Archived
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 jochen
|
64bc138692 |
+3
-5
@@ -21,15 +21,13 @@ ARG GO_BASE=golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7
|
||||
FROM ${GO_BASE} AS build
|
||||
WORKDIR /src
|
||||
|
||||
# **Nothing is fetched** (novox/hq to-be 45 Phase 1): every dependency is in vendor/, committed, so
|
||||
# the image builds from this repository alone — the host's validator among them, whose module no
|
||||
# public proxy is asked for. Dependencies first, so a change to the source does not re-copy them.
|
||||
# Dependencies first, so a change to the source does not refetch them.
|
||||
COPY go.mod go.sum ./
|
||||
COPY vendor/ vendor/
|
||||
RUN go mod download
|
||||
|
||||
COPY . .
|
||||
ARG VERSION=development
|
||||
RUN CGO_ENABLED=0 GOFLAGS=-mod=vendor GOPROXY=off go build -trimpath \
|
||||
RUN CGO_ENABLED=0 go build -trimpath \
|
||||
-ldflags "-s -w -X main.version=${VERSION}" \
|
||||
-o /mesh-controller ./cmd/mesh-controller
|
||||
|
||||
|
||||
@@ -1,353 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// Acting under the lease (novox/hq to-be 45 §6, ADR 0227 rule 1).
|
||||
//
|
||||
// **Only the instance holding the lease acts**: sends a declaration, writes a plan, a condition or a
|
||||
// call. Every one of those passes theLease.epoch, which answers the epoch the act carries or why it may
|
||||
// not happen. Three ways a process stands to the lease:
|
||||
//
|
||||
// - **The serving controller** takes it before it does anything else — before it asserts the bus's
|
||||
// objects, which are the controller's to write — waiting while another holds it, and renews it.
|
||||
// A renewal refused or failed is the lease lost: the gate closes at once and the process exits, so
|
||||
// its service manager restarts it as a candidate (serve, in push.go).
|
||||
// - **A command run at a shell** — `push` in the installer, the lab, a person repairing a mesh whose
|
||||
// controller is down (issue 201) — acts **under the holder's epoch** when a controller holds the
|
||||
// lease: it is the same mesh's word, composed and sent under the store's hold of each machine like
|
||||
// the serving controller's, and the epoch it carries is read at the moment it acts, so a handover
|
||||
// between makes it stale and refused like any other. **When nobody holds the lease, the command
|
||||
// takes it** for as long as it runs and gives it back; a controller starting meanwhile waits for
|
||||
// it, as it would for another controller.
|
||||
// - **A process with no bus** — a test, a command that only reads — acts with no epoch and is
|
||||
// refused nothing: there is nothing to order against, and nothing it does reaches a machine.
|
||||
//
|
||||
// **Unleased, said and temporary.** A serving controller whose bus refuses it the lease's key — the bus's
|
||||
// user list is older than this build and does not grant the bucket yet — and that sees no other holder
|
||||
// serves without one, as every controller did before the lease: declarations carry no epoch, which no
|
||||
// node-engine refuses. Said once, kept as a condition (S12), and tried again every renewal interval; the
|
||||
// first push that sends the bus its new user list grants it, and the next try takes it. Refusing to act
|
||||
// instead would be a controller that can never send the user list that lets it act.
|
||||
|
||||
// actor is this process's standing to the lease.
|
||||
type actor struct {
|
||||
mu sync.Mutex
|
||||
// held is the lease this process holds: the serving controller's, or a command's own.
|
||||
held *lease.Lease
|
||||
// unleased is why a serving controller acts without the lease; empty while it holds it or is not
|
||||
// serving.
|
||||
unleased string
|
||||
// serving is a serving controller, which never borrows another's epoch.
|
||||
serving bool
|
||||
// kv is the lease bucket, for a command to read the holder's epoch from.
|
||||
kv jetstream.KeyValue
|
||||
close func()
|
||||
// noBus is a process with no bus configured.
|
||||
noBus bool
|
||||
// reset is when the lease bucket was found raised again from nothing and its revisions moved past
|
||||
// the highest epoch issued, and what was said of it; zero when it was not (S12).
|
||||
reset time.Time
|
||||
resetSaid string
|
||||
}
|
||||
|
||||
// theLease is this process's standing to the lease.
|
||||
var theLease = &actor{}
|
||||
|
||||
// instance names this process among controller instances: its machine, its process and when it
|
||||
// started. The lease's holder and every call this process keeps carry it.
|
||||
var instance = func() string {
|
||||
host, _ := os.Hostname()
|
||||
return fmt.Sprintf("controller@%s pid %d since %s", host, os.Getpid(), time.Now().UTC().Format(time.RFC3339))
|
||||
}()
|
||||
|
||||
// epoch is the gate: the epoch an act carries — zero for none — or why it may not happen.
|
||||
func (a *actor) epoch(ctx context.Context) (uint64, error) {
|
||||
a.mu.Lock()
|
||||
held, serving, unleased, noBus := a.held, a.serving, a.unleased, a.noBus
|
||||
a.mu.Unlock()
|
||||
switch {
|
||||
case held != nil:
|
||||
return held.Epoch()
|
||||
case serving && unleased != "":
|
||||
return 0, nil
|
||||
case serving:
|
||||
return 0, lease.ErrNotHeld
|
||||
case noBus:
|
||||
return 0, nil
|
||||
}
|
||||
return a.forACommand(ctx)
|
||||
}
|
||||
|
||||
// forACommand is a command's epoch: the holder's, or a lease of its own when nobody holds one.
|
||||
func (a *actor) forACommand(ctx context.Context) (uint64, error) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
if a.held != nil {
|
||||
return a.held.Epoch()
|
||||
}
|
||||
if a.kv == nil {
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
// No bus: this process reaches no machine, and has nothing to order against.
|
||||
a.noBus = true
|
||||
return 0, nil
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("the bus cannot be reached, so whether a controller holds the lease cannot be "+
|
||||
"read and nothing is done: %w", err)
|
||||
}
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
js.Close()
|
||||
return 0, err
|
||||
}
|
||||
reading, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if err := broker.EnsureLeaseBucket(reading, api); err != nil {
|
||||
js.Close()
|
||||
return 0, err
|
||||
}
|
||||
kv, err := api.KeyValue(reading, broker.LeaseBucket)
|
||||
if err != nil {
|
||||
js.Close()
|
||||
return 0, err
|
||||
}
|
||||
a.kv, a.close = kv, js.Close
|
||||
if _, found, err := lease.Current(reading, kv); err == nil && !found {
|
||||
// Nobody: this command takes it for as long as it runs.
|
||||
l, err := lease.Open(reading, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance),
|
||||
Say: func(format string, args ...any) { fmt.Printf(format+"\n", args...) }})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
epoch, err := l.TryTake(reading)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("no controller holds the lease and this command could not take it: %w", err)
|
||||
}
|
||||
keeping, stop := context.WithCancel(context.Background())
|
||||
go l.Keep(keeping)
|
||||
a.held = l
|
||||
closeBus := a.close
|
||||
a.close = func() {
|
||||
stop()
|
||||
l.Release(context.Background())
|
||||
closeBus()
|
||||
}
|
||||
return epoch, nil
|
||||
}
|
||||
}
|
||||
reading, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
holder, found, err := lease.Current(reading, a.kv)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("who holds the controller lease cannot be read, so nothing is done: %w", err)
|
||||
}
|
||||
if !found {
|
||||
return 0, errors.New("the controller that held the lease while this command ran let go of it; nothing " +
|
||||
"more is done under an epoch nobody holds — run the command again")
|
||||
}
|
||||
return holder.Epoch, nil
|
||||
}
|
||||
|
||||
// release gives back what this process holds, at its end.
|
||||
func (a *actor) release() {
|
||||
a.mu.Lock()
|
||||
closing := a.close
|
||||
a.close = nil
|
||||
a.mu.Unlock()
|
||||
if closing != nil {
|
||||
closing()
|
||||
}
|
||||
}
|
||||
|
||||
// holderOf is this process as the lease's holder.
|
||||
func holderOf(instance string) lease.Holder {
|
||||
host, _ := os.Hostname()
|
||||
return lease.Holder{Instance: instance, Host: host, Build: version}
|
||||
}
|
||||
|
||||
// serveUnderTheLease takes the lease for the serving controller, waiting while another holds it, and
|
||||
// keeps it until ctx ends. Lost is closed when it is lost; the caller exits on it.
|
||||
func (a *actor) serveUnderTheLease(ctx context.Context, inv *inventory.Inventory, address string) (lost <-chan struct{}, err error) {
|
||||
a.mu.Lock()
|
||||
a.serving = true
|
||||
a.mu.Unlock()
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the mesh is on the bus at %s and this control plane cannot reach it to take the "+
|
||||
"lease: %w", broker.BareAddress(address), err)
|
||||
}
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
js.Close()
|
||||
return nil, err
|
||||
}
|
||||
asserting, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
err = broker.EnsureLeaseBucket(asserting, api)
|
||||
cancel()
|
||||
if err != nil {
|
||||
js.Close()
|
||||
return nil, err
|
||||
}
|
||||
say := func(format string, args ...any) { fmt.Printf(format+"\n", args...) }
|
||||
l, err := lease.Open(ctx, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance),
|
||||
Floor: inv.HighestEpoch, Say: say, Moved: func(was, floor uint64) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
a.reset = time.Now()
|
||||
a.resetSaid = fmt.Sprintf("the lease bucket was at revision %d with epoch %d already issued: it was "+
|
||||
"raised again from nothing (a bus whose data was replaced), and its revisions were moved past %d so "+
|
||||
"no machine refuses the next epoch", was, floor, floor)
|
||||
}})
|
||||
if err != nil {
|
||||
js.Close()
|
||||
return nil, err
|
||||
}
|
||||
gone := make(chan struct{})
|
||||
epoch, err := l.Take(ctx)
|
||||
switch {
|
||||
case ctx.Err() != nil:
|
||||
js.Close()
|
||||
return nil, ctx.Err()
|
||||
case err != nil && !errors.Is(err, lease.ErrUnwritable):
|
||||
// Whether another controller acts cannot be told: this one does not act, and exits to try again.
|
||||
js.Close()
|
||||
return nil, err
|
||||
case err != nil:
|
||||
// Nobody holds it and the bus will not let it be written: unleased, said, tried again (see above).
|
||||
a.mu.Lock()
|
||||
a.unleased = err.Error()
|
||||
a.mu.Unlock()
|
||||
say("this controller serves WITHOUT the lease: %v. Its declarations carry no epoch; it tries again "+
|
||||
"every %s, and the first push that sends the bus its user list grants it", err, lease.RenewEvery)
|
||||
go a.takeWhenGranted(ctx, l, inv, gone)
|
||||
default:
|
||||
a.took(ctx, l, inv, epoch, gone)
|
||||
}
|
||||
a.mu.Lock()
|
||||
a.close = func() {
|
||||
if held, err := l.Epoch(); err == nil {
|
||||
ending, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
if err := inv.EndEpoch(ending, held, inventory.EpochReleased); err != nil {
|
||||
say("how epoch %d ended could not be recorded: %v", held, err)
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
l.Release(context.Background())
|
||||
js.Close()
|
||||
}
|
||||
a.mu.Unlock()
|
||||
return gone, nil
|
||||
}
|
||||
|
||||
// took is the lease taken: recorded, earlier epochs nobody gave back ended as expired, kept.
|
||||
func (a *actor) took(ctx context.Context, l *lease.Lease, inv *inventory.Inventory, epoch uint64, gone chan struct{}) {
|
||||
a.mu.Lock()
|
||||
a.held, a.unleased = l, ""
|
||||
a.mu.Unlock()
|
||||
h := holderOf(instance)
|
||||
recording, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
expired, err := inv.TookEpoch(recording, inventory.Epoch{Epoch: epoch, Instance: h.Instance, Host: h.Host,
|
||||
Build: h.Build, Taken: time.Now()})
|
||||
cancel()
|
||||
if err != nil {
|
||||
fmt.Printf("epoch %d could not be recorded as taken, so a stale refusal from it will not name it: %v\n", epoch, err)
|
||||
}
|
||||
for _, e := range expired {
|
||||
fmt.Printf("the controller of epoch %d (%s) stopped renewing the lease without giving it back: it is "+
|
||||
"taken over at epoch %d\n", e.Epoch, e.Instance, epoch)
|
||||
}
|
||||
go l.Keep(ctx)
|
||||
go func() {
|
||||
<-l.Lost()
|
||||
if ctx.Err() == nil {
|
||||
why := l.LostWhy()
|
||||
ending, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
_ = inv.EndEpoch(ending, epoch, inventory.EpochLost)
|
||||
cancel()
|
||||
fmt.Printf("the controller lease was lost (epoch %d): %v — this controller stops and exits, to "+
|
||||
"be started again as a candidate\n", epoch, why)
|
||||
}
|
||||
close(gone)
|
||||
}()
|
||||
}
|
||||
|
||||
// takeWhenGranted tries the lease again every renewal interval while serving unleased, and stops this
|
||||
// controller if another took it meanwhile: two serving at once is what the lease is for.
|
||||
func (a *actor) takeWhenGranted(ctx context.Context, l *lease.Lease, inv *inventory.Inventory, gone chan struct{}) {
|
||||
tick := time.NewTicker(lease.RenewEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
epoch, err := l.TryTake(ctx)
|
||||
if errors.Is(err, lease.ErrTaken) {
|
||||
fmt.Printf("another controller took the lease while this one served without it: %v — this one "+
|
||||
"stops and exits\n", err)
|
||||
close(gone)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
// Still not written, or not readable this time: unleased, said by S12, tried again.
|
||||
a.mu.Lock()
|
||||
a.unleased = err.Error()
|
||||
a.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
a.took(ctx, l, inv, epoch, gone)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// standing is what `status` and the self-check say of this process and the lease.
|
||||
type standing struct {
|
||||
Epoch uint64
|
||||
Held bool
|
||||
Renewed time.Time
|
||||
Unleased string
|
||||
// Reset is when the lease bucket was found raised again from nothing, and ResetSaid what of it.
|
||||
Reset time.Time
|
||||
ResetSaid string
|
||||
}
|
||||
|
||||
func (a *actor) standing() standing {
|
||||
a.mu.Lock()
|
||||
held, unleased, reset, resetSaid := a.held, a.unleased, a.reset, a.resetSaid
|
||||
a.mu.Unlock()
|
||||
st := standing{Unleased: unleased, Reset: reset, ResetSaid: resetSaid}
|
||||
if held == nil {
|
||||
return st
|
||||
}
|
||||
epoch, err := held.Epoch()
|
||||
st.Epoch, st.Held, st.Renewed = epoch, err == nil, held.Renewed()
|
||||
return st
|
||||
}
|
||||
|
||||
// The gates, given to what acts: a declaration's send (link) and a plan's write (the inventory).
|
||||
func init() {
|
||||
link.ActingGate = func(ctx context.Context) error {
|
||||
_, err := theLease.epoch(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("this controller may not send: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -677,24 +676,9 @@ type answers struct {
|
||||
// refused, until the switch — and while there is any, the mesh is not all well: the order the
|
||||
// machines' modules are built in is the mesh's to keep, and this is where it says it is not kept.
|
||||
unheld []catalogue.Unheld
|
||||
// conditions is every open condition (novox/hq to-be 45 §2), urgent first and then oldest first:
|
||||
// what leads status, and what its all-well sentence needs to be none of, silenced ones included.
|
||||
// A provider failing a consumer is one of them (ADR 0224). conditionsUnread says why they could
|
||||
// not be read when they could not — never read as none.
|
||||
conditions []conditions.Condition
|
||||
conditionsUnread string
|
||||
// overflowing is every module whose identity overflows the bound of a provision it requires
|
||||
// (novox/hq ADR 0225): its provider leaves it out of the grants and composes everything else, so
|
||||
// this is the one place it is said across the mesh. Not well while there is any.
|
||||
overflowing []catalogue.Overflow
|
||||
// handActs is how many acts were done by hand in the last seven days (novox/hq to-be 45 §7), nil
|
||||
// where the log is not on hand; handActsUnread why it could not be read when it could not.
|
||||
handActs *int
|
||||
handActsUnread string
|
||||
// heals is what the healers did in the last seven days (novox/hq to-be 45 §7): acts, and conditions
|
||||
// handed to the operator; healsUnread why it could not be read.
|
||||
heals *healsCount
|
||||
healsUnread string
|
||||
// failing is every consumer a provider says it keeps failing (novox/hq ADR 0224): a provider's
|
||||
// journal was the only place that said so for a day (04-ISSUES/179).
|
||||
failing []inventory.ProviderStanding
|
||||
}
|
||||
|
||||
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
|
||||
@@ -711,14 +695,12 @@ func heldBy(ctx context.Context) map[string]string {
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read what this mesh has built, so a module naming a "+
|
||||
"base will be told that base is missing: %v\n", err)
|
||||
// empty-on-error: said above; a build that names a base is refused by name for want of it
|
||||
return nil
|
||||
}
|
||||
defer open.Close()
|
||||
held, err := open.inventory.Held(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read what this mesh has built: %v\n", err)
|
||||
// empty-on-error: said above; a build that names a base is refused by name for want of it
|
||||
return nil
|
||||
}
|
||||
address, err := whereABuilderReachesTheStore(ctx, open.inventory)
|
||||
|
||||
@@ -1,162 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The streams and durable consumers the mesh's own traffic needs, derived once (novox/hq to-be 45 §4,
|
||||
// D6 and D7).
|
||||
//
|
||||
// **What the controller asserts at its start and what the self-check expects to find are one
|
||||
// derivation**, run against the bus to make them and against a recorder to list them. Two lists would
|
||||
// drift, and a self-check comparing the bus with a second opinion of what should be there would find
|
||||
// the drift rather than the fault.
|
||||
|
||||
// assertBusObjects brings every stream and consumer into being on r, and answers the machines that
|
||||
// can now hear a declaration.
|
||||
func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Raiser) ([]string, error) {
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names := make([]string, 0, len(nodes))
|
||||
for _, n := range nodes {
|
||||
names = append(names, n.Name)
|
||||
}
|
||||
if err := broker.Raise(r, names); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The work queues of the mesh's own roles (novox/hq ADR 0121). The queue before the holder,
|
||||
// deliberately: work queues until somebody arrives to do it, so assigning a build machine a week
|
||||
// after something started asking for builds flushes the backlog instead of having lost it.
|
||||
// With the seats' holders, so each role's work queue gets the consumer its holder takes
|
||||
// work from. Passed as nil until the first live raise, which left the build machine bound to a
|
||||
// consumer nothing had created (2026-09-28).
|
||||
holders, err := seatHolders(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := broker.RaiseSeats(r, inventory.MeshSeats(), holders); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// And how every module hears what it consumes. Derived from the same records the user list is
|
||||
// composed from, so a module the mesh grants a consumer's subjects has that consumer waiting.
|
||||
// Done on every raise, not only when a credential is issued: every module moved onto this bus
|
||||
// by the rollout was issued on the old one, and came up with nothing to bind to (2026-09-28).
|
||||
consumers, err := moduleConsumers(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Every one tried, and every failure named: one module's consumer the bus refuses is no reason
|
||||
// the modules after it in the list hear nothing (novox/hq issue 208, where this runs on each send).
|
||||
var failed []error
|
||||
for _, c := range consumers {
|
||||
if err := r.EnsureConsumer(c.Consumer); err != nil {
|
||||
failed = append(failed, fmt.Errorf("how %s on %s hears what it consumes: %w", c.Module, c.Node, err))
|
||||
}
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
return nil, errors.Join(failed...)
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// What raises the condition a send says when the objects it implies could not be asserted, and its kind.
|
||||
const (
|
||||
sourceBusObjects = "bus-objects"
|
||||
kindBusObjectsUnasserted = "bus-objects-unasserted"
|
||||
)
|
||||
|
||||
// assertOnSend asserts, on the bus a send is about to use, every object assertBusObjects derives —
|
||||
// **whenever a declaration is sent, not only when the controller starts** (novox/hq issue 208).
|
||||
//
|
||||
// A module assigned after the controller started was sent its declaration and found no consumer to
|
||||
// bind (`consumer not found`, messenger on 2026-10-06), and a seat holder assigned after it found no
|
||||
// worker: the objects a declaration implies were asserted at start and nowhere else, so they existed
|
||||
// only for what was assigned before the last restart. The same derivation, not a second list of what
|
||||
// a send needs: what start asserts, the self-check expects and a send asserts are one answer. Every
|
||||
// part is idempotent, so asserting the whole of it again is the no-op a restart already relies on.
|
||||
//
|
||||
// **A failure is said and raised, and the send goes on.** The objects are the mesh's, not the
|
||||
// machines' being sent: holding every machine back for one consumer that none of them may use would
|
||||
// turn one fault into all of them, and the declarations are not what is wrong. It is never silent —
|
||||
// said in the send's own output and raised as a condition, which the next send that asserts them
|
||||
// clears — and the start-time raise still refuses to serve without them.
|
||||
func assertOnSend(ctx context.Context, inv *inventory.Inventory, r broker.Raiser, indent string) error {
|
||||
_, err := assertBusObjects(ctx, inv, r)
|
||||
var observed []conditions.Observation
|
||||
if err != nil {
|
||||
fmt.Printf("%sTHE BUS DOES NOT HOLD WHAT THIS SEND IMPLIES: %v\n", indent, err)
|
||||
fmt.Printf("%s a module may find no consumer to bind, or a holder no worker; sent anyway, raised as "+
|
||||
"condition %s, and asserted again by the next send\n", indent, unassertedObservation(err).Key())
|
||||
observed = append(observed, unassertedObservation(err))
|
||||
}
|
||||
// Observed when it failed, cleared when it did not: a send that asserted everything is the
|
||||
// observation that the bus holds what it should.
|
||||
if kerr := withKeeper(ctx, func(k *conditions.Keeper) error {
|
||||
return k.Reconcile(ctx, sourceBusObjects, observed)
|
||||
}); kerr != nil {
|
||||
fmt.Printf("%sand whether the bus holds what this send implies could not be kept as a condition: %v\n",
|
||||
indent, kerr)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// unassertedObservation is a send's failure to assert the bus's objects, as a condition.
|
||||
func unassertedObservation(err error) conditions.Observation {
|
||||
return conditions.Observation{Scope: conditions.ScopeBus, ID: "objects", Token: "unasserted",
|
||||
Kind: kindBusObjectsUnasserted, Severity: conditions.Warning, Source: sourceBusObjects,
|
||||
Summary: "the bus's streams and consumers could not be asserted when a declaration was sent: " +
|
||||
"a module may find no consumer to bind, or a seat's holder no worker",
|
||||
Said: err.Error()}
|
||||
}
|
||||
|
||||
// moduleConsumers is every module's durable consumer, from the records the user list is composed from.
|
||||
func moduleConsumers(ctx context.Context, inv *inventory.Inventory) ([]broker.ModuleConsumer, error) {
|
||||
records, err := inv.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return broker.ConsumersOf(users), nil
|
||||
}
|
||||
|
||||
// moduleConsumerCount is how many modules hear what they consume, for the raise's one line.
|
||||
func moduleConsumerCount(ctx context.Context, inv *inventory.Inventory) (int, error) {
|
||||
consumers, err := moduleConsumers(ctx, inv)
|
||||
return len(consumers), err
|
||||
}
|
||||
|
||||
// expectedBusObjects is every stream and consumer assertBusObjects would make, made nowhere.
|
||||
func expectedBusObjects(ctx context.Context, inv *inventory.Inventory) ([]broker.Stream, []broker.Consumer, error) {
|
||||
var rec recordingRaiser
|
||||
if _, err := assertBusObjects(ctx, inv, &rec); err != nil {
|
||||
return nil, nil, fmt.Errorf("what the bus should hold cannot be worked out: %w", err)
|
||||
}
|
||||
return rec.streams, rec.consumers, nil
|
||||
}
|
||||
|
||||
// recordingRaiser keeps what it was asked to assert and asserts nothing.
|
||||
type recordingRaiser struct {
|
||||
streams []broker.Stream
|
||||
consumers []broker.Consumer
|
||||
}
|
||||
|
||||
func (r *recordingRaiser) EnsureStream(s broker.Stream) error {
|
||||
r.streams = append(r.streams, s)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *recordingRaiser) EnsureConsumer(c broker.Consumer) error {
|
||||
r.consumers = append(r.consumers, c)
|
||||
return nil
|
||||
}
|
||||
@@ -1,199 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// novox/hq issue 208: the bus's objects a declaration implies are asserted whenever one is sent, not
|
||||
// only when the controller starts.
|
||||
|
||||
// aCarriedConsumer is the runtime on laptop and, carried by it, a module that consumes an event: the
|
||||
// shape of messenger on 2026-10-06, assigned after the controller started.
|
||||
func aCarriedConsumer(t *testing.T, open *stores) broker.Consumer {
|
||||
t.Helper()
|
||||
register(t, open, catalogue.Manifest{Module: catalogue.RuntimeModule, Version: "1",
|
||||
OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/var/lib/mesh/node-tools/broker"}}})
|
||||
register(t, open, catalogue.Manifest{Module: "messenger", Version: "1",
|
||||
Consumes: []string{"billing.order.placed"}})
|
||||
for _, m := range []string{catalogue.RuntimeModule, "messenger"} {
|
||||
if _, err := open.inventory.Assign(t.Context(), "laptop", m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
consumers, err := moduleConsumers(t.Context(), open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range consumers {
|
||||
if c.Module == "messenger" && c.Node == "laptop" {
|
||||
return c.Consumer
|
||||
}
|
||||
}
|
||||
t.Fatalf("messenger on laptop is derived no consumer: %+v", consumers)
|
||||
return broker.Consumer{}
|
||||
}
|
||||
|
||||
// aLateHolder is a module holding the build agent's seat on anchor, assigned after the controller
|
||||
// started, and the worker its seat's queue should have for it.
|
||||
func aLateHolder(t *testing.T, open *stores) broker.Consumer {
|
||||
t.Helper()
|
||||
register(t, open, catalogue.Manifest{Module: "late-builder", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}})
|
||||
if _, err := open.inventory.Assign(t.Context(), "anchor", "late-builder"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, s := range inventory.MeshSeats() {
|
||||
if s.Name == "node-build-agent" {
|
||||
c, needed := broker.HolderConsumerFor("anchor", "late-builder", s)
|
||||
if !needed {
|
||||
t.Fatal("the build agent's seat needs no worker")
|
||||
}
|
||||
return c
|
||||
}
|
||||
}
|
||||
t.Fatal("the mesh declares no build agent's seat")
|
||||
return broker.Consumer{}
|
||||
}
|
||||
|
||||
// asserted says whether a recording holds a consumer by stream and name.
|
||||
func asserted(rec *recordingRaiser, want broker.Consumer) bool {
|
||||
for _, c := range rec.consumers {
|
||||
if c.Stream == want.Stream && c.Name == want.Name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// What a send asserts includes what was assigned after the start's assertion: a carried module's
|
||||
// consumer and a late holder's worker. The derivation is the start's own, so this holds without a bus.
|
||||
func TestASendAssertsWhatWasAssignedAfterStart(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
var atStart recordingRaiser
|
||||
if _, err := assertBusObjects(t.Context(), open.inventory, &atStart); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
consumer := aCarriedConsumer(t, open)
|
||||
worker := aLateHolder(t, open)
|
||||
if asserted(&atStart, consumer) || asserted(&atStart, worker) {
|
||||
t.Fatal("the start asserted what was not yet assigned; the test proves nothing")
|
||||
}
|
||||
var onSend recordingRaiser
|
||||
if err := assertOnSend(t.Context(), open.inventory, &onSend, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !asserted(&onSend, consumer) {
|
||||
t.Fatalf("messenger's consumer %s on %s is not asserted by the send: %+v", consumer.Name, consumer.Stream, onSend.consumers)
|
||||
}
|
||||
if !asserted(&onSend, worker) {
|
||||
t.Fatalf("the late holder's worker %s on %s is not asserted by the send: %+v", worker.Name, worker.Stream, onSend.consumers)
|
||||
}
|
||||
}
|
||||
|
||||
// failingRaiser refuses one consumer by name and records everything it was asked.
|
||||
type failingRaiser struct {
|
||||
recordingRaiser
|
||||
refuse string
|
||||
}
|
||||
|
||||
func (f *failingRaiser) EnsureConsumer(c broker.Consumer) error {
|
||||
f.consumers = append(f.consumers, c)
|
||||
if c.Name == f.refuse {
|
||||
return errors.New("nats: API error: code=503 description=insufficient resources")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// A send whose objects cannot be asserted says so in its output and raises a condition — and the next
|
||||
// send that asserts them clears it. The consumers after the refused one are still asked for.
|
||||
func TestASendThatCannotAssertTheBusSaysSoAndRaisesACondition(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
consumer := aCarriedConsumer(t, open)
|
||||
worker := aLateHolder(t, open)
|
||||
failing := &failingRaiser{refuse: consumer.Name}
|
||||
|
||||
var sendErr error
|
||||
out := stdoutOf(t, func() error {
|
||||
sendErr = assertOnSend(t.Context(), open.inventory, failing, " ")
|
||||
return nil
|
||||
})
|
||||
if sendErr == nil || !strings.Contains(sendErr.Error(), "how messenger on laptop hears what it consumes") {
|
||||
t.Fatalf("the failure is not answered: %v", sendErr)
|
||||
}
|
||||
if !strings.Contains(out, "THE BUS DOES NOT HOLD WHAT THIS SEND IMPLIES") ||
|
||||
!strings.Contains(out, "insufficient resources") || !strings.Contains(out, "bus.objects.unasserted") {
|
||||
t.Fatalf("the failure is not said in the send's output:\n%s", out)
|
||||
}
|
||||
if !asserted(&failing.recordingRaiser, worker) {
|
||||
t.Fatal("the seat's worker was not asked for")
|
||||
}
|
||||
c, open1, err := conditionsFrom.Get(t.Context(), "bus.objects.unasserted")
|
||||
if err != nil || !open1 {
|
||||
t.Fatalf("no condition raised: %v", err)
|
||||
}
|
||||
if c.Kind != kindBusObjectsUnasserted || c.Source != sourceBusObjects ||
|
||||
len(c.Evidence) == 0 || !strings.Contains(c.Evidence[0].Said, "insufficient resources") {
|
||||
t.Fatalf("the condition does not say what failed: %+v", c)
|
||||
}
|
||||
|
||||
// The next send asserts them, and that observation clears it.
|
||||
if err := assertOnSend(t.Context(), open.inventory, &recordingRaiser{}, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, stillOpen, err := conditionsFrom.Get(t.Context(), "bus.objects.unasserted"); err != nil || stillOpen {
|
||||
t.Fatalf("a send that asserted everything left the condition open: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Against a real bus, through the send's own grant: a module assigned after start has its consumer
|
||||
// once a declaration is sent, and a holder assigned after start its seat's worker.
|
||||
func TestNatsAModuleAssignedAfterStartGetsItsConsumerAtItsFirstSend(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
open := aMesh(t)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
|
||||
_ = js.Context().DeleteStream(s)
|
||||
}
|
||||
// The controller starting, before either was assigned.
|
||||
if _, err := assertBusObjects(t.Context(), open.inventory, js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
consumer := aCarriedConsumer(t, open)
|
||||
worker := aLateHolder(t, open)
|
||||
_ = js.Context().DeleteConsumer(worker.Stream, worker.Name)
|
||||
for _, c := range []broker.Consumer{consumer, worker} {
|
||||
if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); !errors.Is(err, nats.ErrConsumerNotFound) {
|
||||
t.Fatalf("%s on %s exists before any send; the test proves nothing: %v", c.Name, c.Stream, err)
|
||||
}
|
||||
}
|
||||
|
||||
server := link.ConnectNats(js, nil, nil)
|
||||
if err := (overTheBus{open: open, server: server}).grant(t.Context(), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range []broker.Consumer{consumer, worker} {
|
||||
if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); err != nil {
|
||||
t.Fatalf("%s on %s is not on the bus after a send: %v", c.Name, c.Stream, err)
|
||||
}
|
||||
}
|
||||
if _, raised, _ := conditionsFrom.Get(t.Context(), "bus.objects.unasserted"); raised {
|
||||
t.Fatal("a send that asserted everything raised a condition")
|
||||
}
|
||||
}
|
||||
@@ -35,10 +35,10 @@ func TestAPushAnswersBeforeItSends(t *testing.T) {
|
||||
args map[string]any
|
||||
want bool
|
||||
}{
|
||||
{"push", map[string]any{"node": "anchor", "why": "w"}, true},
|
||||
{"push", map[string]any{"why": "w"}, true},
|
||||
{"command", map[string]any{"command": "push anchor --why w"}, true},
|
||||
{"command", map[string]any{"command": "push --behind --why=w"}, true},
|
||||
{"push", map[string]any{"node": "anchor"}, true},
|
||||
{"push", map[string]any{}, true},
|
||||
{"command", map[string]any{"command": "push anchor"}, true},
|
||||
{"command", map[string]any{"command": "push --behind"}, true},
|
||||
{"command", map[string]any{"command": "builds"}, false},
|
||||
{"status", map[string]any{}, false},
|
||||
{"assign", map[string]any{"node": "anchor", "module": "m"}, false},
|
||||
|
||||
@@ -25,17 +25,7 @@ import (
|
||||
// mesh seat is judged fully only at registration. A seat another module declares is unknown unless
|
||||
// that module's manifest is passed too. Both are printed as a note, not as a problem — a check that
|
||||
// refused what it could not see would teach people to ignore it.
|
||||
//
|
||||
// **And every identity against every bound it meets** (novox/hq ADR 0225, issue 263): each module's
|
||||
// identity, on a machine whose name is `longestMachine` characters, against the bound of every
|
||||
// provision it wants that a manifest given here offers. An overflow is refused in the pull request
|
||||
// that introduces it — a new requirement, a lowered bound, a longer slug — instead of on the
|
||||
// provider's machine when a real machine's name first meets the module's.
|
||||
func moduleCheck(paths []string, out io.Writer) error {
|
||||
return moduleCheckFor(paths, catalogue.DefaultLongestMachine, out)
|
||||
}
|
||||
|
||||
func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
if len(paths) == 0 {
|
||||
return errors.New("module check <manifest.json>... — one file per module; pass every " +
|
||||
"manifest of a repository together so the rules between them are checked too")
|
||||
@@ -84,15 +74,6 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
}
|
||||
failed += len(problems)
|
||||
|
||||
// Between the manifests too: an identity against the bounds of the provisions it wants, which
|
||||
// only the provider's manifest states.
|
||||
identities := catalogue.IdentityProblems(shelf, longestMachine)
|
||||
sort.Strings(identities)
|
||||
for _, p := range identities {
|
||||
fmt.Fprintln(out, p)
|
||||
}
|
||||
failed += len(identities)
|
||||
|
||||
var names []string
|
||||
for name := range shelf {
|
||||
names = append(names, name)
|
||||
@@ -128,8 +109,7 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
}
|
||||
fmt.Fprintf(out, "%d manifest(s) checked. Judged against the seats this binary carries; a claim on "+
|
||||
"one of the mesh's own seats is judged fully at registration, and a seat declared by a "+
|
||||
"module not given here reads as unknown. Identities judged on a %d-character machine name, "+
|
||||
"against the bounds of the providers given here\n", len(paths), longestMachine)
|
||||
"module not given here reads as unknown\n", len(paths))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -1,441 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// What is wrong, kept until observation says it is not (novox/hq to-be 45 §2, ADR 0227).
|
||||
//
|
||||
// **`status` used to be the only place the mesh said it was wrong, and only to whoever asked.** Every
|
||||
// core failure of research 031 was found by a person looking. A condition is the mesh saying it: raised
|
||||
// by a watchdog when a signal is late (signals.go), by a probe when an invariant does not hold
|
||||
// (doctor.go), or by an event a provider sends (standing.go); kept on the bus with since-when and
|
||||
// evidence; said on the bus as it changes, for the operator's channel to carry; and cleared when an
|
||||
// observation says it is resolved. Nobody resolves one by hand. A person who knows silences it, for a
|
||||
// while, with a reason, and that is recorded as a hand act.
|
||||
|
||||
// conditionsFrom is the serving controller's keeper; nil in any other process, which opens its own.
|
||||
var conditionsFrom *conditions.Keeper
|
||||
|
||||
// keeperOn is a keeper over the store on a connection, saying its transitions on that connection.
|
||||
func keeperOn(ctx context.Context, conn *nats.Conn) (*conditions.Keeper, error) {
|
||||
store, history, err := conditions.OnTheBus(ctx, conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js, err := conn.JetStream()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history,
|
||||
Teller: link.OverNATS{Conn: conn, JS: js},
|
||||
Say: func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) },
|
||||
// What status leads with changed: composed again soon (a nudge outside the serving controller
|
||||
// does nothing).
|
||||
Changed: statusFrom.nudge,
|
||||
// Written under the lease, carrying its epoch (novox/hq to-be 45 §6).
|
||||
Epoch: func() (uint64, error) { return theLease.epoch(context.WithoutCancel(ctx)) }}), nil
|
||||
}
|
||||
|
||||
// withKeeper runs f with the serving controller's keeper, or one of its own that says everything
|
||||
// it was given before it returns.
|
||||
func withKeeper(ctx context.Context, f func(*conditions.Keeper) error) error {
|
||||
if conditionsFrom != nil {
|
||||
return f(conditionsFrom)
|
||||
}
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
k, err := keeperOn(ctx, conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
flushing, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
k.Close(flushing)
|
||||
}()
|
||||
return f(k)
|
||||
})
|
||||
}
|
||||
|
||||
// openConditions is every open condition, for `status` and `node show`: from the serving keeper, or
|
||||
// read from the bus. Where there is no bus to read it from, it says so — never "none open".
|
||||
func openConditions(ctx context.Context) ([]conditions.Condition, error) {
|
||||
if conditionsFrom != nil {
|
||||
return conditionsFrom.Open(ctx)
|
||||
}
|
||||
if _, err := broker.BusAddress(); err != nil {
|
||||
return nil, fmt.Errorf("this process has no bus to read the conditions from: %w", err)
|
||||
}
|
||||
var out []conditions.Condition
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
store, _, err := conditions.OnTheBus(ctx, conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
reading, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
out, err = conditions.Read(reading, store)
|
||||
return err
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
// conditionsUsage is how the verb is typed.
|
||||
const conditionsUsage = "conditions [--scope S] [--severity urgent|warning] [--machine M] [--json] | " +
|
||||
"conditions show <key> | conditions silence <key> --for <duration> --why <text> | " +
|
||||
"conditions history [--days N] [--key K] [--json]"
|
||||
|
||||
// conditionsCommand is `conditions`, `conditions show`, `conditions silence` and `conditions history`.
|
||||
func conditionsCommand(ctx context.Context, args []string) error {
|
||||
sub := "list"
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
sub, args = args[0], args[1:]
|
||||
}
|
||||
switch sub {
|
||||
case "list":
|
||||
return listConditions(ctx, args)
|
||||
case "show":
|
||||
return showCondition(ctx, args)
|
||||
case "silence":
|
||||
return silenceCondition(ctx, args)
|
||||
case "history":
|
||||
return conditionHistory(ctx, args)
|
||||
}
|
||||
return errors.New(conditionsUsage)
|
||||
}
|
||||
|
||||
func listConditions(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("conditions", flag.ContinueOnError)
|
||||
scope := set.String("scope", "", "only this scope: "+strings.Join(conditions.Scopes, ", "))
|
||||
severity := set.String("severity", "", "only urgent, or only warning")
|
||||
machine := set.String("machine", "", "only those about this machine")
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New(conditionsUsage)
|
||||
}
|
||||
if *severity != "" && *severity != string(conditions.Urgent) && *severity != string(conditions.Warning) {
|
||||
return fmt.Errorf("a severity is urgent or warning, not %q", *severity)
|
||||
}
|
||||
open, err := openConditions(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var out []conditions.Condition
|
||||
for _, c := range open {
|
||||
if (*scope == "" || c.Subject.Scope == *scope) && (*severity == "" || string(c.Severity) == *severity) &&
|
||||
(*machine == "" || concerns(c, *machine)) {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
if *asJSON {
|
||||
if out == nil {
|
||||
out = []conditions.Condition{}
|
||||
}
|
||||
return printJSON(map[string]any{"conditions": out, "open": len(open),
|
||||
"note": "urgent first, then oldest first; a condition clears when observation says so, never by hand"})
|
||||
}
|
||||
if len(out) == 0 {
|
||||
if len(open) == 0 {
|
||||
fmt.Println("no open conditions")
|
||||
} else {
|
||||
fmt.Printf("none of the %d open condition(s) is about that\n", len(open))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for _, line := range conditionLines(out, time.Now()) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// concerns says whether a condition is about a machine: it names it, or its key does.
|
||||
func concerns(c conditions.Condition, machine string) bool {
|
||||
if c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) {
|
||||
return true
|
||||
}
|
||||
for _, part := range strings.Split(c.Key, ".") {
|
||||
if part == machine {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// conditionLines is how a list of conditions reads: one line each, its silence under it.
|
||||
func conditionLines(list []conditions.Condition, now time.Time) []string {
|
||||
var out []string
|
||||
for _, c := range list {
|
||||
times := ""
|
||||
if c.Count > 1 {
|
||||
times = fmt.Sprintf(", raised %d times", c.Count)
|
||||
}
|
||||
out = append(out, fmt.Sprintf(" %-7s %s — %s (since %s%s)", strings.ToUpper(string(c.Severity)),
|
||||
c.Key, c.Summary, c.Raised.Local().Format("2006-01-02 15:04"), times))
|
||||
if c.SilencedAt(now) {
|
||||
out = append(out, fmt.Sprintf(" silenced until %s by %s: %s",
|
||||
c.Silenced.Until.Local().Format("2006-01-02 15:04"), c.Silenced.By, c.Silenced.Why))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func showCondition(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("conditions show", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
rest, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(rest) != 1 {
|
||||
return errors.New("conditions show <key>")
|
||||
}
|
||||
key := rest[0]
|
||||
var c conditions.Condition
|
||||
var found bool
|
||||
if conditionsFrom != nil {
|
||||
c, found, err = conditionsFrom.Get(ctx, key)
|
||||
} else {
|
||||
err = onTheBus(func(conn *nats.Conn) error {
|
||||
store, _, err := conditions.OnTheBus(ctx, conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c, found, err = conditions.ReadOne(ctx, store, key)
|
||||
return err
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("no condition %s is open — `conditions` lists those that are, and `conditions "+
|
||||
"history --key %s` what became of it", key, key)
|
||||
}
|
||||
if *asJSON {
|
||||
return printJSON(c)
|
||||
}
|
||||
now := time.Now()
|
||||
fmt.Printf("%s %s\n %s\n\n", strings.ToUpper(string(c.Severity)), c.Key, c.Summary)
|
||||
fmt.Printf(" kind %s\n about %s %s", c.Kind, c.Subject.Scope, c.Subject.ID)
|
||||
if c.Subject.Machine != "" {
|
||||
fmt.Printf(", on %s", c.Subject.Machine)
|
||||
}
|
||||
fmt.Printf("\n raised by %s\n since %s (%s ago), observed %d time(s), last %s ago\n",
|
||||
c.Source, c.Raised.Local().Format("2006-01-02 15:04:05"), roughly(now.Sub(c.Raised)), c.Observations,
|
||||
now.Sub(c.LastObserved).Round(time.Second))
|
||||
if c.Count > 1 {
|
||||
fmt.Printf(" raised %d times, each within ten minutes of clearing\n", c.Count)
|
||||
}
|
||||
fmt.Printf(" resolved by %s\n", resolverWords(c.Resolver))
|
||||
if c.Silenced != nil {
|
||||
fmt.Printf(" silenced until %s by %s: %s\n", c.Silenced.Until.Local().Format("2006-01-02 15:04"),
|
||||
c.Silenced.By, c.Silenced.Why)
|
||||
}
|
||||
if len(c.Tried) > 0 {
|
||||
fmt.Println("\n tried:")
|
||||
for _, t := range c.Tried {
|
||||
fmt.Printf(" %s %s — %s: %s\n", t.At.Local().Format("2006-01-02 15:04"), orHealer(t.By), t.What, t.Outcome)
|
||||
}
|
||||
}
|
||||
fmt.Println("\n evidence, newest first:")
|
||||
for _, e := range c.Evidence {
|
||||
fmt.Printf(" %s %s\n", e.At.Local().Format("2006-01-02 15:04:05"), e.Said)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func resolverWords(r string) string {
|
||||
switch r {
|
||||
case conditions.ResolverSelf:
|
||||
return "itself: it clears when observation says it is resolved"
|
||||
case conditions.ResolverOperator:
|
||||
return "the operator: nothing in the mesh will repair it"
|
||||
case conditions.ResolverAgent:
|
||||
return "an agent"
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// silenceCondition stops a condition's messages for a while (to-be 45 §2). A hand act: recorded with
|
||||
// who and why before it is done, its cause the condition's kind unless one is given.
|
||||
func silenceCondition(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("conditions silence", flag.ContinueOnError)
|
||||
forFlag := set.String("for", "", "how long: 30m, 4h, 2d — at most 7d")
|
||||
acts := addHandActFlags(set)
|
||||
rest, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(rest) != 1 {
|
||||
return errors.New("conditions silence <key> --for <duration> --why <text>")
|
||||
}
|
||||
key := rest[0]
|
||||
if err := acts.require("conditions silence"); err != nil {
|
||||
return err
|
||||
}
|
||||
d, err := parseFor(*forFlag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d > conditions.MaxSilence {
|
||||
return fmt.Errorf("a condition is silenced for at most %s at once; past it, say so again", conditions.MaxSilence)
|
||||
}
|
||||
return withKeeper(ctx, func(k *conditions.Keeper) error {
|
||||
c, found, err := k.Get(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("no condition %s is open — `conditions` lists them. Nothing was silenced", key)
|
||||
}
|
||||
if strings.TrimSpace(*acts.cause) == "" {
|
||||
*acts.cause = c.Kind
|
||||
}
|
||||
*acts.condition = key
|
||||
acts.record(ctx, "conditions silence", []string{key, "--for", *forFlag})
|
||||
held, err := k.Silence(ctx, key, d, link.Caller(), *acts.why)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s is silenced until %s: no message is sent for it until then. It is still open, and "+
|
||||
"`status` still says it; it clears when observation says it is resolved\n",
|
||||
held.Key, held.Silenced.Until.Local().Format("2006-01-02 15:04"))
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// parseFor reads a duration, days included.
|
||||
func parseFor(s string) (time.Duration, error) {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return 0, errors.New("say for how long: --for 30m, 4h or 2d")
|
||||
}
|
||||
if days, ok := strings.CutSuffix(s, "d"); ok {
|
||||
n, err := strconv.Atoi(days)
|
||||
if err != nil || n <= 0 {
|
||||
return 0, fmt.Errorf("%q is not a number of days", s)
|
||||
}
|
||||
return time.Duration(n) * 24 * time.Hour, nil
|
||||
}
|
||||
d, err := time.ParseDuration(s)
|
||||
if err != nil || d <= 0 {
|
||||
return 0, fmt.Errorf("%q is not a duration: 30m, 4h or 2d", s)
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func conditionHistory(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("conditions history", flag.ContinueOnError)
|
||||
days := set.Int("days", 7, "how many days back, at most 90")
|
||||
key := set.String("key", "", "only this condition")
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New("conditions history [--days N] [--key K] [--json]")
|
||||
}
|
||||
since := time.Now().Add(-time.Duration(*days) * 24 * time.Hour)
|
||||
var events []conditions.Event
|
||||
read := func(h conditions.History) error {
|
||||
var err error
|
||||
events, err = h.Since(ctx, since)
|
||||
return err
|
||||
}
|
||||
var err error
|
||||
if conditionsFrom != nil {
|
||||
events, err = conditionsFrom.HistorySince(ctx, since)
|
||||
} else {
|
||||
err = onTheBus(func(conn *nats.Conn) error {
|
||||
_, history, err := conditions.OnTheBus(ctx, conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return read(history)
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var out []conditions.Event
|
||||
for _, e := range events {
|
||||
if *key == "" || e.Key == *key {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
if *asJSON {
|
||||
if out == nil {
|
||||
out = []conditions.Event{}
|
||||
}
|
||||
return printJSON(map[string]any{"history": out, "days": *days})
|
||||
}
|
||||
if len(out) == 0 {
|
||||
fmt.Printf("nothing was raised, changed or cleared in the last %d day(s)\n", *days)
|
||||
return nil
|
||||
}
|
||||
for _, e := range out {
|
||||
line := fmt.Sprintf("%s %-13s %s", e.At.Local().Format("2006-01-02 15:04:05"), e.Change, e.Key)
|
||||
switch e.Change {
|
||||
case conditions.ChangeRaised, conditions.ChangeReopened:
|
||||
line += " — " + e.Summary
|
||||
case conditions.ChangeSeverity:
|
||||
line += fmt.Sprintf(" — %s, was %s", e.Severity, e.Was)
|
||||
case conditions.ChangeResolver:
|
||||
line += fmt.Sprintf(" — %s, was %s", e.Resolver, e.Was)
|
||||
default:
|
||||
if e.Why != "" {
|
||||
line += " — " + e.Why
|
||||
}
|
||||
}
|
||||
fmt.Println(line)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// printConditions is the status section that leads it: every open condition, urgent first, oldest
|
||||
// first, silenced ones with their expiry (to-be 45 §2). A store that could not be read is said, and
|
||||
// is not "none open".
|
||||
func printConditions(list []conditions.Condition, unread string, now time.Time) {
|
||||
if unread != "" {
|
||||
fmt.Printf("the open conditions could NOT be read, so whether anything is wrong is not known: %s\n\n", unread)
|
||||
return
|
||||
}
|
||||
if len(list) == 0 {
|
||||
return
|
||||
}
|
||||
urgent := 0
|
||||
for _, c := range list {
|
||||
if c.Severity == conditions.Urgent {
|
||||
urgent++
|
||||
}
|
||||
}
|
||||
fmt.Printf("%d open condition(s), %d urgent:\n\n", len(list), urgent)
|
||||
for _, line := range conditionLines(list, now) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
fmt.Printf("\n `conditions show <key>` says more; each clears when observation says it is resolved, " +
|
||||
"never by hand — `conditions silence <key> --for <d> --why <text>` stops its messages\n\n")
|
||||
}
|
||||
|
||||
// orHealer is who tried, as an attempt names it.
|
||||
func orHealer(by string) string {
|
||||
if by == "" {
|
||||
return "a healer"
|
||||
}
|
||||
return by
|
||||
}
|
||||
@@ -1,107 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// The verbs of the condition store (novox/hq to-be 45 §2) as the console reaches them, and status led
|
||||
// by what is open.
|
||||
|
||||
func TestTheConditionsVerbComposesEachShape(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
args map[string]any
|
||||
want string
|
||||
}{
|
||||
{map[string]any{}, "conditions --json"},
|
||||
{map[string]any{"severity": "urgent", "machine": "ace"}, "conditions --json --severity urgent --machine ace"},
|
||||
{map[string]any{"key": "machine.ace.silent"}, "conditions show machine.ace.silent --json"},
|
||||
{map[string]any{"history": "true", "days": "3", "key": "machine.ace.silent"},
|
||||
"conditions history --json --days 3 --key machine.ace.silent"},
|
||||
{map[string]any{"silence": "machine.ace.silent", "for": "2h", "why": "on the train"},
|
||||
"conditions silence machine.ace.silent --for 2h --why on the train"},
|
||||
{map[string]any{"run": "true"}, "doctor run --json"},
|
||||
{map[string]any{}, "doctor --json"},
|
||||
} {
|
||||
verb := "conditions"
|
||||
if strings.HasPrefix(c.want, "doctor") {
|
||||
verb = "doctor"
|
||||
}
|
||||
argv, err := argvFor(verb, c.args)
|
||||
if err != nil || strings.Join(argv, " ") != c.want {
|
||||
t.Errorf("%s %v composed %q (%v), want %q", verb, c.args, strings.Join(argv, " "), err, c.want)
|
||||
}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
}{
|
||||
{"conditions", map[string]any{"silence": "machine.ace.silent", "why": "x"}}, // no for
|
||||
{"doctor", map[string]any{"run": "true", "signals": "true"}}, // two at once
|
||||
{"conditions", map[string]any{"silence": "machine.ace.silent", "for": "1h", "why": "x", "days": "3"}}, // passed over
|
||||
} {
|
||||
if argv, err := argvFor(c.verb, c.args); err == nil {
|
||||
t.Errorf("%s %v composed %v", c.verb, c.args, argv)
|
||||
}
|
||||
}
|
||||
if repairingCommand([]string{"conditions", "silence", "k"}) != "conditions silence" {
|
||||
t.Error("a silence is not a hand act")
|
||||
}
|
||||
}
|
||||
|
||||
// **A silence through the verb is recorded and bounded**; the condition stays open.
|
||||
func TestASilenceThroughTheVerbHoldsAndTheConditionStaysOpen(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
ctx := t.Context()
|
||||
if _, err := k.Observe(ctx, conditions.Observation{Scope: conditions.ScopeMachine, ID: "ace", Kind: "silent",
|
||||
Severity: conditions.Warning, Summary: "ace is silent", Source: "S1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := conditionsCommand(ctx, []string{"silence", "machine.ace.silent", "--for", "2d"}); err == nil {
|
||||
t.Fatal("silenced without saying why")
|
||||
}
|
||||
if err := conditionsCommand(ctx, []string{"silence", "machine.ace.silent", "--for", "8d", "--why", "x"}); err == nil {
|
||||
t.Fatal("silenced for more than a week")
|
||||
}
|
||||
said := printed(t, func() error {
|
||||
return conditionsCommand(ctx, []string{"silence", "machine.ace.silent", "--for", "2d", "--why", "on the train"})
|
||||
})
|
||||
if !strings.Contains(said, "is silenced until") {
|
||||
t.Fatalf("%s", said)
|
||||
}
|
||||
c, found, _ := k.Get(ctx, "machine.ace.silent")
|
||||
if !found || c.Silenced == nil || c.Silenced.Why != "on the train" {
|
||||
t.Fatalf("%+v", c)
|
||||
}
|
||||
listed := printed(t, func() error { return conditionsCommand(ctx, nil) })
|
||||
if !strings.Contains(listed, "machine.ace.silent") || !strings.Contains(listed, "silenced until") {
|
||||
t.Fatalf("%s", listed)
|
||||
}
|
||||
}
|
||||
|
||||
// **Conditions that cannot be read are not none open**: status says so, and is not well.
|
||||
func TestUnreadableConditionsAreNotAWellMesh(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
_, store := withConditionsInMemory(t)
|
||||
store.Fail = errors.New("the bus is away")
|
||||
asked, err := theThreeQuestions(t.Context(), open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if asked.well() || asked.conditionsUnread == "" {
|
||||
t.Fatalf("well with its conditions unread: %+v", asked.conditionsUnread)
|
||||
}
|
||||
said := printed(t, func() error { return printStatus(asked) })
|
||||
if !strings.HasPrefix(said, "the open conditions could NOT be read") || strings.Contains(said, "no open conditions") {
|
||||
t.Fatalf("%s", said)
|
||||
}
|
||||
store.Fail = nil
|
||||
asked, _ = theThreeQuestions(t.Context(), open)
|
||||
said = printed(t, func() error { return printStatus(asked) })
|
||||
if asked.well() && !strings.Contains(said, "no open conditions;") {
|
||||
t.Fatalf("the all-well sentence does not say no conditions are open:\n%s", said)
|
||||
}
|
||||
}
|
||||
@@ -1,545 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The self-check: `doctor` (novox/hq to-be 45 §4, ADR 0227 rule 6).
|
||||
//
|
||||
// **The design's invariants, run against the running mesh.** A probe is one live invariant of a
|
||||
// design — every machine's declaration composes and validates, every resolver answers, every seat's
|
||||
// holder answers, every stream and consumer is there as defined — with an id, a bound, and the
|
||||
// condition it raises when the invariant does not hold. The serving controller runs the registry every
|
||||
// five minutes, each probe given thirty seconds; a probe that errors or does not finish raises
|
||||
// `probe-failed` for itself, because an unanswered probe is never a pass. Every run ends with a
|
||||
// heartbeat on the bus (`doctor-heartbeat`, S10), which mesh-watcher listens for from a second
|
||||
// machine: a controller that stops checking is itself said, through a channel that does not pass
|
||||
// through it.
|
||||
//
|
||||
// `doctor` answers the last run's verdict at once; `doctor run` runs now; `doctor probes` lists the
|
||||
// registry; `doctor signals` says, for every row of the signals table, the age of its newest signal.
|
||||
|
||||
// The self-check's clocks.
|
||||
var (
|
||||
// doctorEvery is how often the registry runs; doctorFirstAfter how long after the controller
|
||||
// starts the first run waits, so what the controller hears at its start has arrived.
|
||||
doctorEvery = 5 * time.Minute
|
||||
doctorFirstAfter = time.Minute
|
||||
// probeWithin is each probe's bound.
|
||||
probeWithin = 30 * time.Second
|
||||
)
|
||||
|
||||
// The verdicts a probe can have.
|
||||
const (
|
||||
verdictPass = "pass"
|
||||
verdictFail = "fail"
|
||||
verdictFailedToRun = "failed-to-run"
|
||||
verdictDeferred = "deferred"
|
||||
)
|
||||
|
||||
// probe is one live invariant.
|
||||
type probe struct {
|
||||
ID string
|
||||
Asserts string
|
||||
From string
|
||||
// Kind is the condition kind raised when the invariant does not hold.
|
||||
Kind string
|
||||
// Raises are the other kinds its findings carry, each its own (a consumer missing, one far behind):
|
||||
// what a healer may be registered against (healers.go).
|
||||
Raises []string
|
||||
Phase int
|
||||
// Deferred says why it is not run yet; empty for one that is.
|
||||
Deferred string
|
||||
// Asks are the seat verbs it calls. A probe may call no other (askSeatTool refuses), and the
|
||||
// controller's grant names every one (a test over this registry): a probe whose question the bus
|
||||
// refuses checks nothing (D8, 2026-10-06).
|
||||
Asks []broker.SeatVerb
|
||||
run func(ctx context.Context, d *doctor) ([]conditions.Observation, error)
|
||||
}
|
||||
|
||||
// probeRegistry is the registry, in to-be 45's order. **The registry is the design's live form**: a
|
||||
// probe added to a design is a row added here.
|
||||
var probeRegistry = []probe{
|
||||
{ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation",
|
||||
From: "issues 236, 263", Kind: "declaration-refused", Phase: 1, run: probeDeclarations},
|
||||
{ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6",
|
||||
From: "issue 262", Kind: "resolver-wrong", Phase: 1, run: probeResolvers},
|
||||
{ID: "D3", Asserts: "every seat on record that serves verbs has a live holder that answers, on every " +
|
||||
"machine that is heard from", From: "issues 208, 218", Kind: "holder-silent", Phase: 1, run: probeHolders},
|
||||
{ID: "D4", Asserts: "every kept archive is held by a manifest", From: "issue 253",
|
||||
Kind: "archives-unheld", Phase: 1, run: probeArchives},
|
||||
{ID: "D5", Asserts: "exactly one lease holder — the key names this controller at its epoch, and the record " +
|
||||
"holds no other epoch open; no message from a stale epoch refused in the last interval",
|
||||
From: "issue 204", Kind: "lease-split", Phase: 2, run: probeLease},
|
||||
{ID: "D6", Asserts: "every durable consumer the mesh expects exists with its definition, and is near its " +
|
||||
"stream's head", From: "issues 248, 266", Kind: "consumer-wrong", Raises: []string{"consumer-lost", kindConsumerBehind},
|
||||
Phase: 1, run: probeConsumers},
|
||||
{ID: "D7", Asserts: "every stream the controller defines exists with its definition, and its own buckets",
|
||||
From: "issue 208", Kind: "stream-wrong", Phase: 1, run: probeStreams},
|
||||
{ID: "D8", Asserts: "no address the mesh owns — a machine's private address or its endpoint — is in a ban list",
|
||||
From: "issue 238", Kind: "own-address-banned", Phase: 1, run: probeBans,
|
||||
Asks: []broker.SeatVerb{{Seat: "node-intrusion-prevention", Verb: "banned"}}},
|
||||
{ID: "D9", Asserts: "status answers in full within ten seconds, from a summary composed lately",
|
||||
From: "issue 265", Kind: "status-slow", Phase: 1, run: probeStatus},
|
||||
{ID: "D10", Asserts: "every machine runs the node-engine and node tools builds the mesh holds, or is inside " +
|
||||
"a plan's window", From: "the version split", Kind: "core-behind", Phase: 1, run: probeCoreBuilds},
|
||||
{ID: "D11", Asserts: "no provider holds a consumer retired more than thirty days without a person deciding " +
|
||||
"its cleanup", From: "ADR 0230", Kind: kindCleanupWaiting, Phase: 2, run: probeRetired},
|
||||
{ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals",
|
||||
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs},
|
||||
}
|
||||
|
||||
// probeVerdict is one probe's outcome in a run.
|
||||
type probeVerdict struct {
|
||||
ID string `json:"id"`
|
||||
Verdict string `json:"verdict"`
|
||||
Found []string `json:"found,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Took string `json:"took,omitempty"`
|
||||
}
|
||||
|
||||
// doctorCounts are a run's verdicts, counted.
|
||||
type doctorCounts struct {
|
||||
Passed int `json:"passed"`
|
||||
Failed int `json:"failed"`
|
||||
FailedToRun int `json:"failed-to-run"`
|
||||
Deferred int `json:"deferred"`
|
||||
}
|
||||
|
||||
// doctorRun is one run of the registry, and the body of its heartbeat.
|
||||
type doctorRun struct {
|
||||
Run string `json:"run"`
|
||||
At time.Time `json:"at"`
|
||||
Started time.Time `json:"started"`
|
||||
Took string `json:"took"`
|
||||
IntervalSeconds int `json:"interval-seconds"`
|
||||
Counts doctorCounts `json:"counts"`
|
||||
Probes []probeVerdict `json:"probes"`
|
||||
// Controller is the machine that ran it, and Why what started it: the schedule, or a person.
|
||||
Controller string `json:"controller"`
|
||||
Why string `json:"why"`
|
||||
// Unsaid is how many condition transitions this controller could not say, since it started.
|
||||
Unsaid int `json:"unsaid,omitempty"`
|
||||
}
|
||||
|
||||
// doctor is the registry and what its probes need.
|
||||
type doctor struct {
|
||||
open *stores
|
||||
js *broker.JetStream
|
||||
keeper *conditions.Keeper
|
||||
teller conditions.Teller
|
||||
watchdogs *watchdogs
|
||||
host string
|
||||
|
||||
running sync.Mutex
|
||||
mu sync.Mutex
|
||||
last *doctorRun
|
||||
ended time.Time
|
||||
}
|
||||
|
||||
// lastRunEnded is when the last run ended; zero before the first.
|
||||
func (d *doctor) lastRunEnded() time.Time {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
return d.ended
|
||||
}
|
||||
|
||||
// lastRun is the last run's verdict; nil before the first.
|
||||
func (d *doctor) lastRun() *doctorRun {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
return d.last
|
||||
}
|
||||
|
||||
// keep runs the registry on its schedule until ctx ends.
|
||||
func (d *doctor) keep(ctx context.Context) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(doctorFirstAfter):
|
||||
}
|
||||
tick := time.NewTicker(doctorEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
// Only the controller acting checks the mesh on a schedule: one standing by would say a
|
||||
// heartbeat for a self-check that is not the mesh's.
|
||||
if d.watchdogs == nil || d.watchdogs.acting == nil || d.watchdogs.acting() {
|
||||
d.runOnce(ctx, "the schedule")
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var doctorRuns struct {
|
||||
sync.Mutex
|
||||
n uint64
|
||||
}
|
||||
|
||||
// runOnce runs every probe the registry runs, keeps what each found, and says the heartbeat. One run
|
||||
// at a time: a person's `doctor run` during a scheduled one waits for it.
|
||||
func (d *doctor) runOnce(ctx context.Context, why string) doctorRun {
|
||||
d.running.Lock()
|
||||
defer d.running.Unlock()
|
||||
doctorRuns.Lock()
|
||||
doctorRuns.n++
|
||||
n := doctorRuns.n
|
||||
doctorRuns.Unlock()
|
||||
started := time.Now()
|
||||
run := doctorRun{Run: fmt.Sprintf("doctor-%d-%d", started.Unix(), n), Started: started.UTC(),
|
||||
IntervalSeconds: int(doctorEvery / time.Second), Controller: d.host, Why: why}
|
||||
|
||||
type result struct {
|
||||
obs []conditions.Observation
|
||||
err error
|
||||
took time.Duration
|
||||
}
|
||||
results := make([]result, len(probeRegistry))
|
||||
var wg sync.WaitGroup
|
||||
for i, p := range probeRegistry {
|
||||
if p.run == nil {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(i int, p probe) {
|
||||
defer wg.Done()
|
||||
probing, cancel := context.WithTimeout(context.WithValue(ctx, probeAsksKey{}, p), probeWithin)
|
||||
defer cancel()
|
||||
began := time.Now()
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
done <- result{err: fmt.Errorf("the probe panicked: %v", r)}
|
||||
}
|
||||
}()
|
||||
obs, err := p.run(probing, d)
|
||||
done <- result{obs: obs, err: err}
|
||||
}()
|
||||
select {
|
||||
case r := <-done:
|
||||
r.took = time.Since(began)
|
||||
results[i] = r
|
||||
case <-probing.Done():
|
||||
results[i] = result{err: fmt.Errorf("it did not finish within %s", probeWithin), took: time.Since(began)}
|
||||
}
|
||||
}(i, p)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
var blind []conditions.Observation
|
||||
for i, p := range probeRegistry {
|
||||
v := probeVerdict{ID: p.ID}
|
||||
r := results[i]
|
||||
switch {
|
||||
case p.run == nil:
|
||||
v.Verdict = verdictDeferred
|
||||
run.Counts.Deferred++
|
||||
case r.err != nil:
|
||||
v.Verdict, v.Error = verdictFailedToRun, r.err.Error()
|
||||
run.Counts.FailedToRun++
|
||||
blind = append(blind, conditions.Observation{Scope: conditions.ScopeProbe, ID: p.ID, Kind: "probe-failed",
|
||||
Token: "failed", Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the probe %s (%s) could not run: what it checks is not known — never a pass", p.ID, p.Asserts),
|
||||
Said: firstLine(r.err.Error())})
|
||||
default:
|
||||
if err := d.keeper.Reconcile(ctx, p.ID, kindedAs(r.obs, p.Kind)); err != nil {
|
||||
v.Error = "what it found could not be kept: " + err.Error()
|
||||
}
|
||||
if len(r.obs) == 0 {
|
||||
v.Verdict = verdictPass
|
||||
run.Counts.Passed++
|
||||
} else {
|
||||
v.Verdict = verdictFail
|
||||
run.Counts.Failed++
|
||||
for _, o := range r.obs {
|
||||
v.Found = append(v.Found, o.Summary)
|
||||
}
|
||||
}
|
||||
}
|
||||
if p.run != nil {
|
||||
v.Took = r.took.Round(time.Millisecond).String()
|
||||
}
|
||||
run.Probes = append(run.Probes, v)
|
||||
}
|
||||
if err := d.keeper.Reconcile(ctx, sourceDoctor, blind); err != nil {
|
||||
fmt.Printf("the self-check's own failures could not be kept: %v\n", err)
|
||||
}
|
||||
ended := time.Now()
|
||||
run.At, run.Took, run.Unsaid = ended.UTC(), ended.Sub(started).Round(time.Millisecond).String(), d.keeper.Unsaid()
|
||||
d.mu.Lock()
|
||||
d.last, d.ended = &run, ended
|
||||
d.mu.Unlock()
|
||||
d.sayHeartbeat(ctx, run)
|
||||
return run
|
||||
}
|
||||
|
||||
// sourceDoctor is what raises a probe's own failure to run.
|
||||
const sourceDoctor = "doctor"
|
||||
|
||||
// kindedAs gives each observation of a probe the probe's kind where it named none.
|
||||
func kindedAs(obs []conditions.Observation, kind string) []conditions.Observation {
|
||||
out := make([]conditions.Observation, 0, len(obs))
|
||||
for _, o := range obs {
|
||||
if o.Kind == "" {
|
||||
o.Kind = kind
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sayHeartbeat publishes the run's heartbeat. Not said is said here, and S10 on the second machine
|
||||
// says it outward: the watcher hears nothing.
|
||||
func (d *doctor) sayHeartbeat(ctx context.Context, run doctorRun) {
|
||||
if d.teller == nil {
|
||||
return
|
||||
}
|
||||
body, err := json.Marshal(run)
|
||||
if err != nil {
|
||||
fmt.Printf("the self-check's heartbeat could not be written: %v\n", err)
|
||||
return
|
||||
}
|
||||
saying, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if err := d.teller.PublishSeatEvent(saying, conditions.Seat, conditions.HeartbeatEvent, body); err != nil {
|
||||
fmt.Printf("the self-check's heartbeat (%s) could NOT be said, so the watcher on the second machine "+
|
||||
"will say the self-check is silent: %v\n", run.Run, err)
|
||||
}
|
||||
}
|
||||
|
||||
// doctorFrom is the serving controller's self-check; nil in any other process.
|
||||
var doctorFrom *doctor
|
||||
|
||||
// doctorCommand is `doctor`, `doctor run`, `doctor probes` and `doctor signals`.
|
||||
func doctorCommand(ctx context.Context, args []string) error {
|
||||
sub := ""
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
sub, args = args[0], args[1:]
|
||||
}
|
||||
set := flag.NewFlagSet("doctor", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New("doctor [run|probes|signals] [--json]")
|
||||
}
|
||||
answer, err := doctorAnswer(ctx, sub)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return printJSON(answer)
|
||||
}
|
||||
fmt.Print(doctorText(answer))
|
||||
return nil
|
||||
}
|
||||
|
||||
// doctorAnswer is what the verb answers, as data.
|
||||
func doctorAnswer(ctx context.Context, sub string) (any, error) {
|
||||
switch sub {
|
||||
case "":
|
||||
if doctorFrom != nil {
|
||||
if run := doctorFrom.lastRun(); run != nil {
|
||||
return verdictAnswer(*run, time.Now()), nil
|
||||
}
|
||||
return nil, fmt.Errorf("the self-check has not finished its first run yet: it runs %s after the "+
|
||||
"controller starts, then every %s — `doctor run` runs it now", doctorFirstAfter, doctorEvery)
|
||||
}
|
||||
run, err := lastHeartbeat(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return verdictAnswer(run, time.Now()), nil
|
||||
case "run":
|
||||
d := doctorFrom
|
||||
if d == nil {
|
||||
local, closeIt, err := localDoctor(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer closeIt()
|
||||
d = local
|
||||
}
|
||||
return verdictAnswer(d.runOnce(ctx, "asked by "+link.Caller()), time.Now()), nil
|
||||
case "probes":
|
||||
return probesAnswer(), nil
|
||||
case "signals":
|
||||
if doctorFrom == nil || doctorFrom.watchdogs == nil {
|
||||
return nil, errors.New("the age of each signal is known to the serving controller alone, which " +
|
||||
"hears them: ask it through the mesh-controller seat's doctor verb")
|
||||
}
|
||||
return signalsAnswer(doctorFrom.watchdogs.lastFacts(), doctorFrom.watchdogs.lastTick()), nil
|
||||
}
|
||||
return nil, fmt.Errorf("doctor answers the last run, or `run`, `probes` or `signals` — not %q", sub)
|
||||
}
|
||||
|
||||
// verdictAnswer is a run as the verb answers it, with its age.
|
||||
func verdictAnswer(run doctorRun, now time.Time) map[string]any {
|
||||
return map[string]any{"run": run, "age": now.Sub(run.At).Round(time.Second).String(),
|
||||
"note": "a probe that could not run is never a pass; each failure is an open condition until a run passes it"}
|
||||
}
|
||||
|
||||
// probesAnswer is the registry.
|
||||
func probesAnswer() map[string]any {
|
||||
var out []map[string]any
|
||||
for _, p := range probeRegistry {
|
||||
row := map[string]any{"id": p.ID, "asserts": p.Asserts, "from": p.From, "kind": p.Kind, "phase": p.Phase}
|
||||
if len(p.Raises) > 0 {
|
||||
row["raises"] = p.Raises
|
||||
}
|
||||
if p.Deferred != "" {
|
||||
row["deferred"] = p.Deferred
|
||||
}
|
||||
out = append(out, row)
|
||||
}
|
||||
return map[string]any{"probes": out, "every": doctorEvery.String(), "each within": probeWithin.String()}
|
||||
}
|
||||
|
||||
// signalsAnswer is every row of the signals table with the age of its newest signal.
|
||||
func signalsAnswer(f *signalFacts, ticked time.Time) map[string]any {
|
||||
var rows []map[string]any
|
||||
for _, r := range signalsTable {
|
||||
row := map[string]any{"row": r.Row, "signal": r.Signal, "emitter": r.Emitter, "bound": r.Bound,
|
||||
"kind": r.Kind, "severity": r.Severity, "phase": r.Phase}
|
||||
switch {
|
||||
case r.Deferred != "":
|
||||
row["deferred"] = r.Deferred
|
||||
case f == nil:
|
||||
row["newest"] = "not yet looked at"
|
||||
default:
|
||||
if err := r.needs(f); err != nil {
|
||||
row["blind"] = err.Error()
|
||||
} else if newest := r.newest(f); newest.IsZero() {
|
||||
row["newest"] = "none heard"
|
||||
} else {
|
||||
row["newest"] = newest.UTC().Format(time.RFC3339)
|
||||
row["age"] = f.now.Sub(newest).Round(time.Second).String()
|
||||
}
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
out := map[string]any{"signals": rows, "every": watchEvery.String()}
|
||||
if !ticked.IsZero() {
|
||||
out["looked"] = ticked.UTC().Format(time.RFC3339)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// doctorText is an answer as a person reads it.
|
||||
func doctorText(answer any) string {
|
||||
body, _ := json.Marshal(answer)
|
||||
var b strings.Builder
|
||||
var verdict struct {
|
||||
Run doctorRun `json:"run"`
|
||||
Age string `json:"age"`
|
||||
}
|
||||
if json.Unmarshal(body, &verdict) == nil && verdict.Run.Run != "" {
|
||||
r := verdict.Run
|
||||
fmt.Fprintf(&b, "%s, %s ago (took %s, %s): %d passed, %d failed, %d could not run, %d not built yet\n\n",
|
||||
r.Run, verdict.Age, r.Took, r.Why, r.Counts.Passed, r.Counts.Failed, r.Counts.FailedToRun, r.Counts.Deferred)
|
||||
for _, p := range r.Probes {
|
||||
fmt.Fprintf(&b, " %-4s %-14s %s\n", p.ID, p.Verdict, p.Took)
|
||||
for _, f := range p.Found {
|
||||
fmt.Fprintf(&b, " %s\n", f)
|
||||
}
|
||||
if p.Error != "" {
|
||||
fmt.Fprintf(&b, " %s\n", p.Error)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
pretty, _ := json.MarshalIndent(answer, "", " ")
|
||||
return string(pretty) + "\n"
|
||||
}
|
||||
|
||||
// lastHeartbeat is the newest run's heartbeat, read from the events stream: what a process other than
|
||||
// the serving controller answers `doctor` from.
|
||||
func lastHeartbeat(ctx context.Context) (doctorRun, error) {
|
||||
var run doctorRun
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
js, err := conn.JetStream(nats.Context(ctx))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg, err := js.GetLastMsg(broker.EventsStream, link.SeatEventSubject(conditions.Seat, conditions.HeartbeatEvent))
|
||||
if errors.Is(err, nats.ErrMsgNotFound) {
|
||||
return errors.New("the self-check has said no heartbeat on the bus in the last week: it is not running")
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("the self-check's last heartbeat cannot be read: %w", err)
|
||||
}
|
||||
return json.Unmarshal(msg.Data, &run)
|
||||
})
|
||||
return run, err
|
||||
}
|
||||
|
||||
// localDoctor is a self-check run by a process other than the serving controller: its own stores,
|
||||
// its own connection, and its own keeper, closed after.
|
||||
func localDoctor(ctx context.Context) (*doctor, func(), error) {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
js, err := dialTheBus()
|
||||
if err != nil {
|
||||
open.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
k, err := keeperOn(ctx, js.Conn())
|
||||
if err != nil {
|
||||
js.Close()
|
||||
open.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
jsCtx := js.Context()
|
||||
d := &doctor{open: open, js: js, keeper: k, teller: link.OverNATS{Conn: js.Conn(), JS: jsCtx},
|
||||
host: controlHost(ctx, open.inventory)}
|
||||
return d, func() {
|
||||
flushing, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
k.Close(flushing)
|
||||
js.Close()
|
||||
open.Close()
|
||||
}, nil
|
||||
}
|
||||
|
||||
// sortedFound is a probe's findings in a stated order, so two runs over one mesh say the same.
|
||||
func sortedFound(obs []conditions.Observation) []conditions.Observation {
|
||||
sort.Slice(obs, func(i, j int) bool { return obs[i].Key() < obs[j].Key() })
|
||||
return obs
|
||||
}
|
||||
|
||||
// probeAsksKey carries the running probe, so a seat verb it calls is checked against what it declares.
|
||||
type probeAsksKey struct{}
|
||||
|
||||
// declaredBy says whether the probe running in ctx declared a seat verb; outside a probe, false.
|
||||
func declaredBy(ctx context.Context, seat, verb string) (string, bool) {
|
||||
p, ok := ctx.Value(probeAsksKey{}).(probe)
|
||||
if !ok {
|
||||
return "a caller outside the self-check", false
|
||||
}
|
||||
for _, v := range p.Asks {
|
||||
if v.Seat == seat && v.Verb == verb {
|
||||
return p.ID, true
|
||||
}
|
||||
}
|
||||
return p.ID, false
|
||||
}
|
||||
@@ -1,425 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"golang.org/x/net/dns/dnsmessage"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The self-check (novox/hq to-be 45 §4): a probe that fails raises its condition, one that cannot run
|
||||
// raises probe-failed for itself and is never a pass, and every run ends with its heartbeat.
|
||||
|
||||
// withProbes runs the test with a registry of its own.
|
||||
func withProbes(t *testing.T, probes ...probe) {
|
||||
t.Helper()
|
||||
before := probeRegistry
|
||||
probeRegistry = probes
|
||||
t.Cleanup(func() { probeRegistry = before })
|
||||
}
|
||||
|
||||
func TestARunKeepsWhatEachProbeFoundAndSaysItsHeartbeat(t *testing.T) {
|
||||
failing := conditions.Observation{Scope: conditions.ScopeMachine, ID: "anchor", Token: "refused",
|
||||
Severity: conditions.Urgent, Summary: "anchor's node-engine would refuse its declaration"}
|
||||
var broken atomic.Bool
|
||||
broken.Store(true)
|
||||
withProbes(t,
|
||||
probe{ID: "P1", Asserts: "passes", Kind: "never", Phase: 1,
|
||||
run: func(context.Context, *doctor) ([]conditions.Observation, error) { return nil, nil }},
|
||||
probe{ID: "P2", Asserts: "finds a fault", Kind: "declaration-refused", Phase: 1,
|
||||
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
|
||||
if broken.Load() {
|
||||
return []conditions.Observation{failing}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}},
|
||||
probe{ID: "P3", Asserts: "cannot run", Kind: "x", Phase: 1,
|
||||
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
|
||||
if broken.Load() {
|
||||
return nil, errors.New("the store is away")
|
||||
}
|
||||
return nil, nil
|
||||
}},
|
||||
probe{ID: "P4", Asserts: "hangs", Kind: "x", Phase: 1,
|
||||
run: func(ctx context.Context, _ *doctor) ([]conditions.Observation, error) {
|
||||
if broken.Load() {
|
||||
<-ctx.Done()
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
return nil, nil
|
||||
}},
|
||||
probe{ID: "P5", Asserts: "later", Kind: "x", Phase: 2, Deferred: "not yet"},
|
||||
)
|
||||
before := probeWithin
|
||||
probeWithin = 200 * time.Millisecond
|
||||
t.Cleanup(func() { probeWithin = before })
|
||||
|
||||
store := conditions.NewInMemory()
|
||||
told := &conditions.Told{}
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||
defer k.Close(context.Background())
|
||||
d := &doctor{keeper: k, teller: told, host: "anchor"}
|
||||
run := d.runOnce(t.Context(), "a test")
|
||||
if run.Counts != (doctorCounts{Passed: 1, Failed: 1, FailedToRun: 2, Deferred: 1}) {
|
||||
t.Fatalf("counted %+v", run.Counts)
|
||||
}
|
||||
open, err := k.Open(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keys []string
|
||||
for _, c := range open {
|
||||
keys = append(keys, c.Key+"="+c.Kind)
|
||||
}
|
||||
for _, want := range []string{"machine.anchor.refused=declaration-refused", "probe.P3.failed=probe-failed",
|
||||
"probe.P4.failed=probe-failed"} {
|
||||
if !slices.Contains(keys, want) {
|
||||
t.Errorf("%s is not open: %v", want, keys)
|
||||
}
|
||||
}
|
||||
// The heartbeat, in the shape mesh-watcher reads (the contract with the operator's channel).
|
||||
if len(told.Names) != 1 || told.Names[0] != conditions.HeartbeatEvent {
|
||||
t.Fatalf("said %v", told.Names)
|
||||
}
|
||||
if d.lastRunEnded().IsZero() || d.lastRun().Run != run.Run {
|
||||
t.Fatal("the run is not the last verdict")
|
||||
}
|
||||
body, _ := json.Marshal(run)
|
||||
var shape map[string]any
|
||||
_ = json.Unmarshal(body, &shape)
|
||||
for _, field := range []string{"run", "at", "interval-seconds", "counts", "probes", "controller"} {
|
||||
if _, ok := shape[field]; !ok {
|
||||
t.Errorf("the heartbeat carries no %q: %s", field, body)
|
||||
}
|
||||
}
|
||||
|
||||
// Mended: the next run clears every one of them.
|
||||
broken.Store(false)
|
||||
d.runOnce(t.Context(), "a test")
|
||||
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||
t.Fatalf("a passing run left open %+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
// **The registry says what each probe asserts**, and a probe not built says why and when.
|
||||
func TestTheRegistryIsTheDesignsLiveForm(t *testing.T) {
|
||||
seen := map[string]bool{}
|
||||
for _, p := range probeRegistry {
|
||||
if seen[p.ID] {
|
||||
t.Errorf("%s twice", p.ID)
|
||||
}
|
||||
seen[p.ID] = true
|
||||
if p.Asserts == "" || p.From == "" || p.Kind == "" {
|
||||
t.Errorf("%s does not say what it asserts, where from, or what it raises", p.ID)
|
||||
}
|
||||
if (p.run == nil) != (p.Deferred != "") || (p.Deferred != "" && p.Phase <= 1) {
|
||||
t.Errorf("%s is run and deferred, or neither, or deferred out of Phase 1: %+v", p.ID, p)
|
||||
}
|
||||
}
|
||||
for _, id := range []string{"D1", "D2", "D3", "D4", "D5", "D6", "D7", "D8", "D9", "D10"} {
|
||||
if !seen[id] {
|
||||
t.Errorf("to-be 45 §4 has %s and the registry does not", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **The doctor and the watchdogs watch each other**: watchdogs that stopped are DW; a self-check that
|
||||
// stopped is S10 (signals_test.go).
|
||||
func TestWatchdogsThatStoppedAreSaid(t *testing.T) {
|
||||
w := &watchdogs{started: time.Now().Add(-time.Hour)}
|
||||
d := &doctor{watchdogs: w, host: "anchor"}
|
||||
got, err := probeWatchdogs(t.Context(), d)
|
||||
if err != nil || len(got) != 1 || got[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("%+v %v", got, err)
|
||||
}
|
||||
w.ticked = time.Now()
|
||||
if got, _ := probeWatchdogs(t.Context(), d); len(got) != 0 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **D1 composes every machine of a healthy mesh and the host's own validator takes each.**
|
||||
func TestEveryMachineOfAHealthyMeshComposesAndValidates(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
got, err := probeDeclarations(t.Context(), &doctor{open: open})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 0 {
|
||||
t.Fatalf("a healthy mesh failed D1: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **D1 names a machine nothing can be sent to**, and the network that cannot be computed for it.
|
||||
func TestAMachineWhoseDeclarationDoesNotComposeIsSaid(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
one, two := rivals()
|
||||
register(t, open, one)
|
||||
register(t, open, two)
|
||||
for _, m := range []string{"rival-one", "rival-two"} {
|
||||
if _, err := assign(ctx, open, "laptop", m); err != nil && m == "rival-one" {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, err := probeDeclarations(ctx, &doctor{open: open})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "the-seat") {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **D2: a resolver answering NXDOMAIN for IPv6 is wrong** — musl takes it as no such name (issue 262).
|
||||
func TestAResolverAnsweringNoSuchNameForIPv6IsWrong(t *testing.T) {
|
||||
answerAs := func(rcode dnsmessage.RCode) string {
|
||||
conn, err := net.ListenPacket("udp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = conn.Close() })
|
||||
go func() {
|
||||
buf := make([]byte, 1500)
|
||||
for {
|
||||
n, from, err := conn.ReadFrom(buf)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var q dnsmessage.Message
|
||||
if q.Unpack(buf[:n]) != nil {
|
||||
continue
|
||||
}
|
||||
reply := dnsmessage.Message{Header: dnsmessage.Header{ID: q.ID, Response: true}, Questions: q.Questions}
|
||||
if q.Questions[0].Type == dnsmessage.TypeA {
|
||||
reply.Answers = []dnsmessage.Resource{{Header: dnsmessage.ResourceHeader{Name: q.Questions[0].Name,
|
||||
Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET}, Body: &dnsmessage.AResource{A: [4]byte{10, 77, 0, 1}}}}
|
||||
} else {
|
||||
reply.RCode = rcode
|
||||
}
|
||||
packed, _ := reply.Pack()
|
||||
_, _ = conn.WriteTo(packed, from)
|
||||
}
|
||||
}()
|
||||
_, port, _ := net.SplitHostPort(conn.LocalAddr().String())
|
||||
return port
|
||||
}
|
||||
before := resolverPort
|
||||
t.Cleanup(func() { resolverPort = before })
|
||||
|
||||
resolverPort = answerAs(dnsmessage.RCodeSuccess)
|
||||
v4, rcode, err := askResolver(t.Context(), "127.0.0.1", "anchor.internal", dnsmessage.TypeA)
|
||||
if err != nil || rcode != dnsmessage.RCodeSuccess || !slices.Equal(v4, []string{"10.77.0.1"}) {
|
||||
t.Fatalf("%v %v %v", v4, rcode, err)
|
||||
}
|
||||
v6, rcode, err := askResolver(t.Context(), "127.0.0.1", "anchor.internal", dnsmessage.TypeAAAA)
|
||||
if err != nil || rcode != dnsmessage.RCodeSuccess || len(v6) != 0 {
|
||||
t.Fatalf("NODATA read as %v %v %v", v6, rcode, err)
|
||||
}
|
||||
resolverPort = answerAs(dnsmessage.RCodeNameError)
|
||||
if _, rcode, _ := askResolver(t.Context(), "127.0.0.1", "anchor.internal", dnsmessage.TypeAAAA); rcode != dnsmessage.RCodeNameError {
|
||||
t.Fatalf("NXDOMAIN read as %v", rcode)
|
||||
}
|
||||
}
|
||||
|
||||
// **D6, D7: what the controller defines is what it finds**, and a consumer deleted or a stream
|
||||
// redefined is said — against a real bus, raised by the same derivation the controller starts with.
|
||||
func TestNatsTheBusIsWhatTheControllerDefines(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
open := aMesh(t)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
|
||||
_ = js.Context().DeleteStream(s)
|
||||
}
|
||||
if _, err := assertBusObjects(t.Context(), open.inventory, js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &doctor{open: open, js: js}
|
||||
for _, p := range []func(context.Context, *doctor) ([]conditions.Observation, error){probeConsumers, probeStreams} {
|
||||
got, err := p(t.Context(), d)
|
||||
if err != nil || len(got) != 0 {
|
||||
t.Fatalf("a bus just raised fails: %+v %v", got, err)
|
||||
}
|
||||
}
|
||||
if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := js.Context().StreamInfo("EVENTS")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := info.Config
|
||||
cfg.MaxMsgsPerSubject = 3
|
||||
if _, err := js.Context().UpdateStream(&cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
consumers, err := probeConsumers(t.Context(), d)
|
||||
if err != nil || len(consumers) != 1 || consumers[0].Key() != "bus.NODES.laptop.missing" {
|
||||
t.Fatalf("the deleted consumer: %+v %v", consumers, err)
|
||||
}
|
||||
streams, err := probeStreams(t.Context(), d)
|
||||
if err != nil || len(streams) != 1 || !strings.Contains(streams[0].Summary, "per subject") {
|
||||
t.Fatalf("the redefined stream: %+v %v", streams, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **S9 hears the bus**: a consumer that gives up on a message, and one deleted, as the server says.
|
||||
func TestNatsTheBusSaysAConsumerGaveUpAndOneWasDeleted(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
heard := make(chan *nats.Msg, 16)
|
||||
for _, subject := range broker.BusAdvisories {
|
||||
if _, err := conn.ChanSubscribe(subject, heard); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
api, _ := jetstream.New(conn)
|
||||
_ = api.DeleteStream(t.Context(), "SEAT_ADVISED")
|
||||
stream, err := api.CreateStream(t.Context(), jetstream.StreamConfig{Name: "SEAT_ADVISED", Subjects: []string{"advised.>"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = api.DeleteStream(context.Background(), "SEAT_ADVISED") }()
|
||||
consumer, err := stream.CreateConsumer(t.Context(), jetstream.ConsumerConfig{Durable: "SEAT_ADVISED_worker",
|
||||
AckPolicy: jetstream.AckExplicitPolicy, MaxDeliver: 1, AckWait: 100 * time.Millisecond})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := api.Publish(t.Context(), "advised.x", []byte("x")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := consumer.Fetch(1, jetstream.FetchMaxWait(time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A seat's worker, so the deletion is of a consumer the mesh names (link.MeshNamed).
|
||||
// Not acknowledged: after its one delivery the consumer gives up on it — on the next fetch.
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
_, _ = consumer.Fetch(1, jetstream.FetchMaxWait(300*time.Millisecond))
|
||||
if err := stream.DeleteConsumer(t.Context(), "SEAT_ADVISED_worker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kinds := map[string]string{}
|
||||
deadline := time.After(5 * time.Second)
|
||||
for len(kinds) < 2 {
|
||||
select {
|
||||
case m := <-heard:
|
||||
if a, ok := link.ReadAdvisory(m.Subject, m.Data); ok {
|
||||
kinds[a.Kind] = a.Said
|
||||
}
|
||||
case <-deadline:
|
||||
t.Fatalf("the bus said only %v", kinds)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(kinds["max-deliveries"], "gave up") || !strings.Contains(kinds["consumer-lost"], "was deleted") {
|
||||
t.Fatalf("%v", kinds)
|
||||
}
|
||||
}
|
||||
|
||||
// **D10 compares a node-engine with what it is delivered as, not with its commit** (2026-10-06: every
|
||||
// machine read as behind right after a push sent it the current build — it says the digest-named
|
||||
// directory it runs from, and the mesh holds a commit).
|
||||
func TestANodeEngineIsJudgedByTheVersionItIsDeliveredAs(t *testing.T) {
|
||||
m := catalogue.Manifest{Module: "mesh-host", Resources: []map[string]any{
|
||||
{"id": "launcher", "type": "file", "path": "/usr/lib/nox-mesh-host/launch"},
|
||||
{"id": "host", "type": "archive", "path": "/usr/lib/nox-mesh-host/versions/31045596c83a"},
|
||||
{"id": "unfilled", "type": "archive", "path": "/usr/lib/x/versions/${version}"},
|
||||
}}
|
||||
delivered := deliveredVersions(m)
|
||||
if !slices.Equal(delivered, []string{"31045596c83a"}) {
|
||||
t.Fatalf("%v", delivered)
|
||||
}
|
||||
commit := "1545b00a9f0c"
|
||||
for _, c := range []struct {
|
||||
reported string
|
||||
behind bool
|
||||
}{
|
||||
{"31045596c83a", false}, // the live case: current, and was called behind
|
||||
{"0123456789ab", true}, // another delivery
|
||||
{"1545b00a", false}, // placed by hand, stamped with the commit
|
||||
{"", false}, // not said
|
||||
} {
|
||||
if got := engineBehind(c.reported, delivered, commit); got != c.behind {
|
||||
t.Errorf("%q behind = %v, want %v", c.reported, got, c.behind)
|
||||
}
|
||||
}
|
||||
if engineBehind("31045596c83a", nil, commit) {
|
||||
t.Error("behind a mesh that holds no delivered build")
|
||||
}
|
||||
}
|
||||
|
||||
// **Every seat verb a probe calls is one it declares, and one the controller is granted** — derived
|
||||
// from the registry, so a probe added with a question the bus would refuse fails here, not live.
|
||||
func TestEverySeatVerbAProbeAsksIsGranted(t *testing.T) {
|
||||
granted, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asked := 0
|
||||
for _, p := range probeRegistry {
|
||||
for _, v := range p.Asks {
|
||||
asked++
|
||||
subject := link.NodeSeatToolSubject(v.Seat, v.Verb, "anchor")
|
||||
if !slices.ContainsFunc(granted.Publish, func(pattern string) bool { return subjectMatches(pattern, subject) }) {
|
||||
t.Errorf("%s asks %s.%s and the controller may not publish %s", p.ID, v.Seat, v.Verb, subject)
|
||||
}
|
||||
if !slices.Contains(broker.VerbsTheSelfCheckAsks, v) {
|
||||
t.Errorf("%s asks %s.%s, which broker.VerbsTheSelfCheckAsks does not name", p.ID, v.Seat, v.Verb)
|
||||
}
|
||||
}
|
||||
}
|
||||
if asked == 0 {
|
||||
t.Fatal("no probe asks a seat verb: D8 lost its declaration")
|
||||
}
|
||||
// And a probe asking what it did not declare is refused before anything is sent.
|
||||
ctx := context.WithValue(t.Context(), probeAsksKey{}, probe{ID: "DX"})
|
||||
if _, err := askSeatTool(ctx, nil, "node-intrusion-prevention", "banned", "anchor"); err == nil ||
|
||||
!strings.Contains(err.Error(), "does not declare") {
|
||||
t.Fatalf("an undeclared question was asked: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// subjectMatches is the bus's matching of a permission pattern against a subject.
|
||||
func subjectMatches(pattern, subject string) bool {
|
||||
p, s := strings.Split(pattern, "."), strings.Split(subject, ".")
|
||||
for i, tok := range p {
|
||||
if tok == ">" {
|
||||
return len(s) > i
|
||||
}
|
||||
if i >= len(s) || (tok != "*" && tok != s[i]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(p) == len(s)
|
||||
}
|
||||
@@ -1,170 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// What the core's bounds are set from (novox/hq to-be 45 Phase 0).
|
||||
//
|
||||
// **A bound is set from what was measured, not from what seemed reasonable.** Phase 1 puts a watchdog
|
||||
// on each row of the signals table, and each has a bound: S1 three heartbeat intervals, S2 three times
|
||||
// a machine's last apply, S3 a tier's build and apply time, S6 a build's timeout. Marked provisional
|
||||
// in the design until a fortnight of these says what the mesh actually takes. Recorded by the serving
|
||||
// controller as it hears each — a send's first report, a machine's next word, a plan leaving a tier, a
|
||||
// build's outcome — and summarised here per machine, repository or module.
|
||||
|
||||
// recordBuildDuration measures one build from its ask to its outcome heard.
|
||||
func recordBuildDuration(ctx context.Context, inv *inventory.Inventory, result link.BuildResult, asked time.Time) {
|
||||
if asked.IsZero() || result.ID == "" {
|
||||
return
|
||||
}
|
||||
subject := result.Module
|
||||
if subject == "" {
|
||||
subject = result.Repository
|
||||
}
|
||||
detail := "built"
|
||||
if result.Failed != "" {
|
||||
detail = "failed: " + firstLine(result.Failed)
|
||||
}
|
||||
if err := inv.RecordDuration(ctx, inventory.Duration{Kind: inventory.DurationBuild, Subject: subject,
|
||||
Node: result.On, Ref: result.ID, Started: asked, Took: time.Since(asked), Detail: detail}); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: how long it took could not be recorded: %v\n", result.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// durationSummary is one subject's measurements of one kind.
|
||||
type durationSummary struct {
|
||||
Kind string `json:"kind"`
|
||||
Subject string `json:"subject"`
|
||||
Count int `json:"count"`
|
||||
Median string `json:"median"`
|
||||
P90 string `json:"p90"`
|
||||
Max string `json:"max"`
|
||||
// Bound is what to-be 45's rule would make of these, where the rule is a multiple of a measured
|
||||
// time: three times the slowest apply (S2), three times the median word interval (S1).
|
||||
Suggests string `json:"suggests,omitempty"`
|
||||
}
|
||||
|
||||
func summarise(ds []inventory.Duration) []durationSummary {
|
||||
type key struct{ kind, subject string }
|
||||
by := map[key][]time.Duration{}
|
||||
for _, d := range ds {
|
||||
k := key{d.Kind, d.Subject}
|
||||
by[k] = append(by[k], d.Took)
|
||||
}
|
||||
var out []durationSummary
|
||||
for k, took := range by {
|
||||
slices.Sort(took)
|
||||
at := func(q float64) time.Duration { return took[int(q*float64(len(took)-1))] }
|
||||
s := durationSummary{Kind: k.kind, Subject: k.subject, Count: len(took),
|
||||
Median: round(at(0.5)), P90: round(at(0.9)), Max: round(took[len(took)-1])}
|
||||
switch k.kind {
|
||||
case inventory.DurationApply:
|
||||
s.Suggests = "S2 bound max(2m, 3×last apply) ≈ " + round(max(2*time.Minute, 3*at(0.9))) + " at the p90"
|
||||
case inventory.DurationHeartbeatGap:
|
||||
s.Suggests = "S1 bound 3×interval ≈ " + round(3*at(0.5))
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
ki, kj := slices.Index(inventory.DurationKinds, out[i].Kind), slices.Index(inventory.DurationKinds, out[j].Kind)
|
||||
if ki != kj {
|
||||
return ki < kj
|
||||
}
|
||||
return out[i].Subject < out[j].Subject
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func round(d time.Duration) string {
|
||||
switch {
|
||||
case d < time.Second:
|
||||
return d.Round(time.Millisecond).String()
|
||||
case d < time.Minute:
|
||||
return d.Round(100 * time.Millisecond).String()
|
||||
default:
|
||||
return d.Round(time.Second).String()
|
||||
}
|
||||
}
|
||||
|
||||
// durationsCommand is `durations`: the summary per kind and subject, or every measurement as data.
|
||||
func durationsCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("durations", flag.ContinueOnError)
|
||||
kind := set.String("kind", "", "one kind: "+strings.Join(inventory.DurationKinds, ", "))
|
||||
days := set.Int("days", 14, "how many days back")
|
||||
asJSON := set.Bool("json", false, "the summary as data")
|
||||
all := set.Bool("all", false, "every measurement rather than the summary")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
}
|
||||
if *kind != "" && !slices.Contains(inventory.DurationKinds, *kind) {
|
||||
return fmt.Errorf("%q is not a kind of duration: %s", *kind, strings.Join(inventory.DurationKinds, ", "))
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
ds, err := open.inventory.Durations(ctx, *kind, time.Now().Add(-time.Duration(*days)*24*time.Hour))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *all {
|
||||
body, err := json.MarshalIndent(ds, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
summary := summarise(ds)
|
||||
if *asJSON {
|
||||
body, err := json.MarshalIndent(map[string]any{"days": *days, "durations": summary}, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
if len(summary) == 0 {
|
||||
fmt.Printf("nothing measured in the last %d day(s): the serving controller records apply, heartbeat-gap, "+
|
||||
"plan-tier and build durations as it hears them\n", *days)
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("durations over the last %d day(s) — what the core's bounds are set from (to-be 45 Phase 0)\n\n", *days)
|
||||
fmt.Printf(" %-14s %-28s %6s %10s %10s %10s\n", "kind", "of", "count", "median", "p90", "max")
|
||||
for _, s := range summary {
|
||||
fmt.Printf(" %-14s %-28s %6d %10s %10s %10s\n", s.Kind, s.Subject, s.Count, s.Median, s.P90, s.Max)
|
||||
if s.Suggests != "" {
|
||||
fmt.Printf(" %-14s %-28s %s\n", "", "", s.Suggests)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// forgettingOldDurations removes what is older than a month, at start and daily after.
|
||||
func forgettingOldDurations(ctx context.Context, inv *inventory.Inventory) {
|
||||
for {
|
||||
if n, err := inv.ForgetOldDurations(ctx); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "durations older than %s could not be removed: %v\n", inventory.DurationsKeptFor, err)
|
||||
} else if n > 0 {
|
||||
fmt.Printf("removed %d duration(s) older than %s\n", n, inventory.DurationsKeptFor)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(24 * time.Hour):
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,115 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A declaration carries the lease's epoch (novox/hq to-be 45 §6) — to a machine whose node-engine said
|
||||
// it reads one, and to no other: an older node-engine refuses a key it does not know, whole.
|
||||
|
||||
// bodiesDelivery records each send as the mesh does, and keeps the bodies.
|
||||
type bodiesDelivery struct {
|
||||
recordedDelivery
|
||||
bodies map[string][]byte
|
||||
}
|
||||
|
||||
func (b *bodiesDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
|
||||
b.bodies[s.node] = body
|
||||
return b.recordedDelivery.declare(ctx, s, body)
|
||||
}
|
||||
|
||||
func TestAMachineIsSentTheEpochOnlyOnceItSaysItReadsOne(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
epoch := uint64(57)
|
||||
was := epochForActs
|
||||
epochForActs = func(context.Context) (uint64, error) { return epoch, nil }
|
||||
t.Cleanup(func() { epochForActs = was })
|
||||
|
||||
anchor, err := inv.NodeByName(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordReadsEpoch(ctx, anchor.ID, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &bodiesDelivery{recordedDelivery: recordedDelivery{inv: inv}, bodies: map[string][]byte{}}
|
||||
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, composeForPush(open, gens), d, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
carried := func(node string) (epoch float64, has bool) {
|
||||
var envelope map[string]any
|
||||
if err := json.Unmarshal(d.bodies[node], &envelope); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
epoch, has = envelope["epoch"].(float64)
|
||||
return epoch, has
|
||||
}
|
||||
if e, has := carried("anchor"); !has || e != 57 {
|
||||
t.Fatalf("the machine that reads an epoch was sent %v: %s", e, d.bodies["anchor"])
|
||||
}
|
||||
if _, has := carried("laptop"); has {
|
||||
t.Fatalf("a machine that never said it reads an epoch was sent one: %s", d.bodies["laptop"])
|
||||
}
|
||||
|
||||
// A new holder of the lease is not a change of the machine: neither reads as behind.
|
||||
epoch = 58
|
||||
would, err := wouldSend(ctx, open, mustNodes(t, open))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
sent, err := inv.Outstanding(ctx, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if would[node] != sent {
|
||||
t.Fatalf("%s reads as behind after the lease changed hands, with nothing else changed", node)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A process that may not act composes nothing and sends nothing: its number is not taken.
|
||||
func TestNothingIsComposedOrSentWithoutTheLease(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
was := epochForActs
|
||||
epochForActs = func(context.Context) (uint64, error) { return 0, errors.New("this controller lost the lease") }
|
||||
t.Cleanup(func() { epochForActs = was })
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &bodiesDelivery{recordedDelivery: recordedDelivery{inv: open.inventory}, bodies: map[string][]byte{}}
|
||||
refused, err := sendRound(ctx, open, []string{"anchor"}, composeForPush(open, gens), d, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(d.bodies) != 0 || len(refused) != 1 || !strings.Contains(refused[0], "lost the lease") {
|
||||
t.Fatalf("a controller without the lease composed %d and refused %v", len(d.bodies), refused)
|
||||
}
|
||||
anchor, _ := open.inventory.NodeByName(ctx, "anchor")
|
||||
if seq, _ := open.inventory.Sequence(ctx, anchor.ID); seq != 0 {
|
||||
t.Fatalf("a controller without the lease took sequence %d", seq)
|
||||
}
|
||||
|
||||
// And at the send itself: the gate every declaration passes.
|
||||
gate := link.ActingGate
|
||||
link.ActingGate = func(context.Context) error { return errors.New("this controller lost the lease") }
|
||||
t.Cleanup(func() { link.ActingGate = gate })
|
||||
if err := link.Declare(ctx, nil, nil, "anchor", []byte(`{"declaration":1}`), 0); err == nil ||
|
||||
!strings.Contains(err.Error(), "lost the lease") {
|
||||
t.Fatalf("a declaration was let through the gate: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -1,90 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A value given by hand lives only until the module's first good start (novox/hq ADR 0228).
|
||||
//
|
||||
// The serving controller hears every report; a clean one about the declaration a machine was last
|
||||
// sent is the signal the store asks about (inventory.ReplaceGivenAfterStart). What it replaced is
|
||||
// sent at once, said in the log and stated on the bus as the controller seat's `secret-replaced`,
|
||||
// so a replacement is never silent. **Not in the hand-act log**: nobody acted by hand, and that log
|
||||
// is read as the count of repairs a healer is wanted for.
|
||||
|
||||
// SecretReplaced is the controller seat's fact that a value given by hand was replaced
|
||||
// (link.KeySecretReplaced). Never the value: neither the old one, which the mesh cannot read,
|
||||
// nor the new one, sealed to the machine as it was made.
|
||||
type SecretReplaced struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Name string `json:"name"`
|
||||
Given time.Time `json:"given"`
|
||||
// Sent are the machines sent so the module starts again on the new value; Unsent says why
|
||||
// they could not be, in which case the next push carries it.
|
||||
Sent []string `json:"sent"`
|
||||
Unsent string `json:"unsent,omitempty"`
|
||||
Why string `json:"why"`
|
||||
}
|
||||
|
||||
// givenEvents is where the serving controller states it; nil in a command.
|
||||
var givenEvents link.Bus
|
||||
|
||||
// replacing keeps one replacement per machine at a time: two reports arriving together find the
|
||||
// same rows, and the store's claim makes one of them the replacer, but the sends need not race.
|
||||
var replacing sync.Map
|
||||
|
||||
// startedWell says a report is a machine's clean account of a declaration: everything applied,
|
||||
// nothing failed or refused. Whether it is the declaration last sent is the store's to answer.
|
||||
func startedWell(report link.Report) bool {
|
||||
return report.Declared != "" && report.Refused == "" && len(report.Failed) == 0 && report.Applied != nil
|
||||
}
|
||||
|
||||
const givenWhy = "a value given by hand lives only until its module's first good start under the mesh (novox/hq ADR 0228)"
|
||||
|
||||
// replaceGiven replaces what the report makes due, sends the machines, and says so.
|
||||
func replaceGiven(ctx context.Context, open *stores, report link.Report) {
|
||||
if _, busy := replacing.LoadOrStore(report.Node, true); busy {
|
||||
return
|
||||
}
|
||||
defer replacing.Delete(report.Node)
|
||||
replaced, err := open.inventory.ReplaceGivenAfterStart(ctx, report.Node, report.Declared)
|
||||
if err != nil {
|
||||
log.Printf("a value given by hand on %s could not be replaced after its module started: %v", report.Node, err)
|
||||
}
|
||||
for _, r := range replaced {
|
||||
said := SecretReplaced{Node: report.Node, Module: r.Module, Name: r.Name, Given: r.Given.UTC(),
|
||||
Sent: r.Machines, Why: givenWhy}
|
||||
log.Printf("replaced %q of %s on %s, given %s, with a value the mesh made: %s; sending %s",
|
||||
r.Name, r.Module, report.Node, r.Given.UTC().Format(time.RFC3339), givenWhy, strings.Join(r.Machines, ", "))
|
||||
if err := sendTo(ctx, open, r.Machines); err != nil {
|
||||
said.Sent, said.Unsent = nil, err.Error()
|
||||
log.Printf("the new %q of %s is sealed and not yet delivered to %s — the next push carries it: %v",
|
||||
r.Name, r.Module, strings.Join(r.Machines, ", "), err)
|
||||
}
|
||||
stateReplaced(ctx, said)
|
||||
}
|
||||
}
|
||||
|
||||
func stateReplaced(ctx context.Context, said SecretReplaced) {
|
||||
if givenEvents == nil {
|
||||
return
|
||||
}
|
||||
body, err := json.Marshal(said)
|
||||
if err != nil {
|
||||
log.Printf("could not say that %s's %q was replaced: %v", said.Module, said.Name, err)
|
||||
return
|
||||
}
|
||||
stating, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if err := givenEvents.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeySecretReplaced, body); err != nil {
|
||||
log.Printf("could not say that %s's %q was replaced: %v", said.Module, said.Name, err)
|
||||
}
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A rotation asked through the seat carries why to the command, which records it in the hand-act
|
||||
// log (novox/hq ADR 0228); why with a provision is refused as passed over, not dropped.
|
||||
func TestARotationThroughTheSeatCarriesWhy(t *testing.T) {
|
||||
argv, err := argvFor("rotate", map[string]any{"node": "anchor", "module": "letta",
|
||||
"secret": "server-password", "why": "leaked into logs", "cause": "leaked"})
|
||||
if err != nil || strings.Join(argv, " ") != "secret rotate anchor letta server-password --why leaked into logs --cause leaked" {
|
||||
t.Fatalf("%v %v", argv, err)
|
||||
}
|
||||
argv, err = argvFor("rotate", map[string]any{"node": "anchor", "module": "letta", "secret": "server-password"})
|
||||
if err != nil || strings.Join(argv, " ") != "secret rotate anchor letta server-password" {
|
||||
t.Fatalf("without why: %v %v", argv, err)
|
||||
}
|
||||
if argv, err := argvFor("rotate", map[string]any{"provision": "postgres-database", "why": "leaked"}); err == nil {
|
||||
t.Fatalf("why beside a provision was passed over: %v", argv)
|
||||
}
|
||||
}
|
||||
|
||||
// Only a clean account of a declaration is a good start; a refusal, a failure or a bare word that
|
||||
// the machine is there is not (novox/hq ADR 0228).
|
||||
func TestAGoodStartIsACleanAccountOfADeclaration(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
report link.Report
|
||||
good bool
|
||||
}{
|
||||
{link.Report{Node: "anchor", Declared: "d", Applied: []string{"container:letta"}}, true},
|
||||
{link.Report{Node: "anchor", Declared: "d", Applied: []string{}}, true},
|
||||
{link.Report{Node: "anchor"}, false},
|
||||
{link.Report{Node: "anchor", Applied: []string{"x"}}, false},
|
||||
{link.Report{Node: "anchor", Declared: "d", Applied: []string{"x"}, Failed: map[string]string{"y": "no"}}, false},
|
||||
{link.Report{Node: "anchor", Declared: "d", Refused: "older"}, false},
|
||||
} {
|
||||
if got := startedWell(c.report); got != c.good {
|
||||
t.Errorf("%+v: a good start = %v, want %v", c.report, got, c.good)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,197 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// Acts done by hand, and why (novox/hq to-be 45 §7).
|
||||
//
|
||||
// **Which verbs ask why.** `plans close` and `plans stop`, `broker consumer-reset` and `hand-act
|
||||
// record` refuse without it everywhere: nothing automated runs them, so a call without a reason is a
|
||||
// person who has not given one. A named `push` asks for it through the mesh-controller seat, which is
|
||||
// how a person or an agent acts by hand on the mesh; at a shell `--why` is recorded when given and not
|
||||
// required, because the installer and the lab push by command line as a step of what they do, and a
|
||||
// step of a procedure is not a repair. `conditions silence` joins them when the condition store does
|
||||
// (Phase 1).
|
||||
|
||||
// handActFlags are the flags every repairing verb takes.
|
||||
type handActFlags struct {
|
||||
why, cause, condition *string
|
||||
}
|
||||
|
||||
func addHandActFlags(set *flag.FlagSet) handActFlags {
|
||||
return handActFlags{
|
||||
why: set.String("why", "", "why this is done by hand — recorded in the hand-act log (novox/hq to-be 45 §7)"),
|
||||
cause: set.String("cause", "", "the cause, in a word or a condition's kind; the verb's own name when not given"),
|
||||
condition: set.String("condition", "", "the key of the condition this act addresses, if any"),
|
||||
}
|
||||
}
|
||||
|
||||
// given is whether a reason was given.
|
||||
func (f handActFlags) given() bool { return strings.TrimSpace(*f.why) != "" }
|
||||
|
||||
// require refuses an act without a reason, before anything is done.
|
||||
func (f handActFlags) require(verb string) error {
|
||||
if f.given() {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%s is a repair done by hand, and says why: --why <text> (recorded in the hand-act "+
|
||||
"log, novox/hq to-be 45 §7). Nothing was done", verb)
|
||||
}
|
||||
|
||||
// handActConn is the serving controller's connection, for what it reads of the log itself; a
|
||||
// command dials its own.
|
||||
var handActConn *nats.Conn
|
||||
|
||||
// onTheBus runs f with a connection to the bus: the serving controller's, or one of its own.
|
||||
func onTheBus(f func(*nats.Conn) error) error {
|
||||
if handActConn != nil {
|
||||
return f(handActConn)
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot reach the bus: %w", err)
|
||||
}
|
||||
defer js.Close()
|
||||
return f(js.Conn())
|
||||
}
|
||||
|
||||
// record writes the entry for an act about to be done. **Before the act, and never instead of it**:
|
||||
// a log that cannot be written is said loudly, and the repair it was about still happens — a mesh
|
||||
// whose bus is down is exactly the mesh somebody is repairing by hand.
|
||||
func (f handActFlags) record(ctx context.Context, verb string, args []string) {
|
||||
if !f.given() {
|
||||
return
|
||||
}
|
||||
act := link.HandAct{Verb: verb, Args: args, Why: strings.TrimSpace(*f.why),
|
||||
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
written, err := link.RecordHandAct(ctx, conn, act)
|
||||
act = written
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "this act by hand could NOT be recorded in the hand-act log, and is done anyway: %v\n", err)
|
||||
return
|
||||
}
|
||||
fmt.Printf("recorded as %s in the hand-act log: %s, because %q (cause: %s)\n", act.ID, act.By, act.Why, act.Cause)
|
||||
}
|
||||
|
||||
// handActCommand is `hand-act record` and `hand-acts`.
|
||||
func handActCommand(ctx context.Context, args []string) error {
|
||||
if len(args) > 0 && args[0] == "record" {
|
||||
set := flag.NewFlagSet("hand-act record", flag.ContinueOnError)
|
||||
f := addHandActFlags(set)
|
||||
positionals, err := parseAround(set, args[1:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
what := strings.TrimSpace(strings.Join(positionals, " "))
|
||||
if what == "" {
|
||||
return errors.New("hand-act record <what was done> --why <text> [--cause <word>] [--condition <key>]")
|
||||
}
|
||||
if err := f.require("hand-act record"); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(*f.cause) == "" {
|
||||
return errors.New("hand-act record says the cause too: --cause <word>, the word a second " +
|
||||
"act for the same reason will use — it is how a repair done twice is found")
|
||||
}
|
||||
act := link.HandAct{Verb: "hand-act record", Args: []string{what}, Why: strings.TrimSpace(*f.why),
|
||||
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
written, err := link.RecordHandAct(ctx, conn, act)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the act could not be recorded: %w", err)
|
||||
}
|
||||
fmt.Printf("recorded as %s: %s did %q, because %q (cause: %s)\n", written.ID, written.By, what,
|
||||
written.Why, written.Cause)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if len(args) > 0 && args[0] != "list" && !strings.HasPrefix(args[0], "-") {
|
||||
return errors.New("hand-act record <what> --why <text> --cause <word> | hand-acts [--days N] [--json]")
|
||||
}
|
||||
if len(args) > 0 && args[0] == "list" {
|
||||
args = args[1:]
|
||||
}
|
||||
set := flag.NewFlagSet("hand-acts", flag.ContinueOnError)
|
||||
days := set.Int("days", 14, "how many days back")
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
}
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
now := time.Now()
|
||||
acts, err := link.HandActs(ctx, conn, now.Add(-time.Duration(*days)*24*time.Hour))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
repeated := link.RepeatedCauses(acts, now)
|
||||
if *asJSON {
|
||||
body, err := json.MarshalIndent(map[string]any{"acts": acts, "repeated": repeated}, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
if len(acts) == 0 {
|
||||
fmt.Printf("nothing was done by hand in the last %d day(s)\n", *days)
|
||||
return nil
|
||||
}
|
||||
for i := len(acts) - 1; i >= 0; i-- {
|
||||
a := acts[i]
|
||||
fmt.Printf("%s %s %s %s\n by %s — %s (cause: %s", a.At.Local().Format("2006-01-02 15:04"), a.ID,
|
||||
a.Verb, strings.Join(a.Args, " "), a.By, a.Why, a.Cause)
|
||||
if a.Condition != "" {
|
||||
fmt.Printf(", condition %s", a.Condition)
|
||||
}
|
||||
fmt.Println(")")
|
||||
}
|
||||
if len(repeated) > 0 {
|
||||
causes := make([]string, 0, len(repeated))
|
||||
for c, n := range repeated {
|
||||
causes = append(causes, fmt.Sprintf("%s ×%d", c, n))
|
||||
}
|
||||
sort.Strings(causes)
|
||||
fmt.Printf("\ndone by hand more than once in a fortnight — a healer is wanted (to-be 45 S15): %s\n",
|
||||
strings.Join(causes, ", "))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// handActsThisWeek is how many acts were done by hand in the last seven days, for `status`; -1 when
|
||||
// the log could not be read, which status says rather than reading as none.
|
||||
func handActsThisWeek(ctx context.Context) (int, string) {
|
||||
n := -1
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
reading, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
acts, err := link.HandActs(reading, conn, time.Now().Add(-7*24*time.Hour))
|
||||
n = len(acts)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return -1, err.Error()
|
||||
}
|
||||
return n, ""
|
||||
}
|
||||
@@ -1,94 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **Every verb that repairs by hand takes a required why** (novox/hq to-be 45 §7): refused before
|
||||
// anything is done, through the seat, through `command`, and at a shell where nothing automated runs
|
||||
// the verb.
|
||||
func TestARepairByHandWithoutAReasonIsRefused(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
}{
|
||||
{"push", map[string]any{"node": "anchor"}},
|
||||
{"push", map[string]any{}},
|
||||
{"plans", map[string]any{"close": "plan-1"}},
|
||||
{"plans", map[string]any{"stop": "plan-1"}},
|
||||
{"hand-act", map[string]any{"what": "restarted the proxy", "cause": "proxy-stuck"}},
|
||||
{"command", map[string]any{"command": "push anchor"}},
|
||||
{"command", map[string]any{"command": "plans close plan-1"}},
|
||||
{"command", map[string]any{"command": "broker consumer-reset EVENTS controller"}},
|
||||
{"command", map[string]any{"command": "hand-act record restarted --cause x"}},
|
||||
} {
|
||||
argv, err := argvFor(c.verb, c.args)
|
||||
if c.verb == "plans" && err == nil {
|
||||
// The seat composes the command line; the command refuses it, before opening anything.
|
||||
err = plansCommand(context.Background(), argv[1:])
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), "why") {
|
||||
t.Errorf("%s %v was not refused for want of why: %v %v", c.verb, c.args, argv, err)
|
||||
}
|
||||
}
|
||||
for _, args := range [][]string{{"EVENTS", "controller"}} {
|
||||
if err := consumerReset(context.Background(), args); err == nil || !strings.Contains(err.Error(), "--why") {
|
||||
t.Errorf("consumer-reset without why: %v", err)
|
||||
}
|
||||
}
|
||||
if err := handActCommand(context.Background(), []string{"record", "restarted the proxy", "--cause", "x"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "--why") {
|
||||
t.Errorf("hand-act record without why: %v", err)
|
||||
}
|
||||
if err := handActCommand(context.Background(), []string{"record", "restarted the proxy", "--why", "it hung"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "--cause") {
|
||||
t.Errorf("hand-act record without a cause: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// With a reason, the seat passes it to the command, and a verb that only reads is not held to one.
|
||||
func TestARepairByHandCarriesItsReason(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
want string
|
||||
}{
|
||||
{"push", map[string]any{"node": "anchor", "why": "stuck", "cause": "sent-not-reported"},
|
||||
"push anchor --wait 0 --why stuck --cause sent-not-reported"},
|
||||
{"plans", map[string]any{"close": "plan-1", "why": "the report will not come"},
|
||||
"plans close plan-1 --why the report will not come"},
|
||||
{"plans", map[string]any{"retry": "plan-1"}, "plans retry plan-1"},
|
||||
{"hand-act", map[string]any{"what": "restarted", "why": "hung", "cause": "proxy", "condition": "machine.a.silent"},
|
||||
"hand-act record restarted --why hung --cause proxy --condition machine.a.silent"},
|
||||
{"command", map[string]any{"command": "push anchor --why stuck"}, "push anchor --why stuck"},
|
||||
{"command", map[string]any{"command": "plans plan-1"}, "plans plan-1"},
|
||||
} {
|
||||
argv, err := argvFor(c.verb, c.args)
|
||||
if err != nil || strings.Join(argv, " ") != c.want {
|
||||
t.Errorf("%s %v: %v %v, want %q", c.verb, c.args, argv, err, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The summary of durations says, per kind and subject, what a bound would be set from.
|
||||
func TestDurationsAreSummarisedPerSubject(t *testing.T) {
|
||||
var ds []inventory.Duration
|
||||
for i := 1; i <= 10; i++ {
|
||||
ds = append(ds, inventory.Duration{Kind: inventory.DurationApply, Subject: "anchor",
|
||||
Took: time.Duration(i) * time.Second})
|
||||
}
|
||||
ds = append(ds, inventory.Duration{Kind: inventory.DurationHeartbeatGap, Subject: "anchor", Took: time.Minute})
|
||||
got := summarise(ds)
|
||||
if len(got) != 2 || got[0].Kind != inventory.DurationApply || got[0].Count != 10 ||
|
||||
got[0].Max != "10s" || got[0].Median != "5s" || got[0].P90 != "9s" {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
if !strings.Contains(got[1].Suggests, "3m0s") {
|
||||
t.Fatalf("a minute between words suggests %q", got[1].Suggests)
|
||||
}
|
||||
}
|
||||
@@ -1,865 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The healers (novox/hq to-be 45 §7, ADR 0227 rule 7, Phase 3).
|
||||
//
|
||||
// **A known failure heals itself, under a brake, and every repair is said.** Research 031 counted the
|
||||
// repairs people made by hand in six days: a push to unstick a plan waiting on a report (four times),
|
||||
// a controller restarted to make an object again (twice), a plan closed (twice), a consumer re-made from
|
||||
// now (once). Each was the ordinary path, taken again by a person who noticed. A healer is that act,
|
||||
// registered against the one condition kind it answers, so the mesh takes it itself:
|
||||
//
|
||||
// - **its repair is the ordinary path again** — the send a push makes, the plan's own close, the
|
||||
// assertion every send makes, the consumer reset the verb makes — never a withdrawal, a deletion of
|
||||
// data or a recreation of it;
|
||||
// - **its budget** is how many acts it may take against one thing in a window, and **its settle** how
|
||||
// long after an act it leaves the observation to say whether it worked;
|
||||
// - **success is never the healer's to say.** The condition clears when the watchdog or the probe that
|
||||
// raised it no longer sees it. A healer marking its own work done would be the second opinion of a
|
||||
// fact that rule 1 forbids;
|
||||
// - **a spent budget stops it**: the condition is the operator's, urgent, with every attempt in
|
||||
// `tried`, and no healer touches it again until observation clears it;
|
||||
// - **every act is said**: begun in the store before it is made (so a controller dying mid-act has
|
||||
// still spent it), kept in the condition's `tried` as `healer Hn`, and said on the bus as the
|
||||
// controller seat's `healer-acted`. A heal is not a hand act and is never in the hand-act log —
|
||||
// which is how S15 can tell a repair the mesh made from one a person had to.
|
||||
//
|
||||
// **And the mesh-wide brake**: more than healBrakeLimit acts in an hour, all healers together, and every
|
||||
// healer stops — an urgent condition says so — until an hour has passed with none. A healer looping is
|
||||
// then at most a dozen acts, said, and never the incident itself.
|
||||
//
|
||||
// Only the controller holding the lease heals, under its epoch: a controller serving without the lease
|
||||
// (S12) heals nothing, because a repair is the one act that can always wait for the lease.
|
||||
|
||||
// The mesh-wide brake (to-be 45 §7): how many acts all healers together may take in an hour.
|
||||
const (
|
||||
healBrakeLimit = 12
|
||||
healBrakeWindow = time.Hour
|
||||
)
|
||||
|
||||
// healEvery is how often the healers look at what is open.
|
||||
var healEvery = 30 * time.Second
|
||||
|
||||
// What raises the healers' own conditions, and their kinds.
|
||||
const (
|
||||
sourceHealers = "healers"
|
||||
kindHealersBraked = "healers-braked"
|
||||
kindConsumerBehind = "consumer-behind"
|
||||
kindHealersBlind = "probe-failed"
|
||||
)
|
||||
|
||||
// healerRow is one row of the registry.
|
||||
type healerRow struct {
|
||||
ID string
|
||||
// Kinds are the condition kinds it answers: from the signals table, the probe registry, or an event.
|
||||
Kinds []string
|
||||
// Condition, Repair, Then, From are the row in words, as to-be 45 §7 and `healers` say it.
|
||||
Condition string
|
||||
Repair string
|
||||
Then string
|
||||
From string
|
||||
// Budget acts against one budget key within Window; Settle after an act before the next, or the
|
||||
// escalation, so the observation has had its turn to say whether it worked.
|
||||
Budget int
|
||||
Window time.Duration
|
||||
Settle time.Duration
|
||||
// ActsIn is where the repair runs: the controller, or a module that repairs its own (H5).
|
||||
ActsIn string
|
||||
// Event is what each act is said as.
|
||||
Event string
|
||||
// applies says whether this condition is one the healer may act on, and what its budget is
|
||||
// counted against; why when it is not. Reads, never acts. Nil where the repair is a module's.
|
||||
applies func(ctx context.Context, h *healing, c conditions.Condition) (budget string, ok bool, why string, err error)
|
||||
// repair is the act: what it did in words, what came of it, or an error when it could not be done.
|
||||
repair func(ctx context.Context, h *healing, c conditions.Condition) (act, said string, err error)
|
||||
}
|
||||
|
||||
// actsInController is a healer whose repair the controller makes.
|
||||
const actsInController = "controller"
|
||||
|
||||
// healerRegistry is to-be 45 §7's table, compiled in. **The registry is the design's live form**: a
|
||||
// test generated from it holds every row to a condition kind the mesh raises, a budget, a brake and its
|
||||
// event (healers_test.go).
|
||||
var healerRegistry = []healerRow{
|
||||
{ID: "H1", Kinds: []string{"sent-not-reported"},
|
||||
Condition: "a machine was sent a declaration and has not reported it (S2)",
|
||||
Repair: "ask the machine's node-engine to say again what it last applied (the `report` verb); if what " +
|
||||
"comes back is not the declaration it was sent, or nothing comes, send it its current declaration " +
|
||||
"again — what a push of that one machine does, never moving a build a policy or a plan holds back",
|
||||
Then: "resolver operator, urgent", From: "a push by hand to unstick a plan waiting on a report (230, 257, 264, 267)",
|
||||
Budget: 2, Window: 6 * time.Hour, Settle: 3 * time.Minute, ActsIn: actsInController, Event: link.KeyHealerActed,
|
||||
applies: appliesToAMachine, repair: repairReport},
|
||||
{ID: "H2", Kinds: []string{"stalled"},
|
||||
Condition: "a plan stalled on a wait that is superseded — a newer plan of its repository and branch " +
|
||||
"exists — or already finished — every module of every tier built or failed, and every one that " +
|
||||
"rolls out sent (S3)",
|
||||
Repair: "close the plan with its note, as `plans close` does: superseded, naming the newer plan, or " +
|
||||
"done; what it asked still builds and registers",
|
||||
Then: "resolver operator, urgent", From: "a stuck plan closed by hand (214, 254)",
|
||||
Budget: 1, Window: 24 * time.Hour, Settle: 2 * time.Minute, ActsIn: actsInController, Event: link.KeyHealerActed,
|
||||
applies: appliesToAStalePlan, repair: repairPlan},
|
||||
{ID: "H3", Kinds: []string{"holder-silent", "consumer-lost"},
|
||||
Condition: "a seat's holder that does not answer (D3), or a durable consumer the mesh expects and the " +
|
||||
"bus does not hold (D6, S9)",
|
||||
Repair: "assert the bus's streams, consumers and seat workers again — the assertion every send makes " +
|
||||
"(issue 208)",
|
||||
Then: "resolver operator, urgent", From: "a controller restarted to make a missing object again (208, 248)",
|
||||
Budget: 1, Window: time.Hour, Settle: 6 * time.Minute, ActsIn: actsInController, Event: link.KeyHealerActed,
|
||||
applies: appliesToAnObject, repair: repairObjects},
|
||||
{ID: "H4", Kinds: []string{kindConsumerBehind},
|
||||
Condition: "a durable consumer far behind its stream's head (D6) that the stream table marks resettable",
|
||||
Repair: "re-make the consumer to deliver from now — `broker consumer-reset` (issue 248); what it drops " +
|
||||
"is caught up where the table says",
|
||||
Then: "resolver operator, urgent", From: "a consumer re-made from now by hand (248)",
|
||||
Budget: 1, Window: 24 * time.Hour, Settle: 6 * time.Minute, ActsIn: actsInController, Event: link.KeyHealerActed,
|
||||
applies: appliesToAResettableConsumer, repair: repairConsumer},
|
||||
{ID: "H5", Kinds: []string{kindProviderFailing},
|
||||
Condition: "the identity provider's administrator refusing the mesh's secret (provider-failing, " +
|
||||
"credentials-rejected)",
|
||||
Repair: "the provider repairs it itself through the server's own bootstrap command, checks again and " +
|
||||
"says what it did (ADR 0224 §5); the controller keeps its word as the condition",
|
||||
Then: "announced failing by the provider, braked from ten minutes doubling to six hours (ADR 0224 §5)",
|
||||
From: "the identity provider's admin reset through its bootstrap command (179)",
|
||||
// Its budget and brake are the module's own: ten minutes, doubling, to six hours.
|
||||
Budget: 1, Window: 10 * time.Minute, Settle: 10 * time.Minute,
|
||||
ActsIn: "the provider module (keycloak), ADR 0224 §5", Event: "provisioner.failing, provisioner.recovered"},
|
||||
}
|
||||
|
||||
// healerFor is the healer registered for a kind, and false when none acts in the controller.
|
||||
func healerFor(kind string) (healerRow, bool) {
|
||||
for _, r := range healerRegistry {
|
||||
if r.repair != nil && slices.Contains(r.Kinds, kind) {
|
||||
return r, true
|
||||
}
|
||||
}
|
||||
return healerRow{}, false
|
||||
}
|
||||
|
||||
// healerNamedFor is any healer registered for a kind, wherever it acts: what S15 says beside a cause.
|
||||
func healerNamedFor(kind string) string {
|
||||
for _, r := range healerRegistry {
|
||||
if slices.Contains(r.Kinds, kind) {
|
||||
return r.ID
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// healing is the healers' runner and what their repairs reach.
|
||||
type healing struct {
|
||||
open *stores
|
||||
keeper *conditions.Keeper
|
||||
teller conditions.Teller
|
||||
// js is the bus, for the repairs that assert or reset its objects; nil where there is none.
|
||||
js *broker.JetStream
|
||||
// epoch is the lease's gate; acting says this controller is the one acting, not one standing by.
|
||||
epoch func(ctx context.Context) (uint64, error)
|
||||
acting func() bool
|
||||
now func() time.Time
|
||||
say func(format string, args ...any)
|
||||
|
||||
// The acts, as seams a test replaces: asking a machine to report, sending it again, asserting the
|
||||
// bus's objects, resetting a consumer.
|
||||
askReport func(ctx context.Context, node string) error
|
||||
sendAgain func(ctx context.Context, node string) error
|
||||
assertObjects func(ctx context.Context) error
|
||||
resetConsumer func(stream, name string) (string, error)
|
||||
// reportWait is how long H1 waits for the machine's report after asking.
|
||||
reportWait time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
// declined is why each condition was last passed over, so it is said once and `healers` can show it.
|
||||
declined map[string]string
|
||||
paused string
|
||||
}
|
||||
|
||||
// newHealing is the serving controller's runner, its acts the real ones.
|
||||
func newHealing(open *stores, keeper *conditions.Keeper, teller conditions.Teller, js *broker.JetStream) *healing {
|
||||
h := &healing{open: open, keeper: keeper, teller: teller, js: js, acting: link.Holding,
|
||||
epoch: func(ctx context.Context) (uint64, error) { return theLease.epoch(ctx) },
|
||||
now: time.Now, say: func(format string, args ...any) { fmt.Printf(format+"\n", args...) },
|
||||
reportWait: 45 * time.Second, declined: map[string]string{}}
|
||||
h.askReport = func(ctx context.Context, node string) error {
|
||||
if h.js == nil {
|
||||
return errors.New("this controller is not on the bus")
|
||||
}
|
||||
return askToReport(ctx, h.js.Conn(), node)
|
||||
}
|
||||
h.sendAgain = func(ctx context.Context, node string) error {
|
||||
// **Never what a policy or a plan holds back** (ADR 0221): a send by the mesh itself is a push
|
||||
// that did not name the machine — a person's word sends a held build, a healer's does not.
|
||||
held, err := heldMachines(ctx, open, []string{node})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if why := held[node]; len(why) > 0 {
|
||||
return fmt.Errorf("not sent again: it would move what a policy or a plan holds back (ADR 0221) — %s; "+
|
||||
"`push %s` sends it, on a person's word", strings.Join(why, "; "), node)
|
||||
}
|
||||
return sendTo(ctx, open, []string{node})
|
||||
}
|
||||
h.assertObjects = func(ctx context.Context) error {
|
||||
if h.js == nil {
|
||||
return errors.New("this controller is not on the bus")
|
||||
}
|
||||
return assertOnSend(ctx, open.inventory, h.js, " ")
|
||||
}
|
||||
h.resetConsumer = func(stream, name string) (string, error) {
|
||||
if h.js == nil {
|
||||
return "", errors.New("this controller is not on the bus")
|
||||
}
|
||||
before, after, err := h.js.ResetConsumer(stream, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("it was %d behind with %d unacknowledged; it delivers from now, %d pending", before.Pending,
|
||||
before.AckPending, after.Pending), nil
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// askToReport asks one machine's node-engine to say again what it last applied (to-be 45 §6). On core
|
||||
// NATS, fired and flushed: the answer is the machine's ordinary report, read from the store.
|
||||
func askToReport(ctx context.Context, conn *nats.Conn, node string) error {
|
||||
body, err := json.Marshal(map[string]any{"asked": time.Now().UTC(), "by": "the controller's healer H1"})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := conn.Publish(broker.AskReportSubject(node), body); err != nil {
|
||||
return err
|
||||
}
|
||||
flushing, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
return conn.FlushWithContext(flushing)
|
||||
}
|
||||
|
||||
// keep runs the healers until ctx ends.
|
||||
func (h *healing) keep(ctx context.Context) {
|
||||
tick := time.NewTicker(healEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
h.tick(ctx)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// tick is one look at what is open, and every act it calls for.
|
||||
func (h *healing) tick(ctx context.Context) {
|
||||
if h.acting != nil && !h.acting() {
|
||||
return
|
||||
}
|
||||
epoch, err := h.epoch(ctx)
|
||||
switch {
|
||||
case err != nil:
|
||||
h.pause(fmt.Sprintf("this controller may not act: %v", err))
|
||||
return
|
||||
case epoch == 0:
|
||||
h.pause("this controller serves without the lease (S12): a repair waits for it")
|
||||
return
|
||||
}
|
||||
inv := h.open.inventory
|
||||
now := h.now()
|
||||
heals, err := inv.HealsSince(ctx, now.Add(-24*time.Hour))
|
||||
if err != nil {
|
||||
// Blind: nothing is done, and that is said — never read as "no heals, budgets whole".
|
||||
h.reconcile(ctx, []conditions.Observation{{Scope: conditions.ScopeProbe, ID: "healers", Token: "failed",
|
||||
Kind: kindHealersBlind, Severity: conditions.Warning,
|
||||
Summary: "the healers cannot read what they did, so they count no budget and act on nothing",
|
||||
Said: firstLine(err.Error())}})
|
||||
return
|
||||
}
|
||||
open, err := h.keeper.Open(ctx)
|
||||
if err != nil {
|
||||
h.say("the healers cannot read the open conditions, and act on nothing: %v", err)
|
||||
return
|
||||
}
|
||||
acts := actsWithin(heals, now.Add(-healBrakeWindow))
|
||||
if braked, said := brakeHolds(acts, open, now); braked {
|
||||
h.reconcile(ctx, []conditions.Observation{brakeObservation(acts, said)})
|
||||
h.pause("the mesh-wide brake holds: " + said)
|
||||
return
|
||||
}
|
||||
h.reconcile(ctx, nil)
|
||||
h.resume()
|
||||
for _, c := range open {
|
||||
row, ok := healerFor(c.Kind)
|
||||
if !ok || c.Escalated() {
|
||||
continue
|
||||
}
|
||||
budget, applies, why, err := row.applies(ctx, h, c)
|
||||
if err != nil {
|
||||
h.decline(c.Key, row.ID, "could not tell whether it applies: "+err.Error())
|
||||
continue
|
||||
}
|
||||
if !applies {
|
||||
h.decline(c.Key, row.ID, why)
|
||||
continue
|
||||
}
|
||||
h.forget(c.Key)
|
||||
spent := spentOn(heals, row, budget, now)
|
||||
if n := len(spent); n > 0 && now.Sub(spent[n-1].At) < row.Settle {
|
||||
continue // its last act is still the observation's to judge
|
||||
}
|
||||
if len(spent) >= row.Budget {
|
||||
h.escalate(ctx, row, c, budget, spent)
|
||||
continue
|
||||
}
|
||||
if len(acts) >= healBrakeLimit {
|
||||
return // the brake takes hold on the next look, said there
|
||||
}
|
||||
if h.act(ctx, row, c, budget, len(spent)) {
|
||||
acts = append(acts, inventory.Heal{At: now})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// act is one healer's act on one condition: begun in the store, made, finished, kept in the
|
||||
// condition's tried and said. False when it could not even be begun.
|
||||
func (h *healing) act(ctx context.Context, row healerRow, c conditions.Condition, budget string, spent int) bool {
|
||||
inv := h.open.inventory
|
||||
begun, err := inv.BeginHeal(ctx, inventory.Heal{Healer: row.ID, ConditionKey: c.Key, Kind: c.Kind,
|
||||
BudgetKey: budget, Act: row.Repair, At: h.now()})
|
||||
if err != nil {
|
||||
h.say("healer %s did not act on %s: %v", row.ID, c.Key, err)
|
||||
return false
|
||||
}
|
||||
act, said, err := row.repair(ctx, h, c)
|
||||
outcome := inventory.HealActed
|
||||
if err != nil {
|
||||
outcome, said = inventory.HealFailed, err.Error()
|
||||
}
|
||||
if act == "" {
|
||||
act = row.Repair
|
||||
}
|
||||
finishing, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
if ferr := inv.FinishHeal(finishing, begun.ID, outcome, act+" — "+said); ferr != nil {
|
||||
h.say("healer %s acted on %s and could not record what came of it: %v", row.ID, c.Key, ferr)
|
||||
}
|
||||
budgetWords := fmt.Sprintf("act %d of %d within %s", spent+1, row.Budget, row.Window)
|
||||
attempt := conditions.Attempt{What: act, Outcome: outcome + ": " + said + " (" + budgetWords + ")",
|
||||
By: "healer " + row.ID}
|
||||
if _, _, terr := h.keeper.Tried(finishing, c.Key, attempt, conditions.ResolverHealer(row.ID)); terr != nil {
|
||||
h.say("healer %s acted on %s and could not keep it in the condition: %v", row.ID, c.Key, terr)
|
||||
}
|
||||
h.tell(finishing, healerActed{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: act, Outcome: outcome,
|
||||
Said: said, Budget: budgetWords, Epoch: begun.Epoch})
|
||||
h.say("healer %s %s %s: %s — %s (%s)", row.ID, outcome, c.Key, act, said, budgetWords)
|
||||
return true
|
||||
}
|
||||
|
||||
// escalate is a spent budget: the condition is the operator's now, urgent, with what was tried. Said
|
||||
// once: an escalated condition is passed over by every healer until observation clears it.
|
||||
func (h *healing) escalate(ctx context.Context, row healerRow, c conditions.Condition, budget string, spent []inventory.Heal) {
|
||||
var tried []string
|
||||
for _, s := range spent {
|
||||
tried = append(tried, fmt.Sprintf("%s at %s (%s)", s.Outcome, s.At.UTC().Format("15:04 MST"), firstLine(s.Said)))
|
||||
}
|
||||
said := fmt.Sprintf("its budget of %d act(s) within %s is spent and %s is still open: %s", row.Budget, row.Window,
|
||||
c.Key, strings.Join(tried, "; "))
|
||||
if _, err := h.open.inventory.BeginHeal(ctx, inventory.Heal{Healer: row.ID, ConditionKey: c.Key, Kind: c.Kind,
|
||||
BudgetKey: budget, Act: "handed the condition to the operator", Outcome: inventory.HealEscalated, Said: said,
|
||||
At: h.now()}); err != nil {
|
||||
h.say("healer %s could not record handing %s to the operator, and does not: %v", row.ID, c.Key, err)
|
||||
return
|
||||
}
|
||||
attempt := conditions.Attempt{What: "handed to the operator: nothing in the mesh will repair it now",
|
||||
Outcome: inventory.HealEscalated + ": " + said, By: "healer " + row.ID}
|
||||
if _, _, err := h.keeper.Escalate(ctx, c.Key, attempt); err != nil {
|
||||
h.say("healer %s could not hand %s to the operator: %v", row.ID, c.Key, err)
|
||||
}
|
||||
h.tell(ctx, healerActed{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: "handed to the operator",
|
||||
Outcome: inventory.HealEscalated, Said: said, Budget: fmt.Sprintf("%d of %d within %s", len(spent), row.Budget, row.Window)})
|
||||
h.say("healer %s handed %s to the operator: %s", row.ID, c.Key, said)
|
||||
}
|
||||
|
||||
// healerActed is the body of `healer-acted` (to-be 45 §7): the healer, the condition, the act and its
|
||||
// outcome, and where the budget stands. **A contract**, like a condition's events: the operator-channel's
|
||||
// holder may say it.
|
||||
type healerActed struct {
|
||||
Event string `json:"event"`
|
||||
At time.Time `json:"at"`
|
||||
Healer string `json:"healer"`
|
||||
By string `json:"by"`
|
||||
Condition string `json:"condition"`
|
||||
Kind string `json:"kind"`
|
||||
Act string `json:"act"`
|
||||
// Outcome is acted, failed or escalated. Acted says the act was made, not that it worked: the
|
||||
// condition clearing says that.
|
||||
Outcome string `json:"outcome"`
|
||||
Said string `json:"said"`
|
||||
Budget string `json:"budget"`
|
||||
Epoch uint64 `json:"epoch,omitempty"`
|
||||
Show string `json:"show"`
|
||||
}
|
||||
|
||||
// tell says a heal on the bus. Not said is said in the log: the act and the condition's tried still
|
||||
// hold it.
|
||||
func (h *healing) tell(ctx context.Context, e healerActed) {
|
||||
e.Event, e.At, e.By = link.KeyHealerActed, h.now().UTC(), "healer "+e.Healer
|
||||
e.Show = "mesh-controller.conditions key=" + e.Condition
|
||||
if h.teller == nil {
|
||||
return
|
||||
}
|
||||
body, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
saying, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if err := h.teller.PublishSeatEvent(saying, conditions.Seat, link.KeyHealerActed, body); err != nil {
|
||||
h.say("healer %s %s %s, and that could NOT be said on the bus: %v", e.Healer, e.Outcome, e.Condition, err)
|
||||
}
|
||||
}
|
||||
|
||||
// reconcile keeps the healers' own conditions: the brake, and their being blind.
|
||||
func (h *healing) reconcile(ctx context.Context, observed []conditions.Observation) {
|
||||
if err := h.keeper.Reconcile(ctx, sourceHealers, observed); err != nil {
|
||||
h.say("the healers' own conditions could not be kept: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *healing) pause(why string) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.paused != why {
|
||||
h.say("the healers act on nothing: %s", why)
|
||||
}
|
||||
h.paused = why
|
||||
}
|
||||
|
||||
func (h *healing) resume() {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.paused != "" {
|
||||
h.say("the healers act again")
|
||||
}
|
||||
h.paused = ""
|
||||
}
|
||||
|
||||
// decline remembers why a healer passed a condition over, said once.
|
||||
func (h *healing) decline(key, healer, why string) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.declined[key] != why {
|
||||
h.say("healer %s leaves %s alone: %s", healer, key, why)
|
||||
}
|
||||
h.declined[key] = why
|
||||
}
|
||||
|
||||
func (h *healing) forget(key string) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
delete(h.declined, key)
|
||||
}
|
||||
|
||||
// actsWithin are the heals since a moment that were acts — begun, made or failed; an escalation is a
|
||||
// saying, not an act, and the brake does not count it.
|
||||
func actsWithin(heals []inventory.Heal, since time.Time) []inventory.Heal {
|
||||
var out []inventory.Heal
|
||||
for _, h := range heals {
|
||||
if !h.At.Before(since) && h.Outcome != inventory.HealEscalated {
|
||||
out = append(out, h)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// spentOn is one healer's acts against one budget key within its window, oldest first.
|
||||
func spentOn(heals []inventory.Heal, row healerRow, budget string, now time.Time) []inventory.Heal {
|
||||
var out []inventory.Heal
|
||||
for _, h := range actsWithin(heals, now.Add(-row.Window)) {
|
||||
if h.Healer == row.ID && h.BudgetKey == budget {
|
||||
out = append(out, h)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// brakeHolds says whether the mesh-wide brake holds: the limit reached within the hour, or the brake
|
||||
// already said and an act within the hour still — it lets go an hour after the last act, not the first.
|
||||
func brakeHolds(acts []inventory.Heal, open []conditions.Condition, now time.Time) (bool, string) {
|
||||
said := fmt.Sprintf("%d act(s) by the healers in the last %s, the limit %d", len(acts), healBrakeWindow, healBrakeLimit)
|
||||
if len(acts) >= healBrakeLimit {
|
||||
return true, said
|
||||
}
|
||||
held := slices.ContainsFunc(open, func(c conditions.Condition) bool { return c.Kind == kindHealersBraked })
|
||||
if held && len(acts) > 0 {
|
||||
last := acts[len(acts)-1].At
|
||||
return true, fmt.Sprintf("%s; held until an hour after the last, at %s", said,
|
||||
last.Add(healBrakeWindow).UTC().Format("15:04 MST"))
|
||||
}
|
||||
return false, said
|
||||
}
|
||||
|
||||
// brakeObservation is the brake, as the urgent condition that says it.
|
||||
func brakeObservation(acts []inventory.Heal, said string) conditions.Observation {
|
||||
counts := map[string]int{}
|
||||
for _, a := range acts {
|
||||
if a.Healer != "" {
|
||||
counts[a.Healer+" on "+a.ConditionKey]++
|
||||
}
|
||||
}
|
||||
var most []string
|
||||
for what, n := range counts {
|
||||
most = append(most, fmt.Sprintf("%s ×%d", what, n))
|
||||
}
|
||||
sort.Strings(most)
|
||||
return conditions.Observation{Scope: conditions.ScopeMesh, ID: "healers", Token: "braked", Kind: kindHealersBraked,
|
||||
Severity: conditions.Urgent,
|
||||
Summary: "every healer has stopped: the healers acted more often in an hour than the mesh allows, which is a " +
|
||||
"healer looping, not repairing — " + said,
|
||||
Said: strings.Join(most, ", ")}
|
||||
}
|
||||
|
||||
// --- H1 ----------------------------------------------------------------------------------------------
|
||||
|
||||
// appliesToAMachine is H1's: a machine named, its budget counted against the condition.
|
||||
func appliesToAMachine(_ context.Context, _ *healing, c conditions.Condition) (string, bool, string, error) {
|
||||
if c.Subject.Machine == "" {
|
||||
return "", false, "it names no machine", nil
|
||||
}
|
||||
return c.Key, true, "", nil
|
||||
}
|
||||
|
||||
// repairReport is H1: ask the machine to report; if what comes back is not what it was sent, or nothing
|
||||
// comes, send it again.
|
||||
func repairReport(ctx context.Context, h *healing, c conditions.Condition) (string, string, error) {
|
||||
node := c.Subject.Machine
|
||||
inv := h.open.inventory
|
||||
// The store's clock, as the report's time is: not the runner's, which a test moves by hand.
|
||||
asked := time.Now()
|
||||
askErr := h.askReport(ctx, node)
|
||||
current, heard := false, false
|
||||
if askErr == nil {
|
||||
deadline := time.Now().Add(h.reportWait)
|
||||
for {
|
||||
r, found, err := lastReportOf(ctx, inv, node)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if found && r.At != nil && r.At.After(asked) {
|
||||
heard, current = true, r.Current
|
||||
if current {
|
||||
break
|
||||
}
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return "", "", ctx.Err()
|
||||
case <-time.After(min(2*time.Second, h.reportWait/4+time.Millisecond)):
|
||||
}
|
||||
}
|
||||
}
|
||||
if current {
|
||||
return "asked " + node + "'s node-engine to say again what it last applied",
|
||||
"it reported the declaration it was sent: its report had not reached the mesh", nil
|
||||
}
|
||||
why := "it said nothing within " + h.reportWait.String() + " — its node-engine may be older than the report verb"
|
||||
switch {
|
||||
case askErr != nil:
|
||||
why = "it could not be asked: " + askErr.Error()
|
||||
case heard:
|
||||
why = "it reported a declaration other than the one it was sent"
|
||||
}
|
||||
if err := h.sendAgain(ctx, node); err != nil {
|
||||
return "asked " + node + " to report, then sent it its current declaration again", why + "; the send failed",
|
||||
err
|
||||
}
|
||||
return "asked " + node + " to report, then sent it its current declaration again", why + "; sent again", nil
|
||||
}
|
||||
|
||||
// lastReportOf is one machine's last report beside its last send.
|
||||
func lastReportOf(ctx context.Context, inv *inventory.Inventory, node string) (inventory.Reported, bool, error) {
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return inventory.Reported{}, false, err
|
||||
}
|
||||
for _, r := range reports {
|
||||
if r.Node == node {
|
||||
return r, true, nil
|
||||
}
|
||||
}
|
||||
return inventory.Reported{}, false, nil
|
||||
}
|
||||
|
||||
// --- H2 ----------------------------------------------------------------------------------------------
|
||||
|
||||
// planStale is why a plan's wait is superseded or finished, and the state closing it leaves it in; empty
|
||||
// when it is neither, which is not H2's to repair.
|
||||
func planStale(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) (state, why string, err error) {
|
||||
recent, err := inv.RecentPlans(ctx, 50)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
for _, newer := range recent {
|
||||
if newer.ID == p.ID || !newer.Created.After(p.Created) || !repositoryMatches(newer.Repository, p.Repository) ||
|
||||
newer.Branch != p.Branch || newer.State == inventory.PlanSuperseded {
|
||||
continue
|
||||
}
|
||||
return inventory.PlanSuperseded, fmt.Sprintf("superseded by %s (%s %s), a newer merge of the same repository "+
|
||||
"and branch", newer.ID, newer.Repository, short(newer.Commit)), nil
|
||||
}
|
||||
for _, tier := range p.Tiers {
|
||||
for _, m := range tier {
|
||||
s := p.Modules[m]
|
||||
if s == nil || (s.State != "built" && s.State != "failed") {
|
||||
return "", "", nil
|
||||
}
|
||||
if s.State == "built" && s.SentAt == nil {
|
||||
u, err := inv.UpgradeOf(ctx, m)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if u.RollOut {
|
||||
return "", "", nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return inventory.PlanDone, "finished: every module of every tier is built or failed, and every one that rolls " +
|
||||
"out was sent — nothing is left to wait on", nil
|
||||
}
|
||||
|
||||
// appliesToAStalePlan is H2's: the plan is open, and its wait is superseded or finished.
|
||||
func appliesToAStalePlan(ctx context.Context, h *healing, c conditions.Condition) (string, bool, string, error) {
|
||||
p, err := h.open.inventory.PlanByID(ctx, c.Subject.ID)
|
||||
if err != nil {
|
||||
return "", false, "", err
|
||||
}
|
||||
if !p.Open() {
|
||||
return "", false, "the plan is " + p.State + " already: its condition clears on the next look", nil
|
||||
}
|
||||
state, _, err := planStale(ctx, h.open.inventory, p)
|
||||
if err != nil {
|
||||
return "", false, "", err
|
||||
}
|
||||
if state == "" {
|
||||
return "", false, "its wait is neither superseded nor finished: it waits on something still to come, " +
|
||||
"which its own signal says", nil
|
||||
}
|
||||
return c.Key, true, "", nil
|
||||
}
|
||||
|
||||
// repairPlan is H2: the plan closed with its note, under the plans' hold, by compare-and-set.
|
||||
func repairPlan(ctx context.Context, h *healing, c conditions.Condition) (string, string, error) {
|
||||
inv := h.open.inventory
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
defer release()
|
||||
p, err := inv.PlanByID(ctx, c.Subject.ID)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if !p.Open() {
|
||||
return "closed nothing", "the plan is " + p.State + " already", nil
|
||||
}
|
||||
state, why, err := planStale(ctx, inv, p)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if state == "" {
|
||||
return "closed nothing", "its wait is no longer superseded or finished", nil
|
||||
}
|
||||
p.State = state
|
||||
p.Note = fmt.Sprintf("closed by healer H2 at tier %d: %s", p.Tier, why)
|
||||
if state != inventory.PlanDone {
|
||||
sayUnsent(&p, func(m string) bool {
|
||||
u, err := inv.UpgradeOf(ctx, m)
|
||||
return err == nil && u.RollOut
|
||||
})
|
||||
}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return "closed the plan " + p.ID + " (" + state + ")", why, nil
|
||||
}
|
||||
|
||||
// --- H3 ----------------------------------------------------------------------------------------------
|
||||
|
||||
// appliesToAnObject is H3's: every holder or consumer the probe or the bus names, its budget per object.
|
||||
func appliesToAnObject(_ context.Context, h *healing, c conditions.Condition) (string, bool, string, error) {
|
||||
if h.assertObjects == nil {
|
||||
return "", false, "this controller is not on the bus", nil
|
||||
}
|
||||
return c.Key, true, "", nil
|
||||
}
|
||||
|
||||
// repairObjects is H3: the send's own assertion of every stream, consumer and seat worker.
|
||||
func repairObjects(ctx context.Context, h *healing, _ conditions.Condition) (string, string, error) {
|
||||
if err := h.assertObjects(ctx); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return "asserted the bus's streams, consumers and seat workers again",
|
||||
"every object the mesh defines is asserted; the probe that raised it says on its next run whether it is there", nil
|
||||
}
|
||||
|
||||
// --- H4 ----------------------------------------------------------------------------------------------
|
||||
|
||||
// resettable is why a consumer may be reset by the mesh itself, from the stream table; empty when not.
|
||||
func resettable(stream, name string) string {
|
||||
for _, c := range broker.MeshConsumers() {
|
||||
if c.Stream == stream && c.Name == name {
|
||||
return c.Resettable
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// consumerOf is the stream and consumer a bus condition names: `<stream>.<consumer>`.
|
||||
func consumerOf(c conditions.Condition) (string, string, bool) {
|
||||
stream, name, found := strings.Cut(c.Subject.ID, ".")
|
||||
return stream, name, found && stream != "" && name != ""
|
||||
}
|
||||
|
||||
// appliesToAResettableConsumer is H4's: only a consumer the stream table marks resettable.
|
||||
func appliesToAResettableConsumer(_ context.Context, _ *healing, c conditions.Condition) (string, bool, string, error) {
|
||||
stream, name, ok := consumerOf(c)
|
||||
if !ok {
|
||||
return "", false, "it names no consumer", nil
|
||||
}
|
||||
if resettable(stream, name) == "" {
|
||||
return "", false, fmt.Sprintf("%s on %s is not marked resettable in the stream table: what a reset drops, "+
|
||||
"nothing would catch up", name, stream), nil
|
||||
}
|
||||
return c.Key, true, "", nil
|
||||
}
|
||||
|
||||
// repairConsumer is H4: `broker consumer-reset`, made by the mesh.
|
||||
func repairConsumer(_ context.Context, h *healing, c conditions.Condition) (string, string, error) {
|
||||
stream, name, _ := consumerOf(c)
|
||||
said, err := h.resetConsumer(stream, name)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return fmt.Sprintf("re-made %s on %s to deliver from now", name, stream),
|
||||
said + "; " + resettable(stream, name), nil
|
||||
}
|
||||
|
||||
// --- the verb ----------------------------------------------------------------------------------------
|
||||
|
||||
// healersCommand is `healers`: the registry, what the healers did lately, and the brake.
|
||||
func healersCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("healers", flag.ContinueOnError)
|
||||
daysFlag := set.Int("days", 7, "how many days of acts back")
|
||||
jsonFlag := set.Bool("json", false, "as data")
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New("healers [--days N] [--json]")
|
||||
}
|
||||
days, asJSON := *daysFlag, *jsonFlag
|
||||
if days <= 0 {
|
||||
return fmt.Errorf("healers --days takes a number of days, not %d", days)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
now := time.Now()
|
||||
heals, err := open.inventory.HealsSince(ctx, now.Add(-time.Duration(days)*24*time.Hour))
|
||||
if err != nil {
|
||||
return fmt.Errorf("what the healers did cannot be read: %w", err)
|
||||
}
|
||||
conds, condErr := openConditions(ctx)
|
||||
braked, said := brakeHolds(actsWithin(heals, now.Add(-healBrakeWindow)), conds, now)
|
||||
brake := map[string]any{"holds": braked, "said": said, "limit": healBrakeLimit, "window": healBrakeWindow.String()}
|
||||
if condErr != nil {
|
||||
brake["conditions"] = "the open conditions could not be read, so whether the brake was said is not known: " +
|
||||
condErr.Error()
|
||||
}
|
||||
var rows []map[string]any
|
||||
for _, r := range healerRegistry {
|
||||
rows = append(rows, map[string]any{"id": r.ID, "kinds": r.Kinds, "condition": r.Condition, "repair": r.Repair,
|
||||
"budget": fmt.Sprintf("%d act(s) within %s, %s apart at least", r.Budget, r.Window, r.Settle),
|
||||
"then": r.Then, "acts-in": r.ActsIn, "event": r.Event, "from": r.From})
|
||||
}
|
||||
if heals == nil {
|
||||
heals = []inventory.Heal{}
|
||||
}
|
||||
if asJSON {
|
||||
return printJSON(map[string]any{"healers": rows, "heals": heals, "brake": brake, "days": days,
|
||||
"note": "a heal is never a hand act; whether it repaired anything is the condition's clearing to say"})
|
||||
}
|
||||
for _, r := range healerRegistry {
|
||||
fmt.Printf("%s %s\n → %s\n budget %d within %s; then %s (in %s)\n", r.ID, r.Condition, r.Repair, r.Budget,
|
||||
r.Window, r.Then, r.ActsIn)
|
||||
}
|
||||
fmt.Printf("\nthe brake: %s — %s\n\n", map[bool]string{true: "HOLDS, every healer has stopped", false: "off"}[braked], said)
|
||||
if len(heals) == 0 {
|
||||
fmt.Printf("no healer acted in the last %d day(s)\n", days)
|
||||
return nil
|
||||
}
|
||||
for i := len(heals) - 1; i >= 0; i-- {
|
||||
x := heals[i]
|
||||
fmt.Printf("%s %s %-9s %s\n %s\n", x.At.Local().Format("2006-01-02 15:04"), x.Healer, x.Outcome, x.ConditionKey,
|
||||
firstLine(x.Said))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// forgettingOldHeals removes heals past their keeping once a day, by the controller acting only.
|
||||
func forgettingOldHeals(ctx context.Context, inv *inventory.Inventory) {
|
||||
for {
|
||||
if link.Holding() {
|
||||
if n, err := inv.ForgetOldHeals(ctx); err != nil {
|
||||
fmt.Printf("heals older than %s could not be removed: %v\n", inventory.HealsKeptFor, err)
|
||||
} else if n > 0 {
|
||||
fmt.Printf("removed %d heal(s) older than %s\n", n, inventory.HealsKeptFor)
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(24 * time.Hour):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// healsCount is what the healers did, counted for `status`.
|
||||
type healsCount struct {
|
||||
Acts int `json:"acts"`
|
||||
Escalated int `json:"escalated"`
|
||||
}
|
||||
|
||||
// countHeals counts acts and escalations.
|
||||
func countHeals(heals []inventory.Heal) *healsCount {
|
||||
out := &healsCount{}
|
||||
for _, h := range heals {
|
||||
if h.Outcome == inventory.HealEscalated {
|
||||
out.Escalated++
|
||||
} else {
|
||||
out.Acts++
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1,647 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The test generated from the healer registry (novox/hq to-be 45 §7, ADR 0227 rule 7 "how it is
|
||||
// checked"): **every row is walked.** Its kinds are ones the mesh raises — a row of the signals table, a
|
||||
// probe of the self-check, or a provider's event — it has a budget, a window and a settle inside it,
|
||||
// what happens when the budget is spent, and the event each act is said as; a healer the controller runs
|
||||
// has its applies and its repair, and an induced failure below that sees it act, say so and brake. A
|
||||
// row added without one fails, so the registry cannot grow a healer nobody has seen act.
|
||||
|
||||
// raisedKinds is every condition kind the mesh raises, and what raises it.
|
||||
func raisedKinds() map[string]string {
|
||||
out := map[string]string{kindProviderFailing: "the provisioner.failing event (ADR 0224)"}
|
||||
for _, r := range signalsTable {
|
||||
for _, k := range kindsOf(r) {
|
||||
out[k] = r.Row
|
||||
}
|
||||
}
|
||||
for _, p := range probeRegistry {
|
||||
out[p.Kind] = p.ID
|
||||
for _, k := range p.Raises {
|
||||
out[k] = p.ID
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// inducedFailures are the healers seen acting in this file, by id: a row without one fails.
|
||||
var inducedFailures = map[string]string{
|
||||
"H1": "TestH1AsksAMachineToReportAndSendsItAgain",
|
||||
"H2": "TestH2ClosesAPlanAnotherHasTakenOver",
|
||||
"H3": "TestNatsH3AssertsAMissingConsumerAgainAndBrakesAfterItsBudget",
|
||||
"H4": "TestNatsH4ResetsTheControllersEventsConsumerAndItStillDelivers",
|
||||
}
|
||||
|
||||
func TestEveryHealerAnswersAKindTheMeshRaisesWithABudgetABrakeAndItsEvent(t *testing.T) {
|
||||
kinds := raisedKinds()
|
||||
source, err := os.ReadFile("healers_test.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
answered := map[string]string{}
|
||||
for _, r := range healerRegistry {
|
||||
t.Run(r.ID, func(t *testing.T) {
|
||||
if seen[r.ID] {
|
||||
t.Fatalf("%s is in the registry twice", r.ID)
|
||||
}
|
||||
seen[r.ID] = true
|
||||
if len(r.Kinds) == 0 || r.Condition == "" || r.Repair == "" || r.Then == "" || r.From == "" || r.ActsIn == "" {
|
||||
t.Fatalf("%s does not say what it answers, what it does, what then, and which hand act it replaces: %+v", r.ID, r)
|
||||
}
|
||||
for _, k := range r.Kinds {
|
||||
if _, ok := kinds[k]; !ok {
|
||||
t.Errorf("%s answers %q, which nothing in the mesh raises", r.ID, k)
|
||||
}
|
||||
if other, twice := answered[k]; twice {
|
||||
t.Errorf("%q is answered by %s and %s: one healer per kind", k, other, r.ID)
|
||||
}
|
||||
answered[k] = r.ID
|
||||
}
|
||||
if r.Budget <= 0 || r.Window <= 0 || r.Settle <= 0 || r.Settle > r.Window {
|
||||
t.Errorf("%s has no budget it can spend: %d within %s, settled after %s", r.ID, r.Budget, r.Window, r.Settle)
|
||||
}
|
||||
if r.Event == "" {
|
||||
t.Errorf("%s says nothing when it acts", r.ID)
|
||||
}
|
||||
if r.ActsIn != actsInController {
|
||||
if r.repair != nil || r.applies != nil {
|
||||
t.Errorf("%s acts in %s and the controller would act for it too", r.ID, r.ActsIn)
|
||||
}
|
||||
return
|
||||
}
|
||||
if r.repair == nil || r.applies == nil {
|
||||
t.Fatalf("%s acts in the controller and has no repair or no applies", r.ID)
|
||||
}
|
||||
if r.Event != link.KeyHealerActed || !slices.Contains(broker.ControllerStates, r.Event) {
|
||||
t.Errorf("%s is said as %q, which the controller's grant does not permit", r.ID, r.Event)
|
||||
}
|
||||
if inducedFailures[r.ID] == "" {
|
||||
t.Errorf("%s has no induced failure: a healer nobody has seen act", r.ID)
|
||||
} else if !strings.Contains(string(source), "func "+inducedFailures[r.ID]+"(t *testing.T)") {
|
||||
t.Errorf("%s's induced failure %s is not a test in this file", r.ID, inducedFailures[r.ID])
|
||||
}
|
||||
})
|
||||
}
|
||||
for id := range inducedFailures {
|
||||
if !seen[id] {
|
||||
t.Errorf("an induced failure for %s, which the registry does not have", id)
|
||||
}
|
||||
}
|
||||
if healBrakeLimit <= 0 || healBrakeWindow <= 0 {
|
||||
t.Error("the mesh-wide brake holds nothing")
|
||||
}
|
||||
}
|
||||
|
||||
// heard keeps the healer-acted events said.
|
||||
type heard struct {
|
||||
mu sync.Mutex
|
||||
acts []healerActed
|
||||
}
|
||||
|
||||
func (h *heard) PublishSeatEvent(_ context.Context, seat, event string, body []byte) error {
|
||||
if seat != conditions.Seat || event != link.KeyHealerActed {
|
||||
return errors.New("said under the wrong seat or name: " + seat + " " + event)
|
||||
}
|
||||
var e healerActed
|
||||
if err := json.Unmarshal(body, &e); err != nil {
|
||||
return err
|
||||
}
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.acts = append(h.acts, e)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *heard) said() []healerActed {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return append([]healerActed(nil), h.acts...)
|
||||
}
|
||||
|
||||
// testClock is a moment a test moves by hand.
|
||||
type testClock struct {
|
||||
mu sync.Mutex
|
||||
at time.Time
|
||||
}
|
||||
|
||||
func (c *testClock) now() time.Time {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.at
|
||||
}
|
||||
|
||||
func (c *testClock) pass(d time.Duration) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.at = c.at.Add(d)
|
||||
}
|
||||
|
||||
// healingOn is a runner over a mesh's stores and its condition store, under epoch 57, every act a
|
||||
// fake that fails the test unless the test gives it.
|
||||
func healingOn(t *testing.T, open *stores) (*healing, *heard, *testClock) {
|
||||
t.Helper()
|
||||
if conditionsFrom == nil {
|
||||
t.Fatal("the mesh has no condition store")
|
||||
}
|
||||
told, clock := &heard{}, &testClock{at: time.Now()}
|
||||
// The store's gate, as the serving controller's is the lease's (stores.go).
|
||||
open.inventory.ActsUnder(func(context.Context) (uint64, error) { return 57, nil })
|
||||
h := &healing{open: open, keeper: conditionsFrom, teller: told,
|
||||
epoch: func(context.Context) (uint64, error) { return 57, nil }, now: clock.now,
|
||||
say: func(f string, a ...any) { t.Logf(f, a...) }, reportWait: 300 * time.Millisecond,
|
||||
declined: map[string]string{}}
|
||||
h.askReport = func(context.Context, string) error { t.Error("asked a machine to report"); return nil }
|
||||
h.sendAgain = func(context.Context, string) error { t.Error("sent a machine again"); return nil }
|
||||
h.assertObjects = func(context.Context) error { t.Error("asserted the bus's objects"); return nil }
|
||||
h.resetConsumer = func(string, string) (string, error) { t.Error("reset a consumer"); return "", nil }
|
||||
return h, told, clock
|
||||
}
|
||||
|
||||
// sentNotReported raises S2 for a machine, as the watchdog does.
|
||||
func sentNotReported(t *testing.T, node string) string {
|
||||
t.Helper()
|
||||
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: "sent-not-reported", Machine: node,
|
||||
Severity: conditions.Warning, Summary: node + " was sent a declaration and has not reported it", Source: "S2"}
|
||||
if _, err := conditionsFrom.Observe(t.Context(), o); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return o.Key()
|
||||
}
|
||||
|
||||
// **H1, the commonest hand act** (031/01 §f: a push by hand to unstick a plan waiting on a report, four
|
||||
// times): the machine is asked to report; a report that names what it was sent is all it takes, and one
|
||||
// that does not — or none — is a send of its current declaration again. Each act kept in `tried` as
|
||||
// `healer H1`, said as healer-acted, counted; twice, then the operator's, urgent; then nothing more.
|
||||
func TestH1AsksAMachineToReportAndSendsItAgain(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
h, told, clock := healingOn(t, open)
|
||||
record, err := open.inventory.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSent(ctx, record.ID, "d2", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := sentNotReported(t, "laptop")
|
||||
|
||||
// First: the report had not reached the mesh, and asking for it brings it.
|
||||
asked := 0
|
||||
h.askReport = func(ctx context.Context, node string) error {
|
||||
asked++
|
||||
if node != "laptop" {
|
||||
t.Errorf("asked %s", node)
|
||||
}
|
||||
_, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeApplied,
|
||||
At: time.Now().Add(time.Second), Declared: "d2"})
|
||||
return err
|
||||
}
|
||||
h.tick(ctx)
|
||||
c, found, err := conditionsFrom.Get(ctx, key)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("the condition is gone after the act — a healer cleared it, not an observation: %v", err)
|
||||
}
|
||||
if asked != 1 || len(c.Tried) != 1 || c.Tried[0].By != "healer H1" || !strings.Contains(c.Tried[0].Outcome, "acted") ||
|
||||
c.Resolver != "healer:H1" {
|
||||
t.Fatalf("after the first act: asked %d, %+v", asked, c)
|
||||
}
|
||||
if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H1" || acts[0].Outcome != inventory.HealActed ||
|
||||
acts[0].Condition != key || acts[0].By != "healer H1" || acts[0].Epoch != 57 {
|
||||
t.Fatalf("the act was not said as healer-acted: %+v", acts)
|
||||
}
|
||||
|
||||
// Within its settle, nothing more: the observation has its turn.
|
||||
clock.pass(time.Minute)
|
||||
h.tick(ctx)
|
||||
if asked != 1 {
|
||||
t.Fatalf("acted again inside the settle: asked %d", asked)
|
||||
}
|
||||
|
||||
// Second: the machine says nothing, so it is sent again.
|
||||
clock.pass(3 * time.Minute)
|
||||
sent := 0
|
||||
h.askReport = func(context.Context, string) error { asked++; return nil }
|
||||
h.sendAgain = func(_ context.Context, node string) error { sent++; return nil }
|
||||
h.tick(ctx)
|
||||
if asked != 2 || sent != 1 {
|
||||
t.Fatalf("the second act: asked %d, sent %d", asked, sent)
|
||||
}
|
||||
c, _, _ = conditionsFrom.Get(ctx, key)
|
||||
if len(c.Tried) != 2 || !strings.Contains(c.Tried[1].Outcome, "sent again") || !strings.Contains(c.Tried[1].Outcome, "act 2 of 2") {
|
||||
t.Fatalf("tried %+v", c.Tried)
|
||||
}
|
||||
|
||||
// The budget is spent: the operator's, urgent, said; and no healer touches it again.
|
||||
clock.pass(4 * time.Minute)
|
||||
h.tick(ctx)
|
||||
c, _, _ = conditionsFrom.Get(ctx, key)
|
||||
if !c.Escalated() || c.Severity != conditions.Urgent || len(c.Tried) != 3 ||
|
||||
!strings.Contains(c.Tried[2].Outcome, "budget of 2") {
|
||||
t.Fatalf("not handed to the operator: %+v", c)
|
||||
}
|
||||
if acts := told.said(); len(acts) != 3 || acts[2].Outcome != inventory.HealEscalated {
|
||||
t.Fatalf("the escalation was not said: %+v", acts)
|
||||
}
|
||||
clock.pass(time.Hour)
|
||||
h.tick(ctx)
|
||||
if asked != 2 || sent != 1 || len(told.said()) != 3 {
|
||||
t.Fatalf("a healer acted on a condition the operator holds: asked %d sent %d said %d", asked, sent, len(told.said()))
|
||||
}
|
||||
heals, err := open.inventory.HealsSince(ctx, time.Now().Add(-time.Hour))
|
||||
if err != nil || len(heals) != 3 || heals[0].Outcome != inventory.HealActed || heals[1].Outcome != inventory.HealActed ||
|
||||
heals[2].Outcome != inventory.HealEscalated || heals[0].Epoch != 57 {
|
||||
t.Fatalf("the heals kept: %+v %v", heals, err)
|
||||
}
|
||||
// And a heal is not a hand act: what S15 counts never sees it.
|
||||
for _, x := range heals {
|
||||
if x.Healer == "" || strings.HasPrefix(x.Act, "hand-act") {
|
||||
t.Errorf("a heal reads as a hand act: %+v", x)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **Only the controller holding the lease heals** (to-be 45 §6): unleased, or standing by, nothing.
|
||||
func TestNoHealerActsWithoutTheLease(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
h, told, _ := healingOn(t, open)
|
||||
sentNotReported(t, "laptop")
|
||||
h.epoch = func(context.Context) (uint64, error) { return 0, nil }
|
||||
h.tick(ctx)
|
||||
h.epoch = func(context.Context) (uint64, error) { return 0, errors.New("the lease is not held") }
|
||||
h.tick(ctx)
|
||||
h.epoch = func(context.Context) (uint64, error) { return 57, nil }
|
||||
h.acting = func() bool { return false }
|
||||
h.tick(ctx)
|
||||
if len(told.said()) != 0 {
|
||||
t.Fatalf("a healer acted without the lease: %+v", told.said())
|
||||
}
|
||||
}
|
||||
|
||||
// **The mesh-wide brake**: a dozen acts in an hour and every healer stops, said urgently, until an hour
|
||||
// after the last.
|
||||
func TestTheBrakeStopsEveryHealerAndSaysSo(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
h, told, clock := healingOn(t, open)
|
||||
for i := 0; i < healBrakeLimit; i++ {
|
||||
if _, err := open.inventory.BeginHeal(ctx, inventory.Heal{Healer: "H3", ConditionKey: "seat.x.anchor.silent",
|
||||
Kind: "holder-silent", Act: "asserted", Outcome: inventory.HealActed,
|
||||
At: clock.now().Add(-time.Duration(healBrakeLimit-i) * time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
sentNotReported(t, "laptop")
|
||||
h.tick(ctx) // the fakes fail the test if anything acts
|
||||
braked, found, err := conditionsFrom.Get(ctx, "mesh.healers.braked")
|
||||
if err != nil || !found || braked.Severity != conditions.Urgent || !strings.Contains(braked.Evidence[0].Said, "H3 on seat.x.anchor.silent ×12") {
|
||||
t.Fatalf("the brake was not said: %+v %v", braked, err)
|
||||
}
|
||||
if len(told.said()) != 0 {
|
||||
t.Fatalf("a healer acted under the brake: %+v", told.said())
|
||||
}
|
||||
// Past the hour of the first act, still held: it lets go an hour after the last.
|
||||
clock.pass(30 * time.Minute)
|
||||
h.tick(ctx)
|
||||
if _, held, _ := conditionsFrom.Get(ctx, "mesh.healers.braked"); !held {
|
||||
t.Fatal("the brake let go before an hour had passed since the last act")
|
||||
}
|
||||
clock.pass(31 * time.Minute)
|
||||
asked := 0
|
||||
h.askReport = func(context.Context, string) error { asked++; return nil }
|
||||
h.sendAgain = func(context.Context, string) error { return nil }
|
||||
h.tick(ctx)
|
||||
if _, held, _ := conditionsFrom.Get(ctx, "mesh.healers.braked"); held {
|
||||
t.Fatal("the brake held an hour after the last act")
|
||||
}
|
||||
if asked != 1 {
|
||||
t.Fatalf("the healers did not act again after the brake let go: asked %d", asked)
|
||||
}
|
||||
}
|
||||
|
||||
// **H2 closes a plan another has taken over**, with its note — and leaves a plan alone whose wait is
|
||||
// still to come: that one is its own signal's, not a healer's.
|
||||
func TestH2ClosesAPlanAnotherHasTakenOver(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
h, told, _ := healingOn(t, open)
|
||||
created := time.Now().UTC().Add(-time.Hour)
|
||||
older := inventory.Plan{ID: "plan-old", Repository: "novox/app", Branch: "main", Commit: "0ld0ld0", Created: created,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "asked"}}}
|
||||
waiting := inventory.Plan{ID: "plan-other", Repository: "novox/other", Branch: "main", Commit: "07he707", Created: created,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"b"}}, Modules: map[string]*inventory.PlanModule{"b": {State: "asked"}}}
|
||||
newer := inventory.Plan{ID: "plan-new", Repository: "novox/app", Branch: "main", Commit: "new0new", Created: created.Add(time.Minute),
|
||||
State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "built"}}}
|
||||
for _, p := range []*inventory.Plan{&older, &waiting, &newer} {
|
||||
if err := open.inventory.SavePlan(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, id := range []string{"plan-old", "plan-other"} {
|
||||
if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopePlan, ID: id, Kind: "stalled",
|
||||
Severity: conditions.Warning, Summary: id + " is stalled", Source: "S3"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
h.tick(ctx)
|
||||
closed, err := open.inventory.PlanByID(ctx, "plan-old")
|
||||
if err != nil || closed.State != inventory.PlanSuperseded || !strings.Contains(closed.Note, "closed by healer H2") ||
|
||||
!strings.Contains(closed.Note, "plan-new") {
|
||||
t.Fatalf("the superseded plan: %+v %v", closed, err)
|
||||
}
|
||||
if other, _ := open.inventory.PlanByID(ctx, "plan-other"); other.State != inventory.PlanBuilding {
|
||||
t.Fatalf("a plan still waiting was closed: %+v", other)
|
||||
}
|
||||
if c, _, _ := conditionsFrom.Get(ctx, "plan.plan-other.stalled"); len(c.Tried) != 0 || c.Resolver != conditions.ResolverSelf {
|
||||
t.Fatalf("a plan H2 does not repair was touched: %+v", c)
|
||||
}
|
||||
if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H2" || acts[0].Condition != "plan.plan-old.stalled" {
|
||||
t.Fatalf("said %+v", acts)
|
||||
}
|
||||
// Finished: every module built, none rolled out unsent — closed as done.
|
||||
done := inventory.Plan{ID: "plan-done", Repository: "novox/third", Branch: "main", Commit: "d0ned0n", Created: created,
|
||||
State: inventory.PlanRolling, Tier: 0, Tiers: [][]string{{"c"}}, Modules: map[string]*inventory.PlanModule{"c": {State: "built"}}}
|
||||
if err := open.inventory.SavePlan(ctx, &done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state, why, err := planStale(ctx, open.inventory, done); err != nil || state != inventory.PlanDone || !strings.Contains(why, "finished") {
|
||||
t.Fatalf("a finished plan reads %q %q %v", state, why, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **H4 only for a consumer the stream table marks resettable**: a module's consumer far behind is said
|
||||
// and left — what a reset drops, nothing would catch up for it.
|
||||
func TestH4ResetsOnlyWhatTheTableMarksResettable(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
h, _, _ := healingOn(t, open)
|
||||
marked := 0
|
||||
for _, c := range broker.MeshConsumers() {
|
||||
if c.Resettable != "" {
|
||||
marked++
|
||||
if c.Stream != broker.EventsStream || c.Name != broker.ControllerName {
|
||||
t.Errorf("%s on %s is marked resettable: only the controller's own events consumer is", c.Name, c.Stream)
|
||||
}
|
||||
}
|
||||
}
|
||||
if marked != 1 {
|
||||
t.Fatalf("%d consumers are marked resettable, want the controller's events consumer alone", marked)
|
||||
}
|
||||
for _, who := range []string{"EVENTS.anchor_shop", "CONTROL.controller"} {
|
||||
if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "behind",
|
||||
Kind: kindConsumerBehind, Severity: conditions.Warning, Summary: who + " is far behind", Source: "D6"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
h.tick(ctx) // the fake reset fails the test if it is called
|
||||
reset := ""
|
||||
h.resetConsumer = func(stream, name string) (string, error) {
|
||||
reset = stream + "." + name
|
||||
return "it was 1500 behind", nil
|
||||
}
|
||||
if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopeBus, ID: "EVENTS.controller",
|
||||
Token: "behind", Kind: kindConsumerBehind, Severity: conditions.Warning, Summary: "far behind", Source: "D6"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.tick(ctx)
|
||||
if reset != "EVENTS.controller" {
|
||||
t.Fatalf("reset %q", reset)
|
||||
}
|
||||
}
|
||||
|
||||
// **H3 against a real bus** (issue 208's note): a consumer the mesh expects, deleted; D6 says so; H3
|
||||
// asserts the bus's objects the way a send does, and D6's next run clears it — the healer never does.
|
||||
// Deleted again within the hour, the budget is spent: the operator's, urgent, and H3 stops.
|
||||
func TestNatsH3AssertsAMissingConsumerAgainAndBrakesAfterItsBudget(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
|
||||
_ = js.Context().DeleteStream(s)
|
||||
}
|
||||
if _, err := assertBusObjects(ctx, open.inventory, js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h, told, clock := healingOn(t, open)
|
||||
h.js = js
|
||||
h.assertObjects = func(ctx context.Context) error { return assertOnSend(ctx, open.inventory, js, " ") }
|
||||
d := &doctor{open: open, js: js}
|
||||
probe := func() {
|
||||
t.Helper()
|
||||
found, err := probeConsumers(ctx, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := conditionsFrom.Reconcile(ctx, "D6", kindedAs(found, "consumer-wrong")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
const key = "bus.NODES.laptop.missing"
|
||||
|
||||
if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
probe()
|
||||
if c, raised, _ := conditionsFrom.Get(ctx, key); !raised || c.Kind != "consumer-lost" {
|
||||
t.Fatalf("the deleted consumer was not raised: %+v", c)
|
||||
}
|
||||
h.tick(ctx)
|
||||
if _, err := js.Context().ConsumerInfo("NODES", "laptop"); err != nil {
|
||||
t.Fatalf("H3 did not make the consumer again: %v", err)
|
||||
}
|
||||
c, still, _ := conditionsFrom.Get(ctx, key)
|
||||
if !still || len(c.Tried) != 1 || c.Tried[0].By != "healer H3" || c.Resolver != "healer:H3" {
|
||||
t.Fatalf("the act is not in the condition, or the healer cleared it: %+v", c)
|
||||
}
|
||||
probe()
|
||||
if _, still, _ := conditionsFrom.Get(ctx, key); still {
|
||||
t.Fatal("the probe's next run did not clear what H3 repaired")
|
||||
}
|
||||
// Kept in the history as the keeper says it, a moment later.
|
||||
var cleared *conditions.Event
|
||||
for wait := time.Now().Add(5 * time.Second); cleared == nil && time.Now().Before(wait); time.Sleep(20 * time.Millisecond) {
|
||||
history, err := conditionsFrom.HistorySince(ctx, time.Now().Add(-time.Minute))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := range history {
|
||||
if history[i].Key == key && history[i].Change == conditions.ChangeCleared {
|
||||
cleared = &history[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
if cleared == nil || !strings.Contains(cleared.Why, "D6 no longer observes it") || len(cleared.Tried) != 1 {
|
||||
t.Fatalf("the clearing is not the probe's, or forgets what was tried: %+v", cleared)
|
||||
}
|
||||
|
||||
// Again within the hour: the budget is one, so after its settle the operator is told, and H3 stops.
|
||||
clock.pass(10 * time.Minute)
|
||||
if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
probe()
|
||||
h.assertObjects = func(context.Context) error { t.Error("H3 acted past its budget"); return nil }
|
||||
h.tick(ctx)
|
||||
c, _, _ = conditionsFrom.Get(ctx, key)
|
||||
if !c.Escalated() || c.Severity != conditions.Urgent || c.Count != 2 {
|
||||
t.Fatalf("the spent budget was not handed to the operator: %+v", c)
|
||||
}
|
||||
acts := told.said()
|
||||
if len(acts) != 2 || acts[0].Outcome != inventory.HealActed || acts[1].Outcome != inventory.HealEscalated {
|
||||
t.Fatalf("said %+v", acts)
|
||||
}
|
||||
clock.pass(2 * time.Hour)
|
||||
h.tick(ctx)
|
||||
if len(told.said()) != 2 {
|
||||
t.Fatal("a healer acted on what the operator holds")
|
||||
}
|
||||
}
|
||||
|
||||
// **H4 against a real bus** (issue 248): the controller's events consumer a long way behind — the week it
|
||||
// once replayed — is re-made from now by the mesh itself, and the controller's bound subscription still
|
||||
// receives what comes next: a reset that left the controller deaf would be the incident.
|
||||
func TestNatsH4ResetsTheControllersEventsConsumerAndItStillDelivers(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
|
||||
_ = js.Context().DeleteStream(s)
|
||||
}
|
||||
if _, err := assertBusObjects(ctx, open.inventory, js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Bound as the controller binds it (link/receive_nats.go), taking one and acknowledging none.
|
||||
events := make(chan *nats.Msg, 64)
|
||||
sub, err := js.Context().ChanSubscribe("", events, nats.Bind(broker.EventsStream, broker.ControllerName))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
followed := broker.ControllerFollows[0]
|
||||
for i := 0; i < consumerFarBehind+200; i++ {
|
||||
if _, err := js.Context().Publish(followed, []byte(`{"n":`+strconv.Itoa(i)+`}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
d := &doctor{open: open, js: js}
|
||||
probe := func() []conditions.Observation {
|
||||
t.Helper()
|
||||
found, err := probeConsumers(ctx, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := conditionsFrom.Reconcile(ctx, "D6", kindedAs(found, "consumer-wrong")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return found
|
||||
}
|
||||
probe()
|
||||
const key = "bus.EVENTS.controller.behind"
|
||||
if c, raised, _ := conditionsFrom.Get(ctx, key); !raised || c.Kind != kindConsumerBehind {
|
||||
t.Fatalf("a consumer %d behind was not raised: %+v", consumerFarBehind+200, c)
|
||||
}
|
||||
h, told, _ := healingOn(t, open)
|
||||
h.resetConsumer = func(stream, name string) (string, error) {
|
||||
before, after, err := js.ResetConsumer(stream, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "it was " + strconv.FormatUint(before.Pending, 10) + " behind; " + strconv.FormatUint(after.Pending, 10) +
|
||||
" pending now", nil
|
||||
}
|
||||
h.tick(ctx)
|
||||
if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H4" || acts[0].Outcome != inventory.HealActed {
|
||||
t.Fatalf("said %+v", acts)
|
||||
}
|
||||
if found := probe(); len(found) != 0 {
|
||||
t.Fatalf("after the reset the probe still finds %+v", found)
|
||||
}
|
||||
if _, still, _ := conditionsFrom.Get(ctx, key); still {
|
||||
t.Fatal("the probe's next run did not clear what H4 repaired")
|
||||
}
|
||||
// What comes next still reaches the controller.
|
||||
for len(events) > 0 {
|
||||
<-events
|
||||
}
|
||||
if _, err := js.Context().Publish(followed, []byte(`{"after":"the reset"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deadline := time.After(10 * time.Second)
|
||||
for {
|
||||
select {
|
||||
case m := <-events:
|
||||
if strings.Contains(string(m.Data), "after") {
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
case <-deadline:
|
||||
t.Fatal("the controller's subscription heard nothing after its consumer was reset: it would be deaf")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **H1's question reaches the machine**, on the subject its grant lets it hear and nothing else.
|
||||
func TestNatsAskToReportReachesTheMachine(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(conn.Close)
|
||||
asked, err := conn.SubscribeSync(broker.AskReportSubject("laptop"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := askToReport(t.Context(), conn, "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
msg, err := asked.NextMsg(5 * time.Second)
|
||||
if err != nil || !strings.Contains(string(msg.Data), "healer H1") {
|
||||
t.Fatalf("the machine heard %v, %v", msg, err)
|
||||
}
|
||||
perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindNode, Node: "laptop", PasswordHash: "x"})
|
||||
if err != nil || !slices.Contains(perms.Subscribe, "mesh.node.laptop.ask.report") ||
|
||||
slices.Contains(perms.Subscribe, "mesh.node.anchor.ask.report") {
|
||||
t.Fatalf("a machine's grant for the question: %v %v", perms.Subscribe, err)
|
||||
}
|
||||
}
|
||||
@@ -128,7 +128,7 @@ func (r *recordedDelivery) grant(context.Context, []readyNode) error { return ni
|
||||
|
||||
func (r *recordedDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
|
||||
r.declared = append(r.declared, s.node)
|
||||
return recordSent(ctx, r.inv, s.node, body, s.declared.Builds, s.declared.Epoch)
|
||||
return recordSent(ctx, r.inv, s.node, body, s.declared.Builds)
|
||||
}
|
||||
|
||||
// aResolver is a module built from a repository, at a commit, with something on the machine that
|
||||
|
||||
@@ -1,150 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0225, issue 263: a consumer's identity is bounded by the provision it requires, an
|
||||
// overflow is refused before merge by `module check`, and a provider's machine is never refused for
|
||||
// one consumer's identity.
|
||||
|
||||
// `module check` refuses the pull request that introduces an overflow, naming the module.
|
||||
func TestModuleCheckRefusesAnIdentityThatOverflowsWhatItRequires(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
write := func(name, body string) string {
|
||||
p := filepath.Join(dir, name+".json")
|
||||
if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
objects := write("objects", `{"module":"objects","version":"1",
|
||||
"provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}],
|
||||
"receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`)
|
||||
resolver := write("resolver", `{"module":"resolver","version":"1",
|
||||
"provides":[{"name":"wildcard-resolution","scope":"mesh","identity":false}]}`)
|
||||
album := write("photoalbum", `{"module":"photoalbum","version":"1","requires":["s3-bucket"]}`)
|
||||
nm := write("networkmanager", `{"module":"networkmanager","version":"1","requires":["wildcard-resolution"]}`)
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := moduleCheckFor([]string{resolver, nm}, 6, &out); err != nil {
|
||||
t.Fatalf("a long name requiring a keyless provision was refused (issue 263): %v\n%s", err, out.String())
|
||||
}
|
||||
out.Reset()
|
||||
err := moduleCheckFor([]string{objects, album, resolver, nm}, 6, &out)
|
||||
if err == nil {
|
||||
t.Fatalf("an identity overflowing an S3 access key passed:\n%s", out.String())
|
||||
}
|
||||
if !strings.Contains(out.String(), "photoalbum wants s3-bucket") ||
|
||||
!strings.Contains(out.String(), "`slug` of at most 8 characters") ||
|
||||
strings.Contains(out.String(), "networkmanager wants") {
|
||||
t.Fatalf("the refusal does not name the one overflowing module and its remedy:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Tonight's case, through the commands: networkmanager on a six-character machine requires the
|
||||
// resolver provision, and a second consumer there overflows an object store's access key. The
|
||||
// provider's machine still composes; the overflowing consumer is left out of its grants and named,
|
||||
// by push and by `status`, and the keyless consumer is granted with its long name.
|
||||
func TestAnOverflowingConsumerNeverRefusesItsProvidersMachine(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "objects", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "s3-bucket", Scope: catalogue.ScopeMesh,
|
||||
Identity: &catalogue.OfferIdentity{Max: 20, In: "an S3 access key"}}},
|
||||
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}})
|
||||
register(t, open, catalogue.Manifest{Module: "resolver", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}})
|
||||
register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1",
|
||||
Requires: []string{"wildcard-resolution"}})
|
||||
register(t, open, catalogue.Manifest{Module: "photoalbum", Version: "1", Requires: []string{"s3-bucket"}})
|
||||
register(t, open, catalogue.Manifest{Module: "files", Version: "1", Requires: []string{"s3-bucket"}})
|
||||
for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "resolver"},
|
||||
{"laptop", "networkmanager"}, {"laptop", "photoalbum"}, {"laptop", "files"}} {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
|
||||
// The consumer's machine resolves, and says which of its modules no provider will grant.
|
||||
consumer, _, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
over := consumer.Overflowing()
|
||||
if len(over) != 1 || over[0].Module != "photoalbum" || over[0].Provision != "s3-bucket" {
|
||||
t.Fatalf("the consumer's side does not name exactly photoalbum: %+v", over)
|
||||
}
|
||||
|
||||
// The provider's machine composes. Under ADR 0049's one bound this was a refusal naming
|
||||
// networkmanager, and no push to the provider could go through.
|
||||
plan, settings, err := planFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationFor(ctx, open, "anchor", plan, settings)
|
||||
if err != nil {
|
||||
t.Fatalf("one consumer's identity refused its provider's whole machine: %v", err)
|
||||
}
|
||||
if len(declared.withheld) != 1 || declared.withheld[0].Identity != "mesh_laptop_photoalbum" {
|
||||
t.Fatalf("the overflowing consumer is not the one withheld: %+v", declared.withheld)
|
||||
}
|
||||
grants, _, err := grantsFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keyless bool
|
||||
for _, g := range grants {
|
||||
keyless = keyless || g.Provision == "wildcard-resolution" && g.From == "networkmanager"
|
||||
}
|
||||
if !keyless {
|
||||
t.Fatalf("networkmanager, 26 characters, is not granted the keyless resolver provision: %+v", grants)
|
||||
}
|
||||
var granted []string
|
||||
for _, c := range declared.Received["objects"]["s3-bucket"] {
|
||||
granted = append(granted, c.From)
|
||||
}
|
||||
if strings.Join(granted, ",") != "files" {
|
||||
t.Fatalf("the object store grants %v; files and only files fit", granted)
|
||||
}
|
||||
said := printed(t, func() error { reportLeftOut("anchor", declared); return nil })
|
||||
if !strings.Contains(said, `photoalbum on laptop requires s3-bucket from anchor`) ||
|
||||
!strings.Contains(said, "left out of anchor's grants") {
|
||||
t.Fatalf("the push does not say whom it leaves out:\n%s", said)
|
||||
}
|
||||
|
||||
// And `status` names it, and does not call the mesh well while it stands.
|
||||
asked, err := theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(asked.overflowing) != 1 || asked.overflowing[0].Module != "photoalbum" {
|
||||
t.Fatalf("status does not carry the overflow: %+v", asked.overflowing)
|
||||
}
|
||||
if asked.well() {
|
||||
t.Fatal("a mesh with a consumer left out of its grants reads as well")
|
||||
}
|
||||
shown := printed(t, func() error { return printStatus(asked) })
|
||||
if !strings.Contains(shown, "identified too long for a provision they require") ||
|
||||
!strings.Contains(shown, "mesh_laptop_photoalbum") {
|
||||
t.Fatalf("status does not say it:\n%s", shown)
|
||||
}
|
||||
body, err := statusAsJSON(asked)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var doc struct {
|
||||
Overflowing []catalogue.Overflow `json:"overflowing"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &doc); err != nil || len(doc.Overflowing) != 1 ||
|
||||
doc.Overflowing[0].Bound.Max != 20 {
|
||||
t.Fatalf("the document does not carry it: %v\n%s", err, body)
|
||||
}
|
||||
}
|
||||
@@ -45,11 +45,11 @@ func TestADeclarationComposedEarlierIsNumberedLowerWhateverOrderItIsSent(t *test
|
||||
// fails leaves nothing composed for that machine, and the others are still composed.
|
||||
func TestTheNumberIsTakenBeforeComposingAndItsFailureIsARefusal(t *testing.T) {
|
||||
calls := 0
|
||||
allot := func(node string) (order, error) {
|
||||
allot := func(node string) (int64, error) {
|
||||
if node == "anchor" {
|
||||
return order{}, context.DeadlineExceeded
|
||||
return 0, context.DeadlineExceeded
|
||||
}
|
||||
return order{sequence: 7}, nil
|
||||
return 7, nil
|
||||
}
|
||||
sending, refusals := composeEach([]string{"anchor", "laptop"}, allot, func(node string) (sendable, error) {
|
||||
calls++
|
||||
@@ -77,7 +77,7 @@ func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) {
|
||||
}
|
||||
cancel() // the sender is going away: its context is cancelled between the send and the record
|
||||
body := []byte(`{"declaration":1,"resources":[]}`)
|
||||
digest, err := recordSent(ctx, inv, "anchor", body, nil, 0)
|
||||
digest, err := recordSent(ctx, inv, "anchor", body, nil)
|
||||
if err != nil {
|
||||
// NodeByName on the cancelled context may itself refuse; the record must still be possible
|
||||
// through the detached context, so look the node up again on a live one.
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
)
|
||||
|
||||
// A command run at a shell (novox/hq to-be 45 §6): under the holder's epoch while a controller holds the
|
||||
// lease, read at the moment it acts; under a lease of its own while none does, given back as it ends.
|
||||
func TestACommandActsUnderTheHoldersEpochOrItsOwn(t *testing.T) {
|
||||
url, kv := aBusForTheLease(t)
|
||||
t.Setenv(broker.NATSVar, url)
|
||||
ctx := t.Context()
|
||||
|
||||
// Nobody holds it: the command takes it, and gives it back.
|
||||
cmd := &actor{}
|
||||
own, err := cmd.epoch(ctx)
|
||||
if err != nil || own == 0 {
|
||||
t.Fatalf("a command with nobody holding the lease acts as %d (%v)", own, err)
|
||||
}
|
||||
if h, found, _ := lease.Current(ctx, kv); !found || h.Epoch != own {
|
||||
t.Fatalf("the command's lease is not on the bus: %+v", h)
|
||||
}
|
||||
cmd.release()
|
||||
if _, found, _ := lease.Current(ctx, kv); found {
|
||||
t.Fatal("the command did not give its lease back as it ended")
|
||||
}
|
||||
|
||||
// A controller holds it: a command acts under that epoch.
|
||||
l, err := lease.Open(ctx, mustJetStream(t, url), broker.LeaseBucket, lease.Options{Holder: lease.Holder{Instance: "serving"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held, err := l.TryTake(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
borrower := &actor{}
|
||||
defer borrower.release()
|
||||
if got, err := borrower.epoch(ctx); err != nil || got != held {
|
||||
t.Fatalf("a command acts as %d (%v), want the holder's %d", got, err, held)
|
||||
}
|
||||
// The holder lets go: the command does not go on under an epoch nobody holds.
|
||||
l.Release(ctx)
|
||||
if _, err := borrower.epoch(ctx); err == nil {
|
||||
t.Fatal("a command acted under an epoch nobody holds any more")
|
||||
}
|
||||
}
|
||||
|
||||
// mustJetStream is a connection of its own to the test bus.
|
||||
func mustJetStream(t *testing.T, url string) jetstream.JetStream {
|
||||
t.Helper()
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(conn.Close)
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return js
|
||||
}
|
||||
@@ -1,175 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
)
|
||||
|
||||
// Two controllers at once (novox/hq to-be 45 §6, issue 204; the half of replay R1 that lives here): two
|
||||
// serving controllers over one store and one bus. The second waits while the first holds the lease; on
|
||||
// a handover it takes it at a higher epoch, the record says which held what and how each ended; and the
|
||||
// one that lost it acts no more — no declaration composed, no plan and no condition written — the
|
||||
// moment it lost it.
|
||||
//
|
||||
// MESH_TEST_POSTGRES=… MESH_TEST_NATS=nats://127.0.0.1:14222 go test ./cmd/mesh-controller/ -run Controllers
|
||||
|
||||
// aBusForTheLease is the test bus with the controller's lease bucket new.
|
||||
func aBusForTheLease(t *testing.T) (string, jetstream.KeyValue) {
|
||||
t.Helper()
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(conn.Close)
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = js.DeleteKeyValue(t.Context(), broker.LeaseBucket)
|
||||
if err := broker.EnsureLeaseBucket(t.Context(), js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv, err := js.KeyValue(t.Context(), broker.LeaseBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = js.DeleteKeyValue(context.Background(), broker.LeaseBucket) })
|
||||
return url, kv
|
||||
}
|
||||
|
||||
func TestTwoControllersOneActs(t *testing.T) {
|
||||
url, kv := aBusForTheLease(t)
|
||||
inv := inventory.ForTest(t)
|
||||
ctx := t.Context()
|
||||
|
||||
// Controller A takes the lease.
|
||||
a := &actor{}
|
||||
aCtx, stopA := context.WithCancel(ctx)
|
||||
defer stopA()
|
||||
lostA, err := a.serveUnderTheLease(aCtx, inv, url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
epochA, err := a.epoch(ctx)
|
||||
if err != nil || epochA == 0 {
|
||||
t.Fatalf("A holds no epoch: %d, %v", epochA, err)
|
||||
}
|
||||
|
||||
// Controller B starts while A holds it, and waits — acting on nothing meanwhile.
|
||||
b := &actor{}
|
||||
bCtx, stopB := context.WithCancel(ctx)
|
||||
defer stopB()
|
||||
tookB := make(chan (<-chan struct{}), 1)
|
||||
go func() {
|
||||
lost, err := b.serveUnderTheLease(bCtx, inv, url)
|
||||
if err != nil {
|
||||
t.Errorf("B: %v", err)
|
||||
close(tookB)
|
||||
return
|
||||
}
|
||||
tookB <- lost
|
||||
}()
|
||||
select {
|
||||
case <-tookB:
|
||||
t.Fatal("B took the lease while A held it")
|
||||
case <-time.After(3 * time.Second):
|
||||
}
|
||||
if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) {
|
||||
t.Fatalf("B, waiting, may act: %v", err)
|
||||
}
|
||||
|
||||
// A hands over, as a controller being replaced does: B takes the lease at once, at a higher epoch.
|
||||
stopA()
|
||||
a.release()
|
||||
var lostB <-chan struct{}
|
||||
select {
|
||||
case lostB = <-tookB:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("B did not take the lease A gave back")
|
||||
}
|
||||
select {
|
||||
case <-lostA:
|
||||
default:
|
||||
t.Fatal("A, having given the lease back, is not told it no longer holds it")
|
||||
}
|
||||
epochB, err := b.epoch(ctx)
|
||||
if err != nil || epochB <= epochA {
|
||||
t.Fatalf("B acts as epoch %d after A's %d (%v): an epoch only grows", epochB, epochA, err)
|
||||
}
|
||||
if _, err := a.epoch(ctx); err == nil {
|
||||
t.Fatal("A acts after giving the lease back")
|
||||
}
|
||||
ea, _, _ := inv.EpochOf(ctx, epochA)
|
||||
eb, _, _ := inv.EpochOf(ctx, epochB)
|
||||
if ea.How != inventory.EpochReleased || eb.Ended != nil || eb.Instance != instance {
|
||||
t.Fatalf("the record of the handover reads %+v then %+v", ea, eb)
|
||||
}
|
||||
|
||||
// Something else writes the lease's key — a third controller on a clock that read it as expired:
|
||||
// B's next renewal is refused, and B stops acting at once.
|
||||
if _, err := kv.Put(ctx, lease.Key, []byte(`{"instance":"a third controller","epoch":1}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-lostB:
|
||||
case <-time.After(2 * lease.RenewEvery):
|
||||
t.Fatal("B was not told it lost the lease")
|
||||
}
|
||||
if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) {
|
||||
t.Fatalf("B acts after losing the lease: %v", err)
|
||||
}
|
||||
// Nothing B does is written: a declaration's number is not taken, a plan is not saved, a condition
|
||||
// is not raised.
|
||||
inv.ActsUnder(b.epoch)
|
||||
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saved := inventory.Plan{ID: "plan-after-loss", Repository: "novox/app", Commit: "c0ffee00", Created: time.Now(),
|
||||
State: inventory.PlanBuilding}
|
||||
if err := inv.SavePlan(ctx, &saved); err == nil {
|
||||
t.Fatal("B wrote a plan after losing the lease")
|
||||
}
|
||||
store := conditions.NewInMemory()
|
||||
keeper := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store,
|
||||
Epoch: func() (uint64, error) { return b.epoch(ctx) }})
|
||||
defer keeper.Close(context.Background())
|
||||
if _, err := keeper.Observe(ctx, conditions.Observation{Scope: conditions.ScopeCore, ID: "x", Kind: "x",
|
||||
Severity: conditions.Warning, Summary: "x", Source: "test"}); err == nil {
|
||||
t.Fatal("B raised a condition after losing the lease")
|
||||
}
|
||||
if ended, _, _ := inv.EpochOf(ctx, epochB); ended.How != inventory.EpochLost {
|
||||
t.Fatalf("B's epoch does not say it was lost: %+v", ended)
|
||||
}
|
||||
}
|
||||
|
||||
// A controller whose bus refuses it the lease's key, with nobody holding it, serves without the lease:
|
||||
// it acts with no epoch — refused by no node-engine — says so, and takes the lease once it can.
|
||||
func TestAControllerTheBusRefusesTheLeaseServesUnleasedAndSaysSo(t *testing.T) {
|
||||
a := &actor{serving: true, unleased: "the bus refused the lease's key"}
|
||||
if epoch, err := a.epoch(context.Background()); err != nil || epoch != 0 {
|
||||
t.Fatalf("an unleased controller answers %d, %v: it acts, claiming no epoch", epoch, err)
|
||||
}
|
||||
if st := a.standing(); st.Unleased == "" || st.Held {
|
||||
t.Fatalf("its standing says %+v", st)
|
||||
}
|
||||
// One that neither holds nor is unleased — still waiting — acts on nothing.
|
||||
waiting := &actor{serving: true}
|
||||
if _, err := waiting.epoch(context.Background()); !errors.Is(err, lease.ErrNotHeld) {
|
||||
t.Fatalf("a controller waiting for the lease may act: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -55,9 +55,6 @@ func run() error {
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
// Whatever this process holds of the controller's lease is given back as it ends (novox/hq to-be
|
||||
// 45 §6), so the next controller takes it at once rather than after its age.
|
||||
defer theLease.release()
|
||||
|
||||
switch args[0] {
|
||||
case "build":
|
||||
@@ -148,28 +145,6 @@ func run() error {
|
||||
return seatCommand(ctx, args[1:])
|
||||
case "status":
|
||||
return statusCommand(ctx, args[1:])
|
||||
// Acts done by hand, and why (novox/hq to-be 45 §7).
|
||||
case "hand-act":
|
||||
return handActCommand(ctx, args[1:])
|
||||
case "hand-acts":
|
||||
return handActCommand(ctx, append([]string{"list"}, args[1:]...))
|
||||
// What the mesh's bounds will be set from (novox/hq to-be 45 Phase 0).
|
||||
case "durations":
|
||||
return durationsCommand(ctx, args[1:])
|
||||
// What is wrong, and the self-check (novox/hq to-be 45 §2, §4).
|
||||
case "conditions":
|
||||
return conditionsCommand(ctx, args[1:])
|
||||
case "doctor":
|
||||
return doctorCommand(ctx, args[1:])
|
||||
// What the healers did, and their brake (novox/hq to-be 45 §7).
|
||||
case "healers":
|
||||
return healersCommand(ctx, args[1:])
|
||||
// A consumer the mesh stopped asking for: retired, waiting for a person, deleted only by one
|
||||
// (novox/hq ADR 0230).
|
||||
case "retire":
|
||||
return retireCommand(ctx, args[1:])
|
||||
case "cleanup":
|
||||
return cleanupCommand(ctx, args[1:])
|
||||
case "version":
|
||||
fmt.Println(version)
|
||||
return nil
|
||||
@@ -216,12 +191,6 @@ func usage() {
|
||||
upgrade <name> roll-out [--together] ...send it to the machines running it
|
||||
upgrade <name> record ...record that they are behind, and send nothing
|
||||
status [--json] what is wrong, what is quiet, and what is out of date
|
||||
retire [--json] every provider waiting for a person to approve a retirement (ADR 0230)
|
||||
retire approve <node> <module> --why <text> retire what it waits with: access off, data kept
|
||||
retire reject <node> <module> --why <text> keep them active; a warning stays open
|
||||
cleanup [list] [--json] every retired consumer per provider: age, size, why
|
||||
cleanup delete <node> <module> <consumer> --why <text> the provider deletes that one retired consumer
|
||||
cleanup delete --older-than <days> --why <text> [--confirm] list those older; delete only with --confirm
|
||||
seats [--json] every seat this mesh defines, what it delivers, and who holds it
|
||||
seat rename <from> <to> rename a seat; its former name still resolves (ADR 0122)
|
||||
seat <name> --to <node>/<module> hand a seat to that assignment as one act; never empty in between (ADR 0131)
|
||||
@@ -257,34 +226,19 @@ func usage() {
|
||||
kill <id> end a build where it runs; recorded failed, killed by hand
|
||||
pause [<node>] / resume [<node>] the build seat's holder there, or every holder, takes nothing new / again
|
||||
plans retry <id> ask a failed plan's failed builds again, and carry the plan on
|
||||
plans stop|close <id> --why <text> end a plan by hand; recorded in the hand-act log
|
||||
hand-act record <what> --why <text> --cause <word> [--condition <key>]
|
||||
record an act done by hand outside the mesh (to-be 45 §7)
|
||||
hand-acts [--days N] [--json] what was done by hand lately, why, and which causes repeat
|
||||
conditions [--scope S] [--severity S] [--machine M] [--json]
|
||||
what is wrong now: every open condition, urgent first (to-be 45 §2)
|
||||
conditions show <key> one condition whole, with its evidence
|
||||
conditions silence <key> --for <d> --why <text> send no message for it a while; a hand act
|
||||
conditions history [--days N] [--key K] every raising, change and clearing lately
|
||||
doctor [run|probes|signals] [--json]
|
||||
the self-check: the last verdict, a run now, the probes, the signals' ages
|
||||
healers [--days N] [--json] the healers, what they did lately, and their brake (to-be 45 §7)
|
||||
durations [--kind K] [--days N] [--json]
|
||||
apply, heartbeat, plan-tier and build durations, per machine or module
|
||||
collection [--json] kept archives held/unheld by a manifest, and what the sweep may let go
|
||||
builder issue <name> a broker account for a build machine, scoped to build work,
|
||||
delivered as the builder module's broker secret (module add it first)
|
||||
licence add|list|use|key model access, under the name a person calls it
|
||||
licence manager <name> <node> the node that holds a refreshable licence's refresh token
|
||||
licence refresh <name> mint a new access token and seal it to every holder
|
||||
rotate <provision> [--consumer <n>] [--module <m>] a new credential for every holder, both ends at once
|
||||
rotate <provision> [--consumer <n>] a new credential for every holder, both ends at once
|
||||
ask <module> <tool> [json] call one of a module's tools over the broker, and print its answer
|
||||
pin <node> <provision> <from-node> <module>
|
||||
which provider this one gets a provision from: the module, and its node
|
||||
unpin <node> <provision> put that question back
|
||||
plan <node> [--files|--json] what that node would run, and why
|
||||
push [<node>] [--behind] [--why <text>] send a node everything it should be, or only those
|
||||
that need it; --why records it in the hand-act log
|
||||
push [<node>] [--behind] send a node everything it should be, or only those that need it
|
||||
version what this binary is
|
||||
|
||||
Each context reaches its own store through its own credential (novox/hq ADR 0008), named
|
||||
@@ -337,9 +291,6 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
||||
// When it was asked, so a plan takes as its outcome only a build asked for it or after it
|
||||
// (novox/hq 04-ISSUES/219). Zero when the id does not say.
|
||||
asked, _ := link.BuildAskedAt(result.ID)
|
||||
// And how long it took, asked to heard, which a build's bound will be set from (novox/hq to-be 45
|
||||
// Phase 0). Said if lost; never a reason not to take the build in.
|
||||
recordBuildDuration(ctx, b.inv, result, asked)
|
||||
switch {
|
||||
case err != nil && result.Failed != "":
|
||||
fmt.Printf("%s: %v\n", result.ID, err)
|
||||
@@ -366,7 +317,5 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
||||
result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit))
|
||||
saysWhenThePolicyActs(ctx, b.inv, manifest.Module)
|
||||
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
|
||||
// A module registered may be one a machine is now behind: `status` is composed again.
|
||||
statusFrom.nudge()
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,14 +1,11 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"crypto/ecdh"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
@@ -40,9 +37,6 @@ func aMesh(t *testing.T) *stores {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(open.Close)
|
||||
// A condition store of its own, held in memory (novox/hq to-be 45 §2): status leads with what is
|
||||
// open, and a mesh with no bus would otherwise read as one whose conditions cannot be read.
|
||||
withConditionsInMemory(t)
|
||||
for _, m := range provided {
|
||||
if err := open.inventory.Provide(t.Context(), m); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -112,17 +106,3 @@ func rivals() (catalogue.Manifest, catalogue.Manifest) {
|
||||
return catalogue.Manifest{Module: "rival-one", Version: "1", Claims: claim},
|
||||
catalogue.Manifest{Module: "rival-two", Version: "1", Claims: claim}
|
||||
}
|
||||
|
||||
// withConditionsInMemory gives the test a condition store in memory, as the serving controller's.
|
||||
func withConditionsInMemory(t *testing.T) (*conditions.Keeper, *conditions.InMemory) {
|
||||
t.Helper()
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||
before := conditionsFrom
|
||||
conditionsFrom = k
|
||||
t.Cleanup(func() {
|
||||
conditionsFrom = before
|
||||
k.Close(context.Background())
|
||||
})
|
||||
return k, store
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
@@ -60,13 +59,9 @@ func catchingUpOnMerges(ctx context.Context, open *stores, announced merges) {
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
watchedMerges.begin()
|
||||
err := catchUpOnMerges(ctx, time.Now(), announced, catalogued, f.SourceMoved, func(format string, args ...any) {
|
||||
fmt.Printf(format+"\n", args...)
|
||||
})
|
||||
// What the pass found is what S5 says (novox/hq to-be 45 §3); a pass that could not read
|
||||
// says that instead, and leaves what the last one found standing.
|
||||
watchedMerges.end(time.Now(), err)
|
||||
// A pass that cannot read says so once, not every five minutes, and says when it reads again.
|
||||
why := ""
|
||||
if err != nil {
|
||||
@@ -119,8 +114,6 @@ func catchUpOnMerges(ctx context.Context, now time.Time, announced merges,
|
||||
for _, e := range moves {
|
||||
names = append(names, e.Manifest.Module)
|
||||
}
|
||||
watchedMerges.found(missedMerge{Owner: a.Owner, Repo: a.Repo, Base: a.Base, Commit: a.Commit,
|
||||
At: a.At, Modules: names})
|
||||
say("%s/%s merged into %s (%.8s), announced %s ago, and the controller never acted on it: the bus "+
|
||||
"did not hand the announcement over (novox/hq issue 266). %s %s behind it; acting on it now",
|
||||
a.Owner, a.Repo, a.Base, a.Commit, now.Sub(a.At).Round(time.Minute), readableList(names),
|
||||
@@ -136,54 +129,3 @@ func catchUpOnMerges(ctx context.Context, now time.Time, announced merges,
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// missedMerge is one merge the bus announced and never handed over, as a pass found it.
|
||||
type missedMerge struct {
|
||||
Owner, Repo, Base, Commit string
|
||||
// At is when the bus took the announcement.
|
||||
At time.Time
|
||||
Modules []string
|
||||
}
|
||||
|
||||
// mergeWatch is what the passes found, for S5 (novox/hq to-be 45 §3): **a merge nothing read is
|
||||
// said**, urgent, even though the pass acts on it at once — the bus skipping a message is a fault of
|
||||
// the transport the mesh's every change rides on, and acting late is the repair, not the absence of
|
||||
// the fault. The next pass, finding it acted on, clears it.
|
||||
type mergeWatch struct {
|
||||
mu sync.Mutex
|
||||
passed time.Time
|
||||
err error
|
||||
finding []missedMerge
|
||||
missed []missedMerge
|
||||
}
|
||||
|
||||
var watchedMerges = &mergeWatch{}
|
||||
|
||||
func (w *mergeWatch) begin() {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.finding = nil
|
||||
}
|
||||
|
||||
func (w *mergeWatch) found(m missedMerge) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.finding = append(w.finding, m)
|
||||
}
|
||||
|
||||
func (w *mergeWatch) end(at time.Time, err error) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.passed, w.err = at, err
|
||||
if err == nil {
|
||||
w.missed = w.finding
|
||||
}
|
||||
}
|
||||
|
||||
// last is when the last pass ended, what the last pass that read found, and what the last pass
|
||||
// could not read.
|
||||
func (w *mergeWatch) last() (time.Time, []missedMerge, error) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.passed, append([]missedMerge(nil), w.missed...), w.err
|
||||
}
|
||||
|
||||
@@ -64,19 +64,8 @@ func moduleCommand(ctx context.Context, args []string) error {
|
||||
// `check` needs no mesh, and must not: it is what somebody runs in their own repository before
|
||||
// there is a mesh in reach (novox/hq issue 148). A directory expands to every manifest under it.
|
||||
if args[0] == "check" {
|
||||
set := flag.NewFlagSet("module check", flag.ContinueOnError)
|
||||
// The longest machine name an identity must fit on (novox/hq ADR 0225): a mesh passes its own.
|
||||
longest := set.Int("longest-machine-name", catalogue.DefaultLongestMachine,
|
||||
"judge each module's identity on a machine name this many characters long")
|
||||
given, err := parseAround(set, args[1:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *longest < 1 {
|
||||
return errors.New("--longest-machine-name is a length, at least 1")
|
||||
}
|
||||
var paths []string
|
||||
for _, a := range given {
|
||||
for _, a := range args[1:] {
|
||||
if info, err := os.Stat(a); err == nil && info.IsDir() {
|
||||
under, err := manifestsUnder(a)
|
||||
if err != nil {
|
||||
@@ -87,7 +76,7 @@ func moduleCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
paths = append(paths, a)
|
||||
}
|
||||
return moduleCheckFor(paths, *longest, os.Stdout)
|
||||
return moduleCheck(paths, os.Stdout)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -388,7 +387,7 @@ func brokerCommand(ctx context.Context, args []string) error {
|
||||
return busAccounts(ctx, args[1:])
|
||||
}
|
||||
if len(args) > 0 && args[0] == "consumer-reset" {
|
||||
return consumerReset(ctx, args[1:])
|
||||
return consumerReset(args[1:])
|
||||
}
|
||||
if len(args) == 0 || args[0] != "show" {
|
||||
return errors.New("broker show | broker certificate [--check] --into <directory> | broker accounts --into <file> | " +
|
||||
@@ -415,21 +414,10 @@ func brokerCommand(ctx context.Context, args []string) error {
|
||||
// consumerReset re-makes one consumer on a stream that keeps history to start from now (novox/hq issue
|
||||
// 248): the way out of a consumer replaying a week of announcements, said rather than done by hand. A
|
||||
// person's act — what was pending is dropped — so it is a command, and nothing calls it on its own.
|
||||
func consumerReset(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("broker consumer-reset", flag.ContinueOnError)
|
||||
// A repair by hand, which says why (novox/hq to-be 45 §7).
|
||||
why := addHandActFlags(set)
|
||||
args, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
func consumerReset(args []string) error {
|
||||
if len(args) != 2 {
|
||||
return errors.New("broker consumer-reset <stream> <consumer> --why <text>, e.g. broker consumer-reset EVENTS controller --why ...")
|
||||
return errors.New("broker consumer-reset <stream> <consumer>, e.g. broker consumer-reset EVENTS controller")
|
||||
}
|
||||
if err := why.require("broker consumer-reset"); err != nil {
|
||||
return err
|
||||
}
|
||||
why.record(ctx, "broker consumer-reset", args)
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -517,22 +505,15 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
|
||||
fmt.Printf(" public domain %s\n", domain)
|
||||
}
|
||||
|
||||
// Every open condition about this machine (novox/hq to-be 45 §2) — a provider here failing a
|
||||
// consumer, or a consumer here failed, among them (ADR 0224). Before the capabilities, because it
|
||||
// is something not working now and they are a description. Unreadable is said, not passed over.
|
||||
open, err := openConditions(ctx)
|
||||
// A provider here failing a consumer, or a consumer here failed (novox/hq ADR 0224). Before the
|
||||
// capabilities, because it is something not working now and they are a description.
|
||||
failing, err := failingProviders(ctx, inv)
|
||||
if err != nil {
|
||||
fmt.Printf("\n the open conditions could NOT be read, so whether anything here is wrong is not known: %v\n", err)
|
||||
return err
|
||||
}
|
||||
var here []conditions.Condition
|
||||
for _, c := range open {
|
||||
if concerns(c, name) {
|
||||
here = append(here, c)
|
||||
}
|
||||
}
|
||||
if len(here) > 0 {
|
||||
fmt.Printf("\n %d open condition(s) about this machine:\n", len(here))
|
||||
for _, line := range conditionLines(here, time.Now()) {
|
||||
if here := failingOn(failing, name); len(here) > 0 {
|
||||
fmt.Printf("\n %d consumer(s) a provider keeps failing, here or for a module here:\n", len(here))
|
||||
for _, line := range failingLines(here, time.Now()) {
|
||||
fmt.Printf(" %s\n", line)
|
||||
}
|
||||
}
|
||||
|
||||
+16
-50
@@ -399,7 +399,7 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
||||
}
|
||||
out := sendable{Resources: composed.Resources, Adoption: adoption,
|
||||
Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
|
||||
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld}
|
||||
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut}
|
||||
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
|
||||
// 0221). Read only on the send path: a question about what would be sent records nothing.
|
||||
if choosing == Allocating {
|
||||
@@ -464,11 +464,6 @@ func reportLeftOut(node string, declared sendable) {
|
||||
"what the machine holds for it is kept and its containers are untouched. %s\n",
|
||||
node, m, declared.leftOutWhy[m])
|
||||
}
|
||||
// And whom it serves nothing, because their identity overflows what the provision keeps (ADR
|
||||
// 0225): the machine is sent everything else, and the consumer is named.
|
||||
for _, o := range declared.withheld {
|
||||
fmt.Printf("%s: %s\n", node, o)
|
||||
}
|
||||
}
|
||||
|
||||
// renderingFor is everything a node's declaration is composed with, and the node's record.
|
||||
@@ -476,7 +471,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
plan catalogue.Resolution, settings catalogue.SettingsBy,
|
||||
gens map[string]catalogue.Generator, choosing Choosing) (catalogue.Rendering, inventory.Node, error) {
|
||||
inv := open.inventory
|
||||
grants, withheld, err := grantsFor(ctx, open, node)
|
||||
grants, err := grantsFor(ctx, open, node)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
@@ -799,7 +794,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
|
||||
Kept: kept, Adopted: record.Adopted, OutwardLinks: outwardLinks,
|
||||
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, SeatReach: reach, Built: built,
|
||||
BusUsers: busUsers, Withheld: withheld,
|
||||
BusUsers: busUsers,
|
||||
}, record, nil
|
||||
}
|
||||
|
||||
@@ -935,34 +930,28 @@ func certificateFor(ctx context.Context, open *stores, node string) (string, str
|
||||
// The mirror of what a consumer is given, and the half that makes the credential real: a password
|
||||
// nothing was told to create is a password that authenticates nowhere. Sealed to this node, so
|
||||
// the mesh hands over something it cannot itself use.
|
||||
//
|
||||
// **One consumer's identity never refuses the provider's machine** (novox/hq ADR 0225, issue 263).
|
||||
// A consumer whose identity overflows the provision's bound is left out of the grants and returned
|
||||
// beside them, for push, plan and `status` to say; every other consumer is granted and the provider's
|
||||
// declaration composes. Refusing here once made a whole machine unpushable for one module elsewhere.
|
||||
func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Grant, []catalogue.Overflow, error) {
|
||||
func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Grant, error) {
|
||||
inv := open.inventory
|
||||
issued, err := inv.SecretsFrom(ctx, node)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Where each consumer is, so a provider that must reach back to one does not have to know how
|
||||
// the mesh names machines.
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, err
|
||||
}
|
||||
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// What each consumer actually asked for, taken from that machine's own resolution rather than
|
||||
// from a record beside it. A provider told to create a password and not what to create it for
|
||||
// can do nothing with it, and the name a consumer wants is the consumer's to say.
|
||||
out := make([]catalogue.Grant, 0, len(issued))
|
||||
var withheld []catalogue.Overflow
|
||||
for _, s := range issued {
|
||||
plan, settings, err := planFor(ctx, open, s.Consumer)
|
||||
switch {
|
||||
@@ -975,11 +964,11 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
||||
// The mesh could not be asked what they wanted, which is not the same as their wanting
|
||||
// nothing — and withholding a grant on that reading takes a consumer's access away
|
||||
// (novox/hq 04-ISSUES/152).
|
||||
return nil, nil, fmt.Errorf("what %s asked of %s cannot be read: %w", s.Consumer, s.Name, err)
|
||||
return nil, fmt.Errorf("what %s asked of %s cannot be read: %w", s.Consumer, s.Name, err)
|
||||
}
|
||||
values, asks, err := plan.ContributionsFrom(s.Name, s.ConsumerModule, settings)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, err
|
||||
}
|
||||
// A port in there is the consumer's software port until this. The consumer is on another
|
||||
// machine, so the assignment that moved it is that machine's — fetched here rather than
|
||||
@@ -987,7 +976,7 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
||||
// this case (novox/hq 04-ISSUES/038, the cross-node half).
|
||||
published, err := portsOn(ctx, inv, s.Consumer, s.ConsumerModule)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, err
|
||||
}
|
||||
values = catalogue.AtPublishedPort(values, s.ConsumerModule, published)
|
||||
from := s.ConsumerModule
|
||||
@@ -998,10 +987,9 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
||||
from = ""
|
||||
}
|
||||
// The consumer's identity slug, from its own manifest, carried on the grant so the provider
|
||||
// derives the same login the consumer does (novox/hq ADR 0049). Judged against the bound of
|
||||
// this provision, as the consumer's resolution states it from the provider's offer (ADR
|
||||
// 0225): a consumer it would not fit is left out of the grants and said, rather than a login a
|
||||
// provider silently shortened — and rather than this whole machine refused for it.
|
||||
// derives the same login the consumer does (novox/hq ADR 0049). Refused here if it still would
|
||||
// not fit the tightest backend — the mesh chose the name, so the mesh refuses it, with the
|
||||
// remedy a short slug rather than a login a provider silently shortened.
|
||||
slug := ""
|
||||
for _, mm := range plan.Modules {
|
||||
if mm.Module == s.ConsumerModule {
|
||||
@@ -1010,32 +998,15 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
||||
}
|
||||
}
|
||||
if from != "" {
|
||||
bound := boundOfGrant(plan, s, node)
|
||||
source := catalogue.IdentitySource(slug, s.ConsumerModule)
|
||||
if catalogue.CheckIdentityWithin(s.Consumer, source, bound) != nil {
|
||||
withheld = append(withheld, catalogue.Overflow{Provision: s.Name, Provider: node,
|
||||
Consumer: s.Consumer, Module: s.ConsumerModule,
|
||||
Identity: catalogue.ConsumerIdentity(s.Consumer, source), Bound: bound})
|
||||
continue
|
||||
if err := catalogue.CheckIdentity(s.Consumer, catalogue.IdentitySource(slug, s.ConsumerModule)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
out = append(out, catalogue.Grant{
|
||||
Provision: s.Name, Consumer: s.Consumer, At: onNetwork[s.Consumer],
|
||||
From: from, Values: values, Slug: slug, Sealed: s.ForProvider, Local: s.Local})
|
||||
}
|
||||
return out, withheld, nil
|
||||
}
|
||||
|
||||
// boundOfGrant is the identity bound the consumer's own resolution states for the requirement this
|
||||
// grant answers. A requirement not found there is held to the tightest bound the mesh knows rather
|
||||
// than to none: what the provider keeps of it is not known here.
|
||||
func boundOfGrant(consumer catalogue.Resolution, s inventory.Secret, provider string) catalogue.IdentityBound {
|
||||
for _, n := range consumer.Needs {
|
||||
if n.Name == s.Name && n.For == s.ConsumerModule && n.From == provider {
|
||||
return n.Identity
|
||||
}
|
||||
}
|
||||
return catalogue.DefaultIdentityBound
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// listensLines is what a person is told about what this module would open, and why — the same
|
||||
@@ -1111,11 +1082,6 @@ func planCommand(ctx context.Context, args []string) error {
|
||||
left := plan.LeftOut(settings, record.Adopted)
|
||||
reportLeftOut(args[0], sendable{LeftOut: sortedKeysOf(left), leftOutWhy: left})
|
||||
}
|
||||
// And which of its modules no provider will grant, because the identity overflows the bound of
|
||||
// what it requires (novox/hq ADR 0225) — said on the machine the remedy is for.
|
||||
for _, o := range plan.Overflowing() {
|
||||
fmt.Printf("%s: %s\n", args[0], o)
|
||||
}
|
||||
// And a setting that reaches nothing — refused where it is stored, and said here for one
|
||||
// stored before its definition moved from under it.
|
||||
for _, m := range plan.Modules {
|
||||
|
||||
@@ -280,7 +280,7 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
||||
}
|
||||
}
|
||||
resumed(&p, fmt.Sprintf("tier %d retried by hand: %s asked again", p.Tier, strings.Join(failed, ", ")))
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if p.State != inventory.PlanBuilding {
|
||||
@@ -323,7 +323,7 @@ func joinAPlan(ctx context.Context, open *stores, module string) (bool, string,
|
||||
askModule(ctx, &p, module, byName)
|
||||
s := p.Modules[module]
|
||||
resumed(&p, fmt.Sprintf("tier %d: %s rebuilt by hand", p.Tier, module))
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
if s.State != "asked" {
|
||||
@@ -393,7 +393,7 @@ func retryRollouts(ctx context.Context, open *stores, p *inventory.Plan) (string
|
||||
}
|
||||
p.State = inventory.PlanRolling
|
||||
p.Note = fmt.Sprintf("tier %d retried by hand; sent %s first again", p.Tier, strings.Join(said, "; "))
|
||||
if err := open.inventory.SavePlan(ctx, p); err != nil {
|
||||
if err := open.inventory.SavePlan(ctx, *p); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("%s retried at tier %d of %d: sent %s first again; the rest follow once it reports it "+
|
||||
|
||||
@@ -18,7 +18,7 @@ func TestAControllerLeavesThePlansToTheOneHoldingThem(t *testing.T) {
|
||||
plan := inventory.Plan{ID: "plan-213", Repository: "r", Commit: "abc", Created: now, Updated: now,
|
||||
State: inventory.PlanRolling, Tier: 1, Tiers: [][]string{{"app"}},
|
||||
Modules: map[string]*inventory.PlanModule{"app": {State: "built"}}}
|
||||
if err := open.inventory.SavePlan(ctx, &plan); err != nil {
|
||||
if err := open.inventory.SavePlan(ctx, plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,903 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
"github.com/novox/mesh-host/validate"
|
||||
"golang.org/x/net/dns/dnsmessage"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// The probes of the self-check's registry (doctor.go), one function each. A probe answers what is
|
||||
// wrong now as observations, or an error when it could not tell — never "nothing" for "could not
|
||||
// look" (ADR 0227 rule 4).
|
||||
|
||||
// probeDeclarations is D1: every machine's declaration composes, and the node-engine's own validator
|
||||
// (mesh-host's `validate`, the package the host runs) takes it.
|
||||
func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
open := d.open
|
||||
nodes, err := open.inventory.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []conditions.Observation
|
||||
// The private network every declaration is composed with. Not computable is not "could not
|
||||
// look": it is the finding — no machine's declaration composes without it — and the machines that
|
||||
// do not resolve, which are usually why, are named below.
|
||||
gens, gensErr := generators(ctx, open)
|
||||
if gensErr != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMesh, ID: "private-network",
|
||||
Token: "uncomposable", Severity: conditions.Urgent,
|
||||
Summary: "the private network cannot be computed, so no machine's declaration composes",
|
||||
Said: firstLine(gensErr.Error())})
|
||||
}
|
||||
for _, n := range nodes {
|
||||
plan, settings, err := planFor(ctx, open, n.Name)
|
||||
if err != nil && !unresolvable(err) {
|
||||
return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err)
|
||||
}
|
||||
var problems []string
|
||||
if err == nil && gensErr == nil {
|
||||
var declared sendable
|
||||
if declared, err = declarationWith(ctx, open, n.Name, plan, settings, gens, Reading); err == nil {
|
||||
// With the order it was last sent, so the validator reads the envelope a machine is sent
|
||||
// — its epoch included (novox/hq to-be 45 §6).
|
||||
var serr error
|
||||
if declared.Sequence, serr = open.inventory.Sequence(ctx, n.ID); serr == nil {
|
||||
declared.Epoch, serr = open.inventory.SentEpoch(ctx, n.ID)
|
||||
}
|
||||
if serr != nil {
|
||||
return nil, serr // the store, not the machine: the probe could not run
|
||||
}
|
||||
var body []byte
|
||||
if body, err = declared.Body(); err == nil {
|
||||
problems = validate.Declaration(body)
|
||||
}
|
||||
}
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
switch {
|
||||
case err != nil:
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "uncomposable",
|
||||
Machine: n.Name, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("nothing can be sent to %s: its declaration does not compose — %s", n.Name,
|
||||
oneLine(err.Error())),
|
||||
Said: oneLine(err.Error())})
|
||||
case len(problems) > 0:
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "refused",
|
||||
Machine: n.Name, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("%s's node-engine would refuse its declaration whole: %d problem(s), the first: %s",
|
||||
n.Name, len(problems), problems[0]),
|
||||
Said: strings.Join(problems, "; ")})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// probeResolvers is D2: every holder of the mesh's resolver answers each machine's name with its
|
||||
// address for IPv4, and with no address and no error for IPv6 — NODATA, not NXDOMAIN, which musl
|
||||
// takes as final (issue 262).
|
||||
func probeResolvers(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
inv := d.open.inventory
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
holders, err := replicatedHolders(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resolvers := holders["mesh-dns-resolver"]
|
||||
if len(resolvers) == 0 {
|
||||
return nil, nil // the mesh holds no resolver of its own: nothing is asked of one
|
||||
}
|
||||
places, err := onTheNetwork(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
suffix := overlay.Suffix()
|
||||
var out []conditions.Observation
|
||||
for holder, at := range resolvers {
|
||||
var wrong []string
|
||||
for _, p := range places {
|
||||
if p.Address == "" {
|
||||
continue
|
||||
}
|
||||
name := p.Name + "." + suffix
|
||||
v4, rcode, err := askResolver(ctx, at, name, dnsmessage.TypeA)
|
||||
switch {
|
||||
case err != nil:
|
||||
wrong = append(wrong, fmt.Sprintf("%s for %s: %v", "A", name, err))
|
||||
continue
|
||||
case rcode != dnsmessage.RCodeSuccess:
|
||||
wrong = append(wrong, fmt.Sprintf("%s answered %s for %s (A)", holder, rcode, name))
|
||||
case !slices.Contains(v4, p.Address):
|
||||
wrong = append(wrong, fmt.Sprintf("%s answered %v for %s (A), not %s", holder, v4, name, p.Address))
|
||||
}
|
||||
v6, rcode, err := askResolver(ctx, at, name, dnsmessage.TypeAAAA)
|
||||
switch {
|
||||
case err != nil:
|
||||
wrong = append(wrong, fmt.Sprintf("%s for %s: %v", "AAAA", name, err))
|
||||
case rcode != dnsmessage.RCodeSuccess:
|
||||
wrong = append(wrong, fmt.Sprintf("%s answered %s for %s (AAAA), not NODATA: a musl machine "+
|
||||
"takes that as no such name", holder, rcode, name))
|
||||
case len(v6) > 0:
|
||||
wrong = append(wrong, fmt.Sprintf("%s answered %v for %s (AAAA); the mesh has no IPv6 addresses", holder, v6, name))
|
||||
}
|
||||
}
|
||||
if len(wrong) > 0 {
|
||||
node := strings.TrimSuffix(holder, "."+suffix)
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeSeat, ID: "mesh-dns-resolver." + node,
|
||||
Token: "wrong", Machine: node, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("the mesh's resolver on %s does not answer machine names as it must: %s",
|
||||
node, wrong[0]),
|
||||
Said: strings.Join(wrong, "; ")})
|
||||
}
|
||||
}
|
||||
return sortedFound(out), nil
|
||||
}
|
||||
|
||||
// resolverPort is where a resolver answers; a variable so a test can stand one up.
|
||||
var resolverPort = "53"
|
||||
|
||||
// askResolver asks one resolver one question over UDP, and answers the addresses and the code.
|
||||
func askResolver(ctx context.Context, at, name string, kind dnsmessage.Type) ([]string, dnsmessage.RCode, error) {
|
||||
q, err := dnsmessage.NewName(strings.TrimSuffix(name, ".") + ".")
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
msg := dnsmessage.Message{Header: dnsmessage.Header{ID: uint16(time.Now().UnixNano()), RecursionDesired: true},
|
||||
Questions: []dnsmessage.Question{{Name: q, Type: kind, Class: dnsmessage.ClassINET}}}
|
||||
packed, err := msg.Pack()
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
dialer := net.Dialer{Timeout: 3 * time.Second}
|
||||
conn, err := dialer.DialContext(ctx, "udp", net.JoinHostPort(at, resolverPort))
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
defer conn.Close()
|
||||
_ = conn.SetDeadline(time.Now().Add(3 * time.Second))
|
||||
if _, err := conn.Write(packed); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
buf := make([]byte, 1500)
|
||||
n, err := conn.Read(buf)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("no answer from %s: %w", at, err)
|
||||
}
|
||||
var answer dnsmessage.Message
|
||||
if err := answer.Unpack(buf[:n]); err != nil {
|
||||
return nil, 0, fmt.Errorf("an answer from %s that cannot be read: %w", at, err)
|
||||
}
|
||||
var addresses []string
|
||||
for _, a := range answer.Answers {
|
||||
switch r := a.Body.(type) {
|
||||
case *dnsmessage.AResource:
|
||||
addresses = append(addresses, net.IP(r.A[:]).String())
|
||||
case *dnsmessage.AAAAResource:
|
||||
addresses = append(addresses, net.IP(r.AAAA[:]).String())
|
||||
}
|
||||
}
|
||||
return addresses, answer.RCode, nil
|
||||
}
|
||||
|
||||
// probeHolders is D3: every seat that serves verbs has, on every machine that holds it and is heard
|
||||
// from, a holder answering the bus's discovery for that seat. A machine past its heartbeat's bound is
|
||||
// S1's, and is not asked about here.
|
||||
func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
entries, err := d.open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
recorded, err := d.open.inventory.Holdings(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
heard := heardMachines(d)
|
||||
expected := map[string]map[string]bool{} // seat → machine
|
||||
add := func(seat, node string) {
|
||||
if expected[seat] == nil {
|
||||
expected[seat] = map[string]bool{}
|
||||
}
|
||||
expected[seat][node] = true
|
||||
}
|
||||
for _, s := range catalogue.SeatsWithAProtocol() {
|
||||
if len(s.Serves) == 0 || s.Name == catalogue.ControllerSeatName {
|
||||
continue // a seat with no verb has nothing to answer with; this controller is answering now
|
||||
}
|
||||
onRecord := false
|
||||
for _, h := range recorded {
|
||||
if h.Claim == s.Name && heard[h.Node] {
|
||||
add(s.Name, h.Node)
|
||||
onRecord = true
|
||||
}
|
||||
}
|
||||
if onRecord && s.Scope == catalogue.ScopeMesh {
|
||||
continue
|
||||
}
|
||||
for _, e := range entries {
|
||||
for _, c := range e.Manifest.Claims {
|
||||
if c.Name != s.Name {
|
||||
continue
|
||||
}
|
||||
for _, node := range e.On {
|
||||
if heard[node] && (s.Scope == catalogue.ScopeNode || !onRecord) {
|
||||
add(s.Name, node)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(expected) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
answering, err := discoverHolders(ctx, d.js.Conn())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []conditions.Observation
|
||||
for seat, nodes := range expected {
|
||||
for node := range nodes {
|
||||
if answering[seat][node] {
|
||||
continue
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeSeat, ID: seat + "." + node,
|
||||
Token: "silent", Machine: node, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s's holder on %s does not answer the bus: its verbs reach nothing there", seat, node),
|
||||
Said: fmt.Sprintf("no answer for %s from %s to the bus's discovery", seat, node)})
|
||||
}
|
||||
}
|
||||
return sortedFound(out), nil
|
||||
}
|
||||
|
||||
// heardMachines is every machine within its heartbeat's bound, as the watchdogs last saw.
|
||||
func heardMachines(d *doctor) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
if d.watchdogs == nil {
|
||||
return out
|
||||
}
|
||||
f := d.watchdogs.lastFacts()
|
||||
if f == nil {
|
||||
return out
|
||||
}
|
||||
for _, m := range f.machines {
|
||||
if !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) <= heartbeatBound(m.every) && !m.asleep() {
|
||||
out[m.name] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// discoveryPatience is how long the discovery's answers are waited for after the last arrived, and at
|
||||
// the most (ADR 0197: a large runtime's answer arrives last).
|
||||
const (
|
||||
discoveryQuiet = 1500 * time.Millisecond
|
||||
discoveryPatience = 8 * time.Second
|
||||
)
|
||||
|
||||
// discoverHolders asks the bus's discovery who serves what, and answers seat → machine for every
|
||||
// endpoint a seat's verb is served on.
|
||||
func discoverHolders(ctx context.Context, conn *nats.Conn) (map[string]map[string]bool, error) {
|
||||
inbox := conn.NewRespInbox()
|
||||
sub, err := conn.SubscribeSync(inbox)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
if err := conn.PublishRequest("$SRV.INFO", inbox, nil); err != nil {
|
||||
return nil, fmt.Errorf("asking the bus who serves what: %w", err)
|
||||
}
|
||||
out := map[string]map[string]bool{}
|
||||
deadline := time.Now().Add(discoveryPatience)
|
||||
for time.Now().Before(deadline) {
|
||||
wait, cancel := context.WithTimeout(ctx, discoveryQuiet)
|
||||
msg, err := sub.NextMsgWithContext(wait)
|
||||
cancel()
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
break
|
||||
}
|
||||
var info micro.Info
|
||||
if json.Unmarshal(msg.Data, &info) != nil {
|
||||
continue
|
||||
}
|
||||
for _, e := range info.Endpoints {
|
||||
seat, node := e.Metadata["seat"], e.Metadata["node"]
|
||||
if seat == "" {
|
||||
continue
|
||||
}
|
||||
if node == "" {
|
||||
node = info.ID
|
||||
}
|
||||
if out[seat] == nil {
|
||||
out[seat] = map[string]bool{}
|
||||
}
|
||||
out[seat][node] = true
|
||||
}
|
||||
// A holder that announces itself as its seat, by name and machine.
|
||||
if out[info.Name] == nil {
|
||||
out[info.Name] = map[string]bool{}
|
||||
}
|
||||
out[info.Name][info.ID] = true
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// probeArchives is D4: every archive the mesh keeps is held by its manifest in the artifact store.
|
||||
func probeArchives(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
inv := d.open.inventory
|
||||
kept, err := inv.KeptArchives(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(kept) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
store, err := artifactStoreAddress(ctx, inv, shelf, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if store == "" {
|
||||
return nil, errors.New("the mesh keeps archives and has no artifact store on its network to ask")
|
||||
}
|
||||
var report collectionReport
|
||||
if unasked, stopped := askHeld(ctx, artifacts.Store{Address: store}, kept, &report); stopped != "" {
|
||||
return nil, fmt.Errorf("%d kept archive(s) could not be asked about: %s", unasked, stopped)
|
||||
}
|
||||
var out []conditions.Observation
|
||||
if n := len(report.Unheld); n > 0 {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMesh, ID: "artifact-store", Token: "archives-unheld",
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%d of %d kept archive(s) are not held by a manifest: the store's collector would "+
|
||||
"delete them", n, len(kept)),
|
||||
Said: "first: " + report.Unheld[0]})
|
||||
}
|
||||
if n := len(report.Missing); n > 0 {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMesh, ID: "artifact-store", Token: "archives-missing",
|
||||
Kind: "archives-missing", Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%d kept archive(s) are not in the artifact store at all", n),
|
||||
Said: "first: " + report.Missing[0]})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// consumerFarBehind is how far a durable consumer may be from its stream's head (D6).
|
||||
const consumerFarBehind = 1000
|
||||
|
||||
// probeConsumers is D6: every durable consumer the mesh expects exists, as the controller defines it,
|
||||
// and is near its stream's head.
|
||||
func probeConsumers(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
_, consumers, err := expectedBusObjects(ctx, d.open.inventory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js := d.js.Context()
|
||||
var out []conditions.Observation
|
||||
for _, c := range consumers {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
info, err := js.ConsumerInfo(c.Stream, c.Name, nats.Context(ctx))
|
||||
who := c.Stream + "." + c.Name
|
||||
switch {
|
||||
case errors.Is(err, nats.ErrConsumerNotFound) || errors.Is(err, nats.ErrStreamNotFound):
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "missing",
|
||||
Kind: "consumer-lost", Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s is not on the bus: what it delivers reaches nobody", consumerWords(c)),
|
||||
Said: "no consumer " + c.Name + " on " + c.Stream})
|
||||
continue
|
||||
case err != nil:
|
||||
return nil, fmt.Errorf("%s cannot be read: %w", consumerWords(c), err)
|
||||
}
|
||||
if differs := consumerDiffers(c, info.Config); differs != "" {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "redefined",
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s is not as the controller defines it: %s", consumerWords(c), differs),
|
||||
Said: differs})
|
||||
}
|
||||
if c.Stream != "NODES" && info.NumPending > consumerFarBehind {
|
||||
// Its own kind (novox/hq to-be 45 §7): healer H4 answers a consumer far behind, and only
|
||||
// this — a redefined consumer is not one a reset repairs.
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "behind",
|
||||
Kind: kindConsumerBehind, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s is %d message(s) behind its stream's head", consumerWords(c), info.NumPending),
|
||||
Said: fmt.Sprintf("%d pending, %d handed out and not settled", info.NumPending, info.NumAckPending)})
|
||||
}
|
||||
}
|
||||
return sortedFound(out), nil
|
||||
}
|
||||
|
||||
// consumerWords is a consumer as the mesh says it, with its name.
|
||||
func consumerWords(c broker.Consumer) string {
|
||||
return fmt.Sprintf("%s (%s on %s)", link.ConsumerInWords(c.Stream, c.Name), c.Name, c.Stream)
|
||||
}
|
||||
|
||||
// consumerDiffers is what about a consumer on the bus is not as defined; empty when nothing is. The
|
||||
// delivery subject is not compared: one kept as it was while a holder is bound is the assertion's
|
||||
// stated choice (novox/hq issue 156).
|
||||
func consumerDiffers(want broker.Consumer, have nats.ConsumerConfig) string {
|
||||
var differs []string
|
||||
haveFilters := append([]string(nil), have.FilterSubjects...)
|
||||
if have.FilterSubject != "" {
|
||||
haveFilters = append(haveFilters, have.FilterSubject)
|
||||
}
|
||||
wantFilters := append([]string(nil), want.Filters...)
|
||||
sort.Strings(haveFilters)
|
||||
sort.Strings(wantFilters)
|
||||
if !slices.Equal(haveFilters, wantFilters) {
|
||||
differs = append(differs, fmt.Sprintf("filters %v, defined %v", haveFilters, wantFilters))
|
||||
}
|
||||
if have.AckPolicy != nats.AckExplicitPolicy {
|
||||
differs = append(differs, "acknowledges "+have.AckPolicy.String()+", defined explicit")
|
||||
}
|
||||
if wantMax := want.MaxDeliver; wantMax != 0 && have.MaxDeliver != wantMax {
|
||||
differs = append(differs, fmt.Sprintf("hands a message over %d times, defined %d", have.MaxDeliver, wantMax))
|
||||
}
|
||||
if wantWait := time.Duration(want.AckWaitSeconds) * time.Second; wantWait != 0 && have.AckWait != wantWait {
|
||||
differs = append(differs, fmt.Sprintf("waits %s for an acknowledgement, defined %s", have.AckWait, wantWait))
|
||||
}
|
||||
if want.MaxAckPending != 0 && have.MaxAckPending != want.MaxAckPending {
|
||||
differs = append(differs, fmt.Sprintf("hands out %d at once, defined %d", have.MaxAckPending, want.MaxAckPending))
|
||||
}
|
||||
if wantPush, havePush := want.Push || want.Queue != "", have.DeliverSubject != ""; wantPush != havePush {
|
||||
differs = append(differs, "delivers by the other shape (push or pull) than defined")
|
||||
}
|
||||
return strings.Join(differs, "; ")
|
||||
}
|
||||
|
||||
// probeStreams is D7: every stream the controller defines exists as defined, and its own buckets.
|
||||
func probeStreams(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
streams, _, err := expectedBusObjects(ctx, d.open.inventory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js := d.js.Context()
|
||||
var out []conditions.Observation
|
||||
for _, s := range streams {
|
||||
info, err := js.StreamInfo(s.Name, nats.Context(ctx))
|
||||
switch {
|
||||
case errors.Is(err, nats.ErrStreamNotFound):
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: s.Name, Token: "stream-missing",
|
||||
Kind: "stream-wrong", Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("the stream %s is not on the bus: %s", s.Name, firstLine(s.Why)),
|
||||
Said: "no stream " + s.Name})
|
||||
continue
|
||||
case err != nil:
|
||||
return nil, fmt.Errorf("the stream %s cannot be read: %w", s.Name, err)
|
||||
}
|
||||
if differs := streamDiffers(s, info.Config); differs != "" {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: s.Name, Token: "stream-redefined",
|
||||
Kind: "stream-wrong", Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the stream %s is not as the controller defines it: %s", s.Name, differs),
|
||||
Said: differs})
|
||||
}
|
||||
}
|
||||
for _, bucket := range broker.ControllerBuckets() {
|
||||
if _, err := js.StreamInfo("KV_"+bucket, nats.Context(ctx)); errors.Is(err, nats.ErrStreamNotFound) {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: bucket, Token: "bucket-missing",
|
||||
Kind: "stream-wrong", Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("the controller's bucket %s is not on the bus: what it keeps there is not kept", bucket),
|
||||
Said: "no bucket " + bucket})
|
||||
} else if err != nil {
|
||||
return nil, fmt.Errorf("the bucket %s cannot be read: %w", bucket, err)
|
||||
}
|
||||
}
|
||||
return sortedFound(out), nil
|
||||
}
|
||||
|
||||
// streamDiffers is what about a stream on the bus is not as defined; empty when nothing is.
|
||||
func streamDiffers(want broker.Stream, have nats.StreamConfig) string {
|
||||
var differs []string
|
||||
haveSubjects := append([]string(nil), have.Subjects...)
|
||||
wantSubjects := append([]string(nil), want.Subjects...)
|
||||
sort.Strings(haveSubjects)
|
||||
sort.Strings(wantSubjects)
|
||||
if !slices.Equal(haveSubjects, wantSubjects) {
|
||||
differs = append(differs, fmt.Sprintf("subjects %v, defined %v", haveSubjects, wantSubjects))
|
||||
}
|
||||
retention, perSubject := nats.LimitsPolicy, int64(want.MaxMsgsPerSubject)
|
||||
switch want.Retention {
|
||||
case broker.RetentionWorkQueue:
|
||||
retention = nats.WorkQueuePolicy
|
||||
case broker.RetentionLastPerSubject:
|
||||
perSubject = 1
|
||||
}
|
||||
if have.Retention != retention {
|
||||
differs = append(differs, fmt.Sprintf("keeps by %s, defined %s", have.Retention, retention))
|
||||
}
|
||||
if perSubject != 0 && have.MaxMsgsPerSubject != perSubject {
|
||||
differs = append(differs, fmt.Sprintf("keeps %d per subject, defined %d", have.MaxMsgsPerSubject, perSubject))
|
||||
}
|
||||
return strings.Join(differs, "; ")
|
||||
}
|
||||
|
||||
// ipLike finds the addresses in a ban list, whatever shape its holder answers in.
|
||||
var ipLike = regexp.MustCompile(`\b(?:\d{1,3}\.){3}\d{1,3}\b`)
|
||||
|
||||
// probeBans is D8: no address the mesh owns is in any machine's ban list. The mesh's own addresses are
|
||||
// every machine's private address and the address its endpoint names; the ban lists are asked of each
|
||||
// machine's holder of `node-intrusion-prevention` that is heard from.
|
||||
func probeBans(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
inv := d.open.inventory
|
||||
places, err := inv.Overlays(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
owned := map[string]string{} // address → whose
|
||||
for _, p := range places {
|
||||
if p.Address != "" {
|
||||
owned[p.Address] = p.Name + "'s private address"
|
||||
}
|
||||
if host, _, err := net.SplitHostPort(p.Endpoint); err == nil && host != "" {
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
owned[ip.String()] = p.Name + "'s endpoint"
|
||||
} else if addrs, err := net.DefaultResolver.LookupHost(ctx, host); err == nil {
|
||||
for _, a := range addrs {
|
||||
owned[a] = p.Name + "'s endpoint (" + host + ")"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
heard := heardMachines(d)
|
||||
var holders []string
|
||||
for _, e := range entries {
|
||||
for _, c := range e.Manifest.Claims {
|
||||
if c.Name == "node-intrusion-prevention" {
|
||||
for _, node := range e.On {
|
||||
if heard[node] && !slices.Contains(holders, node) {
|
||||
holders = append(holders, node)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(holders)
|
||||
var out []conditions.Observation
|
||||
// Every machine asked at once: one after another, four holders that each wait their bound
|
||||
// outlast the probe's thirty seconds, and the probe says nothing about any of them.
|
||||
answers := make([]json.RawMessage, len(holders))
|
||||
errs := make([]error, len(holders))
|
||||
var wg sync.WaitGroup
|
||||
for i, node := range holders {
|
||||
wg.Add(1)
|
||||
go func(i int, node string) {
|
||||
defer wg.Done()
|
||||
answers[i], errs[i] = askSeatTool(ctx, d.js.Conn(), "node-intrusion-prevention", "banned", node)
|
||||
}(i, node)
|
||||
}
|
||||
wg.Wait()
|
||||
var unasked []string
|
||||
for i, node := range holders {
|
||||
answer, err := answers[i], errs[i]
|
||||
if err != nil {
|
||||
unasked = append(unasked, err.Error())
|
||||
continue
|
||||
}
|
||||
var banned []string
|
||||
for _, ip := range ipLike.FindAllString(string(answer), -1) {
|
||||
if whose, ours := owned[ip]; ours && !slices.Contains(banned, ip+" ("+whose+")") {
|
||||
banned = append(banned, ip+" ("+whose+")")
|
||||
}
|
||||
}
|
||||
if len(banned) > 0 {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Token: "bans-the-mesh",
|
||||
Machine: node, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("%s's ban list holds %d of the mesh's own address(es): %s — the mesh is locked out "+
|
||||
"of itself there (ADR 0186)", node, len(banned), strings.Join(banned, ", ")),
|
||||
Said: strings.Join(banned, ", ")})
|
||||
}
|
||||
}
|
||||
if len(unasked) > 0 {
|
||||
return nil, fmt.Errorf("a ban list could not be read: %s", strings.Join(unasked, "; "))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// probeStatus is D9: `status` answers in full within ten seconds, from a summary composed lately.
|
||||
func probeStatus(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
started := time.Now()
|
||||
stale := ""
|
||||
if statusFrom != nil {
|
||||
answer, err := statusFrom.answer(ctx)
|
||||
if err != nil {
|
||||
stale = err.Error()
|
||||
} else if m, ok := answer.(map[string]any); ok {
|
||||
if failed, _ := m["lastAttemptFailed"].(string); failed != "" {
|
||||
stale = "its last composition failed: " + failed
|
||||
}
|
||||
if composed, err := time.Parse(time.RFC3339, fmt.Sprint(m["composed"])); err == nil &&
|
||||
time.Since(composed) > 3*statusEvery+statusComposeWithin {
|
||||
stale = fmt.Sprintf("it answers a summary composed %s ago", time.Since(composed).Round(time.Second))
|
||||
}
|
||||
}
|
||||
} else if _, err := theThreeQuestions(ctx, d.open); err != nil {
|
||||
stale = err.Error()
|
||||
}
|
||||
took := time.Since(started)
|
||||
var out []conditions.Observation
|
||||
if took > 10*time.Second || stale != "" {
|
||||
why := stale
|
||||
if why == "" {
|
||||
why = fmt.Sprintf("it took %s", took.Round(time.Millisecond))
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: "controller", Token: "status-slow",
|
||||
Machine: d.host, Severity: conditions.Warning,
|
||||
Summary: "status does not answer in full within ten seconds: " + why, Said: why})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// probeCoreBuilds is D10: every machine runs the node-engine and the node tools the mesh holds, or a
|
||||
// plan is rolling one of them out. The node-engine says its build in each report; the node tools' is
|
||||
// the build the machine was last sent, once it reported applying that send.
|
||||
func probeCoreBuilds(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
inv := d.open.inventory
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hostVersions := deliveredVersions(shelf[hostModule])
|
||||
rolling := map[string]bool{}
|
||||
for _, p := range plans {
|
||||
for m := range p.Modules {
|
||||
rolling[m] = true
|
||||
}
|
||||
}
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
applied := map[string]bool{}
|
||||
for _, r := range reports {
|
||||
applied[r.Node] = r.Current
|
||||
}
|
||||
heard := heardMachines(d)
|
||||
var out []conditions.Observation
|
||||
for _, n := range nodes {
|
||||
if !heard[n.Name] {
|
||||
continue // a machine not heard from is S1's
|
||||
}
|
||||
var behind []string
|
||||
// **A node-engine says its version as the directory it was delivered into** — its archive's
|
||||
// digest, twelve characters (catalogue.versionOf, ADR 0141) — not the commit it was built
|
||||
// from. Compared as a commit, every machine read as behind right after a push sent it the
|
||||
// current one (2026-10-06). An engine placed by hand reports its link-time stamp instead,
|
||||
// which a commit can match.
|
||||
if !rolling[hostModule] && engineBehind(n.HostVersion, hostVersions, current[hostModule].Commit) {
|
||||
behind = append(behind, fmt.Sprintf("the node-engine %s, the mesh holds %s (built from %s)",
|
||||
n.HostVersion, strings.Join(hostVersions, " or "), short(current[hostModule].Commit)))
|
||||
}
|
||||
assigned, err := inv.Assigned(ctx, n.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tools := current[broker.RuntimeModule].Commit
|
||||
if tools != "" && slices.Contains(assigned, broker.RuntimeModule) && !rolling[broker.RuntimeModule] {
|
||||
sent, known, err := inv.SentBuilds(ctx, n.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch {
|
||||
case !known:
|
||||
case !sameCommit(sent[broker.RuntimeModule], tools):
|
||||
behind = append(behind, fmt.Sprintf("the node tools %s, the mesh holds %s",
|
||||
short(orNotKnown(sent[broker.RuntimeModule])), short(tools)))
|
||||
case !applied[n.Name]:
|
||||
behind = append(behind, "the node tools it was last sent, not yet reported applied")
|
||||
}
|
||||
}
|
||||
if len(behind) > 0 {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "core-behind",
|
||||
Machine: n.Name, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s runs %s, and no plan is rolling them out", n.Name, strings.Join(behind, "; ")),
|
||||
Said: strings.Join(behind, "; ")})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// deliveredVersions are the versions a module's registered build is delivered as: the last element
|
||||
// of every resource path under a `versions/` directory, which registration filled from the artifact's
|
||||
// digest (catalogue `${version}`). The node-engine names itself by that directory.
|
||||
func deliveredVersions(m catalogue.Manifest) []string {
|
||||
var out []string
|
||||
for _, r := range m.Resources {
|
||||
path, _ := r["path"].(string)
|
||||
before, version, found := strings.Cut(path, "/versions/")
|
||||
if !found || before == "" || version == "" || strings.Contains(version, "/") || strings.Contains(version, "$") {
|
||||
continue
|
||||
}
|
||||
if !slices.Contains(out, version) {
|
||||
out = append(out, version)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// engineBehind says a node-engine's reported version is not the build the mesh holds: neither the
|
||||
// directory that build is delivered as, nor (for an engine placed by hand) its commit. A machine that
|
||||
// has not said, or a mesh that holds no delivered build, is not behind anything.
|
||||
func engineBehind(reported string, delivered []string, commit string) bool {
|
||||
if reported == "" || len(delivered) == 0 {
|
||||
return false
|
||||
}
|
||||
return !slices.Contains(delivered, reported) && !sameCommit(reported, commit)
|
||||
}
|
||||
|
||||
// hostModule is the node-engine's module.
|
||||
const hostModule = "mesh-host"
|
||||
|
||||
// sameCommit says two commits are one, either written short.
|
||||
func sameCommit(a, b string) bool {
|
||||
if a == "" || b == "" {
|
||||
return false
|
||||
}
|
||||
return strings.HasPrefix(a, b) || strings.HasPrefix(b, a)
|
||||
}
|
||||
|
||||
func orNotKnown(s string) string {
|
||||
if s == "" {
|
||||
return "(not known)"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// probeWatchdogs is DW: the watchdogs ran within three of their intervals. The doctor and the
|
||||
// watchdogs watch each other: S10 is the other half.
|
||||
func probeWatchdogs(_ context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
if d.watchdogs == nil {
|
||||
return nil, nil // a self-check run outside the serving controller has none to watch
|
||||
}
|
||||
ticked := d.watchdogs.lastTick()
|
||||
since := ticked
|
||||
if since.IsZero() {
|
||||
since = d.watchdogs.started
|
||||
}
|
||||
if time.Since(since) <= 3*watchEvery {
|
||||
return nil, nil
|
||||
}
|
||||
return []conditions.Observation{{Scope: conditions.ScopeCore, ID: "watchdogs", Token: "silent",
|
||||
Machine: d.host, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("the watchdogs of the signals table have not run since %s: no late signal is being said",
|
||||
since.UTC().Format("2006-01-02 15:04 MST")),
|
||||
Said: fmt.Sprintf("no tick for %s", time.Since(since).Round(time.Second))}}, nil
|
||||
}
|
||||
|
||||
// askSeatTool asks one machine's holder of a node seat a verb and answers its result; the holder's
|
||||
// own refusal is an error.
|
||||
func askSeatTool(ctx context.Context, conn *nats.Conn, seat, verb, node string) (json.RawMessage, error) {
|
||||
if who, declared := declaredBy(ctx, seat, verb); !declared {
|
||||
return nil, fmt.Errorf("%s asks %s.%s, which it does not declare in the probe registry — and so the "+
|
||||
"controller is not granted it", who, seat, verb)
|
||||
}
|
||||
answer, err := link.AskSeatTool(ctx, conn, seat, verb, node, map[string]any{}, 10*time.Second)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return nil, fmt.Errorf("%s's %s.%s refused: %s", node, seat, verb, answer.Error)
|
||||
}
|
||||
return answer.Result, nil
|
||||
}
|
||||
|
||||
// oneLine is a message of several lines said on one, its runs of space made one.
|
||||
func oneLine(s string) string { return strings.Join(strings.Fields(s), " ") }
|
||||
|
||||
// probeLease is D5 (novox/hq to-be 45 §4, §6): exactly one lease holder — the key on the bus names this
|
||||
// controller at the epoch it acts under, and the mesh's record has that epoch and no other open — and no
|
||||
// message from a stale epoch refused in the last interval.
|
||||
func probeLease(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
if d.js == nil {
|
||||
return nil, errors.New("this controller is not on the bus")
|
||||
}
|
||||
st := theLease.standing()
|
||||
var out []conditions.Observation
|
||||
split := func(token, summary, said string) {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: "controller.lease", Token: token,
|
||||
Machine: d.host, Severity: conditions.Urgent, Summary: summary, Said: said})
|
||||
}
|
||||
if st.Unleased != "" {
|
||||
split("unheld", "this controller acts without the lease, so nothing keeps another from acting beside it: "+
|
||||
st.Unleased, st.Unleased)
|
||||
return out, nil
|
||||
}
|
||||
api, err := jetstream.New(d.js.Conn())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, broker.LeaseBucket)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the lease bucket cannot be read: %w", err)
|
||||
}
|
||||
holder, found, err := lease.Current(ctx, kv)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the lease cannot be read: %w", err)
|
||||
}
|
||||
switch {
|
||||
case !found:
|
||||
split("split", fmt.Sprintf("nobody holds the lease on the bus, and this controller acts as epoch %d", st.Epoch),
|
||||
"the lease's key is absent")
|
||||
case holder.Instance != instance || holder.Epoch != st.Epoch:
|
||||
split("split", fmt.Sprintf("the lease on the bus names %s at epoch %d, and this controller (%s) acts as "+
|
||||
"epoch %d: two controllers believe they may act", holder.Instance, holder.Epoch, instance, st.Epoch),
|
||||
fmt.Sprintf("held by %s, epoch %d", holder.Instance, holder.Epoch))
|
||||
}
|
||||
// The record: one epoch open, this one.
|
||||
epochs, err := d.open.inventory.EpochsSince(ctx, time.Now().Add(-time.Hour))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the epochs the mesh issued cannot be read: %w", err)
|
||||
}
|
||||
var open []string
|
||||
for _, e := range epochs {
|
||||
if e.Ended == nil && e.Epoch != st.Epoch {
|
||||
open = append(open, fmt.Sprintf("epoch %d (%s)", e.Epoch, e.Instance))
|
||||
}
|
||||
}
|
||||
if len(open) > 0 {
|
||||
split("open-epochs", fmt.Sprintf("the mesh's record holds %s open beside this controller's epoch %d: a "+
|
||||
"holder that neither gave the lease back nor was found expired", strings.Join(open, ", "), st.Epoch),
|
||||
strings.Join(open, ", "))
|
||||
}
|
||||
// And no message from a stale epoch in the last interval.
|
||||
for _, w := range link.StaleRefusals.Within(time.Now().Add(-doctorEvery)) {
|
||||
if w.Epoch <= 0 || uint64(w.Epoch) >= st.Epoch {
|
||||
continue
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: fmt.Sprintf("controller.epoch-%d", w.Epoch),
|
||||
Token: "stale-epoch", Machine: firstOf(w.Receivers), Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("%d declaration(s) from epoch %d — older than this controller's %d — reached %s in the "+
|
||||
"last %s and were refused: a controller that lost the lease is still sending", w.Count, w.Epoch, st.Epoch,
|
||||
strings.Join(w.Receivers, ", "), doctorEvery),
|
||||
Said: fmt.Sprintf("%d refused, the last at %s", w.Count, w.Last.UTC().Format(time.RFC3339))})
|
||||
}
|
||||
return sortedFound(out), nil
|
||||
}
|
||||
+58
-174
@@ -8,7 +8,6 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
@@ -63,7 +62,7 @@ func connectLink(ctx context.Context, inv *inventory.Inventory, enroller link.En
|
||||
return link.ConnectNats(js, enroller, listener), nil
|
||||
}
|
||||
|
||||
func serve(ctx context.Context) (err error) {
|
||||
func serve(ctx context.Context) error {
|
||||
// The one process whose log is read over time, so the one that says each change to a node's
|
||||
// unmet seat dependencies once (novox/hq ADR 0207).
|
||||
logUnheldChanges = true
|
||||
@@ -104,47 +103,9 @@ func serve(ctx context.Context) (err error) {
|
||||
// being live is refused, because a mesh half on each is one where a declaration goes out on one
|
||||
// and the report comes back on the other, and every component logs success while it happens.
|
||||
|
||||
// **The lease, before anything that acts** (novox/hq to-be 45 §6): asserting the bus's objects is the
|
||||
// controller's to do, and so is everything after. A controller starting while another holds it waits
|
||||
// here, said; one that loses it stops: every act's gate closes at once, and ctx ends so the process
|
||||
// exits and is started again as a candidate.
|
||||
busAddress, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lost, err := theLease.serveUnderTheLease(ctx, inv, busAddress)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Given back before the store closes, so the epoch is recorded as given back rather than found
|
||||
// expired by the next holder.
|
||||
defer theLease.release()
|
||||
ctx, stopActing := context.WithCancel(ctx)
|
||||
defer stopActing()
|
||||
go func() {
|
||||
select {
|
||||
case <-lost:
|
||||
stopActing()
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}()
|
||||
defer func() {
|
||||
select {
|
||||
case <-lost:
|
||||
if err == nil {
|
||||
err = errors.New("the controller lease was lost; this controller stopped acting and exits, to " +
|
||||
"be started again as a candidate")
|
||||
}
|
||||
default:
|
||||
}
|
||||
}()
|
||||
|
||||
work := link.Enrolment{Inventory: inv, Identity: ident, Broker: known,
|
||||
OnNATS: true}
|
||||
// `status` from a summary kept current here (novox/hq to-be 45 Phase 0): a machine saying
|
||||
// something new is one thing that moves it, so the listener nudges it.
|
||||
statusFrom = newStatusSummary(composeStatus(open))
|
||||
server, err := connectLink(ctx, inv, work, nudgingListener{Enrolment: work, summary: statusFrom, open: open})
|
||||
server, err := connectLink(ctx, inv, work, work)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -160,8 +121,6 @@ func serve(ctx context.Context) (err error) {
|
||||
// Open plans move on a timer as well as on outcomes (novox/hq ADR 0162): a tier waiting for
|
||||
// machines to report moves when they have, and a plan left by a replaced controller resumes.
|
||||
go planTicker(ctx, open)
|
||||
// The durations the core's bounds are set from are kept a month (novox/hq to-be 45 Phase 0).
|
||||
go forgettingOldDurations(ctx, inv)
|
||||
// And what the catalogue decided a build meant. The builder's own result is already handled
|
||||
// above; this is the other half — the control plane is the only one of the three that knows
|
||||
// which machines run the thing, so it is the one that acts (novox/hq ADR 0072).
|
||||
@@ -178,13 +137,7 @@ func serve(ctx context.Context) (err error) {
|
||||
}
|
||||
// And what providers say about consumers they keep failing, kept for `status` (novox/hq ADR
|
||||
// 0224): a provider's journal must not be the only place that says so.
|
||||
if err := server.Watches(standings{keeper: func() *conditions.Keeper { return conditionsFrom }}); err != nil {
|
||||
return err
|
||||
}
|
||||
// And what they say about consumers the mesh stopped asking for: one waiting for a person is an
|
||||
// urgent condition, and an act asked of a provider some other way is recorded by hand (ADR 0230).
|
||||
if err := server.KeepsRetirements(retirements{keeper: func() *conditions.Keeper { return conditionsFrom },
|
||||
record: recordHandActOnTheBus}); err != nil {
|
||||
if err := server.Watches(standings{inv}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -205,33 +158,8 @@ func serve(ctx context.Context) (err error) {
|
||||
if !isNATS {
|
||||
return errors.New("the mesh's verbs are served over the bus, and this control plane is not on it")
|
||||
}
|
||||
// The hand-act log is counted for `status` on this connection rather than a new one a minute.
|
||||
handActConn = bus.Conn
|
||||
// And says when it replaced a value given by hand (novox/hq ADR 0228).
|
||||
givenEvents = bus
|
||||
// Composed now and kept current, before the verb that answers from it is served.
|
||||
go statusFrom.keep(ctx)
|
||||
// Every call carries the lease's epoch, and its record is written only under the lease (novox/hq
|
||||
// to-be 45 §6).
|
||||
link.Calls.UnderLease(func() (uint64, error) { return theLease.epoch(ctx) })
|
||||
// A call that outlasts its caller's patience is followed by `calls` (novox/hq issue 265).
|
||||
link.Calls.Follow = catalogue.ControllerSeatName + ".calls"
|
||||
// And every call is kept on the bus, so a restart of this process keeps what came of each
|
||||
// (novox/hq to-be 45 §6). A bus without the bucket is said and served from memory, as before:
|
||||
// answering no calls at all would be worse than answering them without the record.
|
||||
said := log.New(os.Stdout, "", log.LstdFlags)
|
||||
if keeper, err := link.CallsOnTheBus(ctx, bus.Conn); err != nil {
|
||||
fmt.Printf("calls are kept in memory only, and lost when this controller stops: %v\n", err)
|
||||
} else if err := link.Calls.Durably(ctx, keeper, instance, said); err != nil {
|
||||
fmt.Printf("calls are kept on the bus from now on; the ones kept before could not be read: %v\n", err)
|
||||
}
|
||||
// What is wrong, kept and said (novox/hq to-be 45 §2): the condition store, the watchdogs of the
|
||||
// signals table, what the bus says about itself, and the self-check. A store that cannot be opened
|
||||
// is said and the controller serves on: status then says the conditions cannot be read, and is
|
||||
// not well — louder than not serving at all, and the push that repairs the bus still runs.
|
||||
if stopWatching := watchTheMesh(ctx, open, server, bus); stopWatching != nil {
|
||||
defer stopWatching()
|
||||
}
|
||||
stopServing, err := bus.ServeSeatTools(catalogue.ControllerSeatName, handlers, log.New(os.Stdout, "", log.LstdFlags))
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -301,12 +229,7 @@ func declare(ctx context.Context, args []string) error {
|
||||
// is still what the machine was last told, and status must not read it as current for the one
|
||||
// the mesh would compose. Which builds it carried is recorded as not known (novox/hq issue 259):
|
||||
// the mesh did not compose it, so a push that does not name this machine treats it as held.
|
||||
// The epoch it carried, if a person wrote one in, is what the machine heard.
|
||||
var carried struct {
|
||||
Epoch uint64 `json:"epoch"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &carried)
|
||||
if _, err := recordSent(ctx, inv, node, raw, nil, carried.Epoch); err != nil {
|
||||
if _, err := recordSent(ctx, inv, node, raw, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw))
|
||||
@@ -337,9 +260,6 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
// (novox/hq ADR 0010). 0 waits for nothing, which is the old fire-and-forget.
|
||||
wait := set.Duration("wait", 0,
|
||||
"for a named node, how long to wait for it to report applying what it was sent (0: do not wait)")
|
||||
// A push by hand is a repair, and says why (novox/hq to-be 45 §7): required through the seat,
|
||||
// recorded when given at a shell — see handacts.go for why a shell is not refused.
|
||||
why := addHandActFlags(set)
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -354,11 +274,6 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
return errors.New("push <node> or push --behind, not both: one names a machine and the " +
|
||||
"other asks which machines need one")
|
||||
}
|
||||
recorded := append([]string(nil), args...)
|
||||
if *behind {
|
||||
recorded = append(recorded, "--behind")
|
||||
}
|
||||
why.record(ctx, "push", recorded)
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -635,7 +550,7 @@ type readyNode struct {
|
||||
//
|
||||
// The all-or-nothing rule is kept where it means something — sendTo, which rotates a credential
|
||||
// across two machines that must agree — and dropped here, where it never did.
|
||||
func composeEach(names []string, allot func(node string) (order, error),
|
||||
func composeEach(names []string, allot func(node string) (int64, error),
|
||||
compose func(node string) (sendable, error)) ([]readyNode, []string) {
|
||||
|
||||
var sending []readyNode
|
||||
@@ -649,7 +564,7 @@ func composeEach(names []string, allot func(node string) (order, error),
|
||||
// took the older content as the newer word: on 2026-10-02 a runtime assigned and applied on
|
||||
// two machines was undone two seconds later by exactly that. Taken here, before the first
|
||||
// read, what was composed earlier is numbered lower whatever order the sends happen in.
|
||||
numbered, err := allot(name)
|
||||
seq, err := allot(name)
|
||||
if err != nil {
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
@@ -659,7 +574,7 @@ func composeEach(names []string, allot func(node string) (order, error),
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
}
|
||||
declared.Sequence, declared.Epoch = numbered.sequence, numbered.epoch
|
||||
declared.Sequence = seq
|
||||
if len(declared.Resources) == 0 {
|
||||
// Sent, not skipped (novox/hq issue 127). A node whose declaration composes to
|
||||
// nothing may have HELD something before — the broker opening a placement gave it,
|
||||
@@ -827,13 +742,6 @@ type overTheBus struct {
|
||||
}
|
||||
|
||||
func (b overTheBus) grant(ctx context.Context, sending []readyNode) error {
|
||||
// The bus's objects first, which every declaration implies (novox/hq issue 208): said and raised
|
||||
// when they cannot be, and never what holds the send back (assertOnSend).
|
||||
if bus, ok := b.server.Bus().(link.OverNATS); ok {
|
||||
js := broker.OnConn(bus.Conn)
|
||||
js.Note = func(format string, args ...any) { fmt.Printf(b.indent+" "+format+"\n", args...) }
|
||||
_ = assertOnSend(ctx, b.open.inventory, js, b.indent)
|
||||
}
|
||||
return issueMemberships(ctx, b.open, b.server, sending)
|
||||
}
|
||||
|
||||
@@ -843,7 +751,7 @@ func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (stri
|
||||
}
|
||||
// After it is away, not before. A digest recorded for something that failed to send would make
|
||||
// the machine look current for a declaration it never received.
|
||||
digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds, s.declared.Epoch)
|
||||
digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -996,7 +904,7 @@ func sendToEach(ctx context.Context, open *stores, names []string) ([]string, er
|
||||
var refusals []string
|
||||
for _, name := range names {
|
||||
// Numbered before composing, for the reason composeEach gives (novox/hq issue 204).
|
||||
numbered, err := allot(ctx, inv, name)
|
||||
seq, err := allot(ctx, inv, name)
|
||||
if err != nil {
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
@@ -1012,7 +920,7 @@ func sendToEach(ctx context.Context, open *stores, names []string) ([]string, er
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
}
|
||||
declared.Sequence, declared.Epoch = numbered.sequence, numbered.epoch
|
||||
declared.Sequence = seq
|
||||
reportLeftOut(name, declared)
|
||||
sending = append(sending, readyNode{name, declared})
|
||||
}
|
||||
@@ -1131,20 +1039,6 @@ func digestOf(body []byte) string {
|
||||
// be worked out" is a different problem with a different remedy, and `plan` is where it is said.
|
||||
func wouldSend(ctx context.Context, open *stores,
|
||||
nodes []inventory.Node) (map[string]string, error) {
|
||||
return wouldSendFrom(ctx, open, nodes, nil)
|
||||
}
|
||||
|
||||
// planned is one machine's plan as planFor answered it, for a caller that already asked.
|
||||
type planned struct {
|
||||
plan catalogue.Resolution
|
||||
settings catalogue.SettingsBy
|
||||
}
|
||||
|
||||
// wouldSendFrom is wouldSend reusing the plans a caller worked out a moment before: resolving a
|
||||
// machine is most of what `status` costs, and it used to resolve every machine twice (novox/hq
|
||||
// to-be 45 Phase 0). A machine absent from plans is worked out here.
|
||||
func wouldSendFrom(ctx context.Context, open *stores,
|
||||
nodes []inventory.Node, plans map[string]planned) (map[string]string, error) {
|
||||
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
@@ -1152,12 +1046,9 @@ func wouldSendFrom(ctx context.Context, open *stores,
|
||||
}
|
||||
out := map[string]string{}
|
||||
for _, n := range nodes {
|
||||
known, have := plans[n.Name]
|
||||
plan, settings := known.plan, known.settings
|
||||
if !have {
|
||||
if plan, settings, err = planFor(ctx, open, n.Name); err != nil {
|
||||
continue
|
||||
}
|
||||
plan, settings, err := planFor(ctx, open, n.Name)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
declared, err := declarationWith(ctx, open, n.Name, plan, settings, gens, Reading)
|
||||
if err != nil {
|
||||
@@ -1169,11 +1060,6 @@ func wouldSendFrom(ctx context.Context, open *stores,
|
||||
if declared.Sequence, err = open.inventory.Sequence(ctx, n.ID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// And the epoch it was last sent under, for the same reason: a new holder of the lease is not a
|
||||
// change of the machine (novox/hq to-be 45 §6).
|
||||
if declared.Epoch, err = open.inventory.SentEpoch(ctx, n.ID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1221,22 +1107,35 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string)
|
||||
}
|
||||
}
|
||||
|
||||
// The mesh's own streams and consumers, the seats' work queues and their workers, and how every
|
||||
// machine hears its declaration — one derivation, which the self-check reads as well (D6, D7).
|
||||
names, err := assertBusObjects(ctx, inv, js)
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
names := make([]string, 0, len(nodes))
|
||||
for _, n := range nodes {
|
||||
names = append(names, n.Name)
|
||||
}
|
||||
if err := broker.Raise(js, names); err != nil {
|
||||
return err
|
||||
}
|
||||
// The work queues of the mesh's own roles (novox/hq ADR 0121). The queue before the holder,
|
||||
// deliberately: work queues until somebody arrives to do it, so assigning a build machine a week
|
||||
// after something started asking for builds flushes the backlog instead of having lost it.
|
||||
// With the seats' holders, so each role's work queue gets the consumer its holder takes
|
||||
// work from. Passed as nil until the first live raise, which left the build machine bound to a
|
||||
// consumer nothing had created (2026-09-28).
|
||||
holders, err := seatHolders(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := broker.RaiseSeats(js, inventory.MeshSeats(), holders); err != nil {
|
||||
return err
|
||||
}
|
||||
// And each work queue's cancelled set (novox/hq ADR 0219), so a holder taking an ask can ask
|
||||
// whether it was cancelled the moment it took it.
|
||||
if err := broker.RaiseCancelledSets(js, inventory.MeshSeats()); err != nil {
|
||||
return err
|
||||
}
|
||||
// And the controller's own buckets (novox/hq to-be 45 §1): the calls it serves and the acts done
|
||||
// by hand, kept where a restart of this process does not take them.
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
return err
|
||||
}
|
||||
// Every module's state (novox/hq ADR 0201), from the catalogue: a bucket exists from
|
||||
// registration, so a module reading one may watch it before its owner runs anywhere. One that
|
||||
// nothing declares any more is said and kept — what it holds is data.
|
||||
@@ -1252,11 +1151,25 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string)
|
||||
fmt.Printf("the bus holds state nothing declares any more, kept because it is data: %s — "+
|
||||
"removing it is a person's act\n", strings.Join(undeclared, ", "))
|
||||
}
|
||||
// And how every module hears what it consumes: asserted with the rest above, counted here.
|
||||
hearing, err := moduleConsumerCount(ctx, inv)
|
||||
// And how every module hears what it consumes. Derived from the same records the user list is
|
||||
// composed from, so a module the mesh grants a consumer's subjects has that consumer waiting.
|
||||
// Done on every raise, not only when a credential is issued: every module moved onto this bus
|
||||
// by the rollout was issued on the old one, and came up with nothing to bind to (2026-09-28).
|
||||
records, err := inv.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hearing := 0
|
||||
for _, c := range broker.ConsumersOf(users) {
|
||||
if err := js.EnsureConsumer(c.Consumer); err != nil {
|
||||
return fmt.Errorf("how %s on %s hears what it consumes: %w", c.Module, c.Node, err)
|
||||
}
|
||||
hearing++
|
||||
}
|
||||
fmt.Printf("the bus at %s has its streams, %d machine(s) can hear a declaration, %d module(s) "+
|
||||
"can hear what they consume, and %d bucket(s) of state\n", broker.BareAddress(address), len(names), hearing, len(buckets))
|
||||
return nil
|
||||
@@ -1299,46 +1212,17 @@ func seatHolders(ctx context.Context, inv *inventory.Inventory) (map[string]brok
|
||||
|
||||
// number gives one send the next sequence for its node (novox/hq 04-ISSUES/107).
|
||||
// allotting is allot over one inventory, in the shape composeEach takes.
|
||||
func allotting(ctx context.Context, inv *inventory.Inventory) func(node string) (order, error) {
|
||||
return func(node string) (order, error) { return allot(ctx, inv, node) }
|
||||
func allotting(ctx context.Context, inv *inventory.Inventory) func(node string) (int64, error) {
|
||||
return func(node string) (int64, error) { return allot(ctx, inv, node) }
|
||||
}
|
||||
|
||||
// epochForActs is the lease's gate as a composition asks it; a variable so a test can act under an epoch
|
||||
// without a bus.
|
||||
var epochForActs = func(ctx context.Context) (uint64, error) { return theLease.epoch(ctx) }
|
||||
|
||||
// order is what a declaration carries of its writer's order (link/order.go): its sequence, and the
|
||||
// epoch of the lease it is composed under — zero for a machine that has not said it reads one.
|
||||
type order struct {
|
||||
sequence int64
|
||||
epoch uint64
|
||||
}
|
||||
|
||||
// allot takes the next sequence for a machine — the number its next declaration carries — under the
|
||||
// lease: a process that may not act takes none, and composes nothing (novox/hq to-be 45 §6).
|
||||
func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, error) {
|
||||
epoch, err := epochForActs(ctx)
|
||||
if err != nil {
|
||||
return order{}, fmt.Errorf("nothing was composed for %s: %w", node, err)
|
||||
}
|
||||
// allot takes the next sequence for a machine — the number its next declaration carries.
|
||||
func allot(ctx context.Context, inv *inventory.Inventory, node string) (int64, error) {
|
||||
record, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return order{}, err
|
||||
return 0, err
|
||||
}
|
||||
if epoch > 0 {
|
||||
reads, err := inv.ReadsEpoch(ctx, record.ID)
|
||||
if err != nil {
|
||||
return order{}, err
|
||||
}
|
||||
if !reads {
|
||||
epoch = 0
|
||||
}
|
||||
}
|
||||
seq, err := inv.NextSequence(ctx, record.ID)
|
||||
if err != nil {
|
||||
return order{}, err
|
||||
}
|
||||
return order{sequence: seq, epoch: epoch}, nil
|
||||
return inv.NextSequence(ctx, record.ID)
|
||||
}
|
||||
|
||||
// recordSent writes down what a machine was just sent, and returns the digest.
|
||||
@@ -1353,7 +1237,7 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, e
|
||||
// And the build of each module it carried (novox/hq issue 259, ADR 0221), nil when that is not known:
|
||||
// what tells a machine held back by a policy or a plan from one a push left behind.
|
||||
func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte,
|
||||
builds map[string]string, epoch uint64) (string, error) {
|
||||
builds map[string]string) (string, error) {
|
||||
kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
record, err := inv.NodeByName(kept, node)
|
||||
@@ -1361,7 +1245,7 @@ func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body
|
||||
return "", err
|
||||
}
|
||||
digest := digestOf(body)
|
||||
if err := inv.RecordSentUnder(kept, record.ID, digest, builds, epoch); err != nil {
|
||||
if err := inv.RecordSent(kept, record.ID, digest, builds); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return digest, nil
|
||||
|
||||
@@ -76,7 +76,7 @@ func TestASkippedMachineIsStillAnError(t *testing.T) {
|
||||
}
|
||||
|
||||
// numbered is an allotter for tests: one higher per call, as the inventory's is per machine.
|
||||
func numbered() func(string) (order, error) {
|
||||
func numbered() func(string) (int64, error) {
|
||||
var n int64
|
||||
return func(string) (order, error) { n++; return order{sequence: n}, nil }
|
||||
return func(string) (int64, error) { n++; return n, nil }
|
||||
}
|
||||
|
||||
@@ -64,7 +64,7 @@ func TestAFailedPlanIsRetriedAndGoesOnThroughItsLaterTiers(t *testing.T) {
|
||||
Note: "a failed to build in tier 0",
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1",
|
||||
Why: link.KilledByHand}}}
|
||||
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
|
||||
if err := open.inventory.SavePlan(ctx, failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -156,7 +156,7 @@ func TestARebuildJoinsThePlanHoldingTheModule(t *testing.T) {
|
||||
Created: before, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Note: "a failed to build in tier 0",
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", Why: link.CancelledByHand}}}
|
||||
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
|
||||
if err := open.inventory.SavePlan(ctx, failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := rebuildCommand(ctx, []string{"a"}); err != nil {
|
||||
@@ -433,7 +433,7 @@ func TestCancelDeletesTheAskAndFailsThePlanThatAskedIt(t *testing.T) {
|
||||
plan := inventory.Plan{ID: "plan-cancel", Repository: "novox/a", Commit: "c0ffee", Created: asked,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: id}}}
|
||||
if err := open.inventory.SavePlan(ctx, &plan); err != nil {
|
||||
if err := open.inventory.SavePlan(ctx, plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -631,21 +631,21 @@ func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) {
|
||||
Note: "a stopped at its first machine in tier 0: laptop refused what it was sent",
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "built", BuiltAt: &long, Commit: "c0ffee",
|
||||
First: []string{"laptop"}, FirstAt: &long, Why: "laptop refused what it was sent"}}}
|
||||
if err := open.inventory.SavePlan(ctx, &stopped); err != nil {
|
||||
if err := open.inventory.SavePlan(ctx, stopped); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A newer plan holding a refuses it: sending the older build would put it back.
|
||||
newer := inventory.Plan{ID: "plan-newer", Repository: "novox/other", Commit: "d00d", Created: long.Add(time.Hour),
|
||||
State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{}}
|
||||
if err := open.inventory.SavePlan(ctx, &newer); err != nil {
|
||||
if err := open.inventory.SavePlan(ctx, newer); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := retryPlan(ctx, open, stopped.ID); err == nil || !strings.Contains(err.Error(), "plan-newer") {
|
||||
t.Fatalf("retried under a newer plan: %v", err)
|
||||
}
|
||||
newer.State = inventory.PlanSuperseded
|
||||
if err := open.inventory.SavePlan(ctx, &newer); err != nil {
|
||||
if err := open.inventory.SavePlan(ctx, newer); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -683,7 +683,7 @@ func TestAPlanIsAnsweredOnlyByTheBuildItAskedFor(t *testing.T) {
|
||||
plan := inventory.Plan{ID: "plan-own", Repository: "novox/a", Commit: "c0ffee", Created: asked,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: "build-own"}}}
|
||||
if err := open.inventory.SavePlan(ctx, &plan); err != nil {
|
||||
if err := open.inventory.SavePlan(ctx, plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
planBuilt(ctx, open, "a", "0ldc0mm1t", "", time.Now().UTC(), "build-replay")
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"sort"
|
||||
"time"
|
||||
@@ -79,28 +78,11 @@ type meshStatus struct {
|
||||
// that machine holds, with the modules that could hold it (novox/hq ADR 0207). Absent when every
|
||||
// dependency is met. Reported, not refused, until the switch.
|
||||
Unheld []catalogue.Unheld `json:"unheld,omitempty"`
|
||||
// HandActsThisWeek is how many acts were done by hand in the last seven days (novox/hq to-be 45
|
||||
// §7): every one is a repair a healer could have made. Absent where the log is not on hand;
|
||||
// HandActsUnread says why when it could not be read, rather than reading as none.
|
||||
HandActsThisWeek *int `json:"handActsThisWeek,omitempty"`
|
||||
HandActsUnread string `json:"handActsUnread,omitempty"`
|
||||
// HealsThisWeek is what the healers did in the last seven days (novox/hq to-be 45 §7); HealsUnread
|
||||
// why it could not be read.
|
||||
HealsThisWeek *healsCount `json:"healsThisWeek,omitempty"`
|
||||
HealsUnread string `json:"healsUnread,omitempty"`
|
||||
// Conditions is every open condition, urgent first and then oldest first (novox/hq to-be 45 §2):
|
||||
// what is wrong, as the watchdogs, the self-check and the providers say it. Always present — an
|
||||
// empty list is "none open" — unless they could not be read, which ConditionsUnread says.
|
||||
Conditions []conditions.Condition `json:"conditions"`
|
||||
ConditionsUnread string `json:"conditionsUnread,omitempty"`
|
||||
// Failing is every consumer a provider says it keeps failing (novox/hq ADR 0224): the open
|
||||
// conditions of that kind, carried here as well because ADR 0224 names this field. Absent when no
|
||||
// provider says so. A document without it called the mesh well while the identity provider
|
||||
// refused every consumer for a day (04-ISSUES/179).
|
||||
Failing []conditions.Condition `json:"failing,omitempty"`
|
||||
// Overflowing is every module whose identity overflows the bound of a provision it requires, and
|
||||
// so is left out of its provider's grants (novox/hq ADR 0225). Absent when every identity fits.
|
||||
Overflowing []catalogue.Overflow `json:"overflowing,omitempty"`
|
||||
// Failing is every consumer a provider says it keeps failing, with the class of error, since
|
||||
// when, and when it was last said (novox/hq ADR 0224). Absent when no provider says so. A
|
||||
// document without this called the mesh well while the identity provider refused every consumer
|
||||
// for a day (04-ISSUES/179).
|
||||
Failing []inventory.ProviderStanding `json:"failing,omitempty"`
|
||||
}
|
||||
|
||||
// machineFiltered is one rule set on a converged machine that the mesh did not write and that
|
||||
@@ -233,14 +215,7 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
||||
}
|
||||
}
|
||||
out.Unheld = asked.unheld
|
||||
out.HandActsThisWeek, out.HandActsUnread = asked.handActs, asked.handActsUnread
|
||||
out.HealsThisWeek, out.HealsUnread = asked.heals, asked.healsUnread
|
||||
out.Conditions, out.ConditionsUnread = asked.conditions, asked.conditionsUnread
|
||||
if out.Conditions == nil {
|
||||
out.Conditions = []conditions.Condition{}
|
||||
}
|
||||
out.Failing = providerStandings(asked.conditions)
|
||||
out.Overflowing = asked.overflowing
|
||||
out.Failing = asked.failing
|
||||
for name := range asked.refused {
|
||||
out.Unresolved = append(out.Unresolved, machineUnresolved{
|
||||
Node: name, Problem: asked.refused[name]})
|
||||
|
||||
@@ -441,7 +441,7 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string,
|
||||
state.BuiltAt = &now
|
||||
state.Commit = commit
|
||||
}
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
if err := inv.SavePlan(ctx, *p); err != nil {
|
||||
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
|
||||
continue
|
||||
}
|
||||
@@ -500,7 +500,7 @@ func advanceHeld(ctx context.Context, open *stores) {
|
||||
// Kept in the plan, so `plans` says why it has not moved rather than the log alone;
|
||||
// the state is left as it was and the step is tried again on the next tick.
|
||||
p.Note = "tier " + fmt.Sprint(p.Tier) + ": " + err.Error() + " — tried again"
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
if err := inv.SavePlan(ctx, *p); err != nil {
|
||||
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
|
||||
}
|
||||
break
|
||||
@@ -508,7 +508,7 @@ func advanceHeld(ctx context.Context, open *stores) {
|
||||
if p.State == inventory.PlanFailed {
|
||||
sayUnsent(p, rollsOut)
|
||||
}
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
if err := inv.SavePlan(ctx, *p); err != nil {
|
||||
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
|
||||
break
|
||||
}
|
||||
@@ -986,18 +986,10 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
whatIf := set.String("what-if", "", "owner/repository: the plan a merge there would produce, saving nothing — with --paths or --modules")
|
||||
paths := set.String("paths", "", "the files the merge would change, comma-separated, from the repository's root")
|
||||
modules := set.String("modules", "", "or the modules it would change, comma-separated")
|
||||
// Ending a plan by hand is a repair, and says why (novox/hq to-be 45 §7).
|
||||
why := addHandActFlags(set)
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) == 2 && (positionals[0] == "stop" || positionals[0] == "close") {
|
||||
// Refused before anything is opened: a repair by hand says why.
|
||||
if err := why.require("plans " + positionals[0]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1069,14 +1061,13 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
if !p.Open() {
|
||||
return fmt.Errorf("%s is already %s", p.ID, p.State)
|
||||
}
|
||||
why.record(ctx, "plans "+positionals[0], positionals[1:])
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = how + " by hand at tier " + fmt.Sprint(p.Tier) + ": " + strings.TrimSpace(*why.why)
|
||||
p.Note = how + " by hand at tier " + fmt.Sprint(p.Tier)
|
||||
sayUnsent(&p, func(m string) bool {
|
||||
u, err := inv.UpgradeOf(ctx, m)
|
||||
return err == nil && u.RollOut
|
||||
})
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s %s at tier %d of %d; what was asked still builds and registers, nothing further is asked\n",
|
||||
|
||||
@@ -1,442 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The verbs a person answers a provider's retirement with, and cleans up with (novox/hq ADR 0230).
|
||||
//
|
||||
// **The controller asks; the provider acts.** Approving, rejecting and deleting are each a question to
|
||||
// the provider on the machine it runs on — its `provisioner_*` tools — because the provider owns its
|
||||
// backend and the controller touches none. Every one says why, is written in the hand-act log before
|
||||
// it is asked, and the provider says what it did as its `provisioner.retirement` event.
|
||||
|
||||
const retireUsage = "retire [--json] | retire approve <node> <module> --why <text> | retire reject <node> <module> --why <text>"
|
||||
|
||||
const cleanupUsage = "cleanup [list] [--json] | cleanup delete <node> <module> <consumer> --why <text> | " +
|
||||
"cleanup delete --older-than <days> --why <text> [--confirm]"
|
||||
|
||||
func isNothingServes(err error) bool { return errors.Is(err, link.ErrNothingServes) }
|
||||
|
||||
func unmarshalAnswer(a link.Answer, v any) error {
|
||||
if len(a.Result) == 0 {
|
||||
return errors.New("an empty answer")
|
||||
}
|
||||
return json.Unmarshal(a.Result, v)
|
||||
}
|
||||
|
||||
// retireCommand is `retire`, `retire approve` and `retire reject`.
|
||||
func retireCommand(ctx context.Context, args []string) error {
|
||||
sub := "list"
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
sub, args = args[0], args[1:]
|
||||
}
|
||||
switch sub {
|
||||
case "list":
|
||||
set := flag.NewFlagSet("retire", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New(retireUsage)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
return onTheBus(func(conn *nats.Conn) error { return listRetiring(ctx, open.inventory, conn, *asJSON) })
|
||||
case "approve", "reject":
|
||||
set := flag.NewFlagSet("retire "+sub, flag.ContinueOnError)
|
||||
f := addHandActFlags(set)
|
||||
rest, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(rest) != 2 {
|
||||
return errors.New(retireUsage)
|
||||
}
|
||||
if err := f.require("retire " + sub); err != nil {
|
||||
return err
|
||||
}
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
return answerRetirement(ctx, conn, providerInstance{Node: rest[0], Module: rest[1]}, sub == "approve", f)
|
||||
})
|
||||
}
|
||||
return errors.New(retireUsage)
|
||||
}
|
||||
|
||||
// retiringRow is one provider's waiting or rejected set, as `retire` lists it.
|
||||
type retiringRow struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Waiting []link.RetiredConsumer `json:"waiting,omitempty"`
|
||||
Since string `json:"since,omitempty"`
|
||||
Held int `json:"held,omitempty"`
|
||||
Bound string `json:"bound,omitempty"`
|
||||
Rejected []link.RetiredConsumer `json:"rejected,omitempty"`
|
||||
RejectWhy string `json:"rejected-why,omitempty"`
|
||||
Unasked string `json:"unasked,omitempty"`
|
||||
}
|
||||
|
||||
func listRetiring(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn, asJSON bool) error {
|
||||
instances, err := providerInstances(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var rows []retiringRow
|
||||
for _, p := range instances {
|
||||
row := retiringRow{Node: p.Node, Module: p.Module}
|
||||
state, err := askRetirement(ctx, conn, p)
|
||||
switch {
|
||||
case isNothingServes(err):
|
||||
row.Unasked = "answers no retirement question: it predates ADR 0230"
|
||||
case err != nil:
|
||||
row.Unasked = err.Error()
|
||||
default:
|
||||
if state.Waiting != nil {
|
||||
row.Waiting, row.Since, row.Held = state.Waiting.Consumers, state.Waiting.Since, state.Waiting.Held
|
||||
}
|
||||
if state.Rejected != nil {
|
||||
row.Rejected, row.RejectWhy = state.Rejected.Consumers, orWhy(state.Rejected.By, state.Rejected.Why)
|
||||
}
|
||||
row.Bound = state.Bound
|
||||
if row.Waiting == nil && row.Rejected == nil {
|
||||
continue
|
||||
}
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
if asJSON {
|
||||
if rows == nil {
|
||||
rows = []retiringRow{}
|
||||
}
|
||||
return printJSON(map[string]any{"providers": rows})
|
||||
}
|
||||
said := false
|
||||
for _, r := range rows {
|
||||
switch {
|
||||
case r.Unasked != "":
|
||||
fmt.Printf("%s on %s: not asked — %s\n", r.Module, r.Node, r.Unasked)
|
||||
default:
|
||||
if r.Waiting != nil {
|
||||
said = true
|
||||
fmt.Printf("%s on %s WAITS since %s to retire %d of the %d it holds (%s): %s\n"+
|
||||
" retire approve %s %s --why … | retire reject %s %s --why …\n",
|
||||
r.Module, r.Node, r.Since, len(r.Waiting), r.Held, orBound(r.Bound), consumerList(r.Waiting),
|
||||
r.Node, r.Module, r.Node, r.Module)
|
||||
}
|
||||
if r.Rejected != nil {
|
||||
said = true
|
||||
fmt.Printf("%s on %s keeps active, by a rejection (%s): %s\n", r.Module, r.Node, r.RejectWhy,
|
||||
consumerList(r.Rejected))
|
||||
}
|
||||
}
|
||||
}
|
||||
if !said {
|
||||
fmt.Println("no provider waits for a person to approve a retirement")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// answerRetirement approves or rejects what one provider waits with. **The set sent is the set the
|
||||
// provider says it waits with, read now**, and the provider refuses any other: a person approves what
|
||||
// they were shown, never a set that moved since.
|
||||
func answerRetirement(ctx context.Context, conn *nats.Conn, p providerInstance, approve bool, f handActFlags) error {
|
||||
state, err := askRetirement(ctx, conn, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
verb, tool := "retire reject", link.ToolRetireReject
|
||||
var set []link.RetiredConsumer
|
||||
if state.Waiting != nil {
|
||||
set = state.Waiting.Consumers
|
||||
}
|
||||
if approve {
|
||||
verb, tool = "retire approve", link.ToolRetireApprove
|
||||
if set == nil && state.Rejected != nil {
|
||||
// A rejection can be taken back: the consumers it kept are retired after all.
|
||||
set = state.Rejected.Consumers
|
||||
}
|
||||
}
|
||||
if len(set) == 0 {
|
||||
return fmt.Errorf("%s on %s waits for nobody to approve or reject a retirement. Nothing was done", p.Module, p.Node)
|
||||
}
|
||||
names := make([]string, 0, len(set))
|
||||
for _, c := range set {
|
||||
names = append(names, c.Consumer)
|
||||
}
|
||||
if strings.TrimSpace(*f.cause) == "" {
|
||||
*f.cause = kindRetireWaiting
|
||||
}
|
||||
if strings.TrimSpace(*f.condition) == "" {
|
||||
*f.condition = retireWaitingKey(p.Module, p.Node)
|
||||
}
|
||||
f.record(ctx, verb, append([]string{p.Node, p.Module}, names...))
|
||||
answer, err := link.AskModuleToolOn(ctx, conn, p.Module, tool, p.Node, map[string]any{
|
||||
"consumers": names, "why": strings.TrimSpace(*f.why), "by": link.Caller(), "via": link.ViaController,
|
||||
}, retirementAsk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return fmt.Errorf("%s on %s refused: %s", p.Module, p.Node, answer.Error)
|
||||
}
|
||||
if approve && state.RetiresBy == "mark-only" {
|
||||
fmt.Printf("%s on %s marked %s retired, MARK ONLY: this provider cannot disable a consumer, so they keep "+
|
||||
"their access until `cleanup delete`; `cleanup list` shows them\n", p.Module, p.Node, strings.Join(names, ", "))
|
||||
} else if approve {
|
||||
fmt.Printf("%s on %s retired %s: access disabled, data kept; `cleanup list` shows them\n", p.Module, p.Node,
|
||||
strings.Join(names, ", "))
|
||||
} else {
|
||||
fmt.Printf("%s on %s keeps %s active; the warning stays open until the mesh asks for them again or the "+
|
||||
"retirement is approved\n", p.Module, p.Node, strings.Join(names, ", "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// cleanupCommand is `cleanup list` and `cleanup delete`.
|
||||
func cleanupCommand(ctx context.Context, args []string) error {
|
||||
sub := "list"
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
sub, args = args[0], args[1:]
|
||||
}
|
||||
switch sub {
|
||||
case "list":
|
||||
set := flag.NewFlagSet("cleanup", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New(cleanupUsage)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
listing, err := gatherRetired(ctx, open.inventory, conn, time.Now())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return printRetired(listing, *asJSON)
|
||||
})
|
||||
case "delete":
|
||||
set := flag.NewFlagSet("cleanup delete", flag.ContinueOnError)
|
||||
f := addHandActFlags(set)
|
||||
olderThan := set.Int("older-than", 0, "every retired consumer older than this many days")
|
||||
confirm := set.Bool("confirm", false, "with --older-than: delete what is listed, rather than only list it")
|
||||
rest, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := f.require("cleanup delete"); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(*f.cause) == "" {
|
||||
*f.cause = kindCleanupWaiting
|
||||
}
|
||||
switch {
|
||||
case *olderThan > 0 && len(rest) == 0:
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
return deleteOlderThan(ctx, open.inventory, conn, *olderThan, *confirm, f, time.Now())
|
||||
})
|
||||
case *olderThan == 0 && len(rest) == 3 && !*confirm:
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
return deleteRetired(ctx, conn, providerInstance{Node: rest[0], Module: rest[1]}, rest[2], f)
|
||||
})
|
||||
}
|
||||
return errors.New(cleanupUsage)
|
||||
}
|
||||
return errors.New(cleanupUsage)
|
||||
}
|
||||
|
||||
// retiredRow is one retired consumer, as `cleanup list` shows it.
|
||||
type retiredRow struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Consumer string `json:"consumer"`
|
||||
// ConsumerNode is where the consumer was, when the provider knows.
|
||||
ConsumerNode string `json:"consumer-node,omitempty"`
|
||||
Kind string `json:"kind,omitempty"`
|
||||
RetiredAt string `json:"retired-at,omitempty"`
|
||||
// AgeDays is whole days since it was retired; -1 when the provider could not say when.
|
||||
AgeDays int `json:"age-days"`
|
||||
SizeBytes *int64 `json:"size-bytes,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
// Access is "kept" for a consumer of a mark-only provider: retired on record, still reachable.
|
||||
Access string `json:"access,omitempty"`
|
||||
}
|
||||
|
||||
// retiredListing is every provider's retired consumers, and the providers that could not say.
|
||||
type retiredListing struct {
|
||||
Retired []retiredRow `json:"retired"`
|
||||
// Unasked names each provider not asked, and why: one older than the question, or one that failed.
|
||||
Unasked []string `json:"unasked,omitempty"`
|
||||
}
|
||||
|
||||
func gatherRetired(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn, now time.Time) (retiredListing, error) {
|
||||
instances, err := providerInstances(ctx, inv)
|
||||
if err != nil {
|
||||
return retiredListing{}, err
|
||||
}
|
||||
return retiredOf(ctx, conn, instances, now), nil
|
||||
}
|
||||
|
||||
func retiredOf(ctx context.Context, conn *nats.Conn, instances []providerInstance, now time.Time) retiredListing {
|
||||
out := retiredListing{Retired: []retiredRow{}}
|
||||
for _, p := range instances {
|
||||
state, err := askRetirement(ctx, conn, p)
|
||||
if err != nil {
|
||||
if isNothingServes(err) {
|
||||
out.Unasked = append(out.Unasked, fmt.Sprintf("%s on %s answers no retirement question: it predates ADR 0230", p.Module, p.Node))
|
||||
} else {
|
||||
out.Unasked = append(out.Unasked, err.Error())
|
||||
}
|
||||
continue
|
||||
}
|
||||
for _, c := range state.Retired {
|
||||
row := retiredRow{Node: p.Node, Module: p.Module, Consumer: c.Consumer, ConsumerNode: c.Node, Kind: c.Kind,
|
||||
RetiredAt: c.RetiredAt, AgeDays: -1, SizeBytes: c.SizeBytes, Why: c.Why, Access: c.Access}
|
||||
if at := retiredAt(c); !at.IsZero() {
|
||||
row.AgeDays = int(now.Sub(at).Hours() / 24)
|
||||
}
|
||||
out.Retired = append(out.Retired, row)
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out.Retired, func(i, j int) bool { return out.Retired[i].AgeDays > out.Retired[j].AgeDays })
|
||||
return out
|
||||
}
|
||||
|
||||
func printRetired(l retiredListing, asJSON bool) error {
|
||||
if asJSON {
|
||||
return printJSON(l)
|
||||
}
|
||||
if len(l.Retired) == 0 {
|
||||
fmt.Println("no provider holds a retired consumer")
|
||||
}
|
||||
for _, r := range l.Retired {
|
||||
age := "age unknown"
|
||||
if r.AgeDays >= 0 {
|
||||
age = strconv.Itoa(r.AgeDays) + " day(s)"
|
||||
}
|
||||
kind := ""
|
||||
if r.Kind != "" && r.Kind != "consumer" {
|
||||
kind = " [" + r.Kind + "]"
|
||||
}
|
||||
if r.Access == "kept" {
|
||||
kind += " [MARK ONLY: access kept until deleted]"
|
||||
}
|
||||
fmt.Printf("%s on %s: %s%s — retired %s, %s, %s\n %s\n", r.Module, r.Node, r.Consumer, kind, age,
|
||||
sizeWords(r.SizeBytes), r.RetiredAt, orWhy("", r.Why))
|
||||
}
|
||||
for _, u := range l.Unasked {
|
||||
fmt.Printf("not asked: %s\n", u)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// deleteRetired asks one provider to delete one consumer it holds retired — never an active one: the
|
||||
// provider refuses that, and this refuses it first, from what the provider says it holds.
|
||||
func deleteRetired(ctx context.Context, conn *nats.Conn, p providerInstance, consumer string, f handActFlags) error {
|
||||
state, err := askRetirement(ctx, conn, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
found := false
|
||||
for _, c := range state.Retired {
|
||||
found = found || c.Consumer == consumer
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("%s on %s holds no retired consumer %s — only a retired consumer is deleted. Nothing was done",
|
||||
p.Module, p.Node, consumer)
|
||||
}
|
||||
f.record(ctx, "cleanup delete", []string{p.Node, p.Module, consumer})
|
||||
answer, err := link.AskModuleToolOn(ctx, conn, p.Module, link.ToolRetiredDelete, p.Node, map[string]any{
|
||||
"consumer": consumer, "confirm": consumer, "why": strings.TrimSpace(*f.why), "by": link.Caller(),
|
||||
"via": link.ViaController,
|
||||
}, retirementAsk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return fmt.Errorf("%s on %s refused to delete %s: %s", p.Module, p.Node, consumer, answer.Error)
|
||||
}
|
||||
var done struct {
|
||||
FreedBytes *int64 `json:"freed_bytes"`
|
||||
}
|
||||
_ = unmarshalAnswer(answer, &done)
|
||||
fmt.Printf("%s on %s deleted %s (%s freed)\n", p.Module, p.Node, consumer, sizeWords(done.FreedBytes))
|
||||
return nil
|
||||
}
|
||||
|
||||
// deleteOlderThan lists every consumer retired more than days ago, and deletes them only with confirm.
|
||||
// One whose age the provider cannot say is never in it.
|
||||
func deleteOlderThan(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn, days int, confirm bool,
|
||||
f handActFlags, now time.Time) error {
|
||||
listing, err := gatherRetired(ctx, inv, conn, now)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return deleteFrom(ctx, conn, listing, days, confirm, f)
|
||||
}
|
||||
|
||||
func deleteFrom(ctx context.Context, conn *nats.Conn, listing retiredListing, days int, confirm bool, f handActFlags) error {
|
||||
var due []retiredRow
|
||||
unknown := 0
|
||||
for _, r := range listing.Retired {
|
||||
switch {
|
||||
case r.AgeDays < 0:
|
||||
unknown++
|
||||
case r.AgeDays > days:
|
||||
due = append(due, r)
|
||||
}
|
||||
}
|
||||
for _, u := range listing.Unasked {
|
||||
fmt.Printf("not asked: %s\n", u)
|
||||
}
|
||||
if unknown > 0 {
|
||||
fmt.Printf("%d retired consumer(s) whose age their provider cannot say are left out\n", unknown)
|
||||
}
|
||||
if len(due) == 0 {
|
||||
fmt.Printf("nothing has been retired more than %d day(s)\n", days)
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("retired more than %d day(s):\n", days)
|
||||
for _, r := range due {
|
||||
fmt.Printf(" %s on %s: %s — %d day(s), %s\n", r.Module, r.Node, r.Consumer, r.AgeDays, sizeWords(r.SizeBytes))
|
||||
}
|
||||
if !confirm {
|
||||
fmt.Printf("nothing was deleted: add --confirm to delete these %d\n", len(due))
|
||||
return nil
|
||||
}
|
||||
var failed []string
|
||||
for _, r := range due {
|
||||
if err := deleteRetired(ctx, conn, providerInstance{Node: r.Node, Module: r.Module}, r.Consumer, f); err != nil {
|
||||
failed = append(failed, err.Error())
|
||||
}
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
return fmt.Errorf("%d of %d not deleted: %s", len(failed), len(due), strings.Join(failed, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,343 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A consumer the mesh stops asking for is retired, not withdrawn, and deleted only by a person
|
||||
// (novox/hq ADR 0230, the operator's decision of 2026-10-06; it replaces ADR 0229's withdrawal brake).
|
||||
//
|
||||
// A provider retires a consumer — disables its access, keeps its data, marks it with when and why —
|
||||
// once it has seen the same consumers go unasked for in five passes. More than three at once, or more
|
||||
// than half of those it holds where it holds more than one, is a person actively working on the mesh,
|
||||
// so the provider retires nothing and waits: the controller keeps that as an urgent condition naming
|
||||
// what would go and the two verbs that answer it. A retired consumer is deleted only by `cleanup
|
||||
// delete`, which the provider executes on its own backend — the controller never touches one — and
|
||||
// one retired more than thirty days is a warning that cleanup is waiting (D11).
|
||||
|
||||
// The kinds a provider's retirement word raises.
|
||||
const (
|
||||
kindRetireWaiting = "retire-waiting"
|
||||
kindRetireRejected = "retire-rejected"
|
||||
kindCleanupWaiting = "cleanup-waiting"
|
||||
// sourceRetirement is what raised a retirement condition: the provider's own event.
|
||||
sourceRetirement = "provisioner.retirement"
|
||||
)
|
||||
|
||||
// cleanupAfter is how long a consumer may stay retired before cleanup is said to be waiting (D11).
|
||||
const cleanupAfter = 30 * 24 * time.Hour
|
||||
|
||||
// retirementAsk is how long one provider is given to answer one question about its retired consumers.
|
||||
var retirementAsk = 20 * time.Second
|
||||
|
||||
func retireWaitingKey(module, node string) string {
|
||||
return conditions.Key(conditions.ScopeProvider, module+"."+node, "retire")
|
||||
}
|
||||
|
||||
func retireRejectedKey(module, node string) string {
|
||||
return conditions.Key(conditions.ScopeProvider, module+"."+node, "retire-rejected")
|
||||
}
|
||||
|
||||
// retirements keeps what providers say about consumers they retire, as conditions.
|
||||
type retirements struct {
|
||||
keeper func() *conditions.Keeper
|
||||
// record writes an act done by hand that reached a provider some other way than this controller's
|
||||
// verbs; nil records nothing (a test).
|
||||
record func(ctx context.Context, act link.HandAct) error
|
||||
}
|
||||
|
||||
func consumerList(cs []link.RetiredConsumer) string {
|
||||
parts := make([]string, 0, len(cs))
|
||||
for _, c := range cs {
|
||||
p := c.Consumer
|
||||
if c.Node != "" {
|
||||
p += " (" + c.Node + ")"
|
||||
}
|
||||
parts = append(parts, p)
|
||||
}
|
||||
return strings.Join(parts, ", ")
|
||||
}
|
||||
|
||||
// waitingObservation is a provider waiting for a person to approve or reject a retirement.
|
||||
func waitingObservation(r link.Retirement) conditions.Observation {
|
||||
since := r.Since
|
||||
if since.IsZero() {
|
||||
since = r.At
|
||||
}
|
||||
summary := fmt.Sprintf("%s on %s would retire %d consumer(s) the mesh no longer asks for — %s — more than its "+
|
||||
"bound (%s), so it retires nothing until a person answers: `retire approve %s %s --why …` or `retire reject "+
|
||||
"%s %s --why …`", r.Module, r.ProviderNode, len(r.Consumers), consumerList(r.Consumers), orBound(r.Bound),
|
||||
r.ProviderNode, r.Module, r.ProviderNode, r.Module)
|
||||
said := fmt.Sprintf("waiting since %s, %d held: %s", since.UTC().Format("2006-01-02 15:04 MST"), r.Held,
|
||||
consumerList(r.Consumers))
|
||||
return conditions.Observation{Scope: conditions.ScopeProvider, ID: r.Module + "." + r.ProviderNode,
|
||||
Token: "retire", Kind: kindRetireWaiting, Machine: r.ProviderNode, Also: consumerNodes(r),
|
||||
Severity: conditions.Urgent, Summary: summary, Said: said, Source: sourceRetirement,
|
||||
Resolver: conditions.ResolverOperator}
|
||||
}
|
||||
|
||||
// rejectedObservation is consumers kept active by a person's rejection though the mesh asks for them no more.
|
||||
func rejectedObservation(r link.Retirement) conditions.Observation {
|
||||
summary := fmt.Sprintf("%s on %s keeps %s active although the mesh no longer asks for them: a person rejected "+
|
||||
"their retirement (%s). Assign them again, or `retire approve %s %s --why …`", r.Module, r.ProviderNode,
|
||||
consumerList(r.Consumers), orWhy(r.By, r.Why), r.ProviderNode, r.Module)
|
||||
return conditions.Observation{Scope: conditions.ScopeProvider, ID: r.Module + "." + r.ProviderNode,
|
||||
Token: "retire-rejected", Kind: kindRetireRejected, Machine: r.ProviderNode, Also: consumerNodes(r),
|
||||
Severity: conditions.Warning, Summary: summary, Said: "rejected: " + orWhy(r.By, r.Why),
|
||||
Source: sourceRetirement, Resolver: conditions.ResolverOperator}
|
||||
}
|
||||
|
||||
func orBound(b string) string {
|
||||
if b == "" {
|
||||
return "more than 3, or more than half of those held"
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func orWhy(by, why string) string {
|
||||
switch {
|
||||
case by != "" && why != "":
|
||||
return by + ": " + why
|
||||
case why != "":
|
||||
return why
|
||||
case by != "":
|
||||
return "by " + by
|
||||
}
|
||||
return "no reason given"
|
||||
}
|
||||
|
||||
// consumerNodes are the other machines a retirement concerns: where its consumers are.
|
||||
func consumerNodes(r link.Retirement) []string {
|
||||
seen := map[string]bool{r.ProviderNode: true}
|
||||
var out []string
|
||||
for _, c := range r.Consumers {
|
||||
if c.Node != "" && !seen[c.Node] {
|
||||
seen[c.Node] = true
|
||||
out = append(out, c.Node)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// Retired keeps one word. Waiting raises the urgent condition; a rejection turns it into a warning; a
|
||||
// retirement, an approval or the set settling clears both. An approval, a rejection or a deletion that
|
||||
// did not come through this controller's verbs is recorded in the hand-act log here, so every one is.
|
||||
func (s retirements) Retired(ctx context.Context, r link.Retirement) error {
|
||||
k := s.keeper()
|
||||
if k == nil {
|
||||
return fmt.Errorf("the condition store is not open in this controller: %w", link.ErrTryAgain)
|
||||
}
|
||||
waiting, rejected := retireWaitingKey(r.Module, r.ProviderNode), retireRejectedKey(r.Module, r.ProviderNode)
|
||||
clear := func(keys ...string) error {
|
||||
for _, key := range keys {
|
||||
if _, err := k.Clear(ctx, key, fmt.Sprintf("%s on %s says %s", r.Module, r.ProviderNode, r.Change)); err != nil {
|
||||
return storeAway(err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
var err error
|
||||
switch r.Change {
|
||||
case link.RetireWaiting:
|
||||
if _, err = k.Observe(ctx, waitingObservation(r)); err != nil {
|
||||
return storeAway(err)
|
||||
}
|
||||
case link.RetireRejected:
|
||||
if err = clear(waiting); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = k.Observe(ctx, rejectedObservation(r)); err != nil {
|
||||
return storeAway(err)
|
||||
}
|
||||
case link.RetireApproved, link.RetireRetired, link.RetireSettled:
|
||||
if err = clear(waiting, rejected); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
switch r.Change {
|
||||
case link.RetireApproved, link.RetireRejected, link.RetireDeleted:
|
||||
if r.Via != link.ViaController && s.record != nil {
|
||||
verb := map[string]string{link.RetireApproved: "retire approve", link.RetireRejected: "retire reject",
|
||||
link.RetireDeleted: "cleanup delete"}[r.Change]
|
||||
cause := kindRetireWaiting
|
||||
if r.Change == link.RetireDeleted {
|
||||
cause = kindCleanupWaiting
|
||||
}
|
||||
why := r.Why
|
||||
if strings.TrimSpace(why) == "" {
|
||||
why = "no reason given to the provider"
|
||||
}
|
||||
act := link.HandAct{Verb: verb, Args: append([]string{r.ProviderNode, r.Module}, r.Names()...),
|
||||
Why: why + " (asked of the provider directly, not through the controller)", Cause: cause,
|
||||
By: r.By, At: r.At}
|
||||
if act.By == "" {
|
||||
act.By = "unknown, asked of " + r.Module + " on " + r.ProviderNode + " directly"
|
||||
}
|
||||
if err := s.record(ctx, act); err != nil {
|
||||
return fmt.Errorf("%s on %s %s by hand, and it could not be recorded: %v: %w", r.Module,
|
||||
r.ProviderNode, r.Change, err, link.ErrTryAgain)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// recordHandActOnTheBus writes an act through the serving controller's connection.
|
||||
func recordHandActOnTheBus(ctx context.Context, act link.HandAct) error {
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
_, err := link.RecordHandAct(ctx, conn, act)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
// providerInstance is one provider module on one machine.
|
||||
type providerInstance struct {
|
||||
Node, Module string
|
||||
}
|
||||
|
||||
// providerInstances are every module assigned on every machine whose manifest receives contributions:
|
||||
// every provider, which serves the retirement tools (ADR 0230) — or is older than them.
|
||||
func providerInstances(ctx context.Context, inv *inventory.Inventory) ([]providerInstance, error) {
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []providerInstance
|
||||
for _, n := range nodes {
|
||||
modules, err := inv.Assigned(ctx, n.Name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("what %s is assigned cannot be read: %w", n.Name, err)
|
||||
}
|
||||
for _, m := range modules {
|
||||
if man, ok := shelf[m]; ok && len(man.Receives) > 0 {
|
||||
out = append(out, providerInstance{Node: n.Name, Module: m})
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].Node != out[j].Node {
|
||||
return out[i].Node < out[j].Node
|
||||
}
|
||||
return out[i].Module < out[j].Module
|
||||
})
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// askRetirement asks one provider for what it holds retired and what waits.
|
||||
func askRetirement(ctx context.Context, conn *nats.Conn, p providerInstance) (link.RetirementState, error) {
|
||||
var state link.RetirementState
|
||||
answer, err := link.AskModuleToolOn(ctx, conn, p.Module, link.ToolRetirement, p.Node, map[string]any{}, retirementAsk)
|
||||
if err != nil {
|
||||
return state, err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return state, fmt.Errorf("%s on %s answered %s with an error: %s", p.Module, p.Node, link.ToolRetirement, answer.Error)
|
||||
}
|
||||
if err := unmarshalAnswer(answer, &state); err != nil {
|
||||
return state, fmt.Errorf("%s on %s answered %s with something unreadable: %w", p.Module, p.Node, link.ToolRetirement, err)
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
// retiredAt reads a retired consumer's moment; zero when the provider could not say.
|
||||
func retiredAt(c link.RetiredConsumer) time.Time {
|
||||
t, err := time.Parse(time.RFC3339, c.RetiredAt)
|
||||
if err != nil {
|
||||
return time.Time{}
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// cleanupObservation is a provider holding consumers retired longer than cleanupAfter.
|
||||
func cleanupObservation(p providerInstance, old []link.RetiredConsumer, now time.Time) conditions.Observation {
|
||||
parts := make([]string, 0, len(old))
|
||||
for _, c := range old {
|
||||
parts = append(parts, fmt.Sprintf("%s (%d days, %s)", c.Consumer, int(now.Sub(retiredAt(c)).Hours()/24),
|
||||
sizeWords(c.SizeBytes)))
|
||||
}
|
||||
return conditions.Observation{Scope: conditions.ScopeProvider, ID: p.Module + "." + p.Node, Token: "cleanup",
|
||||
Kind: kindCleanupWaiting, Machine: p.Node, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s on %s holds %d consumer(s) retired more than %d days, waiting for a person to "+
|
||||
"delete or bring them back: %s — `cleanup list`, then `cleanup delete %s %s <consumer> --why …`",
|
||||
p.Module, p.Node, len(old), int(cleanupAfter.Hours()/24), strings.Join(parts, ", "), p.Node, p.Module),
|
||||
Resolver: conditions.ResolverOperator}
|
||||
}
|
||||
|
||||
func sizeWords(size *int64) string {
|
||||
if size == nil || *size < 0 {
|
||||
return "size unknown"
|
||||
}
|
||||
b := float64(*size)
|
||||
for _, unit := range []string{"B", "KB", "MB", "GB"} {
|
||||
if b < 1024 || unit == "GB" {
|
||||
if unit == "B" {
|
||||
return fmt.Sprintf("%d B", *size)
|
||||
}
|
||||
return fmt.Sprintf("%.1f %s", b, unit)
|
||||
}
|
||||
b /= 1024
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// probeRetired is D11: no provider holds a consumer retired more than thirty days. Every provider
|
||||
// assigned is asked what it holds retired; one that does not serve the question — an older build, a
|
||||
// TypeScript provider not yet on the SDK that retires — has nothing it can say and is passed over,
|
||||
// which is not a failure of the probe. A provider that cannot be asked otherwise is.
|
||||
func probeRetired(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
if d.js == nil {
|
||||
return nil, fmt.Errorf("no bus to ask the providers over")
|
||||
}
|
||||
instances, err := providerInstances(ctx, d.open.inventory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return retiredTooLong(ctx, d.js.Conn(), instances, time.Now())
|
||||
}
|
||||
|
||||
// retiredTooLong asks each provider and answers one observation per provider holding anything retired
|
||||
// longer than cleanupAfter.
|
||||
func retiredTooLong(ctx context.Context, conn *nats.Conn, instances []providerInstance,
|
||||
now time.Time) ([]conditions.Observation, error) {
|
||||
var out []conditions.Observation
|
||||
var problems []string
|
||||
for _, p := range instances {
|
||||
state, err := askRetirement(ctx, conn, p)
|
||||
if err != nil {
|
||||
if isNothingServes(err) {
|
||||
continue
|
||||
}
|
||||
problems = append(problems, err.Error())
|
||||
continue
|
||||
}
|
||||
var old []link.RetiredConsumer
|
||||
for _, c := range state.Retired {
|
||||
if at := retiredAt(c); !at.IsZero() && now.Sub(at) > cleanupAfter {
|
||||
old = append(old, c)
|
||||
}
|
||||
}
|
||||
if len(old) > 0 {
|
||||
out = append(out, cleanupObservation(p, old, now))
|
||||
}
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
// Not "nothing retired": a provider that could not be asked may hold the oldest of all.
|
||||
return nil, fmt.Errorf("%s", strings.Join(problems, "; "))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -1,483 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A consumer the mesh stops asking for is retired, not withdrawn, and deleted only by a person
|
||||
// (novox/hq ADR 0230): what a provider says becomes a condition a person answers, and the verbs that
|
||||
// answer it ask the provider — never anything else — for exactly what it said.
|
||||
|
||||
func waitingWord(module, node string, names ...string) link.Retirement {
|
||||
r := link.Retirement{Module: module, Provider: "postgres-database", ProviderNode: node, Change: link.RetireWaiting,
|
||||
Held: 7, Bound: "more than 3, or more than half of the 7 held", At: time.Now(), Since: time.Now()}
|
||||
for _, n := range names {
|
||||
r.Consumers = append(r.Consumers, link.RetiredConsumer{Consumer: n, Node: "laptop"})
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func openKeys(t *testing.T, k *conditions.Keeper) map[string]conditions.Condition {
|
||||
t.Helper()
|
||||
all, err := k.Open(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := map[string]conditions.Condition{}
|
||||
for _, c := range all {
|
||||
out[c.Key] = c
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestARetirementWaitingIsUrgentARejectionAWarningAndARetirementClears(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
var recorded []link.HandAct
|
||||
r := retirements{keeper: func() *conditions.Keeper { return k },
|
||||
record: func(_ context.Context, a link.HandAct) error { recorded = append(recorded, a); return nil }}
|
||||
ctx := t.Context()
|
||||
waiting := waitingWord("postgres", "anchor", "a", "b", "c", "d")
|
||||
if err := r.Retired(ctx, waiting); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open := openKeys(t, k)
|
||||
c, ok := open["provider.postgres.anchor.retire"]
|
||||
if !ok || c.Kind != kindRetireWaiting || c.Severity != conditions.Urgent {
|
||||
t.Fatalf("waiting is not an urgent retire-waiting condition: %+v", open)
|
||||
}
|
||||
for _, want := range []string{"a (laptop), b (laptop), c (laptop), d (laptop)", "retire approve anchor postgres",
|
||||
"retire reject anchor postgres", "more than 3"} {
|
||||
if !strings.Contains(c.Summary, want) {
|
||||
t.Errorf("the condition does not say %q: %s", want, c.Summary)
|
||||
}
|
||||
}
|
||||
|
||||
// Rejected, through the controller: the urgent one becomes a warning, and nothing is recorded here —
|
||||
// the verb recorded it before it asked.
|
||||
rejected := waiting
|
||||
rejected.Change, rejected.By, rejected.Why, rejected.Via = link.RetireRejected, "operator", "moving them", link.ViaController
|
||||
if err := r.Retired(ctx, rejected); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open = openKeys(t, k)
|
||||
if _, still := open["provider.postgres.anchor.retire"]; still {
|
||||
t.Fatal("a rejection left the provider waiting")
|
||||
}
|
||||
if c, ok := open["provider.postgres.anchor.retire-rejected"]; !ok || c.Severity != conditions.Warning ||
|
||||
c.Kind != kindRetireRejected || !strings.Contains(c.Summary, "moving them") {
|
||||
t.Fatalf("a rejection is not a warning saying why: %+v", open)
|
||||
}
|
||||
if len(recorded) != 0 {
|
||||
t.Fatalf("an act through the controller was recorded twice: %+v", recorded)
|
||||
}
|
||||
|
||||
// Approved afterwards, asked of the provider directly: both cleared, and recorded by hand here.
|
||||
approved := waiting
|
||||
approved.Change, approved.By, approved.Why, approved.Via = link.RetireApproved, "someone", "done moving", ""
|
||||
if err := r.Retired(ctx, approved); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if open := openKeys(t, k); len(open) != 0 {
|
||||
t.Fatalf("an approval left conditions open: %+v", open)
|
||||
}
|
||||
if len(recorded) != 1 || recorded[0].Verb != "retire approve" || recorded[0].By != "someone" ||
|
||||
!strings.Contains(recorded[0].Why, "directly") || !slices.Contains(recorded[0].Args, "d") {
|
||||
t.Fatalf("an approval outside the controller was not recorded: %+v", recorded)
|
||||
}
|
||||
|
||||
// Waiting again, then the set settles (asked for again): cleared. And retired clears too.
|
||||
for _, change := range []string{link.RetireSettled, link.RetireRetired} {
|
||||
if err := r.Retired(ctx, waiting); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
done := waiting
|
||||
done.Change = change
|
||||
if err := r.Retired(ctx, done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if open := openKeys(t, k); len(open) != 0 {
|
||||
t.Fatalf("%s left conditions open: %+v", change, open)
|
||||
}
|
||||
}
|
||||
|
||||
// A deletion asked of the provider directly is recorded too.
|
||||
deleted := link.Retirement{Module: "postgres", ProviderNode: "anchor", Change: link.RetireDeleted, By: "x",
|
||||
Why: "gone for good", At: time.Now(), Consumers: []link.RetiredConsumer{{Consumer: "a"}}}
|
||||
if err := r.Retired(ctx, deleted); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(recorded) != 2 || recorded[1].Verb != "cleanup delete" || recorded[1].Cause != kindCleanupWaiting {
|
||||
t.Fatalf("a deletion outside the controller was not recorded: %+v", recorded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARetirementWordIsKeptAsItsConditionNamingTheEmitterFromTheSubject(t *testing.T) {
|
||||
body, _ := json.Marshal(map[string]any{"provider": "oidc-client", "provider-node": "anchor", "change": "waiting",
|
||||
"held": 2, "consumers": []map[string]any{{"consumer": "x"}, {"consumer": "y"}}, "module": "liar"})
|
||||
r, err := link.ReadRetirement("mesh.mod.idp.event.provisioner.retirement", body)
|
||||
if err != nil || r.Module != "idp" || len(r.Consumers) != 2 {
|
||||
t.Fatalf("%+v %v", r, err)
|
||||
}
|
||||
if _, err := link.ReadRetirement("mesh.mod.idp.event.provisioner.retirement",
|
||||
[]byte(`{"provider-node":"anchor","change":"vanished"}`)); err == nil {
|
||||
t.Fatal("a change the mesh has no name for was read")
|
||||
}
|
||||
if _, err := link.ReadRetirement("mesh.mod.idp.event.provisioner.failing", body); err == nil {
|
||||
t.Fatal("a failing word was read as a retirement")
|
||||
}
|
||||
k, _ := withConditionsInMemory(t)
|
||||
if err := (retirements{keeper: func() *conditions.Keeper { return k }}).Retired(t.Context(), r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := openKeys(t, k)["provider.idp.anchor.retire"]; !ok {
|
||||
t.Fatal("not kept under the emitter the subject names")
|
||||
}
|
||||
}
|
||||
|
||||
func TestARetirementConditionOfAnUnassignedProviderClears(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "pg", Version: "1",
|
||||
Receives: map[string]string{"postgres-database": "/var/lib/mesh/pg/mesh.json"}})
|
||||
if _, err := assign(ctx, open, "anchor", "pg"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := retirements{keeper: func() *conditions.Keeper { return conditionsFrom }}
|
||||
for _, w := range []link.Retirement{waitingWord("pg", "anchor", "a", "b", "c", "d"), waitingWord("gone", "anchor", "a", "b", "c", "d")} {
|
||||
if err := r.Retired(ctx, w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := conditionsFrom.Reconcile(ctx, "D11", []conditions.Observation{
|
||||
cleanupObservation(providerInstance{Node: "anchor", Module: "gone"},
|
||||
[]link.RetiredConsumer{{Consumer: "a", RetiredAt: time.Now().Add(-40 * 24 * time.Hour).Format(time.RFC3339)}}, time.Now()),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
all, _ := conditionsFrom.Open(ctx)
|
||||
if len(all) != 3 {
|
||||
t.Fatalf("%+v", all)
|
||||
}
|
||||
if err := unassignedProviders(ctx, open.inventory, conditionsFrom, providerConditions(all)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left := openKeys(t, conditionsFrom)
|
||||
if len(left) != 1 || left["provider.pg.anchor.retire"].Key == "" {
|
||||
t.Fatalf("only the assigned provider's waiting should stay: %+v", left)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeProvider answers the retirement tools on one machine, over a real bus, keeping what it was asked.
|
||||
type fakeProvider struct {
|
||||
mu sync.Mutex
|
||||
state link.RetirementState
|
||||
asked []map[string]any
|
||||
deleted []string
|
||||
approved []string
|
||||
rejected []string
|
||||
}
|
||||
|
||||
func (f *fakeProvider) serve(t *testing.T, conn *nats.Conn, module, node string) {
|
||||
t.Helper()
|
||||
answer := func(m *nats.Msg, result any, refusal string) {
|
||||
body, _ := json.Marshal(map[string]any{"result": result, "error": refusal, "node": node})
|
||||
_ = m.Respond(body)
|
||||
}
|
||||
names := func(args map[string]any) []string {
|
||||
var out []string
|
||||
for _, v := range args["consumers"].([]any) {
|
||||
out = append(out, v.(string))
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
set := func(cs []link.RetiredConsumer) []string {
|
||||
var out []string
|
||||
for _, c := range cs {
|
||||
out = append(out, c.Consumer)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
for _, tool := range []string{link.ToolRetirement, link.ToolRetireApprove, link.ToolRetireReject, link.ToolRetiredDelete} {
|
||||
tool := tool
|
||||
sub, err := conn.Subscribe(link.ModuleToolOn(module, tool, node), func(m *nats.Msg) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var args map[string]any
|
||||
_ = json.Unmarshal(m.Data, &args)
|
||||
f.asked = append(f.asked, map[string]any{"tool": tool, "args": args})
|
||||
switch tool {
|
||||
case link.ToolRetirement:
|
||||
answer(m, f.state, "")
|
||||
case link.ToolRetireApprove:
|
||||
if f.state.Waiting == nil || !slices.Equal(names(args), set(f.state.Waiting.Consumers)) {
|
||||
answer(m, nil, "not the set I wait with")
|
||||
return
|
||||
}
|
||||
f.approved = names(args)
|
||||
answer(m, map[string]any{"retired": f.approved}, "")
|
||||
case link.ToolRetireReject:
|
||||
if f.state.Waiting == nil || !slices.Equal(names(args), set(f.state.Waiting.Consumers)) {
|
||||
answer(m, nil, "not the set I wait with")
|
||||
return
|
||||
}
|
||||
f.rejected = names(args)
|
||||
answer(m, map[string]any{"kept": f.rejected}, "")
|
||||
case link.ToolRetiredDelete:
|
||||
if args["confirm"] != args["consumer"] || args["why"] == "" || args["via"] != link.ViaController {
|
||||
answer(m, nil, "confirm, why and via")
|
||||
return
|
||||
}
|
||||
f.deleted = append(f.deleted, args["consumer"].(string))
|
||||
answer(m, map[string]any{"deleted": args["consumer"], "freed_bytes": 1024}, "")
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
}
|
||||
if err := conn.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func onATestBus(t *testing.T) *nats.Conn {
|
||||
t.Helper()
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := handActConn
|
||||
handActConn = js.Conn()
|
||||
t.Cleanup(func() { handActConn = before })
|
||||
return js.Conn()
|
||||
}
|
||||
|
||||
func whyFlags(t *testing.T, why string) handActFlags {
|
||||
t.Helper()
|
||||
set := flag.NewFlagSet("t", flag.ContinueOnError)
|
||||
f := addHandActFlags(set)
|
||||
if err := set.Parse([]string{"--why", why}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
func handActsBy(t *testing.T, conn *nats.Conn, verb string) []link.HandAct {
|
||||
t.Helper()
|
||||
acts, err := link.HandActs(t.Context(), conn, time.Now().Add(-time.Minute))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out []link.HandAct
|
||||
for _, a := range acts {
|
||||
if a.Verb == verb {
|
||||
out = append(out, a)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func retiredDaysAgo(name string, days int) link.RetiredConsumer {
|
||||
size := int64(4096)
|
||||
return link.RetiredConsumer{Consumer: name, Kind: "consumer", Why: "the mesh stopped asking for it",
|
||||
RetiredAt: time.Now().Add(-time.Duration(days) * 24 * time.Hour).UTC().Format(time.RFC3339), SizeBytes: &size}
|
||||
}
|
||||
|
||||
func TestNatsApproveSendsTheExactSetTheProviderWaitsWith(t *testing.T) {
|
||||
conn := onATestBus(t)
|
||||
fake := &fakeProvider{}
|
||||
fake.state.Waiting = &link.RetirementWaiting{Consumers: []link.RetiredConsumer{{Consumer: "d"}, {Consumer: "b"}, {Consumer: "a"}, {Consumer: "c"}}, Held: 6}
|
||||
fake.serve(t, conn, "pg-approve", "anchor")
|
||||
p := providerInstance{Node: "anchor", Module: "pg-approve"}
|
||||
said := printed(t, func() error { return answerRetirement(t.Context(), conn, p, true, whyFlags(t, "moved them")) })
|
||||
if !slices.Equal(fake.approved, []string{"a", "b", "c", "d"}) || !strings.Contains(said, "retired d, b, a, c") {
|
||||
t.Fatalf("approved %v; said %s", fake.approved, said)
|
||||
}
|
||||
acts := handActsBy(t, conn, "retire approve")
|
||||
if len(acts) == 0 || acts[len(acts)-1].Cause != kindRetireWaiting ||
|
||||
acts[len(acts)-1].Condition != "provider.pg-approve.anchor.retire" {
|
||||
t.Fatalf("the approval is not in the hand-act log: %+v", acts)
|
||||
}
|
||||
|
||||
// Reject, on another provider: the same set, and nothing approved.
|
||||
other := &fakeProvider{state: fake.state}
|
||||
other.serve(t, conn, "pg-reject", "anchor")
|
||||
printed(t, func() error {
|
||||
return answerRetirement(t.Context(), conn, providerInstance{Node: "anchor", Module: "pg-reject"}, false,
|
||||
whyFlags(t, "still moving"))
|
||||
})
|
||||
if !slices.Equal(other.rejected, []string{"a", "b", "c", "d"}) || other.approved != nil {
|
||||
t.Fatalf("rejected %v approved %v", other.rejected, other.approved)
|
||||
}
|
||||
|
||||
// Nothing waiting: refused, nothing asked but the question.
|
||||
idle := &fakeProvider{}
|
||||
idle.serve(t, conn, "pg-idle", "anchor")
|
||||
if err := answerRetirement(t.Context(), conn, providerInstance{Node: "anchor", Module: "pg-idle"}, true,
|
||||
whyFlags(t, "x")); err == nil || !strings.Contains(err.Error(), "waits for nobody") {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
if len(idle.asked) != 1 {
|
||||
t.Fatalf("an idle provider was asked more than its state: %+v", idle.asked)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNatsCleanupDeletesOnlyTheNamedRetiredConsumer(t *testing.T) {
|
||||
conn := onATestBus(t)
|
||||
fake := &fakeProvider{}
|
||||
fake.state.Held = []string{"active"}
|
||||
fake.state.Retired = []link.RetiredConsumer{retiredDaysAgo("old", 40), retiredDaysAgo("young", 2)}
|
||||
fake.serve(t, conn, "pg-clean", "anchor")
|
||||
p := providerInstance{Node: "anchor", Module: "pg-clean"}
|
||||
said := printed(t, func() error { return deleteRetired(t.Context(), conn, p, "old", whyFlags(t, "not needed")) })
|
||||
if !slices.Equal(fake.deleted, []string{"old"}) || !strings.Contains(said, "deleted old (1.0 KB freed)") {
|
||||
t.Fatalf("deleted %v; said %s", fake.deleted, said)
|
||||
}
|
||||
// An active consumer, or one it does not hold, is refused before the provider is asked to delete.
|
||||
for _, name := range []string{"active", "nobody"} {
|
||||
if err := deleteRetired(t.Context(), conn, p, name, whyFlags(t, "x")); err == nil ||
|
||||
!strings.Contains(err.Error(), "only a retired consumer is deleted") {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(fake.deleted, []string{"old"}) {
|
||||
t.Fatalf("more was deleted: %v", fake.deleted)
|
||||
}
|
||||
if acts := handActsBy(t, conn, "cleanup delete"); len(acts) == 0 || acts[len(acts)-1].Args[2] != "old" {
|
||||
t.Fatalf("the deletion is not in the hand-act log: %+v", acts)
|
||||
}
|
||||
|
||||
// Older than: listed, and nothing deleted without confirm; with it, only the old one.
|
||||
listing := retiredOf(t.Context(), conn, []providerInstance{p}, time.Now())
|
||||
if len(listing.Retired) != 2 || listing.Retired[0].Consumer != "old" || listing.Retired[0].AgeDays != 40 {
|
||||
t.Fatalf("%+v", listing)
|
||||
}
|
||||
fake.deleted = nil
|
||||
said = printed(t, func() error { return deleteFrom(t.Context(), conn, listing, 30, false, whyFlags(t, "tidy")) })
|
||||
if fake.deleted != nil || !strings.Contains(said, "nothing was deleted: add --confirm") || !strings.Contains(said, "old") ||
|
||||
strings.Contains(said, "young") {
|
||||
t.Fatalf("deleted %v; said %s", fake.deleted, said)
|
||||
}
|
||||
printed(t, func() error { return deleteFrom(t.Context(), conn, listing, 30, true, whyFlags(t, "tidy")) })
|
||||
if !slices.Equal(fake.deleted, []string{"old"}) {
|
||||
t.Fatalf("confirmed, deleted %v", fake.deleted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNatsD11SaysCleanupWaitsAfterThirtyDays(t *testing.T) {
|
||||
conn := onATestBus(t)
|
||||
old := &fakeProvider{}
|
||||
old.state.Retired = []link.RetiredConsumer{retiredDaysAgo("mesh_a_letta", 31), retiredDaysAgo("fresh", 1)}
|
||||
old.serve(t, conn, "pg-d11-old", "anchor")
|
||||
young := &fakeProvider{}
|
||||
young.state.Retired = []link.RetiredConsumer{retiredDaysAgo("x", 29)}
|
||||
young.serve(t, conn, "pg-d11-young", "anchor")
|
||||
instances := []providerInstance{{Node: "anchor", Module: "pg-d11-old"}, {Node: "anchor", Module: "pg-d11-young"},
|
||||
// Nothing serves this one: a provider older than the question is passed over, not a failure.
|
||||
{Node: "anchor", Module: "pg-d11-predates"}}
|
||||
found, err := retiredTooLong(t.Context(), conn, instances, time.Now())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(found) != 1 || found[0].Key() != "provider.pg-d11-old.anchor.cleanup" || found[0].Kind != kindCleanupWaiting ||
|
||||
found[0].Severity != conditions.Warning || !strings.Contains(found[0].Summary, "mesh_a_letta (31 days") ||
|
||||
strings.Contains(found[0].Summary, "fresh") {
|
||||
t.Fatalf("%+v", found)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheRetireAndCleanupVerbsComposeTheirCommandLines(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
want string
|
||||
}{
|
||||
{"retire", map[string]any{}, "retire --json"},
|
||||
{"retire", map[string]any{"answer": "approve", "node": "anchor", "module": "postgres", "why": "moved"},
|
||||
"retire approve anchor postgres --why moved"},
|
||||
{"retire", map[string]any{"answer": "reject", "node": "anchor", "module": "postgres", "why": "no"},
|
||||
"retire reject anchor postgres --why no"},
|
||||
{"cleanup", map[string]any{}, "cleanup list --json"},
|
||||
{"cleanup", map[string]any{"node": "anchor", "module": "postgres", "consumer": "x", "why": "gone"},
|
||||
"cleanup delete anchor postgres x --why gone"},
|
||||
{"cleanup", map[string]any{"older-than": "30", "why": "tidy"}, "cleanup delete --older-than 30 --why tidy"},
|
||||
{"cleanup", map[string]any{"older-than": "30", "why": "tidy", "confirm": "true"},
|
||||
"cleanup delete --older-than 30 --why tidy --confirm"},
|
||||
} {
|
||||
argv, err := argvFor(c.verb, c.args)
|
||||
if err != nil || strings.Join(argv, " ") != c.want {
|
||||
t.Errorf("%s %v: %q %v, want %q", c.verb, c.args, argv, err, c.want)
|
||||
}
|
||||
}
|
||||
for _, args := range []map[string]any{
|
||||
{"answer": "approve", "node": "anchor", "module": "postgres"}, // no why
|
||||
{"answer": "maybe", "node": "anchor", "module": "postgres", "why": "x"},
|
||||
} {
|
||||
if _, err := argvFor("retire", args); err == nil {
|
||||
t.Errorf("retire %v was composed", args)
|
||||
}
|
||||
}
|
||||
for _, args := range []map[string]any{
|
||||
{"node": "anchor", "module": "postgres", "consumer": "x"}, // no why
|
||||
{"older-than": "30"},
|
||||
{"consumer": "x", "older-than": "30", "node": "a", "module": "b", "why": "y"},
|
||||
} {
|
||||
if _, err := argvFor("cleanup", args); err == nil {
|
||||
t.Errorf("cleanup %v was composed", args)
|
||||
}
|
||||
}
|
||||
if repairingCommand([]string{"cleanup", "delete", "a", "b", "c"}) == "" ||
|
||||
repairingCommand([]string{"retire", "approve", "a", "b"}) == "" || repairingCommand([]string{"retire"}) != "" {
|
||||
t.Error("the generic verb would let a retirement or a deletion through without a why")
|
||||
}
|
||||
}
|
||||
|
||||
// A mark-only provider's retired consumer keeps its access; the listing and the approval say so
|
||||
// rather than claiming it was disabled (ADR 0230).
|
||||
func TestNatsAMarkOnlyRetirementIsSaidAsSuch(t *testing.T) {
|
||||
conn := onATestBus(t)
|
||||
fake := &fakeProvider{}
|
||||
kept := retiredDaysAgo("ledger", 3)
|
||||
kept.Access = "kept"
|
||||
fake.state.Retired = []link.RetiredConsumer{kept}
|
||||
fake.state.RetiresBy = "mark-only"
|
||||
fake.state.Waiting = &link.RetirementWaiting{Consumers: []link.RetiredConsumer{{Consumer: "a"}, {Consumer: "b"}}, Held: 2}
|
||||
fake.serve(t, conn, "vault-mark", "anchor")
|
||||
p := providerInstance{Node: "anchor", Module: "vault-mark"}
|
||||
listing := retiredOf(t.Context(), conn, []providerInstance{p}, time.Now())
|
||||
said := printed(t, func() error { return printRetired(listing, false) })
|
||||
if !strings.Contains(said, "MARK ONLY") || listing.Retired[0].Access != "kept" {
|
||||
t.Fatalf("%s %+v", said, listing)
|
||||
}
|
||||
said = printed(t, func() error { return answerRetirement(t.Context(), conn, p, true, whyFlags(t, "gone")) })
|
||||
if !strings.Contains(said, "MARK ONLY") || strings.Contains(said, "access disabled") {
|
||||
t.Fatal(said)
|
||||
}
|
||||
}
|
||||
@@ -6,8 +6,6 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// rotateCommand replaces a credential and moves both ends together.
|
||||
@@ -35,16 +33,12 @@ func rotateCommand(ctx context.Context, args []string) error {
|
||||
// One consumer rather than all of them. Ordinary: a credential is suspected on one machine,
|
||||
// and rotating the other nine would be a great deal of disruption for one suspicion.
|
||||
only := set.String("consumer", "", "only this machine's credential, rather than every holder's")
|
||||
// One consuming module rather than every module on the machine. A machine runs many consumers
|
||||
// of one provision, each with its own credential; one module that leaked its credential (novox/hq
|
||||
// issue 268) is no reason to restart every other one on the machine.
|
||||
module := set.String("module", "", "only this consuming module's credential")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("rotate <provision> [--consumer <machine>] [--module <module>]")
|
||||
return errors.New("rotate <provision> [--consumer <machine>]")
|
||||
}
|
||||
provision := positionals[0]
|
||||
|
||||
@@ -59,12 +53,6 @@ func rotateCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
holders = ofModule(holders, *module)
|
||||
if len(holders) == 0 && *module != "" {
|
||||
return fmt.Errorf(
|
||||
"no module %s%s holds a credential for %q, so there is nothing to rotate. `plan <machine>` "+
|
||||
"says what a machine holds", *module, onMachine(*only), provision)
|
||||
}
|
||||
if len(holders) == 0 {
|
||||
// Said, not silent. "Nobody holds this" and "this did not run" must never look the same —
|
||||
// and a rotation somebody believes happened is worse than one they know did not.
|
||||
@@ -128,27 +116,6 @@ func rotateCommand(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ofModule is the holders whose consuming module is this one; all of them when none is named.
|
||||
func ofModule(holders []inventory.Holder, module string) []inventory.Holder {
|
||||
if module == "" {
|
||||
return holders
|
||||
}
|
||||
var out []inventory.Holder
|
||||
for _, h := range holders {
|
||||
if h.ConsumerModule == module {
|
||||
out = append(out, h)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func onMachine(machine string) string {
|
||||
if machine == "" {
|
||||
return ""
|
||||
}
|
||||
return " on " + machine
|
||||
}
|
||||
|
||||
// asLocal names the credential inside the consumer where it holds several (ADR 0094).
|
||||
func asLocal(local string) string {
|
||||
if local == "" {
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// One consuming module's credential, and not its neighbours' on the same machine (novox/hq issue
|
||||
// 268): a module that leaked its database password is no reason to restart every other consumer.
|
||||
func TestARotationNarrowedToAModuleTouchesOnlyThatModulesCredential(t *testing.T) {
|
||||
holders := []inventory.Holder{
|
||||
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "letta", Provider: "ace"},
|
||||
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "n8n", Provider: "ace"},
|
||||
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "letta", Local: "reader", Provider: "ace"},
|
||||
}
|
||||
got := ofModule(holders, "letta")
|
||||
if len(got) != 2 || got[0].ConsumerModule != "letta" || got[1].Local != "reader" {
|
||||
t.Fatalf("narrowed to letta: %+v", got)
|
||||
}
|
||||
if len(ofModule(holders, "")) != 3 {
|
||||
t.Fatal("no module named narrowed anyway")
|
||||
}
|
||||
if len(ofModule(holders, "absent")) != 0 {
|
||||
t.Fatal("a module holding nothing matched")
|
||||
}
|
||||
}
|
||||
@@ -9,11 +9,9 @@ import (
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
"os"
|
||||
"os/exec"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
@@ -241,12 +239,6 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
if len(argv) == 0 {
|
||||
return nil, errors.New("command names no command")
|
||||
}
|
||||
// The generic verb is no way round the hand-act log (novox/hq to-be 45 §7): a repair through
|
||||
// it says why, as it would through its own verb.
|
||||
if repair := repairingCommand(argv); repair != "" && !slices.ContainsFunc(argv, isWhyFlag) {
|
||||
return nil, fmt.Errorf("%s is a repair done by hand, and says why: add --why <text> to the command "+
|
||||
"line (recorded in the hand-act log). Nothing was done", repair)
|
||||
}
|
||||
return argv, nil
|
||||
case "tools":
|
||||
return nil, errors.New("tools is answered from the records, not by a command")
|
||||
@@ -288,18 +280,7 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
}
|
||||
for _, act := range []string{"stop", "close", "retry"} {
|
||||
if id := str(act); id != "" {
|
||||
argv := []string{"plans", act, id}
|
||||
if act == "retry" {
|
||||
return argv, nil
|
||||
}
|
||||
// Ending a plan by hand says why (novox/hq to-be 45 §7); the command refuses it without.
|
||||
if w := str("why"); w != "" {
|
||||
argv = append(argv, "--why", w)
|
||||
}
|
||||
if c := str("cause"); c != "" {
|
||||
argv = append(argv, "--cause", c)
|
||||
}
|
||||
return argv, nil
|
||||
return []string{"plans", act, id}, nil
|
||||
}
|
||||
}
|
||||
if id := str("id"); id != "" {
|
||||
@@ -374,155 +355,22 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
// Sent and not waited for: the asker reads `status` for what the machine did, which is
|
||||
// what a person at a shell does too. A tool call that blocked for a push's whole apply would
|
||||
// time out on every machine that takes a minute, and say nothing about the ones that did not.
|
||||
// A push through the seat is a push by hand, and says why (novox/hq to-be 45 §7).
|
||||
if err := need("why"); err != nil {
|
||||
return nil, fmt.Errorf("%w: a push by hand is a repair, recorded in the hand-act log with why", err)
|
||||
}
|
||||
why := []string{"--why", str("why")}
|
||||
if c := str("cause"); c != "" {
|
||||
why = append(why, "--cause", c)
|
||||
}
|
||||
if n := str("node"); n != "" {
|
||||
// behind is not read here: given with a machine, it is refused as passed over — naming
|
||||
// a machine and asking for every machine behind are two requests, and guessing one
|
||||
// would push a machine nobody named, or not push one somebody did.
|
||||
return append([]string{"push", n, "--wait", "0"}, why...), nil
|
||||
return []string{"push", n, "--wait", "0"}, nil
|
||||
}
|
||||
// No machine: the whole mesh, whether or not behind said so. The command's answer says it
|
||||
// first, so a caller who meant one machine reads that it was not one.
|
||||
on("behind")
|
||||
return append([]string{"push", "--behind", "--wait", "0"}, why...), nil
|
||||
case "hand-act":
|
||||
if err := need("what", "why", "cause"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argv := []string{"hand-act", "record", str("what"), "--why", str("why"), "--cause", str("cause")}
|
||||
if c := str("condition"); c != "" {
|
||||
argv = append(argv, "--condition", c)
|
||||
}
|
||||
return argv, nil
|
||||
case "hand-acts":
|
||||
argv := []string{"hand-acts", "--json"}
|
||||
if d := str("days"); d != "" {
|
||||
argv = append(argv, "--days", d)
|
||||
}
|
||||
return argv, nil
|
||||
case "healers":
|
||||
argv := []string{"healers", "--json"}
|
||||
if d := str("days"); d != "" {
|
||||
argv = append(argv, "--days", d)
|
||||
}
|
||||
return argv, nil
|
||||
case "durations":
|
||||
argv := []string{"durations", "--json"}
|
||||
if k := str("kind"); k != "" {
|
||||
argv = append(argv, "--kind", k)
|
||||
}
|
||||
if d := str("days"); d != "" {
|
||||
argv = append(argv, "--days", d)
|
||||
}
|
||||
return argv, nil
|
||||
case "conditions":
|
||||
// One verb, four shapes, as `plans` (novox/hq to-be 45 §2): a silence, one condition, the
|
||||
// history, or the open ones filtered.
|
||||
if key := str("silence"); key != "" {
|
||||
if err := need("for", "why"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argv := []string{"conditions", "silence", key, "--for", str("for"), "--why", str("why")}
|
||||
if c := str("cause"); c != "" {
|
||||
argv = append(argv, "--cause", c)
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
if on("history") {
|
||||
argv := []string{"conditions", "history", "--json"}
|
||||
if d := str("days"); d != "" {
|
||||
argv = append(argv, "--days", d)
|
||||
}
|
||||
if k := str("key"); k != "" {
|
||||
argv = append(argv, "--key", k)
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
if k := str("key"); k != "" {
|
||||
return []string{"conditions", "show", k, "--json"}, nil
|
||||
}
|
||||
argv := []string{"conditions", "--json"}
|
||||
for _, filter := range []string{"scope", "severity", "machine"} {
|
||||
if v := str(filter); v != "" {
|
||||
argv = append(argv, "--"+filter, v)
|
||||
}
|
||||
}
|
||||
return argv, nil
|
||||
case "retire":
|
||||
answer := str("answer")
|
||||
if answer == "" {
|
||||
return []string{"retire", "--json"}, nil
|
||||
}
|
||||
if answer != "approve" && answer != "reject" {
|
||||
return nil, fmt.Errorf("retire answers approve or reject, not %q", answer)
|
||||
}
|
||||
if err := need("node", "module", "why"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argv := []string{"retire", answer, str("node"), str("module"), "--why", str("why")}
|
||||
if c := str("cause"); c != "" {
|
||||
argv = append(argv, "--cause", c)
|
||||
}
|
||||
return argv, nil
|
||||
case "cleanup":
|
||||
consumer, older := str("consumer"), str("older-than")
|
||||
switch {
|
||||
case consumer != "" && older != "":
|
||||
return nil, errors.New("cleanup deletes one consumer or those older than some days, not both")
|
||||
case consumer != "":
|
||||
if err := need("node", "module", "why"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argv := []string{"cleanup", "delete", str("node"), str("module"), consumer, "--why", str("why")}
|
||||
if c := str("cause"); c != "" {
|
||||
argv = append(argv, "--cause", c)
|
||||
}
|
||||
return argv, nil
|
||||
case older != "":
|
||||
if err := need("why"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argv := []string{"cleanup", "delete", "--older-than", older, "--why", str("why")}
|
||||
if on("confirm") {
|
||||
argv = append(argv, "--confirm")
|
||||
}
|
||||
if c := str("cause"); c != "" {
|
||||
argv = append(argv, "--cause", c)
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
return []string{"cleanup", "list", "--json"}, nil
|
||||
case "doctor":
|
||||
which := 0
|
||||
argv := []string{"doctor"}
|
||||
for _, sub := range []string{"run", "probes", "signals"} {
|
||||
if on(sub) {
|
||||
which++
|
||||
argv = append(argv, sub)
|
||||
}
|
||||
}
|
||||
if which > 1 {
|
||||
return nil, errors.New("doctor answers one of run, probes or signals at a time")
|
||||
}
|
||||
return append(argv, "--json"), nil
|
||||
return []string{"push", "--behind", "--wait", "0"}, nil
|
||||
case "rotate":
|
||||
if p := str("provision"); p != "" {
|
||||
argv := []string{"rotate", p}
|
||||
if c := str("consumer"); c != "" {
|
||||
argv = append(argv, "--consumer", c)
|
||||
}
|
||||
// With a provision, module narrows to one consuming module (novox/hq issue 268); node
|
||||
// and secret stay the other shape's, and are refused as passed over.
|
||||
if m := str("module"); m != "" {
|
||||
argv = append(argv, "--module", m)
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
_, node := a.given["node"]
|
||||
@@ -532,15 +380,7 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
if err := need("node", "module", "secret"); err != nil {
|
||||
return nil, fmt.Errorf("%w: a module's own secret is named by node, module and secret together", err)
|
||||
}
|
||||
argv := []string{"secret", "rotate", str("node"), str("module"), str("secret")}
|
||||
// Why, recorded in the hand-act log (novox/hq ADR 0228); a cause only beside a why.
|
||||
if w := str("why"); w != "" {
|
||||
argv = append(argv, "--why", w)
|
||||
if c := str("cause"); c != "" {
|
||||
argv = append(argv, "--cause", c)
|
||||
}
|
||||
}
|
||||
return argv, nil
|
||||
return []string{"secret", "rotate", str("node"), str("module"), str("secret")}, nil
|
||||
}
|
||||
// Neither shape: the command says its usage, which names both, and that is the answer the
|
||||
// caller needs.
|
||||
@@ -597,34 +437,7 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
}
|
||||
|
||||
// jsonVerbs are the verbs whose command speaks JSON, so the answer carries it as data as well.
|
||||
var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true,
|
||||
"hand-acts": true, "durations": true, "conditions": true, "doctor": true, "retire": true, "cleanup": true}
|
||||
|
||||
// repairingCommand names a command line that repairs by hand, and so says why: a push, a plan stopped
|
||||
// or closed, a consumer re-made (novox/hq to-be 45 §7). Empty for any other.
|
||||
func repairingCommand(argv []string) string {
|
||||
switch {
|
||||
case argv[0] == "push":
|
||||
return "push"
|
||||
case argv[0] == "plans" && len(argv) > 1 && (argv[1] == "stop" || argv[1] == "close"):
|
||||
return "plans " + argv[1]
|
||||
case argv[0] == "broker" && len(argv) > 1 && argv[1] == "consumer-reset":
|
||||
return "broker consumer-reset"
|
||||
case argv[0] == "hand-act":
|
||||
return "hand-act record"
|
||||
case argv[0] == "conditions" && len(argv) > 1 && argv[1] == "silence":
|
||||
return "conditions silence"
|
||||
case argv[0] == "retire" && len(argv) > 1 && (argv[1] == "approve" || argv[1] == "reject"):
|
||||
return "retire " + argv[1]
|
||||
case argv[0] == "cleanup" && len(argv) > 1 && argv[1] == "delete":
|
||||
return "cleanup delete"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func isWhyFlag(word string) bool {
|
||||
return word == "--why" || word == "-why" || strings.HasPrefix(word, "--why=") || strings.HasPrefix(word, "-why=")
|
||||
}
|
||||
var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true}
|
||||
|
||||
// runVerb runs this binary with the given command line and gathers what it said.
|
||||
func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
||||
@@ -636,12 +449,6 @@ func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
||||
// The same environment: the stores' credentials, the bus, the broker — everything a command run
|
||||
// from a shell in this container would have, because it is that.
|
||||
cmd.Env = os.Environ()
|
||||
// And who asked, so an act it does by hand is recorded as theirs (novox/hq to-be 45 §7).
|
||||
caller := link.CallerIn(ctx)
|
||||
if caller == "" {
|
||||
caller = "a seat call whose caller the bus did not name"
|
||||
}
|
||||
cmd.Env = append(cmd.Env, link.CallerVar+"="+caller+", through the "+catalogue.ControllerSeatName+" seat")
|
||||
// Two buffers, one answer. What the command *says* is both streams, in the order a person at
|
||||
// a shell would read them; what it *answers as data* is standard output alone — `status --json`
|
||||
// prints its warnings beside the document, and a JSON parsed from the two together parsed
|
||||
@@ -694,19 +501,6 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
||||
if verb == "calls" {
|
||||
return callsAnswer(link.Calls, a.given["call"])
|
||||
}
|
||||
if verb == "doctor" {
|
||||
// From the serving controller, which runs the self-check and hears the signals
|
||||
// (novox/hq to-be 45 §4): the last verdict at once, or a run now.
|
||||
argv, err := a.commandLine()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sub := ""
|
||||
if len(argv) > 2 {
|
||||
sub = argv[1]
|
||||
}
|
||||
return doctorAnswer(ctx, sub)
|
||||
}
|
||||
return seatTools(), nil
|
||||
}
|
||||
continue
|
||||
@@ -745,14 +539,6 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if verb == "status" && statusFrom != nil {
|
||||
// At once, from the summary the serving controller keeps (novox/hq to-be 45 Phase 0).
|
||||
return statusFrom.answer(ctx)
|
||||
}
|
||||
if !readingVerbs[verb] && !(verb == "plans" && !actsOnAPlan(args)) {
|
||||
// Whatever it did, `status` is composed again once it has.
|
||||
defer statusFrom.nudge()
|
||||
}
|
||||
if answersFirst(argv) {
|
||||
// Before anything is sent: a push sends the bus's own machine first, and a broker
|
||||
// reloading its user list forgets the answer it was about to permit (novox/hq issue 265).
|
||||
@@ -764,19 +550,9 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
||||
return handlers, behind, nil
|
||||
}
|
||||
|
||||
// actsOnAPlan is `plans` asked to stop, close or retry one rather than to show them.
|
||||
func actsOnAPlan(args map[string]any) bool {
|
||||
for _, act := range []string{"stop", "close", "retry"} {
|
||||
if v, _ := args[act].(string); strings.TrimSpace(v) != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// inProcess are the verbs answered by this process rather than by a command it runs: `tools` from
|
||||
// the records, `calls` from what this process served.
|
||||
var inProcess = map[string]bool{"tools": true, "calls": true, "doctor": true}
|
||||
var inProcess = map[string]bool{"tools": true, "calls": true}
|
||||
|
||||
// answersFirst is a command line whose caller is answered before it runs: a push, by its verb or
|
||||
// through `command`. A push sends the machine holding the bus first when its user list changed, the
|
||||
@@ -787,41 +563,22 @@ func answersFirst(argv []string) bool {
|
||||
}
|
||||
|
||||
// callsAnswer is what `calls` answers: the kept calls, newest first, without their answers — or
|
||||
// one call whole. Kept on the bus, so a call a controller before this one served is answered too
|
||||
// (novox/hq to-be 45 §6); where the bus cannot be read, what this process served is answered and
|
||||
// the reason said beside it.
|
||||
// one call whole.
|
||||
func callsAnswer(log *link.CallLog, id string) (any, error) {
|
||||
if id != "" {
|
||||
c, ok, err := log.Get(id)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("call %s is not in this controller's memory, and the calls kept on the "+
|
||||
"bus could not be read: %w", id, err)
|
||||
}
|
||||
c, ok := log.Get(id)
|
||||
if !ok {
|
||||
if log.IsDurable() {
|
||||
return nil, fmt.Errorf("no call %s is kept: the bus keeps the last %d calls, or %s, and this "+
|
||||
"is not among them — `calls` lists them", id, broker.KeptCallsDurably, broker.CallsKeptFor)
|
||||
}
|
||||
return nil, fmt.Errorf("no call %s is kept here: calls are kept by the controller that "+
|
||||
"answered them, the last %d, and not across a restart — `calls` lists them", id, link.KeptCalls)
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
recent, err := log.Recent()
|
||||
recent := log.Recent()
|
||||
for i := range recent {
|
||||
recent[i].Answer = nil
|
||||
}
|
||||
answer := map[string]any{"calls": recent, "note": "newest first; `calls` with a call's id gives its whole answer"}
|
||||
if log.IsDurable() {
|
||||
answer["kept"] = fmt.Sprintf("the last %d calls, or %s, on the bus — across a restart of the controller",
|
||||
broker.KeptCallsDurably, broker.CallsKeptFor)
|
||||
} else {
|
||||
answer["kept"] = fmt.Sprintf("the last %d calls this controller served, in its memory only", link.KeptCalls)
|
||||
}
|
||||
if err != nil {
|
||||
answer["unread"] = err.Error()
|
||||
}
|
||||
return answer, nil
|
||||
return map[string]any{"calls": recent, "kept": link.KeptCalls,
|
||||
"note": "newest first; `calls` with a call's id gives its whole answer"}, nil
|
||||
}
|
||||
|
||||
// seatTools is what `tools` answers: every seat with a protocol, and the tools each serves, from the
|
||||
|
||||
@@ -145,17 +145,17 @@ func TestAnArgumentAVerbDoesNotDeclareIsRefused(t *testing.T) {
|
||||
// The push that was the cause: a machine named is that machine; none named is the whole mesh, and
|
||||
// naming one beside behind is refused rather than one of the two guessed.
|
||||
func TestAPushIsOneMachineOrSaysItIsTheWholeMesh(t *testing.T) {
|
||||
argv, err := argvFor("push", map[string]any{"node": "g1", "why": "w"})
|
||||
if err != nil || strings.Join(argv, " ") != "push g1 --wait 0 --why w" {
|
||||
argv, err := argvFor("push", map[string]any{"node": "g1"})
|
||||
if err != nil || strings.Join(argv, " ") != "push g1 --wait 0" {
|
||||
t.Fatalf("a named push: %v %v", argv, err)
|
||||
}
|
||||
for _, args := range []map[string]any{{"why": "w"}, {"behind": "true", "why": "w"}} {
|
||||
for _, args := range []map[string]any{{}, {"behind": "true"}} {
|
||||
argv, err := argvFor("push", args)
|
||||
if err != nil || strings.Join(argv, " ") != "push --behind --wait 0 --why w" {
|
||||
if err != nil || strings.Join(argv, " ") != "push --behind --wait 0" {
|
||||
t.Fatalf("a push of the whole mesh %v: %v %v", args, argv, err)
|
||||
}
|
||||
}
|
||||
if _, err := argvFor("push", map[string]any{"node": "g1", "behind": "true", "why": "w"}); err == nil ||
|
||||
if _, err := argvFor("push", map[string]any{"node": "g1", "behind": "true"}); err == nil ||
|
||||
!strings.Contains(err.Error(), `"behind"`) {
|
||||
t.Fatalf("a named push with behind was taken: %v", err)
|
||||
}
|
||||
@@ -268,17 +268,6 @@ var accountedFlags = map[string]map[string]string{
|
||||
},
|
||||
"builds": {"n": "=limit"},
|
||||
"plans": {"n": "=limit", "what-if": "=repository"},
|
||||
"durations": {
|
||||
"json": "set by the verb: the answer is data",
|
||||
"all": "withheld: every measurement of a fortnight is more than a call should carry; `command` reaches it",
|
||||
},
|
||||
"hand-acts": {"json": "set by the verb: the answer is data"},
|
||||
"conditions": {"json": "set by the verb: the answer is data"},
|
||||
"retire": {"json": "set by the verb: the answer is data"},
|
||||
"cleanup": {"json": "set by the verb: the answer is data"},
|
||||
"conditions history": {"json": "set by the verb: the answer is data"},
|
||||
"conditions show": {"json": "set by the verb: the answer is data"},
|
||||
"healers": {"json": "set by the verb: the answer is data"},
|
||||
}
|
||||
|
||||
// **Every flag of the command a verb runs is in the verb's schema, or accounted for here.** Derived
|
||||
|
||||
@@ -43,10 +43,6 @@ func TestRotateTakesAProvisionOrAnOwnSecret(t *testing.T) {
|
||||
if strings.Join(argv, " ") != "rotate postgres-database --consumer ace" {
|
||||
t.Fatalf("a pair credential: %v", argv)
|
||||
}
|
||||
argv, _ = argvFor("rotate", map[string]any{"provision": "postgres-database", "consumer": "ace", "module": "letta"})
|
||||
if strings.Join(argv, " ") != "rotate postgres-database --consumer ace --module letta" {
|
||||
t.Fatalf("one consuming module's pair credential: %v", argv)
|
||||
}
|
||||
argv, _ = argvFor("rotate", map[string]any{"node": "ace", "module": "nodered", "secret": "api-token"})
|
||||
if strings.Join(argv, " ") != "secret rotate ace nodered api-token" {
|
||||
t.Fatalf("an own secret: %v", argv)
|
||||
@@ -107,8 +103,8 @@ func TestAVerbMissingWhatItNeedsIsRefused(t *testing.T) {
|
||||
// A push and a build are sent, not waited for: the asker reads status, or the build's log by its
|
||||
// id, for what happened. A repository given as a forge path is said to be one (issue 176).
|
||||
func TestActsDoNotBlockTheCall(t *testing.T) {
|
||||
argv, _ := argvFor("push", map[string]any{"node": "one", "why": "w"})
|
||||
if strings.Join(argv, " ") != "push one --wait 0 --why w" {
|
||||
argv, _ := argvFor("push", map[string]any{"node": "one"})
|
||||
if strings.Join(argv, " ") != "push one --wait 0" {
|
||||
t.Fatalf("push waits: %v", argv)
|
||||
}
|
||||
argv, _ = argvFor("build", map[string]any{"repository": "novox/x", "path": "modules/x"})
|
||||
|
||||
@@ -93,31 +93,18 @@ func secretCommand(ctx context.Context, args []string) error {
|
||||
fmt.Printf(" run `push %s` and `push %s` to send it\n", *provider, node)
|
||||
return nil
|
||||
}
|
||||
untilStart, err := open.inventory.AcceptGivenSecret(ctx, node, module, name, value)
|
||||
if err != nil {
|
||||
if err := open.inventory.AcceptSecretForModule(ctx, node, module, name, value); err != nil {
|
||||
return err
|
||||
}
|
||||
// Not printed back, and there is nowhere it could be printed from: it is sealed to that
|
||||
// machine and the mesh cannot read it again.
|
||||
fmt.Printf("%s on %s now holds %q, sealed to that machine.\n", module, node, name)
|
||||
if untilStart {
|
||||
// Accepted, and said what it is for (novox/hq ADR 0228): a value given by hand adopts
|
||||
// something that already holds it, and lives only until the module has started on it.
|
||||
fmt.Printf(" it lives until %s next starts well under the mesh on it, and is then replaced with a value\n"+
|
||||
" the mesh makes, sealed and sent (ADR 0228): a value given by hand is for adopting something\n"+
|
||||
" already running that holds it\n", module)
|
||||
if record, err := open.inventory.NodeByName(ctx, node); err == nil && !record.Adopted {
|
||||
fmt.Printf(" %s is not an adopted machine: if %s is installed fresh there, it needs no given value —\n"+
|
||||
" the mesh makes one at the first push\n", node, module)
|
||||
}
|
||||
} else {
|
||||
fmt.Printf(" the mesh cannot read it back, and will not replace it with one of its own\n")
|
||||
}
|
||||
fmt.Printf(" the mesh cannot read it back, and will not replace it with one of its own\n")
|
||||
fmt.Printf(" run `push %s` to send it\n", node)
|
||||
return nil
|
||||
}
|
||||
|
||||
const secretUsage = "secret rotate <node> <module> <name> [--why <text> [--cause <word>]]\n" +
|
||||
const secretUsage = "secret rotate <node> <module> <name>\n" +
|
||||
"secret accept <node> <module> <name> [--from <file>] [--provider <node> [--local <name>]]\n" +
|
||||
"secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>] [--provider <node>]\n" +
|
||||
"secret export [--out <file>]"
|
||||
@@ -380,20 +367,9 @@ func valueFor(node, module, name, from string) (string, error) {
|
||||
// secretRotate makes a module's own secret anew and sends the machine, so the module starts again on
|
||||
// the new value (novox/hq ADR 0114, issue 180). A pair credential rotates with `rotate <provision>`;
|
||||
// this is the secret with one party. Said in the log with who asked and when, never the value.
|
||||
//
|
||||
// **A value given to the mesh rotates the same way** (novox/hq ADR 0228): what a module reads at start
|
||||
// is held by nobody else, so the old value is not needed to replace it. Refused for a value an
|
||||
// outside party issued, which no value of the mesh's would replace. Why it was rotated is recorded in
|
||||
// the hand-act log when given — a rotation asked by a person is an act by hand, and a leak is a cause
|
||||
// worth counting.
|
||||
func secretRotate(ctx context.Context, args []string) error {
|
||||
rest, flags := split(args)
|
||||
set := flag.NewFlagSet("secret rotate", flag.ContinueOnError)
|
||||
why := addHandActFlags(set)
|
||||
if err := set.Parse(flags); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(rest) != 3 || set.NArg() != 0 {
|
||||
rest, _ := split(args)
|
||||
if len(rest) != 3 {
|
||||
return errors.New(secretUsage)
|
||||
}
|
||||
node, module, name := rest[0], rest[1], rest[2]
|
||||
@@ -402,10 +378,6 @@ func secretRotate(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
origin, given, err := open.inventory.OwnSecretOrigin(ctx, node, module, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := open.inventory.RotateModuleSecret(ctx, node, module, name); err != nil {
|
||||
var refused inventory.ErrNotRotatable
|
||||
if errors.As(err, &refused) {
|
||||
@@ -413,13 +385,8 @@ func secretRotate(ctx context.Context, args []string) error {
|
||||
}
|
||||
return err
|
||||
}
|
||||
why.record(ctx, "secret rotate", []string{node, module, name})
|
||||
fmt.Printf("rotated %q of %s on %s at %s, asked by %s; the value is sealed and not shown\n",
|
||||
name, module, node, time.Now().UTC().Format(time.RFC3339), whoAsked())
|
||||
if origin == inventory.OriginAccepted {
|
||||
fmt.Printf(" it replaces the value given to the mesh on %s: the mesh made this one, so `secret rotate` "+
|
||||
"replaces it again whenever asked (ADR 0228)\n", given.UTC().Format("2006-01-02"))
|
||||
}
|
||||
// A shared credential (ADR 0158) has as many holders as the provision has consumers, and all
|
||||
// of them are sent in one act, so no machine is left reading a value the provider no longer takes.
|
||||
machines, err := open.inventory.SharedHolders(ctx, node, module, name)
|
||||
|
||||
@@ -22,11 +22,6 @@ type sendable struct {
|
||||
// under the node's hold just before the body is made (novox/hq 04-ISSUES/107). Zero is not sent
|
||||
// at all, which a host reads as "no order claimed" — the shape of every declaration before this.
|
||||
Sequence int64
|
||||
// Epoch is the controller lease's epoch it was composed under (novox/hq to-be 45 §6): a machine that
|
||||
// heard a later epoch refuses it. Zero is not sent at all — every machine whose node-engine has not
|
||||
// said it reads one is sent none, because an older node-engine refuses a key it does not know, whole
|
||||
// (link/order.go, the contract).
|
||||
Epoch uint64
|
||||
// Adoption is nil for a converged node, and then the body is byte for byte what it was before
|
||||
// adoption existed: an older host parses the envelope strictly and would refuse the key.
|
||||
Adoption *adoptionEnvelope
|
||||
@@ -48,9 +43,6 @@ type sendable struct {
|
||||
LeftOut []string
|
||||
// leftOutWhy is why each was, for push and plan to say; never on the wire.
|
||||
leftOutWhy map[string]string
|
||||
// withheld is every consumer this machine's grants leave out, because its identity overflows the
|
||||
// provision's bound (novox/hq ADR 0225); for push and plan to say, never on the wire.
|
||||
withheld []catalogue.Overflow
|
||||
// Builds is the build of each module this declaration carries — module to the commit its build
|
||||
// was made from — recorded with the send and never on the wire (novox/hq issue 259, ADR 0221).
|
||||
// Composed only on the send path; nil records that it is not known.
|
||||
@@ -75,9 +67,6 @@ func (s sendable) Body() ([]byte, error) {
|
||||
if s.Sequence > 0 {
|
||||
envelope["sequence"] = s.Sequence
|
||||
}
|
||||
if s.Epoch > 0 {
|
||||
envelope["epoch"] = s.Epoch
|
||||
}
|
||||
if len(s.LeftOut) > 0 {
|
||||
envelope["left_out"] = s.LeftOut
|
||||
}
|
||||
|
||||
@@ -1,645 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The signals table (novox/hq to-be 45 §3, ADR 0227 rule 5), compiled in.
|
||||
//
|
||||
// **Every signal the core expects has a watchdog; its absence is a condition.** A row says what is
|
||||
// expected, from whom, after what, within what bound, and what is raised when it does not come. One
|
||||
// table, read three ways: by the watchdog loop (watchdogs.go), which runs every row's watch over the
|
||||
// facts it gathered; by `doctor signals`, which says the age of every row's newest signal; and by the
|
||||
// test generated from it (signals_test.go), which suppresses each signal in turn and asserts its
|
||||
// condition — so a row added without a watch, or a watch that does not fire, fails the build.
|
||||
//
|
||||
// A row not watched yet says why, and in which phase it will be: a watchdog of a signal nothing emits
|
||||
// would be a condition that can only cry wolf. Bounds marked provisional are set from what Phase 0
|
||||
// measured (`durations`) and corrected in Phase 1's first live week; a corrected bound is a change to
|
||||
// this table, reviewed like code.
|
||||
|
||||
// signalRow is one row of the table.
|
||||
type signalRow struct {
|
||||
Row string
|
||||
Signal string
|
||||
Emitter string
|
||||
Trigger string
|
||||
Bound string
|
||||
Kind string
|
||||
Severity conditions.Severity
|
||||
// Phase is the phase of to-be 45 its watchdog is built in.
|
||||
Phase int
|
||||
// Deferred says why it is not watched yet; empty for a row that is.
|
||||
Deferred string
|
||||
// needs is the part of the facts the row reads: an error there is the row blind, which is
|
||||
// itself said (probe-failed) rather than read as nothing wrong.
|
||||
needs func(f *signalFacts) error
|
||||
// watch is what is wrong now, from the facts.
|
||||
watch func(f *signalFacts) []conditions.Observation
|
||||
// newest is the time of the newest signal of this row, for `doctor signals`; zero when none.
|
||||
newest func(f *signalFacts) time.Time
|
||||
}
|
||||
|
||||
// The bounds, provisional where to-be 45 says so.
|
||||
const (
|
||||
// heartbeatEvery is the interval a node-engine or node tools that say none have always used.
|
||||
heartbeatEvery = 60 * time.Second
|
||||
// heartbeatsMissed is how many intervals may pass in silence (S1, S11).
|
||||
heartbeatsMissed = 3
|
||||
// controlNodeUrgentAfter is how long the control node may be silent before it is urgent (S1).
|
||||
controlNodeUrgentAfter = 30 * time.Minute
|
||||
// reportAtLeast is the least a machine is given to report a send (S2).
|
||||
reportAtLeast = 2 * time.Minute
|
||||
// tierAtLeast is the least a plan's tier is given (S3), the bound `status` calls a plan late at.
|
||||
tierAtLeast = planWaitBound
|
||||
// loopDeafAfter is how long the event loop may take nothing while its consumers hold some (S4).
|
||||
loopDeafAfter = 2 * time.Minute
|
||||
// askAtLeast is the least a build ask is given, and askDefault the bound while nothing is
|
||||
// measured (S6): the build seat declares no timeout of its own.
|
||||
askAtLeast = 20 * time.Minute
|
||||
askDefault = time.Hour
|
||||
// callDefault is the bound of a verb that declares none (S7); push's and build's are longer.
|
||||
callDefault = 10 * time.Minute
|
||||
// advisoryQuiet is how long the bus must be quiet about a thing before its advisory clears (S9).
|
||||
advisoryQuiet = time.Hour
|
||||
// staleRefusalsAllowed in staleRefusalsWithin are what S13 lets pass from one writer.
|
||||
staleRefusalsAllowed = 5
|
||||
staleRefusalsWithin = 5 * time.Minute
|
||||
// leaseBound is how long the lease may go unrenewed (S12): the key's age.
|
||||
leaseBound = broker.LeaseTTL
|
||||
)
|
||||
|
||||
// callBounds are the verbs that may run longer than callDefault, and how long (S7).
|
||||
var callBounds = map[string]time.Duration{
|
||||
"push": 30 * time.Minute, "rotate": 30 * time.Minute, "assign": 15 * time.Minute,
|
||||
"unassign": 15 * time.Minute, "command": 30 * time.Minute, "doctor": 3 * time.Minute,
|
||||
}
|
||||
|
||||
// callBound is a verb's bound.
|
||||
func callBound(verb string) time.Duration {
|
||||
if b, ok := callBounds[verb]; ok {
|
||||
return b
|
||||
}
|
||||
return callDefault
|
||||
}
|
||||
|
||||
// signalsTable is the table, in to-be 45's order.
|
||||
var signalsTable = []signalRow{
|
||||
{Row: "S1", Signal: "machine heartbeat", Emitter: "node-engine", Trigger: "its interval",
|
||||
Bound: "3 × the interval it says (60 s when it says none), provisional; not raised while the machine " +
|
||||
"said it is asleep or shutting down (ADR 0211); urgent after 30 min for a control node",
|
||||
Kind: "silent", Severity: conditions.Warning, Phase: 1,
|
||||
needs: func(f *signalFacts) error { return f.machinesErr }, watch: watchHeartbeats,
|
||||
newest: func(f *signalFacts) time.Time {
|
||||
return newestOf(f.machines, func(m machineFacts) time.Time { return m.lastHeard })
|
||||
}},
|
||||
{Row: "S2", Signal: "report after a send", Emitter: "node-engine", Trigger: "each declaration sent",
|
||||
Bound: "max(2 min, 3 × that machine's last apply duration), provisional; not while the machine is silent or asleep",
|
||||
Kind: "sent-not-reported", Severity: conditions.Warning, Phase: 1,
|
||||
needs: func(f *signalFacts) error { return f.machinesErr }, watch: watchReports,
|
||||
newest: func(f *signalFacts) time.Time {
|
||||
return newestOf(f.machines, func(m machineFacts) time.Time { return m.reportedAt })
|
||||
}},
|
||||
{Row: "S3", Signal: "plan tier progress", Emitter: "controller's plan", Trigger: "each tier entered",
|
||||
Bound: "max(30 min, 3 × the p90 of that repository's measured tiers), provisional; not while the " +
|
||||
"build seat is paused under it",
|
||||
Kind: "stalled", Severity: conditions.Warning, Phase: 1,
|
||||
needs: func(f *signalFacts) error { return f.plansErr }, watch: watchPlans,
|
||||
newest: func(f *signalFacts) time.Time {
|
||||
return newestOf(f.plans, func(p planFacts) time.Time { return p.entered })
|
||||
}},
|
||||
{Row: "S4", Signal: "the controller's event loop takes a message", Emitter: "controller",
|
||||
Trigger: "while its consumers have pending messages", Bound: "2 min",
|
||||
Kind: "controller-deaf", Severity: conditions.Urgent, Phase: 1,
|
||||
needs: func(f *signalFacts) error { return f.loopErr }, watch: watchLoop,
|
||||
newest: func(f *signalFacts) time.Time { return f.loop.took }},
|
||||
{Row: "S5", Signal: "a merge announced becomes a plan, or nothing reads it", Emitter: "announcer → controller",
|
||||
Trigger: "each merge", Bound: "10 min (the catch-up pass of issue 266)",
|
||||
Kind: "merge-not-acted", Severity: conditions.Urgent, Phase: 1,
|
||||
needs: func(f *signalFacts) error { return f.mergesErr }, watch: watchMerges,
|
||||
newest: func(f *signalFacts) time.Time { return f.mergesPassed }},
|
||||
{Row: "S6", Signal: "a build asked → its outcome", Emitter: "build seat", Trigger: "each ask",
|
||||
Bound: "max(20 min, 3 × the p90 of measured builds), 1 h while nothing is measured, provisional — the " +
|
||||
"build seat declares no timeout; and an ask the queue gave up on (dead) at once",
|
||||
Kind: "ask-lost", Severity: conditions.Warning, Phase: 1,
|
||||
needs: func(f *signalFacts) error { return f.asksErr }, watch: watchAsks,
|
||||
newest: func(f *signalFacts) time.Time { return newestOf(f.asks, func(a askFacts) time.Time { return a.since }) }},
|
||||
{Row: "S7", Signal: "a call running → finished", Emitter: "controller", Trigger: "each call",
|
||||
Bound: "the verb's bound: push, rotate and command 30 min, assign and unassign 15 min, doctor 3 min, " +
|
||||
"any other 10 min",
|
||||
Kind: "call-hung", Severity: conditions.Warning, Phase: 1,
|
||||
needs: func(*signalFacts) error { return nil }, watch: watchCalls,
|
||||
newest: func(f *signalFacts) time.Time {
|
||||
return newestOf(f.calls, func(c link.Call) time.Time { return c.Started })
|
||||
}},
|
||||
{Row: "S8", Signal: "a provider's failing word repeated", Emitter: "provider",
|
||||
Trigger: "every 15 min while failing (ADR 0224)", Bound: "30 min",
|
||||
Kind: kindProviderSilent, Severity: conditions.Warning, Phase: 1,
|
||||
needs: func(f *signalFacts) error { return f.standingsErr }, watch: watchProviders,
|
||||
newest: func(f *signalFacts) time.Time {
|
||||
return newestOf(f.standings, func(c conditions.Condition) time.Time { return c.LastObserved })
|
||||
}},
|
||||
{Row: "S9", Signal: "bus advisories: maximum deliveries, consumer deleted; the controller's own slow " +
|
||||
"consumer and refused subjects", Emitter: "bus server's advisory subjects; the controller's connection",
|
||||
Trigger: "any", Bound: "any occurrence; clears after an hour without another, and a deleted consumer " +
|
||||
"once it exists again or the mesh no longer expects it",
|
||||
Kind: "slow-consumer, max-deliveries, refused, consumer-lost", Severity: conditions.Warning, Phase: 1,
|
||||
needs: func(f *signalFacts) error { return f.advisoriesErr }, watch: watchAdvisories,
|
||||
newest: func(f *signalFacts) time.Time {
|
||||
return newestOf(f.advisories, func(a link.Advisory) time.Time { return a.Last })
|
||||
}},
|
||||
{Row: "S10", Signal: "the self-check's heartbeat", Emitter: "controller's doctor", Trigger: "every run",
|
||||
Bound: "2 × its interval; watched from a second machine by mesh-watcher, and here as well",
|
||||
Kind: "self-check-silent", Severity: conditions.Urgent, Phase: 1,
|
||||
needs: func(*signalFacts) error { return nil }, watch: watchSelfCheck,
|
||||
newest: func(f *signalFacts) time.Time { return f.selfCheck.last }},
|
||||
{Row: "S11", Signal: "node tools heartbeat", Emitter: "node tools", Trigger: "its interval",
|
||||
Bound: "3 × the interval it says (60 s when it says none), provisional; only where node-tools is assigned",
|
||||
Kind: "tools-silent", Severity: conditions.Warning, Phase: 1,
|
||||
needs: func(f *signalFacts) error { return f.machinesErr }, watch: watchTools,
|
||||
newest: func(f *signalFacts) time.Time {
|
||||
return newestOf(f.machines, func(m machineFacts) time.Time { return m.toolsHeard })
|
||||
}},
|
||||
{Row: "S12", Signal: "the controller lease renewed", Emitter: "controller", Trigger: "every 5 s",
|
||||
Bound: "15 s (the key's age); a holder that lost the lease, or stopped renewing and was taken over, and a " +
|
||||
"lease bucket found raised again from nothing, are said for an hour after; a controller serving without " +
|
||||
"the lease, for as long as it does",
|
||||
Kind: "lease-lost", Severity: conditions.Urgent, Phase: 2,
|
||||
needs: func(f *signalFacts) error { return f.leaseErr }, watch: watchLease,
|
||||
newest: func(f *signalFacts) time.Time { return f.lease.renewed }},
|
||||
{Row: "S13", Signal: "stale refusals", Emitter: "every receiver (rule 2)", Trigger: "each refusal",
|
||||
Bound: "more than 5 from one writer in 5 min: a controller epoch, a controller that claimed none, or a " +
|
||||
"machine's node-engine whose accounts the controller refused",
|
||||
Kind: "stale-writer", Severity: conditions.Warning, Phase: 2,
|
||||
needs: func(*signalFacts) error { return nil }, watch: watchStaleRefusals,
|
||||
newest: func(f *signalFacts) time.Time {
|
||||
return newestOf(f.staleRefusals, func(w link.WriterRefusals) time.Time { return w.Last })
|
||||
}},
|
||||
{Row: "S14", Signal: "facts snapshot exported", Emitter: "controller", Trigger: "daily",
|
||||
Bound: "2 days", Kind: "facts-stale", Severity: conditions.Warning, Phase: 5,
|
||||
Deferred: "the facts snapshot is built in Phase 5 (to-be 45 §9): nothing exports one yet"},
|
||||
{Row: "S15", Signal: "a hand act with a cause already recorded", Emitter: "hand-act log",
|
||||
Trigger: "each act", Bound: "the second within 14 days; clears when fewer than two remain within 14 days",
|
||||
Kind: "healer-wanted", Severity: conditions.Warning, Phase: 3,
|
||||
needs: func(f *signalFacts) error { return f.handActsErr }, watch: watchHandActs,
|
||||
newest: func(f *signalFacts) time.Time {
|
||||
return newestOf(f.handActs, func(a link.HandAct) time.Time { return a.At })
|
||||
}},
|
||||
}
|
||||
|
||||
// newestOf is the newest time among things.
|
||||
func newestOf[T any](list []T, at func(T) time.Time) time.Time {
|
||||
var newest time.Time
|
||||
for _, x := range list {
|
||||
if t := at(x); t.After(newest) {
|
||||
newest = t
|
||||
}
|
||||
}
|
||||
return newest
|
||||
}
|
||||
|
||||
// ago is a duration as the summaries say it.
|
||||
func ago(d time.Duration) string {
|
||||
if d < time.Minute {
|
||||
return d.Round(time.Second).String()
|
||||
}
|
||||
return d.Round(time.Minute).String()
|
||||
}
|
||||
|
||||
// heartbeatBound is a machine's S1 or S11 bound from the interval it says.
|
||||
func heartbeatBound(every time.Duration) time.Duration {
|
||||
if every <= 0 {
|
||||
every = heartbeatEvery
|
||||
}
|
||||
return heartbeatsMissed * every
|
||||
}
|
||||
|
||||
func watchHeartbeats(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, m := range f.machines {
|
||||
if m.lastHeard.IsZero() || m.asleep() {
|
||||
// Never heard is a machine that has not joined, which status says; asleep is not lost.
|
||||
continue
|
||||
}
|
||||
bound := heartbeatBound(m.every)
|
||||
silent := f.now.Sub(m.lastHeard)
|
||||
if silent <= bound {
|
||||
continue
|
||||
}
|
||||
severity := conditions.Warning
|
||||
if m.control && silent > controlNodeUrgentAfter {
|
||||
severity = conditions.Urgent
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name, Kind: "silent",
|
||||
Machine: m.name, Severity: severity,
|
||||
Summary: fmt.Sprintf("%s has not been heard from since %s (bound %s)", m.name,
|
||||
m.lastHeard.UTC().Format("2006-01-02 15:04 MST"), bound),
|
||||
Said: fmt.Sprintf("silent for %s", ago(silent))})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func watchReports(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, m := range f.machines {
|
||||
if m.sentAt.IsZero() || m.reportedCurrent || m.asleep() {
|
||||
continue
|
||||
}
|
||||
if !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) > heartbeatBound(m.every) {
|
||||
continue // silent: S1 says it, and a silent machine reports nothing
|
||||
}
|
||||
bound := max(reportAtLeast, 3*m.lastApply)
|
||||
waited := f.now.Sub(m.sentAt)
|
||||
if waited <= bound {
|
||||
continue
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name,
|
||||
Kind: "sent-not-reported", Machine: m.name, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s was sent a declaration at %s and has not reported applying it (bound %s)",
|
||||
m.name, m.sentAt.UTC().Format("2006-01-02 15:04 MST"), ago(bound)),
|
||||
Said: fmt.Sprintf("waiting %s for the report of the declaration sent", ago(waited))})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func watchPlans(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, p := range f.plans {
|
||||
if p.paused {
|
||||
continue
|
||||
}
|
||||
in := f.now.Sub(p.entered)
|
||||
if in <= p.bound {
|
||||
continue
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.id, Kind: "stalled",
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the plan for %s %s has been at tier %d of %d since %s (bound %s): %s",
|
||||
p.repository, short(p.commit), p.tier+1, p.tiers, p.entered.UTC().Format("2006-01-02 15:04 MST"),
|
||||
ago(p.bound), p.waiting),
|
||||
Said: fmt.Sprintf("at tier %d for %s: %s", p.tier+1, ago(in), p.waiting)})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func watchLoop(f *signalFacts) []conditions.Observation {
|
||||
if f.loop.pending == 0 {
|
||||
return nil
|
||||
}
|
||||
since := f.loop.took
|
||||
if since.IsZero() {
|
||||
since = f.started
|
||||
}
|
||||
if f.now.Sub(since) <= loopDeafAfter {
|
||||
return nil
|
||||
}
|
||||
return []conditions.Observation{{Scope: conditions.ScopeCore, ID: "controller", Kind: "controller-deaf",
|
||||
Token: "deaf", Machine: f.host, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("the controller's event loop has taken nothing for %s while its consumers hold %d "+
|
||||
"message(s): reports, builds and merges are not being acted on", ago(f.now.Sub(since)), f.loop.pending),
|
||||
Said: fmt.Sprintf("%d pending (%s), last taken %s", f.loop.pending, f.loop.where, since.UTC().Format(time.RFC3339))}}
|
||||
}
|
||||
|
||||
func watchMerges(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, m := range f.merges {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMerge, ID: m.Repo + "." + short(m.Commit),
|
||||
Kind: "merge-not-acted", Token: "not-acted", Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("%s/%s merged into %s (%s) was never handed to the controller by the bus; "+
|
||||
"acted on late by the catch-up", m.Owner, m.Repo, m.Base, short(m.Commit)),
|
||||
Said: fmt.Sprintf("announced %s, %s behind it", m.At.UTC().Format(time.RFC3339), readableList(m.Modules))})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func watchAsks(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, a := range f.asks {
|
||||
var said string
|
||||
switch {
|
||||
case a.state == link.AskDead:
|
||||
said = fmt.Sprintf("the %s queue handed it out as often as it may and it was never settled", a.seat)
|
||||
case a.state == link.AskInFlight && f.now.Sub(a.since) > a.bound:
|
||||
said = fmt.Sprintf("in flight on %s for %s (bound %s)", orSomewhere(a.on), ago(f.now.Sub(a.since)), ago(a.bound))
|
||||
default:
|
||||
continue
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeBuild, ID: a.id, Kind: "ask-lost",
|
||||
Token: "lost", Machine: a.on, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the build %s of %s has no outcome: %s", a.id, a.what, said), Said: said})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func orSomewhere(node string) string {
|
||||
if node == "" {
|
||||
return "a machine that did not say which"
|
||||
}
|
||||
return node
|
||||
}
|
||||
|
||||
func watchCalls(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, c := range f.calls {
|
||||
bound := callBound(c.Verb)
|
||||
running := f.now.Sub(c.Started)
|
||||
if running <= bound {
|
||||
continue
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeCall, ID: c.ID, Kind: "call-hung",
|
||||
Token: "hung", Machine: f.host, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s.%s (call %s) has been running since %s, past its bound of %s", c.Seat, c.Verb,
|
||||
c.ID, c.Started.UTC().Format("2006-01-02 15:04 MST"), ago(bound)),
|
||||
Said: fmt.Sprintf("running %s, asked by %s", ago(running), orSomebody(c.Caller))})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func orSomebody(caller string) string {
|
||||
if caller == "" {
|
||||
return "a caller the bus did not name"
|
||||
}
|
||||
return caller
|
||||
}
|
||||
|
||||
func watchProviders(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, c := range f.standings {
|
||||
quiet := f.now.Sub(c.LastObserved)
|
||||
if quiet <= providerSaysAgainWithin {
|
||||
continue
|
||||
}
|
||||
module, node, consumer, ok := providerOf(c)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeProvider, ID: module + "." + node + "." + consumer,
|
||||
Token: "silent", Kind: kindProviderSilent, Machine: node, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s on %s said it keeps failing %s and has said nothing since %s: it stopped "+
|
||||
"saying anything, so its last word is all the mesh has", module, node, consumer,
|
||||
c.LastObserved.UTC().Format("2006-01-02 15:04 MST")),
|
||||
Said: fmt.Sprintf("not said again for %s", ago(quiet))})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func watchAdvisories(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, a := range f.advisories {
|
||||
if f.now.Sub(a.Last) > advisoryQuiet {
|
||||
continue
|
||||
}
|
||||
if a.Kind == link.AdvisoryConsumerLost && !f.lostConsumers[a.Stream+"."+a.Consumer] {
|
||||
continue // it exists again, or the mesh no longer expects it: a removal, not a loss
|
||||
}
|
||||
severity := conditions.Warning
|
||||
times := ""
|
||||
if a.Count > 1 {
|
||||
times = fmt.Sprintf(" (%d times since %s)", a.Count, a.First.UTC().Format("15:04 MST"))
|
||||
}
|
||||
machine := ""
|
||||
if a.ID == "controller" {
|
||||
machine = f.host
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: a.ID, Kind: a.Kind,
|
||||
Machine: machine, Severity: severity, Summary: a.Said + times, Said: a.Said})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func watchSelfCheck(f *signalFacts) []conditions.Observation {
|
||||
every := f.selfCheck.every
|
||||
if every <= 0 {
|
||||
every = doctorEvery
|
||||
}
|
||||
since := f.selfCheck.last
|
||||
if since.IsZero() {
|
||||
since = f.started
|
||||
}
|
||||
if f.now.Sub(since) <= 2*every {
|
||||
return nil
|
||||
}
|
||||
return []conditions.Observation{{Scope: conditions.ScopeCore, ID: "doctor", Kind: "self-check-silent",
|
||||
Machine: f.host, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("the self-check has not finished a run since %s (it runs every %s): the mesh's "+
|
||||
"invariants are not being checked", since.UTC().Format("2006-01-02 15:04 MST"), every),
|
||||
Said: fmt.Sprintf("no run for %s", ago(f.now.Sub(since)))}}
|
||||
}
|
||||
|
||||
func watchTools(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, m := range f.machines {
|
||||
if !m.tools || m.asleep() {
|
||||
continue
|
||||
}
|
||||
bound := heartbeatBound(m.toolsEvery)
|
||||
since := m.toolsHeard
|
||||
if since.IsZero() {
|
||||
// Not heard since this controller started: silent since then at the most.
|
||||
since = f.toolsHeardFrom
|
||||
}
|
||||
silent := f.now.Sub(since)
|
||||
if silent <= bound {
|
||||
continue
|
||||
}
|
||||
heard := "not since this controller started at " + since.UTC().Format("2006-01-02 15:04 MST")
|
||||
if !m.toolsHeard.IsZero() {
|
||||
heard = "since " + m.toolsHeard.UTC().Format("2006-01-02 15:04 MST")
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name, Kind: "tools-silent",
|
||||
Machine: m.name, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the node tools on %s have not said they are there %s (bound %s): nothing can "+
|
||||
"ask that machine anything", m.name, heard, bound),
|
||||
Said: fmt.Sprintf("silent for %s", ago(silent))})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func watchStaleRefusals(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, w := range f.staleRefusals {
|
||||
if w.Count <= staleRefusalsAllowed {
|
||||
continue
|
||||
}
|
||||
scope, id, machine := conditions.ScopeCore, "controller.unnamed", ""
|
||||
named := w.Writer
|
||||
switch {
|
||||
case w.Epoch > 0:
|
||||
id = fmt.Sprintf("controller.epoch-%d", w.Epoch)
|
||||
if e, ok := f.epochs[w.Epoch]; ok {
|
||||
named = fmt.Sprintf("the controller of epoch %d (%s%s)", w.Epoch, e.Instance, endedWords(e))
|
||||
}
|
||||
case w.Writer == link.WriterNodeEngine(firstOf(w.Receivers)) || strings.HasPrefix(w.Writer, "the node-engine on "):
|
||||
node := strings.TrimPrefix(w.Writer, "the node-engine on ")
|
||||
scope, id, machine = conditions.ScopeMachine, node, node
|
||||
}
|
||||
if machine == "" && len(w.Receivers) > 0 {
|
||||
machine = w.Receivers[0]
|
||||
}
|
||||
var also []string
|
||||
for _, r := range w.Receivers {
|
||||
if r != machine && r != "controller" {
|
||||
also = append(also, r)
|
||||
}
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: scope, ID: id, Kind: "stale-writer", Token: "stale-writer",
|
||||
Machine: machine, Also: also, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s was refused %d time(s) in %s as older than what its receivers hold (%s): a "+
|
||||
"writer is sending what the mesh has moved past", named, w.Count, staleRefusalsWithin,
|
||||
strings.Join(w.Receivers, ", ")),
|
||||
Said: fmt.Sprintf("%d stale refusals in %s, the last at %s", w.Count, staleRefusalsWithin,
|
||||
w.Last.UTC().Format(time.RFC3339))})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// firstOf is a list's first, empty for none.
|
||||
func firstOf(list []string) string {
|
||||
if len(list) == 0 {
|
||||
return ""
|
||||
}
|
||||
return list[0]
|
||||
}
|
||||
|
||||
// endedWords is how an epoch ended, for a sentence naming it; nothing while it is held.
|
||||
func endedWords(e inventory.Epoch) string {
|
||||
if e.Ended == nil {
|
||||
return ", still holding the lease"
|
||||
}
|
||||
return fmt.Sprintf(", %s at %s", e.How, e.Ended.UTC().Format("15:04:05 MST"))
|
||||
}
|
||||
|
||||
// watchLease is S12: the lease held and renewed by this controller, and every holder that lost it.
|
||||
func watchLease(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
l := f.lease
|
||||
if l.unleased != "" {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: "controller.lease", Token: "unleased",
|
||||
Kind: "lease-lost", Machine: f.host, Severity: conditions.Urgent,
|
||||
Summary: "the controller serves WITHOUT the lease: nothing keeps a second controller from acting " +
|
||||
"beside it, and its declarations carry no epoch. A bus whose user list is older than this " +
|
||||
"controller does not grant it the lease's bucket: a push of the machine holding the bus sends " +
|
||||
"the list that does, and the controller takes the lease within five seconds — " + l.unleased,
|
||||
Said: l.unleased})
|
||||
} else if l.held && !l.renewed.IsZero() && f.now.Sub(l.renewed) > leaseBound {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: "controller.lease", Token: "late",
|
||||
Kind: "lease-lost", Machine: f.host, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("the controller has not renewed its lease (epoch %d) since %s, past the key's age "+
|
||||
"of %s: another controller may take it", l.epoch, l.renewed.UTC().Format(time.RFC3339), leaseBound),
|
||||
Said: fmt.Sprintf("not renewed for %s", ago(f.now.Sub(l.renewed)))})
|
||||
}
|
||||
if !l.reset.IsZero() && f.now.Sub(l.reset) <= advisoryQuiet {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: "controller.lease", Token: "reset",
|
||||
Kind: "lease-lost", Machine: f.host, Severity: conditions.Urgent,
|
||||
Summary: "the controller lease's bucket was raised again from nothing: " + l.resetSaid,
|
||||
Said: l.resetSaid})
|
||||
}
|
||||
var lost []string
|
||||
newest := inventory.Epoch{}
|
||||
for _, e := range l.ended {
|
||||
if e.Ended == nil || e.How == inventory.EpochReleased || f.now.Sub(*e.Ended) > advisoryQuiet {
|
||||
continue
|
||||
}
|
||||
lost = append(lost, fmt.Sprintf("epoch %d (%s) %s at %s", e.Epoch, e.Instance, e.How,
|
||||
e.Ended.UTC().Format("15:04:05 MST")))
|
||||
if newest.Ended == nil || e.Ended.After(*newest.Ended) {
|
||||
newest = e
|
||||
}
|
||||
}
|
||||
if len(lost) > 0 {
|
||||
how := "lost it: its renewal was refused or could not be made"
|
||||
if newest.How == inventory.EpochExpired {
|
||||
how = "stopped renewing it without giving it back, and was taken over"
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: "controller.lease", Token: "lost",
|
||||
Kind: "lease-lost", Machine: newest.Host, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("the controller of epoch %d (%s) %s; the controller of epoch %d acts now", newest.Epoch,
|
||||
newest.Instance, how, l.epoch),
|
||||
Said: strings.Join(lost, "; ")})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// handActsWithin is how far back a repeated cause counts (S15).
|
||||
const handActsWithin = 14 * 24 * time.Hour
|
||||
|
||||
// personsDecision are the causes of hand acts that are a person's decision by design, never a repair a
|
||||
// healer could take over: approving or rejecting a retirement, and deleting what was retired (novox/hq
|
||||
// ADR 0230 — nothing is retired past the bound or deleted without a person). Repeated, they are the
|
||||
// mesh working as decided, not a healer wanted.
|
||||
var personsDecision = map[string]bool{kindRetireWaiting: true, kindCleanupWaiting: true}
|
||||
|
||||
// watchHandActs is S15: a cause recorded by hand twice within a fortnight is a healer wanted, named by
|
||||
// the cause. **A heal is never a hand act** (healers.go), so a cause a healer exists for and a person
|
||||
// still repaired twice says the healer is not enough — its reach or its budget — and is said so.
|
||||
func watchHandActs(f *signalFacts) []conditions.Observation {
|
||||
recent := make([]link.HandAct, 0, len(f.handActs))
|
||||
for _, a := range f.handActs {
|
||||
if personsDecision[a.Cause] {
|
||||
continue
|
||||
}
|
||||
if f.now.Sub(a.At) <= handActsWithin {
|
||||
recent = append(recent, a)
|
||||
}
|
||||
}
|
||||
repeated := link.RepeatedCauses(recent, f.now)
|
||||
causes := make([]string, 0, len(repeated))
|
||||
for c := range repeated {
|
||||
causes = append(causes, c)
|
||||
}
|
||||
sort.Strings(causes)
|
||||
var out []conditions.Observation
|
||||
for _, cause := range causes {
|
||||
var acts []string
|
||||
var newest link.HandAct
|
||||
for _, a := range recent {
|
||||
if a.Cause != cause {
|
||||
continue
|
||||
}
|
||||
acts = append(acts, fmt.Sprintf("%s %s by %s: %s", a.At.UTC().Format("2006-01-02 15:04"),
|
||||
strings.TrimSpace(a.Verb+" "+strings.Join(a.Args, " ")), a.By, a.Why))
|
||||
if a.At.After(newest.At) {
|
||||
newest = a
|
||||
}
|
||||
}
|
||||
wanted := "a healer is wanted for it"
|
||||
if h := healerNamedFor(cause); h != "" {
|
||||
wanted = fmt.Sprintf("healer %s answers this cause and a person still repaired it: its reach or its "+
|
||||
"budget is not enough", h)
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMesh, ID: "hand-acts." + cause,
|
||||
Token: "healer-wanted", Kind: "healer-wanted", Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%q was repaired by hand %d times in %d days, the last by %s: %s", cause,
|
||||
repeated[cause], int(handActsWithin.Hours()/24), newest.By, wanted),
|
||||
Said: strings.Join(acts, "; ")})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// watchedRows are the rows a watchdog runs for.
|
||||
func watchedRows() []signalRow {
|
||||
var out []signalRow
|
||||
for _, r := range signalsTable {
|
||||
if r.watch != nil {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// kindsOf is a row's condition kinds, one or several.
|
||||
func kindsOf(r signalRow) []string {
|
||||
var out []string
|
||||
for _, k := range strings.Split(r.Kind, ",") {
|
||||
out = append(out, strings.TrimSpace(k))
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1,392 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The test generated from the signals table (novox/hq to-be 45 §3, ADR 0227 rule 5, "how it is
|
||||
// checked"): **every row is walked**. A watched row's signal is suppressed just inside its bound —
|
||||
// nothing raised — and just past it — its condition raised, with its kind and severity — and restored,
|
||||
// and the condition clears. A row that is not watched says why. A row added to the table without a
|
||||
// suppression here fails, so the table cannot grow a watchdog nobody has seen fire.
|
||||
|
||||
// calm is a mesh whose every signal is fresh: one control node heard ten seconds ago, its last send
|
||||
// reported, a plan a minute into its tier, the loop taking, the merges read, nothing asked, no call
|
||||
// running, the self-check a minute old.
|
||||
func calm(now time.Time) *signalFacts {
|
||||
return &signalFacts{now: now, started: now.Add(-time.Hour), host: "anchor", toolsHeardFrom: now.Add(-time.Hour),
|
||||
machines: []machineFacts{{name: "anchor", control: true, lastHeard: now.Add(-10 * time.Second),
|
||||
every: time.Minute, sentAt: now.Add(-time.Hour), reportedCurrent: true, reportedAt: now.Add(-59 * time.Minute),
|
||||
lastApply: 20 * time.Second, tools: true, toolsHeard: now.Add(-10 * time.Second), toolsEvery: time.Minute}},
|
||||
plans: []planFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee00", tier: 0, tiers: 2,
|
||||
entered: now.Add(-time.Minute), bound: 30 * time.Minute, waiting: "building"}},
|
||||
loop: loopFacts{took: now.Add(-time.Second), pending: 1},
|
||||
mergesPassed: now.Add(-time.Minute),
|
||||
selfCheck: selfCheckFacts{last: now.Add(-time.Minute), every: 5 * time.Minute},
|
||||
lostConsumers: map[string]bool{}, epochs: map[int64]inventory.Epoch{},
|
||||
lease: leaseFacts{held: true, epoch: 57, renewed: now.Add(-2 * time.Second)},
|
||||
}
|
||||
}
|
||||
|
||||
// refusedBy is n refusals of one writer, the last a moment ago.
|
||||
func refusedBy(now time.Time, epoch int64, n int) []link.WriterRefusals {
|
||||
return []link.WriterRefusals{{Writer: link.WriterEpoch(epoch), Epoch: epoch, Count: n,
|
||||
Receivers: []string{"anchor", "laptop"}, Last: now.Add(-time.Second)}}
|
||||
}
|
||||
|
||||
// suppression is one row's signal held back: inside its bound, and past it.
|
||||
type suppression struct{ inside, past func(f *signalFacts) }
|
||||
|
||||
// suppressions are every watched row's, by row.
|
||||
var suppressions = map[string]suppression{
|
||||
"S1": {
|
||||
inside: func(f *signalFacts) { f.machines[0].lastHeard = f.now.Add(-3*time.Minute + time.Second) },
|
||||
past: func(f *signalFacts) { f.machines[0].lastHeard = f.now.Add(-3*time.Minute - time.Second) },
|
||||
},
|
||||
"S2": {
|
||||
inside: func(f *signalFacts) {
|
||||
f.machines[0].sentAt, f.machines[0].reportedCurrent = f.now.Add(-110*time.Second), false
|
||||
},
|
||||
past: func(f *signalFacts) {
|
||||
f.machines[0].sentAt, f.machines[0].reportedCurrent = f.now.Add(-121*time.Second), false
|
||||
},
|
||||
},
|
||||
"S3": {
|
||||
inside: func(f *signalFacts) { f.plans[0].entered = f.now.Add(-29 * time.Minute) },
|
||||
past: func(f *signalFacts) { f.plans[0].entered = f.now.Add(-31 * time.Minute) },
|
||||
},
|
||||
"S4": {
|
||||
inside: func(f *signalFacts) { f.loop.took = f.now.Add(-119 * time.Second) },
|
||||
past: func(f *signalFacts) { f.loop.took = f.now.Add(-121 * time.Second) },
|
||||
},
|
||||
"S5": {
|
||||
inside: func(f *signalFacts) {},
|
||||
past: func(f *signalFacts) {
|
||||
f.merges = []missedMerge{{Owner: "novox", Repo: "app", Base: "main", Commit: "c0ffee0011", At: f.now.Add(-11 * time.Minute)}}
|
||||
},
|
||||
},
|
||||
"S6": {
|
||||
inside: func(f *signalFacts) {
|
||||
f.asks = []askFacts{{id: "build-1", seat: "node-build-agent", what: "novox/app", state: link.AskInFlight,
|
||||
on: "anchor", since: f.now.Add(-59 * time.Minute), bound: time.Hour}}
|
||||
},
|
||||
past: func(f *signalFacts) {
|
||||
f.asks = []askFacts{{id: "build-1", seat: "node-build-agent", what: "novox/app", state: link.AskInFlight,
|
||||
on: "anchor", since: f.now.Add(-61 * time.Minute), bound: time.Hour}}
|
||||
},
|
||||
},
|
||||
"S7": {
|
||||
inside: func(f *signalFacts) {
|
||||
f.calls = []link.Call{{ID: "call-1", Seat: "mesh-controller", Verb: "push", Started: f.now.Add(-29 * time.Minute)}}
|
||||
},
|
||||
past: func(f *signalFacts) {
|
||||
f.calls = []link.Call{{ID: "call-1", Seat: "mesh-controller", Verb: "push", Started: f.now.Add(-31 * time.Minute)}}
|
||||
},
|
||||
},
|
||||
"S8": {
|
||||
inside: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-29 * time.Minute))} },
|
||||
past: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-31 * time.Minute))} },
|
||||
},
|
||||
"S9": {
|
||||
inside: func(f *signalFacts) {
|
||||
f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop", Said: "gave up",
|
||||
First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-61 * time.Minute), Count: 1}}
|
||||
},
|
||||
past: func(f *signalFacts) {
|
||||
f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop", Said: "gave up",
|
||||
First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-59 * time.Minute), Count: 1}}
|
||||
},
|
||||
},
|
||||
"S10": {
|
||||
inside: func(f *signalFacts) { f.selfCheck.last = f.now.Add(-9 * time.Minute) },
|
||||
past: func(f *signalFacts) { f.selfCheck.last = f.now.Add(-11 * time.Minute) },
|
||||
},
|
||||
"S11": {
|
||||
inside: func(f *signalFacts) { f.machines[0].toolsHeard = f.now.Add(-179 * time.Second) },
|
||||
past: func(f *signalFacts) { f.machines[0].toolsHeard = f.now.Add(-181 * time.Second) },
|
||||
},
|
||||
"S12": {
|
||||
inside: func(f *signalFacts) { f.lease.renewed = f.now.Add(-15 * time.Second) },
|
||||
past: func(f *signalFacts) { f.lease.renewed = f.now.Add(-16 * time.Second) },
|
||||
},
|
||||
"S13": {
|
||||
inside: func(f *signalFacts) { f.staleRefusals = refusedBy(f.now, 41, 5) },
|
||||
past: func(f *signalFacts) { f.staleRefusals = refusedBy(f.now, 41, 6) },
|
||||
},
|
||||
// Twice by hand within a fortnight is a healer wanted; once, or the first of two a day too old, is not.
|
||||
"S15": {
|
||||
inside: func(f *signalFacts) {
|
||||
f.handActs = []link.HandAct{actByHand(f.now.Add(-15*24*time.Hour), "consumer-behind"),
|
||||
actByHand(f.now.Add(-time.Hour), "consumer-behind")}
|
||||
},
|
||||
past: func(f *signalFacts) {
|
||||
f.handActs = []link.HandAct{actByHand(f.now.Add(-13*24*time.Hour), "consumer-behind"),
|
||||
actByHand(f.now.Add(-time.Hour), "consumer-behind")}
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// actByHand is one entry in the hand-act log, with its cause.
|
||||
func actByHand(at time.Time, cause string) link.HandAct {
|
||||
return link.HandAct{ID: "act-" + at.Format("150405"), At: at, By: "jochen at a shell on the laptop",
|
||||
Verb: "broker consumer-reset", Args: []string{"EVENTS", "controller"}, Why: "it replayed a week", Cause: cause}
|
||||
}
|
||||
|
||||
// **S15 names the cause and, where a healer answers it, that the healer was not enough.**
|
||||
func TestARepeatedHandActNamesItsCauseAndItsHealer(t *testing.T) {
|
||||
now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
|
||||
f := calm(now)
|
||||
f.handActs = []link.HandAct{actByHand(now.Add(-2*time.Hour), "consumer-behind"),
|
||||
actByHand(now.Add(-time.Hour), "consumer-behind"), actByHand(now.Add(-time.Hour), "restarted the proxy"),
|
||||
actByHand(now.Add(-time.Minute), "restarted the proxy")}
|
||||
got := watchHandActs(f)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
if got[0].Key() != "mesh.hand-acts.consumer-behind.healer-wanted" || !strings.Contains(got[0].Summary, "healer H4") {
|
||||
t.Errorf("a cause a healer answers: %s — %s", got[0].Key(), got[0].Summary)
|
||||
}
|
||||
if got[1].Key() != "mesh.hand-acts.restarted_the_proxy.healer-wanted" ||
|
||||
!strings.Contains(got[1].Summary, "a healer is wanted") {
|
||||
t.Errorf("a cause no healer answers: %s — %s", got[1].Key(), got[1].Summary)
|
||||
}
|
||||
}
|
||||
|
||||
// **Approving a retirement and deleting what was retired are a person's decision by design** (ADR
|
||||
// 0230): repeated, they are not a healer wanted.
|
||||
func TestARetirementDecisionRepeatedWantsNoHealer(t *testing.T) {
|
||||
now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
|
||||
f := calm(now)
|
||||
f.handActs = []link.HandAct{actByHand(now.Add(-2*time.Hour), kindRetireWaiting),
|
||||
actByHand(now.Add(-time.Hour), kindRetireWaiting), actByHand(now.Add(-time.Hour), kindCleanupWaiting),
|
||||
actByHand(now.Add(-time.Minute), kindCleanupWaiting)}
|
||||
if got := watchHandActs(f); len(got) != 0 {
|
||||
t.Fatalf("a person's decision asked for a healer: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// standingSaid is a provider's failing word last said at a moment.
|
||||
func standingSaid(at time.Time) conditions.Condition {
|
||||
return conditions.Condition{Key: "provider.idp.anchor.app.failing", Kind: kindProviderFailing, LastObserved: at}
|
||||
}
|
||||
|
||||
func TestEveryRowOfTheSignalsTableIsWatchedRaisedAndCleared(t *testing.T) {
|
||||
now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
|
||||
seen := map[string]bool{}
|
||||
for _, row := range signalsTable {
|
||||
t.Run(row.Row, func(t *testing.T) {
|
||||
if seen[row.Row] {
|
||||
t.Fatalf("%s is in the table twice", row.Row)
|
||||
}
|
||||
seen[row.Row] = true
|
||||
if row.Signal == "" || row.Emitter == "" || row.Trigger == "" || row.Bound == "" || row.Kind == "" ||
|
||||
(row.Severity != conditions.Urgent && row.Severity != conditions.Warning) {
|
||||
t.Fatalf("%s does not say what it expects, from whom, within what, and what it raises: %+v", row.Row, row)
|
||||
}
|
||||
if row.Deferred != "" {
|
||||
if row.watch != nil || row.Phase <= 1 {
|
||||
t.Fatalf("%s is deferred and watched, or deferred out of Phase 1's own rows: %+v", row.Row, row)
|
||||
}
|
||||
if _, has := suppressions[row.Row]; has {
|
||||
t.Fatalf("%s is deferred and has a suppression: one of the two is stale", row.Row)
|
||||
}
|
||||
return
|
||||
}
|
||||
if row.watch == nil || row.needs == nil || row.newest == nil {
|
||||
t.Fatalf("%s is watched and lacks its watch, its needs or its newest", row.Row)
|
||||
}
|
||||
s, ok := suppressions[row.Row]
|
||||
if !ok {
|
||||
t.Fatalf("%s has no suppression in this test: a watchdog nobody has seen fire", row.Row)
|
||||
}
|
||||
if got := row.watch(calm(now)); len(got) != 0 {
|
||||
t.Fatalf("%s raised on a calm mesh: %+v", row.Row, got)
|
||||
}
|
||||
inside := calm(now)
|
||||
s.inside(inside)
|
||||
if got := row.watch(inside); len(got) != 0 {
|
||||
t.Fatalf("%s raised inside its bound: %+v", row.Row, got)
|
||||
}
|
||||
past := calm(now)
|
||||
s.past(past)
|
||||
got := row.watch(past)
|
||||
if len(got) == 0 {
|
||||
t.Fatalf("%s raised nothing past its bound", row.Row)
|
||||
}
|
||||
for _, o := range got {
|
||||
if !slices.Contains(kindsOf(row), o.Kind) {
|
||||
t.Errorf("%s raised %q, which is not its kind %q", row.Row, o.Kind, row.Kind)
|
||||
}
|
||||
if o.Severity != row.Severity {
|
||||
t.Errorf("%s raised %s, the table says %s", row.Row, o.Severity, row.Severity)
|
||||
}
|
||||
if strings.TrimSpace(o.Summary) == "" {
|
||||
t.Errorf("%s raised a condition that says nothing", row.Row)
|
||||
}
|
||||
}
|
||||
|
||||
// Through the store: raised past the bound, cleared when the signal returns.
|
||||
store := conditions.NewInMemory()
|
||||
told := &conditions.Told{}
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: told,
|
||||
Now: func() time.Time { return now }})
|
||||
defer k.Close(context.Background())
|
||||
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
||||
w.see(t.Context(), past)
|
||||
open, err := k.Open(t.Context())
|
||||
if err != nil || len(open) != len(got) {
|
||||
t.Fatalf("%s past its bound left %d open (%v), want %d", row.Row, len(open), err, len(got))
|
||||
}
|
||||
w.see(t.Context(), calm(now))
|
||||
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||
t.Fatalf("%s's condition stayed open after the signal returned: %+v", row.Row, open)
|
||||
}
|
||||
})
|
||||
}
|
||||
for name := range suppressions {
|
||||
if !seen[name] {
|
||||
t.Errorf("a suppression for %s, which the table does not have", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **The control node silent for half an hour is urgent** (S1); any other machine stays a warning.
|
||||
func TestTheControlNodeSilentIsUrgentAfterHalfAnHour(t *testing.T) {
|
||||
now := time.Now()
|
||||
f := calm(now)
|
||||
f.machines[0].lastHeard = now.Add(-31 * time.Minute)
|
||||
f.machines = append(f.machines, machineFacts{name: "laptop", lastHeard: now.Add(-31 * time.Minute)})
|
||||
got := watchHeartbeats(f)
|
||||
if len(got) != 2 || got[0].Severity != conditions.Urgent || got[1].Severity != conditions.Warning {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **A machine that said it sleeps is not silent** (ADR 0211), nor late to report; one that woke is.
|
||||
func TestAMachineThatSaidItSleepsIsNotSilent(t *testing.T) {
|
||||
now := time.Now()
|
||||
f := calm(now)
|
||||
f.machines[0].lastHeard = now.Add(-2 * time.Hour)
|
||||
f.machines[0].sentAt, f.machines[0].reportedCurrent = now.Add(-time.Hour), false
|
||||
f.machines[0].power = link.PowerState{State: "sleeping", At: now.Add(-2 * time.Hour)}
|
||||
if got := append(watchHeartbeats(f), append(watchReports(f), watchTools(f)...)...); len(got) != 0 {
|
||||
t.Fatalf("a sleeping machine raised %+v", got)
|
||||
}
|
||||
f.machines[0].power = link.PowerState{State: "woke", At: now.Add(-time.Hour)}
|
||||
if got := watchHeartbeats(f); len(got) != 1 {
|
||||
t.Fatalf("a woken machine silent past its bound raised %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **A watchdog that cannot see says so, and clears nothing it raised** (ADR 0227 rule 4): the store
|
||||
// unreadable is a probe-failed of its own, and the machine's silence stays open until it can see again.
|
||||
func TestABlindWatchdogSaysSoAndClearsNothing(t *testing.T) {
|
||||
now := time.Now()
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||
defer k.Close(context.Background())
|
||||
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
||||
silent := calm(now)
|
||||
silent.machines[0].lastHeard = now.Add(-10 * time.Minute)
|
||||
w.see(t.Context(), silent)
|
||||
blind := calm(now)
|
||||
blind.machines, blind.machinesErr = nil, errors.New("the store is away")
|
||||
w.see(t.Context(), blind)
|
||||
open, err := k.Open(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keys []string
|
||||
for _, c := range open {
|
||||
keys = append(keys, c.Key)
|
||||
}
|
||||
for _, want := range []string{"machine.anchor.silent", "probe.S1.failed", "probe.S2.failed", "probe.S11.failed"} {
|
||||
if !slices.Contains(keys, want) {
|
||||
t.Errorf("%s is not open while the machines cannot be read: %v", want, keys)
|
||||
}
|
||||
}
|
||||
w.see(t.Context(), calm(now))
|
||||
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||
t.Fatalf("seeing again left open %+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
// **A controller standing by sees nothing and says nothing**: it hears no heartbeat, and would call
|
||||
// every machine silent.
|
||||
func TestAControllerStandingBySaysNothing(t *testing.T) {
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||
defer k.Close(context.Background())
|
||||
w := &watchdogs{keeper: k, started: time.Now(), acting: func() bool { return false }}
|
||||
w.tick(t.Context())
|
||||
if w.lastTick().IsZero() {
|
||||
t.Fatal("a tick standing by was not counted")
|
||||
}
|
||||
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||
t.Fatalf("%+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
// **A stale writer is named** (novox/hq to-be 45 §3, S13): by the controller instance that held the epoch
|
||||
// its refused declarations claimed, and how that epoch ended — the question issue 204 could not answer.
|
||||
func TestAStaleWriterIsNamedByItsEpoch(t *testing.T) {
|
||||
now := time.Now()
|
||||
f := calm(now)
|
||||
ended := now.Add(-time.Minute)
|
||||
f.staleRefusals = refusedBy(now, 41, 6)
|
||||
f.epochs[41] = inventory.Epoch{Epoch: 41, Instance: "controller@anchor pid 7 since 2026-10-06T10:00:00Z",
|
||||
Host: "anchor", Ended: &ended, How: inventory.EpochExpired}
|
||||
got := watchStaleRefusals(f)
|
||||
if len(got) != 1 || got[0].Key() != "core.controller.epoch-41.stale-writer" ||
|
||||
!strings.Contains(got[0].Summary, "pid 7") || !strings.Contains(got[0].Summary, "expired") ||
|
||||
got[0].Machine != "anchor" || !slices.Equal(got[0].Also, []string{"laptop"}) {
|
||||
t.Fatalf("the stale writer is not named: %+v", got)
|
||||
}
|
||||
// An account the controller refused names the machine whose node-engine sent it.
|
||||
f.staleRefusals = []link.WriterRefusals{{Writer: link.WriterNodeEngine("laptop"), Count: 6,
|
||||
Receivers: []string{"controller"}, Last: now}}
|
||||
got = watchStaleRefusals(f)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.stale-writer" || got[0].Machine != "laptop" {
|
||||
t.Fatalf("a node-engine sending older accounts is not named: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **The lease lost is said for an hour, and serving without it for as long as it lasts** (S12).
|
||||
func TestALeaseLostOrMissingIsSaid(t *testing.T) {
|
||||
now := time.Now()
|
||||
f := calm(now)
|
||||
expired := now.Add(-59 * time.Minute)
|
||||
f.lease.ended = []inventory.Epoch{{Epoch: 41, Instance: "controller@anchor pid 7", Host: "anchor",
|
||||
Ended: &expired, How: inventory.EpochExpired}}
|
||||
got := watchLease(f)
|
||||
if len(got) != 1 || got[0].Key() != "core.controller.lease.lost" || !strings.Contains(got[0].Summary, "epoch 41") ||
|
||||
got[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("a holder that stopped renewing was not said: %+v", got)
|
||||
}
|
||||
long := now.Add(-61 * time.Minute)
|
||||
f.lease.ended[0].Ended = &long
|
||||
if got := watchLease(f); len(got) != 0 {
|
||||
t.Fatalf("a loss an hour old is still said: %+v", got)
|
||||
}
|
||||
f.lease.ended[0].How, f.lease.ended[0].Ended = inventory.EpochReleased, &expired
|
||||
if got := watchLease(f); len(got) != 0 {
|
||||
t.Fatalf("a lease given back is said as lost: %+v", got)
|
||||
}
|
||||
f.lease = leaseFacts{held: true, epoch: 501, renewed: now, reset: now.Add(-time.Minute), resetSaid: "moved past 500"}
|
||||
if got := watchLease(f); len(got) != 1 || got[0].Key() != "core.controller.lease.reset" {
|
||||
t.Fatalf("a lease bucket raised again from nothing was not said: %+v", got)
|
||||
}
|
||||
f.lease = leaseFacts{unleased: "the bus refused the key"}
|
||||
if got := watchLease(f); len(got) != 1 || got[0].Key() != "core.controller.lease.unleased" {
|
||||
t.Fatalf("serving without the lease was not said: %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,6 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
@@ -135,28 +134,19 @@ func seatBase(world catalogue.World, seatName string) (string, error) {
|
||||
// seatBases is the clone base of every seat a recipe's context may name, for a build request
|
||||
// (novox/hq ADR 0155). A seat nobody holds is left out rather than refused here: the build may not
|
||||
// name it at all, and if it does the builder refuses with the seat's name.
|
||||
//
|
||||
// **What cannot be read is said, not passed over as no seats** (novox/hq to-be 45 Phase 2, the
|
||||
// empty-on-error lint): the build still goes ahead — one that names no seat needs none — and one that
|
||||
// does is refused naming it, but the reason is the store, and that is said here where it is known.
|
||||
func seatBases(ctx context.Context) map[string]string {
|
||||
unread := func(what string, err error) map[string]string {
|
||||
fmt.Fprintf(os.Stderr, "could not read %s, so a build naming a seat's clone base will be told that "+
|
||||
"seat is not held: %v\n", what, err)
|
||||
return nil
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return unread("the mesh's store", err)
|
||||
return nil
|
||||
}
|
||||
defer open.Close()
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return unread("the catalogue", err)
|
||||
return nil
|
||||
}
|
||||
world, err := theRestOfTheMesh(ctx, open.inventory, shelf, "")
|
||||
if err != nil {
|
||||
return unread("who holds which seat", err)
|
||||
return nil
|
||||
}
|
||||
bases := map[string]string{}
|
||||
for _, seatName := range []string{gitSeat} {
|
||||
|
||||
+76
-145
@@ -2,175 +2,91 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A provider that keeps failing a consumer is a problem the controller reports (novox/hq ADR 0224) —
|
||||
// **the condition store's first kind** (to-be 45 §2, ADR 0227).
|
||||
// A provider that keeps failing a consumer is a problem the controller reports (novox/hq ADR 0224).
|
||||
//
|
||||
// On 2026-10-05 the identity provider's provisioner failed every consumer from shortly after midnight
|
||||
// until it was fixed by hand that night — 31,000 refused logins after its database was moved and its
|
||||
// admin kept an older password — and `status` called the mesh well all day (novox/hq issue 179). A
|
||||
// provider announces a consumer it has failed for minutes; the controller keeps it until the provider
|
||||
// says it recovered; and `status`, its JSON and `node show` name it, breaking "all well". Unchanged in
|
||||
// what it says and when; kept as a condition, `provider.<module>.<node>.<consumer>.failing`, rather
|
||||
// than a row of its own, so it is said outward like every other fault and silenced like one.
|
||||
// provider now announces a consumer it has failed for minutes; the controller keeps it until the
|
||||
// provider says it recovered; and `status`, its JSON and `node show` name it, breaking "all well".
|
||||
|
||||
// Kinds of the provider standing.
|
||||
const (
|
||||
kindProviderFailing = "provider-failing"
|
||||
kindProviderSilent = "provider-silent"
|
||||
// sourceProvisioner is what raised a standing: the provider's own event.
|
||||
sourceProvisioner = "provisioner.failing"
|
||||
)
|
||||
// standings keeps what providers say, in the inventory.
|
||||
type standings struct{ inv *inventory.Inventory }
|
||||
|
||||
// providerSaysAgainWithin is how long a failing word stays current without being said again: twice
|
||||
// the quarter of an hour a provider repeats it at (ADR 0224). Past it, S8.
|
||||
const providerSaysAgainWithin = 30 * time.Minute
|
||||
|
||||
// standings keeps what providers say, as conditions.
|
||||
type standings struct {
|
||||
keeper func() *conditions.Keeper
|
||||
}
|
||||
|
||||
// standingObservation is a provider's failing word as an observation: the provider, its machine and
|
||||
// the consumer name it, so the same consumer failed again is the same condition.
|
||||
func standingObservation(st link.Standing) conditions.Observation {
|
||||
whom := st.Consumer
|
||||
if st.Node != "" {
|
||||
whom += " on " + st.Node
|
||||
}
|
||||
summary := fmt.Sprintf("%s on %s keeps failing %s: %s, %d attempt(s) since %s", st.Module, st.ProviderNode,
|
||||
whom, orUnclassed(st.Class), st.Attempts, st.Since.UTC().Format("2006-01-02 15:04 MST"))
|
||||
said := orUnclassed(st.Class)
|
||||
if e := firstLine(st.Error); e != "" {
|
||||
said += ": " + e
|
||||
}
|
||||
if st.Provider != "" {
|
||||
said += fmt.Sprintf(" (provision %s, %d attempts)", st.Provider, st.Attempts)
|
||||
}
|
||||
return conditions.Observation{Scope: conditions.ScopeProvider,
|
||||
ID: st.Module + "." + st.ProviderNode + "." + st.Consumer,
|
||||
Token: "failing", Kind: kindProviderFailing, Machine: st.ProviderNode, Also: alsoOn(st.Node, st.ProviderNode),
|
||||
Severity: conditions.Warning, Summary: summary, Said: said, Source: sourceProvisioner}
|
||||
}
|
||||
|
||||
// Stood keeps a provider's newest word: failing raises or observes its condition, recovered clears
|
||||
// it. An error is the store away, and the link holds the message to be asked again — a recovery is
|
||||
// said once, and dropping it would leave a consumer named failing that is fine.
|
||||
func (s standings) Stood(ctx context.Context, st link.Standing) (bool, error) {
|
||||
k := s.keeper()
|
||||
if k == nil {
|
||||
return false, fmt.Errorf("the condition store is not open in this controller: %w", link.ErrTryAgain)
|
||||
}
|
||||
o := standingObservation(st)
|
||||
if !st.Failing {
|
||||
why := "the provider says it recovered"
|
||||
if st.Why != "" {
|
||||
why += ": " + st.Why
|
||||
}
|
||||
cleared, err := k.Clear(ctx, o.Key(), why)
|
||||
return cleared, storeAway(err)
|
||||
}
|
||||
_, err := k.Observe(ctx, o)
|
||||
return false, storeAway(err)
|
||||
return s.inv.KeepStanding(ctx, st.Failing, inventory.ProviderStanding{
|
||||
Module: st.Module, ProviderNode: st.ProviderNode, Provision: st.Provider,
|
||||
Consumer: st.Consumer, ConsumerNode: st.Node,
|
||||
Class: st.Class, Error: st.Error, Since: st.Since, Attempts: st.Attempts,
|
||||
})
|
||||
}
|
||||
|
||||
// storeAway reads the condition store failing as the bus being away for the moment: the link holds the
|
||||
// message and asks again, as it does for a store restarting (ADR 0083), rather than taking it unkept.
|
||||
func storeAway(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%v: %w", err, link.ErrTryAgain)
|
||||
}
|
||||
|
||||
// providerStandings is every open provider-failing condition, from what is open.
|
||||
func providerStandings(open []conditions.Condition) []conditions.Condition {
|
||||
var out []conditions.Condition
|
||||
for _, c := range open {
|
||||
if c.Kind == kindProviderFailing {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// providerConditions is every open condition a provider's word raised: a consumer failing (ADR 0224),
|
||||
// and a retirement waiting for a person, kept by a rejection, or cleanup waiting (ADR 0230).
|
||||
func providerConditions(open []conditions.Condition) []conditions.Condition {
|
||||
var out []conditions.Condition
|
||||
for _, c := range open {
|
||||
switch c.Kind {
|
||||
case kindProviderFailing, kindRetireWaiting, kindRetireRejected, kindCleanupWaiting:
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// providerOf reads a provider condition's module and machine back from its key: a standing's
|
||||
// `provider.<module>.<node>.<consumer>.failing`, and a retirement's `provider.<module>.<node>.<token>`,
|
||||
// which names no consumer.
|
||||
func providerOf(c conditions.Condition) (module, node, consumer string, ok bool) {
|
||||
parts := strings.Split(c.Key, ".")
|
||||
if parts[0] != conditions.ScopeProvider {
|
||||
return "", "", "", false
|
||||
}
|
||||
switch len(parts) {
|
||||
case 5:
|
||||
return parts[1], parts[2], parts[3], true
|
||||
case 4:
|
||||
return parts[1], parts[2], "", true
|
||||
}
|
||||
return "", "", "", false
|
||||
}
|
||||
|
||||
// unassignedProviders clears the standing of every provider no longer assigned where it ran — and its
|
||||
// retirement and cleanup conditions with it (ADR 0230): nothing runs there to retire or delete anything.
|
||||
// failingProviders is every consumer a provider still assigned where it ran says it keeps failing.
|
||||
//
|
||||
// **A provider no longer assigned is not asked about** (ADR 0224 §4): nothing runs there to fail
|
||||
// anybody, and nothing there will ever say it recovered. The observation that resolves it is the
|
||||
// assignment. Assigned again, its first failure raises it again.
|
||||
func unassignedProviders(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper,
|
||||
open []conditions.Condition) error {
|
||||
// **A provider no longer assigned is not asked about.** Its last word stays in the store, and is
|
||||
// not a problem: nothing runs there to fail anybody. Assigned again, its first success for each
|
||||
// consumer clears it.
|
||||
func failingProviders(ctx context.Context, inv *inventory.Inventory) ([]inventory.ProviderStanding, error) {
|
||||
all, err := inv.FailingProviders(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("what providers say they keep failing cannot be read: %w", err)
|
||||
}
|
||||
assigned := map[string]map[string]bool{}
|
||||
for _, c := range providerConditions(open) {
|
||||
module, node, _, ok := providerOf(c)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
on, asked := assigned[node]
|
||||
var out []inventory.ProviderStanding
|
||||
for _, s := range all {
|
||||
on, asked := assigned[s.ProviderNode]
|
||||
if !asked {
|
||||
modules, err := inv.Assigned(ctx, node)
|
||||
modules, err := inv.Assigned(ctx, s.ProviderNode)
|
||||
if err != nil {
|
||||
if errors.Is(err, inventory.ErrNoSuchNode) {
|
||||
modules = nil // a machine the mesh no longer knows runs nothing
|
||||
} else {
|
||||
return fmt.Errorf("what %s is assigned cannot be read: %w", node, err)
|
||||
}
|
||||
// A provider on a machine the mesh no longer knows has nothing running to fail anybody.
|
||||
modules = nil
|
||||
}
|
||||
on = map[string]bool{}
|
||||
for _, m := range modules {
|
||||
on[m] = true
|
||||
}
|
||||
assigned[node] = on
|
||||
assigned[s.ProviderNode] = on
|
||||
}
|
||||
if !on[module] {
|
||||
if _, err := k.Clear(ctx, c.Key, module+" is no longer assigned to "+node+
|
||||
": nothing runs there to fail anybody"); err != nil {
|
||||
return err
|
||||
}
|
||||
if on[s.Module] {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// failingLines is how status says them: one consumer per entry, the error under it, and a provider
|
||||
// that stopped repeating itself said so.
|
||||
func failingLines(list []inventory.ProviderStanding, now time.Time) []string {
|
||||
var out []string
|
||||
for _, s := range list {
|
||||
where := s.Module
|
||||
if s.ProviderNode != "" {
|
||||
where += " on " + s.ProviderNode
|
||||
}
|
||||
whom := s.Consumer
|
||||
if s.ConsumerNode != "" {
|
||||
whom += " (" + s.ConsumerNode + ")"
|
||||
}
|
||||
out = append(out, fmt.Sprintf(" %-24s fails %s: %s, for %s (%d attempts since %s)",
|
||||
where, whom, orUnclassed(s.Class), roughly(now.Sub(s.Since)), s.Attempts,
|
||||
s.Since.Local().Format("2006-01-02 15:04")))
|
||||
if e := strings.TrimSpace(s.Error); e != "" {
|
||||
out = append(out, fmt.Sprintf(" %-24s %s", "", firstLine(e)))
|
||||
}
|
||||
if s.Quiet(now) {
|
||||
out = append(out, fmt.Sprintf(" %-24s not said again for %s — the provider has stopped "+
|
||||
"saying anything, so this is its last word", "", roughly(now.Sub(s.SaidAt))))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func orUnclassed(class string) string {
|
||||
@@ -180,10 +96,25 @@ func orUnclassed(class string) string {
|
||||
return class
|
||||
}
|
||||
|
||||
// alsoOn is a consumer's machine, when it is not the provider's.
|
||||
func alsoOn(consumerNode, providerNode string) []string {
|
||||
if consumerNode == "" || consumerNode == providerNode {
|
||||
return nil
|
||||
// printFailing is the status section, said when there is anything to say.
|
||||
func printFailing(list []inventory.ProviderStanding, now time.Time) {
|
||||
if len(list) == 0 {
|
||||
return
|
||||
}
|
||||
return []string{consumerNode}
|
||||
fmt.Printf("%d consumer(s) a provider keeps failing (ADR 0224):\n\n", len(list))
|
||||
for _, line := range failingLines(list, now) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
fmt.Printf("\n the provider's journal has every attempt; it says recovered on its next success\n\n")
|
||||
}
|
||||
|
||||
// failingOn is the standings that concern one machine: a provider running there, or a consumer.
|
||||
func failingOn(list []inventory.ProviderStanding, node string) []inventory.ProviderStanding {
|
||||
var out []inventory.ProviderStanding
|
||||
for _, s := range list {
|
||||
if s.ProviderNode == node || s.ConsumerNode == node {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -7,14 +7,13 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A provider that keeps failing a consumer is a problem `status` names (novox/hq ADR 0224), kept as
|
||||
// the condition store's first kind (to-be 45 §2). On 2026-10-05 the identity provider refused every
|
||||
// consumer for a day and status called the mesh well (04-ISSUES/179): this is that day, told to the
|
||||
// controller the way the provider now tells it.
|
||||
// A provider that keeps failing a consumer is a problem `status` names (novox/hq ADR 0224). On
|
||||
// 2026-10-05 the identity provider refused every consumer for a day and status called the mesh well
|
||||
// (04-ISSUES/179): this is that day, told to the controller the way the provider now tells it.
|
||||
func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
@@ -23,7 +22,7 @@ func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
|
||||
if _, err := assign(ctx, open, "anchor", "idp"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kept := standings{keeper: func() *conditions.Keeper { return conditionsFrom }}
|
||||
kept := standings{open.inventory}
|
||||
since := time.Now().Add(-23 * time.Hour)
|
||||
failing := link.Standing{Module: "idp", Failing: true, Provider: "oidc-client", ProviderNode: "anchor",
|
||||
Consumer: "mesh_laptop_dashboard", Node: "laptop", Class: "credentials-rejected",
|
||||
@@ -40,8 +39,8 @@ func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
|
||||
t.Fatal("a mesh whose identity provider fails a consumer reads as well")
|
||||
}
|
||||
said := printed(t, func() error { return printStatus(asked) })
|
||||
for _, want := range []string{"1 open condition(s)", "provider.idp.anchor.mesh_laptop_dashboard.failing",
|
||||
"idp on anchor keeps failing mesh_laptop_dashboard on laptop", "credentials-rejected", "31000 attempt(s)"} {
|
||||
for _, want := range []string{"1 consumer(s) a provider keeps failing", "idp on anchor",
|
||||
"mesh_laptop_dashboard (laptop)", "credentials-rejected", "31000 attempts", "invalid_grant"} {
|
||||
if !strings.Contains(said, want) {
|
||||
t.Fatalf("status does not say %q:\n%s", want, said)
|
||||
}
|
||||
@@ -49,25 +48,20 @@ func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
|
||||
if strings.Contains(said, "all doing what they were told") {
|
||||
t.Fatalf("status said all well beside a failing provider:\n%s", said)
|
||||
}
|
||||
if !strings.HasPrefix(said, "1 open condition(s)") {
|
||||
t.Fatalf("status does not lead with what is open:\n%s", said)
|
||||
}
|
||||
body, err := statusAsJSON(asked)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var doc struct {
|
||||
Conditions []conditions.Condition `json:"conditions"`
|
||||
Failing []conditions.Condition `json:"failing"`
|
||||
Failing []inventory.ProviderStanding `json:"failing"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &doc); err != nil || len(doc.Failing) != 1 || len(doc.Conditions) != 1 ||
|
||||
!strings.Contains(doc.Failing[0].Evidence[0].Said, "invalid_grant") {
|
||||
if err := json.Unmarshal(body, &doc); err != nil || len(doc.Failing) != 1 || doc.Failing[0].Consumer != "mesh_laptop_dashboard" {
|
||||
t.Fatalf("the document does not carry it: %v\n%s", err, body)
|
||||
}
|
||||
// Both machines' `node show` name it: where the provider runs, and where the consumer is.
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
shown := printed(t, func() error { return showNode(ctx, open.inventory, node) })
|
||||
if !strings.Contains(shown, "open condition(s) about this machine") || !strings.Contains(shown, "mesh_laptop_dashboard") {
|
||||
if !strings.Contains(shown, "a provider keeps failing") || !strings.Contains(shown, "mesh_laptop_dashboard") {
|
||||
t.Fatalf("node show %s does not name it:\n%s", node, shown)
|
||||
}
|
||||
}
|
||||
@@ -81,38 +75,41 @@ func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(asked.conditions) != 0 {
|
||||
t.Fatalf("a recovered consumer is still named: %+v", asked.conditions)
|
||||
if len(asked.failing) != 0 {
|
||||
t.Fatalf("a recovered consumer is still named: %+v", asked.failing)
|
||||
}
|
||||
}
|
||||
|
||||
// A provider no longer assigned where it ran has nothing running to fail anybody: its last word is
|
||||
// cleared on the next look, said as resolved by the assignment.
|
||||
func TestAnUnassignedProvidersLastWordIsCleared(t *testing.T) {
|
||||
// not a problem.
|
||||
func TestAnUnassignedProvidersLastWordIsNotAProblem(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
kept := standings{keeper: func() *conditions.Keeper { return conditionsFrom }}
|
||||
if _, err := kept.Stood(ctx, link.Standing{Module: "gone", Failing: true,
|
||||
if _, err := (standings{open.inventory}).Stood(ctx, link.Standing{Module: "gone", Failing: true,
|
||||
ProviderNode: "anchor", Consumer: "x", Since: time.Now()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
all, err := conditionsFrom.Open(ctx)
|
||||
if err != nil || len(all) != 1 {
|
||||
t.Fatalf("%+v %v", all, err)
|
||||
}
|
||||
if err := unassignedProviders(ctx, open.inventory, conditionsFrom, all); err != nil {
|
||||
asked, err := theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if all, _ := conditionsFrom.Open(ctx); len(all) != 0 {
|
||||
t.Fatalf("%+v", all)
|
||||
if len(asked.failing) != 0 {
|
||||
t.Fatalf("%+v", asked.failing)
|
||||
}
|
||||
}
|
||||
|
||||
// **The store away holds a recovery** (ADR 0224 §3): a standing that cannot be kept is asked again.
|
||||
func TestAStandingTheStoreCannotKeepIsAskedAgain(t *testing.T) {
|
||||
kept := standings{keeper: func() *conditions.Keeper { return nil }}
|
||||
if _, err := kept.Stood(t.Context(), link.Standing{Module: "idp", ProviderNode: "anchor", Consumer: "x"}); err == nil ||
|
||||
!strings.Contains(err.Error(), link.ErrTryAgain.Error()) {
|
||||
t.Fatalf("answered %v", err)
|
||||
func TestAProviderThatStoppedRepeatingItselfIsSaidToHaveGoneQuiet(t *testing.T) {
|
||||
now := time.Now()
|
||||
lines := strings.Join(failingLines([]inventory.ProviderStanding{{
|
||||
Module: "idp", ProviderNode: "anchor", Consumer: "c", Class: "unreachable", Error: "connection refused\nmore",
|
||||
Since: now.Add(-3 * time.Hour), SaidAt: now.Add(-2 * time.Hour), Attempts: 9,
|
||||
}}, now), "\n")
|
||||
for _, want := range []string{"unreachable, for 3h", "connection refused", "not said again for 2h"} {
|
||||
if !strings.Contains(lines, want) {
|
||||
t.Fatalf("%q not in:\n%s", want, lines)
|
||||
}
|
||||
}
|
||||
if strings.Contains(lines, "more") {
|
||||
t.Fatalf("more than the first line of an error:\n%s", lines)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
@@ -82,12 +81,8 @@ func printStatus(asked answers) error {
|
||||
wrong, nodes, quiet := asked.wrong, asked.nodes, asked.quiet
|
||||
behind, sources := asked.behind, asked.sources
|
||||
|
||||
// **What is wrong leads** (novox/hq to-be 45 §2): every open condition, urgent first, oldest
|
||||
// first, silenced ones with when their silence ends.
|
||||
printConditions(asked.conditions, asked.conditionsUnread, time.Now())
|
||||
|
||||
if len(asked.refused) > 0 {
|
||||
// First of what follows. A machine that cannot be worked out is not running an old
|
||||
// First, above everything else. A machine that cannot be worked out is not running an old
|
||||
// declaration — it has no declaration, and nothing below this line is about it.
|
||||
var names []string
|
||||
for name := range asked.refused {
|
||||
@@ -134,6 +129,10 @@ func printStatus(asked answers) error {
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
// A provider failing a consumer, beside machines failing what they were told: both are something
|
||||
// not working now (novox/hq ADR 0224).
|
||||
printFailing(asked.failing, time.Now())
|
||||
|
||||
if len(quiet) > 0 {
|
||||
var said []string
|
||||
for _, n := range quiet {
|
||||
@@ -303,38 +302,6 @@ func printStatus(asked answers) error {
|
||||
fmt.Printf("\n `assign <node> <holder>` meets it; reported until every machine has its holders, then refused\n\n")
|
||||
}
|
||||
|
||||
if len(asked.overflowing) > 0 {
|
||||
// **Reported, and the provider still pushed** (novox/hq ADR 0225, issue 263). Each is left
|
||||
// out of its provider's grants, so the module holds a login nothing created; the provider's
|
||||
// machine is sent everything else rather than refused for one consumer elsewhere.
|
||||
fmt.Printf("%d module(s) identified too long for a provision they require, and not granted it:\n",
|
||||
len(asked.overflowing))
|
||||
for _, o := range asked.overflowing {
|
||||
fmt.Printf(" %-12s %-20s %-22s %q is %d, %s keeps %d\n", o.Consumer, o.Module, o.Provision,
|
||||
o.Identity, len(o.Identity), o.Bound.In, o.Bound.Max)
|
||||
}
|
||||
fmt.Printf("\n a shorter `slug` in the module's definition fits it; `module check` refuses one before merge\n\n")
|
||||
}
|
||||
|
||||
// Repairs done by hand this week (novox/hq to-be 45 §7). Not a fault, so it does not break "all
|
||||
// well"; each is a healer the mesh does not have yet, and the count is how that is watched.
|
||||
switch {
|
||||
case asked.handActsUnread != "":
|
||||
fmt.Printf("the hand-act log could not be read, so how much was done by hand this week is not known: %s\n\n",
|
||||
asked.handActsUnread)
|
||||
case asked.handActs != nil && *asked.handActs > 0:
|
||||
fmt.Printf("%d act(s) done by hand in the last seven days — `hand-acts` lists them, and why\n\n", *asked.handActs)
|
||||
}
|
||||
|
||||
// And what the mesh repaired by itself (novox/hq to-be 45 §7): seen, not only done.
|
||||
switch {
|
||||
case asked.healsUnread != "":
|
||||
fmt.Printf("what the healers did could not be read: %s\n\n", asked.healsUnread)
|
||||
case asked.heals != nil && (asked.heals.Acts > 0 || asked.heals.Escalated > 0):
|
||||
fmt.Printf("%d repair(s) made by the healers in the last seven days, %d handed to the operator — `healers` "+
|
||||
"lists them, and each is in its condition's tried\n\n", asked.heals.Acts, asked.heals.Escalated)
|
||||
}
|
||||
|
||||
if adopted := adoptedNodes(nodes); len(adopted) > 0 {
|
||||
// Said, because nothing forces the flip: a node left adopted is visible here rather than
|
||||
// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
|
||||
@@ -344,9 +311,8 @@ func printStatus(asked answers) error {
|
||||
|
||||
if asked.well() {
|
||||
// Said plainly. "Nothing to report" and "nothing was checked" must never look the same,
|
||||
// and getting here means every question was asked and answered. **No open conditions first**
|
||||
// (novox/hq to-be 45 §2): it is what the sentence means now, silenced ones included.
|
||||
fmt.Printf("no open conditions; %d machine(s), all doing what they were told, all heard from, running what "+
|
||||
// and getting here means every question was asked and answered.
|
||||
fmt.Printf("%d machine(s), all doing what they were told, all heard from, running what "+
|
||||
"the mesh would send them, and every module current with its source\n", len(nodes))
|
||||
// **And what that sentence does not cover**, because for eleven hours it was true of a mesh
|
||||
// in which no module could reach another (novox/hq 04-ISSUES/145). Every question above is
|
||||
@@ -444,27 +410,21 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
||||
// ADR 0207). Each machine resolved again rather than threaded through whoResolves, whose answer
|
||||
// the private network is built from and should say nothing else; a machine that does not
|
||||
// resolve is already in refused, and is passed over here.
|
||||
plans := map[string]planned{}
|
||||
for _, n := range out.nodes {
|
||||
plan, settings, err := planFor(ctx, open, n.Name)
|
||||
plan, _, err := planFor(ctx, open, n.Name)
|
||||
if err != nil {
|
||||
if unresolvable(err) {
|
||||
continue
|
||||
}
|
||||
return answers{}, err
|
||||
}
|
||||
plans[n.Name] = planned{plan, settings}
|
||||
out.unheld = append(out.unheld, plan.Unheld...)
|
||||
// And which of its modules a provider leaves out of its grants, for an identity too long
|
||||
// for what the provision keeps (novox/hq ADR 0225) — judged from the consumer's own
|
||||
// resolution, as the provider's composition judges it.
|
||||
out.overflowing = append(out.overflowing, plan.Overflowing()...)
|
||||
}
|
||||
// And every open condition (novox/hq to-be 45 §2): what the watchdogs, the self-check and the
|
||||
// providers' own words say is wrong — a provider failing a consumer among them (ADR 0224). Kept on
|
||||
// the bus; a process that cannot read them says so, and the mesh is then not called well.
|
||||
if out.conditions, err = openConditions(ctx); err != nil {
|
||||
out.conditionsUnread = err.Error()
|
||||
// And every consumer a provider says it keeps failing (novox/hq ADR 0224). Read from what the
|
||||
// providers announced: nothing else in the mesh knows whether a provision is being made.
|
||||
out.failing, err = failingProviders(ctx, inv)
|
||||
if err != nil {
|
||||
return answers{}, err
|
||||
}
|
||||
out.plans, err = inv.RecentPlans(ctx, 5)
|
||||
if err != nil {
|
||||
@@ -472,23 +432,6 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
||||
}
|
||||
// Whether the build seat takes work, for a plan waiting on it (novox/hq ADR 0219).
|
||||
out.paused = buildSeatPause(ctx, inv, out.plans)
|
||||
// And how many repairs were done by hand this week (novox/hq to-be 45 §7) — where there is a bus
|
||||
// to read the log from; a process with none has no log to count.
|
||||
if _, onBus := broker.BusAddress(); onBus == nil {
|
||||
n, unread := handActsThisWeek(ctx)
|
||||
if unread != "" {
|
||||
out.handActsUnread = unread
|
||||
} else {
|
||||
out.handActs = &n
|
||||
}
|
||||
}
|
||||
|
||||
// And what the healers did this week (novox/hq to-be 45 §7).
|
||||
if heals, err := inv.HealsSince(ctx, time.Now().Add(-7*24*time.Hour)); err != nil {
|
||||
out.healsUnread = err.Error()
|
||||
} else {
|
||||
out.heals = countHeals(heals)
|
||||
}
|
||||
|
||||
// And which machines are not running what the mesh would send them. The same question as a
|
||||
// module being behind its source, one level down: that one says the catalogue is out of date,
|
||||
@@ -500,7 +443,7 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
||||
// said nothing at all and `status --json` emitted prose to stderr and no JSON anywhere. The
|
||||
// reason is kept and reported as data; every question that does not depend on it is still
|
||||
// answered.
|
||||
would, err := wouldSendFrom(ctx, open, out.nodes, plans)
|
||||
would, err := wouldSend(ctx, open, out.nodes)
|
||||
if err != nil {
|
||||
out.network = err.Error()
|
||||
would = map[string]string{}
|
||||
@@ -595,8 +538,7 @@ func untakenModules(ctx context.Context, inv *inventory.Inventory, nodes []inven
|
||||
func (a answers) well() bool {
|
||||
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
|
||||
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 &&
|
||||
len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.overflowing) == 0 &&
|
||||
len(a.conditions) == 0 && a.conditionsUnread == ""
|
||||
len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.failing) == 0
|
||||
}
|
||||
|
||||
// hostSplit is which machines report which host version, for every version more than one machine
|
||||
|
||||
@@ -1,208 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// `status` answered from a summary the serving controller keeps current (novox/hq to-be 45 Phase 0,
|
||||
// §4's D9 and §8's health of the controller).
|
||||
//
|
||||
// **Asked, it was composed: every machine resolved, twice, while its caller waited.** On 2026-10-06
|
||||
// the verb took eighteen seconds on a mesh of four machines, so its caller read "still running" and
|
||||
// had to ask `calls` for the answer to "is the mesh alright" — the one question that must answer at
|
||||
// once, and the one a self-check and a rollout gate will ask every few minutes. So the serving
|
||||
// controller composes it in the background — at its start, after anything that changes what it says
|
||||
// (a machine's report, a build, a verb that acts), and every minute regardless — and the verb answers
|
||||
// the last composition at once, saying when it was composed and how long that took. A caller who
|
||||
// needs it newer than that reads the time and asks again; nothing is answered as current that is not.
|
||||
|
||||
// statusEvery is how often the summary is composed with nothing having nudged it; statusSettle how
|
||||
// long a nudge waits for the next, so a push answered by four machines is composed once.
|
||||
var (
|
||||
statusEvery = time.Minute
|
||||
statusSettle = 2 * time.Second
|
||||
// statusComposeWithin bounds one composition, so a store that hangs cannot stop the summary for
|
||||
// good; the attempt is said as failed, and the last summary stands with its age.
|
||||
statusComposeWithin = 2 * time.Minute
|
||||
)
|
||||
|
||||
// statusSummary is the last composed `status --json`, and when.
|
||||
type statusSummary struct {
|
||||
compose func(context.Context) ([]byte, error)
|
||||
// every, settle and within are the clocks above, read once when it is made.
|
||||
every, settle, within time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
body []byte
|
||||
composedAt time.Time
|
||||
took time.Duration
|
||||
failed string
|
||||
failedAt time.Time
|
||||
started time.Time
|
||||
first chan struct{} // closed when the first attempt ends, either way
|
||||
|
||||
nudged chan struct{}
|
||||
}
|
||||
|
||||
func newStatusSummary(compose func(context.Context) ([]byte, error)) *statusSummary {
|
||||
return &statusSummary{compose: compose, started: time.Now(), first: make(chan struct{}),
|
||||
nudged: make(chan struct{}, 1), every: statusEvery, settle: statusSettle, within: statusComposeWithin}
|
||||
}
|
||||
|
||||
// statusFrom is the serving controller's summary; nil in any other process, where `status` is
|
||||
// composed when asked, as at a shell.
|
||||
var statusFrom *statusSummary
|
||||
|
||||
// nudge asks for a composition soon. Never blocks: one pending is as good as many.
|
||||
func (s *statusSummary) nudge() {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case s.nudged <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// keep composes until ctx ends: now, on a nudge once things settle, and every statusEvery.
|
||||
func (s *statusSummary) keep(ctx context.Context) {
|
||||
once := sync.Once{}
|
||||
for {
|
||||
s.composeOnce(ctx)
|
||||
once.Do(func() { close(s.first) })
|
||||
timer := time.NewTimer(s.every)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return
|
||||
case <-timer.C:
|
||||
case <-s.nudged:
|
||||
timer.Stop()
|
||||
// Let what else is arriving arrive, then compose once for all of it.
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(s.settle):
|
||||
}
|
||||
select {
|
||||
case <-s.nudged:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *statusSummary) composeOnce(ctx context.Context) {
|
||||
start := time.Now()
|
||||
asking, cancel := context.WithTimeout(ctx, s.within)
|
||||
body, err := s.compose(asking)
|
||||
cancel()
|
||||
took := time.Since(start)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if err != nil {
|
||||
s.failed, s.failedAt = err.Error(), time.Now()
|
||||
fmt.Printf("status could not be composed (after %s): %v — `status` answers the last summary, "+
|
||||
"with its age\n", took.Round(time.Millisecond), err)
|
||||
return
|
||||
}
|
||||
s.body, s.composedAt, s.took, s.failed = body, start, took, ""
|
||||
}
|
||||
|
||||
// answer is what the `status` verb answers: the last summary at once, the same document `status
|
||||
// --json` prints, with when it was composed. Before the first composition has ended it waits for it,
|
||||
// but never past the caller's window; a controller that has none says so and why, rather than
|
||||
// answering an empty mesh as a well one.
|
||||
func (s *statusSummary) answer(ctx context.Context) (any, error) {
|
||||
wait := time.NewTimer(link.AnswerWithin - time.Second)
|
||||
defer wait.Stop()
|
||||
select {
|
||||
case <-s.first:
|
||||
case <-wait.C:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.body == nil {
|
||||
why := "its first composition has not finished"
|
||||
if s.failed != "" {
|
||||
why = "it could not be composed: " + s.failed
|
||||
}
|
||||
return nil, fmt.Errorf("this controller started %s ago and has no status to answer yet — %s. "+
|
||||
"Ask again shortly", time.Since(s.started).Round(time.Second), why)
|
||||
}
|
||||
var parsed any
|
||||
_ = json.Unmarshal(s.body, &parsed)
|
||||
out := map[string]any{
|
||||
"output": string(s.body), "ok": true, "answer": parsed,
|
||||
"composed": s.composedAt.UTC().Format(time.RFC3339),
|
||||
"age": time.Since(s.composedAt).Round(time.Second).String(),
|
||||
"composedIn": s.took.Round(time.Millisecond).String(),
|
||||
"note": "composed by the serving controller at its start, after each report, build or act, and " +
|
||||
"every minute; answered at once from the last composition",
|
||||
}
|
||||
if s.failed != "" && s.failedAt.After(s.composedAt) {
|
||||
out["lastAttemptFailed"] = fmt.Sprintf("%s: %s — this summary is the last that could be composed",
|
||||
s.failedAt.UTC().Format(time.RFC3339), s.failed)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// composeStatus is `status --json`, composed in this process against its stores.
|
||||
func composeStatus(open *stores) func(context.Context) ([]byte, error) {
|
||||
return func(ctx context.Context) ([]byte, error) {
|
||||
asked, err := theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return statusAsJSON(asked)
|
||||
}
|
||||
}
|
||||
|
||||
// readingVerbs are the verbs that only read; after any other, what `status` says may have changed, so the summary is
|
||||
// composed again; a verb that only reads leaves it alone, or a console polling `nodes` would keep the
|
||||
// controller composing for ever.
|
||||
var readingVerbs = map[string]bool{
|
||||
"tools": true, "calls": true, "status": true, "nodes": true, "node": true, "modules": true,
|
||||
"seats": true, "builds": true, "plan": true, "queue": true, "durations": true, "hand-acts": true,
|
||||
"doctor": true, "conditions": true, "healers": true,
|
||||
}
|
||||
|
||||
// nudgingListener is the enrolment, nudging the summary when a machine said something new.
|
||||
type nudgingListener struct {
|
||||
link.Enrolment
|
||||
summary *statusSummary
|
||||
// open is the serving controller's stores, for replacing a given value after a module's first
|
||||
// good start (novox/hq ADR 0228).
|
||||
open *stores
|
||||
}
|
||||
|
||||
func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, error) {
|
||||
// A declaration refused as older than the one the machine holds is counted by its writer — the
|
||||
// controller epoch it claimed (novox/hq to-be 45 §6, S13) — and so is what the machine's own count
|
||||
// says it refused beyond the refusals heard.
|
||||
now := time.Now()
|
||||
if report.StaleRefusalOf() {
|
||||
link.StaleRefusals.Refused(link.Refusal{Writer: link.WriterEpoch(report.Epoch), Epoch: report.Epoch,
|
||||
Receiver: report.Node, At: now})
|
||||
}
|
||||
if report.Ordered() {
|
||||
link.StaleRefusals.Lifetime(report.Node, report.RefusedOlder, now)
|
||||
}
|
||||
news, err := l.Enrolment.Heard(ctx, report)
|
||||
if news {
|
||||
l.summary.nudge()
|
||||
}
|
||||
if err == nil && l.open != nil && startedWell(report) {
|
||||
// Off the report's path: replacing a given value sends the machine, and a report waits for
|
||||
// nothing it caused (novox/hq ADR 0228).
|
||||
go replaceGiven(context.WithoutCancel(ctx), l.open, report)
|
||||
}
|
||||
return news, err
|
||||
}
|
||||
@@ -1,152 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// quickly shortens the summary's clocks for one test.
|
||||
func quickly(t *testing.T) {
|
||||
t.Helper()
|
||||
every, settle := statusEvery, statusSettle
|
||||
statusEvery, statusSettle = time.Hour, 10*time.Millisecond
|
||||
t.Cleanup(func() { statusEvery, statusSettle = every, settle })
|
||||
}
|
||||
|
||||
// **`status` answers in full within ten seconds, five times in a row** (novox/hq to-be 45 Phase 0,
|
||||
// D9) — however long composing it takes. On 2026-10-06 composing took eighteen seconds and the
|
||||
// verb answered "still running"; from the summary it answers at once, in full, saying when.
|
||||
func TestStatusAnswersAtOnceHoweverLongComposingTakes(t *testing.T) {
|
||||
quickly(t)
|
||||
var composed atomic.Int32
|
||||
slow := make(chan struct{})
|
||||
s := newStatusSummary(func(ctx context.Context) ([]byte, error) {
|
||||
if composed.Add(1) > 1 {
|
||||
<-slow // every composition after the first outlasts any caller
|
||||
}
|
||||
return []byte(`{"wrong":[],"machines":4}`), nil
|
||||
})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
defer close(slow)
|
||||
go s.keep(ctx)
|
||||
for i := 0; i < 5; i++ {
|
||||
s.nudge() // a composition is under way and does not finish
|
||||
start := time.Now()
|
||||
got, err := s.answer(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if took := time.Since(start); took > time.Second {
|
||||
t.Fatalf("answer %d took %s", i+1, took)
|
||||
}
|
||||
m := got.(map[string]any)
|
||||
if m["answer"].(map[string]any)["machines"] != float64(4) || m["composed"] == "" || m["ok"] != true {
|
||||
t.Fatalf("answer %d was not in full: %v", i+1, m)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The first composition is waited for, never past the caller's window; a controller with none yet
|
||||
// says so rather than answering an empty mesh as a well one.
|
||||
func TestStatusBeforeItsFirstCompositionSaysSo(t *testing.T) {
|
||||
quickly(t)
|
||||
was := link.AnswerWithin
|
||||
link.AnswerWithin = 1100 * time.Millisecond
|
||||
t.Cleanup(func() { link.AnswerWithin = was })
|
||||
never := make(chan struct{})
|
||||
defer close(never)
|
||||
s := newStatusSummary(func(context.Context) ([]byte, error) { <-never; return nil, nil })
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go s.keep(ctx)
|
||||
start := time.Now()
|
||||
_, err := s.answer(ctx)
|
||||
if err == nil || !strings.Contains(err.Error(), "has no status to answer yet") {
|
||||
t.Fatalf("answered %v", err)
|
||||
}
|
||||
if took := time.Since(start); took > link.AnswerWithin {
|
||||
t.Fatalf("waited %s, past the caller's window", took)
|
||||
}
|
||||
}
|
||||
|
||||
// A nudge composes it again, once for several close together; a failed composition leaves the last
|
||||
// summary standing and says it is the last that could be composed.
|
||||
func TestANudgeComposesAgainAndAFailureKeepsTheLastSummary(t *testing.T) {
|
||||
quickly(t)
|
||||
var composed atomic.Int32
|
||||
fail := atomic.Bool{}
|
||||
s := newStatusSummary(func(context.Context) ([]byte, error) {
|
||||
n := composed.Add(1)
|
||||
if fail.Load() {
|
||||
return nil, errors.New("the store did not answer")
|
||||
}
|
||||
body, _ := json.Marshal(map[string]any{"n": n})
|
||||
return body, nil
|
||||
})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go s.keep(ctx)
|
||||
if _, err := s.answer(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.nudge()
|
||||
s.nudge()
|
||||
s.nudge()
|
||||
waitFor(t, func() bool { return composed.Load() == 2 })
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if n := composed.Load(); n != 2 {
|
||||
t.Fatalf("three nudges together composed %d times after the first", n-1)
|
||||
}
|
||||
fail.Store(true)
|
||||
s.nudge()
|
||||
waitFor(t, func() bool { return composed.Load() == 3 })
|
||||
waitFor(t, func() bool {
|
||||
got, err := s.answer(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := got.(map[string]any)
|
||||
return m["lastAttemptFailed"] != nil && m["answer"].(map[string]any)["n"] == float64(2)
|
||||
})
|
||||
}
|
||||
|
||||
// The seat's `status` answers from the summary when this process keeps one.
|
||||
func TestTheStatusVerbAnswersFromTheSummary(t *testing.T) {
|
||||
quickly(t)
|
||||
s := newStatusSummary(func(context.Context) ([]byte, error) { return []byte(`{"from":"summary"}`), nil })
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go s.keep(ctx)
|
||||
statusFrom = s
|
||||
t.Cleanup(func() { statusFrom = nil })
|
||||
handlers, _, err := seatToolHandlers()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := handlers["status"](ctx, json.RawMessage(`{}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.(map[string]any)["answer"].(map[string]any)["from"] != "summary" {
|
||||
t.Fatalf("answered %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func waitFor(t *testing.T, ok func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for !ok() {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("never happened")
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
@@ -110,8 +110,6 @@ func openInventory(ctx context.Context) (*inventory.Inventory, error) {
|
||||
inv.Close()
|
||||
return nil, err
|
||||
}
|
||||
// A plan is written only under the lease, carrying its epoch (novox/hq to-be 45 §6).
|
||||
inv.ActsUnder(theLease.epoch)
|
||||
// Load the seat set from the store, so the control plane reads the set as data rather than as
|
||||
// the slice it was compiled with (novox/hq ADR 0122). A store not yet seeded — or one whose
|
||||
// seat table a migration has not reached — returns nothing, and UseSeats leaves the compiled
|
||||
|
||||
@@ -72,28 +72,24 @@ func TestAPersonClosesAStuckPlan(t *testing.T) {
|
||||
stuck := inventory.Plan{ID: "plan-97b1b2b", Repository: "novox/mesh-catalog", Commit: "97b1b2b",
|
||||
Created: time.Now().UTC(), State: inventory.PlanRolling, Tier: 1, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "built"}, "b": {}}}
|
||||
if err := open.inventory.SavePlan(ctx, &stuck); err != nil {
|
||||
if err := open.inventory.SavePlan(ctx, stuck); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := plansCommand(ctx, []string{"close", stuck.ID}); err == nil || !strings.Contains(err.Error(), "--why") {
|
||||
t.Fatalf("a plan was closed by hand without saying why: %v", err)
|
||||
}
|
||||
if err := plansCommand(ctx, []string{"close", stuck.ID, "--why", "its report will not come"}); err != nil {
|
||||
if err := plansCommand(ctx, []string{"close", stuck.ID}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
closed, err := open.inventory.PlanByID(ctx, stuck.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if closed.State != inventory.PlanFailed || !strings.Contains(closed.Note, "closed by hand") ||
|
||||
!strings.Contains(closed.Note, "its report will not come") {
|
||||
if closed.State != inventory.PlanFailed || !strings.Contains(closed.Note, "closed by hand") {
|
||||
t.Fatalf("the plan was left %s: %q", closed.State, closed.Note)
|
||||
}
|
||||
if err := plansCommand(ctx, []string{"close", stuck.ID, "--why", "again"}); err == nil {
|
||||
if err := plansCommand(ctx, []string{"close", stuck.ID}); err == nil {
|
||||
t.Fatal("a plan already closed was closed again")
|
||||
}
|
||||
if argv, err := argvFor("plans", map[string]any{"close": stuck.ID, "why": "w"}); err != nil ||
|
||||
!reflect.DeepEqual(argv, []string{"plans", "close", stuck.ID, "--why", "w"}) {
|
||||
if argv, err := argvFor("plans", map[string]any{"close": stuck.ID}); err != nil ||
|
||||
!reflect.DeepEqual(argv, []string{"plans", "close", stuck.ID}) {
|
||||
t.Fatalf("the seat's verb does not close a plan: %v %v", argv, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -383,13 +383,13 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
||||
fmt.Printf(" the last tier depends on itself: %s — built together, in no order\n",
|
||||
strings.Join(plan.Tiers[len(plan.Tiers)-1], ", "))
|
||||
}
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
if err := inv.SavePlan(ctx, plan); err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
// Closed after the newer plan is kept, never before: a controller replaced between the two leaves
|
||||
// both open, which the next merge settles, rather than neither.
|
||||
for _, old := range superseded {
|
||||
if err := inv.SavePlan(ctx, &old); err != nil {
|
||||
if err := inv.SavePlan(ctx, old); err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
fmt.Printf(" %s (%s at %s) is %s\n", old.ID, old.Repository, short(old.Commit), old.Note)
|
||||
@@ -411,7 +411,7 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
||||
if err := askTier(ctx, inv, &plan); err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
if err := inv.SavePlan(ctx, plan); err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -1,620 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The watchdogs (novox/hq to-be 45 §3): every row of the signals table, run over what the serving
|
||||
// controller knows, every half minute.
|
||||
//
|
||||
// **One loop, one gathering, every row.** The facts are gathered once a tick — the store's machines,
|
||||
// plans and durations, the bus's queue and consumers, what this process heard — and each row's watch
|
||||
// is a pure reading of them, which is what lets the test generated from the table suppress a signal by
|
||||
// changing a fact. A part that cannot be gathered makes the rows that read it blind: each says so as
|
||||
// a condition of its own (`probe.<row>.failed`) and keeps what it raised before, because a watchdog
|
||||
// that cannot see must not read as one that sees nothing wrong (ADR 0227 rule 4).
|
||||
//
|
||||
// **The watchdogs are themselves watched.** Each tick is recorded; the self-check (doctor.go) fails
|
||||
// its probe DW when the ticks stop, and the watchdogs raise S10 when the self-check stops — two loops
|
||||
// watching each other, and mesh-watcher on a second machine watching the self-check's heartbeat for
|
||||
// the case both stop with the process.
|
||||
|
||||
// watchEvery is how often the watchdogs run.
|
||||
var watchEvery = 30 * time.Second
|
||||
|
||||
// signalFacts is what one tick of the watchdogs reads.
|
||||
type signalFacts struct {
|
||||
now time.Time
|
||||
started time.Time
|
||||
// host is the machine this controller runs on, as the mesh names it, for a condition about itself.
|
||||
host string
|
||||
|
||||
machines []machineFacts
|
||||
machinesErr error
|
||||
// toolsHeardFrom is when this process began hearing node tools: one not heard since is silent
|
||||
// since then at the most.
|
||||
toolsHeardFrom time.Time
|
||||
|
||||
plans []planFacts
|
||||
plansErr error
|
||||
|
||||
loop loopFacts
|
||||
loopErr error
|
||||
|
||||
merges []missedMerge
|
||||
mergesPassed time.Time
|
||||
mergesErr error
|
||||
|
||||
asks []askFacts
|
||||
asksErr error
|
||||
|
||||
calls []link.Call
|
||||
|
||||
standings []conditions.Condition
|
||||
standingsErr error
|
||||
// providerWords are every open condition a provider's word raised — failing, waiting for a person
|
||||
// to approve a retirement, kept by a rejection, cleanup waiting (ADR 0224, 0230) — so a provider no
|
||||
// longer assigned has them all cleared.
|
||||
providerWords []conditions.Condition
|
||||
|
||||
advisories []link.Advisory
|
||||
lostConsumers map[string]bool
|
||||
advisoriesErr error
|
||||
|
||||
selfCheck selfCheckFacts
|
||||
|
||||
// staleRefusals are the writers refused as older lately, and epochs the mesh's record of each epoch
|
||||
// they name (novox/hq to-be 45 §6, S13).
|
||||
staleRefusals []link.WriterRefusals
|
||||
epochs map[int64]inventory.Epoch
|
||||
|
||||
// handActs are the acts done by hand within the fortnight S15 counts.
|
||||
handActs []link.HandAct
|
||||
handActsErr error
|
||||
|
||||
// lease is this controller's standing to the lease, and the epochs that ended lately (S12).
|
||||
lease leaseFacts
|
||||
leaseErr error
|
||||
}
|
||||
|
||||
type leaseFacts struct {
|
||||
held bool
|
||||
epoch uint64
|
||||
renewed time.Time
|
||||
unleased string
|
||||
ended []inventory.Epoch
|
||||
// reset is when the lease bucket was found raised again from nothing; resetSaid what of it.
|
||||
reset time.Time
|
||||
resetSaid string
|
||||
}
|
||||
|
||||
type machineFacts struct {
|
||||
name string
|
||||
control bool
|
||||
// lastHeard is the store's last word from it, any word; zero when never.
|
||||
lastHeard time.Time
|
||||
// every is the heartbeat interval it said; zero when it said none.
|
||||
every time.Duration
|
||||
power link.PowerState
|
||||
// sentAt is when it was last sent a declaration; reportedCurrent whether it has reported that one.
|
||||
sentAt time.Time
|
||||
reportedCurrent bool
|
||||
reportedAt time.Time
|
||||
// lastApply is its newest measured apply.
|
||||
lastApply time.Duration
|
||||
// tools is whether node-tools is assigned there; toolsHeard and toolsEvery its heartbeat.
|
||||
tools bool
|
||||
toolsHeard time.Time
|
||||
toolsEvery time.Duration
|
||||
}
|
||||
|
||||
// asleep says the machine said it would be away and has not said it is back (ADR 0211).
|
||||
func (m machineFacts) asleep() bool { return m.power.Away() }
|
||||
|
||||
type planFacts struct {
|
||||
id, repository, commit string
|
||||
tier, tiers int
|
||||
entered time.Time
|
||||
bound time.Duration
|
||||
waiting string
|
||||
paused bool
|
||||
}
|
||||
|
||||
type loopFacts struct {
|
||||
took time.Time
|
||||
pending uint64
|
||||
where string
|
||||
}
|
||||
|
||||
type askFacts struct {
|
||||
id, seat, what, state, on string
|
||||
since time.Time
|
||||
bound time.Duration
|
||||
}
|
||||
|
||||
type selfCheckFacts struct {
|
||||
last time.Time
|
||||
every time.Duration
|
||||
}
|
||||
|
||||
// watchdogs is the loop and what it needs.
|
||||
type watchdogs struct {
|
||||
open *stores
|
||||
server *link.Server
|
||||
js *broker.JetStream
|
||||
keeper *conditions.Keeper
|
||||
doctor *doctor
|
||||
|
||||
started time.Time
|
||||
// acting says this controller is the one acting, not one standing by (link.Holding): a controller
|
||||
// standing by hears no heartbeat and would call every machine silent.
|
||||
acting func() bool
|
||||
|
||||
mu sync.Mutex
|
||||
ticked time.Time
|
||||
last *signalFacts
|
||||
failed string
|
||||
}
|
||||
|
||||
// lastTick is when the watchdogs last finished a tick.
|
||||
func (w *watchdogs) lastTick() time.Time {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.ticked
|
||||
}
|
||||
|
||||
// lastFacts is the facts of the newest tick, for `doctor signals`; nil before the first.
|
||||
func (w *watchdogs) lastFacts() *signalFacts {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.last
|
||||
}
|
||||
|
||||
// keep runs the watchdogs until ctx ends.
|
||||
func (w *watchdogs) keep(ctx context.Context) {
|
||||
tick := time.NewTicker(watchEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
w.tick(ctx)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// tick is one run of every row.
|
||||
func (w *watchdogs) tick(ctx context.Context) {
|
||||
if w.acting != nil && !w.acting() {
|
||||
// Standing by: nothing seen, nothing said, and the tick counted — this process's self-check
|
||||
// is not the one that matters while another acts.
|
||||
w.mu.Lock()
|
||||
w.ticked = time.Now()
|
||||
w.mu.Unlock()
|
||||
return
|
||||
}
|
||||
running, cancel := context.WithTimeout(ctx, watchEvery)
|
||||
defer cancel()
|
||||
w.see(running, w.gather(running))
|
||||
}
|
||||
|
||||
// see runs every watched row over one gathering, keeps what each found, and records the tick.
|
||||
func (w *watchdogs) see(running context.Context, f *signalFacts) {
|
||||
var problems []string
|
||||
var blind []conditions.Observation
|
||||
for _, row := range watchedRows() {
|
||||
if err := row.needs(f); err != nil {
|
||||
blind = append(blind, blindRow(row, err))
|
||||
continue
|
||||
}
|
||||
if err := w.keeper.Reconcile(running, row.Row, row.watch(f)); err != nil {
|
||||
problems = append(problems, row.Row+": "+err.Error())
|
||||
}
|
||||
}
|
||||
// The rows that could not see, said; the ones that see again, cleared.
|
||||
if err := w.keeper.Reconcile(running, sourceWatchdogs, blind); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
// A provider no longer assigned where it ran: its standing is resolved by the assignment.
|
||||
if f.standingsErr == nil && w.open != nil {
|
||||
if err := unassignedProviders(running, w.open.inventory, w.keeper, f.providerWords); err != nil {
|
||||
problems = append(problems, "S8: "+err.Error())
|
||||
}
|
||||
}
|
||||
if err := w.keeper.EndSilences(running); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
failed := ""
|
||||
if len(problems) > 0 {
|
||||
failed = fmt.Sprintf("%v", problems)
|
||||
}
|
||||
w.mu.Lock()
|
||||
said := w.failed
|
||||
w.ticked, w.last, w.failed = time.Now(), f, failed
|
||||
w.mu.Unlock()
|
||||
if failed != said {
|
||||
if failed != "" {
|
||||
fmt.Printf("the watchdogs could not keep what they saw: %s\n", failed)
|
||||
} else if said != "" {
|
||||
fmt.Println("the watchdogs keep what they see again")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sourceWatchdogs is what raises a blind row's condition.
|
||||
const sourceWatchdogs = "watchdogs"
|
||||
|
||||
// blindRow is a row whose facts could not be gathered, as a condition of its own.
|
||||
func blindRow(row signalRow, err error) conditions.Observation {
|
||||
return conditions.Observation{Scope: conditions.ScopeProbe, ID: row.Row, Kind: "probe-failed", Token: "failed",
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the watchdog of %s (%s) cannot see: what it reads could not be read, so nothing "+
|
||||
"it would raise can be — and nothing it raised before is cleared", row.Row, row.Signal),
|
||||
Said: firstLine(err.Error())}
|
||||
}
|
||||
|
||||
// gather reads every fact a tick needs. Each part's failure is kept beside it, never an empty part.
|
||||
func (w *watchdogs) gather(ctx context.Context) *signalFacts {
|
||||
now := time.Now()
|
||||
f := &signalFacts{now: now, started: w.started, toolsHeardFrom: link.ToolsBeats.Started(), calls: link.Calls.Running(),
|
||||
staleRefusals: link.StaleRefusals.Within(now.Add(-staleRefusalsWithin)), lostConsumers: map[string]bool{},
|
||||
epochs: map[int64]inventory.Epoch{}}
|
||||
if w.doctor != nil {
|
||||
f.selfCheck = selfCheckFacts{last: w.doctor.lastRunEnded(), every: doctorEvery}
|
||||
}
|
||||
inv := w.open.inventory
|
||||
f.host = controlHost(ctx, inv)
|
||||
f.lease, f.leaseErr = gatherLease(ctx, inv, now)
|
||||
for _, r := range f.staleRefusals {
|
||||
if r.Epoch <= 0 {
|
||||
continue
|
||||
}
|
||||
// Named where the record has it; a writer the record cannot name is still said by its epoch.
|
||||
if e, found, err := inv.EpochOf(ctx, uint64(r.Epoch)); err == nil && found {
|
||||
f.epochs[r.Epoch] = e
|
||||
}
|
||||
}
|
||||
f.machines, f.machinesErr = w.gatherMachines(ctx, inv, now)
|
||||
f.plans, f.plansErr = gatherPlans(ctx, inv, now)
|
||||
f.loop, f.loopErr = w.gatherLoop()
|
||||
f.mergesPassed, f.merges, f.mergesErr = watchedMerges.last()
|
||||
if f.mergesErr == nil && !f.mergesPassed.IsZero() && now.Sub(f.mergesPassed) > 3*mergeCatchUpEvery {
|
||||
f.mergesErr = fmt.Errorf("the catch-up of merges has not passed since %s", f.mergesPassed.UTC().Format(time.RFC3339))
|
||||
}
|
||||
f.asks, f.asksErr = w.gatherAsks(ctx, inv)
|
||||
if open, err := w.keeper.Open(ctx); err != nil {
|
||||
f.standingsErr = err
|
||||
} else {
|
||||
f.standings = providerStandings(open)
|
||||
f.providerWords = providerConditions(open)
|
||||
}
|
||||
f.advisories = link.Advisories.Since(now.Add(-advisoryQuiet))
|
||||
f.lostConsumers, f.advisoriesErr = w.lostConsumers(ctx, f.advisories)
|
||||
f.handActs, f.handActsErr = w.gatherHandActs(ctx, now)
|
||||
return f
|
||||
}
|
||||
|
||||
// gatherLease is this controller's standing to the lease and the epochs that ended within the hour.
|
||||
func gatherLease(ctx context.Context, inv *inventory.Inventory, now time.Time) (leaseFacts, error) {
|
||||
st := theLease.standing()
|
||||
f := leaseFacts{held: st.Held, epoch: st.Epoch, renewed: st.Renewed, unleased: st.Unleased, reset: st.Reset,
|
||||
resetSaid: st.ResetSaid}
|
||||
ended, err := inv.EpochsSince(ctx, now.Add(-advisoryQuiet))
|
||||
if err != nil {
|
||||
return f, fmt.Errorf("the epochs the mesh issued cannot be read: %w", err)
|
||||
}
|
||||
f.ended = ended
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// controlHost is the machine running the controller, as the mesh names it: the one the controller
|
||||
// module is assigned to, or this process's host name where that is not one machine.
|
||||
func controlHost(ctx context.Context, inv *inventory.Inventory) string {
|
||||
if on, err := inv.Running(ctx, "mesh-controller"); err == nil && len(on) == 1 {
|
||||
return on[0]
|
||||
}
|
||||
host, _ := os.Hostname()
|
||||
return host
|
||||
}
|
||||
|
||||
func (w *watchdogs) gatherMachines(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]machineFacts, error) {
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the machines cannot be read: %w", err)
|
||||
}
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("what each machine last reported cannot be read: %w", err)
|
||||
}
|
||||
reported := map[string]inventory.Reported{}
|
||||
for _, r := range reports {
|
||||
reported[r.Node] = r
|
||||
}
|
||||
control, err := inv.Running(ctx, "mesh-controller")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("where the controller runs cannot be read: %w", err)
|
||||
}
|
||||
tooled, err := inv.Running(ctx, broker.RuntimeModule)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("where the node tools run cannot be read: %w", err)
|
||||
}
|
||||
applies, err := inv.Durations(ctx, inventory.DurationApply, now.Add(-7*24*time.Hour))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the measured applies cannot be read: %w", err)
|
||||
}
|
||||
lastApply := map[string]time.Duration{}
|
||||
for _, d := range applies { // oldest first: the last one read is the newest
|
||||
lastApply[d.Node] = d.Took
|
||||
}
|
||||
var out []machineFacts
|
||||
anyPast := false
|
||||
for _, n := range nodes {
|
||||
m := machineFacts{name: n.Name, control: slices.Contains(control, n.Name), lastHeard: n.LastSeen,
|
||||
lastApply: lastApply[n.Name], tools: slices.Contains(tooled, n.Name)}
|
||||
if beat, ok := link.HostBeats.Of(n.Name); ok {
|
||||
m.every = beat.Every
|
||||
}
|
||||
if beat, ok := link.ToolsBeats.Of(n.Name); ok {
|
||||
m.toolsHeard, m.toolsEvery = beat.At, beat.Every
|
||||
}
|
||||
if r, ok := reported[n.Name]; ok {
|
||||
if r.Sent != nil {
|
||||
m.sentAt = *r.Sent
|
||||
}
|
||||
if r.At != nil {
|
||||
m.reportedAt = *r.At
|
||||
}
|
||||
m.reportedCurrent = r.Current
|
||||
}
|
||||
if !m.lastHeard.IsZero() && now.Sub(m.lastHeard) > heartbeatBound(m.every) {
|
||||
anyPast = true
|
||||
}
|
||||
if m.tools && now.Sub(later(m.toolsHeard, link.ToolsBeats.Started())) > heartbeatBound(m.toolsEvery) {
|
||||
anyPast = true
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
// What a machine past its bound last said of its power, read only then: a machine that said it
|
||||
// is asleep is not lost (ADR 0211). Unreadable is said: a sleeping laptop is then called silent,
|
||||
// which is the louder mistake and the right one to make.
|
||||
if anyPast && w.server != nil {
|
||||
states, err := w.server.PowerStates(ctx, now.Add(-7*24*time.Hour))
|
||||
if err != nil {
|
||||
fmt.Printf("what machines said of their power cannot be read, so a sleeping one is called silent: %v\n", err)
|
||||
}
|
||||
for i := range out {
|
||||
out[i].power = states[out[i].name]
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// gatherPlans is every open plan, its tier's bound from what was measured, and what it waits on.
|
||||
func gatherPlans(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]planFacts, error) {
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the open plans cannot be read: %w", err)
|
||||
}
|
||||
if len(plans) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
tiers, err := inv.Durations(ctx, inventory.DurationPlanTier, now.Add(-14*24*time.Hour))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the measured plan tiers cannot be read: %w", err)
|
||||
}
|
||||
measured := map[string][]time.Duration{}
|
||||
for _, d := range tiers {
|
||||
measured[d.Subject] = append(measured[d.Subject], d.Took)
|
||||
}
|
||||
pause := buildSeatPause(ctx, inv, plans)
|
||||
var out []planFacts
|
||||
for _, p := range plans {
|
||||
_, paused := pausedWaiting(p, pause, now)
|
||||
out = append(out, planFacts{id: p.ID, repository: p.Repository, commit: p.Commit, tier: p.Tier,
|
||||
tiers: len(p.Tiers), entered: p.TierEntered, bound: max(tierAtLeast, 3*p90(measured[p.Repository])),
|
||||
waiting: planLineWith(p, now, pause), paused: paused})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// p90 is the ninetieth percentile of measurements; zero for none.
|
||||
func p90(took []time.Duration) time.Duration {
|
||||
if len(took) == 0 {
|
||||
return 0
|
||||
}
|
||||
sorted := append([]time.Duration(nil), took...)
|
||||
sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] })
|
||||
return sorted[int(0.9*float64(len(sorted)-1))]
|
||||
}
|
||||
|
||||
// gatherLoop is when the event loop last took a message and what its consumers hold.
|
||||
func (w *watchdogs) gatherLoop() (loopFacts, error) {
|
||||
f := loopFacts{took: link.Loop.Last()}
|
||||
if w.js == nil {
|
||||
return f, errors.New("this controller is not on the bus")
|
||||
}
|
||||
var where []string
|
||||
for _, c := range broker.MeshConsumers() {
|
||||
info, err := w.js.Context().ConsumerInfo(c.Stream, c.Name)
|
||||
if err != nil {
|
||||
return f, fmt.Errorf("the controller's consumer on %s cannot be read: %w", c.Stream, err)
|
||||
}
|
||||
if held := info.NumPending + uint64(info.NumAckPending); held > 0 {
|
||||
f.pending += held
|
||||
where = append(where, fmt.Sprintf("%d on %s", held, c.Stream))
|
||||
}
|
||||
}
|
||||
f.where = fmt.Sprint(where)
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// gatherAsks is every ask in the build seat's queue that is in flight or dead, with its bound.
|
||||
func (w *watchdogs) gatherAsks(ctx context.Context, inv *inventory.Inventory) ([]askFacts, error) {
|
||||
if w.js == nil {
|
||||
return nil, errors.New("this controller is not on the bus")
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the catalogue cannot be read: %w", err)
|
||||
}
|
||||
seat := buildSeatAmong(entries)
|
||||
q, err := link.ReadQueue(ctx, w.js, seat)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
builds, err := inv.Durations(ctx, inventory.DurationBuild, time.Now().Add(-14*24*time.Hour))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the measured builds cannot be read: %w", err)
|
||||
}
|
||||
var took []time.Duration
|
||||
for _, d := range builds {
|
||||
took = append(took, d.Took)
|
||||
}
|
||||
bound := askDefault
|
||||
if len(took) > 0 {
|
||||
bound = max(askAtLeast, 3*p90(took))
|
||||
}
|
||||
var out []askFacts
|
||||
for _, a := range q.Asks {
|
||||
if a.State != link.AskInFlight && a.State != link.AskDead {
|
||||
continue
|
||||
}
|
||||
since := a.Started
|
||||
if since.IsZero() {
|
||||
since = a.AskedAt
|
||||
}
|
||||
what := a.Repository
|
||||
if a.Path != "" {
|
||||
what += " " + a.Path
|
||||
}
|
||||
out = append(out, askFacts{id: a.ID, seat: seat, what: what, state: a.State, on: a.On, since: since, bound: bound})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// lostConsumers is, of the consumers the bus said were deleted, each that is still missing and that
|
||||
// the mesh expects: a consumer removed because its module was unassigned is not lost.
|
||||
func (w *watchdogs) lostConsumers(ctx context.Context, heard []link.Advisory) (map[string]bool, error) {
|
||||
out := map[string]bool{}
|
||||
var deleted []link.Advisory
|
||||
for _, a := range heard {
|
||||
if a.Kind == link.AdvisoryConsumerLost {
|
||||
deleted = append(deleted, a)
|
||||
}
|
||||
}
|
||||
if len(deleted) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
if w.js == nil {
|
||||
return nil, errors.New("this controller is not on the bus")
|
||||
}
|
||||
_, expected, err := expectedBusObjects(ctx, w.open.inventory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
want := map[string]bool{}
|
||||
for _, c := range expected {
|
||||
want[c.Stream+"."+c.Name] = true
|
||||
}
|
||||
for _, a := range deleted {
|
||||
_, err := w.js.Context().ConsumerInfo(a.Stream, a.Consumer)
|
||||
switch {
|
||||
case errors.Is(err, nats.ErrConsumerNotFound):
|
||||
out[a.Stream+"."+a.Consumer] = want[a.Stream+"."+a.Consumer]
|
||||
case err != nil:
|
||||
return nil, fmt.Errorf("whether %s exists cannot be read: %w", link.ConsumerInWords(a.Stream, a.Consumer), err)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// gatherHandActs is the hand-act log's fortnight (S15), read from the bus.
|
||||
func (w *watchdogs) gatherHandActs(ctx context.Context, now time.Time) ([]link.HandAct, error) {
|
||||
if w.js == nil {
|
||||
return nil, errors.New("this controller is not on the bus")
|
||||
}
|
||||
reading, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
acts, err := link.HandActs(reading, w.js.Conn(), now.Add(-handActsWithin))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the hand-act log cannot be read: %w", err)
|
||||
}
|
||||
return acts, nil
|
||||
}
|
||||
|
||||
// later is the later of two moments.
|
||||
func later(a, b time.Time) time.Time {
|
||||
if a.After(b) {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// watchTheMesh opens the condition store and starts the watchdogs, the bus's advisories and the
|
||||
// self-check, for the serving controller; the returned function stops them. Nil when the store could
|
||||
// not be opened, which is said.
|
||||
func watchTheMesh(ctx context.Context, open *stores, server *link.Server, bus link.OverNATS) func() {
|
||||
keeper, err := keeperOn(ctx, bus.Conn)
|
||||
if err != nil {
|
||||
fmt.Printf("the condition store could NOT be opened, so nothing that goes wrong is kept or said, and "+
|
||||
"status says the conditions cannot be read: %v\n", err)
|
||||
return nil
|
||||
}
|
||||
conditionsFrom = keeper
|
||||
logf := func(format string, args ...any) { fmt.Printf(format+"\n", args...) }
|
||||
stopHearing, err := server.HearAdvisories(logf)
|
||||
if err != nil {
|
||||
fmt.Printf("what the bus says about itself cannot be heard (S9 is blind): %v\n", err)
|
||||
stopHearing = func() {}
|
||||
}
|
||||
host := controlHost(ctx, open.inventory)
|
||||
w := &watchdogs{open: open, server: server, js: server.JetStream(), keeper: keeper, started: time.Now(),
|
||||
acting: link.Holding}
|
||||
d := &doctor{open: open, js: server.JetStream(), keeper: keeper, teller: bus, watchdogs: w, host: host}
|
||||
w.doctor = d
|
||||
doctorFrom = d
|
||||
watching, stop := context.WithCancel(ctx)
|
||||
go w.keep(watching)
|
||||
go d.keep(watching)
|
||||
// And the healers (novox/hq to-be 45 §7): what is open that a registered healer answers, repaired
|
||||
// under the lease and the brake, every act said.
|
||||
healers := newHealing(open, keeper, bus, server.JetStream())
|
||||
go healers.keep(watching)
|
||||
go forgettingOldHeals(watching, open.inventory)
|
||||
fmt.Printf("watching the mesh: %d signal(s) every %s, %d probe(s) every %s; what is wrong is kept in %s "+
|
||||
"and said as %s events\n", len(watchedRows()), watchEvery, len(runnableProbes()), doctorEvery,
|
||||
broker.ConditionsBucket, conditions.Seat)
|
||||
return func() {
|
||||
stop()
|
||||
stopHearing()
|
||||
flushing, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
keeper.Close(flushing)
|
||||
}
|
||||
}
|
||||
|
||||
// runnableProbes are the probes the registry runs.
|
||||
func runnableProbes() []probe {
|
||||
var out []probe
|
||||
for _, p := range probeRegistry {
|
||||
if p.run != nil {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -5,9 +5,7 @@ go 1.26.0
|
||||
require (
|
||||
github.com/jackc/pgx/v5 v5.10.0
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
github.com/novox/mesh-host v0.0.0
|
||||
golang.org/x/crypto v0.57.0
|
||||
golang.org/x/net v0.58.0
|
||||
)
|
||||
|
||||
require (
|
||||
@@ -17,15 +15,8 @@ require (
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/net v0.58.0 // indirect
|
||||
golang.org/x/sync v0.23.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/text v0.42.0 // indirect
|
||||
)
|
||||
|
||||
// The node-engine's own validator (mesh-host/validate, novox/hq to-be 45 D1): one validator, the host's.
|
||||
// The host's module path names no forge a build can fetch from, so the module is read from the one
|
||||
// that holds it, at the host's commit — **once, by whoever moves the pin, into vendor/**, which is
|
||||
// committed. Every build (the build agent's `go build`, the Dockerfile) compiles from vendor/ and
|
||||
// fetches nothing; go refuses to build when vendor/ and this file disagree, so a pin moved without
|
||||
// `go mod vendor` fails loudly, at once, everywhere.
|
||||
replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261006095519-3e80b7ae325e
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261006095519-3e80b7ae325e h1:g9h4QRaAMg5yaJLwqtb0FoOs23DVGUYpW6qvnQ3oY5A=
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261006095519-3e80b7ae325e/go.mod h1:VlilMCRZ5yyNXg7SNigNBLr0Gt32jrGw5KSNq5JAVYs=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
|
||||
@@ -1,174 +0,0 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// The controller's own key-value buckets (novox/hq to-be 45 §1, §6, §7).
|
||||
//
|
||||
// **What the controller must remember across its own restart, it keeps on the bus.** A call's
|
||||
// outcome lived in the memory of the process that served it (novox/hq issue 265), so a controller
|
||||
// replaced while a push ran answered "no such call" for the one thing its caller had been told to
|
||||
// ask about. The bus already outlives the controller and is the shape ADR 0201 gives a module's
|
||||
// current state: one value per key, written by one owner, read by anybody granted it. These are the
|
||||
// controller's, written by it alone — the writers table of to-be 45 §1 — and asserted on every start
|
||||
// like the streams, so a bus raised from nothing has them before the first call is served.
|
||||
|
||||
// CallsBucket keeps every call of the mesh's own verbs and what came of it; HandActsBucket every act
|
||||
// a person did by hand, with why; ConditionsBucket every condition open now (to-be 45 §2), one key
|
||||
// each, and ConditionHistoryBucket every transition of one — raised, changed, silenced, cleared —
|
||||
// for ninety days.
|
||||
//
|
||||
// **The history is a bucket of its own** because its keys expire and an open condition's must not:
|
||||
// a bucket has one age for every key, and a condition open longer than the history is kept would
|
||||
// otherwise vanish from the store while still true.
|
||||
var (
|
||||
CallsBucket = BucketName(ControllerSeat, "calls")
|
||||
HandActsBucket = BucketName(ControllerSeat, "hand-acts")
|
||||
ConditionsBucket = BucketName(ControllerSeat, "conditions")
|
||||
ConditionHistoryBucket = BucketName(ControllerSeat, "condition-history")
|
||||
// LeaseBucket holds the controller's lease (to-be 45 §6): one key, `holder`, which the instance
|
||||
// allowed to act writes by compare-and-set and renews; its revision when taken is the epoch.
|
||||
LeaseBucket = BucketName(ControllerSeat, "lease")
|
||||
)
|
||||
|
||||
// LeaseTTL is how long the lease's key lives unrenewed (to-be 45 §6): fifteen seconds, renewed
|
||||
// every five. The bucket's age, so the bus forgets a holder that stopped renewing.
|
||||
const LeaseTTL = 15 * time.Second
|
||||
|
||||
// The bounds to-be 45 §6 sets for calls: the last thousand, or fourteen days, whichever is fewer.
|
||||
// A call is two keys — its record, and its answer apart so a listing does not read every answer —
|
||||
// so the stream holds twice as many messages as it keeps calls.
|
||||
const (
|
||||
KeptCallsDurably = 1000
|
||||
CallsKeptFor = 14 * 24 * time.Hour
|
||||
// CallAnswerBytes is the most of one answer kept: a whole declaration is far smaller, and an
|
||||
// answer larger is cut and says so.
|
||||
CallAnswerBytes = 64 << 10
|
||||
// HandActsKeptFor is as long as a condition's history (to-be 45 §2): an act by hand is read
|
||||
// back beside what it addressed.
|
||||
HandActsKeptFor = 90 * 24 * time.Hour
|
||||
// ConditionHistoryKeptFor is how long a condition's transitions are kept (to-be 45 §2).
|
||||
ConditionHistoryKeptFor = 90 * 24 * time.Hour
|
||||
)
|
||||
|
||||
// IsControllerBucket says a bucket is the controller's own, not a module's state nothing declares.
|
||||
func IsControllerBucket(bucket string) bool {
|
||||
return bucket == CallsBucket || bucket == HandActsBucket || bucket == ConditionsBucket ||
|
||||
bucket == ConditionHistoryBucket || bucket == LeaseBucket
|
||||
}
|
||||
|
||||
// ControllerBuckets are the controller's own buckets, in the order they are asserted.
|
||||
func ControllerBuckets() []string {
|
||||
return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket}
|
||||
}
|
||||
|
||||
// ControllerBucketsAsserter is what raising the controller's buckets needs of a connection.
|
||||
type ControllerBucketsAsserter interface {
|
||||
EnsureControllerBuckets() error
|
||||
}
|
||||
|
||||
// EnsureControllerBuckets creates the controller's buckets if absent and brings their options to
|
||||
// match. An update, never a delete: what they hold is the record of what the mesh was asked.
|
||||
func (j *JetStream) EnsureControllerBuckets() error {
|
||||
js, err := jetstream.New(j.conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if err := EnsureLeaseBucket(ctx, js); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: CallsBucket,
|
||||
Description: "the calls of the mesh's own verbs and what came of each (novox/hq to-be 45 §6, issue " +
|
||||
"265): written by the controller alone, read through `calls`; the last thousand, or fourteen days",
|
||||
History: 1,
|
||||
TTL: CallsKeptFor,
|
||||
MaxValueSize: CallAnswerBytes + 4<<10,
|
||||
MaxBytes: 2 * KeptCallsDurably * (CallAnswerBytes + 4<<10),
|
||||
Storage: jetstream.FileStorage,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", CallsBucket, err)
|
||||
}
|
||||
// **The count, on the stream under the bucket.** A bucket has an age and a size and no count;
|
||||
// the stream it is made of does, and with one value per key the oldest message is the oldest
|
||||
// call. Asserted after the bucket, every time, because asserting the bucket writes the stream's
|
||||
// configuration whole and puts the count back to none.
|
||||
stream, err := js.Stream(ctx, "KV_"+CallsBucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading the stream under %s: %w", CallsBucket, err)
|
||||
}
|
||||
cfg := stream.CachedInfo().Config
|
||||
if cfg.MaxMsgs != 2*KeptCallsDurably {
|
||||
cfg.MaxMsgs = 2 * KeptCallsDurably
|
||||
cfg.Discard = jetstream.DiscardOld
|
||||
if _, err := js.UpdateStream(ctx, cfg); err != nil {
|
||||
return fmt.Errorf("bounding %s to the last %d calls: %w", CallsBucket, KeptCallsDurably, err)
|
||||
}
|
||||
}
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: HandActsBucket,
|
||||
Description: "every act a person did by hand, with why (novox/hq to-be 45 §7): written by the " +
|
||||
"controller's repairing verbs and `hand-act record`, read through `hand-acts`",
|
||||
History: 1,
|
||||
TTL: HandActsKeptFor,
|
||||
MaxValueSize: 16 << 10,
|
||||
MaxBytes: 64 << 20,
|
||||
Storage: jetstream.FileStorage,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", HandActsBucket, err)
|
||||
}
|
||||
// **No age on the open conditions.** A condition is removed when observation clears it and at no
|
||||
// other moment: one that expired would be a fault the store forgot while it was still true.
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: ConditionsBucket,
|
||||
Description: "every condition open now, one key each (novox/hq to-be 45 §2): written by the " +
|
||||
"controller alone, raised and cleared by observation, read through `conditions`",
|
||||
History: 1,
|
||||
MaxValueSize: 64 << 10,
|
||||
MaxBytes: 64 << 20,
|
||||
Storage: jetstream.FileStorage,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", ConditionsBucket, err)
|
||||
}
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: ConditionHistoryBucket,
|
||||
Description: "every transition of a condition — raised, changed, silenced, cleared — kept ninety " +
|
||||
"days (novox/hq to-be 45 §2): written by the controller alone, read through `conditions history`",
|
||||
History: 1,
|
||||
TTL: ConditionHistoryKeptFor,
|
||||
MaxValueSize: 64 << 10,
|
||||
MaxBytes: 256 << 20,
|
||||
Storage: jetstream.FileStorage,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", ConditionHistoryBucket, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// EnsureLeaseBucket creates the lease's bucket if absent and brings its options to match (to-be 45
|
||||
// §6). **Before the lease is taken, by any candidate**: it is the lease's own precondition, and asserting
|
||||
// a bucket that exists changes nothing. File storage, so its revisions — the epochs — outlive a restart of
|
||||
// the bus; one raised again from nothing is moved past the highest epoch issued when the lease is taken.
|
||||
func EnsureLeaseBucket(ctx context.Context, js jetstream.JetStream) error {
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: LeaseBucket,
|
||||
Description: "the controller's lease (novox/hq to-be 45 §6): the key `holder`, written by compare-and-set " +
|
||||
"by the one controller instance that may act and renewed every five seconds; its revision when taken " +
|
||||
"is the epoch every declaration and plan write carries",
|
||||
History: 1,
|
||||
TTL: LeaseTTL,
|
||||
MaxValueSize: 4 << 10,
|
||||
MaxBytes: 1 << 20,
|
||||
Storage: jetstream.FileStorage,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", LeaseBucket, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,61 +0,0 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// **The controller may write every bucket it writes** (novox/hq to-be 45 §1, issue 269). Writing a
|
||||
// key is a publish to the bucket's own subject, which the management interface's grant does not
|
||||
// cover: the cancelled sets' writes timed out for want of this, and the controller's own buckets
|
||||
// would have.
|
||||
func TestTheControllerMayWriteEveryBucketItWrites(t *testing.T) {
|
||||
p, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{"$KV." + CallsBucket + ".>", "$KV." + HandActsBucket + ".>"}
|
||||
for _, seat := range seatsTheControllerAsks {
|
||||
if hasCancelledSet(seat) {
|
||||
want = append(want, "$KV."+CancelledSetName(seat)+".>")
|
||||
}
|
||||
}
|
||||
for _, subject := range want {
|
||||
if !slices.Contains(p.Publish, subject) {
|
||||
t.Errorf("the controller may not publish %s, so it cannot write that bucket", subject)
|
||||
}
|
||||
}
|
||||
if slices.Contains(p.Publish, "$KV.>") {
|
||||
t.Error("the controller may write any bucket, a module's state included")
|
||||
}
|
||||
}
|
||||
|
||||
// **A machine's node tools may say they are there, as that machine and no other** (novox/hq to-be 45
|
||||
// S11), and the controller may hear the bus's advisories and ask who answers — read-only, named.
|
||||
func TestTheWatchedSignalsMayBeSaidAndHeard(t *testing.T) {
|
||||
tools, err := PermissionsFor(Principal{Kind: KindNodeTools, Node: "anchor", Module: RuntimeModule, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Contains(tools.Publish, "mesh.control.anchor.tools-alive") {
|
||||
t.Error("the node tools may not say they are there")
|
||||
}
|
||||
for _, s := range tools.Publish {
|
||||
if strings.Contains(s, "tools-alive") && s != "mesh.control.anchor.tools-alive" {
|
||||
t.Errorf("the node tools may say %s", s)
|
||||
}
|
||||
}
|
||||
controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, s := range BusAdvisories {
|
||||
if !slices.Contains(controller.Subscribe, s) {
|
||||
t.Errorf("the controller may not hear %s", s)
|
||||
}
|
||||
}
|
||||
if !slices.Contains(controller.Publish, "$SRV.INFO") || slices.Contains(controller.Subscribe, "$JS.EVENT.>") {
|
||||
t.Error("the controller may not ask who answers, or hears every API call")
|
||||
}
|
||||
}
|
||||
@@ -54,11 +54,6 @@ type Consumer struct {
|
||||
// that exists keeps where it is, whatever this says; only its making is decided here.
|
||||
FromNow bool
|
||||
Why string
|
||||
// Resettable says why this consumer may be re-made to deliver from now by the mesh itself, with
|
||||
// nobody asked (novox/hq to-be 45 §7, healer H4): what a reset drops, something else catches up.
|
||||
// Empty for every consumer where nothing would — a module's, whose events would be lost to it.
|
||||
// Not a property of the consumer on the bus: nothing here is sent to the server.
|
||||
Resettable string
|
||||
}
|
||||
|
||||
// seatStreamName is the stream holding a seat's inbound work. Named after the seat rather than
|
||||
|
||||
+2
-58
@@ -125,16 +125,6 @@ type Principal struct {
|
||||
// goes with the retired seat row.
|
||||
var seatsTheControllerAsks = []string{"node-build-agent", "mesh-build-machine"}
|
||||
|
||||
// SeatVerb is one verb of one seat, on every machine holding it.
|
||||
type SeatVerb struct{ Seat, Verb string }
|
||||
|
||||
// VerbsTheSelfCheckAsks are the seat verbs the controller's self-check and watchdogs call (novox/hq
|
||||
// to-be 45 §4): D8 reads every machine's ban list. **Named one by one, and the test that holds them
|
||||
// to the probe registry is the reason they cannot drift** — a probe that calls a verb its grant does
|
||||
// not name is refused by the bus on every run (found live on 2026-10-06: D8 timed out on each
|
||||
// machine, refused). Asked of any machine (`.*`), read-only verbs, nothing else of the seat.
|
||||
var VerbsTheSelfCheckAsks = []SeatVerb{{Seat: "node-intrusion-prevention", Verb: "banned"}}
|
||||
|
||||
// perMachineEvents are a node-scoped seat's events about the holder itself, whose last token is the
|
||||
// holder's machine (novox/hq ADR 0219): `paused.<node>`, the build agent saying whether it takes work.
|
||||
var perMachineEvents = map[string]bool{"paused.*": true}
|
||||
@@ -179,10 +169,6 @@ func (p Principal) Username() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// AskReportSubject is where the mesh asks one machine's node-engine to say again what it last applied
|
||||
// (novox/hq to-be 45 §6): its answer is an ordinary report, on its own report subject.
|
||||
func AskReportSubject(node string) string { return "mesh.node." + node + ".ask.report" }
|
||||
|
||||
// inbox is a principal's own reply space. No user is ever granted a bare `_INBOX.>` (design 25
|
||||
// §4): with one account, inbox privacy is the permission list or it is nothing, so each user's
|
||||
// inbox is derived from its own identity and its permissions name that prefix and no other.
|
||||
@@ -233,12 +219,7 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// stream definitions (design 25 §3), so it alone reaches the JetStream API — and it alone
|
||||
// issues memberships (novox/hq ADR 0160), which it publishes into the assignments stream
|
||||
// after each push; refused by the server on 2026-10-01 until this line named them.
|
||||
//
|
||||
// **Not what the machines say** (novox/hq to-be 45 §1): `mesh.control.>` is subscribed, never
|
||||
// published — a machine's report has one writer, its node-engine, and the controller granted
|
||||
// that subject would be a second (the writers table refuses it). It was granted from the first
|
||||
// composition and nothing ever published under it.
|
||||
pub = []string{"mesh.node.>", "mesh.assignment.>", "$JS.API.>"}
|
||||
pub = []string{"mesh.control.>", "mesh.node.>", "mesh.assignment.>", "$JS.API.>"}
|
||||
// **And where its consumers deliver.** A push consumer delivers on `_DELIVER.<its name>`,
|
||||
// and a client bound to it subscribes exactly that; the server refused it for every
|
||||
// principal the first time one bound a consumer (2026-09-28). Each kind below is granted
|
||||
@@ -256,12 +237,6 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// building, kill it, pause it, resume it. The queue is the controller's to show and to
|
||||
// change, and what one machine is doing with an ask it took only that machine can say.
|
||||
pub = append(pub, "mesh.seat."+seat+".tool.>")
|
||||
// **And its cancelled set** (novox/hq ADR 0219, issue 269): a cancel writes the ask's id
|
||||
// there before it deletes the ask, and a write is a publish to the bucket's subject, which
|
||||
// `$JS.API.>` does not cover — so every cancel timed out, refused by this list.
|
||||
if hasCancelledSet(seat) {
|
||||
pub = append(pub, "$KV."+CancelledSetName(seat)+".>")
|
||||
}
|
||||
}
|
||||
// **And what the mesh says it did** (novox/hq ADR 0134). The control plane states its own
|
||||
// facts under the seat it holds, because a role's events belong to the role and keep their
|
||||
@@ -285,13 +260,6 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
sub = append(sub, "mesh.seat."+ControllerSeat+".tool.>")
|
||||
// And says so (novox/hq ADR 0197): it answers discovery for the seat it serves.
|
||||
sub = append(sub, announcing(ControllerSeat)...)
|
||||
// And the verbs the self-check reads with, of any machine's holder (novox/hq to-be 45 §4).
|
||||
for _, v := range VerbsTheSelfCheckAsks {
|
||||
pub = append(pub, "mesh.seat."+v.Seat+".tool."+v.Verb+".*")
|
||||
}
|
||||
// And asks who answers (novox/hq to-be 45 §4, D3): the self-check finds every seat's holder by
|
||||
// the same discovery the console reads. The question only; the answers come to its own inbox.
|
||||
pub = append(pub, "$SRV.INFO")
|
||||
|
||||
// The events it reacts to, and its ack subject on the stream they arrive from
|
||||
// (streams.go). **Each named, not a pattern**: `mesh.mod.*.event.>` would make the
|
||||
@@ -320,18 +288,6 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// enrolments and can reach nothing else.
|
||||
pub = append(pub, "_INBOX."+enrolmentPrefix+".>")
|
||||
|
||||
// **And its own buckets** (novox/hq to-be 45 §1): the calls it served and the acts done by
|
||||
// hand, which it alone writes. A put is a publish to the bucket's subject, which `$JS.API.>`
|
||||
// does not cover; each bucket named, not `$KV.>`, which would let it write any module's state.
|
||||
for _, bucket := range ControllerBuckets() {
|
||||
pub = append(pub, "$KV."+bucket+".>")
|
||||
}
|
||||
// **And what the bus says about itself, read-only** (novox/hq to-be 45 §3, S9): a durable
|
||||
// consumer that gave up on a message, or one that was deleted. The server already publishes
|
||||
// both in the mesh's own account; the controller says each as a condition in the mesh's words.
|
||||
// Named, not `$JS.EVENT.>`: the other advisories are every API call the mesh makes.
|
||||
sub = append(sub, BusAdvisories...)
|
||||
|
||||
case KindPerson:
|
||||
// Tools, and nothing else. Every subject a person may publish is a tool call; a person
|
||||
// who could publish an event would be able to claim a module said something.
|
||||
@@ -382,11 +338,7 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// next assertion moves it, and a permission that only allowed the new shape would refuse
|
||||
// every node in the mesh for exactly as long as that took.
|
||||
sub = []string{"mesh.node." + p.Node + ".declare",
|
||||
"_DELIVER." + p.Node, "_DELIVER." + p.Node + ".>",
|
||||
// And the mesh asking it to say again what it last applied (novox/hq to-be 45 §6, the
|
||||
// `report` verb healer H1 asks): its own machine's, on core NATS and off any stream. It
|
||||
// answers through its report, the one thing it already says — no reply to anybody's inbox.
|
||||
AskReportSubject(p.Node)}
|
||||
"_DELIVER." + p.Node, "_DELIVER." + p.Node + ".>"}
|
||||
|
||||
case KindModule:
|
||||
// 1. Its own namespace: it publishes its events there and serves its tools there. Nothing
|
||||
@@ -547,9 +499,6 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// that varies is the module, so the pattern is the machine's own assignments.
|
||||
sub = append(sub, "mesh.assignment."+p.Node+".*")
|
||||
pub = append(pub, "$JS.API.DIRECT.GET."+AssignmentsStream+".mesh.assignment."+p.Node+".*")
|
||||
// And that it is there (novox/hq to-be 45 §3, S11): its own heartbeat, under its machine's
|
||||
// name and no other's, on core NATS like the host's.
|
||||
pub = append(pub, "mesh.control."+p.Node+".tools-alive")
|
||||
// And every tool on the mesh (ADR 0175, decision 5): any node may call any tool on any
|
||||
// node, as the console already could — the runtime is the console's serving mode.
|
||||
invoked, err := invokedSubjects([]string{"*"})
|
||||
@@ -617,11 +566,6 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
|
||||
sort.Strings(pub)
|
||||
sort.Strings(sub)
|
||||
// One writer per piece of state (novox/hq to-be 45 §1): a grant that would make a second is
|
||||
// refused here, at composition, naming the state and its writer.
|
||||
if err := CheckWriters(p, pub); err != nil {
|
||||
return Permissions{}, err
|
||||
}
|
||||
return Permissions{
|
||||
Publish: pub,
|
||||
Subscribe: sub,
|
||||
|
||||
@@ -160,9 +160,8 @@ func RaiseBuckets(a BucketAsserter, buckets []Bucket) (undeclared []string, err
|
||||
return nil, fmt.Errorf("listing the bus's state: %w", err)
|
||||
}
|
||||
for _, n := range names {
|
||||
// A seat's cancelled set is the mesh's own (novox/hq ADR 0219), not a module's state; so are
|
||||
// the controller's own buckets (novox/hq to-be 45 §1).
|
||||
if !declared[n] && !IsCancelledSet(n) && !IsControllerBucket(n) {
|
||||
// A seat's cancelled set is the mesh's own (novox/hq ADR 0219), not a module's state.
|
||||
if !declared[n] && !IsCancelledSet(n) {
|
||||
undeclared = append(undeclared, n)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
@@ -21,19 +20,12 @@ func TestTheFactsTheGrantPermitsAreTheFactsTheMeshStates(t *testing.T) {
|
||||
t.Fatalf("the grant is written for the %q seat and the mesh states its facts under %q",
|
||||
broker.ControllerSeat, link.MeshControllerSeat)
|
||||
}
|
||||
// And what is wrong, as it changes, and the self-check's heartbeat (novox/hq to-be 45 §2, §4).
|
||||
states := append([]string{link.KeyApplied, link.KeyRefused, link.KeyBuiltBefore}, conditions.Events...)
|
||||
states = append(states, conditions.HeartbeatEvent)
|
||||
// And a value given by hand, replaced after its module's first good start (novox/hq ADR 0228).
|
||||
states = append(states, link.KeySecretReplaced)
|
||||
// And every act a healer takes (novox/hq to-be 45 §7).
|
||||
states = append(states, link.KeyHealerActed)
|
||||
for _, event := range states {
|
||||
for _, event := range []string{link.KeyApplied, link.KeyRefused, link.KeyBuiltBefore} {
|
||||
if !slices.Contains(broker.ControllerStates, event) {
|
||||
t.Errorf("the mesh states %q and its account may not publish it", event)
|
||||
}
|
||||
}
|
||||
if len(broker.ControllerStates) != len(states) {
|
||||
if len(broker.ControllerStates) != 3 {
|
||||
t.Errorf("the grant permits %v, which is more than the mesh states", broker.ControllerStates)
|
||||
}
|
||||
// **And the seat says it.** A seat carries the protocol of its role (novox/hq ADR 0129), so the
|
||||
|
||||
@@ -201,28 +201,7 @@ const ControllerName = "controller"
|
||||
// something to say.
|
||||
const ControllerSeat = "mesh-controller"
|
||||
|
||||
var ControllerStates = []string{"applied", "refused", "built-before",
|
||||
// What is wrong, said as it changes (novox/hq to-be 45 §2): a condition raised, changed in
|
||||
// severity, resolver or silence, and cleared. The operator-channel's holder and any other surface
|
||||
// consume them; the controller tells nobody itself.
|
||||
"condition-raised", "condition-changed", "condition-cleared",
|
||||
// And the self-check's heartbeat, at the end of every run (to-be 45 §4, S10): watched from a
|
||||
// machine that is not the control node, so the controller going quiet is itself said.
|
||||
"doctor-heartbeat",
|
||||
// And a value given by hand, replaced after its module's first good start (novox/hq ADR 0228).
|
||||
"secret-replaced",
|
||||
// And every act a healer takes on a condition (novox/hq to-be 45 §7, Phase 3): a repair the mesh
|
||||
// made by itself is said like one a person made, never quietly.
|
||||
"healer-acted"}
|
||||
|
||||
// BusAdvisories are what the bus server says about the mesh's own account that the controller
|
||||
// reads (novox/hq to-be 45 §3, S9): a durable consumer that handed a message over as often as it
|
||||
// may and gave up on it, and one that was deleted. Read-only: an advisory is the server's to
|
||||
// publish, and the controller's subscription changes nothing on the bus.
|
||||
var BusAdvisories = []string{
|
||||
"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>",
|
||||
"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>",
|
||||
}
|
||||
var ControllerStates = []string{"applied", "refused", "built-before"}
|
||||
|
||||
// ControllerFollows are the events the controller reacts to: the catalogue saying a module's
|
||||
// current version moved, and a catalogue that has just started saying it may have missed builds.
|
||||
@@ -255,18 +234,13 @@ var ControllerFollows = []string{
|
||||
// the index is a name.
|
||||
moduleEventSubject("*", ProvisionerFailing),
|
||||
moduleEventSubject("*", ProvisionerRecovered),
|
||||
// **What becomes of a consumer the mesh stopped asking for** (novox/hq ADR 0230): retired, waiting
|
||||
// for a person, re-enabled, deleted — a provider's third word, from whichever module provides.
|
||||
// Appended, because the index is a name.
|
||||
moduleEventSubject("*", ProvisionerRetirement),
|
||||
}
|
||||
|
||||
// The provider standing events, by their local names. Written here as well as in the catalogue
|
||||
// (catalogue.ProvisionerEvents), which this package cannot import; a test keeps them agreeing.
|
||||
const (
|
||||
ProvisionerFailing = "provisioner.failing"
|
||||
ProvisionerRecovered = "provisioner.recovered"
|
||||
ProvisionerRetirement = "provisioner.retirement"
|
||||
ProvisionerFailing = "provisioner.failing"
|
||||
ProvisionerRecovered = "provisioner.recovered"
|
||||
)
|
||||
|
||||
// moduleEventSubject is where one module's event lands. The same derivation PermissionsFor uses, so
|
||||
@@ -312,13 +286,6 @@ func MeshConsumers() []Consumer {
|
||||
// client and come back to be acted on again.
|
||||
MaxAckPending: 1,
|
||||
FromNow: true,
|
||||
// **The one consumer the mesh resets by itself** (healer H4, issue 248): it fell a week
|
||||
// behind once and held every merge after it. What a reset drops is caught up elsewhere —
|
||||
// a merge by the catch-up pass that reads the forge (issue 266), a build's outcome from the
|
||||
// build records a plan settles from (issue 214), a provider's failing word by the provider
|
||||
// saying it again every quarter of an hour (ADR 0224).
|
||||
Resettable: "what it drops is caught up: merges by the catch-up pass (issue 266), build outcomes " +
|
||||
"from the build records (issue 214), a provider's failing word said again (ADR 0224)",
|
||||
Why: "the events the mesh's own controller reacts to, one at a time; after " +
|
||||
"max-deliver it dead-letters, because an announcement it cannot act on will not " +
|
||||
"become actionable",
|
||||
|
||||
+3
-3
@@ -24,8 +24,8 @@ accounts {
|
||||
jetstream: enabled
|
||||
users = [
|
||||
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] }
|
||||
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>"] }
|
||||
subscribe: { allow: ["$JS.API.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
{ user: "enrol.one", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: {
|
||||
@@ -34,7 +34,7 @@ accounts {
|
||||
} }
|
||||
{ user: "node.one", password: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn", permissions: {
|
||||
publish: { allow: ["$JS.ACK.NODES.one.>", "$JS.API.CONSUMER.INFO.NODES.one", "mesh.control.one.>"] }
|
||||
subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.node.one.ask.report", "mesh.node.one.declare"] }
|
||||
subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.node.one.declare"] }
|
||||
} }
|
||||
{ user: "one.telegram", password: "$2a$11$tttttttttttttttttttttt", permissions: {
|
||||
publish: { allow: ["$JS.ACK.EVENTS.one_telegram.>", "$JS.ACK.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker.>", "$JS.API.CONSUMER.INFO.EVENTS.one_telegram", "$JS.API.CONSUMER.INFO.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_telegram", "$JS.API.CONSUMER.MSG.NEXT.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.one.telegram", "mesh.seat.telegram-sender.event.delivered", "mesh.seat.telegram-sender.event.failed"] }
|
||||
|
||||
@@ -1,172 +0,0 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The writers table (novox/hq to-be 45 §1, ADR 0227 rule 1), compiled in.
|
||||
//
|
||||
// **Every kind of state the core keeps has one writer; anyone else asks it.** The table is the design's,
|
||||
// row for row, with what each row writes on the bus where it writes there. Two things read it: the
|
||||
// composition of every principal's grants (PermissionsFor), which **refuses a grant that lets a
|
||||
// principal publish on a subject another writes** — so a second writer cannot be granted by accident,
|
||||
// and the composition says which state and whose — and the test beside it, which walks the rows
|
||||
// against the design's list and the composition of a whole mesh. Changing a writer is a change to
|
||||
// this table, through a decision.
|
||||
|
||||
// WriterRow is one row of the writers table.
|
||||
type WriterRow struct {
|
||||
State string
|
||||
Writer string
|
||||
KeptIn string
|
||||
Others string
|
||||
// Subjects are the bus subjects a write of this state is a publish to; none for state kept off the
|
||||
// bus (the controller's store, a module's own) or not built yet.
|
||||
Subjects []string
|
||||
// Writes says a principal granted a publish pattern overlapping Subjects is this state's writer —
|
||||
// given the pattern, because a machine writes its own report and no other's.
|
||||
Writes func(p Principal, pattern string) bool
|
||||
// Shared says why more than one principal may publish here; empty for one writer.
|
||||
Shared string
|
||||
}
|
||||
|
||||
// isController, and the other writers' tests, are who a row's writer is as a principal.
|
||||
func isController(p Principal, _ string) bool { return p.Kind == KindController }
|
||||
|
||||
// ownMachine is a node writing a subject whose third token is its own name, and no wildcard there.
|
||||
func ownMachine(p Principal, pattern string) bool {
|
||||
tokens := strings.Split(pattern, ".")
|
||||
return p.Kind == KindNode && len(tokens) > 2 && tokens[2] == p.Node
|
||||
}
|
||||
|
||||
// holdsSeatOf is a principal holding the seat a `mesh.seat.<seat>.…` pattern names: its module's own
|
||||
// principal, or the node tools carrying a module that holds it (ADR 0175, the runtime is its modules).
|
||||
func holdsSeatOf(p Principal, pattern string) bool {
|
||||
tokens := strings.Split(pattern, ".")
|
||||
if len(tokens) < 3 {
|
||||
return false
|
||||
}
|
||||
seat := tokens[2]
|
||||
holds := func(seats []Seat) bool {
|
||||
return slices.ContainsFunc(seats, func(s Seat) bool { return s.Name == seat })
|
||||
}
|
||||
switch p.Kind {
|
||||
case KindModule:
|
||||
return holds(p.Holds)
|
||||
case KindNodeTools:
|
||||
return slices.ContainsFunc(p.Carries, func(d Declared) bool { return holds(d.Holds) })
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ownModule is a module publishing under its own name, or the node tools carrying it.
|
||||
func ownModule(p Principal, pattern string) bool {
|
||||
tokens := strings.Split(pattern, ".")
|
||||
if len(tokens) < 3 {
|
||||
return false
|
||||
}
|
||||
module := tokens[2]
|
||||
switch p.Kind {
|
||||
case KindModule:
|
||||
return p.Module == module
|
||||
case KindNodeTools:
|
||||
return slices.ContainsFunc(p.Carries, func(d Declared) bool { return d.Module == module })
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// kvOf is a bucket's write subjects.
|
||||
func kvOf(bucket string) []string { return []string{"$KV." + bucket + ".>"} }
|
||||
|
||||
// WritersTable is to-be 45 §1, in its order.
|
||||
var WritersTable = []WriterRow{
|
||||
{State: "a machine's declaration", Writer: "controller (lease holder)", KeptIn: "the bus, last per subject",
|
||||
Others: "read", Subjects: []string{"mesh.node.*.declare"}, Writes: isController},
|
||||
{State: "a machine's applied state and its report", Writer: "the node-engine's apply queue",
|
||||
KeptIn: "the machine; the report on the bus", Others: "the reconcile and a delivery enqueue, never apply",
|
||||
Subjects: []string{"mesh.control.*.report"}, Writes: ownMachine},
|
||||
{State: "the controller lease", Writer: "the controller instance holding it", KeptIn: "key-value " + LeaseBucket,
|
||||
Others: "a candidate waits", Subjects: kvOf(LeaseBucket), Writes: isController},
|
||||
{State: "plans and their tiers", Writer: "controller (lease holder), compare-and-set on the plan's revision",
|
||||
KeptIn: "the controller's store", Others: "read through plans"},
|
||||
{State: "conditions", Writer: "controller", KeptIn: "key-value " + ConditionsBucket + " (and its history, " +
|
||||
ConditionHistoryBucket + ")", Others: "raise or clear only through observations the controller reads",
|
||||
Subjects: append(kvOf(ConditionsBucket), kvOf(ConditionHistoryBucket)...), Writes: isController},
|
||||
{State: "calls and their outcomes", Writer: "controller", KeptIn: "key-value " + CallsBucket,
|
||||
Others: "read by id", Subjects: kvOf(CallsBucket), Writes: isController},
|
||||
{State: "the hand-act log", Writer: "controller, through the verbs that act", KeptIn: "key-value " + HandActsBucket,
|
||||
Others: "—", Subjects: kvOf(HandActsBucket), Writes: isController},
|
||||
// What the healers did (novox/hq to-be 45 §7, Phase 3): the controller's alone, in its store — the
|
||||
// budgets and the mesh-wide brake are counted from it, so a controller restarting cannot reset them.
|
||||
{State: "the healers' acts and their brake", Writer: "controller (lease holder), each act begun before it is made",
|
||||
KeptIn: "the controller's store", Others: "read through healers; each act said as healer-acted and in its condition's tried"},
|
||||
{State: "stream definitions and bus permissions", Writer: "controller", KeptIn: "the bus", Others: "—",
|
||||
// A stream's definition, and a durable consumer's by the API that names it so. Not every
|
||||
// consumer create: a module watching its own bucket makes and deletes an ordered consumer on the
|
||||
// bucket's stream (ADR 0201), which defines nothing the mesh keeps.
|
||||
Subjects: []string{"$JS.API.STREAM.CREATE.>", "$JS.API.STREAM.UPDATE.>", "$JS.API.STREAM.DELETE.>",
|
||||
"$JS.API.CONSUMER.DURABLE.CREATE.>"},
|
||||
Writes: isController},
|
||||
{State: "builds and their outcomes", Writer: "the build seat's holder", KeptIn: "its own state",
|
||||
Others: "the controller asks",
|
||||
Subjects: []string{"mesh.seat.node-build-agent.event.built", "mesh.seat.mesh-build-machine.event.built"},
|
||||
Writes: holdsSeatOf,
|
||||
Shared: "every machine holding the build seat answers the asks it took; each outcome names its ask"},
|
||||
{State: "a merge announced", Writer: "one announcer per forge (the hook, or the poll when the hook is absent — never both)",
|
||||
KeptIn: "the bus", Others: "—", Subjects: []string{"mesh.mod.*.event.pull.merged"}, Writes: ownModule},
|
||||
{State: "a provider's standing", Writer: "the provider", KeptIn: "the provider's events",
|
||||
Others: "the controller keeps the newest word as a condition",
|
||||
Subjects: []string{"mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered",
|
||||
"mesh.mod.*.event.provisioner.retirement"},
|
||||
Writes: ownModule},
|
||||
{State: "the operator-channel's open messages", Writer: "the seat's holder", KeptIn: "its own key-value state",
|
||||
Others: "—"},
|
||||
{State: "the facts snapshot", Writer: "controller", KeptIn: "the artifact store, facts/latest",
|
||||
Others: "the build seat reads"},
|
||||
}
|
||||
|
||||
// CheckWriters refuses a grant that lets a principal publish on a subject the writers table gives
|
||||
// another writer (to-be 45 §1): which state, whose, and the pattern that would make a second writer.
|
||||
func CheckWriters(p Principal, publish []string) error {
|
||||
var problems []string
|
||||
for _, pattern := range publish {
|
||||
for _, row := range WritersTable {
|
||||
if row.Writes == nil {
|
||||
continue
|
||||
}
|
||||
for _, subject := range row.Subjects {
|
||||
if !SubjectsOverlap(pattern, subject) || row.Writes(p, pattern) {
|
||||
continue
|
||||
}
|
||||
problems = append(problems, fmt.Sprintf("%s may publish %s, which writes %s (%s), whose writer is %s",
|
||||
p.Username(), pattern, row.State, subject, row.Writer))
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(problems) == 0 {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("a second writer would be granted (novox/hq to-be 45 §1, one writer per piece of state):\n %s",
|
||||
strings.Join(problems, "\n "))
|
||||
}
|
||||
|
||||
// SubjectsOverlap says some subject matches both patterns: `*` is one token, `>` one or more to the end.
|
||||
func SubjectsOverlap(a, b string) bool {
|
||||
x, y := strings.Split(a, "."), strings.Split(b, ".")
|
||||
for i := 0; ; i++ {
|
||||
switch {
|
||||
case i == len(x) && i == len(y):
|
||||
return true
|
||||
case i == len(x) || i == len(y):
|
||||
return false
|
||||
case x[i] == ">" || y[i] == ">":
|
||||
return true
|
||||
case x[i] == "*" || y[i] == "*" || x[i] == y[i]:
|
||||
continue
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,144 +0,0 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The writers table, checked (novox/hq to-be 45 §1, ADR 0227 rule 1, "how it is checked"): it is the
|
||||
// design's table row for row; every row that writes on the bus names its writer; a whole mesh composes
|
||||
// with one writer per piece of state; and a grant that would make a second writer is refused, naming
|
||||
// the state and whose it is.
|
||||
|
||||
// designRows are the states of to-be 45 §1, in its order. A row added to the design is added here and
|
||||
// to the table; one dropped from either fails.
|
||||
var designRows = []string{
|
||||
"a machine's declaration",
|
||||
"a machine's applied state and its report",
|
||||
"the controller lease",
|
||||
"plans and their tiers",
|
||||
"conditions",
|
||||
"calls and their outcomes",
|
||||
"the hand-act log",
|
||||
"the healers' acts and their brake",
|
||||
"stream definitions and bus permissions",
|
||||
"builds and their outcomes",
|
||||
"a merge announced",
|
||||
"a provider's standing",
|
||||
"the operator-channel's open messages",
|
||||
"the facts snapshot",
|
||||
}
|
||||
|
||||
func TestTheWritersTableIsTheDesigns(t *testing.T) {
|
||||
var states []string
|
||||
for _, row := range WritersTable {
|
||||
states = append(states, row.State)
|
||||
if row.Writer == "" || row.KeptIn == "" || row.Others == "" {
|
||||
t.Errorf("%q does not say who writes it, where it is kept and what others do", row.State)
|
||||
}
|
||||
if len(row.Subjects) > 0 && row.Writes == nil {
|
||||
t.Errorf("%q is written on the bus and names no writer among the principals", row.State)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(states, designRows) {
|
||||
t.Fatalf("the writers table is not to-be 45 §1's:\n have %q\n want %q", states, designRows)
|
||||
}
|
||||
}
|
||||
|
||||
// A mesh of every kind of principal composes: nobody is granted a second writer's subject.
|
||||
func TestAWholeMeshComposesWithOneWriterPerState(t *testing.T) {
|
||||
builder := Seat{Name: "node-build-agent", Scope: "node", Accepts: []string{"build"}, Emits: []string{"built", "started"}}
|
||||
principals := []Principal{
|
||||
{Kind: KindController, PasswordHash: "x"},
|
||||
{Kind: KindNode, Node: "one", PasswordHash: "x"},
|
||||
{Kind: KindEnrolment, Node: "two", PasswordHash: "x"},
|
||||
{Kind: KindPerson, Module: "jochen", Invokes: []string{"*"}, PasswordHash: "x"},
|
||||
{Kind: KindModule, Node: "one", Module: "gitea", Emits: []string{"pull.merged"}, PasswordHash: "x"},
|
||||
{Kind: KindModule, Node: "one", Module: "postgres", Emits: []string{"provisioner.failing", "provisioner.recovered", "provisioner.retirement"},
|
||||
PasswordHash: "x"},
|
||||
{Kind: KindModule, Node: "one", Module: "build-agent", Holds: []Seat{builder}, PasswordHash: "x"},
|
||||
{Kind: KindNodeTools, Node: "one", Module: RuntimeModule, PasswordHash: "x", Carries: []Declared{
|
||||
{Module: "gitea", Emits: []string{"pull.merged"}},
|
||||
{Module: "build-agent", Holds: []Seat{builder}}}},
|
||||
}
|
||||
for _, p := range principals {
|
||||
if _, err := PermissionsFor(p); err != nil {
|
||||
t.Errorf("%s does not compose: %v", p.Username(), err)
|
||||
}
|
||||
}
|
||||
if _, err := ComposeAccounts(principals); err != nil {
|
||||
t.Fatalf("the mesh does not compose: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASecondWriterIsRefusedAtComposition(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
p Principal
|
||||
publish []string
|
||||
state string
|
||||
}{
|
||||
{"the controller publishing what machines say", Principal{Kind: KindController},
|
||||
[]string{"mesh.control.>"}, "a machine's applied state and its report"},
|
||||
{"a machine publishing another's report", Principal{Kind: KindNode, Node: "one"},
|
||||
[]string{"mesh.control.two.report"}, "a machine's applied state and its report"},
|
||||
{"a machine publishing every machine's", Principal{Kind: KindNode, Node: "one"},
|
||||
[]string{"mesh.control.*.>"}, "a machine's applied state and its report"},
|
||||
{"a module writing the lease", Principal{Kind: KindModule, Module: "shop"},
|
||||
[]string{"$KV.mesh-controller_lease.>"}, "the controller lease"},
|
||||
{"a module sending a declaration", Principal{Kind: KindModule, Module: "shop"},
|
||||
[]string{"mesh.node.one.declare"}, "a machine's declaration"},
|
||||
{"a module defining a stream", Principal{Kind: KindModule, Module: "shop"},
|
||||
[]string{"$JS.API.>"}, "stream definitions and bus permissions"},
|
||||
{"a module announcing another forge's merge", Principal{Kind: KindModule, Module: "shop"},
|
||||
[]string{"mesh.mod.gitea.event.pull.merged"}, "a merge announced"},
|
||||
{"a module saying a build it did not do", Principal{Kind: KindModule, Module: "shop"},
|
||||
[]string{"mesh.seat.node-build-agent.event.built"}, "builds and their outcomes"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
err := CheckWriters(c.p, c.publish)
|
||||
if err == nil {
|
||||
t.Fatalf("%v granted to %s was not refused", c.publish, c.p.Username())
|
||||
}
|
||||
if !strings.Contains(err.Error(), c.state) {
|
||||
t.Fatalf("the refusal does not name %q: %v", c.state, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
// And the writers themselves are not refused.
|
||||
for _, ok := range []struct {
|
||||
p Principal
|
||||
publish []string
|
||||
}{
|
||||
{Principal{Kind: KindNode, Node: "one"}, []string{"mesh.control.one.>"}},
|
||||
{Principal{Kind: KindController}, []string{"mesh.node.>", "$JS.API.>", "$KV.mesh-controller_lease.>"}},
|
||||
{Principal{Kind: KindModule, Module: "gitea"}, []string{"mesh.mod.gitea.event.pull.merged"}},
|
||||
{Principal{Kind: KindModule, Module: "shop"}, []string{"$JS.API.CONSUMER.CREATE.KV_shop_carts.>"}},
|
||||
} {
|
||||
if err := CheckWriters(ok.p, ok.publish); err != nil {
|
||||
t.Errorf("a writer was refused its own: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubjectsOverlap(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
a, b string
|
||||
want bool
|
||||
}{
|
||||
{"mesh.control.>", "mesh.control.*.report", true},
|
||||
{"mesh.control.one.>", "mesh.control.*.report", true},
|
||||
{"mesh.control.one.alive", "mesh.control.*.report", false},
|
||||
{"mesh.control.*", "mesh.control.*.report", false},
|
||||
{"$JS.API.>", "$JS.API.STREAM.CREATE.>", true},
|
||||
{"$JS.API.CONSUMER.CREATE.KV_x.>", "$JS.API.STREAM.CREATE.>", false},
|
||||
{"a.b", "a.b", true},
|
||||
{"a.b", "a.b.c", false},
|
||||
{"a.>", "a", false},
|
||||
} {
|
||||
if got := SubjectsOverlap(c.a, c.b); got != c.want || SubjectsOverlap(c.b, c.a) != c.want {
|
||||
t.Errorf("%s ~ %s: %v, want %v", c.a, c.b, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -119,41 +119,3 @@ func TestEveryCatalogueStoreSaysHowItIsBackedUp(t *testing.T) {
|
||||
t.Fatal("no module in the catalogue provides a store, so this proved nothing")
|
||||
}
|
||||
}
|
||||
|
||||
// TestEveryCatalogueIdentityFitsWhatItRequires is ADR 0225's check over the real catalogue: every
|
||||
// module's identity, on the longest machine name, fits the bound of every provision it wants. An
|
||||
// overflow fails here, in the pull request that introduces it, rather than on the provider's machine
|
||||
// the first time a real machine's name meets the module's (issue 263).
|
||||
func TestEveryCatalogueIdentityFitsWhatItRequires(t *testing.T) {
|
||||
root := catalogueRoot(t)
|
||||
found, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil || len(found) == 0 {
|
||||
t.Fatalf("no manifests under %s: %v", root, err)
|
||||
}
|
||||
shelf := Shelf{}
|
||||
for _, p := range found {
|
||||
raw, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", p, err)
|
||||
}
|
||||
m, err := ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", p, err)
|
||||
}
|
||||
shelf[m.Module] = m
|
||||
}
|
||||
for _, p := range IdentityProblems(shelf, DefaultLongestMachine) {
|
||||
t.Error(p)
|
||||
}
|
||||
// The night it was found: the resolver provision bounds nothing, and the object store still 20.
|
||||
if dns, ok := shelf["dnsmasq"]; ok {
|
||||
if b := dns.IdentityBoundOf("wildcard-resolution"); b.Bounded() {
|
||||
t.Errorf("the resolver provision bounds its consumers' identities: %+v", b)
|
||||
}
|
||||
}
|
||||
if store, ok := shelf["minio"]; ok {
|
||||
if b := store.IdentityBoundOf("s3-bucket"); b.Max != 20 {
|
||||
t.Errorf("the object store's access key is not bounded at 20: %+v", b)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -114,9 +114,7 @@ func consumerInto(value, as string) (string, error) {
|
||||
// asDNSLabel writes a minted identity as a DNS label.
|
||||
//
|
||||
// The mesh's identities are already lower-case letters, digits and `_` (ConsumerIdentity), and
|
||||
// already inside the bound of the provision serving them: a provider that serves one as a DNS label
|
||||
// bounds it at 63 or less (CheckServes, novox/hq ADR 0225), and one that serves it at all without
|
||||
// saying is held to twenty (IdentityBoundOf). So
|
||||
// already short enough for the tightest backend they reach (CheckIdentity, twenty characters). So
|
||||
// this is the separator and nothing else — no lower-casing of what is already lower case, no
|
||||
// truncation to a limit the identity is already inside, no padding of a name that is already long
|
||||
// enough. Each of those would be the mesh guessing at a rule it has not been given.
|
||||
@@ -143,31 +141,11 @@ func CheckServes(m Manifest) []string {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s serves %s, and the value it serves as %q %s", m.Module, provision, key, err))
|
||||
}
|
||||
// A label longer than DNS keeps is not truncated here (asDNSLabel), so the offer's own
|
||||
// bound has to keep the identity inside one (ADR 0225).
|
||||
if strings.Contains(text, "${consumer:as:dns}") {
|
||||
if b := m.IdentityBoundOf(provision); !b.Bounded() || b.Max > dnsLabelLimit {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s serves %s's consumers their identity as a DNS label in %q, and bounds "+
|
||||
"that identity at %s: a label keeps %d — state an `identity` of at most %d",
|
||||
m.Module, provision, key, boundWords(b), dnsLabelLimit, dnsLabelLimit))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// dnsLabelLimit is the longest DNS label (RFC 1035 §2.3.4).
|
||||
const dnsLabelLimit = 63
|
||||
|
||||
func boundWords(b IdentityBound) string {
|
||||
if !b.Bounded() {
|
||||
return "nothing"
|
||||
}
|
||||
return fmt.Sprintf("%d", b.Max)
|
||||
}
|
||||
|
||||
func sortedServes(serves map[string]map[string]any) []string {
|
||||
out := make([]string, 0, len(serves))
|
||||
for k := range serves {
|
||||
|
||||
@@ -170,10 +170,6 @@ type Rendering struct {
|
||||
// rather than resolved, because who consumes a node is a fact about the rest of the mesh and
|
||||
// resolution answers questions about one machine.
|
||||
Grants []Grant
|
||||
// Withheld is every consumer left out of Grants because its identity overflows the provision's
|
||||
// bound (novox/hq ADR 0225). Composed into nothing; carried so the machine's declaration can say
|
||||
// whom it does not serve, and why, beside what it does.
|
||||
Withheld []Overflow
|
||||
|
||||
// Ports is where this machine puts what each module needs reachable, by module and by the
|
||||
// port the software itself uses (novox/hq ADR 0038).
|
||||
|
||||
@@ -168,7 +168,7 @@ func placedManifest(m Manifest, with Rendering) (Manifest, error) {
|
||||
if err != nil {
|
||||
return m, err
|
||||
}
|
||||
own[name] = OwnSecret{Path: filled, Taken: s.Taken, IssuedBy: s.IssuedBy}
|
||||
own[name] = OwnSecret{Path: filled, Taken: s.Taken}
|
||||
}
|
||||
m.OwnSecrets = own
|
||||
}
|
||||
|
||||
@@ -164,19 +164,13 @@ func consumePattern(pattern string) error {
|
||||
// The events a provider says about its consumers (novox/hq ADR 0224): a consumer it has failed
|
||||
// without one success for minutes, and that consumer succeeding again or being withdrawn. The
|
||||
// controller follows them from every module and `status` names a consumer failing until it recovers.
|
||||
//
|
||||
// **And what becomes of a consumer the mesh stopped asking for** (novox/hq ADR 0230): retired — its
|
||||
// access disabled and its data kept — after the same answer in five passes, waiting for a person when
|
||||
// more would go than the bound allows, re-enabled when asked for again, and deleted only by a person's
|
||||
// `cleanup delete`. One event, its `change` saying which.
|
||||
const (
|
||||
ProvisionerFailing = "provisioner.failing"
|
||||
ProvisionerRecovered = "provisioner.recovered"
|
||||
ProvisionerRetirement = "provisioner.retirement"
|
||||
ProvisionerFailing = "provisioner.failing"
|
||||
ProvisionerRecovered = "provisioner.recovered"
|
||||
)
|
||||
|
||||
// ProvisionerEvents are all three, in the order they are said.
|
||||
var ProvisionerEvents = []string{ProvisionerFailing, ProvisionerRecovered, ProvisionerRetirement}
|
||||
// ProvisionerEvents are both, in the order they are said.
|
||||
var ProvisionerEvents = []string{ProvisionerFailing, ProvisionerRecovered}
|
||||
|
||||
// EmitsAll is every event a module may publish: what it declares and, for a module that receives
|
||||
// contributions — a provider, running a provisioner over them — the provider's standing events.
|
||||
|
||||
@@ -3,7 +3,6 @@ package catalogue
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -57,36 +56,13 @@ func ConsumerIdentity(node, module string) string {
|
||||
return IdentityPrefix + clean(node) + "_" + clean(module)
|
||||
}
|
||||
|
||||
// IdentityBound is the longest consumer identity one provision's backend keeps, and what keeps it
|
||||
// (novox/hq ADR 0049, refined by ADR 0225). Max zero means no bound: the provision keeps no name
|
||||
// derived from its consumer, or keeps one in something with no limit the mesh need respect.
|
||||
type IdentityBound struct {
|
||||
// Max is the longest identity that backend keeps, in characters; zero for none.
|
||||
Max int `json:"max,omitempty"`
|
||||
// In is what keeps it, in words a refusal can quote: "an S3 access key", "a PostgreSQL role".
|
||||
In string `json:"in,omitempty"`
|
||||
}
|
||||
// identityLimit is the shortest identifier limit among the systems these names reach: an S3 access
|
||||
// key's 20 (novox/hq 04-ISSUES/010). PostgreSQL keeps 63 and MinIO 20, so 20 is the one that binds —
|
||||
// the comment used to name PostgreSQL and was wrong. A name over it is refused, with the remedy a
|
||||
// short slug (ADR 0049), not silently cut to fit.
|
||||
const identityLimit = 20
|
||||
|
||||
// Bounded is whether this bound refuses anything.
|
||||
func (b IdentityBound) Bounded() bool { return b.Max > 0 }
|
||||
|
||||
// DefaultIdentityLimit is the bound on a provision whose provider receives its consumers and does
|
||||
// not say how long a name it keeps: an S3 access key's 20 (novox/hq 04-ISSUES/010, 034), the
|
||||
// tightest backend the mesh has met. It was the bound on every provision until ADR 0225; it stays
|
||||
// the bound on any that has not said otherwise, because a provider that is told each consumer's
|
||||
// identity may create a name from it in a backend nobody has measured.
|
||||
const DefaultIdentityLimit = 20
|
||||
|
||||
// DefaultIdentityBound is DefaultIdentityLimit, said as a bound.
|
||||
var DefaultIdentityBound = IdentityBound{Max: DefaultIdentityLimit,
|
||||
In: "a backend that has not said its limit (the tightest known, an S3 access key's)"}
|
||||
|
||||
// identityLimit is the bound CheckIdentity applies, for a caller that does not know which provision
|
||||
// the identity is for.
|
||||
const identityLimit = DefaultIdentityLimit
|
||||
|
||||
// CheckIdentity refuses an identity that would not fit the tightest backend the mesh knows. A caller
|
||||
// that knows the provision uses CheckIdentityWithin and that provision's own bound (ADR 0225).
|
||||
// CheckIdentity refuses an identity that would not fit the tightest backend a consumer reaches.
|
||||
//
|
||||
// **Truncation is not an error in most of these systems** — a name past the limit is cut to fit and
|
||||
// the statement succeeds, so two consumers agreeing for the first N bytes would become one login
|
||||
@@ -94,127 +70,12 @@ const identityLimit = DefaultIdentityLimit
|
||||
// name and is the only thing that can choose another. The remedy is a first-class one: give the
|
||||
// module a short `slug` (ADR 0049), or shorten the machine's name.
|
||||
func CheckIdentity(node, module string) error {
|
||||
return CheckIdentityWithin(node, module, IdentityBound{Max: identityLimit, In: "a backend (an S3 access key)"})
|
||||
}
|
||||
|
||||
// CheckIdentityWithin refuses an identity that would not fit one provision's bound, and accepts any
|
||||
// identity for a provision with none (novox/hq ADR 0225).
|
||||
func CheckIdentityWithin(node, module string, bound IdentityBound) error {
|
||||
if !bound.Bounded() {
|
||||
return nil
|
||||
}
|
||||
got := ConsumerIdentity(node, module)
|
||||
if len(got) <= bound.Max {
|
||||
if len(got) <= identityLimit {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf(
|
||||
"%s on %s is identified as %q, %d characters where %s keeps %d — "+
|
||||
"%s on %s is identified as %q, %d characters where a backend (an S3 access key) keeps %d — "+
|
||||
"give the module a shorter `slug` or shorten the machine's name",
|
||||
module, node, got, len(got), bound.In, bound.Max)
|
||||
}
|
||||
|
||||
// Overflow is one consumer whose identity does not fit the provision it requires: left out of its
|
||||
// provider's grants and reported, never a reason to refuse the provider's machine (ADR 0225).
|
||||
type Overflow struct {
|
||||
// Provision is what was required, Provider the machine answering it.
|
||||
Provision string `json:"provision"`
|
||||
Provider string `json:"provider"`
|
||||
// Consumer is the machine, Module the module on it that required it.
|
||||
Consumer string `json:"consumer"`
|
||||
Module string `json:"module"`
|
||||
// Identity is the name the mesh derived, and Bound what it overflows.
|
||||
Identity string `json:"identity"`
|
||||
Bound IdentityBound `json:"bound"`
|
||||
}
|
||||
|
||||
func (o Overflow) String() string {
|
||||
return fmt.Sprintf("%s on %s requires %s from %s and is identified as %q, %d characters where %s "+
|
||||
"keeps %d — left out of %s's grants until the module's `slug` is shorter",
|
||||
o.Module, o.Consumer, o.Provision, o.Provider, o.Identity, len(o.Identity), o.Bound.In,
|
||||
o.Bound.Max, o.Provider)
|
||||
}
|
||||
|
||||
// Overflowing is every requirement of this machine's modules whose identity overflows the bound of
|
||||
// the provision answering it. The same judgement the provider's composition makes before it grants
|
||||
// (grantsFor), made from the consumer's side so `status` can say it about every machine.
|
||||
func (r Resolution) Overflowing() []Overflow {
|
||||
slugs := map[string]string{}
|
||||
for _, m := range r.Modules {
|
||||
slugs[m.Module] = m.Slug
|
||||
}
|
||||
var out []Overflow
|
||||
seen := map[string]bool{}
|
||||
for _, n := range r.Needs {
|
||||
if n.ByRecord || !n.Identity.Bounded() {
|
||||
continue
|
||||
}
|
||||
source := IdentitySource(slugs[n.For], n.For)
|
||||
if CheckIdentityWithin(r.Node, source, n.Identity) == nil {
|
||||
continue
|
||||
}
|
||||
key := n.Name + "\x00" + n.From + "\x00" + n.For
|
||||
if seen[key] {
|
||||
continue // one line per requirement, however many local names it has
|
||||
}
|
||||
seen[key] = true
|
||||
out = append(out, Overflow{Provision: n.Name, Provider: n.From, Consumer: r.Node, Module: n.For,
|
||||
Identity: ConsumerIdentity(r.Node, source), Bound: n.Identity})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].Module != out[j].Module {
|
||||
return out[i].Module < out[j].Module
|
||||
}
|
||||
return out[i].Provision < out[j].Provision
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// DefaultLongestMachine is the machine name the catalogue check judges identities on when it is not
|
||||
// told one: the longest name of the mesh this catalogue is written for, so a catalogue that passes
|
||||
// passes on every machine that mesh has. `module check --longest-machine-name` says another mesh's;
|
||||
// a mesh that names a longer machine raises this in the same change (ADR 0225).
|
||||
const DefaultLongestMachine = 6
|
||||
|
||||
// IdentityProblems is every module whose identity would overflow a provision it wants, on a machine
|
||||
// whose name is `longestMachine` characters — judged before merge, over the catalogue alone, so the
|
||||
// pull request that introduces an overflow is the one refused (novox/hq ADR 0225, issue 263). A
|
||||
// provision no module in the shelf offers is not judged: its bound is not known here.
|
||||
func IdentityProblems(shelf Shelf, longestMachine int) []string {
|
||||
offeredBy := map[string][]string{}
|
||||
for _, name := range shelfOrder(shelf) {
|
||||
for _, o := range shelf[name].Offers() {
|
||||
offeredBy[o] = append(offeredBy[o], name)
|
||||
}
|
||||
}
|
||||
machine := strings.Repeat("n", longestMachine)
|
||||
var problems []string
|
||||
for _, name := range shelfOrder(shelf) {
|
||||
m := shelf[name]
|
||||
source := IdentitySource(m.Slug, m.Module)
|
||||
for _, want := range m.Wants() {
|
||||
// The tightest bound among the modules offering it: whichever one answers on a given
|
||||
// machine, the identity has to fit it.
|
||||
tightest, by := IdentityBound{}, ""
|
||||
for _, provider := range offeredBy[want] {
|
||||
if provider == name {
|
||||
continue // a module answering its own requirement is not its own consumer
|
||||
}
|
||||
b := shelf[provider].IdentityBoundOf(want)
|
||||
if b.Bounded() && (!tightest.Bounded() || b.Max < tightest.Max) {
|
||||
tightest, by = b, provider
|
||||
}
|
||||
}
|
||||
if CheckIdentityWithin(machine, source, tightest) == nil {
|
||||
continue
|
||||
}
|
||||
got := ConsumerIdentity(machine, source)
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s wants %s, and %s keeps its consumers' identities in %s of at most %d characters: "+
|
||||
"on a machine with a %d-character name it is identified as %q, %d — give %s a "+
|
||||
"`slug` of at most %d characters",
|
||||
name, want, by, tightest.In, tightest.Max, longestMachine, got, len(got), name,
|
||||
tightest.Max-len(IdentityPrefix)-longestMachine-1))
|
||||
}
|
||||
}
|
||||
return problems
|
||||
module, node, got, len(got), identityLimit)
|
||||
}
|
||||
|
||||
@@ -1,190 +0,0 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Each test names the decision it defends: novox/hq ADR 0225, which refines ADR 0049 after issue 263.
|
||||
|
||||
// The provision a module requires sets the bound on its identity, not the tightest backend anywhere.
|
||||
func TestABoundIsTheProvisionsOwn(t *testing.T) {
|
||||
store := Manifest{Module: "objects", Provides: []Offer{{Name: "s3-bucket", Scope: ScopeMesh,
|
||||
Identity: &OfferIdentity{Max: 20, In: "an S3 access key"}}},
|
||||
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}}
|
||||
database := Manifest{Module: "db", Provides: []Offer{{Name: "postgres-database", Scope: ScopeMesh,
|
||||
Identity: &OfferIdentity{Max: 63, In: "a PostgreSQL role"}}},
|
||||
Receives: map[string]string{"postgres-database": "/var/lib/mesh/db/mesh.json"}}
|
||||
if b := store.IdentityBoundOf("s3-bucket"); b.Max != 20 || b.In != "an S3 access key" {
|
||||
t.Errorf("an object store's stated bound was not taken: %+v", b)
|
||||
}
|
||||
if b := database.IdentityBoundOf("postgres-database"); b.Max != 63 {
|
||||
t.Errorf("a database's stated bound was not taken: %+v", b)
|
||||
}
|
||||
// mesh_workstation_keycloak is 25: refused by the object store, accepted by the database.
|
||||
if CheckIdentityWithin("workstation", "keycloak", store.IdentityBoundOf("s3-bucket")) == nil {
|
||||
t.Error("a 25-character identity fit a 20-character access key")
|
||||
}
|
||||
if err := CheckIdentityWithin("workstation", "keycloak", database.IdentityBoundOf("postgres-database")); err != nil {
|
||||
t.Errorf("a database consumer paid the object store's limit: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// What an offer leaves unsaid follows from whether its provider can keep a name at all.
|
||||
func TestAnUnstatedBoundFollowsWhatTheProviderIsTold(t *testing.T) {
|
||||
// Told nothing about its consumers — no receives, nothing served from their identity: no bound.
|
||||
// The resolver provision is exactly this, and its consumers paid an object store's limit (263).
|
||||
resolver := Manifest{Module: "resolver", Provides: []Offer{{Name: "wildcard-resolution", Scope: ScopeMesh}}}
|
||||
if b := resolver.IdentityBoundOf("wildcard-resolution"); b.Bounded() {
|
||||
t.Errorf("a provision that is told nothing of its consumers bounds them: %+v", b)
|
||||
}
|
||||
// Told each consumer and silent about its backend: the old global bound, not none.
|
||||
told := Manifest{Module: "told", Provides: []Offer{{Name: "thing", Scope: ScopeMesh}},
|
||||
Receives: map[string]string{"thing": "/var/lib/mesh/told/mesh.json"}}
|
||||
if b := told.IdentityBoundOf("thing"); b.Max != DefaultIdentityLimit {
|
||||
t.Errorf("a provider told its consumers and silent about its backend is not held to %d: %+v",
|
||||
DefaultIdentityLimit, b)
|
||||
}
|
||||
// Serving a value built from the identity is being told it, too.
|
||||
serving := Manifest{Module: "serving", Provides: []Offer{{Name: "bucket", Scope: ScopeMesh}},
|
||||
Serves: map[string]map[string]any{"bucket": {"name": "b-${consumer:as:dns}"}}}
|
||||
if b := serving.IdentityBoundOf("bucket"); b.Max != DefaultIdentityLimit {
|
||||
t.Errorf("a provider deriving a name from its consumers is not bounded: %+v", b)
|
||||
}
|
||||
// And `false` says it outright, even for a provider that receives.
|
||||
routes := Manifest{Module: "routes", Provides: []Offer{{Name: "route", Scope: ScopeMesh,
|
||||
Identity: &OfferIdentity{None: true}}},
|
||||
Receives: map[string]string{"route": "/var/lib/mesh/routes/mesh.json"}}
|
||||
if b := routes.IdentityBoundOf("route"); b.Bounded() {
|
||||
t.Errorf("`identity: false` still bounds: %+v", b)
|
||||
}
|
||||
}
|
||||
|
||||
// The field reads as written and writes back the same, and refuses what says nothing.
|
||||
func TestAnOffersIdentityIsParsedStrictly(t *testing.T) {
|
||||
for _, raw := range []string{
|
||||
`{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}`,
|
||||
`{"name":"wildcard-resolution","scope":"mesh","identity":false}`,
|
||||
`{"name":"redis-cache","scope":"mesh","identity":{"in":"a Redis ACL user"}}`,
|
||||
} {
|
||||
var o Offer
|
||||
if err := json.Unmarshal([]byte(raw), &o); err != nil {
|
||||
t.Fatalf("%s: %v", raw, err)
|
||||
}
|
||||
back, err := json.Marshal(o)
|
||||
if err != nil || string(back) != raw {
|
||||
t.Errorf("did not round-trip:\n%s\n%s (%v)", raw, back, err)
|
||||
}
|
||||
}
|
||||
for _, raw := range []string{
|
||||
`{"name":"x","identity":true}`,
|
||||
`{"name":"x","identity":{"max":20,"in":"y","most":3}}`,
|
||||
} {
|
||||
var o Offer
|
||||
if err := json.Unmarshal([]byte(raw), &o); err == nil {
|
||||
t.Errorf("accepted %s", raw)
|
||||
}
|
||||
}
|
||||
bad := Manifest{Module: "bad", Version: "1", Provides: []Offer{
|
||||
{Name: "unsaid", Scope: ScopeMesh, Identity: &OfferIdentity{Max: 20}},
|
||||
{Name: "tiny", Scope: ScopeMesh, Identity: &OfferIdentity{Max: 4, In: "nothing usable"}},
|
||||
}}
|
||||
raw, err := json.Marshal(bad)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = ParseManifest(raw)
|
||||
if err == nil || !strings.Contains(err.Error(), "without saying what keeps them") ||
|
||||
!strings.Contains(err.Error(), "the shortest the mesh makes") {
|
||||
t.Fatalf("a bound with no `in`, or too short for any identity, was accepted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Refused before merge: the catalogue check judges each module's identity, on the longest machine
|
||||
// name, against the bound of every provision it wants — and names the module and the slug to set.
|
||||
func TestTheCatalogueCheckRefusesAnIdentityThatOverflowsWhatItRequires(t *testing.T) {
|
||||
shelf := Shelf{
|
||||
"objects": {Module: "objects", Provides: []Offer{{Name: "s3-bucket", Scope: ScopeMesh,
|
||||
Identity: &OfferIdentity{Max: 20, In: "an S3 access key"}}},
|
||||
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}},
|
||||
"photoalbum": {Module: "photoalbum", Requires: []string{"s3-bucket"}},
|
||||
"files": {Module: "files", Requires: []string{"s3-bucket"}},
|
||||
}
|
||||
problems := IdentityProblems(shelf, 6)
|
||||
if len(problems) != 1 || !strings.Contains(problems[0], "photoalbum wants s3-bucket") ||
|
||||
!strings.Contains(problems[0], `"mesh_nnnnnn_photoalbum", 22`) ||
|
||||
!strings.Contains(problems[0], "`slug` of at most 8 characters") {
|
||||
t.Fatalf("one overflow, named with its remedy, was expected: %q", problems)
|
||||
}
|
||||
// A longer machine name refuses more: the check is about the mesh's machines, not one.
|
||||
if got := IdentityProblems(shelf, 10); len(got) != 2 {
|
||||
t.Fatalf("on a 10-character name both overflow (mesh_nnnnnnnnnn_files is 21): %q", got)
|
||||
}
|
||||
// And a slug is the remedy it names.
|
||||
album := shelf["photoalbum"]
|
||||
album.Slug = "album"
|
||||
shelf["photoalbum"] = album
|
||||
if got := IdentityProblems(shelf, 6); len(got) != 0 {
|
||||
t.Fatalf("a slug that fits is still refused: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Tonight's case (issue 263): networkmanager, no slug, requiring the mesh's resolver provision on a
|
||||
// machine with a six-character name. Under ADR 0049's one bound it was refused, and its provider's
|
||||
// whole machine with it; the resolver keeps no name, so it is not refused at all.
|
||||
func TestARequirementOnAKeylessProvisionComposesWithALongName(t *testing.T) {
|
||||
shelf := Shelf{
|
||||
"resolver": {Module: "resolver", Provides: []Offer{{Name: "wildcard-resolution", Scope: ScopeMesh}}},
|
||||
"networkmanager": {Module: "networkmanager", Requires: []string{"wildcard-resolution"}},
|
||||
}
|
||||
if CheckIdentity("laptop", "networkmanager") == nil {
|
||||
t.Fatal("the regression is not reproduced: mesh_laptop_networkmanager fits the old global bound")
|
||||
}
|
||||
if got := IdentityProblems(shelf, 6); len(got) != 0 {
|
||||
t.Fatalf("a requirement on a keyless provision was refused for its length: %q", got)
|
||||
}
|
||||
r := Resolution{Node: "laptop", Modules: []Manifest{shelf["networkmanager"]},
|
||||
Needs: []Needed{{Name: "wildcard-resolution", From: "anchor", For: "networkmanager",
|
||||
Identity: shelf["resolver"].IdentityBoundOf("wildcard-resolution")}}}
|
||||
if got := r.Overflowing(); len(got) != 0 {
|
||||
t.Fatalf("a keyless requirement is reported as overflowing: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The consumer's side of the same judgement the provider's composition makes: what `status` says.
|
||||
func TestAnOverflowingRequirementIsNamedFromTheConsumersSide(t *testing.T) {
|
||||
bound := IdentityBound{Max: 20, In: "an S3 access key"}
|
||||
r := Resolution{Node: "laptop",
|
||||
Modules: []Manifest{{Module: "photoalbum"}, {Module: "files"}, {Module: "gallery", Slug: "gal"}},
|
||||
Needs: []Needed{
|
||||
{Name: "s3-bucket", From: "anchor", For: "photoalbum", Identity: bound},
|
||||
{Name: "s3-bucket", From: "anchor", For: "photoalbum", Local: "second", Identity: bound},
|
||||
{Name: "s3-bucket", From: "anchor", For: "files", Identity: bound},
|
||||
{Name: "s3-bucket", From: "anchor", For: "gallery", Identity: bound},
|
||||
{Name: "licence", From: "records", For: "photoalbum", ByRecord: true, Identity: bound},
|
||||
}}
|
||||
got := r.Overflowing()
|
||||
if len(got) != 1 || got[0].Module != "photoalbum" || got[0].Identity != "mesh_laptop_photoalbum" ||
|
||||
got[0].Provider != "anchor" {
|
||||
t.Fatalf("one overflow, once, was expected: %+v", got)
|
||||
}
|
||||
if said := got[0].String(); !strings.Contains(said, "an S3 access key keeps 20") ||
|
||||
!strings.Contains(said, "slug") {
|
||||
t.Fatalf("the overflow does not say what keeps it or the remedy: %s", said)
|
||||
}
|
||||
}
|
||||
|
||||
// An identity served as a DNS label is never truncated, so the offer's bound must keep it in one.
|
||||
func TestAnIdentityServedAsADNSLabelIsBoundedToOne(t *testing.T) {
|
||||
serves := map[string]map[string]any{"bucket": {"name": "${consumer:as:dns}"}}
|
||||
wide := Manifest{Module: "wide", Serves: serves, Provides: []Offer{{Name: "bucket", Scope: ScopeMesh,
|
||||
Identity: &OfferIdentity{Max: 255, In: "a client id"}}}}
|
||||
if got := CheckServes(wide); len(got) != 1 || !strings.Contains(got[0], "a label keeps 63") {
|
||||
t.Fatalf("a 255-character bound on a DNS label passed: %q", got)
|
||||
}
|
||||
unsaid := Manifest{Module: "unsaid", Serves: serves, Provides: []Offer{{Name: "bucket", Scope: ScopeMesh}}}
|
||||
if got := CheckServes(unsaid); len(got) != 0 {
|
||||
t.Fatalf("the default bound (20) on a DNS label was refused: %q", got)
|
||||
}
|
||||
}
|
||||
+19
-149
@@ -8,7 +8,6 @@ package catalogue
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
@@ -154,84 +153,6 @@ type Offer struct {
|
||||
// a seat's holder gated by the machine's graphical session, and pulling one in for whatever asked
|
||||
// is the misassignment research 026 found. Unmet, the requirement is refused naming who could.
|
||||
Reach string `json:"reach,omitempty"`
|
||||
// Identity is the longest consumer identity this provision's backend keeps (novox/hq ADR 0225):
|
||||
// `{"max": 63, "in": "a PostgreSQL role"}`, or `false` for a provision that keeps no name derived
|
||||
// from its consumer. Unsaid, the mesh assumes the tightest backend it knows when the provider is
|
||||
// told its consumers, and no bound when it is not — see IdentityBoundOf.
|
||||
Identity *OfferIdentity `json:"identity,omitempty"`
|
||||
}
|
||||
|
||||
// OfferIdentity is what an offer says about the names its backend keeps for its consumers.
|
||||
type OfferIdentity struct {
|
||||
// None is set by `"identity": false`: the provision keeps no name derived from its consumer.
|
||||
None bool
|
||||
// Max is the longest identity kept, in characters; zero with In set means no limit worth stating.
|
||||
Max int
|
||||
// In is what keeps it, for a refusal to quote.
|
||||
In string
|
||||
}
|
||||
|
||||
// UnmarshalJSON accepts `false` or `{"max": N, "in": "..."}`.
|
||||
func (i *OfferIdentity) UnmarshalJSON(raw []byte) error {
|
||||
var flag bool
|
||||
if err := json.Unmarshal(raw, &flag); err == nil {
|
||||
if flag {
|
||||
return errors.New("an offer's identity is false (it keeps no name) or {max, in}; true says nothing")
|
||||
}
|
||||
*i = OfferIdentity{None: true}
|
||||
return nil
|
||||
}
|
||||
var full struct {
|
||||
Max int `json:"max,omitempty"`
|
||||
In string `json:"in"`
|
||||
}
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&full); err != nil {
|
||||
return fmt.Errorf("an offer's identity is false or {max, in}: %w", err)
|
||||
}
|
||||
*i = OfferIdentity{Max: full.Max, In: full.In}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarshalJSON writes it back in the form it was written.
|
||||
func (i OfferIdentity) MarshalJSON() ([]byte, error) {
|
||||
if i.None {
|
||||
return []byte("false"), nil
|
||||
}
|
||||
return json.Marshal(struct {
|
||||
Max int `json:"max,omitempty"`
|
||||
In string `json:"in"`
|
||||
}{i.Max, i.In})
|
||||
}
|
||||
|
||||
// IdentityBoundOf is the bound this module's offer of a provision puts on its consumers' identities
|
||||
// (novox/hq ADR 0225).
|
||||
//
|
||||
// **What an offer says, it gets.** Where it says nothing, the bound follows from whether the
|
||||
// provider can keep a name at all: a provider that receives the provision, or serves its consumers
|
||||
// a value built from their identity, is told who each consumer is and may create a name from it in
|
||||
// a backend nobody measured — so it keeps the old global bound, DefaultIdentityBound. One that does
|
||||
// neither is told nothing about its consumers and keeps nothing of them: no bound. The resolver
|
||||
// provision is that case, and its consumers paid an object store's limit until this (issue 263).
|
||||
func (m Manifest) IdentityBoundOf(provision string) IdentityBound {
|
||||
for _, o := range m.Provides {
|
||||
if o.Name != provision || o.Identity == nil {
|
||||
continue
|
||||
}
|
||||
if o.Identity.None {
|
||||
return IdentityBound{}
|
||||
}
|
||||
return IdentityBound{Max: o.Identity.Max, In: o.Identity.In}
|
||||
}
|
||||
if _, receives := m.Receives[provision]; receives {
|
||||
return DefaultIdentityBound
|
||||
}
|
||||
if served, err := json.Marshal(m.Serves[provision]); err == nil &&
|
||||
bytes.Contains(served, []byte("${consumer:as")) {
|
||||
return DefaultIdentityBound
|
||||
}
|
||||
return IdentityBound{}
|
||||
}
|
||||
|
||||
// MachineReach is whether a provision is usable only on its provider's own machine.
|
||||
@@ -285,21 +206,20 @@ func (o *Offer) UnmarshalJSON(raw []byte) error {
|
||||
Scope string `json:"scope,omitempty"`
|
||||
Credential *OfferCredential `json:"credential,omitempty"`
|
||||
Reach string `json:"reach,omitempty"`
|
||||
Identity *OfferIdentity `json:"identity,omitempty"`
|
||||
}
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&full); err != nil {
|
||||
return fmt.Errorf("a provided name is either a string or {name, scope, credential, reach, identity}: %w", err)
|
||||
return fmt.Errorf("a provided name is either a string or {name, scope, credential, reach}: %w", err)
|
||||
}
|
||||
o.Name, o.Scope, o.Credential, o.Reach, o.Identity = full.Name, full.Scope, full.Credential, full.Reach, full.Identity
|
||||
o.Name, o.Scope, o.Credential, o.Reach = full.Name, full.Scope, full.Credential, full.Reach
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarshalJSON writes back the short form when there is nothing else to say, so a manifest that
|
||||
// went through the mesh comes out looking like the one that went in.
|
||||
func (o Offer) MarshalJSON() ([]byte, error) {
|
||||
if o.Scope == "" && o.Credential == nil && o.Reach == "" && o.Identity == nil {
|
||||
if o.Scope == "" && o.Credential == nil && o.Reach == "" {
|
||||
return json.Marshal(o.Name)
|
||||
}
|
||||
return json.Marshal(struct {
|
||||
@@ -307,8 +227,7 @@ func (o Offer) MarshalJSON() ([]byte, error) {
|
||||
Scope string `json:"scope,omitempty"`
|
||||
Credential *OfferCredential `json:"credential,omitempty"`
|
||||
Reach string `json:"reach,omitempty"`
|
||||
Identity *OfferIdentity `json:"identity,omitempty"`
|
||||
}{o.Name, o.Scope, o.Credential, o.Reach, o.Identity})
|
||||
}{o.Name, o.Scope, o.Credential, o.Reach})
|
||||
}
|
||||
|
||||
// Manifest is everything a module says about itself.
|
||||
@@ -318,10 +237,9 @@ type Manifest struct {
|
||||
|
||||
// Slug is a short identifier the mesh uses in place of the module name when it derives a
|
||||
// consumer's login (novox/hq ADR 0049). Optional: a module with a short name needs none. It
|
||||
// exists because `mesh_<node>_<module>` must fit the bound of every provision the module
|
||||
// requires — an S3 access key's 20 characters is the tightest — and a long module name would
|
||||
// overflow it (ADR 0225: the bound is the provision's own, and a keyless one has none). A person
|
||||
// choosing `kc` for keycloak keeps the identity legible where a hash would not.
|
||||
// exists because `mesh_<node>_<module>` must fit the tightest backend a consumer reaches — an S3
|
||||
// access key is 20 characters — and a long module name would overflow it. A person choosing
|
||||
// `kc` for keycloak keeps the identity legible where a hash would not.
|
||||
Slug string `json:"slug,omitempty"`
|
||||
|
||||
// Provides are the names other modules may require. A module always provides its own name;
|
||||
@@ -558,10 +476,7 @@ type Manifest struct {
|
||||
// that takes it only once, so a rotation must be staged beside the current value — the form the
|
||||
// mesh does not build yet, and refuses by name. A secret that says neither is not rotated by
|
||||
// the mesh: the one fault worse than an unrotated credential is a rotated one the software
|
||||
// never saw. `"issued-by": "outside"` says a party outside the mesh issues the value — an API
|
||||
// key, a bot token, a licence — so the mesh never puts one of its own in its place (novox/hq
|
||||
// ADR 0228); any other at-start secret given by hand lives only until the module's first good
|
||||
// start, and is then replaced.
|
||||
// never saw.
|
||||
OwnSecrets OwnSecrets `json:"own-secrets,omitempty"`
|
||||
|
||||
// SecretsOwner is who the files holding this module's secrets belong to on the machine —
|
||||
@@ -1350,10 +1265,8 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
||||
"%q is not a usable module name: lower-case letters, digits, dashes and dots", m.Module))
|
||||
}
|
||||
// A slug is a short identifier the mesh derives a login from (novox/hq ADR 0049). The same
|
||||
// charset as a name; its length is judged against the bound of each provision it wants on the
|
||||
// longest machine name by the catalogue check (IdentityProblems, ADR 0225), and against the
|
||||
// machine it is on when its provider grants it — a slug that is fine on one machine's short name
|
||||
// can overflow another's.
|
||||
// charset as a name; its length is checked against a backend's limit at assignment, where the
|
||||
// node it joins is known — a slug that is fine on one machine's short name can overflow another's.
|
||||
if m.Slug != "" && !name.MatchString(m.Slug) {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%q is not a usable slug: lower-case letters, digits, dashes and dots", m.Slug))
|
||||
@@ -1390,20 +1303,6 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
||||
if !name.MatchString(p) {
|
||||
problems = append(problems, fmt.Sprintf("%q is not a usable name to provide", p))
|
||||
}
|
||||
// A bound says what keeps the name, so the refusal it causes can say it (ADR 0225); and it
|
||||
// leaves room for the shortest identity the mesh makes, or it would refuse every consumer.
|
||||
if id := offer.Identity; id != nil && !id.None {
|
||||
if strings.TrimSpace(id.In) == "" {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s bounds the identities of %s's consumers without saying what keeps them: `in`",
|
||||
m.Module, p))
|
||||
}
|
||||
if shortest := len(IdentityPrefix) + 3; id.Max < 0 || id.Max > 0 && id.Max < shortest {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s bounds %s's consumers' identities at %d characters, and the shortest the mesh "+
|
||||
"makes is %d", m.Module, p, id.Max, shortest))
|
||||
}
|
||||
}
|
||||
if offer.Credential != nil {
|
||||
own, declared := m.OwnSecrets[offer.Credential.Own]
|
||||
switch {
|
||||
@@ -1804,13 +1703,6 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
||||
"or %q (applied by the module's own code to a backend that takes it once)",
|
||||
m.Module, name, own.Taken, TakenAtStart, TakenApplied))
|
||||
}
|
||||
if own.IssuedBy != "" && own.IssuedBy != IssuedOutside {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s says its secret %q is issued by %q; a secret says %q when a party outside the mesh "+
|
||||
"issues it — a vendor's key, a bot's token, a licence — and says nothing when the mesh "+
|
||||
"may make it (novox/hq ADR 0228)",
|
||||
m.Module, name, own.IssuedBy, IssuedOutside))
|
||||
}
|
||||
}
|
||||
localOf := map[string]string{}
|
||||
for _, to := range m.SecretRequirements() {
|
||||
@@ -2273,24 +2165,10 @@ const (
|
||||
TakenApplied = "applied"
|
||||
)
|
||||
|
||||
// IssuedOutside says a value was issued by a party outside the mesh — a vendor's API key, a bot's
|
||||
// token, a licence — so no value the mesh makes would work in its place (novox/hq ADR 0228).
|
||||
const IssuedOutside = "outside"
|
||||
|
||||
// OwnSecret is where one of a module's own secrets lands, how the module takes it, and — for a
|
||||
// value only an outside party can issue — that it is one.
|
||||
// OwnSecret is where one of a module's own secrets lands, and how the module takes it.
|
||||
type OwnSecret struct {
|
||||
Path string
|
||||
Taken string
|
||||
IssuedBy string
|
||||
}
|
||||
|
||||
// MeshMayMake says the mesh may put a value it makes in place of the one the secret holds: the
|
||||
// module reads it as it starts, and nobody outside the mesh issued it (novox/hq ADR 0228). A value
|
||||
// given to the mesh for such a secret lives only until the module's first good start under the
|
||||
// mesh; anything else given stays as given.
|
||||
func (s OwnSecret) MeshMayMake() bool {
|
||||
return s.Taken == TakenAtStart && s.IssuedBy != IssuedOutside
|
||||
Path string
|
||||
Taken string
|
||||
}
|
||||
|
||||
// OwnSecrets is a module's own secrets by name. On the wire each is a path, or an object naming
|
||||
@@ -2311,16 +2189,15 @@ func (o *OwnSecrets) UnmarshalJSON(raw []byte) error {
|
||||
continue
|
||||
}
|
||||
var long struct {
|
||||
Path string `json:"path"`
|
||||
Taken string `json:"taken,omitempty"`
|
||||
IssuedBy string `json:"issued-by,omitempty"`
|
||||
Path string `json:"path"`
|
||||
Taken string `json:"taken,omitempty"`
|
||||
}
|
||||
dec := json.NewDecoder(bytes.NewReader(body))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&long); err != nil {
|
||||
return fmt.Errorf("own-secrets.%s: a path, or {\"path\", \"taken\", \"issued-by\"}: %w", name, err)
|
||||
return fmt.Errorf("own-secrets.%s: a path, or {\"path\", \"taken\"}: %w", name, err)
|
||||
}
|
||||
out[name] = OwnSecret{Path: long.Path, Taken: long.Taken, IssuedBy: long.IssuedBy}
|
||||
out[name] = OwnSecret{Path: long.Path, Taken: long.Taken}
|
||||
}
|
||||
*o = out
|
||||
return nil
|
||||
@@ -2329,18 +2206,11 @@ func (o *OwnSecrets) UnmarshalJSON(raw []byte) error {
|
||||
func (o OwnSecrets) MarshalJSON() ([]byte, error) {
|
||||
entries := make(map[string]any, len(o))
|
||||
for name, s := range o {
|
||||
if s.Taken == "" && s.IssuedBy == "" {
|
||||
if s.Taken == "" {
|
||||
entries[name] = s.Path
|
||||
continue
|
||||
}
|
||||
long := map[string]string{"path": s.Path}
|
||||
if s.Taken != "" {
|
||||
long["taken"] = s.Taken
|
||||
}
|
||||
if s.IssuedBy != "" {
|
||||
long["issued-by"] = s.IssuedBy
|
||||
}
|
||||
entries[name] = long
|
||||
entries[name] = map[string]string{"path": s.Path, "taken": s.Taken}
|
||||
}
|
||||
return json.Marshal(entries)
|
||||
}
|
||||
|
||||
@@ -52,42 +52,3 @@ func TestAnOwnSecretTakenSomeOtherWayIsRefused(t *testing.T) {
|
||||
t.Fatal("an unknown field on an own secret was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// A secret an outside party issues says so (novox/hq ADR 0228), is written back as it was read, and
|
||||
// is the one at-start secret the mesh may not make; any other word for who issued it is refused.
|
||||
func TestAnOwnSecretSaysWhenAnOutsidePartyIssuesIt(t *testing.T) {
|
||||
m, err := ParseManifest([]byte(`{"module":"letta","version":"1","own-secrets":{
|
||||
"server-password":{"path":"/var/lib/letta/server-password","taken":"at-start"},
|
||||
"openai-api-key":{"path":"/var/lib/letta/openai-api-key","taken":"at-start","issued-by":"outside"},
|
||||
"telegram-token":{"path":"/var/lib/letta/telegram-token","issued-by":"outside"},
|
||||
"logflare":{"path":"/var/lib/letta/logflare","taken":"applied"},
|
||||
"broker":"/var/lib/mesh/letta/broker"}}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, may := range map[string]bool{"server-password": true, "openai-api-key": false,
|
||||
"telegram-token": false, "logflare": false, "broker": false} {
|
||||
if got := m.OwnSecrets[name].MeshMayMake(); got != may {
|
||||
t.Errorf("%s: the mesh may make it = %v, want %v", name, got, may)
|
||||
}
|
||||
}
|
||||
out, err := json.Marshal(m.OwnSecrets)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var again OwnSecrets
|
||||
if err := json.Unmarshal(out, &again); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again["openai-api-key"] != m.OwnSecrets["openai-api-key"] || again["telegram-token"] != m.OwnSecrets["telegram-token"] {
|
||||
t.Fatalf("the round trip lost who issued it: %s", out)
|
||||
}
|
||||
if strings.Contains(string(out), `"taken":""`) {
|
||||
t.Fatalf("a secret that says only who issued it gained an empty taken: %s", out)
|
||||
}
|
||||
_, err = ParseManifest([]byte(`{"module":"letta","version":"1","own-secrets":{
|
||||
"openai-api-key":{"path":"/var/lib/letta/openai-api-key","issued-by":"openai"}}}`))
|
||||
if err == nil || !strings.Contains(err.Error(), `issued by "openai"`) {
|
||||
t.Fatalf("an unknown word for who issued a secret was accepted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -194,11 +194,6 @@ type Needed struct {
|
||||
// state, not a consumer missing its key. Set by the plan, which is the only layer that knows a
|
||||
// licence's manager; empty for every consumer.
|
||||
Manager bool
|
||||
// Identity is the longest consumer identity the answering provision keeps (novox/hq ADR 0225),
|
||||
// from the provider's own offer: what the mesh judges this consumer's identity against, on the
|
||||
// consumer's side for `status` and on the provider's before it grants. No bound for a provision
|
||||
// answered by a record, which keeps no name of anybody's.
|
||||
Identity IdentityBound
|
||||
}
|
||||
|
||||
// Refusal is why a set of assignments cannot become a declaration.
|
||||
@@ -408,7 +403,7 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
|
||||
needs = append(needs, Needed{
|
||||
Name: want, From: node.Name, At: at,
|
||||
Serves: servedByOne(by, want), For: because[want],
|
||||
SharedOwn: sharedByOne(by, want), Identity: by.IdentityBoundOf(want)})
|
||||
SharedOwn: sharedByOne(by, want)})
|
||||
} else if served := servedByOne(by, want); len(served) > 0 {
|
||||
// Answered here with no credential to mint, but the provider serves facts the
|
||||
// consumer cannot guess — a port, a model name — and so still needs a binding.
|
||||
@@ -452,15 +447,11 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
|
||||
return
|
||||
}
|
||||
shared := ""
|
||||
// A provider whose definition is not in hand is held to the tightest bound the
|
||||
// mesh knows, not to none: what it keeps is not known here (ADR 0225).
|
||||
bound := DefaultIdentityBound
|
||||
if pm, known := catalogue[p.Module]; known {
|
||||
shared, _ = pm.SharedCredentialOf(want)
|
||||
bound = pm.IdentityBoundOf(want)
|
||||
}
|
||||
needs = append(needs, Needed{Name: want, From: p.Node, At: p.At,
|
||||
Serves: p.Serves, For: because[want], SharedOwn: shared, Identity: bound})
|
||||
Serves: p.Serves, For: because[want], SharedOwn: shared})
|
||||
}
|
||||
switch {
|
||||
case world.Unchecked:
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Every provider may say what becomes of a consumer the mesh stopped asking for (novox/hq ADR 0230),
|
||||
// whatever its manifest lists — as it may say a consumer it keeps failing (ADR 0224) — and a module
|
||||
// that receives nothing may not.
|
||||
func TestEveryProviderMaySayWhatItRetires(t *testing.T) {
|
||||
provider := Manifest{Module: "pg", Receives: map[string]string{"postgres-database": "/x/mesh.json"},
|
||||
Emits: []string{"database.provisioned"}}
|
||||
for _, e := range []string{ProvisionerFailing, ProvisionerRecovered, ProvisionerRetirement, "database.provisioned"} {
|
||||
if !slices.Contains(provider.EmitsAll(), e) {
|
||||
t.Errorf("a provider may not emit %s: %v", e, provider.EmitsAll())
|
||||
}
|
||||
}
|
||||
if slices.Contains(Manifest{Module: "app", Emits: []string{"x"}}.EmitsAll(), ProvisionerRetirement) {
|
||||
t.Error("a module that provides nothing may say what it retires")
|
||||
}
|
||||
}
|
||||
@@ -83,13 +83,7 @@ var defaultSeats = append([]Seat{
|
||||
// `assign` and the rest are a role's interface, not a container's, and stay addressable while
|
||||
// the control plane is replaced.
|
||||
{Name: ControllerSeatName, Scope: ScopeMesh, Decision: "novox/hq ADR 0079",
|
||||
// And what is wrong, as it changes, and the self-check's heartbeat (novox/hq to-be 45 §2, §4).
|
||||
Emits: []string{"applied", "refused", "built-before",
|
||||
"condition-raised", "condition-changed", "condition-cleared", "doctor-heartbeat",
|
||||
// A value given by hand, replaced after its module's first good start (novox/hq ADR 0228).
|
||||
"secret-replaced",
|
||||
// Every act a healer takes (novox/hq to-be 45 §7).
|
||||
"healer-acted"},
|
||||
Emits: []string{"applied", "refused", "built-before"},
|
||||
Serves: ControllerVerbs},
|
||||
// The store's first verbs (novox/hq ADR 0159): the smallest set that makes the store askable,
|
||||
// served by whichever module holds the seat with tools of these names.
|
||||
|
||||
@@ -110,8 +110,6 @@ var ControllerVerbs = []Verb{
|
||||
"paths": "with repository: the files the merge would change, comma-separated, from the repository's root",
|
||||
"modules": "with repository: or the modules it would change, comma-separated",
|
||||
"limit": "how many plans to list (default 10); only when listing",
|
||||
"why": "with stop or close: why it is ended by hand — required, and recorded in the hand-act log (novox/hq to-be 45 §7)",
|
||||
"cause": "with stop or close: the cause in a word, or a condition's kind (optional)",
|
||||
}, nil)},
|
||||
{Name: "plan", Description: "What one machine would run, and why: the declaration the mesh would send it — " +
|
||||
"or, with files, the files it would be given.",
|
||||
@@ -139,27 +137,21 @@ var ControllerVerbs = []Verb{
|
||||
{Name: "push", Description: "Send one machine everything it should be. With no machine named it is a push of the " +
|
||||
"WHOLE mesh — every machine that is behind — and the answer says so first; behind says that outright. " +
|
||||
"Answers at once that it is running, with a call id: `calls` with that id says what it sent " +
|
||||
"(a push can reload the bus, which then refuses any answer still to come). A push by hand is a repair, " +
|
||||
"and says why: recorded in the hand-act log (novox/hq to-be 45 §7).",
|
||||
"(a push can reload the bus, which then refuses any answer still to come).",
|
||||
Input: schema(map[string]string{
|
||||
"node": "the machine's name; without it, every machine that is behind",
|
||||
"behind": "\"true\": every machine that is behind, the whole mesh — the same as naming none, said outright; not with node",
|
||||
"why": "why this is pushed by hand: recorded in the hand-act log",
|
||||
"cause": "the cause in a word, or a condition's kind — the word a second push for the same reason uses (optional)",
|
||||
}, []string{"why"}, "behind")},
|
||||
{Name: "rotate", Description: "Replace a credential. A pair credential, by provision (and a consuming machine and module, " +
|
||||
}, nil, "behind")},
|
||||
{Name: "rotate", Description: "Replace a credential. A pair credential, by provision (and a consuming machine, " +
|
||||
"else every holder): both ends are re-sent together. Or a module's own secret, by machine, module and " +
|
||||
"name: made anew and the machine sent, so the module starts again on it — for a secret its definition " +
|
||||
"says it reads at start, whether the mesh made the value or a person gave it (novox/hq ADR 0228); one " +
|
||||
"an outside party issued, or one the module applies to a backend, is refused with the reason.",
|
||||
"name: made anew and the machine sent, so the module starts again on it — only for a secret its " +
|
||||
"definition says it reads at start; a value given to the mesh, or one the module applies to a backend, is refused with the reason.",
|
||||
Input: schema(map[string]string{
|
||||
"provision": "a pair credential: the provision whose credential to replace",
|
||||
"consumer": "with provision: only the holder on this machine (optional)",
|
||||
"node": "an own secret: the machine",
|
||||
"module": "an own secret: the module; with provision: only this consuming module's credential (optional)",
|
||||
"module": "an own secret: the module",
|
||||
"secret": "an own secret: its name in the module's definition",
|
||||
"why": "an own secret: why it is rotated — recorded in the hand-act log (optional)",
|
||||
"cause": "with why: the cause in a word, the word a second rotation for the same reason uses (optional)",
|
||||
}, nil)},
|
||||
{Name: "issue", Description: "Give a module on a machine its account on the bus: minted, and sealed to the " +
|
||||
"machine as the module's own secret named broker, read at the next push of that machine. For a module " +
|
||||
@@ -214,86 +206,6 @@ var ControllerVerbs = []Verb{
|
||||
Input: schema(map[string]string{"node": "one machine; every holder when absent"}, nil)},
|
||||
{Name: "resume", Description: "The build seat's holder on one machine — or every holder — takes builds again.",
|
||||
Input: schema(map[string]string{"node": "one machine; every holder when absent"}, nil)},
|
||||
// Acts done by hand, and what the bounds are set from (novox/hq to-be 45 §7, Phase 0).
|
||||
{Name: "hand-act", Description: "Record an act done by hand outside the mesh — a container restarted, a file " +
|
||||
"edited, a service started on a machine — with why and its cause, in the hand-act log beside the pushes and " +
|
||||
"plans ended by hand (novox/hq to-be 45 §7). A cause recorded twice in a fortnight is a healer wanted.",
|
||||
Input: schema(map[string]string{
|
||||
"what": "what was done, in a line",
|
||||
"why": "why it had to be done by hand",
|
||||
"cause": "the cause in a word, or a condition's kind — the word a second act for the same reason uses",
|
||||
"condition": "the key of the condition it addressed, if any (optional)",
|
||||
}, []string{"what", "why", "cause"})},
|
||||
{Name: "hand-acts", Description: "What was done by hand lately — pushes, plans ended, consumers re-made, acts " +
|
||||
"recorded — who, why and the cause of each, and which causes repeat: each repeat is a healer the mesh lacks.",
|
||||
Input: schema(map[string]string{"days": "how many days back (default 14)"}, nil)},
|
||||
{Name: "durations", Description: "How long things take, as the controller measured them: a send to its machine's " +
|
||||
"report (apply), a machine's silence between words (heartbeat-gap), a plan's tier, a build — per machine, " +
|
||||
"repository or module, with median, p90 and max. What the core's bounds are set from (novox/hq to-be 45 Phase 0).",
|
||||
Input: schema(map[string]string{
|
||||
"kind": "one kind: apply, heartbeat-gap, plan-tier or build; every kind when absent",
|
||||
"days": "how many days back (default 14)",
|
||||
}, nil)},
|
||||
// What is wrong, and the self-check (novox/hq to-be 45 §2, §4).
|
||||
{Name: "conditions", Description: "What is wrong with the mesh now: every open condition, urgent first, " +
|
||||
"then oldest — raised by the watchdogs of the signals table, the self-check's probes and the providers' " +
|
||||
"own words, and cleared when observation says it is resolved, never by hand. Given a key, that one " +
|
||||
"whole with its evidence; with history, every transition lately; with silence, stop one's messages " +
|
||||
"for a while — a hand act, which says why (novox/hq to-be 45 §2).",
|
||||
Input: schema(map[string]string{
|
||||
"key": "a condition's key: that one whole; with history, only its transitions",
|
||||
"scope": "only this scope: machine, plan, call, build, merge, provider, seat, bus, core, probe or mesh",
|
||||
"severity": "only urgent, or only warning",
|
||||
"machine": "only those about this machine",
|
||||
"history": "\"true\": every raising, change, silence and clearing lately, oldest first",
|
||||
"days": "with history: how many days back (default 7, at most 90)",
|
||||
"silence": "a condition's key: send no message for it for a while; it stays open and in status",
|
||||
"for": "with silence: how long — 30m, 4h, 2d; at most 7d",
|
||||
"why": "with silence: why — required, and recorded in the hand-act log",
|
||||
"cause": "with silence: the cause in a word (the condition's kind when absent)",
|
||||
}, nil, "history")},
|
||||
// What the healers did (novox/hq to-be 45 §7).
|
||||
{Name: "healers", Description: "The healers (novox/hq to-be 45 §7): each registered response to one kind " +
|
||||
"of condition — its repair (the ordinary path again), its budget, what happens when it is spent — every act " +
|
||||
"they took lately with its outcome, and the mesh-wide brake. A heal is never a hand act; whether it " +
|
||||
"repaired anything is the condition's clearing to say. Each act is also in its condition's tried.",
|
||||
Input: schema(map[string]string{"days": "how many days of acts back (default 7)"}, nil)},
|
||||
{Name: "doctor", Description: "The self-check (novox/hq to-be 45 §4): the last run's verdict at once — " +
|
||||
"each probe of the design's live invariants passed, failed or could not run, and how long ago. With " +
|
||||
"run, a run now; with probes, the registry; with signals, every row of the signals table and the age " +
|
||||
"of its newest signal. Runs every five minutes on its own; each failure is an open condition.",
|
||||
Input: schema(map[string]string{
|
||||
"run": "\"true\": run every probe now and answer the verdict",
|
||||
"probes": "\"true\": the registry — what each probe asserts, and the condition it raises",
|
||||
"signals": "\"true\": the signals table, each row with the age of its newest signal",
|
||||
}, nil, "run", "probes", "signals")},
|
||||
// A consumer the mesh stopped asking for: retired, waiting for a person, deleted only by one
|
||||
// (novox/hq ADR 0230).
|
||||
{Name: "retire", Description: "A consumer the mesh stops asking for is retired by its provider — access " +
|
||||
"disabled, data kept — after the same answer in five passes; more than three at once, or more than half " +
|
||||
"of those held, waits for a person. With no answer, every provider that waits and what it would retire. " +
|
||||
"With answer approve or reject, a node and a module: retire exactly what that provider waits with, or " +
|
||||
"keep it active — a hand act, which says why (novox/hq ADR 0230).",
|
||||
Input: schema(map[string]string{
|
||||
"answer": "approve or reject: answer what the provider waits with (needs node, module and why)",
|
||||
"node": "with answer: the machine the provider runs on",
|
||||
"module": "with answer: the provider module",
|
||||
"why": "with answer: why — required, and recorded in the hand-act log",
|
||||
"cause": "with answer: the cause in a word (retire-waiting when absent)",
|
||||
}, nil)},
|
||||
{Name: "cleanup", Description: "Every consumer a provider holds retired — its age, its size where the " +
|
||||
"backend knows, and why it was retired. With consumer (and node, module): the provider deletes that one " +
|
||||
"retired consumer — never an active one. With older-than: every retired consumer older than that many " +
|
||||
"days, listed; deleted only with confirm. Deleting is a hand act, which says why (novox/hq ADR 0230).",
|
||||
Input: schema(map[string]string{
|
||||
"node": "with consumer: the machine the provider runs on",
|
||||
"module": "with consumer: the provider module",
|
||||
"consumer": "delete this retired consumer (needs node, module and why)",
|
||||
"older-than": "delete every consumer retired more than this many days (needs why; lists only without confirm)",
|
||||
"confirm": "\"true\": with older-than, delete what is listed",
|
||||
"why": "with consumer or older-than: why — required, and recorded in the hand-act log",
|
||||
"cause": "with consumer or older-than: the cause in a word (cleanup-waiting when absent)",
|
||||
}, nil, "confirm")},
|
||||
{Name: "build", Description: "Have the build machine build a repository. Answers at once with the build's id: " +
|
||||
"`builds` with that id follows it line by line, and the module is registered when the outcome comes.",
|
||||
Input: schema(map[string]string{
|
||||
|
||||
@@ -1,159 +0,0 @@
|
||||
package conditions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// The condition store on the bus (to-be 45 §2, ADR 0201): the controller's two buckets, asserted at
|
||||
// its start like its calls and its hand-act log.
|
||||
|
||||
// OnTheBus opens the store and its history on a connection.
|
||||
func OnTheBus(ctx context.Context, conn *nats.Conn) (Backend, History, error) {
|
||||
api, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
open, err := api.KeyValue(ctx, broker.ConditionsBucket)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the condition store %s is not on the bus — the controller asserts it at "+
|
||||
"its start, so one older than this has not: %w", broker.ConditionsBucket, err)
|
||||
}
|
||||
history, err := api.KeyValue(ctx, broker.ConditionHistoryBucket)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the condition history %s is not on the bus — the controller asserts it "+
|
||||
"at its start, so one older than this has not: %w", broker.ConditionHistoryBucket, err)
|
||||
}
|
||||
return busStore{open}, &busHistory{api: api, kv: history}, nil
|
||||
}
|
||||
|
||||
type busStore struct{ kv jetstream.KeyValue }
|
||||
|
||||
func (b busStore) Get(ctx context.Context, key string) (Entry, bool, error) {
|
||||
e, err := b.kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return Entry{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return Entry{}, false, err
|
||||
}
|
||||
return Entry{Value: e.Value(), Revision: e.Revision()}, true, nil
|
||||
}
|
||||
|
||||
func (b busStore) Create(ctx context.Context, key string, value []byte) error {
|
||||
_, err := b.kv.Create(ctx, key, value)
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return ErrMoved
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (b busStore) Update(ctx context.Context, key string, value []byte, revision uint64) error {
|
||||
_, err := b.kv.Update(ctx, key, value, revision)
|
||||
return moved(err)
|
||||
}
|
||||
|
||||
func (b busStore) Delete(ctx context.Context, key string, revision uint64) error {
|
||||
return moved(b.kv.Delete(ctx, key, jetstream.LastRevision(revision)))
|
||||
}
|
||||
|
||||
// moved reads the server's refusal of a compare-and-set as what it is.
|
||||
func moved(err error) error {
|
||||
var apiErr *jetstream.APIError
|
||||
if errors.As(err, &apiErr) && apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence {
|
||||
return ErrMoved
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// All is every key, read through a watch that hands over each current value and then says it has.
|
||||
func (b busStore) All(ctx context.Context) (map[string]Entry, error) {
|
||||
w, err := b.kv.WatchAll(ctx, jetstream.IgnoreDeletes())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = w.Stop() }()
|
||||
out := map[string]Entry{}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("reading the condition store: %w", ctx.Err())
|
||||
case e := <-w.Updates():
|
||||
if e == nil {
|
||||
return out, nil
|
||||
}
|
||||
out[e.Key()] = Entry{Value: e.Value(), Revision: e.Revision()}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// busHistory keeps each transition under a key of its time and a sequence, and reads them back
|
||||
// from a moment through the stream under the bucket — by time, so a read of the last ten minutes
|
||||
// does not read ninety days.
|
||||
type busHistory struct {
|
||||
api jetstream.JetStream
|
||||
kv jetstream.KeyValue
|
||||
seq atomic.Uint64
|
||||
}
|
||||
|
||||
func (h *busHistory) Append(ctx context.Context, e Event) error {
|
||||
body, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
key := strconv.FormatInt(e.At.UnixNano(), 10) + "-" + strconv.FormatUint(h.seq.Add(1), 10)
|
||||
_, err = h.kv.Put(ctx, key, body)
|
||||
return err
|
||||
}
|
||||
|
||||
// historyQuiet is how long a read of the history waits for one more transition before it takes the
|
||||
// stream as read to its end; it answers at once while it holds something.
|
||||
const historyQuiet = 2 * time.Second
|
||||
|
||||
func (h *busHistory) Since(ctx context.Context, since time.Time) ([]Event, error) {
|
||||
start := since
|
||||
consumer, err := h.api.OrderedConsumer(ctx, "KV_"+broker.ConditionHistoryBucket, jetstream.OrderedConsumerConfig{
|
||||
DeliverPolicy: jetstream.DeliverByStartTimePolicy, OptStartTime: &start,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading the condition history: %w", err)
|
||||
}
|
||||
info, err := consumer.Info(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading the condition history: %w", err)
|
||||
}
|
||||
var out []Event
|
||||
pending := info.NumPending
|
||||
for pending > 0 {
|
||||
msg, err := consumer.Next(jetstream.FetchMaxWait(historyQuiet))
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
// Nothing more within the quiet wait: read to its end.
|
||||
break
|
||||
}
|
||||
meta, err := msg.Metadata()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
pending = meta.NumPending
|
||||
var e Event
|
||||
if len(msg.Data()) > 0 && json.Unmarshal(msg.Data(), &e) == nil && e.Key != "" {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].At.Before(out[j].At) })
|
||||
return out, nil
|
||||
}
|
||||
@@ -1,117 +0,0 @@
|
||||
package conditions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// The condition store against a real server: compare-and-set, an unreadable store refused, and the
|
||||
// history read back by time are claims about what the bus does.
|
||||
|
||||
func busStoreForTest(t *testing.T) (*broker.JetStream, Backend, History) {
|
||||
t.Helper()
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = api.DeleteKeyValue(t.Context(), broker.ConditionsBucket)
|
||||
_ = api.DeleteKeyValue(t.Context(), broker.ConditionHistoryBucket)
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store, history, err := OnTheBus(t.Context(), js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return js, store, history
|
||||
}
|
||||
|
||||
// **A condition outlives the controller that raised it**, and two writers on the bus cannot lose each
|
||||
// other's word: a stale revision is refused as moved.
|
||||
func TestNatsTheStoreKeepsConditionsByCompareAndSet(t *testing.T) {
|
||||
_, store, history := busStoreForTest(t)
|
||||
ctx := t.Context()
|
||||
told := &Told{}
|
||||
k := NewKeeper(ctx, Options{Store: store, History: history, Teller: told})
|
||||
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
k.Close(context.Background())
|
||||
|
||||
again := NewKeeper(ctx, Options{Store: store, History: history})
|
||||
defer again.Close(context.Background())
|
||||
open, err := again.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(open) != 1 || open[0].Observations != 2 {
|
||||
t.Fatalf("a new keeper read %+v", open)
|
||||
}
|
||||
e, _, err := store.Get(ctx, "machine.ace.silent")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.Update(ctx, "machine.ace.silent", []byte(`{}`), e.Revision-1); err != ErrMoved {
|
||||
t.Fatalf("a write at a stale revision answered %v", err)
|
||||
}
|
||||
if err := store.Create(ctx, "machine.ace.silent", []byte(`{}`)); err != ErrMoved {
|
||||
t.Fatalf("creating an open condition answered %v", err)
|
||||
}
|
||||
if err := store.Delete(ctx, "machine.ace.silent", e.Revision-1); err != ErrMoved {
|
||||
t.Fatalf("a delete at a stale revision answered %v", err)
|
||||
}
|
||||
if cleared, err := again.Clear(ctx, "machine.ace.silent", "heard"); err != nil || !cleared {
|
||||
t.Fatalf("cleared %v: %v", cleared, err)
|
||||
}
|
||||
// Raised again at once: the store takes a key whose last word was a delete.
|
||||
if _, err := again.Observe(ctx, silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _, _ := again.Get(ctx, "machine.ace.silent")
|
||||
if got.Count != 2 {
|
||||
t.Fatalf("raised again after its clearing as %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **The history is read back from a moment, oldest first**, through the stream under its bucket.
|
||||
func TestNatsTheHistoryIsReadByTime(t *testing.T) {
|
||||
_, store, history := busStoreForTest(t)
|
||||
ctx := t.Context()
|
||||
k := NewKeeper(ctx, Options{Store: store, History: history})
|
||||
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.Clear(ctx, "machine.ace.silent", "heard"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
k.Close(context.Background())
|
||||
all, err := history.Since(ctx, time.Now().Add(-time.Hour))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(all) != 2 || all[0].Change != ChangeRaised || all[1].Change != ChangeCleared {
|
||||
t.Fatalf("history %+v", all)
|
||||
}
|
||||
none, err := history.Since(ctx, time.Now().Add(time.Hour))
|
||||
if err != nil || len(none) != 0 {
|
||||
t.Fatalf("history from the future: %+v %v", none, err)
|
||||
}
|
||||
}
|
||||
@@ -1,244 +0,0 @@
|
||||
// Package conditions is the condition store (novox/hq to-be 45 §2, ADR 0227 rules 5 and 6).
|
||||
//
|
||||
// **A condition is a durable fact about something the mesh owns that is wrong.** Until this, every
|
||||
// one of the forty-eight core failures of research 031 was noticed because a person or an agent
|
||||
// looked: the mesh's own answers carried the fact for whoever asked, and told nobody. A condition is
|
||||
// raised when an observation says something is wrong past its bound, kept with since-when, evidence
|
||||
// and who can resolve it, said on the bus as it changes, and cleared when an observation says it is
|
||||
// resolved — never by hand.
|
||||
//
|
||||
// The controller is the store's only writer (to-be 45 §1). Two of its processes may write at once —
|
||||
// the serving controller's watchdogs, and a command a person runs to silence one — so every write
|
||||
// is a compare-and-set on the key's revision, and a write that lost the race reads again and redoes
|
||||
// itself rather than overwriting what the other said.
|
||||
package conditions
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Severity is how soon the operator is needed: two levels, no more (to-be 45 §2).
|
||||
type Severity string
|
||||
|
||||
const (
|
||||
// Urgent needs the operator now.
|
||||
Urgent Severity = "urgent"
|
||||
// Warning needs the operator when they can.
|
||||
Warning Severity = "warning"
|
||||
)
|
||||
|
||||
// The scopes a condition's key starts with: what kind of thing is wrong.
|
||||
const (
|
||||
ScopeMachine = "machine"
|
||||
ScopePlan = "plan"
|
||||
ScopeCall = "call"
|
||||
ScopeBuild = "build"
|
||||
ScopeMerge = "merge"
|
||||
ScopeProvider = "provider"
|
||||
ScopeSeat = "seat"
|
||||
ScopeBus = "bus"
|
||||
ScopeCore = "core"
|
||||
ScopeProbe = "probe"
|
||||
ScopeMesh = "mesh"
|
||||
)
|
||||
|
||||
// Scopes is every scope, in the order a person reads them.
|
||||
var Scopes = []string{ScopeMachine, ScopePlan, ScopeCall, ScopeBuild, ScopeMerge, ScopeProvider,
|
||||
ScopeSeat, ScopeBus, ScopeCore, ScopeProbe, ScopeMesh}
|
||||
|
||||
// Who resolves a condition.
|
||||
const (
|
||||
// ResolverSelf clears on observation: the signal returns, the probe passes.
|
||||
ResolverSelf = "self"
|
||||
// ResolverOperator needs a person: a healer's budget spent, or a repair that could only destroy.
|
||||
ResolverOperator = "operator"
|
||||
// ResolverAgent is work handed to an agent (research 017; not raised by anything yet).
|
||||
ResolverAgent = "agent"
|
||||
)
|
||||
|
||||
// ResolverHealer is the resolver of a condition a registered healer works on (Phase 3).
|
||||
func ResolverHealer(name string) string { return "healer:" + name }
|
||||
|
||||
// Subject is what the condition is about: its scope, its id within the scope, and the machine it
|
||||
// concerns when there is one.
|
||||
type Subject struct {
|
||||
Scope string `json:"scope"`
|
||||
ID string `json:"id"`
|
||||
Machine string `json:"machine,omitempty"`
|
||||
// Also are the other machines it concerns: a consumer's, for a provider failing it.
|
||||
Also []string `json:"also,omitempty"`
|
||||
}
|
||||
|
||||
// Evidence is one observation, as it was said.
|
||||
type Evidence struct {
|
||||
At time.Time `json:"at"`
|
||||
Said string `json:"said"`
|
||||
}
|
||||
|
||||
// Attempt is one healer's try at a condition (to-be 45 §7): when, what it did, what came of it, and
|
||||
// which healer — so a person reading `conditions show` sees the mesh repairing itself, by whom.
|
||||
type Attempt struct {
|
||||
At time.Time `json:"at"`
|
||||
What string `json:"what"`
|
||||
Outcome string `json:"outcome"`
|
||||
// By is the healer, as a person reads it: `healer H1`. Never a person: an act by hand is the
|
||||
// hand-act log's, not a condition's attempt.
|
||||
By string `json:"by"`
|
||||
}
|
||||
|
||||
// KeptAttempts is how many attempts a condition keeps, newest last: a healer's budget is a few, and
|
||||
// a condition reopened again and again carries its tries forward only so far.
|
||||
const KeptAttempts = 10
|
||||
|
||||
// Silence is a person saying they know: no messages until it ends (to-be 45 §2). Recorded as a hand
|
||||
// act; the condition stays open, and `status` still says it.
|
||||
type Silence struct {
|
||||
Until time.Time `json:"until"`
|
||||
By string `json:"by"`
|
||||
Why string `json:"why"`
|
||||
Since time.Time `json:"since"`
|
||||
}
|
||||
|
||||
// KeptEvidence is how many observations a condition keeps, newest first.
|
||||
const KeptEvidence = 10
|
||||
|
||||
// MaxSilence is the longest a condition may be silenced at once: past it, a person says so again.
|
||||
const MaxSilence = 7 * 24 * time.Hour
|
||||
|
||||
// ReopenWithin is how soon after it cleared a condition raised again is the same one again, with its
|
||||
// count increased, rather than news (to-be 45 §2).
|
||||
const ReopenWithin = 10 * time.Minute
|
||||
|
||||
// Condition is one open condition, as the store keeps it and its events carry it.
|
||||
type Condition struct {
|
||||
Key string `json:"key"`
|
||||
// Kind is the condition kind: from the signals table, the probe registry or an event kind.
|
||||
Kind string `json:"kind"`
|
||||
Subject Subject `json:"subject"`
|
||||
Severity Severity `json:"severity"`
|
||||
// Summary is one line in the mesh's words.
|
||||
Summary string `json:"summary"`
|
||||
// Evidence is the newest observations, at most KeptEvidence, newest first.
|
||||
Evidence []Evidence `json:"evidence"`
|
||||
// Source is the signals-table row, probe or event that raised it: `S1`, `D3`, `provisioner.failing`.
|
||||
Source string `json:"source"`
|
||||
// Raised is when it was first observed this time; LastObserved the newest observation.
|
||||
Raised time.Time `json:"raised"`
|
||||
LastObserved time.Time `json:"last-observed"`
|
||||
// Observations is how many times it was observed since raised.
|
||||
Observations int `json:"observations"`
|
||||
// Count is how many times it has been raised, a reopening within ReopenWithin counted.
|
||||
Count int `json:"count"`
|
||||
Tried []Attempt `json:"tried,omitempty"`
|
||||
Resolver string `json:"resolver"`
|
||||
// Silenced is null when no silence is in force: said, not left out, so a reader need not guess.
|
||||
Silenced *Silence `json:"silenced"`
|
||||
// Epoch is the controller lease epoch that last wrote it (to-be 45 §6). Zero where it was written
|
||||
// by a controller serving without the lease, or before the lease existed.
|
||||
Epoch uint64 `json:"epoch"`
|
||||
}
|
||||
|
||||
// SilencedAt says whether a person's silence is in force at a moment.
|
||||
func (c Condition) SilencedAt(now time.Time) bool {
|
||||
return c.Silenced != nil && now.Before(c.Silenced.Until)
|
||||
}
|
||||
|
||||
// Escalated says a healer tried and its budget is spent: the operator resolves it now (to-be 45 §2,
|
||||
// "budget spent ──► OPEN, resolver: operator, severity: urgent"). An observation does not lower its
|
||||
// severity again: the watchdog that sees it every half minute would otherwise undo the escalation.
|
||||
func (c Condition) Escalated() bool { return c.Resolver == ResolverOperator && len(c.Tried) > 0 }
|
||||
|
||||
// Show is the verb that shows more about a condition, as a message carries it.
|
||||
func (c Condition) Show() string { return "mesh-controller.conditions key=" + c.Key }
|
||||
|
||||
// Observation is one watchdog, probe or event saying something is wrong now.
|
||||
type Observation struct {
|
||||
Scope string
|
||||
// ID is the thing within the scope; several tokens joined by dots where the thing is named by
|
||||
// several (a provider's module, its machine and the consumer).
|
||||
ID string
|
||||
// Token is the last part of the key, short for the kind: `silent` for a machine, `failing` for a
|
||||
// provider. Kind's own word when empty.
|
||||
Token string
|
||||
Kind string
|
||||
Machine string
|
||||
// Also are the other machines it concerns.
|
||||
Also []string
|
||||
Severity Severity
|
||||
Summary string
|
||||
// Said is this observation's evidence, in the mesh's words; Summary when empty.
|
||||
Said string
|
||||
Source string
|
||||
Resolver string
|
||||
}
|
||||
|
||||
// Key is where the observation's condition is kept: `<scope>.<id>.<kind>`, so the same fault said
|
||||
// again is the same condition.
|
||||
func (o Observation) Key() string {
|
||||
token := o.Token
|
||||
if token == "" {
|
||||
token = o.Kind
|
||||
}
|
||||
return Key(o.Scope, o.ID, token)
|
||||
}
|
||||
|
||||
// unsafeKey is anything a key may not hold: the bus takes letters, digits and `-_/=` in a key's
|
||||
// tokens, and a `*` or `>` would make one a wildcard.
|
||||
var unsafeKey = regexp.MustCompile(`[^A-Za-z0-9_=/-]`)
|
||||
|
||||
// Key composes a condition's key from its parts, each token made safe for the bus: a character the
|
||||
// bus would refuse becomes `_`, so a key is never refused for the name of the thing it is about.
|
||||
func Key(scope, id, token string) string {
|
||||
var parts []string
|
||||
for _, p := range append(append([]string{scope}, strings.Split(id, ".")...), token) {
|
||||
p = unsafeKey.ReplaceAllString(strings.TrimSpace(p), "_")
|
||||
if p == "" {
|
||||
p = "_"
|
||||
}
|
||||
parts = append(parts, p)
|
||||
}
|
||||
return strings.Join(parts, ".")
|
||||
}
|
||||
|
||||
// check refuses an observation that could not be said: a condition with no kind, no scope the mesh
|
||||
// knows, or no severity is one nobody could route.
|
||||
func (o Observation) check() error {
|
||||
known := false
|
||||
for _, s := range Scopes {
|
||||
if s == o.Scope {
|
||||
known = true
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case !known:
|
||||
return fmt.Errorf("a condition's scope is one of %s, not %q", strings.Join(Scopes, ", "), o.Scope)
|
||||
case strings.TrimSpace(o.ID) == "":
|
||||
return fmt.Errorf("a %s condition names what it is about", o.Scope)
|
||||
case strings.TrimSpace(o.Kind) == "":
|
||||
return fmt.Errorf("the condition %s has no kind", o.Key())
|
||||
case o.Severity != Urgent && o.Severity != Warning:
|
||||
return fmt.Errorf("the condition %s is urgent or a warning, not %q", o.Key(), o.Severity)
|
||||
case strings.TrimSpace(o.Summary) == "":
|
||||
return fmt.Errorf("the condition %s says nothing", o.Key())
|
||||
case strings.TrimSpace(o.Source) == "":
|
||||
return fmt.Errorf("the condition %s does not say what raised it", o.Key())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Order sorts conditions as `status` says them: urgent before warning, then oldest first.
|
||||
func Order(list []Condition) {
|
||||
sort.SliceStable(list, func(i, j int) bool {
|
||||
if list[i].Severity != list[j].Severity {
|
||||
return list[i].Severity == Urgent
|
||||
}
|
||||
if !list[i].Raised.Equal(list[j].Raised) {
|
||||
return list[i].Raised.Before(list[j].Raised)
|
||||
}
|
||||
return list[i].Key < list[j].Key
|
||||
})
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
package conditions
|
||||
|
||||
import "time"
|
||||
|
||||
// The events a condition's life emits (to-be 45 §2), as the mesh-controller seat's own: published on
|
||||
// `mesh.seat.mesh-controller.event.<name>`, on the events stream, so a consumer that was away catches
|
||||
// up. **This is a contract**: the operator-channel's holder is written against these names and the
|
||||
// shape of Event, and the controller learns nothing about telling.
|
||||
const (
|
||||
// EventRaised: a condition was raised — new, or the same fault again within ReopenWithin of its
|
||||
// clearing (Change says which). A reopened condition is not news: its key is the one the
|
||||
// first message was about.
|
||||
EventRaised = "condition-raised"
|
||||
// EventChanged: its severity, its resolver or its silence changed. Not every observation: a
|
||||
// condition observed again is written, and says nothing.
|
||||
EventChanged = "condition-changed"
|
||||
// EventCleared: an observation says it is resolved. The condition is removed from the store and
|
||||
// the transition kept in its history.
|
||||
EventCleared = "condition-cleared"
|
||||
)
|
||||
|
||||
// Events is every event a condition's life emits.
|
||||
var Events = []string{EventRaised, EventChanged, EventCleared}
|
||||
|
||||
// HeartbeatEvent is the self-check's heartbeat (to-be 45 §4, S10), said under the same seat at the end
|
||||
// of every run: `{run, at, interval-seconds, counts: {passed, failed, failed-to-run, deferred}, probes,
|
||||
// controller, why}`. mesh-watcher, on a machine that is not the control node, listens for it.
|
||||
const HeartbeatEvent = "doctor-heartbeat"
|
||||
|
||||
// What changed, as an event's Change and a history entry's says it.
|
||||
const (
|
||||
ChangeRaised = "raised"
|
||||
ChangeReopened = "reopened"
|
||||
ChangeSeverity = "severity"
|
||||
ChangeResolver = "resolver"
|
||||
ChangeSilenced = "silenced"
|
||||
ChangeUnsilenced = "silence-ended"
|
||||
ChangeCleared = "cleared"
|
||||
)
|
||||
|
||||
// Event is the body of every condition event, and the shape a history entry keeps: **the condition
|
||||
// itself, at the top level** — key, kind, subject, severity, summary, source, raised, last-observed,
|
||||
// observations, resolver, silenced (null when not), epoch, and the evidence — with what happened to
|
||||
// it beside. One object a consumer reads the same way whichever of the three it is.
|
||||
type Event struct {
|
||||
// Condition is the condition after the transition — as it was last held, for a clearing.
|
||||
Condition
|
||||
// Event is the event's own name, so a body read without its subject still says what it is.
|
||||
Event string `json:"event"`
|
||||
// At is when the transition happened.
|
||||
At time.Time `json:"at"`
|
||||
// Change is what happened: raised, reopened, severity, resolver, silenced, silence-ended, cleared.
|
||||
Change string `json:"change"`
|
||||
// Was is the value before, for a severity or resolver change.
|
||||
Was string `json:"was,omitempty"`
|
||||
// Why says why it cleared, or why it was silenced.
|
||||
Why string `json:"why,omitempty"`
|
||||
// Cleared is when it cleared, on a clearing.
|
||||
Cleared *time.Time `json:"cleared,omitempty"`
|
||||
// Show is the verb that shows more.
|
||||
Show string `json:"show"`
|
||||
}
|
||||
|
||||
// eventFor is the event a change is said under.
|
||||
func eventFor(change string) string {
|
||||
switch change {
|
||||
case ChangeRaised, ChangeReopened:
|
||||
return EventRaised
|
||||
case ChangeCleared:
|
||||
return EventCleared
|
||||
}
|
||||
return EventChanged
|
||||
}
|
||||
@@ -1,147 +0,0 @@
|
||||
package conditions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// InMemory is a store and a history held in this process: for tests, and for nothing else — a
|
||||
// condition kept here is forgotten by a restart, which is the fault the store exists to remove.
|
||||
type InMemory struct {
|
||||
mu sync.Mutex
|
||||
values map[string]Entry
|
||||
revision uint64
|
||||
events []Event
|
||||
// Fail, when set, is what every read and write answers: a store that is away.
|
||||
Fail error
|
||||
}
|
||||
|
||||
// NewInMemory is an empty store.
|
||||
func NewInMemory() *InMemory { return &InMemory{values: map[string]Entry{}} }
|
||||
|
||||
func (m *InMemory) Get(_ context.Context, key string) (Entry, bool, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.Fail != nil {
|
||||
return Entry{}, false, m.Fail
|
||||
}
|
||||
e, ok := m.values[key]
|
||||
return e, ok, nil
|
||||
}
|
||||
|
||||
func (m *InMemory) Create(_ context.Context, key string, value []byte) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.Fail != nil {
|
||||
return m.Fail
|
||||
}
|
||||
if _, ok := m.values[key]; ok {
|
||||
return ErrMoved
|
||||
}
|
||||
m.revision++
|
||||
m.values[key] = Entry{Value: value, Revision: m.revision}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *InMemory) Update(_ context.Context, key string, value []byte, revision uint64) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.Fail != nil {
|
||||
return m.Fail
|
||||
}
|
||||
if e, ok := m.values[key]; !ok || e.Revision != revision {
|
||||
return ErrMoved
|
||||
}
|
||||
m.revision++
|
||||
m.values[key] = Entry{Value: value, Revision: m.revision}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *InMemory) Delete(_ context.Context, key string, revision uint64) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.Fail != nil {
|
||||
return m.Fail
|
||||
}
|
||||
if e, ok := m.values[key]; !ok || e.Revision != revision {
|
||||
return ErrMoved
|
||||
}
|
||||
delete(m.values, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *InMemory) All(context.Context) (map[string]Entry, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.Fail != nil {
|
||||
return nil, m.Fail
|
||||
}
|
||||
out := make(map[string]Entry, len(m.values))
|
||||
for k, v := range m.values {
|
||||
out[k] = v
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (m *InMemory) Append(_ context.Context, e Event) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.Fail != nil {
|
||||
return m.Fail
|
||||
}
|
||||
m.events = append(m.events, e)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *InMemory) Since(_ context.Context, since time.Time) ([]Event, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.Fail != nil {
|
||||
return nil, m.Fail
|
||||
}
|
||||
var out []Event
|
||||
for _, e := range m.events {
|
||||
if !e.At.Before(since) {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].At.Before(out[j].At) })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Told is a teller that remembers what it was told, for tests.
|
||||
type Told struct {
|
||||
mu sync.Mutex
|
||||
Events []Event
|
||||
Names []string
|
||||
Fail error
|
||||
}
|
||||
|
||||
func (t *Told) PublishSeatEvent(_ context.Context, seat, event string, body []byte) error {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if t.Fail != nil {
|
||||
return t.Fail
|
||||
}
|
||||
if seat != Seat {
|
||||
return errors.New("told under the wrong seat: " + seat)
|
||||
}
|
||||
var e Event
|
||||
if err := json.Unmarshal(body, &e); err != nil {
|
||||
return err
|
||||
}
|
||||
t.Events = append(t.Events, e)
|
||||
t.Names = append(t.Names, event)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Said is a copy of what was told so far.
|
||||
func (t *Told) Said() []Event {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
return append([]Event(nil), t.Events...)
|
||||
}
|
||||
@@ -1,629 +0,0 @@
|
||||
package conditions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Backend is where the open conditions are kept: one value per key, written by compare-and-set.
|
||||
type Backend interface {
|
||||
// Get is one key's value and revision; false when it holds none.
|
||||
Get(ctx context.Context, key string) (Entry, bool, error)
|
||||
// Create writes a key that holds nothing, and fails with ErrMoved when it holds something.
|
||||
Create(ctx context.Context, key string, value []byte) error
|
||||
// Update writes a key at the revision it was read at, and fails with ErrMoved when it moved.
|
||||
Update(ctx context.Context, key string, value []byte, revision uint64) error
|
||||
// Delete removes a key at the revision it was read at, and fails with ErrMoved when it moved.
|
||||
Delete(ctx context.Context, key string, revision uint64) error
|
||||
// All is every key's value. An error is an error: never an empty store (ADR 0227 rule 4).
|
||||
All(ctx context.Context) (map[string]Entry, error)
|
||||
}
|
||||
|
||||
// Entry is one key's value, at a revision.
|
||||
type Entry struct {
|
||||
Value []byte
|
||||
Revision uint64
|
||||
}
|
||||
|
||||
// ErrMoved is a compare-and-set that lost: somebody wrote the key since it was read.
|
||||
var ErrMoved = errors.New("the condition was written by somebody else since it was read")
|
||||
|
||||
// History keeps every transition (to-be 45 §2): appended, read back from a moment.
|
||||
type History interface {
|
||||
Append(ctx context.Context, e Event) error
|
||||
// Since is every transition from a moment, oldest first.
|
||||
Since(ctx context.Context, since time.Time) ([]Event, error)
|
||||
}
|
||||
|
||||
// Teller says a transition on the bus, as the mesh-controller seat's event. The link's bus is one.
|
||||
type Teller interface {
|
||||
PublishSeatEvent(ctx context.Context, seat, event string, body []byte) error
|
||||
}
|
||||
|
||||
// Seat is the role the events are said under (novox/hq ADR 0134): the control plane's.
|
||||
const Seat = "mesh-controller"
|
||||
|
||||
// Keeper raises, observes, silences and clears conditions, and says each transition.
|
||||
type Keeper struct {
|
||||
store Backend
|
||||
history History
|
||||
teller Teller
|
||||
now func() time.Time
|
||||
say func(format string, args ...any)
|
||||
changed func()
|
||||
epoch func() (uint64, error)
|
||||
|
||||
mu sync.Mutex
|
||||
// cleared is when each recently cleared condition cleared and how often it had been raised, so
|
||||
// one raised again within ReopenWithin is the same one again.
|
||||
cleared map[string]clearing
|
||||
|
||||
// out is the transitions still to be said and kept, in order: said by one goroutine, so a
|
||||
// condition's events arrive in the order they happened, and offered again while the bus is away.
|
||||
out chan Event
|
||||
drained chan struct{}
|
||||
closing sync.Once
|
||||
// Unsaid counts the transitions given up on, for the self-check to say.
|
||||
unsaid int
|
||||
}
|
||||
|
||||
type clearing struct {
|
||||
at time.Time
|
||||
count int
|
||||
silenced *Silence
|
||||
// tried is what healers tried before it cleared: a reopening is the same fault, and what was
|
||||
// tried on it is still what was tried.
|
||||
tried []Attempt
|
||||
}
|
||||
|
||||
// Options are what a Keeper is made with.
|
||||
type Options struct {
|
||||
Store Backend
|
||||
History History
|
||||
// Teller says the transitions; nil says nothing (a test, or a command run with no bus to say on).
|
||||
Teller Teller
|
||||
Now func() time.Time
|
||||
// Say is where a transition that could not be said or kept is said instead.
|
||||
Say func(format string, args ...any)
|
||||
// Changed is told of every transition, at once — for `status`, which leads with what is open.
|
||||
Changed func()
|
||||
// Epoch is the controller lease this keeper writes under (to-be 45 §6): every write carries its
|
||||
// epoch, and none is made while it is not held. Nil writes with no epoch (a test, the memory store).
|
||||
Epoch func() (uint64, error)
|
||||
}
|
||||
|
||||
// TellFor is how long one transition is offered to the bus before it is said lost.
|
||||
var TellFor = 10 * time.Minute
|
||||
|
||||
// NewKeeper is a keeper over a store. It reads what cleared lately from the history, so a condition
|
||||
// that cleared just before this controller started and is raised again now is a reopening.
|
||||
func NewKeeper(ctx context.Context, o Options) *Keeper {
|
||||
k := &Keeper{store: o.Store, history: o.History, teller: o.Teller, now: o.Now, say: o.Say, changed: o.Changed,
|
||||
epoch: o.Epoch,
|
||||
cleared: map[string]clearing{}, out: make(chan Event, 1024), drained: make(chan struct{})}
|
||||
if k.now == nil {
|
||||
k.now = time.Now
|
||||
}
|
||||
if k.say == nil {
|
||||
k.say = func(string, ...any) {}
|
||||
}
|
||||
if k.history != nil {
|
||||
if recent, err := k.history.Since(ctx, k.now().Add(-ReopenWithin)); err == nil {
|
||||
for _, e := range recent {
|
||||
if e.Change == ChangeCleared {
|
||||
k.cleared[e.Key] = clearing{at: e.At, count: e.Condition.Count, silenced: e.Condition.Silenced,
|
||||
tried: e.Condition.Tried}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
k.say("what cleared lately could not be read from the condition history, so a condition "+
|
||||
"raised again now is said as new rather than reopened: %v", err)
|
||||
}
|
||||
}
|
||||
go k.telling()
|
||||
return k
|
||||
}
|
||||
|
||||
// Close says what is still to be said, waiting at most until ctx ends.
|
||||
func (k *Keeper) Close(ctx context.Context) {
|
||||
k.closing.Do(func() { close(k.out) })
|
||||
select {
|
||||
case <-k.drained:
|
||||
case <-ctx.Done():
|
||||
k.say("%d condition transition(s) were not yet said when this process ended", len(k.out))
|
||||
}
|
||||
}
|
||||
|
||||
// Unsaid is how many transitions were given up on since this keeper started.
|
||||
func (k *Keeper) Unsaid() int {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
return k.unsaid
|
||||
}
|
||||
|
||||
// stamp is the gate every write passes: the epoch it carries, set on the condition, or why this
|
||||
// keeper may not write — a controller that lost the lease raises, observes and clears nothing.
|
||||
func (k *Keeper) stamp(c *Condition) error {
|
||||
if k.epoch == nil {
|
||||
return nil
|
||||
}
|
||||
epoch, err := k.epoch()
|
||||
if err != nil {
|
||||
return fmt.Errorf("the condition %s is not written: %w", c.Key, err)
|
||||
}
|
||||
c.Epoch = epoch
|
||||
return nil
|
||||
}
|
||||
|
||||
// tries bounds one compare-and-set: two writers rarely race more than once.
|
||||
const tries = 8
|
||||
|
||||
// Observe records one observation: raises the condition if it is not open, and otherwise adds the
|
||||
// evidence. Says a raising, a reopening, and a change of severity or resolver; an observation that
|
||||
// changes neither is written and said nowhere.
|
||||
func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error) {
|
||||
if err := o.check(); err != nil {
|
||||
return Condition{}, err
|
||||
}
|
||||
key := o.Key()
|
||||
for i := 0; i < tries; i++ {
|
||||
now := k.now().UTC()
|
||||
said := o.Said
|
||||
if said == "" {
|
||||
said = o.Summary
|
||||
}
|
||||
entry, found, err := k.store.Get(ctx, key)
|
||||
if err != nil {
|
||||
return Condition{}, fmt.Errorf("reading the condition %s: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
c := Condition{Key: key, Kind: o.Kind, Subject: Subject{Scope: o.Scope, ID: o.ID, Machine: o.Machine, Also: o.Also},
|
||||
Severity: o.Severity, Summary: o.Summary, Evidence: []Evidence{{At: now, Said: said}},
|
||||
Source: o.Source, Raised: now, LastObserved: now, Observations: 1, Count: 1,
|
||||
Resolver: orSelf(o.Resolver)}
|
||||
change := ChangeRaised
|
||||
k.mu.Lock()
|
||||
if before, ok := k.cleared[key]; ok && now.Sub(before.at) <= ReopenWithin {
|
||||
c.Count, change = before.count+1, ChangeReopened
|
||||
// A silence a person gave the condition before it cleared still holds: they said
|
||||
// they knew, and the same fault again ten minutes later is what they knew about.
|
||||
if before.silenced != nil && now.Before(before.silenced.Until) {
|
||||
c.Silenced = before.silenced
|
||||
}
|
||||
c.Tried = before.tried
|
||||
}
|
||||
k.mu.Unlock()
|
||||
if err := k.stamp(&c); err != nil {
|
||||
return Condition{}, err
|
||||
}
|
||||
body, err := json.Marshal(c)
|
||||
if err != nil {
|
||||
return Condition{}, err
|
||||
}
|
||||
if err := k.store.Create(ctx, key, body); errors.Is(err, ErrMoved) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return Condition{}, fmt.Errorf("raising the condition %s: %w", key, err)
|
||||
}
|
||||
k.mu.Lock()
|
||||
delete(k.cleared, key)
|
||||
k.mu.Unlock()
|
||||
k.tell(Event{Condition: c, At: now, Change: change})
|
||||
return c, nil
|
||||
}
|
||||
var c Condition
|
||||
if err := json.Unmarshal(entry.Value, &c); err != nil {
|
||||
return Condition{}, fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err)
|
||||
}
|
||||
var changes []Event
|
||||
// A healer's budget spent made it urgent; the watchdog seeing it again does not undo that.
|
||||
if o.Severity != c.Severity && !(c.Escalated() && o.Severity != Urgent) {
|
||||
changes = append(changes, Event{Change: ChangeSeverity, Was: string(c.Severity)})
|
||||
c.Severity = o.Severity
|
||||
}
|
||||
if r := orSelf(o.Resolver); o.Resolver != "" && r != c.Resolver {
|
||||
changes = append(changes, Event{Change: ChangeResolver, Was: c.Resolver})
|
||||
c.Resolver = r
|
||||
}
|
||||
// The kind as the source says it now: a source that gave the same key a kind of its own since
|
||||
// (a probe's finding split out for a healer) is read by that kind from its next observation.
|
||||
c.Kind, c.Summary, c.Source, c.LastObserved = o.Kind, o.Summary, o.Source, now
|
||||
if o.Machine != "" {
|
||||
c.Subject.Machine = o.Machine
|
||||
}
|
||||
if len(o.Also) > 0 {
|
||||
c.Subject.Also = o.Also
|
||||
}
|
||||
c.Observations++
|
||||
c.Evidence = append([]Evidence{{At: now, Said: said}}, c.Evidence...)
|
||||
if len(c.Evidence) > KeptEvidence {
|
||||
c.Evidence = c.Evidence[:KeptEvidence]
|
||||
}
|
||||
if err := k.stamp(&c); err != nil {
|
||||
return Condition{}, err
|
||||
}
|
||||
body, err := json.Marshal(c)
|
||||
if err != nil {
|
||||
return Condition{}, err
|
||||
}
|
||||
if err := k.store.Update(ctx, key, body, entry.Revision); errors.Is(err, ErrMoved) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return Condition{}, fmt.Errorf("observing the condition %s: %w", key, err)
|
||||
}
|
||||
for _, e := range changes {
|
||||
e.At, e.Condition = now, c
|
||||
k.tell(e)
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
return Condition{}, fmt.Errorf("the condition %s kept moving under this write; %d tries", key, tries)
|
||||
}
|
||||
|
||||
// Clear removes a condition an observation says is resolved, and says so. False when none was open.
|
||||
func (k *Keeper) Clear(ctx context.Context, key, why string) (bool, error) {
|
||||
for i := 0; i < tries; i++ {
|
||||
entry, found, err := k.store.Get(ctx, key)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("reading the condition %s: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
return false, nil
|
||||
}
|
||||
var c Condition
|
||||
if err := json.Unmarshal(entry.Value, &c); err != nil {
|
||||
// Unreadable is not resolved: kept, and said, rather than removed unread.
|
||||
return false, fmt.Errorf("the condition %s on the bus cannot be read, so it is not cleared: %w", key, err)
|
||||
}
|
||||
if err := k.stamp(&c); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := k.store.Delete(ctx, key, entry.Revision); errors.Is(err, ErrMoved) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return false, fmt.Errorf("clearing the condition %s: %w", key, err)
|
||||
}
|
||||
now := k.now().UTC()
|
||||
k.mu.Lock()
|
||||
k.cleared[key] = clearing{at: now, count: c.Count, silenced: c.Silenced, tried: c.Tried}
|
||||
k.mu.Unlock()
|
||||
k.tell(Event{Condition: c, At: now, Change: ChangeCleared, Why: why, Cleared: &now})
|
||||
return true, nil
|
||||
}
|
||||
return false, fmt.Errorf("the condition %s kept moving under this clearing; %d tries", key, tries)
|
||||
}
|
||||
|
||||
// Tried records a healer's attempt on an open condition (to-be 45 §7) and makes the healer its
|
||||
// resolver: said as `condition-changed` when the resolver changes, kept in `tried` either way. **It
|
||||
// never clears the condition**: a repair that worked is seen by the observation that raised it, which
|
||||
// clears it — a healer marking its own work done would be a second opinion of the fact. False when no
|
||||
// condition is open under the key: it cleared meanwhile, and there is nothing to record against.
|
||||
func (k *Keeper) Tried(ctx context.Context, key string, a Attempt, resolver string) (Condition, bool, error) {
|
||||
return k.amend(ctx, key, func(c *Condition, now time.Time) []Event {
|
||||
c.Tried = appendAttempt(c.Tried, a, now)
|
||||
var changes []Event
|
||||
if resolver != "" && resolver != c.Resolver && !c.Escalated() {
|
||||
changes = append(changes, Event{Change: ChangeResolver, Was: c.Resolver, Why: a.Outcome})
|
||||
c.Resolver = resolver
|
||||
}
|
||||
return changes
|
||||
})
|
||||
}
|
||||
|
||||
// Escalate is a healer's budget spent, or a repair it may not make (to-be 45 §2, §7): the attempt that
|
||||
// says so is kept in `tried`, the resolver becomes the operator and the severity urgent, each said as
|
||||
// `condition-changed`. Observation still clears it when the fault goes; nothing else does.
|
||||
func (k *Keeper) Escalate(ctx context.Context, key string, a Attempt) (Condition, bool, error) {
|
||||
return k.amend(ctx, key, func(c *Condition, now time.Time) []Event {
|
||||
c.Tried = appendAttempt(c.Tried, a, now)
|
||||
var changes []Event
|
||||
if c.Severity != Urgent {
|
||||
changes = append(changes, Event{Change: ChangeSeverity, Was: string(c.Severity), Why: a.Outcome})
|
||||
c.Severity = Urgent
|
||||
}
|
||||
if c.Resolver != ResolverOperator {
|
||||
changes = append(changes, Event{Change: ChangeResolver, Was: c.Resolver, Why: a.Outcome})
|
||||
c.Resolver = ResolverOperator
|
||||
}
|
||||
return changes
|
||||
})
|
||||
}
|
||||
|
||||
// appendAttempt adds an attempt, stamped when it has no time, keeping the newest KeptAttempts.
|
||||
func appendAttempt(tried []Attempt, a Attempt, now time.Time) []Attempt {
|
||||
if a.At.IsZero() {
|
||||
a.At = now
|
||||
}
|
||||
tried = append(tried, a)
|
||||
if len(tried) > KeptAttempts {
|
||||
tried = tried[len(tried)-KeptAttempts:]
|
||||
}
|
||||
return tried
|
||||
}
|
||||
|
||||
// amend changes one open condition by compare-and-set and says what changed. False when none is open.
|
||||
func (k *Keeper) amend(ctx context.Context, key string, change func(*Condition, time.Time) []Event) (Condition, bool, error) {
|
||||
for i := 0; i < tries; i++ {
|
||||
entry, found, err := k.store.Get(ctx, key)
|
||||
if err != nil {
|
||||
return Condition{}, false, fmt.Errorf("reading the condition %s: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
return Condition{}, false, nil
|
||||
}
|
||||
var c Condition
|
||||
if err := json.Unmarshal(entry.Value, &c); err != nil {
|
||||
return Condition{}, false, fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err)
|
||||
}
|
||||
now := k.now().UTC()
|
||||
changes := change(&c, now)
|
||||
if err := k.stamp(&c); err != nil {
|
||||
return Condition{}, false, err
|
||||
}
|
||||
body, err := json.Marshal(c)
|
||||
if err != nil {
|
||||
return Condition{}, false, err
|
||||
}
|
||||
if err := k.store.Update(ctx, key, body, entry.Revision); errors.Is(err, ErrMoved) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return Condition{}, false, fmt.Errorf("writing the condition %s: %w", key, err)
|
||||
}
|
||||
for _, e := range changes {
|
||||
e.At, e.Condition = now, c
|
||||
k.tell(e)
|
||||
}
|
||||
return c, true, nil
|
||||
}
|
||||
return Condition{}, false, fmt.Errorf("the condition %s kept moving under this write; %d tries", key, tries)
|
||||
}
|
||||
|
||||
// Reconcile is one source's whole observation: every condition it observes is observed, and every
|
||||
// condition it raised before and no longer observes is cleared — the observation says it is
|
||||
// resolved. A source that could not observe must not call this: an empty observation clears all it
|
||||
// raised, which is exactly the fault of saying "none" for "I could not tell" (ADR 0227 rule 4).
|
||||
func (k *Keeper) Reconcile(ctx context.Context, source string, observed []Observation) error {
|
||||
all, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var problems []string
|
||||
for _, o := range observed {
|
||||
o.Source = source
|
||||
seen[o.Key()] = true
|
||||
if _, err := k.Observe(ctx, o); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
for _, c := range all {
|
||||
if c.Source != source || seen[c.Key] {
|
||||
continue
|
||||
}
|
||||
if _, err := k.Clear(ctx, c.Key, source+" no longer observes it"); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
return errors.New(strings.Join(problems, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Silence stops a condition's messages for a while, with a reason, by somebody (to-be 45 §2). The
|
||||
// condition stays open and `status` still says it; recording the act in the hand-act log is the
|
||||
// caller's, which knows who acted.
|
||||
func (k *Keeper) Silence(ctx context.Context, key string, d time.Duration, by, why string) (Condition, error) {
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return Condition{}, errors.New("a silence says why: --why <text>")
|
||||
}
|
||||
if d <= 0 || d > MaxSilence {
|
||||
return Condition{}, fmt.Errorf("a condition is silenced for a while, at most %s — not %s", MaxSilence, d)
|
||||
}
|
||||
for i := 0; i < tries; i++ {
|
||||
entry, found, err := k.store.Get(ctx, key)
|
||||
if err != nil {
|
||||
return Condition{}, fmt.Errorf("reading the condition %s: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
return Condition{}, fmt.Errorf("no condition %s is open — `conditions` lists them", key)
|
||||
}
|
||||
var c Condition
|
||||
if err := json.Unmarshal(entry.Value, &c); err != nil {
|
||||
return Condition{}, fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err)
|
||||
}
|
||||
now := k.now().UTC()
|
||||
c.Silenced = &Silence{Until: now.Add(d), By: by, Why: strings.TrimSpace(why), Since: now}
|
||||
if err := k.stamp(&c); err != nil {
|
||||
return Condition{}, err
|
||||
}
|
||||
body, err := json.Marshal(c)
|
||||
if err != nil {
|
||||
return Condition{}, err
|
||||
}
|
||||
if err := k.store.Update(ctx, key, body, entry.Revision); errors.Is(err, ErrMoved) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return Condition{}, fmt.Errorf("silencing the condition %s: %w", key, err)
|
||||
}
|
||||
k.tell(Event{Condition: c, At: now, Change: ChangeSilenced, Why: c.Silenced.Why})
|
||||
return c, nil
|
||||
}
|
||||
return Condition{}, fmt.Errorf("the condition %s kept moving under this silence; %d tries", key, tries)
|
||||
}
|
||||
|
||||
// EndSilences ends every silence that has run out, and says each: the condition is still open, and
|
||||
// its messages start again.
|
||||
func (k *Keeper) EndSilences(ctx context.Context) error {
|
||||
all, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
now := k.now().UTC()
|
||||
for _, c := range all {
|
||||
if c.Silenced == nil || now.Before(c.Silenced.Until) {
|
||||
continue
|
||||
}
|
||||
for i := 0; i < tries; i++ {
|
||||
entry, found, err := k.store.Get(ctx, c.Key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found {
|
||||
break
|
||||
}
|
||||
var held Condition
|
||||
if err := json.Unmarshal(entry.Value, &held); err != nil {
|
||||
return fmt.Errorf("the condition %s on the bus cannot be read: %w", c.Key, err)
|
||||
}
|
||||
if held.Silenced == nil || now.Before(held.Silenced.Until) {
|
||||
break
|
||||
}
|
||||
was := held.Silenced.Why
|
||||
held.Silenced = nil
|
||||
if err := k.stamp(&held); err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := json.Marshal(held)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := k.store.Update(ctx, c.Key, body, entry.Revision); errors.Is(err, ErrMoved) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
k.tell(Event{Condition: held, At: now, Change: ChangeUnsilenced, Why: was})
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Open is every open condition, urgent first and then oldest first.
|
||||
func (k *Keeper) Open(ctx context.Context) ([]Condition, error) {
|
||||
return Read(ctx, k.store)
|
||||
}
|
||||
|
||||
// Get is one open condition.
|
||||
func (k *Keeper) Get(ctx context.Context, key string) (Condition, bool, error) {
|
||||
return ReadOne(ctx, k.store, key)
|
||||
}
|
||||
|
||||
// HistorySince is every transition from a moment, oldest first.
|
||||
func (k *Keeper) HistorySince(ctx context.Context, since time.Time) ([]Event, error) {
|
||||
if k.history == nil {
|
||||
return nil, errors.New("this keeper has no history to read")
|
||||
}
|
||||
return k.history.Since(ctx, since)
|
||||
}
|
||||
|
||||
// Read is every open condition in a store, in the order status says them. A value that cannot be
|
||||
// read is an error naming its key, never a condition left out (ADR 0227 rule 4).
|
||||
func Read(ctx context.Context, store Backend) ([]Condition, error) {
|
||||
all, err := store.All(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the open conditions cannot be read: %w", err)
|
||||
}
|
||||
out := make([]Condition, 0, len(all))
|
||||
for key, e := range all {
|
||||
var c Condition
|
||||
if err := json.Unmarshal(e.Value, &c); err != nil {
|
||||
return nil, fmt.Errorf("the condition %s cannot be read: %w", key, err)
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
Order(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ReadOne is one open condition from a store.
|
||||
func ReadOne(ctx context.Context, store Backend, key string) (Condition, bool, error) {
|
||||
e, found, err := store.Get(ctx, key)
|
||||
if err != nil || !found {
|
||||
return Condition{}, found, err
|
||||
}
|
||||
var c Condition
|
||||
if err := json.Unmarshal(e.Value, &c); err != nil {
|
||||
return Condition{}, false, fmt.Errorf("the condition %s cannot be read: %w", key, err)
|
||||
}
|
||||
return c, true, nil
|
||||
}
|
||||
|
||||
// tell queues a transition to be kept and said. Never blocks the caller for long: a queue that is
|
||||
// full is a bus away for a long time, and the transition is said lost rather than holding a watchdog.
|
||||
func (k *Keeper) tell(e Event) {
|
||||
e.Event = eventFor(e.Change)
|
||||
e.Show = e.Condition.Show()
|
||||
if k.changed != nil {
|
||||
k.changed()
|
||||
}
|
||||
defer func() {
|
||||
// A keeper closed while a write was in flight: said, not a panic.
|
||||
if recover() != nil {
|
||||
k.lost(e, errors.New("the keeper was closed"))
|
||||
}
|
||||
}()
|
||||
select {
|
||||
case k.out <- e:
|
||||
default:
|
||||
k.lost(e, errors.New("too many transitions are waiting to be said"))
|
||||
}
|
||||
}
|
||||
|
||||
func (k *Keeper) lost(e Event, err error) {
|
||||
k.mu.Lock()
|
||||
k.unsaid++
|
||||
k.mu.Unlock()
|
||||
k.say("the condition %s was %s and that could NOT be said or kept: %v", e.Key, e.Change, err)
|
||||
}
|
||||
|
||||
// telling keeps and says every transition in order, offering each again while the bus is away.
|
||||
func (k *Keeper) telling() {
|
||||
defer close(k.drained)
|
||||
for e := range k.out {
|
||||
body, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
k.lost(e, err)
|
||||
continue
|
||||
}
|
||||
deadline := time.Now().Add(TellFor)
|
||||
wait := 200 * time.Millisecond
|
||||
kept, said := k.history == nil, k.teller == nil
|
||||
for {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
if !kept {
|
||||
kept = k.history.Append(ctx, e) == nil
|
||||
}
|
||||
if !said {
|
||||
said = k.teller.PublishSeatEvent(ctx, Seat, e.Event, body) == nil
|
||||
}
|
||||
cancel()
|
||||
if kept && said {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
what := "said"
|
||||
if !kept {
|
||||
what = "kept in the history"
|
||||
}
|
||||
k.lost(e, fmt.Errorf("not %s within %s", what, TellFor))
|
||||
break
|
||||
}
|
||||
time.Sleep(wait)
|
||||
wait = min(2*wait, 10*time.Second)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func orSelf(resolver string) string {
|
||||
if resolver == "" {
|
||||
return ResolverSelf
|
||||
}
|
||||
return resolver
|
||||
}
|
||||
@@ -1,379 +0,0 @@
|
||||
package conditions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// clock is a time a test moves by hand.
|
||||
type clock struct{ at time.Time }
|
||||
|
||||
func (c *clock) now() time.Time { return c.at }
|
||||
func (c *clock) pass(d time.Duration) { c.at = c.at.Add(d) }
|
||||
func newClock() *clock { return &clock{at: time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)} }
|
||||
func keeper(t *testing.T) (*Keeper, *InMemory, *Told, *clock) {
|
||||
t.Helper()
|
||||
store, told, c := NewInMemory(), &Told{}, newClock()
|
||||
k := NewKeeper(t.Context(), Options{Store: store, History: store, Teller: told, Now: c.now,
|
||||
Say: func(f string, a ...any) { t.Logf(f, a...) }})
|
||||
t.Cleanup(func() { k.Close(context.Background()) })
|
||||
return k, store, told, c
|
||||
}
|
||||
|
||||
// settled waits until the teller has been told n events.
|
||||
func settled(t *testing.T, told *Told, n int) []Event {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
said := told.Said()
|
||||
if len(said) >= n {
|
||||
return said
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("told %d event(s), want %d: %+v", len(said), n, said)
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func silent(node string) Observation {
|
||||
return Observation{Scope: ScopeMachine, ID: node, Kind: "silent", Machine: node, Severity: Warning,
|
||||
Summary: node + " has not been heard from", Source: "S1"}
|
||||
}
|
||||
|
||||
// **A condition is raised once, observed many times, and said on the bus only when it changes**
|
||||
// (to-be 45 §2): an observation that changes nothing is written and said nowhere, or the operator's
|
||||
// channel would hear the same fault every thirty seconds.
|
||||
func TestAConditionIsSaidWhenItChangesNotWhenItIsSeenAgain(t *testing.T) {
|
||||
k, _, told, c := keeper(t)
|
||||
ctx := t.Context()
|
||||
for i := 0; i < 3; i++ {
|
||||
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.pass(time.Minute)
|
||||
}
|
||||
urgent := silent("ace")
|
||||
urgent.Severity = Urgent
|
||||
got, err := k.Observe(ctx, urgent)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Key != "machine.ace.silent" || got.Observations != 4 || got.Count != 1 || got.Severity != Urgent {
|
||||
t.Fatalf("held %+v", got)
|
||||
}
|
||||
if len(got.Evidence) != 4 || !got.Evidence[0].At.Equal(c.at) {
|
||||
t.Fatalf("evidence is not newest first: %+v", got.Evidence)
|
||||
}
|
||||
said := settled(t, told, 2)
|
||||
if said[0].Event != EventRaised || said[0].Change != ChangeRaised || said[1].Event != EventChanged ||
|
||||
said[1].Change != ChangeSeverity || said[1].Was != string(Warning) {
|
||||
t.Fatalf("said %+v", said)
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if n := len(told.Said()); n != 2 {
|
||||
t.Fatalf("said %d events for one raising and one change", n)
|
||||
}
|
||||
for i, name := range told.Names {
|
||||
if name != told.Events[i].Event {
|
||||
t.Errorf("event %d published as %s and says it is %s", i, name, told.Events[i].Event)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **Evidence is bounded**: a condition open for a week keeps its newest ten observations, not all.
|
||||
func TestEvidenceKeepsTheNewestTen(t *testing.T) {
|
||||
k, _, _, c := keeper(t)
|
||||
var got Condition
|
||||
for i := 0; i < 25; i++ {
|
||||
var err error
|
||||
if got, err = k.Observe(t.Context(), silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.pass(time.Minute)
|
||||
}
|
||||
if len(got.Evidence) != KeptEvidence || got.Observations != 25 {
|
||||
t.Fatalf("kept %d evidence of %d observations", len(got.Evidence), got.Observations)
|
||||
}
|
||||
}
|
||||
|
||||
// **Cleared and raised again within ten minutes is the same condition again** (to-be 45 §2): its
|
||||
// count goes up and it is said as reopened, not as news; a person's silence of it still holds.
|
||||
func TestRaisedAgainSoonAfterClearingReopens(t *testing.T) {
|
||||
k, store, told, c := keeper(t)
|
||||
ctx := t.Context()
|
||||
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.Silence(ctx, "machine.ace.silent", time.Hour, "jochen", "the laptop is on the train"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cleared, err := k.Clear(ctx, "machine.ace.silent", "heard again"); err != nil || !cleared {
|
||||
t.Fatalf("cleared %v: %v", cleared, err)
|
||||
}
|
||||
c.pass(5 * time.Minute)
|
||||
again, err := k.Observe(ctx, silent("ace"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again.Count != 2 || again.Silenced == nil {
|
||||
t.Fatalf("reopened as %+v", again)
|
||||
}
|
||||
said := settled(t, told, 4)
|
||||
if said[3].Event != EventRaised || said[3].Change != ChangeReopened {
|
||||
t.Fatalf("the reopening was said as %+v", said[3])
|
||||
}
|
||||
// And from a new keeper — the controller restarted between — reading what cleared from history.
|
||||
if _, err := k.Clear(ctx, "machine.ace.silent", "heard again"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settled(t, told, 5)
|
||||
k.Close(context.Background())
|
||||
next := NewKeeper(ctx, Options{Store: store, History: store, Now: c.now})
|
||||
defer next.Close(context.Background())
|
||||
c.pass(time.Minute)
|
||||
third, err := next.Observe(ctx, silent("ace"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if third.Count != 3 {
|
||||
t.Fatalf("a controller restarted between cleared and raised said it as new: %+v", third)
|
||||
}
|
||||
// Past the window it is news.
|
||||
if _, err := next.Clear(ctx, "machine.ace.silent", "heard again"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.pass(ReopenWithin + time.Minute)
|
||||
fourth, err := next.Observe(ctx, silent("ace"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fourth.Count != 1 || fourth.Silenced != nil {
|
||||
t.Fatalf("raised past the window as %+v", fourth)
|
||||
}
|
||||
}
|
||||
|
||||
// **A source's whole observation clears what it no longer observes, and only its own.** A watchdog
|
||||
// that stops seeing a fault says it is resolved; it does not clear what another raised.
|
||||
func TestReconcileClearsOnlyTheSourcesOwn(t *testing.T) {
|
||||
k, _, told, _ := keeper(t)
|
||||
ctx := t.Context()
|
||||
other := Observation{Scope: ScopeProbe, ID: "D3", Kind: "probe-failed", Token: "failed", Severity: Warning,
|
||||
Summary: "D3 did not answer", Source: "doctor"}
|
||||
if _, err := k.Observe(ctx, other); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := k.Reconcile(ctx, "S1", []Observation{silent("ace"), silent("g14")}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := k.Reconcile(ctx, "S1", []Observation{silent("g14")}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keys []string
|
||||
for _, c := range open {
|
||||
keys = append(keys, c.Key)
|
||||
}
|
||||
if strings.Join(keys, ",") != "machine.g14.silent,probe.D3.failed" {
|
||||
t.Fatalf("open after the second observation: %v", keys)
|
||||
}
|
||||
said := settled(t, told, 4)
|
||||
last := said[3]
|
||||
if last.Event != EventCleared || last.Key != "machine.ace.silent" || last.Why == "" {
|
||||
t.Fatalf("the clearing was said as %+v", last)
|
||||
}
|
||||
}
|
||||
|
||||
// **A store that cannot be read is never an empty one** (ADR 0227 rule 4): reconciling against it
|
||||
// clears nothing and says why.
|
||||
func TestAnUnreadableStoreClearsNothing(t *testing.T) {
|
||||
k, store, _, _ := keeper(t)
|
||||
ctx := t.Context()
|
||||
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store.Fail = errors.New("the bus is away")
|
||||
if err := k.Reconcile(ctx, "S1", nil); err == nil {
|
||||
t.Fatal("reconciled against a store it could not read")
|
||||
}
|
||||
if _, err := k.Open(ctx); err == nil {
|
||||
t.Fatal("an unreadable store answered as read")
|
||||
}
|
||||
store.Fail = nil
|
||||
store.values["machine.g14.silent"] = Entry{Value: []byte("{not a condition"), Revision: 99}
|
||||
if _, err := k.Open(ctx); err == nil || !strings.Contains(err.Error(), "machine.g14.silent") {
|
||||
t.Fatalf("an unreadable condition was left out rather than said: %v", err)
|
||||
}
|
||||
if cleared, err := k.Clear(ctx, "machine.g14.silent", "x"); err == nil || cleared {
|
||||
t.Fatal("an unreadable condition was cleared unread")
|
||||
}
|
||||
}
|
||||
|
||||
// **A silence is bounded, says why, and ends on its own** (to-be 45 §2): the condition stays open
|
||||
// through it, and its messages start again when it ends.
|
||||
func TestASilenceIsBoundedAndEnds(t *testing.T) {
|
||||
k, _, told, c := keeper(t)
|
||||
ctx := t.Context()
|
||||
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.Silence(ctx, "machine.ace.silent", 8*24*time.Hour, "jochen", "away"); err == nil {
|
||||
t.Fatal("silenced for longer than a week")
|
||||
}
|
||||
if _, err := k.Silence(ctx, "machine.ace.silent", time.Hour, "jochen", " "); err == nil {
|
||||
t.Fatal("silenced without a reason")
|
||||
}
|
||||
if _, err := k.Silence(ctx, "machine.nothing.silent", time.Hour, "jochen", "x"); err == nil {
|
||||
t.Fatal("silenced a condition that is not open")
|
||||
}
|
||||
held, err := k.Silence(ctx, "machine.ace.silent", time.Hour, "jochen", "on the train")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !held.SilencedAt(c.at) || held.Silenced.By != "jochen" {
|
||||
t.Fatalf("silenced as %+v", held.Silenced)
|
||||
}
|
||||
c.pass(30 * time.Minute)
|
||||
if err := k.EndSilences(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.pass(31 * time.Minute)
|
||||
if err := k.EndSilences(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _, _ := k.Get(ctx, "machine.ace.silent")
|
||||
if got.Silenced != nil {
|
||||
t.Fatalf("a silence past its end still held: %+v", got.Silenced)
|
||||
}
|
||||
said := settled(t, told, 3)
|
||||
if said[1].Change != ChangeSilenced || said[2].Change != ChangeUnsilenced || said[2].Event != EventChanged {
|
||||
t.Fatalf("said %+v", said)
|
||||
}
|
||||
}
|
||||
|
||||
// **Two writers never lose each other's word.** The serving controller observes while a person's
|
||||
// command silences: the write that lost the compare-and-set reads again and redoes itself.
|
||||
func TestAWriteThatLostTheRaceRedoesItself(t *testing.T) {
|
||||
k, store, _, _ := keeper(t)
|
||||
ctx := t.Context()
|
||||
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
racing := &racingStore{InMemory: store, before: func() {
|
||||
// Another process silences between this keeper's read and its write.
|
||||
other := NewKeeper(ctx, Options{Store: store})
|
||||
defer other.Close(context.Background())
|
||||
if _, err := other.Silence(ctx, "machine.ace.silent", time.Hour, "jochen", "known"); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}}
|
||||
k.store = racing
|
||||
got, err := k.Observe(ctx, silent("ace"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Silenced == nil || got.Observations != 2 {
|
||||
t.Fatalf("the observation overwrote the silence: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// racingStore lets another writer in once, between a read and the write after it.
|
||||
type racingStore struct {
|
||||
*InMemory
|
||||
before func()
|
||||
done bool
|
||||
}
|
||||
|
||||
func (r *racingStore) Update(ctx context.Context, key string, value []byte, revision uint64) error {
|
||||
if !r.done {
|
||||
r.done = true
|
||||
r.before()
|
||||
}
|
||||
return r.InMemory.Update(ctx, key, value, revision)
|
||||
}
|
||||
|
||||
// **An observation that could not be routed is refused**, naming what it lacks.
|
||||
func TestAnObservationSaysWhatItIs(t *testing.T) {
|
||||
k, _, _, _ := keeper(t)
|
||||
for _, o := range []Observation{
|
||||
{Scope: "elsewhere", ID: "x", Kind: "k", Severity: Warning, Summary: "s", Source: "S1"},
|
||||
{Scope: ScopeMachine, ID: "x", Kind: "k", Severity: "loud", Summary: "s", Source: "S1"},
|
||||
{Scope: ScopeMachine, ID: "x", Kind: "k", Severity: Warning, Source: "S1"},
|
||||
{Scope: ScopeMachine, ID: "x", Kind: "k", Severity: Warning, Summary: "s"},
|
||||
} {
|
||||
if _, err := k.Observe(t.Context(), o); err == nil {
|
||||
t.Errorf("observed %+v", o)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **A key holds nothing the bus would refuse or read as a wildcard**, whatever the thing is called.
|
||||
func TestAKeyIsSafeForTheBus(t *testing.T) {
|
||||
if got := Key(ScopeProvider, "keycloak.novox.my app*", "failing"); got != "provider.keycloak.novox.my_app_.failing" {
|
||||
t.Fatalf("key %q", got)
|
||||
}
|
||||
if got := Key(ScopeBus, "EVENTS.>", "consumer-lost"); got != "bus.EVENTS._.consumer-lost" {
|
||||
t.Fatalf("key %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **The event's shape is a contract** (to-be 45 §2): the operator-channel's holder is written against
|
||||
// these field names. A rename here is a channel that reads nothing, so they are held still.
|
||||
func TestTheEventShapeIsTheContract(t *testing.T) {
|
||||
k, _, told, _ := keeper(t)
|
||||
if _, err := k.Observe(t.Context(), silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said := settled(t, told, 1)
|
||||
body, err := json.Marshal(said[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var shape map[string]any
|
||||
if err := json.Unmarshal(body, &shape); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The condition at the top level, kebab-case, beside what happened to it.
|
||||
for _, field := range []string{"event", "at", "change", "show", "key", "kind", "subject", "severity",
|
||||
"summary", "evidence", "source", "raised", "last-observed", "observations", "count", "resolver",
|
||||
"silenced", "epoch"} {
|
||||
if _, ok := shape[field]; !ok {
|
||||
t.Errorf("the event carries no %q: %s", field, body)
|
||||
}
|
||||
}
|
||||
if shape["silenced"] != nil {
|
||||
t.Errorf("an unsilenced condition says silenced %v, not null", shape["silenced"])
|
||||
}
|
||||
subject, _ := shape["subject"].(map[string]any)
|
||||
if subject["scope"] != "machine" || subject["id"] != "ace" || subject["machine"] != "ace" {
|
||||
t.Errorf("subject %v", shape["subject"])
|
||||
}
|
||||
if said[0].Show != "mesh-controller.conditions key=machine.ace.silent" {
|
||||
t.Errorf("show is %q", said[0].Show)
|
||||
}
|
||||
}
|
||||
|
||||
// **A transition the bus will not take is offered again**, and said lost only after TellFor.
|
||||
func TestATransitionIsOfferedAgainWhileTheBusIsAway(t *testing.T) {
|
||||
store, told, c := NewInMemory(), &Told{Fail: errors.New("no responders")}, newClock()
|
||||
k := NewKeeper(t.Context(), Options{Store: store, History: store, Teller: told, Now: c.now})
|
||||
defer k.Close(context.Background())
|
||||
if _, err := k.Observe(t.Context(), silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
told.mu.Lock()
|
||||
told.Fail = nil
|
||||
told.mu.Unlock()
|
||||
said := settled(t, told, 1)
|
||||
if said[0].Key != "machine.ace.silent" || k.Unsaid() != 0 {
|
||||
t.Fatalf("said %+v, unsaid %d", said, k.Unsaid())
|
||||
}
|
||||
}
|
||||
@@ -1,105 +0,0 @@
|
||||
package conditions
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **A healer's attempt is said and kept, and never clears the condition** (to-be 45 §7): the resolver
|
||||
// becomes the healer, `tried` says what it did, and only an observation clears it.
|
||||
func TestAHealersAttemptIsKeptAndClearsNothing(t *testing.T) {
|
||||
k, _, told, c := keeper(t)
|
||||
ctx := t.Context()
|
||||
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held, open, err := k.Tried(ctx, "machine.ace.silent", Attempt{What: "asked ace to report", Outcome: "acted",
|
||||
By: "healer H1"}, ResolverHealer("H1"))
|
||||
if err != nil || !open {
|
||||
t.Fatalf("tried: %v %v", open, err)
|
||||
}
|
||||
if held.Resolver != "healer:H1" || len(held.Tried) != 1 || held.Tried[0].By != "healer H1" || held.Tried[0].At.IsZero() {
|
||||
t.Fatalf("after one attempt: %+v", held)
|
||||
}
|
||||
said := settled(t, told, 2)
|
||||
if said[1].Event != EventChanged || said[1].Change != ChangeResolver || said[1].Was != ResolverSelf {
|
||||
t.Errorf("the healer taking it is not said as a change of resolver: %+v", said[1])
|
||||
}
|
||||
// A second attempt by the same healer is kept and says nothing new.
|
||||
if _, _, err := k.Tried(ctx, "machine.ace.silent", Attempt{What: "sent again", Outcome: "acted", By: "healer H1"},
|
||||
ResolverHealer("H1")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, _, _ := k.Get(ctx, "machine.ace.silent"); len(got.Tried) != 2 {
|
||||
t.Errorf("tried %+v", got.Tried)
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if n := len(told.Said()); n != 2 {
|
||||
t.Errorf("a second attempt said %d events in all, want 2", n)
|
||||
}
|
||||
// Nothing open: nothing recorded, and no error.
|
||||
if _, open, err := k.Tried(ctx, "machine.nobody.silent", Attempt{What: "x"}, ResolverHealer("H1")); open || err != nil {
|
||||
t.Errorf("an attempt on nothing open: %v %v", open, err)
|
||||
}
|
||||
c.pass(time.Minute)
|
||||
}
|
||||
|
||||
// **A spent budget is the operator's, urgent, and the watchdog seeing it again does not undo that.**
|
||||
func TestAnEscalationHoldsAgainstTheNextObservation(t *testing.T) {
|
||||
k, _, told, _ := keeper(t)
|
||||
ctx := t.Context()
|
||||
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := k.Tried(ctx, "machine.ace.silent", Attempt{What: "asked", Outcome: "acted", By: "healer H1"},
|
||||
ResolverHealer("H1")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held, _, err := k.Escalate(ctx, "machine.ace.silent", Attempt{What: "budget spent", Outcome: "escalated", By: "healer H1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if held.Severity != Urgent || held.Resolver != ResolverOperator || !held.Escalated() {
|
||||
t.Fatalf("escalated: %+v", held)
|
||||
}
|
||||
said := settled(t, told, 4)
|
||||
if said[2].Change != ChangeSeverity || said[3].Change != ChangeResolver || said[3].Was != "healer:H1" {
|
||||
t.Errorf("the escalation is not said as severity then resolver: %+v %+v", said[2], said[3])
|
||||
}
|
||||
again, err := k.Observe(ctx, silent("ace")) // a warning, as the watchdog says it
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again.Severity != Urgent || again.Resolver != ResolverOperator {
|
||||
t.Errorf("the next observation undid the escalation: %+v", again)
|
||||
}
|
||||
// A healer trying later does not take it back from the operator.
|
||||
after, _, _ := k.Tried(ctx, "machine.ace.silent", Attempt{What: "x", By: "healer H1"}, ResolverHealer("H1"))
|
||||
if after.Resolver != ResolverOperator {
|
||||
t.Errorf("a later attempt took the condition back from the operator: %s", after.Resolver)
|
||||
}
|
||||
}
|
||||
|
||||
// **What was tried is carried into a reopening**: the same fault again within ten minutes is the
|
||||
// same condition, and what the healers tried on it is still what they tried.
|
||||
func TestAReopeningCarriesWhatWasTried(t *testing.T) {
|
||||
k, _, _, c := keeper(t)
|
||||
ctx := t.Context()
|
||||
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := k.Tried(ctx, "machine.ace.silent", Attempt{What: "asked", By: "healer H1"}, ResolverHealer("H1")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.Clear(ctx, "machine.ace.silent", "heard again"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.pass(5 * time.Minute)
|
||||
again, err := k.Observe(ctx, silent("ace"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again.Count != 2 || len(again.Tried) != 1 || again.Tried[0].What != "asked" {
|
||||
t.Errorf("reopened without what was tried: %+v", again)
|
||||
}
|
||||
}
|
||||
@@ -1,146 +0,0 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// The durations the core's bounds are set from (novox/hq to-be 45 Phase 0): see migration 0066.
|
||||
|
||||
// The kinds of duration recorded.
|
||||
const (
|
||||
DurationApply = "apply"
|
||||
DurationHeartbeatGap = "heartbeat-gap"
|
||||
DurationPlanTier = "plan-tier"
|
||||
DurationBuild = "build"
|
||||
)
|
||||
|
||||
// DurationKinds are every kind, in the order `durations` shows them.
|
||||
var DurationKinds = []string{DurationApply, DurationHeartbeatGap, DurationPlanTier, DurationBuild}
|
||||
|
||||
// DurationsKeptFor is how long a duration is kept: long enough to set a bound from, and to correct it
|
||||
// in Phase 1's first live week.
|
||||
const DurationsKeptFor = 30 * 24 * time.Hour
|
||||
|
||||
// Duration is one measurement.
|
||||
type Duration struct {
|
||||
Kind string `json:"kind"`
|
||||
Subject string `json:"subject"`
|
||||
Node string `json:"node,omitempty"`
|
||||
Ref string `json:"ref"`
|
||||
Started time.Time `json:"started"`
|
||||
Took time.Duration `json:"took"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
}
|
||||
|
||||
// RecordDuration keeps one measurement, once: the same thing measured again is not a second row.
|
||||
func (i *Inventory) RecordDuration(ctx context.Context, d Duration) error {
|
||||
if d.Took < 0 {
|
||||
return nil // a clock that went back measures nothing
|
||||
}
|
||||
_, err := i.store.Pool().Exec(ctx,
|
||||
`insert into duration (kind, subject, node, ref, started, took_ms, detail)
|
||||
values ($1, $2, $3, $4, $5, $6, $7) on conflict do nothing`,
|
||||
d.Kind, d.Subject, d.Node, d.Ref, d.Started, d.Took.Milliseconds(), d.Detail)
|
||||
return err
|
||||
}
|
||||
|
||||
// RecordApplyDuration measures a machine's report of the declaration it was last sent: from the send
|
||||
// to the first report of it. A report of anything else, or of a send already measured, measures
|
||||
// nothing — a machine reconciling reports the same declaration every few minutes.
|
||||
func (i *Inventory) RecordApplyDuration(ctx context.Context, node, declared, outcome string) error {
|
||||
if declared == "" {
|
||||
return nil
|
||||
}
|
||||
_, err := i.store.Pool().Exec(ctx,
|
||||
`insert into duration (kind, subject, node, ref, started, took_ms, detail)
|
||||
select $1, n.name, n.name, n.sent || '@' || to_char(n.sent_at at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US'),
|
||||
n.sent_at, (extract(epoch from (now() - n.sent_at)) * 1000)::bigint, $4
|
||||
from node n
|
||||
where n.name = $2 and n.sent = $3 and n.sent_at is not null
|
||||
on conflict do nothing`,
|
||||
DurationApply, node, declared, outcome)
|
||||
return err
|
||||
}
|
||||
|
||||
// RecordHeartbeatGap measures the silence before a machine's word: from the last word heard before
|
||||
// it, which the caller read before recording this one.
|
||||
func (i *Inventory) RecordHeartbeatGap(ctx context.Context, node string, before time.Time) error {
|
||||
if before.IsZero() {
|
||||
return nil
|
||||
}
|
||||
now := time.Now()
|
||||
return i.RecordDuration(ctx, Duration{Kind: DurationHeartbeatGap, Subject: node, Node: node,
|
||||
Ref: before.UTC().Format(time.RFC3339Nano), Started: before, Took: now.Sub(before)})
|
||||
}
|
||||
|
||||
// Durations is every measurement of a kind since a moment, oldest first; every kind when kind is empty.
|
||||
func (i *Inventory) Durations(ctx context.Context, kind string, since time.Time) ([]Duration, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select kind, subject, node, ref, started, took_ms, detail from duration
|
||||
where ($1 = '' or kind = $1) and recorded >= $2 order by recorded`, kind, since)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Duration
|
||||
for rows.Next() {
|
||||
var d Duration
|
||||
var ms int64
|
||||
if err := rows.Scan(&d.Kind, &d.Subject, &d.Node, &d.Ref, &d.Started, &ms, &d.Detail); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d.Took = time.Duration(ms) * time.Millisecond
|
||||
out = append(out, d)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ForgetOldDurations removes what is older than DurationsKeptFor, and says how many.
|
||||
func (i *Inventory) ForgetOldDurations(ctx context.Context) (int64, error) {
|
||||
tag, err := i.store.Pool().Exec(ctx, `delete from duration where recorded < $1`,
|
||||
time.Now().Add(-DurationsKeptFor))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return tag.RowsAffected(), nil
|
||||
}
|
||||
|
||||
// planTierLeft is the measurement a plan's save makes when it leaves a tier: the tier moved on, or
|
||||
// the plan ended. Read in the save's own transaction, so two saves cannot both measure one tier.
|
||||
func planTierLeft(ctx context.Context, tx pgx.Tx, p Plan, now time.Time) (entered time.Time, err error) {
|
||||
var oldTier int
|
||||
var oldState string
|
||||
var since time.Time
|
||||
err = tx.QueryRow(ctx,
|
||||
`select tier, state, coalesce(tier_entered, created) from release_plan where id = $1 for update`,
|
||||
p.ID).Scan(&oldTier, &oldState, &since)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return now, nil // a new plan enters its first tier now
|
||||
}
|
||||
if err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
wasOpen := oldState == PlanBuilding || oldState == PlanRolling
|
||||
if !wasOpen || (oldTier == p.Tier && p.Open()) {
|
||||
return since, nil // still in the tier, or already ended
|
||||
}
|
||||
var modules []string
|
||||
if oldTier >= 0 && oldTier < len(p.Tiers) {
|
||||
modules = p.Tiers[oldTier]
|
||||
}
|
||||
detail := fmt.Sprintf("plan %s, tier %d of %d (%v), left %s", p.ID, oldTier, len(p.Tiers), modules, p.State)
|
||||
if p.Open() {
|
||||
detail = fmt.Sprintf("plan %s, tier %d of %d (%v), moved on to tier %d", p.ID, oldTier, len(p.Tiers), modules, p.Tier)
|
||||
}
|
||||
_, err = tx.Exec(ctx,
|
||||
`insert into duration (kind, subject, node, ref, started, took_ms, detail)
|
||||
values ($1, $2, '', $3, $4, $5, $6) on conflict do nothing`,
|
||||
DurationPlanTier, p.Repository, fmt.Sprintf("%s/tier-%d", p.ID, oldTier), since,
|
||||
now.Sub(since).Milliseconds(), detail)
|
||||
return now, err
|
||||
}
|
||||
@@ -1,97 +0,0 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **The durations the bounds are set from are recorded once each** (novox/hq to-be 45 Phase 0): a
|
||||
// send measured at its first report and not again at every reconcile that repeats it; a send of
|
||||
// something else measures nothing; a new send is a new measurement.
|
||||
func TestAnApplyIsMeasuredOncePerSend(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
node, err := inv.AddNode(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, node.ID, "d1", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for range 3 { // the report, then two reconciles saying the same
|
||||
if err := inv.RecordApplyDuration(ctx, "anchor", "d1", OutcomeApplied); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.RecordApplyDuration(ctx, "anchor", "d0", OutcomeApplied); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ds, err := inv.Durations(ctx, DurationApply, time.Now().Add(-time.Hour))
|
||||
if err != nil || len(ds) != 1 || ds[0].Subject != "anchor" || ds[0].Detail != OutcomeApplied || ds[0].Took < 0 {
|
||||
t.Fatalf("%v %+v", err, ds)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
if err := inv.RecordSent(ctx, node.ID, "d2", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordApplyDuration(ctx, "anchor", "d2", OutcomeFailed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ds, _ = inv.Durations(ctx, "", time.Now().Add(-time.Hour)); len(ds) != 2 {
|
||||
t.Fatalf("a second send was not measured: %+v", ds)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine's silence is the time since its last word, once per word.
|
||||
func TestASilenceIsMeasuredFromTheLastWord(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
before := time.Now().Add(-90 * time.Second)
|
||||
for range 2 {
|
||||
if err := inv.RecordHeartbeatGap(ctx, "anchor", before); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.RecordHeartbeatGap(ctx, "anchor", time.Time{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ds, err := inv.Durations(ctx, DurationHeartbeatGap, time.Now().Add(-time.Hour))
|
||||
if err != nil || len(ds) != 1 || ds[0].Took < 90*time.Second || ds[0].Took > 2*time.Minute {
|
||||
t.Fatalf("%v %+v", err, ds)
|
||||
}
|
||||
}
|
||||
|
||||
// A plan's tier is measured when the plan leaves it — moving on, or ending — and only then.
|
||||
func TestAPlansTierIsMeasuredWhenItIsLeft(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
p := Plan{ID: "plan-1", Repository: "novox/mesh-tools", Commit: "abc", Created: time.Now().UTC(),
|
||||
State: PlanBuilding, Tiers: [][]string{{"mesh-tools"}, {"builder"}},
|
||||
Modules: map[string]*PlanModule{"mesh-tools": {}, "builder": {}}}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.State = PlanRolling // the same tier, saved again
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ds, _ := inv.Durations(ctx, DurationPlanTier, time.Now().Add(-time.Hour)); len(ds) != 0 {
|
||||
t.Fatalf("a tier not left was measured: %+v", ds)
|
||||
}
|
||||
p.Tier = 1
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.State = PlanDone
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil { // saved again once ended: nothing more to measure
|
||||
t.Fatal(err)
|
||||
}
|
||||
ds, err := inv.Durations(ctx, DurationPlanTier, time.Now().Add(-time.Hour))
|
||||
if err != nil || len(ds) != 2 || ds[0].Subject != "novox/mesh-tools" || ds[0].Ref != "plan-1/tier-0" ||
|
||||
ds[1].Ref != "plan-1/tier-1" {
|
||||
t.Fatalf("%v %+v", err, ds)
|
||||
}
|
||||
}
|
||||
@@ -1,181 +0,0 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// A value given by hand lives only until the module's first good start (novox/hq ADR 0228).
|
||||
//
|
||||
// **A person gives a module's own secret for one reason**: to adopt something already running that
|
||||
// holds that value. A module the mesh installs fresh needs none — the mesh makes it. So for a secret
|
||||
// the mesh may make (catalogue.OwnSecret.MeshMayMake: read at start, issued by nobody outside), a
|
||||
// given value is kept until the module has started under the mesh on it, and then replaced with one
|
||||
// the mesh makes, sealed and sent like any other. Nothing a person handled is left in force.
|
||||
//
|
||||
// What stays as given: a value an outside party issued (an API key, a bot token, a licence), which
|
||||
// no value of the mesh's would replace; a value a module applies to a backend, until the staged
|
||||
// rotation exists (ADR 0114); and a value given before this rule, which waits for a person to ask
|
||||
// `secret rotate` — the mesh does not decide for them that what was given long ago is used nowhere
|
||||
// else.
|
||||
|
||||
// AcceptGivenSecret is `secret accept` for a module's own secret: the value a person gave, sealed as
|
||||
// AcceptSecretForModule seals it, and — for a secret the mesh may make — marked to be replaced after
|
||||
// the module's first good start. It answers whether it was so marked.
|
||||
//
|
||||
// **Only the person's path marks.** The mesh's own code accepts values too — a bus account it
|
||||
// issued, the controller's bus address — and those are the mesh's word to a broker, which a fresh
|
||||
// random value would break; they go through AcceptSecretForModule and are never marked.
|
||||
func (i *Inventory) AcceptGivenSecret(ctx context.Context, node, module, name, value string) (untilStart bool, err error) {
|
||||
return i.acceptOwn(ctx, node, module, name, value, true)
|
||||
}
|
||||
|
||||
// OwnSecretOrigin is where a module's own secret on a machine came from — OriginMade or
|
||||
// OriginAccepted — and when it was made or given; empty for one it does not hold yet.
|
||||
func (i *Inventory) OwnSecretOrigin(ctx context.Context, node, module, name string) (string, time.Time, error) {
|
||||
record, err := i.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return "", time.Time{}, err
|
||||
}
|
||||
var origin string
|
||||
var at time.Time
|
||||
err = i.store.Pool().QueryRow(ctx,
|
||||
`select origin, made_at from module_secret where node = $1 and module = $2 and name = $3`,
|
||||
record.ID, module, name).Scan(&origin, &at)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return "", time.Time{}, nil
|
||||
}
|
||||
return origin, at, err
|
||||
}
|
||||
|
||||
// givenAgo is ", given <date>, N days ago" for a refusal about a value given to the mesh, and empty
|
||||
// when the mesh holds none: how old an outside party's key is, said where a person learns it cannot
|
||||
// be rotated by the mesh.
|
||||
func (i *Inventory) givenAgo(ctx context.Context, nodeID any, module, name string) string {
|
||||
var origin string
|
||||
var at time.Time
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`select origin, made_at from module_secret where node = $1 and module = $2 and name = $3`,
|
||||
nodeID, module, name).Scan(&origin, &at)
|
||||
if err != nil || origin != OriginAccepted {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("; the value held was given %s, %d day(s) ago",
|
||||
at.UTC().Format("2006-01-02"), int(time.Since(at).Hours()/24))
|
||||
}
|
||||
|
||||
// GivenReplaced is one given value the mesh replaced after its module's first good start.
|
||||
type GivenReplaced struct {
|
||||
Module, Name string
|
||||
// Given is when the replaced value was given.
|
||||
Given time.Time
|
||||
// Machines are the machines to send so the module, and every holder of a shared credential,
|
||||
// starts again on the new value.
|
||||
Machines []string
|
||||
}
|
||||
|
||||
// ReplaceGivenAfterStart replaces every given value on a machine whose module has now started well
|
||||
// under the mesh on it (novox/hq ADR 0228), and answers what it replaced; the caller sends the
|
||||
// machines each names.
|
||||
//
|
||||
// **The signal is the machine's clean report of the declaration it was last sent** — every resource
|
||||
// applied, nothing failed or refused — **when that declaration was sent after the value was given**,
|
||||
// so it is the start on the given value that counts, not an older one. On an adopted machine the
|
||||
// module must also be taken: until then the mesh runs nothing of it, and nothing has started under
|
||||
// the mesh. Each row is claimed by clearing its mark before it is remade, so two reports arriving
|
||||
// together replace a value once; a remake that fails puts the mark back, and the next good report
|
||||
// tries again.
|
||||
func (i *Inventory) ReplaceGivenAfterStart(ctx context.Context, node, declared string) ([]GivenReplaced, error) {
|
||||
if declared == "" {
|
||||
return nil, nil
|
||||
}
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select s.node, s.module, s.name, s.replace_after_start
|
||||
from module_secret s
|
||||
join node n on n.id = s.node
|
||||
join assignment a on a.node = s.node and a.module = s.module
|
||||
where n.name = $1 and n.sent = $2 and n.sent_at > s.replace_after_start
|
||||
and s.origin = 'accepted' and s.replace_after_start is not null
|
||||
and (not n.adopted or exists (select 1 from taken t where t.node = s.node and t.module = s.module))
|
||||
order by s.module, s.name`, node, declared)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
type due struct {
|
||||
nodeID any
|
||||
module, name string
|
||||
given time.Time
|
||||
}
|
||||
var dues []due
|
||||
for rows.Next() {
|
||||
var d due
|
||||
if err := rows.Scan(&d.nodeID, &d.module, &d.name, &d.given); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
dues = append(dues, d)
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(dues) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
key, err := i.SealingKeyOf(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if key == "" {
|
||||
return nil, fmt.Errorf("%s has no sealing key, so the given values it holds cannot be replaced", node)
|
||||
}
|
||||
|
||||
var out []GivenReplaced
|
||||
var errs []error
|
||||
for _, d := range dues {
|
||||
claimed, err := i.store.Pool().Exec(ctx,
|
||||
`update module_secret set replace_after_start = null
|
||||
where node = $1 and module = $2 and name = $3 and origin = 'accepted'
|
||||
and replace_after_start = $4`, d.nodeID, d.module, d.name, d.given)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
if claimed.RowsAffected() != 1 {
|
||||
continue // replaced by another report, or given again since
|
||||
}
|
||||
m, err := i.declared(ctx, d.module)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
// The definition is asked again now, not only when the value was given: one that has since
|
||||
// said the value is an outside party's, or applied, keeps it as given, and the mark stays gone.
|
||||
if own, ok := m.OwnSecrets[d.name]; !ok || !own.MeshMayMake() {
|
||||
continue
|
||||
}
|
||||
if err := i.remakeOwn(ctx, d.nodeID, key, m, d.module, d.name); err != nil {
|
||||
if _, back := i.store.Pool().Exec(ctx,
|
||||
`update module_secret set replace_after_start = $4
|
||||
where node = $1 and module = $2 and name = $3 and origin = 'accepted'
|
||||
and replace_after_start is null`, d.nodeID, d.module, d.name, d.given); back != nil {
|
||||
err = errors.Join(err, fmt.Errorf("and its mark could not be put back: %w", back))
|
||||
}
|
||||
errs = append(errs, fmt.Errorf("%s's given %q on %s was not replaced: %w", d.module, d.name, node, err))
|
||||
continue
|
||||
}
|
||||
machines, err := i.SharedHolders(ctx, node, d.module, d.name)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
if len(machines) == 0 {
|
||||
machines = []string{node}
|
||||
}
|
||||
out = append(out, GivenReplaced{Module: d.module, Name: d.name, Given: d.given, Machines: machines})
|
||||
}
|
||||
return out, errors.Join(errs...)
|
||||
}
|
||||
@@ -1,201 +0,0 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// A module with one secret of each kind the rule tells apart (novox/hq ADR 0228), assigned to the
|
||||
// consumer machine.
|
||||
func aModuleWithGivenSecrets(t *testing.T) (*Inventory, context.Context) {
|
||||
t.Helper()
|
||||
inv, ctx := twoNodesWithKeys(t)
|
||||
m := catalogue.Manifest{Module: "letta", Version: "1", OwnSecrets: catalogue.OwnSecrets{
|
||||
"server-password": {Path: "/var/lib/letta/server-password", Taken: catalogue.TakenAtStart},
|
||||
"openai-api-key": {Path: "/var/lib/letta/openai-api-key", Taken: catalogue.TakenAtStart,
|
||||
IssuedBy: catalogue.IssuedOutside},
|
||||
"logflare": {Path: "/var/lib/letta/logflare", Taken: catalogue.TakenApplied},
|
||||
"broker": {Path: "/var/lib/mesh/letta/broker"},
|
||||
}}
|
||||
if err := inv.RegisterModule(ctx, m, Source{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assign(t, inv, ctx, "consumer", "letta")
|
||||
return inv, ctx
|
||||
}
|
||||
|
||||
func assign(t *testing.T, inv *Inventory, ctx context.Context, node, module string) {
|
||||
t.Helper()
|
||||
n, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.store.Pool().Exec(ctx,
|
||||
`insert into assignment (node, module) values ($1, $2) on conflict do nothing`, n.ID, module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// sent records a declaration sent to a machine now, as a push does, and answers its digest.
|
||||
func sent(t *testing.T, inv *Inventory, ctx context.Context, node, digest string) string {
|
||||
t.Helper()
|
||||
n, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, n.ID, digest, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return digest
|
||||
}
|
||||
|
||||
func originOf(t *testing.T, inv *Inventory, ctx context.Context, node, module, name string) string {
|
||||
t.Helper()
|
||||
origin, _, err := inv.OwnSecretOrigin(ctx, node, module, name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return origin
|
||||
}
|
||||
|
||||
// **A given value is replaced once, after the first good start on it, and never again** (ADR 0228):
|
||||
// not on a report of a declaration sent before it was given, not on a report of a declaration the
|
||||
// mesh has moved past, and not a second time.
|
||||
func TestAGivenValueIsReplacedAfterTheFirstGoodStartAndNeverAgain(t *testing.T) {
|
||||
inv, ctx := aModuleWithGivenSecrets(t)
|
||||
before := sent(t, inv, ctx, "consumer", "declaration-before")
|
||||
|
||||
marked, err := inv.AcceptGivenSecret(ctx, "consumer", "letta", "server-password", "typed-by-a-person")
|
||||
if err != nil || !marked {
|
||||
t.Fatalf("a given value for a secret the mesh may make is marked to be replaced: %v %v", marked, err)
|
||||
}
|
||||
// The machine's report of what it was sent before the value was given is not a start on it.
|
||||
if got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", before); err != nil || len(got) != 0 {
|
||||
t.Fatalf("a start before the value was given replaced it: %+v %v", got, err)
|
||||
}
|
||||
carrying := sent(t, inv, ctx, "consumer", "declaration-carrying-it")
|
||||
// A report about a declaration other than the one last sent says nothing about this one.
|
||||
if got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", before); err != nil || len(got) != 0 {
|
||||
t.Fatalf("a report of an older declaration replaced it: %+v %v", got, err)
|
||||
}
|
||||
if originOf(t, inv, ctx, "consumer", "letta", "server-password") != OriginAccepted {
|
||||
t.Fatal("the given value was replaced before its module started on it")
|
||||
}
|
||||
|
||||
got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", carrying)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Module != "letta" || got[0].Name != "server-password" ||
|
||||
len(got[0].Machines) != 1 || got[0].Machines[0] != "consumer" {
|
||||
t.Fatalf("the first good start replaced %+v", got)
|
||||
}
|
||||
if originOf(t, inv, ctx, "consumer", "letta", "server-password") != OriginMade {
|
||||
t.Fatal("the replacement is not the mesh's own")
|
||||
}
|
||||
// Never again: the same report heard twice, and the next declaration's report.
|
||||
if again, err := inv.ReplaceGivenAfterStart(ctx, "consumer", carrying); err != nil || len(again) != 0 {
|
||||
t.Fatalf("the same start replaced it twice: %+v %v", again, err)
|
||||
}
|
||||
next := sent(t, inv, ctx, "consumer", "declaration-after")
|
||||
if again, err := inv.ReplaceGivenAfterStart(ctx, "consumer", next); err != nil || len(again) != 0 {
|
||||
t.Fatalf("a later start replaced the mesh's own value: %+v %v", again, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **What stays as given** (ADR 0228): a value an outside party issued, a value the module applies,
|
||||
// and a value the mesh's own code accepted — a bus account it issued is the mesh's word to a broker,
|
||||
// and a fresh random value would break it.
|
||||
func TestWhatIsNeverReplacedAfterAStart(t *testing.T) {
|
||||
inv, ctx := aModuleWithGivenSecrets(t)
|
||||
for _, name := range []string{"openai-api-key", "logflare"} {
|
||||
marked, err := inv.AcceptGivenSecret(ctx, "consumer", "letta", name, "from-outside")
|
||||
if err != nil || marked {
|
||||
t.Fatalf("%s was marked to be replaced: %v %v", name, marked, err)
|
||||
}
|
||||
}
|
||||
if err := inv.AcceptSecretForModule(ctx, "consumer", "letta", "broker", "issued-by-the-mesh"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared := sent(t, inv, ctx, "consumer", "declaration")
|
||||
if got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", declared); err != nil || len(got) != 0 {
|
||||
t.Fatalf("a value that stays as given was replaced: %+v %v", got, err)
|
||||
}
|
||||
for _, name := range []string{"openai-api-key", "logflare", "broker"} {
|
||||
if originOf(t, inv, ctx, "consumer", "letta", name) != OriginAccepted {
|
||||
t.Fatalf("%s was touched", name)
|
||||
}
|
||||
}
|
||||
// And asked by hand, the outside party's key is refused, saying why and how old it is.
|
||||
var refused ErrNotRotatable
|
||||
err := inv.RotateModuleSecret(ctx, "consumer", "letta", "openai-api-key")
|
||||
if !errors.As(err, &refused) || !strings.Contains(err.Error(), "outside the mesh") ||
|
||||
!strings.Contains(err.Error(), "day(s) ago") || !strings.Contains(err.Error(), "secret accept") {
|
||||
t.Fatalf("rotating an outside party's key must be refused with its age and the way out: %v", err)
|
||||
}
|
||||
if originOf(t, inv, ctx, "consumer", "letta", "openai-api-key") != OriginAccepted {
|
||||
t.Fatal("a refused rotation touched the value")
|
||||
}
|
||||
}
|
||||
|
||||
// On an adopted machine the module must be taken before a start is one under the mesh; and a module
|
||||
// no longer assigned to the machine has no start to wait for.
|
||||
func TestAGivenValueWaitsForTheTakeOnAnAdoptedMachine(t *testing.T) {
|
||||
inv, ctx := aModuleWithGivenSecrets(t)
|
||||
if err := inv.SetAdopted(ctx, "consumer", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.AcceptGivenSecret(ctx, "consumer", "letta", "server-password", "the-found-one"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held := sent(t, inv, ctx, "consumer", "declaration-holding-it")
|
||||
if got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", held); err != nil || len(got) != 0 {
|
||||
t.Fatalf("a module held as found, not yet taken, had its given value replaced: %+v %v", got, err)
|
||||
}
|
||||
if err := inv.Take(ctx, "consumer", "letta"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
taken := sent(t, inv, ctx, "consumer", "declaration-taking-it")
|
||||
got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", taken)
|
||||
if err != nil || len(got) != 1 {
|
||||
t.Fatalf("the first good start after the take did not replace the given value: %+v %v", got, err)
|
||||
}
|
||||
|
||||
// Unassigned: nothing starts, nothing is replaced.
|
||||
if _, err := inv.AcceptGivenSecret(ctx, "consumer", "letta", "server-password", "given-again"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.Unassign(ctx, "consumer", "letta"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gone := sent(t, inv, ctx, "consumer", "declaration-without-it")
|
||||
if got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", gone); err != nil || len(got) != 0 {
|
||||
t.Fatalf("a module no longer assigned had its given value replaced: %+v %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A definition that changed after the value was given is asked again at the start: one that now says
|
||||
// the value is an outside party's keeps it as given.
|
||||
func TestADefinitionThatNowSaysOutsideKeepsTheGivenValue(t *testing.T) {
|
||||
inv, ctx := aModuleWithGivenSecrets(t)
|
||||
if _, err := inv.AcceptGivenSecret(ctx, "consumer", "letta", "server-password", "typed"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := catalogue.Manifest{Module: "letta", Version: "2", OwnSecrets: catalogue.OwnSecrets{
|
||||
"server-password": {Path: "/var/lib/letta/server-password", Taken: catalogue.TakenAtStart,
|
||||
IssuedBy: catalogue.IssuedOutside}}}
|
||||
if err := inv.RegisterModule(ctx, m, Source{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared := sent(t, inv, ctx, "consumer", "declaration")
|
||||
if got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", declared); err != nil || len(got) != 0 {
|
||||
t.Fatalf("a value the definition now says is an outside party's was replaced: %+v %v", got, err)
|
||||
}
|
||||
if originOf(t, inv, ctx, "consumer", "letta", "server-password") != OriginAccepted {
|
||||
t.Fatal("the value was touched")
|
||||
}
|
||||
}
|
||||
@@ -1,126 +0,0 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The healers' acts (novox/hq to-be 45 §7, Phase 3): see migration 0070. The controller is their
|
||||
// only writer, under its lease; `healers` and `conditions` read them.
|
||||
|
||||
// The outcomes of a heal.
|
||||
const (
|
||||
// HealActing is an act begun and not yet finished: counted against the budget and the brake from
|
||||
// the moment it starts, so a controller that dies mid-act has still spent it.
|
||||
HealActing = "acting"
|
||||
// HealActed is an act that did what it does. Whether it repaired anything is the observation's to
|
||||
// say, never the healer's: the condition clears, or it does not.
|
||||
HealActed = "acted"
|
||||
// HealFailed is an act that could not be done.
|
||||
HealFailed = "failed"
|
||||
// HealEscalated is a budget spent, said: the condition was handed to the operator.
|
||||
HealEscalated = "escalated"
|
||||
)
|
||||
|
||||
// HealsKeptFor is how long a heal is kept: as long as a condition's history.
|
||||
const HealsKeptFor = 90 * 24 * time.Hour
|
||||
|
||||
// Heal is one act of a healer.
|
||||
type Heal struct {
|
||||
ID int64 `json:"id"`
|
||||
Healer string `json:"healer"`
|
||||
ConditionKey string `json:"condition"`
|
||||
Kind string `json:"kind"`
|
||||
BudgetKey string `json:"budget-key"`
|
||||
Act string `json:"act"`
|
||||
Outcome string `json:"outcome"`
|
||||
Said string `json:"said,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
Finished *time.Time `json:"finished,omitempty"`
|
||||
Epoch uint64 `json:"epoch,omitempty"`
|
||||
}
|
||||
|
||||
// BeginHeal writes an act about to be taken, under the lease: the act is counted from here. Refused,
|
||||
// and nothing written, by a process that may not act.
|
||||
func (i *Inventory) BeginHeal(ctx context.Context, h Heal) (Heal, error) {
|
||||
epoch, err := i.actingEpoch(ctx)
|
||||
if err != nil {
|
||||
return h, fmt.Errorf("the heal %s of %s is not begun: %w", h.Healer, h.ConditionKey, err)
|
||||
}
|
||||
if strings.TrimSpace(h.Healer) == "" || strings.TrimSpace(h.ConditionKey) == "" || strings.TrimSpace(h.Act) == "" {
|
||||
return h, errors.New("a heal names its healer, its condition and its act")
|
||||
}
|
||||
if h.BudgetKey == "" {
|
||||
h.BudgetKey = h.ConditionKey
|
||||
}
|
||||
if h.Outcome == "" {
|
||||
h.Outcome = HealActing
|
||||
}
|
||||
// At the runner's moment when it gives one, so its budgets and the brake are counted on one clock.
|
||||
var at *time.Time
|
||||
if !h.At.IsZero() {
|
||||
at = &h.At
|
||||
}
|
||||
err = i.store.Pool().QueryRow(ctx,
|
||||
`insert into heal (healer, condition_key, kind, budget_key, act, outcome, said, epoch, at)
|
||||
values ($1, $2, $3, $4, $5, $6, $7, $8, coalesce($9, now())) returning id, at`,
|
||||
h.Healer, h.ConditionKey, h.Kind, h.BudgetKey, h.Act, h.Outcome, h.Said, epoch, at).Scan(&h.ID, &h.At)
|
||||
if err != nil {
|
||||
return h, err
|
||||
}
|
||||
if epoch != nil {
|
||||
h.Epoch = uint64(*epoch)
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// FinishHeal says what came of an act begun. Written whatever the lease says by now: what was done was
|
||||
// done, and its record must not depend on the doer still holding the lease a moment later.
|
||||
func (i *Inventory) FinishHeal(ctx context.Context, id int64, outcome, said string) error {
|
||||
tag, err := i.store.Pool().Exec(ctx,
|
||||
`update heal set outcome = $2, said = $3, finished = now() where id = $1`, id, outcome, said)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return fmt.Errorf("no heal %d to finish", id)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// HealsSince is every heal from a moment, oldest first.
|
||||
func (i *Inventory) HealsSince(ctx context.Context, since time.Time) ([]Heal, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select id, healer, condition_key, kind, budget_key, act, outcome, said, at, finished, epoch
|
||||
from heal where at >= $1 order by at, id`, since)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Heal
|
||||
for rows.Next() {
|
||||
var h Heal
|
||||
var epoch *int64
|
||||
if err := rows.Scan(&h.ID, &h.Healer, &h.ConditionKey, &h.Kind, &h.BudgetKey, &h.Act, &h.Outcome, &h.Said,
|
||||
&h.At, &h.Finished, &epoch); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if epoch != nil {
|
||||
h.Epoch = uint64(*epoch)
|
||||
}
|
||||
out = append(out, h)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ForgetOldHeals removes what is older than HealsKeptFor, and says how many.
|
||||
func (i *Inventory) ForgetOldHeals(ctx context.Context) (int64, error) {
|
||||
tag, err := i.store.Pool().Exec(ctx, `delete from heal where at < $1`, time.Now().Add(-HealsKeptFor))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return tag.RowsAffected(), nil
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **A heal is begun under the lease and kept with its epoch** (novox/hq to-be 45 §7): refused, and
|
||||
// nothing written, by a process that may not act; finished whatever the lease says by then; read back
|
||||
// in the order they were taken.
|
||||
func TestAHealIsBegunUnderTheLeaseAndFinishedAfter(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
inv.ActsUnder(func(context.Context) (uint64, error) { return 0, errors.New("the lease is not held") })
|
||||
if _, err := inv.BeginHeal(ctx, Heal{Healer: "H1", ConditionKey: "machine.anchor.sent-not-reported", Act: "asked"}); err == nil {
|
||||
t.Fatal("a heal was begun by a process that may not act")
|
||||
}
|
||||
inv.ActsUnder(func(context.Context) (uint64, error) { return 57, nil })
|
||||
at := time.Now().Add(-time.Minute).UTC().Truncate(time.Microsecond)
|
||||
first, err := inv.BeginHeal(ctx, Heal{Healer: "H1", ConditionKey: "machine.anchor.sent-not-reported",
|
||||
Kind: "sent-not-reported", Act: "asked", At: at})
|
||||
if err != nil || first.ID == 0 || first.Epoch != 57 || first.BudgetKey != first.ConditionKey ||
|
||||
first.Outcome != HealActing || !first.At.Equal(at) {
|
||||
t.Fatalf("begun: %+v %v", first, err)
|
||||
}
|
||||
if _, err := inv.BeginHeal(ctx, Heal{Healer: "H3"}); err == nil {
|
||||
t.Fatal("a heal naming no condition and no act was begun")
|
||||
}
|
||||
inv.ActsUnder(func(context.Context) (uint64, error) { return 0, errors.New("lost meanwhile") })
|
||||
if err := inv.FinishHeal(ctx, first.ID, HealActed, "it reported"); err != nil {
|
||||
t.Fatalf("what was done could not be recorded once the lease was gone: %v", err)
|
||||
}
|
||||
if err := inv.FinishHeal(ctx, first.ID+1000, HealActed, ""); err == nil {
|
||||
t.Fatal("a heal that was never begun was finished")
|
||||
}
|
||||
heals, err := inv.HealsSince(ctx, time.Now().Add(-time.Hour))
|
||||
if err != nil || len(heals) != 1 || heals[0].Outcome != HealActed || heals[0].Said != "it reported" ||
|
||||
heals[0].Finished == nil || heals[0].Epoch != 57 {
|
||||
t.Fatalf("read back: %+v %v", heals, err)
|
||||
}
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
-- The durations the core's bounds are set from (novox/hq to-be 45 Phase 0).
|
||||
--
|
||||
-- Every watchdog of the core's signals table has a bound — how long a machine may take to report
|
||||
-- after a send, how long it may be silent, how long a plan's tier or a build may take — and a bound
|
||||
-- guessed is a condition that cries wolf or one that never fires. So the controller records each
|
||||
-- duration as it is observed, and Phase 1 sets the bounds from what was recorded:
|
||||
--
|
||||
-- apply a declaration sent → the machine's first report of that declaration, per machine
|
||||
-- heartbeat-gap one word from a machine → the next, per machine
|
||||
-- plan-tier a plan entering a tier → leaving it, per repository
|
||||
-- build a build asked → its outcome heard, per module
|
||||
--
|
||||
-- One row per thing measured: `ref` names it (the send, the earlier word, the plan's tier, the
|
||||
-- build), so a report repeated by a reconcile is not a second measurement. Kept a month; read
|
||||
-- through `durations`.
|
||||
create table duration (
|
||||
kind text not null,
|
||||
subject text not null,
|
||||
node text not null default '',
|
||||
ref text not null,
|
||||
started timestamptz not null,
|
||||
took_ms bigint not null,
|
||||
detail text not null default '',
|
||||
recorded timestamptz not null default now(),
|
||||
primary key (kind, subject, ref)
|
||||
);
|
||||
|
||||
create index duration_by_kind on duration (kind, recorded);
|
||||
|
||||
-- When a plan entered the tier it is at, so leaving it measures the tier.
|
||||
alter table release_plan add column tier_entered timestamptz;
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user