Compare commits
14
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
722682f1c4 | ||
|
|
b853439792 | ||
|
|
9cf47f4429 | ||
|
|
8ddc019cd2 | ||
|
|
fab6b0059e | ||
|
|
e1f5d4fdf0 | ||
|
|
bb1607e424 | ||
|
|
cf4834a36c | ||
|
|
9d8cbe7b81 | ||
|
|
e74c32ed50 | ||
|
|
146c48fd96 | ||
|
|
1f3abd3e0e | ||
|
|
6d620f77c3 | ||
|
|
f8286c063d |
+5
-3
@@ -21,13 +21,15 @@ ARG GO_BASE=golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7
|
|||||||
FROM ${GO_BASE} AS build
|
FROM ${GO_BASE} AS build
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
|
|
||||||
# Dependencies first, so a change to the source does not refetch them.
|
# **Nothing is fetched** (novox/hq to-be 45 Phase 1): every dependency is in vendor/, committed, so
|
||||||
|
# the image builds from this repository alone — the host's validator among them, whose module no
|
||||||
|
# public proxy is asked for. Dependencies first, so a change to the source does not re-copy them.
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
COPY vendor/ vendor/
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
ARG VERSION=development
|
ARG VERSION=development
|
||||||
RUN CGO_ENABLED=0 go build -trimpath \
|
RUN CGO_ENABLED=0 GOFLAGS=-mod=vendor GOPROXY=off go build -trimpath \
|
||||||
-ldflags "-s -w -X main.version=${VERSION}" \
|
-ldflags "-s -w -X main.version=${VERSION}" \
|
||||||
-o /mesh-controller ./cmd/mesh-controller
|
-o /mesh-controller ./cmd/mesh-controller
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@@ -676,9 +677,20 @@ type answers struct {
|
|||||||
// refused, until the switch — and while there is any, the mesh is not all well: the order the
|
// refused, until the switch — and while there is any, the mesh is not all well: the order the
|
||||||
// machines' modules are built in is the mesh's to keep, and this is where it says it is not kept.
|
// machines' modules are built in is the mesh's to keep, and this is where it says it is not kept.
|
||||||
unheld []catalogue.Unheld
|
unheld []catalogue.Unheld
|
||||||
// failing is every consumer a provider says it keeps failing (novox/hq ADR 0224): a provider's
|
// conditions is every open condition (novox/hq to-be 45 §2), urgent first and then oldest first:
|
||||||
// journal was the only place that said so for a day (04-ISSUES/179).
|
// what leads status, and what its all-well sentence needs to be none of, silenced ones included.
|
||||||
failing []inventory.ProviderStanding
|
// A provider failing a consumer is one of them (ADR 0224). conditionsUnread says why they could
|
||||||
|
// not be read when they could not — never read as none.
|
||||||
|
conditions []conditions.Condition
|
||||||
|
conditionsUnread string
|
||||||
|
// overflowing is every module whose identity overflows the bound of a provision it requires
|
||||||
|
// (novox/hq ADR 0225): its provider leaves it out of the grants and composes everything else, so
|
||||||
|
// this is the one place it is said across the mesh. Not well while there is any.
|
||||||
|
overflowing []catalogue.Overflow
|
||||||
|
// handActs is how many acts were done by hand in the last seven days (novox/hq to-be 45 §7), nil
|
||||||
|
// where the log is not on hand; handActsUnread why it could not be read when it could not.
|
||||||
|
handActs *int
|
||||||
|
handActsUnread string
|
||||||
}
|
}
|
||||||
|
|
||||||
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
|
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
|
||||||
|
|||||||
@@ -0,0 +1,162 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The streams and durable consumers the mesh's own traffic needs, derived once (novox/hq to-be 45 §4,
|
||||||
|
// D6 and D7).
|
||||||
|
//
|
||||||
|
// **What the controller asserts at its start and what the self-check expects to find are one
|
||||||
|
// derivation**, run against the bus to make them and against a recorder to list them. Two lists would
|
||||||
|
// drift, and a self-check comparing the bus with a second opinion of what should be there would find
|
||||||
|
// the drift rather than the fault.
|
||||||
|
|
||||||
|
// assertBusObjects brings every stream and consumer into being on r, and answers the machines that
|
||||||
|
// can now hear a declaration.
|
||||||
|
func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Raiser) ([]string, error) {
|
||||||
|
nodes, err := inv.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(nodes))
|
||||||
|
for _, n := range nodes {
|
||||||
|
names = append(names, n.Name)
|
||||||
|
}
|
||||||
|
if err := broker.Raise(r, names); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// The work queues of the mesh's own roles (novox/hq ADR 0121). The queue before the holder,
|
||||||
|
// deliberately: work queues until somebody arrives to do it, so assigning a build machine a week
|
||||||
|
// after something started asking for builds flushes the backlog instead of having lost it.
|
||||||
|
// With the seats' holders, so each role's work queue gets the consumer its holder takes
|
||||||
|
// work from. Passed as nil until the first live raise, which left the build machine bound to a
|
||||||
|
// consumer nothing had created (2026-09-28).
|
||||||
|
holders, err := seatHolders(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := broker.RaiseSeats(r, inventory.MeshSeats(), holders); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// And how every module hears what it consumes. Derived from the same records the user list is
|
||||||
|
// composed from, so a module the mesh grants a consumer's subjects has that consumer waiting.
|
||||||
|
// Done on every raise, not only when a credential is issued: every module moved onto this bus
|
||||||
|
// by the rollout was issued on the old one, and came up with nothing to bind to (2026-09-28).
|
||||||
|
consumers, err := moduleConsumers(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// Every one tried, and every failure named: one module's consumer the bus refuses is no reason
|
||||||
|
// the modules after it in the list hear nothing (novox/hq issue 208, where this runs on each send).
|
||||||
|
var failed []error
|
||||||
|
for _, c := range consumers {
|
||||||
|
if err := r.EnsureConsumer(c.Consumer); err != nil {
|
||||||
|
failed = append(failed, fmt.Errorf("how %s on %s hears what it consumes: %w", c.Module, c.Node, err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(failed) > 0 {
|
||||||
|
return nil, errors.Join(failed...)
|
||||||
|
}
|
||||||
|
return names, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// What raises the condition a send says when the objects it implies could not be asserted, and its kind.
|
||||||
|
const (
|
||||||
|
sourceBusObjects = "bus-objects"
|
||||||
|
kindBusObjectsUnasserted = "bus-objects-unasserted"
|
||||||
|
)
|
||||||
|
|
||||||
|
// assertOnSend asserts, on the bus a send is about to use, every object assertBusObjects derives —
|
||||||
|
// **whenever a declaration is sent, not only when the controller starts** (novox/hq issue 208).
|
||||||
|
//
|
||||||
|
// A module assigned after the controller started was sent its declaration and found no consumer to
|
||||||
|
// bind (`consumer not found`, messenger on 2026-10-06), and a seat holder assigned after it found no
|
||||||
|
// worker: the objects a declaration implies were asserted at start and nowhere else, so they existed
|
||||||
|
// only for what was assigned before the last restart. The same derivation, not a second list of what
|
||||||
|
// a send needs: what start asserts, the self-check expects and a send asserts are one answer. Every
|
||||||
|
// part is idempotent, so asserting the whole of it again is the no-op a restart already relies on.
|
||||||
|
//
|
||||||
|
// **A failure is said and raised, and the send goes on.** The objects are the mesh's, not the
|
||||||
|
// machines' being sent: holding every machine back for one consumer that none of them may use would
|
||||||
|
// turn one fault into all of them, and the declarations are not what is wrong. It is never silent —
|
||||||
|
// said in the send's own output and raised as a condition, which the next send that asserts them
|
||||||
|
// clears — and the start-time raise still refuses to serve without them.
|
||||||
|
func assertOnSend(ctx context.Context, inv *inventory.Inventory, r broker.Raiser, indent string) error {
|
||||||
|
_, err := assertBusObjects(ctx, inv, r)
|
||||||
|
var observed []conditions.Observation
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("%sTHE BUS DOES NOT HOLD WHAT THIS SEND IMPLIES: %v\n", indent, err)
|
||||||
|
fmt.Printf("%s a module may find no consumer to bind, or a holder no worker; sent anyway, raised as "+
|
||||||
|
"condition %s, and asserted again by the next send\n", indent, unassertedObservation(err).Key())
|
||||||
|
observed = append(observed, unassertedObservation(err))
|
||||||
|
}
|
||||||
|
// Observed when it failed, cleared when it did not: a send that asserted everything is the
|
||||||
|
// observation that the bus holds what it should.
|
||||||
|
if kerr := withKeeper(ctx, func(k *conditions.Keeper) error {
|
||||||
|
return k.Reconcile(ctx, sourceBusObjects, observed)
|
||||||
|
}); kerr != nil {
|
||||||
|
fmt.Printf("%sand whether the bus holds what this send implies could not be kept as a condition: %v\n",
|
||||||
|
indent, kerr)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// unassertedObservation is a send's failure to assert the bus's objects, as a condition.
|
||||||
|
func unassertedObservation(err error) conditions.Observation {
|
||||||
|
return conditions.Observation{Scope: conditions.ScopeBus, ID: "objects", Token: "unasserted",
|
||||||
|
Kind: kindBusObjectsUnasserted, Severity: conditions.Warning, Source: sourceBusObjects,
|
||||||
|
Summary: "the bus's streams and consumers could not be asserted when a declaration was sent: " +
|
||||||
|
"a module may find no consumer to bind, or a seat's holder no worker",
|
||||||
|
Said: err.Error()}
|
||||||
|
}
|
||||||
|
|
||||||
|
// moduleConsumers is every module's durable consumer, from the records the user list is composed from.
|
||||||
|
func moduleConsumers(ctx context.Context, inv *inventory.Inventory) ([]broker.ModuleConsumer, error) {
|
||||||
|
records, err := inv.BusRecords(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
users, err := broker.Users(records)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return broker.ConsumersOf(users), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// moduleConsumerCount is how many modules hear what they consume, for the raise's one line.
|
||||||
|
func moduleConsumerCount(ctx context.Context, inv *inventory.Inventory) (int, error) {
|
||||||
|
consumers, err := moduleConsumers(ctx, inv)
|
||||||
|
return len(consumers), err
|
||||||
|
}
|
||||||
|
|
||||||
|
// expectedBusObjects is every stream and consumer assertBusObjects would make, made nowhere.
|
||||||
|
func expectedBusObjects(ctx context.Context, inv *inventory.Inventory) ([]broker.Stream, []broker.Consumer, error) {
|
||||||
|
var rec recordingRaiser
|
||||||
|
if _, err := assertBusObjects(ctx, inv, &rec); err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("what the bus should hold cannot be worked out: %w", err)
|
||||||
|
}
|
||||||
|
return rec.streams, rec.consumers, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// recordingRaiser keeps what it was asked to assert and asserts nothing.
|
||||||
|
type recordingRaiser struct {
|
||||||
|
streams []broker.Stream
|
||||||
|
consumers []broker.Consumer
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *recordingRaiser) EnsureStream(s broker.Stream) error {
|
||||||
|
r.streams = append(r.streams, s)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *recordingRaiser) EnsureConsumer(c broker.Consumer) error {
|
||||||
|
r.consumers = append(r.consumers, c)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,199 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq issue 208: the bus's objects a declaration implies are asserted whenever one is sent, not
|
||||||
|
// only when the controller starts.
|
||||||
|
|
||||||
|
// aCarriedConsumer is the runtime on laptop and, carried by it, a module that consumes an event: the
|
||||||
|
// shape of messenger on 2026-10-06, assigned after the controller started.
|
||||||
|
func aCarriedConsumer(t *testing.T, open *stores) broker.Consumer {
|
||||||
|
t.Helper()
|
||||||
|
register(t, open, catalogue.Manifest{Module: catalogue.RuntimeModule, Version: "1",
|
||||||
|
OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/var/lib/mesh/node-tools/broker"}}})
|
||||||
|
register(t, open, catalogue.Manifest{Module: "messenger", Version: "1",
|
||||||
|
Consumes: []string{"billing.order.placed"}})
|
||||||
|
for _, m := range []string{catalogue.RuntimeModule, "messenger"} {
|
||||||
|
if _, err := open.inventory.Assign(t.Context(), "laptop", m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
consumers, err := moduleConsumers(t.Context(), open.inventory)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, c := range consumers {
|
||||||
|
if c.Module == "messenger" && c.Node == "laptop" {
|
||||||
|
return c.Consumer
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("messenger on laptop is derived no consumer: %+v", consumers)
|
||||||
|
return broker.Consumer{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// aLateHolder is a module holding the build agent's seat on anchor, assigned after the controller
|
||||||
|
// started, and the worker its seat's queue should have for it.
|
||||||
|
func aLateHolder(t *testing.T, open *stores) broker.Consumer {
|
||||||
|
t.Helper()
|
||||||
|
register(t, open, catalogue.Manifest{Module: "late-builder", Version: "1",
|
||||||
|
Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}})
|
||||||
|
if _, err := open.inventory.Assign(t.Context(), "anchor", "late-builder"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, s := range inventory.MeshSeats() {
|
||||||
|
if s.Name == "node-build-agent" {
|
||||||
|
c, needed := broker.HolderConsumerFor("anchor", "late-builder", s)
|
||||||
|
if !needed {
|
||||||
|
t.Fatal("the build agent's seat needs no worker")
|
||||||
|
}
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatal("the mesh declares no build agent's seat")
|
||||||
|
return broker.Consumer{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// asserted says whether a recording holds a consumer by stream and name.
|
||||||
|
func asserted(rec *recordingRaiser, want broker.Consumer) bool {
|
||||||
|
for _, c := range rec.consumers {
|
||||||
|
if c.Stream == want.Stream && c.Name == want.Name {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// What a send asserts includes what was assigned after the start's assertion: a carried module's
|
||||||
|
// consumer and a late holder's worker. The derivation is the start's own, so this holds without a bus.
|
||||||
|
func TestASendAssertsWhatWasAssignedAfterStart(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
var atStart recordingRaiser
|
||||||
|
if _, err := assertBusObjects(t.Context(), open.inventory, &atStart); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
consumer := aCarriedConsumer(t, open)
|
||||||
|
worker := aLateHolder(t, open)
|
||||||
|
if asserted(&atStart, consumer) || asserted(&atStart, worker) {
|
||||||
|
t.Fatal("the start asserted what was not yet assigned; the test proves nothing")
|
||||||
|
}
|
||||||
|
var onSend recordingRaiser
|
||||||
|
if err := assertOnSend(t.Context(), open.inventory, &onSend, ""); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !asserted(&onSend, consumer) {
|
||||||
|
t.Fatalf("messenger's consumer %s on %s is not asserted by the send: %+v", consumer.Name, consumer.Stream, onSend.consumers)
|
||||||
|
}
|
||||||
|
if !asserted(&onSend, worker) {
|
||||||
|
t.Fatalf("the late holder's worker %s on %s is not asserted by the send: %+v", worker.Name, worker.Stream, onSend.consumers)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// failingRaiser refuses one consumer by name and records everything it was asked.
|
||||||
|
type failingRaiser struct {
|
||||||
|
recordingRaiser
|
||||||
|
refuse string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *failingRaiser) EnsureConsumer(c broker.Consumer) error {
|
||||||
|
f.consumers = append(f.consumers, c)
|
||||||
|
if c.Name == f.refuse {
|
||||||
|
return errors.New("nats: API error: code=503 description=insufficient resources")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// A send whose objects cannot be asserted says so in its output and raises a condition — and the next
|
||||||
|
// send that asserts them clears it. The consumers after the refused one are still asked for.
|
||||||
|
func TestASendThatCannotAssertTheBusSaysSoAndRaisesACondition(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
consumer := aCarriedConsumer(t, open)
|
||||||
|
worker := aLateHolder(t, open)
|
||||||
|
failing := &failingRaiser{refuse: consumer.Name}
|
||||||
|
|
||||||
|
var sendErr error
|
||||||
|
out := stdoutOf(t, func() error {
|
||||||
|
sendErr = assertOnSend(t.Context(), open.inventory, failing, " ")
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if sendErr == nil || !strings.Contains(sendErr.Error(), "how messenger on laptop hears what it consumes") {
|
||||||
|
t.Fatalf("the failure is not answered: %v", sendErr)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, "THE BUS DOES NOT HOLD WHAT THIS SEND IMPLIES") ||
|
||||||
|
!strings.Contains(out, "insufficient resources") || !strings.Contains(out, "bus.objects.unasserted") {
|
||||||
|
t.Fatalf("the failure is not said in the send's output:\n%s", out)
|
||||||
|
}
|
||||||
|
if !asserted(&failing.recordingRaiser, worker) {
|
||||||
|
t.Fatal("the seat's worker was not asked for")
|
||||||
|
}
|
||||||
|
c, open1, err := conditionsFrom.Get(t.Context(), "bus.objects.unasserted")
|
||||||
|
if err != nil || !open1 {
|
||||||
|
t.Fatalf("no condition raised: %v", err)
|
||||||
|
}
|
||||||
|
if c.Kind != kindBusObjectsUnasserted || c.Source != sourceBusObjects ||
|
||||||
|
len(c.Evidence) == 0 || !strings.Contains(c.Evidence[0].Said, "insufficient resources") {
|
||||||
|
t.Fatalf("the condition does not say what failed: %+v", c)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The next send asserts them, and that observation clears it.
|
||||||
|
if err := assertOnSend(t.Context(), open.inventory, &recordingRaiser{}, ""); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, stillOpen, err := conditionsFrom.Get(t.Context(), "bus.objects.unasserted"); err != nil || stillOpen {
|
||||||
|
t.Fatalf("a send that asserted everything left the condition open: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Against a real bus, through the send's own grant: a module assigned after start has its consumer
|
||||||
|
// once a declaration is sent, and a holder assigned after start its seat's worker.
|
||||||
|
func TestNatsAModuleAssignedAfterStartGetsItsConsumerAtItsFirstSend(t *testing.T) {
|
||||||
|
url := os.Getenv("MESH_TEST_NATS")
|
||||||
|
if url == "" {
|
||||||
|
t.Skip("MESH_TEST_NATS unset")
|
||||||
|
}
|
||||||
|
open := aMesh(t)
|
||||||
|
js, err := broker.Dial(url)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(js.Close)
|
||||||
|
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
|
||||||
|
_ = js.Context().DeleteStream(s)
|
||||||
|
}
|
||||||
|
// The controller starting, before either was assigned.
|
||||||
|
if _, err := assertBusObjects(t.Context(), open.inventory, js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
consumer := aCarriedConsumer(t, open)
|
||||||
|
worker := aLateHolder(t, open)
|
||||||
|
_ = js.Context().DeleteConsumer(worker.Stream, worker.Name)
|
||||||
|
for _, c := range []broker.Consumer{consumer, worker} {
|
||||||
|
if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); !errors.Is(err, nats.ErrConsumerNotFound) {
|
||||||
|
t.Fatalf("%s on %s exists before any send; the test proves nothing: %v", c.Name, c.Stream, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
server := link.ConnectNats(js, nil, nil)
|
||||||
|
if err := (overTheBus{open: open, server: server}).grant(t.Context(), nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, c := range []broker.Consumer{consumer, worker} {
|
||||||
|
if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); err != nil {
|
||||||
|
t.Fatalf("%s on %s is not on the bus after a send: %v", c.Name, c.Stream, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, raised, _ := conditionsFrom.Get(t.Context(), "bus.objects.unasserted"); raised {
|
||||||
|
t.Fatal("a send that asserted everything raised a condition")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -35,10 +35,10 @@ func TestAPushAnswersBeforeItSends(t *testing.T) {
|
|||||||
args map[string]any
|
args map[string]any
|
||||||
want bool
|
want bool
|
||||||
}{
|
}{
|
||||||
{"push", map[string]any{"node": "anchor"}, true},
|
{"push", map[string]any{"node": "anchor", "why": "w"}, true},
|
||||||
{"push", map[string]any{}, true},
|
{"push", map[string]any{"why": "w"}, true},
|
||||||
{"command", map[string]any{"command": "push anchor"}, true},
|
{"command", map[string]any{"command": "push anchor --why w"}, true},
|
||||||
{"command", map[string]any{"command": "push --behind"}, true},
|
{"command", map[string]any{"command": "push --behind --why=w"}, true},
|
||||||
{"command", map[string]any{"command": "builds"}, false},
|
{"command", map[string]any{"command": "builds"}, false},
|
||||||
{"status", map[string]any{}, false},
|
{"status", map[string]any{}, false},
|
||||||
{"assign", map[string]any{"node": "anchor", "module": "m"}, false},
|
{"assign", map[string]any{"node": "anchor", "module": "m"}, false},
|
||||||
|
|||||||
@@ -25,7 +25,17 @@ import (
|
|||||||
// mesh seat is judged fully only at registration. A seat another module declares is unknown unless
|
// mesh seat is judged fully only at registration. A seat another module declares is unknown unless
|
||||||
// that module's manifest is passed too. Both are printed as a note, not as a problem — a check that
|
// that module's manifest is passed too. Both are printed as a note, not as a problem — a check that
|
||||||
// refused what it could not see would teach people to ignore it.
|
// refused what it could not see would teach people to ignore it.
|
||||||
|
//
|
||||||
|
// **And every identity against every bound it meets** (novox/hq ADR 0225, issue 263): each module's
|
||||||
|
// identity, on a machine whose name is `longestMachine` characters, against the bound of every
|
||||||
|
// provision it wants that a manifest given here offers. An overflow is refused in the pull request
|
||||||
|
// that introduces it — a new requirement, a lowered bound, a longer slug — instead of on the
|
||||||
|
// provider's machine when a real machine's name first meets the module's.
|
||||||
func moduleCheck(paths []string, out io.Writer) error {
|
func moduleCheck(paths []string, out io.Writer) error {
|
||||||
|
return moduleCheckFor(paths, catalogue.DefaultLongestMachine, out)
|
||||||
|
}
|
||||||
|
|
||||||
|
func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||||
if len(paths) == 0 {
|
if len(paths) == 0 {
|
||||||
return errors.New("module check <manifest.json>... — one file per module; pass every " +
|
return errors.New("module check <manifest.json>... — one file per module; pass every " +
|
||||||
"manifest of a repository together so the rules between them are checked too")
|
"manifest of a repository together so the rules between them are checked too")
|
||||||
@@ -74,6 +84,15 @@ func moduleCheck(paths []string, out io.Writer) error {
|
|||||||
}
|
}
|
||||||
failed += len(problems)
|
failed += len(problems)
|
||||||
|
|
||||||
|
// Between the manifests too: an identity against the bounds of the provisions it wants, which
|
||||||
|
// only the provider's manifest states.
|
||||||
|
identities := catalogue.IdentityProblems(shelf, longestMachine)
|
||||||
|
sort.Strings(identities)
|
||||||
|
for _, p := range identities {
|
||||||
|
fmt.Fprintln(out, p)
|
||||||
|
}
|
||||||
|
failed += len(identities)
|
||||||
|
|
||||||
var names []string
|
var names []string
|
||||||
for name := range shelf {
|
for name := range shelf {
|
||||||
names = append(names, name)
|
names = append(names, name)
|
||||||
@@ -109,7 +128,8 @@ func moduleCheck(paths []string, out io.Writer) error {
|
|||||||
}
|
}
|
||||||
fmt.Fprintf(out, "%d manifest(s) checked. Judged against the seats this binary carries; a claim on "+
|
fmt.Fprintf(out, "%d manifest(s) checked. Judged against the seats this binary carries; a claim on "+
|
||||||
"one of the mesh's own seats is judged fully at registration, and a seat declared by a "+
|
"one of the mesh's own seats is judged fully at registration, and a seat declared by a "+
|
||||||
"module not given here reads as unknown\n", len(paths))
|
"module not given here reads as unknown. Identities judged on a %d-character machine name, "+
|
||||||
|
"against the bounds of the providers given here\n", len(paths), longestMachine)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,431 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"slices"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What is wrong, kept until observation says it is not (novox/hq to-be 45 §2, ADR 0227).
|
||||||
|
//
|
||||||
|
// **`status` used to be the only place the mesh said it was wrong, and only to whoever asked.** Every
|
||||||
|
// core failure of research 031 was found by a person looking. A condition is the mesh saying it: raised
|
||||||
|
// by a watchdog when a signal is late (signals.go), by a probe when an invariant does not hold
|
||||||
|
// (doctor.go), or by an event a provider sends (standing.go); kept on the bus with since-when and
|
||||||
|
// evidence; said on the bus as it changes, for the operator's channel to carry; and cleared when an
|
||||||
|
// observation says it is resolved. Nobody resolves one by hand. A person who knows silences it, for a
|
||||||
|
// while, with a reason, and that is recorded as a hand act.
|
||||||
|
|
||||||
|
// conditionsFrom is the serving controller's keeper; nil in any other process, which opens its own.
|
||||||
|
var conditionsFrom *conditions.Keeper
|
||||||
|
|
||||||
|
// keeperOn is a keeper over the store on a connection, saying its transitions on that connection.
|
||||||
|
func keeperOn(ctx context.Context, conn *nats.Conn) (*conditions.Keeper, error) {
|
||||||
|
store, history, err := conditions.OnTheBus(ctx, conn)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
js, err := conn.JetStream()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history,
|
||||||
|
Teller: link.OverNATS{Conn: conn, JS: js},
|
||||||
|
Say: func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) },
|
||||||
|
// What status leads with changed: composed again soon (a nudge outside the serving controller
|
||||||
|
// does nothing).
|
||||||
|
Changed: statusFrom.nudge}), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// withKeeper runs f with the serving controller's keeper, or one of its own that says everything
|
||||||
|
// it was given before it returns.
|
||||||
|
func withKeeper(ctx context.Context, f func(*conditions.Keeper) error) error {
|
||||||
|
if conditionsFrom != nil {
|
||||||
|
return f(conditionsFrom)
|
||||||
|
}
|
||||||
|
return onTheBus(func(conn *nats.Conn) error {
|
||||||
|
k, err := keeperOn(ctx, conn)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
flushing, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
k.Close(flushing)
|
||||||
|
}()
|
||||||
|
return f(k)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// openConditions is every open condition, for `status` and `node show`: from the serving keeper, or
|
||||||
|
// read from the bus. Where there is no bus to read it from, it says so — never "none open".
|
||||||
|
func openConditions(ctx context.Context) ([]conditions.Condition, error) {
|
||||||
|
if conditionsFrom != nil {
|
||||||
|
return conditionsFrom.Open(ctx)
|
||||||
|
}
|
||||||
|
if _, err := broker.BusAddress(); err != nil {
|
||||||
|
return nil, fmt.Errorf("this process has no bus to read the conditions from: %w", err)
|
||||||
|
}
|
||||||
|
var out []conditions.Condition
|
||||||
|
err := onTheBus(func(conn *nats.Conn) error {
|
||||||
|
store, _, err := conditions.OnTheBus(ctx, conn)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
reading, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
out, err = conditions.Read(reading, store)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// conditionsUsage is how the verb is typed.
|
||||||
|
const conditionsUsage = "conditions [--scope S] [--severity urgent|warning] [--machine M] [--json] | " +
|
||||||
|
"conditions show <key> | conditions silence <key> --for <duration> --why <text> | " +
|
||||||
|
"conditions history [--days N] [--key K] [--json]"
|
||||||
|
|
||||||
|
// conditionsCommand is `conditions`, `conditions show`, `conditions silence` and `conditions history`.
|
||||||
|
func conditionsCommand(ctx context.Context, args []string) error {
|
||||||
|
sub := "list"
|
||||||
|
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||||
|
sub, args = args[0], args[1:]
|
||||||
|
}
|
||||||
|
switch sub {
|
||||||
|
case "list":
|
||||||
|
return listConditions(ctx, args)
|
||||||
|
case "show":
|
||||||
|
return showCondition(ctx, args)
|
||||||
|
case "silence":
|
||||||
|
return silenceCondition(ctx, args)
|
||||||
|
case "history":
|
||||||
|
return conditionHistory(ctx, args)
|
||||||
|
}
|
||||||
|
return errors.New(conditionsUsage)
|
||||||
|
}
|
||||||
|
|
||||||
|
func listConditions(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("conditions", flag.ContinueOnError)
|
||||||
|
scope := set.String("scope", "", "only this scope: "+strings.Join(conditions.Scopes, ", "))
|
||||||
|
severity := set.String("severity", "", "only urgent, or only warning")
|
||||||
|
machine := set.String("machine", "", "only those about this machine")
|
||||||
|
asJSON := set.Bool("json", false, "as data")
|
||||||
|
if rest, err := parseAround(set, args); err != nil {
|
||||||
|
return err
|
||||||
|
} else if len(rest) > 0 {
|
||||||
|
return errors.New(conditionsUsage)
|
||||||
|
}
|
||||||
|
if *severity != "" && *severity != string(conditions.Urgent) && *severity != string(conditions.Warning) {
|
||||||
|
return fmt.Errorf("a severity is urgent or warning, not %q", *severity)
|
||||||
|
}
|
||||||
|
open, err := openConditions(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var out []conditions.Condition
|
||||||
|
for _, c := range open {
|
||||||
|
if (*scope == "" || c.Subject.Scope == *scope) && (*severity == "" || string(c.Severity) == *severity) &&
|
||||||
|
(*machine == "" || concerns(c, *machine)) {
|
||||||
|
out = append(out, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if *asJSON {
|
||||||
|
if out == nil {
|
||||||
|
out = []conditions.Condition{}
|
||||||
|
}
|
||||||
|
return printJSON(map[string]any{"conditions": out, "open": len(open),
|
||||||
|
"note": "urgent first, then oldest first; a condition clears when observation says so, never by hand"})
|
||||||
|
}
|
||||||
|
if len(out) == 0 {
|
||||||
|
if len(open) == 0 {
|
||||||
|
fmt.Println("no open conditions")
|
||||||
|
} else {
|
||||||
|
fmt.Printf("none of the %d open condition(s) is about that\n", len(open))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, line := range conditionLines(out, time.Now()) {
|
||||||
|
fmt.Println(line)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// concerns says whether a condition is about a machine: it names it, or its key does.
|
||||||
|
func concerns(c conditions.Condition, machine string) bool {
|
||||||
|
if c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
for _, part := range strings.Split(c.Key, ".") {
|
||||||
|
if part == machine {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// conditionLines is how a list of conditions reads: one line each, its silence under it.
|
||||||
|
func conditionLines(list []conditions.Condition, now time.Time) []string {
|
||||||
|
var out []string
|
||||||
|
for _, c := range list {
|
||||||
|
times := ""
|
||||||
|
if c.Count > 1 {
|
||||||
|
times = fmt.Sprintf(", raised %d times", c.Count)
|
||||||
|
}
|
||||||
|
out = append(out, fmt.Sprintf(" %-7s %s — %s (since %s%s)", strings.ToUpper(string(c.Severity)),
|
||||||
|
c.Key, c.Summary, c.Raised.Local().Format("2006-01-02 15:04"), times))
|
||||||
|
if c.SilencedAt(now) {
|
||||||
|
out = append(out, fmt.Sprintf(" silenced until %s by %s: %s",
|
||||||
|
c.Silenced.Until.Local().Format("2006-01-02 15:04"), c.Silenced.By, c.Silenced.Why))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func showCondition(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("conditions show", flag.ContinueOnError)
|
||||||
|
asJSON := set.Bool("json", false, "as data")
|
||||||
|
rest, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(rest) != 1 {
|
||||||
|
return errors.New("conditions show <key>")
|
||||||
|
}
|
||||||
|
key := rest[0]
|
||||||
|
var c conditions.Condition
|
||||||
|
var found bool
|
||||||
|
if conditionsFrom != nil {
|
||||||
|
c, found, err = conditionsFrom.Get(ctx, key)
|
||||||
|
} else {
|
||||||
|
err = onTheBus(func(conn *nats.Conn) error {
|
||||||
|
store, _, err := conditions.OnTheBus(ctx, conn)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
c, found, err = conditions.ReadOne(ctx, store, key)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
return fmt.Errorf("no condition %s is open — `conditions` lists those that are, and `conditions "+
|
||||||
|
"history --key %s` what became of it", key, key)
|
||||||
|
}
|
||||||
|
if *asJSON {
|
||||||
|
return printJSON(c)
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
fmt.Printf("%s %s\n %s\n\n", strings.ToUpper(string(c.Severity)), c.Key, c.Summary)
|
||||||
|
fmt.Printf(" kind %s\n about %s %s", c.Kind, c.Subject.Scope, c.Subject.ID)
|
||||||
|
if c.Subject.Machine != "" {
|
||||||
|
fmt.Printf(", on %s", c.Subject.Machine)
|
||||||
|
}
|
||||||
|
fmt.Printf("\n raised by %s\n since %s (%s ago), observed %d time(s), last %s ago\n",
|
||||||
|
c.Source, c.Raised.Local().Format("2006-01-02 15:04:05"), roughly(now.Sub(c.Raised)), c.Observations,
|
||||||
|
now.Sub(c.LastObserved).Round(time.Second))
|
||||||
|
if c.Count > 1 {
|
||||||
|
fmt.Printf(" raised %d times, each within ten minutes of clearing\n", c.Count)
|
||||||
|
}
|
||||||
|
fmt.Printf(" resolved by %s\n", resolverWords(c.Resolver))
|
||||||
|
if c.Silenced != nil {
|
||||||
|
fmt.Printf(" silenced until %s by %s: %s\n", c.Silenced.Until.Local().Format("2006-01-02 15:04"),
|
||||||
|
c.Silenced.By, c.Silenced.Why)
|
||||||
|
}
|
||||||
|
if len(c.Tried) > 0 {
|
||||||
|
fmt.Println("\n tried:")
|
||||||
|
for _, t := range c.Tried {
|
||||||
|
fmt.Printf(" %s %s: %s\n", t.At.Local().Format("2006-01-02 15:04"), t.What, t.Outcome)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Println("\n evidence, newest first:")
|
||||||
|
for _, e := range c.Evidence {
|
||||||
|
fmt.Printf(" %s %s\n", e.At.Local().Format("2006-01-02 15:04:05"), e.Said)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolverWords(r string) string {
|
||||||
|
switch r {
|
||||||
|
case conditions.ResolverSelf:
|
||||||
|
return "itself: it clears when observation says it is resolved"
|
||||||
|
case conditions.ResolverOperator:
|
||||||
|
return "the operator: nothing in the mesh will repair it"
|
||||||
|
case conditions.ResolverAgent:
|
||||||
|
return "an agent"
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// silenceCondition stops a condition's messages for a while (to-be 45 §2). A hand act: recorded with
|
||||||
|
// who and why before it is done, its cause the condition's kind unless one is given.
|
||||||
|
func silenceCondition(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("conditions silence", flag.ContinueOnError)
|
||||||
|
forFlag := set.String("for", "", "how long: 30m, 4h, 2d — at most 7d")
|
||||||
|
acts := addHandActFlags(set)
|
||||||
|
rest, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(rest) != 1 {
|
||||||
|
return errors.New("conditions silence <key> --for <duration> --why <text>")
|
||||||
|
}
|
||||||
|
key := rest[0]
|
||||||
|
if err := acts.require("conditions silence"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
d, err := parseFor(*forFlag)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if d > conditions.MaxSilence {
|
||||||
|
return fmt.Errorf("a condition is silenced for at most %s at once; past it, say so again", conditions.MaxSilence)
|
||||||
|
}
|
||||||
|
return withKeeper(ctx, func(k *conditions.Keeper) error {
|
||||||
|
c, found, err := k.Get(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
return fmt.Errorf("no condition %s is open — `conditions` lists them. Nothing was silenced", key)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(*acts.cause) == "" {
|
||||||
|
*acts.cause = c.Kind
|
||||||
|
}
|
||||||
|
*acts.condition = key
|
||||||
|
acts.record(ctx, "conditions silence", []string{key, "--for", *forFlag})
|
||||||
|
held, err := k.Silence(ctx, key, d, link.Caller(), *acts.why)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s is silenced until %s: no message is sent for it until then. It is still open, and "+
|
||||||
|
"`status` still says it; it clears when observation says it is resolved\n",
|
||||||
|
held.Key, held.Silenced.Until.Local().Format("2006-01-02 15:04"))
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseFor reads a duration, days included.
|
||||||
|
func parseFor(s string) (time.Duration, error) {
|
||||||
|
s = strings.TrimSpace(s)
|
||||||
|
if s == "" {
|
||||||
|
return 0, errors.New("say for how long: --for 30m, 4h or 2d")
|
||||||
|
}
|
||||||
|
if days, ok := strings.CutSuffix(s, "d"); ok {
|
||||||
|
n, err := strconv.Atoi(days)
|
||||||
|
if err != nil || n <= 0 {
|
||||||
|
return 0, fmt.Errorf("%q is not a number of days", s)
|
||||||
|
}
|
||||||
|
return time.Duration(n) * 24 * time.Hour, nil
|
||||||
|
}
|
||||||
|
d, err := time.ParseDuration(s)
|
||||||
|
if err != nil || d <= 0 {
|
||||||
|
return 0, fmt.Errorf("%q is not a duration: 30m, 4h or 2d", s)
|
||||||
|
}
|
||||||
|
return d, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func conditionHistory(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("conditions history", flag.ContinueOnError)
|
||||||
|
days := set.Int("days", 7, "how many days back, at most 90")
|
||||||
|
key := set.String("key", "", "only this condition")
|
||||||
|
asJSON := set.Bool("json", false, "as data")
|
||||||
|
if rest, err := parseAround(set, args); err != nil {
|
||||||
|
return err
|
||||||
|
} else if len(rest) > 0 {
|
||||||
|
return errors.New("conditions history [--days N] [--key K] [--json]")
|
||||||
|
}
|
||||||
|
since := time.Now().Add(-time.Duration(*days) * 24 * time.Hour)
|
||||||
|
var events []conditions.Event
|
||||||
|
read := func(h conditions.History) error {
|
||||||
|
var err error
|
||||||
|
events, err = h.Since(ctx, since)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var err error
|
||||||
|
if conditionsFrom != nil {
|
||||||
|
events, err = conditionsFrom.HistorySince(ctx, since)
|
||||||
|
} else {
|
||||||
|
err = onTheBus(func(conn *nats.Conn) error {
|
||||||
|
_, history, err := conditions.OnTheBus(ctx, conn)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return read(history)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var out []conditions.Event
|
||||||
|
for _, e := range events {
|
||||||
|
if *key == "" || e.Key == *key {
|
||||||
|
out = append(out, e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if *asJSON {
|
||||||
|
if out == nil {
|
||||||
|
out = []conditions.Event{}
|
||||||
|
}
|
||||||
|
return printJSON(map[string]any{"history": out, "days": *days})
|
||||||
|
}
|
||||||
|
if len(out) == 0 {
|
||||||
|
fmt.Printf("nothing was raised, changed or cleared in the last %d day(s)\n", *days)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, e := range out {
|
||||||
|
line := fmt.Sprintf("%s %-13s %s", e.At.Local().Format("2006-01-02 15:04:05"), e.Change, e.Key)
|
||||||
|
switch e.Change {
|
||||||
|
case conditions.ChangeRaised, conditions.ChangeReopened:
|
||||||
|
line += " — " + e.Summary
|
||||||
|
case conditions.ChangeSeverity:
|
||||||
|
line += fmt.Sprintf(" — %s, was %s", e.Severity, e.Was)
|
||||||
|
case conditions.ChangeResolver:
|
||||||
|
line += fmt.Sprintf(" — %s, was %s", e.Resolver, e.Was)
|
||||||
|
default:
|
||||||
|
if e.Why != "" {
|
||||||
|
line += " — " + e.Why
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Println(line)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// printConditions is the status section that leads it: every open condition, urgent first, oldest
|
||||||
|
// first, silenced ones with their expiry (to-be 45 §2). A store that could not be read is said, and
|
||||||
|
// is not "none open".
|
||||||
|
func printConditions(list []conditions.Condition, unread string, now time.Time) {
|
||||||
|
if unread != "" {
|
||||||
|
fmt.Printf("the open conditions could NOT be read, so whether anything is wrong is not known: %s\n\n", unread)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if len(list) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
urgent := 0
|
||||||
|
for _, c := range list {
|
||||||
|
if c.Severity == conditions.Urgent {
|
||||||
|
urgent++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Printf("%d open condition(s), %d urgent:\n\n", len(list), urgent)
|
||||||
|
for _, line := range conditionLines(list, now) {
|
||||||
|
fmt.Println(line)
|
||||||
|
}
|
||||||
|
fmt.Printf("\n `conditions show <key>` says more; each clears when observation says it is resolved, " +
|
||||||
|
"never by hand — `conditions silence <key> --for <d> --why <text>` stops its messages\n\n")
|
||||||
|
}
|
||||||
@@ -0,0 +1,107 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The verbs of the condition store (novox/hq to-be 45 §2) as the console reaches them, and status led
|
||||||
|
// by what is open.
|
||||||
|
|
||||||
|
func TestTheConditionsVerbComposesEachShape(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
args map[string]any
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{map[string]any{}, "conditions --json"},
|
||||||
|
{map[string]any{"severity": "urgent", "machine": "ace"}, "conditions --json --severity urgent --machine ace"},
|
||||||
|
{map[string]any{"key": "machine.ace.silent"}, "conditions show machine.ace.silent --json"},
|
||||||
|
{map[string]any{"history": "true", "days": "3", "key": "machine.ace.silent"},
|
||||||
|
"conditions history --json --days 3 --key machine.ace.silent"},
|
||||||
|
{map[string]any{"silence": "machine.ace.silent", "for": "2h", "why": "on the train"},
|
||||||
|
"conditions silence machine.ace.silent --for 2h --why on the train"},
|
||||||
|
{map[string]any{"run": "true"}, "doctor run --json"},
|
||||||
|
{map[string]any{}, "doctor --json"},
|
||||||
|
} {
|
||||||
|
verb := "conditions"
|
||||||
|
if strings.HasPrefix(c.want, "doctor") {
|
||||||
|
verb = "doctor"
|
||||||
|
}
|
||||||
|
argv, err := argvFor(verb, c.args)
|
||||||
|
if err != nil || strings.Join(argv, " ") != c.want {
|
||||||
|
t.Errorf("%s %v composed %q (%v), want %q", verb, c.args, strings.Join(argv, " "), err, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, c := range []struct {
|
||||||
|
verb string
|
||||||
|
args map[string]any
|
||||||
|
}{
|
||||||
|
{"conditions", map[string]any{"silence": "machine.ace.silent", "why": "x"}}, // no for
|
||||||
|
{"doctor", map[string]any{"run": "true", "signals": "true"}}, // two at once
|
||||||
|
{"conditions", map[string]any{"silence": "machine.ace.silent", "for": "1h", "why": "x", "days": "3"}}, // passed over
|
||||||
|
} {
|
||||||
|
if argv, err := argvFor(c.verb, c.args); err == nil {
|
||||||
|
t.Errorf("%s %v composed %v", c.verb, c.args, argv)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if repairingCommand([]string{"conditions", "silence", "k"}) != "conditions silence" {
|
||||||
|
t.Error("a silence is not a hand act")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A silence through the verb is recorded and bounded**; the condition stays open.
|
||||||
|
func TestASilenceThroughTheVerbHoldsAndTheConditionStaysOpen(t *testing.T) {
|
||||||
|
k, _ := withConditionsInMemory(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if _, err := k.Observe(ctx, conditions.Observation{Scope: conditions.ScopeMachine, ID: "ace", Kind: "silent",
|
||||||
|
Severity: conditions.Warning, Summary: "ace is silent", Source: "S1"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := conditionsCommand(ctx, []string{"silence", "machine.ace.silent", "--for", "2d"}); err == nil {
|
||||||
|
t.Fatal("silenced without saying why")
|
||||||
|
}
|
||||||
|
if err := conditionsCommand(ctx, []string{"silence", "machine.ace.silent", "--for", "8d", "--why", "x"}); err == nil {
|
||||||
|
t.Fatal("silenced for more than a week")
|
||||||
|
}
|
||||||
|
said := printed(t, func() error {
|
||||||
|
return conditionsCommand(ctx, []string{"silence", "machine.ace.silent", "--for", "2d", "--why", "on the train"})
|
||||||
|
})
|
||||||
|
if !strings.Contains(said, "is silenced until") {
|
||||||
|
t.Fatalf("%s", said)
|
||||||
|
}
|
||||||
|
c, found, _ := k.Get(ctx, "machine.ace.silent")
|
||||||
|
if !found || c.Silenced == nil || c.Silenced.Why != "on the train" {
|
||||||
|
t.Fatalf("%+v", c)
|
||||||
|
}
|
||||||
|
listed := printed(t, func() error { return conditionsCommand(ctx, nil) })
|
||||||
|
if !strings.Contains(listed, "machine.ace.silent") || !strings.Contains(listed, "silenced until") {
|
||||||
|
t.Fatalf("%s", listed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Conditions that cannot be read are not none open**: status says so, and is not well.
|
||||||
|
func TestUnreadableConditionsAreNotAWellMesh(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
_, store := withConditionsInMemory(t)
|
||||||
|
store.Fail = errors.New("the bus is away")
|
||||||
|
asked, err := theThreeQuestions(t.Context(), open)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if asked.well() || asked.conditionsUnread == "" {
|
||||||
|
t.Fatalf("well with its conditions unread: %+v", asked.conditionsUnread)
|
||||||
|
}
|
||||||
|
said := printed(t, func() error { return printStatus(asked) })
|
||||||
|
if !strings.HasPrefix(said, "the open conditions could NOT be read") || strings.Contains(said, "no open conditions") {
|
||||||
|
t.Fatalf("%s", said)
|
||||||
|
}
|
||||||
|
store.Fail = nil
|
||||||
|
asked, _ = theThreeQuestions(t.Context(), open)
|
||||||
|
said = printed(t, func() error { return printStatus(asked) })
|
||||||
|
if asked.well() && !strings.Contains(said, "no open conditions;") {
|
||||||
|
t.Fatalf("the all-well sentence does not say no conditions are open:\n%s", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,536 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The self-check: `doctor` (novox/hq to-be 45 §4, ADR 0227 rule 6).
|
||||||
|
//
|
||||||
|
// **The design's invariants, run against the running mesh.** A probe is one live invariant of a
|
||||||
|
// design — every machine's declaration composes and validates, every resolver answers, every seat's
|
||||||
|
// holder answers, every stream and consumer is there as defined — with an id, a bound, and the
|
||||||
|
// condition it raises when the invariant does not hold. The serving controller runs the registry every
|
||||||
|
// five minutes, each probe given thirty seconds; a probe that errors or does not finish raises
|
||||||
|
// `probe-failed` for itself, because an unanswered probe is never a pass. Every run ends with a
|
||||||
|
// heartbeat on the bus (`doctor-heartbeat`, S10), which mesh-watcher listens for from a second
|
||||||
|
// machine: a controller that stops checking is itself said, through a channel that does not pass
|
||||||
|
// through it.
|
||||||
|
//
|
||||||
|
// `doctor` answers the last run's verdict at once; `doctor run` runs now; `doctor probes` lists the
|
||||||
|
// registry; `doctor signals` says, for every row of the signals table, the age of its newest signal.
|
||||||
|
|
||||||
|
// The self-check's clocks.
|
||||||
|
var (
|
||||||
|
// doctorEvery is how often the registry runs; doctorFirstAfter how long after the controller
|
||||||
|
// starts the first run waits, so what the controller hears at its start has arrived.
|
||||||
|
doctorEvery = 5 * time.Minute
|
||||||
|
doctorFirstAfter = time.Minute
|
||||||
|
// probeWithin is each probe's bound.
|
||||||
|
probeWithin = 30 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
// The verdicts a probe can have.
|
||||||
|
const (
|
||||||
|
verdictPass = "pass"
|
||||||
|
verdictFail = "fail"
|
||||||
|
verdictFailedToRun = "failed-to-run"
|
||||||
|
verdictDeferred = "deferred"
|
||||||
|
)
|
||||||
|
|
||||||
|
// probe is one live invariant.
|
||||||
|
type probe struct {
|
||||||
|
ID string
|
||||||
|
Asserts string
|
||||||
|
From string
|
||||||
|
// Kind is the condition kind raised when the invariant does not hold.
|
||||||
|
Kind string
|
||||||
|
Phase int
|
||||||
|
// Deferred says why it is not run yet; empty for one that is.
|
||||||
|
Deferred string
|
||||||
|
// Asks are the seat verbs it calls. A probe may call no other (askSeatTool refuses), and the
|
||||||
|
// controller's grant names every one (a test over this registry): a probe whose question the bus
|
||||||
|
// refuses checks nothing (D8, 2026-10-06).
|
||||||
|
Asks []broker.SeatVerb
|
||||||
|
run func(ctx context.Context, d *doctor) ([]conditions.Observation, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeRegistry is the registry, in to-be 45's order. **The registry is the design's live form**: a
|
||||||
|
// probe added to a design is a row added here.
|
||||||
|
var probeRegistry = []probe{
|
||||||
|
{ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation",
|
||||||
|
From: "issues 236, 263", Kind: "declaration-refused", Phase: 1, run: probeDeclarations},
|
||||||
|
{ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6",
|
||||||
|
From: "issue 262", Kind: "resolver-wrong", Phase: 1, run: probeResolvers},
|
||||||
|
{ID: "D3", Asserts: "every seat on record that serves verbs has a live holder that answers, on every " +
|
||||||
|
"machine that is heard from", From: "issues 208, 218", Kind: "holder-silent", Phase: 1, run: probeHolders},
|
||||||
|
{ID: "D4", Asserts: "every kept archive is held by a manifest", From: "issue 253",
|
||||||
|
Kind: "archives-unheld", Phase: 1, run: probeArchives},
|
||||||
|
{ID: "D5", Asserts: "exactly one lease holder; no message from a stale epoch in the last interval",
|
||||||
|
From: "issue 204", Kind: "lease-split", Phase: 2,
|
||||||
|
Deferred: "the lease and epoch are built in Phase 2 (to-be 45 §6): nothing holds one yet"},
|
||||||
|
{ID: "D6", Asserts: "every durable consumer the mesh expects exists with its definition, and is near its " +
|
||||||
|
"stream's head", From: "issues 248, 266", Kind: "consumer-wrong", Phase: 1, run: probeConsumers},
|
||||||
|
{ID: "D7", Asserts: "every stream the controller defines exists with its definition, and its own buckets",
|
||||||
|
From: "issue 208", Kind: "stream-wrong", Phase: 1, run: probeStreams},
|
||||||
|
{ID: "D8", Asserts: "no address the mesh owns — a machine's private address or its endpoint — is in a ban list",
|
||||||
|
From: "issue 238", Kind: "own-address-banned", Phase: 1, run: probeBans,
|
||||||
|
Asks: []broker.SeatVerb{{Seat: "node-intrusion-prevention", Verb: "banned"}}},
|
||||||
|
{ID: "D9", Asserts: "status answers in full within ten seconds, from a summary composed lately",
|
||||||
|
From: "issue 265", Kind: "status-slow", Phase: 1, run: probeStatus},
|
||||||
|
{ID: "D10", Asserts: "every machine runs the node-engine and node tools builds the mesh holds, or is inside " +
|
||||||
|
"a plan's window", From: "the version split", Kind: "core-behind", Phase: 1, run: probeCoreBuilds},
|
||||||
|
{ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals",
|
||||||
|
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs},
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeVerdict is one probe's outcome in a run.
|
||||||
|
type probeVerdict struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Verdict string `json:"verdict"`
|
||||||
|
Found []string `json:"found,omitempty"`
|
||||||
|
Error string `json:"error,omitempty"`
|
||||||
|
Took string `json:"took,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// doctorCounts are a run's verdicts, counted.
|
||||||
|
type doctorCounts struct {
|
||||||
|
Passed int `json:"passed"`
|
||||||
|
Failed int `json:"failed"`
|
||||||
|
FailedToRun int `json:"failed-to-run"`
|
||||||
|
Deferred int `json:"deferred"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// doctorRun is one run of the registry, and the body of its heartbeat.
|
||||||
|
type doctorRun struct {
|
||||||
|
Run string `json:"run"`
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
Started time.Time `json:"started"`
|
||||||
|
Took string `json:"took"`
|
||||||
|
IntervalSeconds int `json:"interval-seconds"`
|
||||||
|
Counts doctorCounts `json:"counts"`
|
||||||
|
Probes []probeVerdict `json:"probes"`
|
||||||
|
// Controller is the machine that ran it, and Why what started it: the schedule, or a person.
|
||||||
|
Controller string `json:"controller"`
|
||||||
|
Why string `json:"why"`
|
||||||
|
// Unsaid is how many condition transitions this controller could not say, since it started.
|
||||||
|
Unsaid int `json:"unsaid,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// doctor is the registry and what its probes need.
|
||||||
|
type doctor struct {
|
||||||
|
open *stores
|
||||||
|
js *broker.JetStream
|
||||||
|
keeper *conditions.Keeper
|
||||||
|
teller conditions.Teller
|
||||||
|
watchdogs *watchdogs
|
||||||
|
host string
|
||||||
|
|
||||||
|
running sync.Mutex
|
||||||
|
mu sync.Mutex
|
||||||
|
last *doctorRun
|
||||||
|
ended time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// lastRunEnded is when the last run ended; zero before the first.
|
||||||
|
func (d *doctor) lastRunEnded() time.Time {
|
||||||
|
d.mu.Lock()
|
||||||
|
defer d.mu.Unlock()
|
||||||
|
return d.ended
|
||||||
|
}
|
||||||
|
|
||||||
|
// lastRun is the last run's verdict; nil before the first.
|
||||||
|
func (d *doctor) lastRun() *doctorRun {
|
||||||
|
d.mu.Lock()
|
||||||
|
defer d.mu.Unlock()
|
||||||
|
return d.last
|
||||||
|
}
|
||||||
|
|
||||||
|
// keep runs the registry on its schedule until ctx ends.
|
||||||
|
func (d *doctor) keep(ctx context.Context) {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-time.After(doctorFirstAfter):
|
||||||
|
}
|
||||||
|
tick := time.NewTicker(doctorEvery)
|
||||||
|
defer tick.Stop()
|
||||||
|
for {
|
||||||
|
// Only the controller acting checks the mesh on a schedule: one standing by would say a
|
||||||
|
// heartbeat for a self-check that is not the mesh's.
|
||||||
|
if d.watchdogs == nil || d.watchdogs.acting == nil || d.watchdogs.acting() {
|
||||||
|
d.runOnce(ctx, "the schedule")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-tick.C:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var doctorRuns struct {
|
||||||
|
sync.Mutex
|
||||||
|
n uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
// runOnce runs every probe the registry runs, keeps what each found, and says the heartbeat. One run
|
||||||
|
// at a time: a person's `doctor run` during a scheduled one waits for it.
|
||||||
|
func (d *doctor) runOnce(ctx context.Context, why string) doctorRun {
|
||||||
|
d.running.Lock()
|
||||||
|
defer d.running.Unlock()
|
||||||
|
doctorRuns.Lock()
|
||||||
|
doctorRuns.n++
|
||||||
|
n := doctorRuns.n
|
||||||
|
doctorRuns.Unlock()
|
||||||
|
started := time.Now()
|
||||||
|
run := doctorRun{Run: fmt.Sprintf("doctor-%d-%d", started.Unix(), n), Started: started.UTC(),
|
||||||
|
IntervalSeconds: int(doctorEvery / time.Second), Controller: d.host, Why: why}
|
||||||
|
|
||||||
|
type result struct {
|
||||||
|
obs []conditions.Observation
|
||||||
|
err error
|
||||||
|
took time.Duration
|
||||||
|
}
|
||||||
|
results := make([]result, len(probeRegistry))
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for i, p := range probeRegistry {
|
||||||
|
if p.run == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
wg.Add(1)
|
||||||
|
go func(i int, p probe) {
|
||||||
|
defer wg.Done()
|
||||||
|
probing, cancel := context.WithTimeout(context.WithValue(ctx, probeAsksKey{}, p), probeWithin)
|
||||||
|
defer cancel()
|
||||||
|
began := time.Now()
|
||||||
|
done := make(chan result, 1)
|
||||||
|
go func() {
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
done <- result{err: fmt.Errorf("the probe panicked: %v", r)}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
obs, err := p.run(probing, d)
|
||||||
|
done <- result{obs: obs, err: err}
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case r := <-done:
|
||||||
|
r.took = time.Since(began)
|
||||||
|
results[i] = r
|
||||||
|
case <-probing.Done():
|
||||||
|
results[i] = result{err: fmt.Errorf("it did not finish within %s", probeWithin), took: time.Since(began)}
|
||||||
|
}
|
||||||
|
}(i, p)
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
var blind []conditions.Observation
|
||||||
|
for i, p := range probeRegistry {
|
||||||
|
v := probeVerdict{ID: p.ID}
|
||||||
|
r := results[i]
|
||||||
|
switch {
|
||||||
|
case p.run == nil:
|
||||||
|
v.Verdict = verdictDeferred
|
||||||
|
run.Counts.Deferred++
|
||||||
|
case r.err != nil:
|
||||||
|
v.Verdict, v.Error = verdictFailedToRun, r.err.Error()
|
||||||
|
run.Counts.FailedToRun++
|
||||||
|
blind = append(blind, conditions.Observation{Scope: conditions.ScopeProbe, ID: p.ID, Kind: "probe-failed",
|
||||||
|
Token: "failed", Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("the probe %s (%s) could not run: what it checks is not known — never a pass", p.ID, p.Asserts),
|
||||||
|
Said: firstLine(r.err.Error())})
|
||||||
|
default:
|
||||||
|
if err := d.keeper.Reconcile(ctx, p.ID, kindedAs(r.obs, p.Kind)); err != nil {
|
||||||
|
v.Error = "what it found could not be kept: " + err.Error()
|
||||||
|
}
|
||||||
|
if len(r.obs) == 0 {
|
||||||
|
v.Verdict = verdictPass
|
||||||
|
run.Counts.Passed++
|
||||||
|
} else {
|
||||||
|
v.Verdict = verdictFail
|
||||||
|
run.Counts.Failed++
|
||||||
|
for _, o := range r.obs {
|
||||||
|
v.Found = append(v.Found, o.Summary)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if p.run != nil {
|
||||||
|
v.Took = r.took.Round(time.Millisecond).String()
|
||||||
|
}
|
||||||
|
run.Probes = append(run.Probes, v)
|
||||||
|
}
|
||||||
|
if err := d.keeper.Reconcile(ctx, sourceDoctor, blind); err != nil {
|
||||||
|
fmt.Printf("the self-check's own failures could not be kept: %v\n", err)
|
||||||
|
}
|
||||||
|
ended := time.Now()
|
||||||
|
run.At, run.Took, run.Unsaid = ended.UTC(), ended.Sub(started).Round(time.Millisecond).String(), d.keeper.Unsaid()
|
||||||
|
d.mu.Lock()
|
||||||
|
d.last, d.ended = &run, ended
|
||||||
|
d.mu.Unlock()
|
||||||
|
d.sayHeartbeat(ctx, run)
|
||||||
|
return run
|
||||||
|
}
|
||||||
|
|
||||||
|
// sourceDoctor is what raises a probe's own failure to run.
|
||||||
|
const sourceDoctor = "doctor"
|
||||||
|
|
||||||
|
// kindedAs gives each observation of a probe the probe's kind where it named none.
|
||||||
|
func kindedAs(obs []conditions.Observation, kind string) []conditions.Observation {
|
||||||
|
out := make([]conditions.Observation, 0, len(obs))
|
||||||
|
for _, o := range obs {
|
||||||
|
if o.Kind == "" {
|
||||||
|
o.Kind = kind
|
||||||
|
}
|
||||||
|
out = append(out, o)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// sayHeartbeat publishes the run's heartbeat. Not said is said here, and S10 on the second machine
|
||||||
|
// says it outward: the watcher hears nothing.
|
||||||
|
func (d *doctor) sayHeartbeat(ctx context.Context, run doctorRun) {
|
||||||
|
if d.teller == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
body, err := json.Marshal(run)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("the self-check's heartbeat could not be written: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
saying, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if err := d.teller.PublishSeatEvent(saying, conditions.Seat, conditions.HeartbeatEvent, body); err != nil {
|
||||||
|
fmt.Printf("the self-check's heartbeat (%s) could NOT be said, so the watcher on the second machine "+
|
||||||
|
"will say the self-check is silent: %v\n", run.Run, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// doctorFrom is the serving controller's self-check; nil in any other process.
|
||||||
|
var doctorFrom *doctor
|
||||||
|
|
||||||
|
// doctorCommand is `doctor`, `doctor run`, `doctor probes` and `doctor signals`.
|
||||||
|
func doctorCommand(ctx context.Context, args []string) error {
|
||||||
|
sub := ""
|
||||||
|
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||||
|
sub, args = args[0], args[1:]
|
||||||
|
}
|
||||||
|
set := flag.NewFlagSet("doctor", flag.ContinueOnError)
|
||||||
|
asJSON := set.Bool("json", false, "as data")
|
||||||
|
if rest, err := parseAround(set, args); err != nil {
|
||||||
|
return err
|
||||||
|
} else if len(rest) > 0 {
|
||||||
|
return errors.New("doctor [run|probes|signals] [--json]")
|
||||||
|
}
|
||||||
|
answer, err := doctorAnswer(ctx, sub)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if *asJSON {
|
||||||
|
return printJSON(answer)
|
||||||
|
}
|
||||||
|
fmt.Print(doctorText(answer))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// doctorAnswer is what the verb answers, as data.
|
||||||
|
func doctorAnswer(ctx context.Context, sub string) (any, error) {
|
||||||
|
switch sub {
|
||||||
|
case "":
|
||||||
|
if doctorFrom != nil {
|
||||||
|
if run := doctorFrom.lastRun(); run != nil {
|
||||||
|
return verdictAnswer(*run, time.Now()), nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("the self-check has not finished its first run yet: it runs %s after the "+
|
||||||
|
"controller starts, then every %s — `doctor run` runs it now", doctorFirstAfter, doctorEvery)
|
||||||
|
}
|
||||||
|
run, err := lastHeartbeat(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return verdictAnswer(run, time.Now()), nil
|
||||||
|
case "run":
|
||||||
|
d := doctorFrom
|
||||||
|
if d == nil {
|
||||||
|
local, closeIt, err := localDoctor(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer closeIt()
|
||||||
|
d = local
|
||||||
|
}
|
||||||
|
return verdictAnswer(d.runOnce(ctx, "asked by "+link.Caller()), time.Now()), nil
|
||||||
|
case "probes":
|
||||||
|
return probesAnswer(), nil
|
||||||
|
case "signals":
|
||||||
|
if doctorFrom == nil || doctorFrom.watchdogs == nil {
|
||||||
|
return nil, errors.New("the age of each signal is known to the serving controller alone, which " +
|
||||||
|
"hears them: ask it through the mesh-controller seat's doctor verb")
|
||||||
|
}
|
||||||
|
return signalsAnswer(doctorFrom.watchdogs.lastFacts(), doctorFrom.watchdogs.lastTick()), nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("doctor answers the last run, or `run`, `probes` or `signals` — not %q", sub)
|
||||||
|
}
|
||||||
|
|
||||||
|
// verdictAnswer is a run as the verb answers it, with its age.
|
||||||
|
func verdictAnswer(run doctorRun, now time.Time) map[string]any {
|
||||||
|
return map[string]any{"run": run, "age": now.Sub(run.At).Round(time.Second).String(),
|
||||||
|
"note": "a probe that could not run is never a pass; each failure is an open condition until a run passes it"}
|
||||||
|
}
|
||||||
|
|
||||||
|
// probesAnswer is the registry.
|
||||||
|
func probesAnswer() map[string]any {
|
||||||
|
var out []map[string]any
|
||||||
|
for _, p := range probeRegistry {
|
||||||
|
row := map[string]any{"id": p.ID, "asserts": p.Asserts, "from": p.From, "kind": p.Kind, "phase": p.Phase}
|
||||||
|
if p.Deferred != "" {
|
||||||
|
row["deferred"] = p.Deferred
|
||||||
|
}
|
||||||
|
out = append(out, row)
|
||||||
|
}
|
||||||
|
return map[string]any{"probes": out, "every": doctorEvery.String(), "each within": probeWithin.String()}
|
||||||
|
}
|
||||||
|
|
||||||
|
// signalsAnswer is every row of the signals table with the age of its newest signal.
|
||||||
|
func signalsAnswer(f *signalFacts, ticked time.Time) map[string]any {
|
||||||
|
var rows []map[string]any
|
||||||
|
for _, r := range signalsTable {
|
||||||
|
row := map[string]any{"row": r.Row, "signal": r.Signal, "emitter": r.Emitter, "bound": r.Bound,
|
||||||
|
"kind": r.Kind, "severity": r.Severity, "phase": r.Phase}
|
||||||
|
switch {
|
||||||
|
case r.Deferred != "":
|
||||||
|
row["deferred"] = r.Deferred
|
||||||
|
case f == nil:
|
||||||
|
row["newest"] = "not yet looked at"
|
||||||
|
default:
|
||||||
|
if err := r.needs(f); err != nil {
|
||||||
|
row["blind"] = err.Error()
|
||||||
|
} else if newest := r.newest(f); newest.IsZero() {
|
||||||
|
row["newest"] = "none heard"
|
||||||
|
} else {
|
||||||
|
row["newest"] = newest.UTC().Format(time.RFC3339)
|
||||||
|
row["age"] = f.now.Sub(newest).Round(time.Second).String()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rows = append(rows, row)
|
||||||
|
}
|
||||||
|
out := map[string]any{"signals": rows, "every": watchEvery.String()}
|
||||||
|
if !ticked.IsZero() {
|
||||||
|
out["looked"] = ticked.UTC().Format(time.RFC3339)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// doctorText is an answer as a person reads it.
|
||||||
|
func doctorText(answer any) string {
|
||||||
|
body, _ := json.Marshal(answer)
|
||||||
|
var b strings.Builder
|
||||||
|
var verdict struct {
|
||||||
|
Run doctorRun `json:"run"`
|
||||||
|
Age string `json:"age"`
|
||||||
|
}
|
||||||
|
if json.Unmarshal(body, &verdict) == nil && verdict.Run.Run != "" {
|
||||||
|
r := verdict.Run
|
||||||
|
fmt.Fprintf(&b, "%s, %s ago (took %s, %s): %d passed, %d failed, %d could not run, %d not built yet\n\n",
|
||||||
|
r.Run, verdict.Age, r.Took, r.Why, r.Counts.Passed, r.Counts.Failed, r.Counts.FailedToRun, r.Counts.Deferred)
|
||||||
|
for _, p := range r.Probes {
|
||||||
|
fmt.Fprintf(&b, " %-4s %-14s %s\n", p.ID, p.Verdict, p.Took)
|
||||||
|
for _, f := range p.Found {
|
||||||
|
fmt.Fprintf(&b, " %s\n", f)
|
||||||
|
}
|
||||||
|
if p.Error != "" {
|
||||||
|
fmt.Fprintf(&b, " %s\n", p.Error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
pretty, _ := json.MarshalIndent(answer, "", " ")
|
||||||
|
return string(pretty) + "\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
// lastHeartbeat is the newest run's heartbeat, read from the events stream: what a process other than
|
||||||
|
// the serving controller answers `doctor` from.
|
||||||
|
func lastHeartbeat(ctx context.Context) (doctorRun, error) {
|
||||||
|
var run doctorRun
|
||||||
|
err := onTheBus(func(conn *nats.Conn) error {
|
||||||
|
js, err := conn.JetStream(nats.Context(ctx))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
msg, err := js.GetLastMsg(broker.EventsStream, link.SeatEventSubject(conditions.Seat, conditions.HeartbeatEvent))
|
||||||
|
if errors.Is(err, nats.ErrMsgNotFound) {
|
||||||
|
return errors.New("the self-check has said no heartbeat on the bus in the last week: it is not running")
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the self-check's last heartbeat cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
return json.Unmarshal(msg.Data, &run)
|
||||||
|
})
|
||||||
|
return run, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// localDoctor is a self-check run by a process other than the serving controller: its own stores,
|
||||||
|
// its own connection, and its own keeper, closed after.
|
||||||
|
func localDoctor(ctx context.Context) (*doctor, func(), error) {
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
js, err := dialTheBus()
|
||||||
|
if err != nil {
|
||||||
|
open.Close()
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
k, err := keeperOn(ctx, js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
js.Close()
|
||||||
|
open.Close()
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
jsCtx := js.Context()
|
||||||
|
d := &doctor{open: open, js: js, keeper: k, teller: link.OverNATS{Conn: js.Conn(), JS: jsCtx},
|
||||||
|
host: controlHost(ctx, open.inventory)}
|
||||||
|
return d, func() {
|
||||||
|
flushing, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
k.Close(flushing)
|
||||||
|
js.Close()
|
||||||
|
open.Close()
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// sortedFound is a probe's findings in a stated order, so two runs over one mesh say the same.
|
||||||
|
func sortedFound(obs []conditions.Observation) []conditions.Observation {
|
||||||
|
sort.Slice(obs, func(i, j int) bool { return obs[i].Key() < obs[j].Key() })
|
||||||
|
return obs
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeAsksKey carries the running probe, so a seat verb it calls is checked against what it declares.
|
||||||
|
type probeAsksKey struct{}
|
||||||
|
|
||||||
|
// declaredBy says whether the probe running in ctx declared a seat verb; outside a probe, false.
|
||||||
|
func declaredBy(ctx context.Context, seat, verb string) (string, bool) {
|
||||||
|
p, ok := ctx.Value(probeAsksKey{}).(probe)
|
||||||
|
if !ok {
|
||||||
|
return "a caller outside the self-check", false
|
||||||
|
}
|
||||||
|
for _, v := range p.Asks {
|
||||||
|
if v.Seat == seat && v.Verb == verb {
|
||||||
|
return p.ID, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return p.ID, false
|
||||||
|
}
|
||||||
@@ -0,0 +1,425 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
"golang.org/x/net/dns/dnsmessage"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The self-check (novox/hq to-be 45 §4): a probe that fails raises its condition, one that cannot run
|
||||||
|
// raises probe-failed for itself and is never a pass, and every run ends with its heartbeat.
|
||||||
|
|
||||||
|
// withProbes runs the test with a registry of its own.
|
||||||
|
func withProbes(t *testing.T, probes ...probe) {
|
||||||
|
t.Helper()
|
||||||
|
before := probeRegistry
|
||||||
|
probeRegistry = probes
|
||||||
|
t.Cleanup(func() { probeRegistry = before })
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestARunKeepsWhatEachProbeFoundAndSaysItsHeartbeat(t *testing.T) {
|
||||||
|
failing := conditions.Observation{Scope: conditions.ScopeMachine, ID: "anchor", Token: "refused",
|
||||||
|
Severity: conditions.Urgent, Summary: "anchor's node-engine would refuse its declaration"}
|
||||||
|
var broken atomic.Bool
|
||||||
|
broken.Store(true)
|
||||||
|
withProbes(t,
|
||||||
|
probe{ID: "P1", Asserts: "passes", Kind: "never", Phase: 1,
|
||||||
|
run: func(context.Context, *doctor) ([]conditions.Observation, error) { return nil, nil }},
|
||||||
|
probe{ID: "P2", Asserts: "finds a fault", Kind: "declaration-refused", Phase: 1,
|
||||||
|
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
|
||||||
|
if broken.Load() {
|
||||||
|
return []conditions.Observation{failing}, nil
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}},
|
||||||
|
probe{ID: "P3", Asserts: "cannot run", Kind: "x", Phase: 1,
|
||||||
|
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
|
||||||
|
if broken.Load() {
|
||||||
|
return nil, errors.New("the store is away")
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}},
|
||||||
|
probe{ID: "P4", Asserts: "hangs", Kind: "x", Phase: 1,
|
||||||
|
run: func(ctx context.Context, _ *doctor) ([]conditions.Observation, error) {
|
||||||
|
if broken.Load() {
|
||||||
|
<-ctx.Done()
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}},
|
||||||
|
probe{ID: "P5", Asserts: "later", Kind: "x", Phase: 2, Deferred: "not yet"},
|
||||||
|
)
|
||||||
|
before := probeWithin
|
||||||
|
probeWithin = 200 * time.Millisecond
|
||||||
|
t.Cleanup(func() { probeWithin = before })
|
||||||
|
|
||||||
|
store := conditions.NewInMemory()
|
||||||
|
told := &conditions.Told{}
|
||||||
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||||
|
defer k.Close(context.Background())
|
||||||
|
d := &doctor{keeper: k, teller: told, host: "anchor"}
|
||||||
|
run := d.runOnce(t.Context(), "a test")
|
||||||
|
if run.Counts != (doctorCounts{Passed: 1, Failed: 1, FailedToRun: 2, Deferred: 1}) {
|
||||||
|
t.Fatalf("counted %+v", run.Counts)
|
||||||
|
}
|
||||||
|
open, err := k.Open(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var keys []string
|
||||||
|
for _, c := range open {
|
||||||
|
keys = append(keys, c.Key+"="+c.Kind)
|
||||||
|
}
|
||||||
|
for _, want := range []string{"machine.anchor.refused=declaration-refused", "probe.P3.failed=probe-failed",
|
||||||
|
"probe.P4.failed=probe-failed"} {
|
||||||
|
if !slices.Contains(keys, want) {
|
||||||
|
t.Errorf("%s is not open: %v", want, keys)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The heartbeat, in the shape mesh-watcher reads (the contract with the operator's channel).
|
||||||
|
if len(told.Names) != 1 || told.Names[0] != conditions.HeartbeatEvent {
|
||||||
|
t.Fatalf("said %v", told.Names)
|
||||||
|
}
|
||||||
|
if d.lastRunEnded().IsZero() || d.lastRun().Run != run.Run {
|
||||||
|
t.Fatal("the run is not the last verdict")
|
||||||
|
}
|
||||||
|
body, _ := json.Marshal(run)
|
||||||
|
var shape map[string]any
|
||||||
|
_ = json.Unmarshal(body, &shape)
|
||||||
|
for _, field := range []string{"run", "at", "interval-seconds", "counts", "probes", "controller"} {
|
||||||
|
if _, ok := shape[field]; !ok {
|
||||||
|
t.Errorf("the heartbeat carries no %q: %s", field, body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mended: the next run clears every one of them.
|
||||||
|
broken.Store(false)
|
||||||
|
d.runOnce(t.Context(), "a test")
|
||||||
|
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||||
|
t.Fatalf("a passing run left open %+v", open)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The registry says what each probe asserts**, and a probe not built says why and when.
|
||||||
|
func TestTheRegistryIsTheDesignsLiveForm(t *testing.T) {
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for _, p := range probeRegistry {
|
||||||
|
if seen[p.ID] {
|
||||||
|
t.Errorf("%s twice", p.ID)
|
||||||
|
}
|
||||||
|
seen[p.ID] = true
|
||||||
|
if p.Asserts == "" || p.From == "" || p.Kind == "" {
|
||||||
|
t.Errorf("%s does not say what it asserts, where from, or what it raises", p.ID)
|
||||||
|
}
|
||||||
|
if (p.run == nil) != (p.Deferred != "") || (p.Deferred != "" && p.Phase <= 1) {
|
||||||
|
t.Errorf("%s is run and deferred, or neither, or deferred out of Phase 1: %+v", p.ID, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, id := range []string{"D1", "D2", "D3", "D4", "D5", "D6", "D7", "D8", "D9", "D10"} {
|
||||||
|
if !seen[id] {
|
||||||
|
t.Errorf("to-be 45 §4 has %s and the registry does not", id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The doctor and the watchdogs watch each other**: watchdogs that stopped are DW; a self-check that
|
||||||
|
// stopped is S10 (signals_test.go).
|
||||||
|
func TestWatchdogsThatStoppedAreSaid(t *testing.T) {
|
||||||
|
w := &watchdogs{started: time.Now().Add(-time.Hour)}
|
||||||
|
d := &doctor{watchdogs: w, host: "anchor"}
|
||||||
|
got, err := probeWatchdogs(t.Context(), d)
|
||||||
|
if err != nil || len(got) != 1 || got[0].Severity != conditions.Urgent {
|
||||||
|
t.Fatalf("%+v %v", got, err)
|
||||||
|
}
|
||||||
|
w.ticked = time.Now()
|
||||||
|
if got, _ := probeWatchdogs(t.Context(), d); len(got) != 0 {
|
||||||
|
t.Fatalf("%+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **D1 composes every machine of a healthy mesh and the host's own validator takes each.**
|
||||||
|
func TestEveryMachineOfAHealthyMeshComposesAndValidates(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
got, err := probeDeclarations(t.Context(), &doctor{open: open})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(got) != 0 {
|
||||||
|
t.Fatalf("a healthy mesh failed D1: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **D1 names a machine nothing can be sent to**, and the network that cannot be computed for it.
|
||||||
|
func TestAMachineWhoseDeclarationDoesNotComposeIsSaid(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
one, two := rivals()
|
||||||
|
register(t, open, one)
|
||||||
|
register(t, open, two)
|
||||||
|
for _, m := range []string{"rival-one", "rival-two"} {
|
||||||
|
if _, err := assign(ctx, open, "laptop", m); err != nil && m == "rival-one" {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
got, err := probeDeclarations(ctx, &doctor{open: open})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "the-seat") {
|
||||||
|
t.Fatalf("%+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **D2: a resolver answering NXDOMAIN for IPv6 is wrong** — musl takes it as no such name (issue 262).
|
||||||
|
func TestAResolverAnsweringNoSuchNameForIPv6IsWrong(t *testing.T) {
|
||||||
|
answerAs := func(rcode dnsmessage.RCode) string {
|
||||||
|
conn, err := net.ListenPacket("udp", "127.0.0.1:0")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = conn.Close() })
|
||||||
|
go func() {
|
||||||
|
buf := make([]byte, 1500)
|
||||||
|
for {
|
||||||
|
n, from, err := conn.ReadFrom(buf)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var q dnsmessage.Message
|
||||||
|
if q.Unpack(buf[:n]) != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
reply := dnsmessage.Message{Header: dnsmessage.Header{ID: q.ID, Response: true}, Questions: q.Questions}
|
||||||
|
if q.Questions[0].Type == dnsmessage.TypeA {
|
||||||
|
reply.Answers = []dnsmessage.Resource{{Header: dnsmessage.ResourceHeader{Name: q.Questions[0].Name,
|
||||||
|
Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET}, Body: &dnsmessage.AResource{A: [4]byte{10, 77, 0, 1}}}}
|
||||||
|
} else {
|
||||||
|
reply.RCode = rcode
|
||||||
|
}
|
||||||
|
packed, _ := reply.Pack()
|
||||||
|
_, _ = conn.WriteTo(packed, from)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
_, port, _ := net.SplitHostPort(conn.LocalAddr().String())
|
||||||
|
return port
|
||||||
|
}
|
||||||
|
before := resolverPort
|
||||||
|
t.Cleanup(func() { resolverPort = before })
|
||||||
|
|
||||||
|
resolverPort = answerAs(dnsmessage.RCodeSuccess)
|
||||||
|
v4, rcode, err := askResolver(t.Context(), "127.0.0.1", "anchor.internal", dnsmessage.TypeA)
|
||||||
|
if err != nil || rcode != dnsmessage.RCodeSuccess || !slices.Equal(v4, []string{"10.77.0.1"}) {
|
||||||
|
t.Fatalf("%v %v %v", v4, rcode, err)
|
||||||
|
}
|
||||||
|
v6, rcode, err := askResolver(t.Context(), "127.0.0.1", "anchor.internal", dnsmessage.TypeAAAA)
|
||||||
|
if err != nil || rcode != dnsmessage.RCodeSuccess || len(v6) != 0 {
|
||||||
|
t.Fatalf("NODATA read as %v %v %v", v6, rcode, err)
|
||||||
|
}
|
||||||
|
resolverPort = answerAs(dnsmessage.RCodeNameError)
|
||||||
|
if _, rcode, _ := askResolver(t.Context(), "127.0.0.1", "anchor.internal", dnsmessage.TypeAAAA); rcode != dnsmessage.RCodeNameError {
|
||||||
|
t.Fatalf("NXDOMAIN read as %v", rcode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **D6, D7: what the controller defines is what it finds**, and a consumer deleted or a stream
|
||||||
|
// redefined is said — against a real bus, raised by the same derivation the controller starts with.
|
||||||
|
func TestNatsTheBusIsWhatTheControllerDefines(t *testing.T) {
|
||||||
|
url := os.Getenv("MESH_TEST_NATS")
|
||||||
|
if url == "" {
|
||||||
|
t.Skip("MESH_TEST_NATS unset")
|
||||||
|
}
|
||||||
|
open := aMesh(t)
|
||||||
|
js, err := broker.Dial(url)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(js.Close)
|
||||||
|
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
|
||||||
|
_ = js.Context().DeleteStream(s)
|
||||||
|
}
|
||||||
|
if _, err := assertBusObjects(t.Context(), open.inventory, js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := js.EnsureControllerBuckets(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
d := &doctor{open: open, js: js}
|
||||||
|
for _, p := range []func(context.Context, *doctor) ([]conditions.Observation, error){probeConsumers, probeStreams} {
|
||||||
|
got, err := p(t.Context(), d)
|
||||||
|
if err != nil || len(got) != 0 {
|
||||||
|
t.Fatalf("a bus just raised fails: %+v %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
info, err := js.Context().StreamInfo("EVENTS")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cfg := info.Config
|
||||||
|
cfg.MaxMsgsPerSubject = 3
|
||||||
|
if _, err := js.Context().UpdateStream(&cfg); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
consumers, err := probeConsumers(t.Context(), d)
|
||||||
|
if err != nil || len(consumers) != 1 || consumers[0].Key() != "bus.NODES.laptop.missing" {
|
||||||
|
t.Fatalf("the deleted consumer: %+v %v", consumers, err)
|
||||||
|
}
|
||||||
|
streams, err := probeStreams(t.Context(), d)
|
||||||
|
if err != nil || len(streams) != 1 || !strings.Contains(streams[0].Summary, "per subject") {
|
||||||
|
t.Fatalf("the redefined stream: %+v %v", streams, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **S9 hears the bus**: a consumer that gives up on a message, and one deleted, as the server says.
|
||||||
|
func TestNatsTheBusSaysAConsumerGaveUpAndOneWasDeleted(t *testing.T) {
|
||||||
|
url := os.Getenv("MESH_TEST_NATS")
|
||||||
|
if url == "" {
|
||||||
|
t.Skip("MESH_TEST_NATS unset")
|
||||||
|
}
|
||||||
|
conn, err := nats.Connect(url)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
heard := make(chan *nats.Msg, 16)
|
||||||
|
for _, subject := range broker.BusAdvisories {
|
||||||
|
if _, err := conn.ChanSubscribe(subject, heard); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
api, _ := jetstream.New(conn)
|
||||||
|
_ = api.DeleteStream(t.Context(), "SEAT_ADVISED")
|
||||||
|
stream, err := api.CreateStream(t.Context(), jetstream.StreamConfig{Name: "SEAT_ADVISED", Subjects: []string{"advised.>"}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() { _ = api.DeleteStream(context.Background(), "SEAT_ADVISED") }()
|
||||||
|
consumer, err := stream.CreateConsumer(t.Context(), jetstream.ConsumerConfig{Durable: "SEAT_ADVISED_worker",
|
||||||
|
AckPolicy: jetstream.AckExplicitPolicy, MaxDeliver: 1, AckWait: 100 * time.Millisecond})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := api.Publish(t.Context(), "advised.x", []byte("x")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := consumer.Fetch(1, jetstream.FetchMaxWait(time.Second)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// A seat's worker, so the deletion is of a consumer the mesh names (link.MeshNamed).
|
||||||
|
// Not acknowledged: after its one delivery the consumer gives up on it — on the next fetch.
|
||||||
|
time.Sleep(300 * time.Millisecond)
|
||||||
|
_, _ = consumer.Fetch(1, jetstream.FetchMaxWait(300*time.Millisecond))
|
||||||
|
if err := stream.DeleteConsumer(t.Context(), "SEAT_ADVISED_worker"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
kinds := map[string]string{}
|
||||||
|
deadline := time.After(5 * time.Second)
|
||||||
|
for len(kinds) < 2 {
|
||||||
|
select {
|
||||||
|
case m := <-heard:
|
||||||
|
if a, ok := link.ReadAdvisory(m.Subject, m.Data); ok {
|
||||||
|
kinds[a.Kind] = a.Said
|
||||||
|
}
|
||||||
|
case <-deadline:
|
||||||
|
t.Fatalf("the bus said only %v", kinds)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !strings.Contains(kinds["max-deliveries"], "gave up") || !strings.Contains(kinds["consumer-lost"], "was deleted") {
|
||||||
|
t.Fatalf("%v", kinds)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **D10 compares a node-engine with what it is delivered as, not with its commit** (2026-10-06: every
|
||||||
|
// machine read as behind right after a push sent it the current build — it says the digest-named
|
||||||
|
// directory it runs from, and the mesh holds a commit).
|
||||||
|
func TestANodeEngineIsJudgedByTheVersionItIsDeliveredAs(t *testing.T) {
|
||||||
|
m := catalogue.Manifest{Module: "mesh-host", Resources: []map[string]any{
|
||||||
|
{"id": "launcher", "type": "file", "path": "/usr/lib/nox-mesh-host/launch"},
|
||||||
|
{"id": "host", "type": "archive", "path": "/usr/lib/nox-mesh-host/versions/31045596c83a"},
|
||||||
|
{"id": "unfilled", "type": "archive", "path": "/usr/lib/x/versions/${version}"},
|
||||||
|
}}
|
||||||
|
delivered := deliveredVersions(m)
|
||||||
|
if !slices.Equal(delivered, []string{"31045596c83a"}) {
|
||||||
|
t.Fatalf("%v", delivered)
|
||||||
|
}
|
||||||
|
commit := "1545b00a9f0c"
|
||||||
|
for _, c := range []struct {
|
||||||
|
reported string
|
||||||
|
behind bool
|
||||||
|
}{
|
||||||
|
{"31045596c83a", false}, // the live case: current, and was called behind
|
||||||
|
{"0123456789ab", true}, // another delivery
|
||||||
|
{"1545b00a", false}, // placed by hand, stamped with the commit
|
||||||
|
{"", false}, // not said
|
||||||
|
} {
|
||||||
|
if got := engineBehind(c.reported, delivered, commit); got != c.behind {
|
||||||
|
t.Errorf("%q behind = %v, want %v", c.reported, got, c.behind)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if engineBehind("31045596c83a", nil, commit) {
|
||||||
|
t.Error("behind a mesh that holds no delivered build")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Every seat verb a probe calls is one it declares, and one the controller is granted** — derived
|
||||||
|
// from the registry, so a probe added with a question the bus would refuse fails here, not live.
|
||||||
|
func TestEverySeatVerbAProbeAsksIsGranted(t *testing.T) {
|
||||||
|
granted, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController, PasswordHash: "x"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
asked := 0
|
||||||
|
for _, p := range probeRegistry {
|
||||||
|
for _, v := range p.Asks {
|
||||||
|
asked++
|
||||||
|
subject := link.NodeSeatToolSubject(v.Seat, v.Verb, "anchor")
|
||||||
|
if !slices.ContainsFunc(granted.Publish, func(pattern string) bool { return subjectMatches(pattern, subject) }) {
|
||||||
|
t.Errorf("%s asks %s.%s and the controller may not publish %s", p.ID, v.Seat, v.Verb, subject)
|
||||||
|
}
|
||||||
|
if !slices.Contains(broker.VerbsTheSelfCheckAsks, v) {
|
||||||
|
t.Errorf("%s asks %s.%s, which broker.VerbsTheSelfCheckAsks does not name", p.ID, v.Seat, v.Verb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if asked == 0 {
|
||||||
|
t.Fatal("no probe asks a seat verb: D8 lost its declaration")
|
||||||
|
}
|
||||||
|
// And a probe asking what it did not declare is refused before anything is sent.
|
||||||
|
ctx := context.WithValue(t.Context(), probeAsksKey{}, probe{ID: "DX"})
|
||||||
|
if _, err := askSeatTool(ctx, nil, "node-intrusion-prevention", "banned", "anchor"); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "does not declare") {
|
||||||
|
t.Fatalf("an undeclared question was asked: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// subjectMatches is the bus's matching of a permission pattern against a subject.
|
||||||
|
func subjectMatches(pattern, subject string) bool {
|
||||||
|
p, s := strings.Split(pattern, "."), strings.Split(subject, ".")
|
||||||
|
for i, tok := range p {
|
||||||
|
if tok == ">" {
|
||||||
|
return len(s) > i
|
||||||
|
}
|
||||||
|
if i >= len(s) || (tok != "*" && tok != s[i]) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return len(p) == len(s)
|
||||||
|
}
|
||||||
@@ -0,0 +1,170 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"slices"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What the core's bounds are set from (novox/hq to-be 45 Phase 0).
|
||||||
|
//
|
||||||
|
// **A bound is set from what was measured, not from what seemed reasonable.** Phase 1 puts a watchdog
|
||||||
|
// on each row of the signals table, and each has a bound: S1 three heartbeat intervals, S2 three times
|
||||||
|
// a machine's last apply, S3 a tier's build and apply time, S6 a build's timeout. Marked provisional
|
||||||
|
// in the design until a fortnight of these says what the mesh actually takes. Recorded by the serving
|
||||||
|
// controller as it hears each — a send's first report, a machine's next word, a plan leaving a tier, a
|
||||||
|
// build's outcome — and summarised here per machine, repository or module.
|
||||||
|
|
||||||
|
// recordBuildDuration measures one build from its ask to its outcome heard.
|
||||||
|
func recordBuildDuration(ctx context.Context, inv *inventory.Inventory, result link.BuildResult, asked time.Time) {
|
||||||
|
if asked.IsZero() || result.ID == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
subject := result.Module
|
||||||
|
if subject == "" {
|
||||||
|
subject = result.Repository
|
||||||
|
}
|
||||||
|
detail := "built"
|
||||||
|
if result.Failed != "" {
|
||||||
|
detail = "failed: " + firstLine(result.Failed)
|
||||||
|
}
|
||||||
|
if err := inv.RecordDuration(ctx, inventory.Duration{Kind: inventory.DurationBuild, Subject: subject,
|
||||||
|
Node: result.On, Ref: result.ID, Started: asked, Took: time.Since(asked), Detail: detail}); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "%s: how long it took could not be recorded: %v\n", result.ID, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// durationSummary is one subject's measurements of one kind.
|
||||||
|
type durationSummary struct {
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Subject string `json:"subject"`
|
||||||
|
Count int `json:"count"`
|
||||||
|
Median string `json:"median"`
|
||||||
|
P90 string `json:"p90"`
|
||||||
|
Max string `json:"max"`
|
||||||
|
// Bound is what to-be 45's rule would make of these, where the rule is a multiple of a measured
|
||||||
|
// time: three times the slowest apply (S2), three times the median word interval (S1).
|
||||||
|
Suggests string `json:"suggests,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func summarise(ds []inventory.Duration) []durationSummary {
|
||||||
|
type key struct{ kind, subject string }
|
||||||
|
by := map[key][]time.Duration{}
|
||||||
|
for _, d := range ds {
|
||||||
|
k := key{d.Kind, d.Subject}
|
||||||
|
by[k] = append(by[k], d.Took)
|
||||||
|
}
|
||||||
|
var out []durationSummary
|
||||||
|
for k, took := range by {
|
||||||
|
slices.Sort(took)
|
||||||
|
at := func(q float64) time.Duration { return took[int(q*float64(len(took)-1))] }
|
||||||
|
s := durationSummary{Kind: k.kind, Subject: k.subject, Count: len(took),
|
||||||
|
Median: round(at(0.5)), P90: round(at(0.9)), Max: round(took[len(took)-1])}
|
||||||
|
switch k.kind {
|
||||||
|
case inventory.DurationApply:
|
||||||
|
s.Suggests = "S2 bound max(2m, 3×last apply) ≈ " + round(max(2*time.Minute, 3*at(0.9))) + " at the p90"
|
||||||
|
case inventory.DurationHeartbeatGap:
|
||||||
|
s.Suggests = "S1 bound 3×interval ≈ " + round(3*at(0.5))
|
||||||
|
}
|
||||||
|
out = append(out, s)
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(i, j int) bool {
|
||||||
|
ki, kj := slices.Index(inventory.DurationKinds, out[i].Kind), slices.Index(inventory.DurationKinds, out[j].Kind)
|
||||||
|
if ki != kj {
|
||||||
|
return ki < kj
|
||||||
|
}
|
||||||
|
return out[i].Subject < out[j].Subject
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func round(d time.Duration) string {
|
||||||
|
switch {
|
||||||
|
case d < time.Second:
|
||||||
|
return d.Round(time.Millisecond).String()
|
||||||
|
case d < time.Minute:
|
||||||
|
return d.Round(100 * time.Millisecond).String()
|
||||||
|
default:
|
||||||
|
return d.Round(time.Second).String()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// durationsCommand is `durations`: the summary per kind and subject, or every measurement as data.
|
||||||
|
func durationsCommand(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("durations", flag.ContinueOnError)
|
||||||
|
kind := set.String("kind", "", "one kind: "+strings.Join(inventory.DurationKinds, ", "))
|
||||||
|
days := set.Int("days", 14, "how many days back")
|
||||||
|
asJSON := set.Bool("json", false, "the summary as data")
|
||||||
|
all := set.Bool("all", false, "every measurement rather than the summary")
|
||||||
|
if _, err := parseAround(set, args); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if *kind != "" && !slices.Contains(inventory.DurationKinds, *kind) {
|
||||||
|
return fmt.Errorf("%q is not a kind of duration: %s", *kind, strings.Join(inventory.DurationKinds, ", "))
|
||||||
|
}
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
ds, err := open.inventory.Durations(ctx, *kind, time.Now().Add(-time.Duration(*days)*24*time.Hour))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if *all {
|
||||||
|
body, err := json.MarshalIndent(ds, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(string(body))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
summary := summarise(ds)
|
||||||
|
if *asJSON {
|
||||||
|
body, err := json.MarshalIndent(map[string]any{"days": *days, "durations": summary}, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(string(body))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if len(summary) == 0 {
|
||||||
|
fmt.Printf("nothing measured in the last %d day(s): the serving controller records apply, heartbeat-gap, "+
|
||||||
|
"plan-tier and build durations as it hears them\n", *days)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
fmt.Printf("durations over the last %d day(s) — what the core's bounds are set from (to-be 45 Phase 0)\n\n", *days)
|
||||||
|
fmt.Printf(" %-14s %-28s %6s %10s %10s %10s\n", "kind", "of", "count", "median", "p90", "max")
|
||||||
|
for _, s := range summary {
|
||||||
|
fmt.Printf(" %-14s %-28s %6d %10s %10s %10s\n", s.Kind, s.Subject, s.Count, s.Median, s.P90, s.Max)
|
||||||
|
if s.Suggests != "" {
|
||||||
|
fmt.Printf(" %-14s %-28s %s\n", "", "", s.Suggests)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// forgettingOldDurations removes what is older than a month, at start and daily after.
|
||||||
|
func forgettingOldDurations(ctx context.Context, inv *inventory.Inventory) {
|
||||||
|
for {
|
||||||
|
if n, err := inv.ForgetOldDurations(ctx); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "durations older than %s could not be removed: %v\n", inventory.DurationsKeptFor, err)
|
||||||
|
} else if n > 0 {
|
||||||
|
fmt.Printf("removed %d duration(s) older than %s\n", n, inventory.DurationsKeptFor)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-time.After(24 * time.Hour):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,197 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Acts done by hand, and why (novox/hq to-be 45 §7).
|
||||||
|
//
|
||||||
|
// **Which verbs ask why.** `plans close` and `plans stop`, `broker consumer-reset` and `hand-act
|
||||||
|
// record` refuse without it everywhere: nothing automated runs them, so a call without a reason is a
|
||||||
|
// person who has not given one. A named `push` asks for it through the mesh-controller seat, which is
|
||||||
|
// how a person or an agent acts by hand on the mesh; at a shell `--why` is recorded when given and not
|
||||||
|
// required, because the installer and the lab push by command line as a step of what they do, and a
|
||||||
|
// step of a procedure is not a repair. `conditions silence` joins them when the condition store does
|
||||||
|
// (Phase 1).
|
||||||
|
|
||||||
|
// handActFlags are the flags every repairing verb takes.
|
||||||
|
type handActFlags struct {
|
||||||
|
why, cause, condition *string
|
||||||
|
}
|
||||||
|
|
||||||
|
func addHandActFlags(set *flag.FlagSet) handActFlags {
|
||||||
|
return handActFlags{
|
||||||
|
why: set.String("why", "", "why this is done by hand — recorded in the hand-act log (novox/hq to-be 45 §7)"),
|
||||||
|
cause: set.String("cause", "", "the cause, in a word or a condition's kind; the verb's own name when not given"),
|
||||||
|
condition: set.String("condition", "", "the key of the condition this act addresses, if any"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// given is whether a reason was given.
|
||||||
|
func (f handActFlags) given() bool { return strings.TrimSpace(*f.why) != "" }
|
||||||
|
|
||||||
|
// require refuses an act without a reason, before anything is done.
|
||||||
|
func (f handActFlags) require(verb string) error {
|
||||||
|
if f.given() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return fmt.Errorf("%s is a repair done by hand, and says why: --why <text> (recorded in the hand-act "+
|
||||||
|
"log, novox/hq to-be 45 §7). Nothing was done", verb)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handActConn is the serving controller's connection, for what it reads of the log itself; a
|
||||||
|
// command dials its own.
|
||||||
|
var handActConn *nats.Conn
|
||||||
|
|
||||||
|
// onTheBus runs f with a connection to the bus: the serving controller's, or one of its own.
|
||||||
|
func onTheBus(f func(*nats.Conn) error) error {
|
||||||
|
if handActConn != nil {
|
||||||
|
return f(handActConn)
|
||||||
|
}
|
||||||
|
address, err := broker.BusAddress()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
js, err := broker.Dial(address)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot reach the bus: %w", err)
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
return f(js.Conn())
|
||||||
|
}
|
||||||
|
|
||||||
|
// record writes the entry for an act about to be done. **Before the act, and never instead of it**:
|
||||||
|
// a log that cannot be written is said loudly, and the repair it was about still happens — a mesh
|
||||||
|
// whose bus is down is exactly the mesh somebody is repairing by hand.
|
||||||
|
func (f handActFlags) record(ctx context.Context, verb string, args []string) {
|
||||||
|
if !f.given() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
act := link.HandAct{Verb: verb, Args: args, Why: strings.TrimSpace(*f.why),
|
||||||
|
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
|
||||||
|
err := onTheBus(func(conn *nats.Conn) error {
|
||||||
|
written, err := link.RecordHandAct(ctx, conn, act)
|
||||||
|
act = written
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "this act by hand could NOT be recorded in the hand-act log, and is done anyway: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Printf("recorded as %s in the hand-act log: %s, because %q (cause: %s)\n", act.ID, act.By, act.Why, act.Cause)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handActCommand is `hand-act record` and `hand-acts`.
|
||||||
|
func handActCommand(ctx context.Context, args []string) error {
|
||||||
|
if len(args) > 0 && args[0] == "record" {
|
||||||
|
set := flag.NewFlagSet("hand-act record", flag.ContinueOnError)
|
||||||
|
f := addHandActFlags(set)
|
||||||
|
positionals, err := parseAround(set, args[1:])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
what := strings.TrimSpace(strings.Join(positionals, " "))
|
||||||
|
if what == "" {
|
||||||
|
return errors.New("hand-act record <what was done> --why <text> [--cause <word>] [--condition <key>]")
|
||||||
|
}
|
||||||
|
if err := f.require("hand-act record"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(*f.cause) == "" {
|
||||||
|
return errors.New("hand-act record says the cause too: --cause <word>, the word a second " +
|
||||||
|
"act for the same reason will use — it is how a repair done twice is found")
|
||||||
|
}
|
||||||
|
act := link.HandAct{Verb: "hand-act record", Args: []string{what}, Why: strings.TrimSpace(*f.why),
|
||||||
|
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
|
||||||
|
return onTheBus(func(conn *nats.Conn) error {
|
||||||
|
written, err := link.RecordHandAct(ctx, conn, act)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the act could not be recorded: %w", err)
|
||||||
|
}
|
||||||
|
fmt.Printf("recorded as %s: %s did %q, because %q (cause: %s)\n", written.ID, written.By, what,
|
||||||
|
written.Why, written.Cause)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if len(args) > 0 && args[0] != "list" && !strings.HasPrefix(args[0], "-") {
|
||||||
|
return errors.New("hand-act record <what> --why <text> --cause <word> | hand-acts [--days N] [--json]")
|
||||||
|
}
|
||||||
|
if len(args) > 0 && args[0] == "list" {
|
||||||
|
args = args[1:]
|
||||||
|
}
|
||||||
|
set := flag.NewFlagSet("hand-acts", flag.ContinueOnError)
|
||||||
|
days := set.Int("days", 14, "how many days back")
|
||||||
|
asJSON := set.Bool("json", false, "as data")
|
||||||
|
if _, err := parseAround(set, args); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return onTheBus(func(conn *nats.Conn) error {
|
||||||
|
now := time.Now()
|
||||||
|
acts, err := link.HandActs(ctx, conn, now.Add(-time.Duration(*days)*24*time.Hour))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
repeated := link.RepeatedCauses(acts, now)
|
||||||
|
if *asJSON {
|
||||||
|
body, err := json.MarshalIndent(map[string]any{"acts": acts, "repeated": repeated}, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(string(body))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if len(acts) == 0 {
|
||||||
|
fmt.Printf("nothing was done by hand in the last %d day(s)\n", *days)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for i := len(acts) - 1; i >= 0; i-- {
|
||||||
|
a := acts[i]
|
||||||
|
fmt.Printf("%s %s %s %s\n by %s — %s (cause: %s", a.At.Local().Format("2006-01-02 15:04"), a.ID,
|
||||||
|
a.Verb, strings.Join(a.Args, " "), a.By, a.Why, a.Cause)
|
||||||
|
if a.Condition != "" {
|
||||||
|
fmt.Printf(", condition %s", a.Condition)
|
||||||
|
}
|
||||||
|
fmt.Println(")")
|
||||||
|
}
|
||||||
|
if len(repeated) > 0 {
|
||||||
|
causes := make([]string, 0, len(repeated))
|
||||||
|
for c, n := range repeated {
|
||||||
|
causes = append(causes, fmt.Sprintf("%s ×%d", c, n))
|
||||||
|
}
|
||||||
|
sort.Strings(causes)
|
||||||
|
fmt.Printf("\ndone by hand more than once in a fortnight — a healer is wanted (to-be 45 S15): %s\n",
|
||||||
|
strings.Join(causes, ", "))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// handActsThisWeek is how many acts were done by hand in the last seven days, for `status`; -1 when
|
||||||
|
// the log could not be read, which status says rather than reading as none.
|
||||||
|
func handActsThisWeek(ctx context.Context) (int, string) {
|
||||||
|
n := -1
|
||||||
|
err := onTheBus(func(conn *nats.Conn) error {
|
||||||
|
reading, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
acts, err := link.HandActs(reading, conn, time.Now().Add(-7*24*time.Hour))
|
||||||
|
n = len(acts)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return -1, err.Error()
|
||||||
|
}
|
||||||
|
return n, ""
|
||||||
|
}
|
||||||
@@ -0,0 +1,94 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **Every verb that repairs by hand takes a required why** (novox/hq to-be 45 §7): refused before
|
||||||
|
// anything is done, through the seat, through `command`, and at a shell where nothing automated runs
|
||||||
|
// the verb.
|
||||||
|
func TestARepairByHandWithoutAReasonIsRefused(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
verb string
|
||||||
|
args map[string]any
|
||||||
|
}{
|
||||||
|
{"push", map[string]any{"node": "anchor"}},
|
||||||
|
{"push", map[string]any{}},
|
||||||
|
{"plans", map[string]any{"close": "plan-1"}},
|
||||||
|
{"plans", map[string]any{"stop": "plan-1"}},
|
||||||
|
{"hand-act", map[string]any{"what": "restarted the proxy", "cause": "proxy-stuck"}},
|
||||||
|
{"command", map[string]any{"command": "push anchor"}},
|
||||||
|
{"command", map[string]any{"command": "plans close plan-1"}},
|
||||||
|
{"command", map[string]any{"command": "broker consumer-reset EVENTS controller"}},
|
||||||
|
{"command", map[string]any{"command": "hand-act record restarted --cause x"}},
|
||||||
|
} {
|
||||||
|
argv, err := argvFor(c.verb, c.args)
|
||||||
|
if c.verb == "plans" && err == nil {
|
||||||
|
// The seat composes the command line; the command refuses it, before opening anything.
|
||||||
|
err = plansCommand(context.Background(), argv[1:])
|
||||||
|
}
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "why") {
|
||||||
|
t.Errorf("%s %v was not refused for want of why: %v %v", c.verb, c.args, argv, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, args := range [][]string{{"EVENTS", "controller"}} {
|
||||||
|
if err := consumerReset(context.Background(), args); err == nil || !strings.Contains(err.Error(), "--why") {
|
||||||
|
t.Errorf("consumer-reset without why: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := handActCommand(context.Background(), []string{"record", "restarted the proxy", "--cause", "x"}); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "--why") {
|
||||||
|
t.Errorf("hand-act record without why: %v", err)
|
||||||
|
}
|
||||||
|
if err := handActCommand(context.Background(), []string{"record", "restarted the proxy", "--why", "it hung"}); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "--cause") {
|
||||||
|
t.Errorf("hand-act record without a cause: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// With a reason, the seat passes it to the command, and a verb that only reads is not held to one.
|
||||||
|
func TestARepairByHandCarriesItsReason(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
verb string
|
||||||
|
args map[string]any
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"push", map[string]any{"node": "anchor", "why": "stuck", "cause": "sent-not-reported"},
|
||||||
|
"push anchor --wait 0 --why stuck --cause sent-not-reported"},
|
||||||
|
{"plans", map[string]any{"close": "plan-1", "why": "the report will not come"},
|
||||||
|
"plans close plan-1 --why the report will not come"},
|
||||||
|
{"plans", map[string]any{"retry": "plan-1"}, "plans retry plan-1"},
|
||||||
|
{"hand-act", map[string]any{"what": "restarted", "why": "hung", "cause": "proxy", "condition": "machine.a.silent"},
|
||||||
|
"hand-act record restarted --why hung --cause proxy --condition machine.a.silent"},
|
||||||
|
{"command", map[string]any{"command": "push anchor --why stuck"}, "push anchor --why stuck"},
|
||||||
|
{"command", map[string]any{"command": "plans plan-1"}, "plans plan-1"},
|
||||||
|
} {
|
||||||
|
argv, err := argvFor(c.verb, c.args)
|
||||||
|
if err != nil || strings.Join(argv, " ") != c.want {
|
||||||
|
t.Errorf("%s %v: %v %v, want %q", c.verb, c.args, argv, err, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The summary of durations says, per kind and subject, what a bound would be set from.
|
||||||
|
func TestDurationsAreSummarisedPerSubject(t *testing.T) {
|
||||||
|
var ds []inventory.Duration
|
||||||
|
for i := 1; i <= 10; i++ {
|
||||||
|
ds = append(ds, inventory.Duration{Kind: inventory.DurationApply, Subject: "anchor",
|
||||||
|
Took: time.Duration(i) * time.Second})
|
||||||
|
}
|
||||||
|
ds = append(ds, inventory.Duration{Kind: inventory.DurationHeartbeatGap, Subject: "anchor", Took: time.Minute})
|
||||||
|
got := summarise(ds)
|
||||||
|
if len(got) != 2 || got[0].Kind != inventory.DurationApply || got[0].Count != 10 ||
|
||||||
|
got[0].Max != "10s" || got[0].Median != "5s" || got[0].P90 != "9s" {
|
||||||
|
t.Fatalf("%+v", got)
|
||||||
|
}
|
||||||
|
if !strings.Contains(got[1].Suggests, "3m0s") {
|
||||||
|
t.Fatalf("a minute between words suggests %q", got[1].Suggests)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,150 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq ADR 0225, issue 263: a consumer's identity is bounded by the provision it requires, an
|
||||||
|
// overflow is refused before merge by `module check`, and a provider's machine is never refused for
|
||||||
|
// one consumer's identity.
|
||||||
|
|
||||||
|
// `module check` refuses the pull request that introduces an overflow, naming the module.
|
||||||
|
func TestModuleCheckRefusesAnIdentityThatOverflowsWhatItRequires(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
write := func(name, body string) string {
|
||||||
|
p := filepath.Join(dir, name+".json")
|
||||||
|
if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
objects := write("objects", `{"module":"objects","version":"1",
|
||||||
|
"provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}],
|
||||||
|
"receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`)
|
||||||
|
resolver := write("resolver", `{"module":"resolver","version":"1",
|
||||||
|
"provides":[{"name":"wildcard-resolution","scope":"mesh","identity":false}]}`)
|
||||||
|
album := write("photoalbum", `{"module":"photoalbum","version":"1","requires":["s3-bucket"]}`)
|
||||||
|
nm := write("networkmanager", `{"module":"networkmanager","version":"1","requires":["wildcard-resolution"]}`)
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
if err := moduleCheckFor([]string{resolver, nm}, 6, &out); err != nil {
|
||||||
|
t.Fatalf("a long name requiring a keyless provision was refused (issue 263): %v\n%s", err, out.String())
|
||||||
|
}
|
||||||
|
out.Reset()
|
||||||
|
err := moduleCheckFor([]string{objects, album, resolver, nm}, 6, &out)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("an identity overflowing an S3 access key passed:\n%s", out.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(out.String(), "photoalbum wants s3-bucket") ||
|
||||||
|
!strings.Contains(out.String(), "`slug` of at most 8 characters") ||
|
||||||
|
strings.Contains(out.String(), "networkmanager wants") {
|
||||||
|
t.Fatalf("the refusal does not name the one overflowing module and its remedy:\n%s", out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tonight's case, through the commands: networkmanager on a six-character machine requires the
|
||||||
|
// resolver provision, and a second consumer there overflows an object store's access key. The
|
||||||
|
// provider's machine still composes; the overflowing consumer is left out of its grants and named,
|
||||||
|
// by push and by `status`, and the keyless consumer is granted with its long name.
|
||||||
|
func TestAnOverflowingConsumerNeverRefusesItsProvidersMachine(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, catalogue.Manifest{Module: "objects", Version: "1",
|
||||||
|
Provides: []catalogue.Offer{{Name: "s3-bucket", Scope: catalogue.ScopeMesh,
|
||||||
|
Identity: &catalogue.OfferIdentity{Max: 20, In: "an S3 access key"}}},
|
||||||
|
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}})
|
||||||
|
register(t, open, catalogue.Manifest{Module: "resolver", Version: "1",
|
||||||
|
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}})
|
||||||
|
register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1",
|
||||||
|
Requires: []string{"wildcard-resolution"}})
|
||||||
|
register(t, open, catalogue.Manifest{Module: "photoalbum", Version: "1", Requires: []string{"s3-bucket"}})
|
||||||
|
register(t, open, catalogue.Manifest{Module: "files", Version: "1", Requires: []string{"s3-bucket"}})
|
||||||
|
for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "resolver"},
|
||||||
|
{"laptop", "networkmanager"}, {"laptop", "photoalbum"}, {"laptop", "files"}} {
|
||||||
|
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||||
|
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The consumer's machine resolves, and says which of its modules no provider will grant.
|
||||||
|
consumer, _, err := planFor(ctx, open, "laptop")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
over := consumer.Overflowing()
|
||||||
|
if len(over) != 1 || over[0].Module != "photoalbum" || over[0].Provision != "s3-bucket" {
|
||||||
|
t.Fatalf("the consumer's side does not name exactly photoalbum: %+v", over)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The provider's machine composes. Under ADR 0049's one bound this was a refusal naming
|
||||||
|
// networkmanager, and no push to the provider could go through.
|
||||||
|
plan, settings, err := planFor(ctx, open, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
declared, err := declarationFor(ctx, open, "anchor", plan, settings)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("one consumer's identity refused its provider's whole machine: %v", err)
|
||||||
|
}
|
||||||
|
if len(declared.withheld) != 1 || declared.withheld[0].Identity != "mesh_laptop_photoalbum" {
|
||||||
|
t.Fatalf("the overflowing consumer is not the one withheld: %+v", declared.withheld)
|
||||||
|
}
|
||||||
|
grants, _, err := grantsFor(ctx, open, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var keyless bool
|
||||||
|
for _, g := range grants {
|
||||||
|
keyless = keyless || g.Provision == "wildcard-resolution" && g.From == "networkmanager"
|
||||||
|
}
|
||||||
|
if !keyless {
|
||||||
|
t.Fatalf("networkmanager, 26 characters, is not granted the keyless resolver provision: %+v", grants)
|
||||||
|
}
|
||||||
|
var granted []string
|
||||||
|
for _, c := range declared.Received["objects"]["s3-bucket"] {
|
||||||
|
granted = append(granted, c.From)
|
||||||
|
}
|
||||||
|
if strings.Join(granted, ",") != "files" {
|
||||||
|
t.Fatalf("the object store grants %v; files and only files fit", granted)
|
||||||
|
}
|
||||||
|
said := printed(t, func() error { reportLeftOut("anchor", declared); return nil })
|
||||||
|
if !strings.Contains(said, `photoalbum on laptop requires s3-bucket from anchor`) ||
|
||||||
|
!strings.Contains(said, "left out of anchor's grants") {
|
||||||
|
t.Fatalf("the push does not say whom it leaves out:\n%s", said)
|
||||||
|
}
|
||||||
|
|
||||||
|
// And `status` names it, and does not call the mesh well while it stands.
|
||||||
|
asked, err := theThreeQuestions(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(asked.overflowing) != 1 || asked.overflowing[0].Module != "photoalbum" {
|
||||||
|
t.Fatalf("status does not carry the overflow: %+v", asked.overflowing)
|
||||||
|
}
|
||||||
|
if asked.well() {
|
||||||
|
t.Fatal("a mesh with a consumer left out of its grants reads as well")
|
||||||
|
}
|
||||||
|
shown := printed(t, func() error { return printStatus(asked) })
|
||||||
|
if !strings.Contains(shown, "identified too long for a provision they require") ||
|
||||||
|
!strings.Contains(shown, "mesh_laptop_photoalbum") {
|
||||||
|
t.Fatalf("status does not say it:\n%s", shown)
|
||||||
|
}
|
||||||
|
body, err := statusAsJSON(asked)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var doc struct {
|
||||||
|
Overflowing []catalogue.Overflow `json:"overflowing"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(body, &doc); err != nil || len(doc.Overflowing) != 1 ||
|
||||||
|
doc.Overflowing[0].Bound.Max != 20 {
|
||||||
|
t.Fatalf("the document does not carry it: %v\n%s", err, body)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -145,6 +145,19 @@ func run() error {
|
|||||||
return seatCommand(ctx, args[1:])
|
return seatCommand(ctx, args[1:])
|
||||||
case "status":
|
case "status":
|
||||||
return statusCommand(ctx, args[1:])
|
return statusCommand(ctx, args[1:])
|
||||||
|
// Acts done by hand, and why (novox/hq to-be 45 §7).
|
||||||
|
case "hand-act":
|
||||||
|
return handActCommand(ctx, args[1:])
|
||||||
|
case "hand-acts":
|
||||||
|
return handActCommand(ctx, append([]string{"list"}, args[1:]...))
|
||||||
|
// What the mesh's bounds will be set from (novox/hq to-be 45 Phase 0).
|
||||||
|
case "durations":
|
||||||
|
return durationsCommand(ctx, args[1:])
|
||||||
|
// What is wrong, and the self-check (novox/hq to-be 45 §2, §4).
|
||||||
|
case "conditions":
|
||||||
|
return conditionsCommand(ctx, args[1:])
|
||||||
|
case "doctor":
|
||||||
|
return doctorCommand(ctx, args[1:])
|
||||||
case "version":
|
case "version":
|
||||||
fmt.Println(version)
|
fmt.Println(version)
|
||||||
return nil
|
return nil
|
||||||
@@ -226,19 +239,33 @@ func usage() {
|
|||||||
kill <id> end a build where it runs; recorded failed, killed by hand
|
kill <id> end a build where it runs; recorded failed, killed by hand
|
||||||
pause [<node>] / resume [<node>] the build seat's holder there, or every holder, takes nothing new / again
|
pause [<node>] / resume [<node>] the build seat's holder there, or every holder, takes nothing new / again
|
||||||
plans retry <id> ask a failed plan's failed builds again, and carry the plan on
|
plans retry <id> ask a failed plan's failed builds again, and carry the plan on
|
||||||
|
plans stop|close <id> --why <text> end a plan by hand; recorded in the hand-act log
|
||||||
|
hand-act record <what> --why <text> --cause <word> [--condition <key>]
|
||||||
|
record an act done by hand outside the mesh (to-be 45 §7)
|
||||||
|
hand-acts [--days N] [--json] what was done by hand lately, why, and which causes repeat
|
||||||
|
conditions [--scope S] [--severity S] [--machine M] [--json]
|
||||||
|
what is wrong now: every open condition, urgent first (to-be 45 §2)
|
||||||
|
conditions show <key> one condition whole, with its evidence
|
||||||
|
conditions silence <key> --for <d> --why <text> send no message for it a while; a hand act
|
||||||
|
conditions history [--days N] [--key K] every raising, change and clearing lately
|
||||||
|
doctor [run|probes|signals] [--json]
|
||||||
|
the self-check: the last verdict, a run now, the probes, the signals' ages
|
||||||
|
durations [--kind K] [--days N] [--json]
|
||||||
|
apply, heartbeat, plan-tier and build durations, per machine or module
|
||||||
collection [--json] kept archives held/unheld by a manifest, and what the sweep may let go
|
collection [--json] kept archives held/unheld by a manifest, and what the sweep may let go
|
||||||
builder issue <name> a broker account for a build machine, scoped to build work,
|
builder issue <name> a broker account for a build machine, scoped to build work,
|
||||||
delivered as the builder module's broker secret (module add it first)
|
delivered as the builder module's broker secret (module add it first)
|
||||||
licence add|list|use|key model access, under the name a person calls it
|
licence add|list|use|key model access, under the name a person calls it
|
||||||
licence manager <name> <node> the node that holds a refreshable licence's refresh token
|
licence manager <name> <node> the node that holds a refreshable licence's refresh token
|
||||||
licence refresh <name> mint a new access token and seal it to every holder
|
licence refresh <name> mint a new access token and seal it to every holder
|
||||||
rotate <provision> [--consumer <n>] a new credential for every holder, both ends at once
|
rotate <provision> [--consumer <n>] [--module <m>] a new credential for every holder, both ends at once
|
||||||
ask <module> <tool> [json] call one of a module's tools over the broker, and print its answer
|
ask <module> <tool> [json] call one of a module's tools over the broker, and print its answer
|
||||||
pin <node> <provision> <from-node> <module>
|
pin <node> <provision> <from-node> <module>
|
||||||
which provider this one gets a provision from: the module, and its node
|
which provider this one gets a provision from: the module, and its node
|
||||||
unpin <node> <provision> put that question back
|
unpin <node> <provision> put that question back
|
||||||
plan <node> [--files|--json] what that node would run, and why
|
plan <node> [--files|--json] what that node would run, and why
|
||||||
push [<node>] [--behind] send a node everything it should be, or only those that need it
|
push [<node>] [--behind] [--why <text>] send a node everything it should be, or only those
|
||||||
|
that need it; --why records it in the hand-act log
|
||||||
version what this binary is
|
version what this binary is
|
||||||
|
|
||||||
Each context reaches its own store through its own credential (novox/hq ADR 0008), named
|
Each context reaches its own store through its own credential (novox/hq ADR 0008), named
|
||||||
@@ -291,6 +318,9 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
|||||||
// When it was asked, so a plan takes as its outcome only a build asked for it or after it
|
// When it was asked, so a plan takes as its outcome only a build asked for it or after it
|
||||||
// (novox/hq 04-ISSUES/219). Zero when the id does not say.
|
// (novox/hq 04-ISSUES/219). Zero when the id does not say.
|
||||||
asked, _ := link.BuildAskedAt(result.ID)
|
asked, _ := link.BuildAskedAt(result.ID)
|
||||||
|
// And how long it took, asked to heard, which a build's bound will be set from (novox/hq to-be 45
|
||||||
|
// Phase 0). Said if lost; never a reason not to take the build in.
|
||||||
|
recordBuildDuration(ctx, b.inv, result, asked)
|
||||||
switch {
|
switch {
|
||||||
case err != nil && result.Failed != "":
|
case err != nil && result.Failed != "":
|
||||||
fmt.Printf("%s: %v\n", result.ID, err)
|
fmt.Printf("%s: %v\n", result.ID, err)
|
||||||
@@ -317,5 +347,7 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
|||||||
result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit))
|
result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit))
|
||||||
saysWhenThePolicyActs(ctx, b.inv, manifest.Module)
|
saysWhenThePolicyActs(ctx, b.inv, manifest.Module)
|
||||||
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
|
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
|
||||||
|
// A module registered may be one a machine is now behind: `status` is composed again.
|
||||||
|
statusFrom.nudge()
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,11 +1,14 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
"crypto/ecdh"
|
"crypto/ecdh"
|
||||||
"crypto/rand"
|
"crypto/rand"
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
@@ -37,6 +40,9 @@ func aMesh(t *testing.T) *stores {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
t.Cleanup(open.Close)
|
t.Cleanup(open.Close)
|
||||||
|
// A condition store of its own, held in memory (novox/hq to-be 45 §2): status leads with what is
|
||||||
|
// open, and a mesh with no bus would otherwise read as one whose conditions cannot be read.
|
||||||
|
withConditionsInMemory(t)
|
||||||
for _, m := range provided {
|
for _, m := range provided {
|
||||||
if err := open.inventory.Provide(t.Context(), m); err != nil {
|
if err := open.inventory.Provide(t.Context(), m); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -106,3 +112,17 @@ func rivals() (catalogue.Manifest, catalogue.Manifest) {
|
|||||||
return catalogue.Manifest{Module: "rival-one", Version: "1", Claims: claim},
|
return catalogue.Manifest{Module: "rival-one", Version: "1", Claims: claim},
|
||||||
catalogue.Manifest{Module: "rival-two", Version: "1", Claims: claim}
|
catalogue.Manifest{Module: "rival-two", Version: "1", Claims: claim}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// withConditionsInMemory gives the test a condition store in memory, as the serving controller's.
|
||||||
|
func withConditionsInMemory(t *testing.T) (*conditions.Keeper, *conditions.InMemory) {
|
||||||
|
t.Helper()
|
||||||
|
store := conditions.NewInMemory()
|
||||||
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||||
|
before := conditionsFrom
|
||||||
|
conditionsFrom = k
|
||||||
|
t.Cleanup(func() {
|
||||||
|
conditionsFrom = before
|
||||||
|
k.Close(context.Background())
|
||||||
|
})
|
||||||
|
return k, store
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package main
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
@@ -59,9 +60,13 @@ func catchingUpOnMerges(ctx context.Context, open *stores, announced merges) {
|
|||||||
return
|
return
|
||||||
case <-tick.C:
|
case <-tick.C:
|
||||||
}
|
}
|
||||||
|
watchedMerges.begin()
|
||||||
err := catchUpOnMerges(ctx, time.Now(), announced, catalogued, f.SourceMoved, func(format string, args ...any) {
|
err := catchUpOnMerges(ctx, time.Now(), announced, catalogued, f.SourceMoved, func(format string, args ...any) {
|
||||||
fmt.Printf(format+"\n", args...)
|
fmt.Printf(format+"\n", args...)
|
||||||
})
|
})
|
||||||
|
// What the pass found is what S5 says (novox/hq to-be 45 §3); a pass that could not read
|
||||||
|
// says that instead, and leaves what the last one found standing.
|
||||||
|
watchedMerges.end(time.Now(), err)
|
||||||
// A pass that cannot read says so once, not every five minutes, and says when it reads again.
|
// A pass that cannot read says so once, not every five minutes, and says when it reads again.
|
||||||
why := ""
|
why := ""
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -114,6 +119,8 @@ func catchUpOnMerges(ctx context.Context, now time.Time, announced merges,
|
|||||||
for _, e := range moves {
|
for _, e := range moves {
|
||||||
names = append(names, e.Manifest.Module)
|
names = append(names, e.Manifest.Module)
|
||||||
}
|
}
|
||||||
|
watchedMerges.found(missedMerge{Owner: a.Owner, Repo: a.Repo, Base: a.Base, Commit: a.Commit,
|
||||||
|
At: a.At, Modules: names})
|
||||||
say("%s/%s merged into %s (%.8s), announced %s ago, and the controller never acted on it: the bus "+
|
say("%s/%s merged into %s (%.8s), announced %s ago, and the controller never acted on it: the bus "+
|
||||||
"did not hand the announcement over (novox/hq issue 266). %s %s behind it; acting on it now",
|
"did not hand the announcement over (novox/hq issue 266). %s %s behind it; acting on it now",
|
||||||
a.Owner, a.Repo, a.Base, a.Commit, now.Sub(a.At).Round(time.Minute), readableList(names),
|
a.Owner, a.Repo, a.Base, a.Commit, now.Sub(a.At).Round(time.Minute), readableList(names),
|
||||||
@@ -129,3 +136,54 @@ func catchUpOnMerges(ctx context.Context, now time.Time, announced merges,
|
|||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// missedMerge is one merge the bus announced and never handed over, as a pass found it.
|
||||||
|
type missedMerge struct {
|
||||||
|
Owner, Repo, Base, Commit string
|
||||||
|
// At is when the bus took the announcement.
|
||||||
|
At time.Time
|
||||||
|
Modules []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// mergeWatch is what the passes found, for S5 (novox/hq to-be 45 §3): **a merge nothing read is
|
||||||
|
// said**, urgent, even though the pass acts on it at once — the bus skipping a message is a fault of
|
||||||
|
// the transport the mesh's every change rides on, and acting late is the repair, not the absence of
|
||||||
|
// the fault. The next pass, finding it acted on, clears it.
|
||||||
|
type mergeWatch struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
passed time.Time
|
||||||
|
err error
|
||||||
|
finding []missedMerge
|
||||||
|
missed []missedMerge
|
||||||
|
}
|
||||||
|
|
||||||
|
var watchedMerges = &mergeWatch{}
|
||||||
|
|
||||||
|
func (w *mergeWatch) begin() {
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
w.finding = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *mergeWatch) found(m missedMerge) {
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
w.finding = append(w.finding, m)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *mergeWatch) end(at time.Time, err error) {
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
w.passed, w.err = at, err
|
||||||
|
if err == nil {
|
||||||
|
w.missed = w.finding
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// last is when the last pass ended, what the last pass that read found, and what the last pass
|
||||||
|
// could not read.
|
||||||
|
func (w *mergeWatch) last() (time.Time, []missedMerge, error) {
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
return w.passed, append([]missedMerge(nil), w.missed...), w.err
|
||||||
|
}
|
||||||
|
|||||||
@@ -59,8 +59,19 @@ func moduleCommand(ctx context.Context, args []string) error {
|
|||||||
// `check` needs no mesh, and must not: it is what somebody runs in their own repository before
|
// `check` needs no mesh, and must not: it is what somebody runs in their own repository before
|
||||||
// there is a mesh in reach (novox/hq issue 148). A directory expands to every manifest under it.
|
// there is a mesh in reach (novox/hq issue 148). A directory expands to every manifest under it.
|
||||||
if args[0] == "check" {
|
if args[0] == "check" {
|
||||||
|
set := flag.NewFlagSet("module check", flag.ContinueOnError)
|
||||||
|
// The longest machine name an identity must fit on (novox/hq ADR 0225): a mesh passes its own.
|
||||||
|
longest := set.Int("longest-machine-name", catalogue.DefaultLongestMachine,
|
||||||
|
"judge each module's identity on a machine name this many characters long")
|
||||||
|
given, err := parseAround(set, args[1:])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if *longest < 1 {
|
||||||
|
return errors.New("--longest-machine-name is a length, at least 1")
|
||||||
|
}
|
||||||
var paths []string
|
var paths []string
|
||||||
for _, a := range args[1:] {
|
for _, a := range given {
|
||||||
if info, err := os.Stat(a); err == nil && info.IsDir() {
|
if info, err := os.Stat(a); err == nil && info.IsDir() {
|
||||||
under, err := manifestsUnder(a)
|
under, err := manifestsUnder(a)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -71,7 +82,7 @@ func moduleCommand(ctx context.Context, args []string) error {
|
|||||||
}
|
}
|
||||||
paths = append(paths, a)
|
paths = append(paths, a)
|
||||||
}
|
}
|
||||||
return moduleCheck(paths, os.Stdout)
|
return moduleCheckFor(paths, *longest, os.Stdout)
|
||||||
}
|
}
|
||||||
open, err := openStores(ctx)
|
open, err := openStores(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -387,7 +388,7 @@ func brokerCommand(ctx context.Context, args []string) error {
|
|||||||
return busAccounts(ctx, args[1:])
|
return busAccounts(ctx, args[1:])
|
||||||
}
|
}
|
||||||
if len(args) > 0 && args[0] == "consumer-reset" {
|
if len(args) > 0 && args[0] == "consumer-reset" {
|
||||||
return consumerReset(args[1:])
|
return consumerReset(ctx, args[1:])
|
||||||
}
|
}
|
||||||
if len(args) == 0 || args[0] != "show" {
|
if len(args) == 0 || args[0] != "show" {
|
||||||
return errors.New("broker show | broker certificate [--check] --into <directory> | broker accounts --into <file> | " +
|
return errors.New("broker show | broker certificate [--check] --into <directory> | broker accounts --into <file> | " +
|
||||||
@@ -414,10 +415,21 @@ func brokerCommand(ctx context.Context, args []string) error {
|
|||||||
// consumerReset re-makes one consumer on a stream that keeps history to start from now (novox/hq issue
|
// consumerReset re-makes one consumer on a stream that keeps history to start from now (novox/hq issue
|
||||||
// 248): the way out of a consumer replaying a week of announcements, said rather than done by hand. A
|
// 248): the way out of a consumer replaying a week of announcements, said rather than done by hand. A
|
||||||
// person's act — what was pending is dropped — so it is a command, and nothing calls it on its own.
|
// person's act — what was pending is dropped — so it is a command, and nothing calls it on its own.
|
||||||
func consumerReset(args []string) error {
|
func consumerReset(ctx context.Context, args []string) error {
|
||||||
if len(args) != 2 {
|
set := flag.NewFlagSet("broker consumer-reset", flag.ContinueOnError)
|
||||||
return errors.New("broker consumer-reset <stream> <consumer>, e.g. broker consumer-reset EVENTS controller")
|
// A repair by hand, which says why (novox/hq to-be 45 §7).
|
||||||
|
why := addHandActFlags(set)
|
||||||
|
args, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
|
if len(args) != 2 {
|
||||||
|
return errors.New("broker consumer-reset <stream> <consumer> --why <text>, e.g. broker consumer-reset EVENTS controller --why ...")
|
||||||
|
}
|
||||||
|
if err := why.require("broker consumer-reset"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
why.record(ctx, "broker consumer-reset", args)
|
||||||
address, err := broker.BusAddress()
|
address, err := broker.BusAddress()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -505,15 +517,22 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
|
|||||||
fmt.Printf(" public domain %s\n", domain)
|
fmt.Printf(" public domain %s\n", domain)
|
||||||
}
|
}
|
||||||
|
|
||||||
// A provider here failing a consumer, or a consumer here failed (novox/hq ADR 0224). Before the
|
// Every open condition about this machine (novox/hq to-be 45 §2) — a provider here failing a
|
||||||
// capabilities, because it is something not working now and they are a description.
|
// consumer, or a consumer here failed, among them (ADR 0224). Before the capabilities, because it
|
||||||
failing, err := failingProviders(ctx, inv)
|
// is something not working now and they are a description. Unreadable is said, not passed over.
|
||||||
|
open, err := openConditions(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
fmt.Printf("\n the open conditions could NOT be read, so whether anything here is wrong is not known: %v\n", err)
|
||||||
}
|
}
|
||||||
if here := failingOn(failing, name); len(here) > 0 {
|
var here []conditions.Condition
|
||||||
fmt.Printf("\n %d consumer(s) a provider keeps failing, here or for a module here:\n", len(here))
|
for _, c := range open {
|
||||||
for _, line := range failingLines(here, time.Now()) {
|
if concerns(c, name) {
|
||||||
|
here = append(here, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(here) > 0 {
|
||||||
|
fmt.Printf("\n %d open condition(s) about this machine:\n", len(here))
|
||||||
|
for _, line := range conditionLines(here, time.Now()) {
|
||||||
fmt.Printf(" %s\n", line)
|
fmt.Printf(" %s\n", line)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+50
-16
@@ -399,7 +399,7 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
|||||||
}
|
}
|
||||||
out := sendable{Resources: composed.Resources, Adoption: adoption,
|
out := sendable{Resources: composed.Resources, Adoption: adoption,
|
||||||
Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
|
Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
|
||||||
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut}
|
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld}
|
||||||
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
|
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
|
||||||
// 0221). Read only on the send path: a question about what would be sent records nothing.
|
// 0221). Read only on the send path: a question about what would be sent records nothing.
|
||||||
if choosing == Allocating {
|
if choosing == Allocating {
|
||||||
@@ -464,6 +464,11 @@ func reportLeftOut(node string, declared sendable) {
|
|||||||
"what the machine holds for it is kept and its containers are untouched. %s\n",
|
"what the machine holds for it is kept and its containers are untouched. %s\n",
|
||||||
node, m, declared.leftOutWhy[m])
|
node, m, declared.leftOutWhy[m])
|
||||||
}
|
}
|
||||||
|
// And whom it serves nothing, because their identity overflows what the provision keeps (ADR
|
||||||
|
// 0225): the machine is sent everything else, and the consumer is named.
|
||||||
|
for _, o := range declared.withheld {
|
||||||
|
fmt.Printf("%s: %s\n", node, o)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// renderingFor is everything a node's declaration is composed with, and the node's record.
|
// renderingFor is everything a node's declaration is composed with, and the node's record.
|
||||||
@@ -471,7 +476,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
|||||||
plan catalogue.Resolution, settings catalogue.SettingsBy,
|
plan catalogue.Resolution, settings catalogue.SettingsBy,
|
||||||
gens map[string]catalogue.Generator, choosing Choosing) (catalogue.Rendering, inventory.Node, error) {
|
gens map[string]catalogue.Generator, choosing Choosing) (catalogue.Rendering, inventory.Node, error) {
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
grants, err := grantsFor(ctx, open, node)
|
grants, withheld, err := grantsFor(ctx, open, node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return catalogue.Rendering{}, inventory.Node{}, err
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
}
|
}
|
||||||
@@ -794,7 +799,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
|||||||
Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
|
Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
|
||||||
Kept: kept, Adopted: record.Adopted, OutwardLinks: outwardLinks,
|
Kept: kept, Adopted: record.Adopted, OutwardLinks: outwardLinks,
|
||||||
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, SeatReach: reach, Built: built,
|
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, SeatReach: reach, Built: built,
|
||||||
BusUsers: busUsers,
|
BusUsers: busUsers, Withheld: withheld,
|
||||||
}, record, nil
|
}, record, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -930,28 +935,34 @@ func certificateFor(ctx context.Context, open *stores, node string) (string, str
|
|||||||
// The mirror of what a consumer is given, and the half that makes the credential real: a password
|
// The mirror of what a consumer is given, and the half that makes the credential real: a password
|
||||||
// nothing was told to create is a password that authenticates nowhere. Sealed to this node, so
|
// nothing was told to create is a password that authenticates nowhere. Sealed to this node, so
|
||||||
// the mesh hands over something it cannot itself use.
|
// the mesh hands over something it cannot itself use.
|
||||||
func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Grant, error) {
|
//
|
||||||
|
// **One consumer's identity never refuses the provider's machine** (novox/hq ADR 0225, issue 263).
|
||||||
|
// A consumer whose identity overflows the provision's bound is left out of the grants and returned
|
||||||
|
// beside them, for push, plan and `status` to say; every other consumer is granted and the provider's
|
||||||
|
// declaration composes. Refusing here once made a whole machine unpushable for one module elsewhere.
|
||||||
|
func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Grant, []catalogue.Overflow, error) {
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
issued, err := inv.SecretsFrom(ctx, node)
|
issued, err := inv.SecretsFrom(ctx, node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Where each consumer is, so a provider that must reach back to one does not have to know how
|
// Where each consumer is, so a provider that must reach back to one does not have to know how
|
||||||
// the mesh names machines.
|
// the mesh names machines.
|
||||||
shelf, err := inv.Catalogue(ctx)
|
shelf, err := inv.Catalogue(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
|
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// What each consumer actually asked for, taken from that machine's own resolution rather than
|
// What each consumer actually asked for, taken from that machine's own resolution rather than
|
||||||
// from a record beside it. A provider told to create a password and not what to create it for
|
// from a record beside it. A provider told to create a password and not what to create it for
|
||||||
// can do nothing with it, and the name a consumer wants is the consumer's to say.
|
// can do nothing with it, and the name a consumer wants is the consumer's to say.
|
||||||
out := make([]catalogue.Grant, 0, len(issued))
|
out := make([]catalogue.Grant, 0, len(issued))
|
||||||
|
var withheld []catalogue.Overflow
|
||||||
for _, s := range issued {
|
for _, s := range issued {
|
||||||
plan, settings, err := planFor(ctx, open, s.Consumer)
|
plan, settings, err := planFor(ctx, open, s.Consumer)
|
||||||
switch {
|
switch {
|
||||||
@@ -964,11 +975,11 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
|||||||
// The mesh could not be asked what they wanted, which is not the same as their wanting
|
// The mesh could not be asked what they wanted, which is not the same as their wanting
|
||||||
// nothing — and withholding a grant on that reading takes a consumer's access away
|
// nothing — and withholding a grant on that reading takes a consumer's access away
|
||||||
// (novox/hq 04-ISSUES/152).
|
// (novox/hq 04-ISSUES/152).
|
||||||
return nil, fmt.Errorf("what %s asked of %s cannot be read: %w", s.Consumer, s.Name, err)
|
return nil, nil, fmt.Errorf("what %s asked of %s cannot be read: %w", s.Consumer, s.Name, err)
|
||||||
}
|
}
|
||||||
values, asks, err := plan.ContributionsFrom(s.Name, s.ConsumerModule, settings)
|
values, asks, err := plan.ContributionsFrom(s.Name, s.ConsumerModule, settings)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
// A port in there is the consumer's software port until this. The consumer is on another
|
// A port in there is the consumer's software port until this. The consumer is on another
|
||||||
// machine, so the assignment that moved it is that machine's — fetched here rather than
|
// machine, so the assignment that moved it is that machine's — fetched here rather than
|
||||||
@@ -976,7 +987,7 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
|||||||
// this case (novox/hq 04-ISSUES/038, the cross-node half).
|
// this case (novox/hq 04-ISSUES/038, the cross-node half).
|
||||||
published, err := portsOn(ctx, inv, s.Consumer, s.ConsumerModule)
|
published, err := portsOn(ctx, inv, s.Consumer, s.ConsumerModule)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
values = catalogue.AtPublishedPort(values, s.ConsumerModule, published)
|
values = catalogue.AtPublishedPort(values, s.ConsumerModule, published)
|
||||||
from := s.ConsumerModule
|
from := s.ConsumerModule
|
||||||
@@ -987,9 +998,10 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
|||||||
from = ""
|
from = ""
|
||||||
}
|
}
|
||||||
// The consumer's identity slug, from its own manifest, carried on the grant so the provider
|
// The consumer's identity slug, from its own manifest, carried on the grant so the provider
|
||||||
// derives the same login the consumer does (novox/hq ADR 0049). Refused here if it still would
|
// derives the same login the consumer does (novox/hq ADR 0049). Judged against the bound of
|
||||||
// not fit the tightest backend — the mesh chose the name, so the mesh refuses it, with the
|
// this provision, as the consumer's resolution states it from the provider's offer (ADR
|
||||||
// remedy a short slug rather than a login a provider silently shortened.
|
// 0225): a consumer it would not fit is left out of the grants and said, rather than a login a
|
||||||
|
// provider silently shortened — and rather than this whole machine refused for it.
|
||||||
slug := ""
|
slug := ""
|
||||||
for _, mm := range plan.Modules {
|
for _, mm := range plan.Modules {
|
||||||
if mm.Module == s.ConsumerModule {
|
if mm.Module == s.ConsumerModule {
|
||||||
@@ -998,15 +1010,32 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if from != "" {
|
if from != "" {
|
||||||
if err := catalogue.CheckIdentity(s.Consumer, catalogue.IdentitySource(slug, s.ConsumerModule)); err != nil {
|
bound := boundOfGrant(plan, s, node)
|
||||||
return nil, err
|
source := catalogue.IdentitySource(slug, s.ConsumerModule)
|
||||||
|
if catalogue.CheckIdentityWithin(s.Consumer, source, bound) != nil {
|
||||||
|
withheld = append(withheld, catalogue.Overflow{Provision: s.Name, Provider: node,
|
||||||
|
Consumer: s.Consumer, Module: s.ConsumerModule,
|
||||||
|
Identity: catalogue.ConsumerIdentity(s.Consumer, source), Bound: bound})
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
out = append(out, catalogue.Grant{
|
out = append(out, catalogue.Grant{
|
||||||
Provision: s.Name, Consumer: s.Consumer, At: onNetwork[s.Consumer],
|
Provision: s.Name, Consumer: s.Consumer, At: onNetwork[s.Consumer],
|
||||||
From: from, Values: values, Slug: slug, Sealed: s.ForProvider, Local: s.Local})
|
From: from, Values: values, Slug: slug, Sealed: s.ForProvider, Local: s.Local})
|
||||||
}
|
}
|
||||||
return out, nil
|
return out, withheld, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// boundOfGrant is the identity bound the consumer's own resolution states for the requirement this
|
||||||
|
// grant answers. A requirement not found there is held to the tightest bound the mesh knows rather
|
||||||
|
// than to none: what the provider keeps of it is not known here.
|
||||||
|
func boundOfGrant(consumer catalogue.Resolution, s inventory.Secret, provider string) catalogue.IdentityBound {
|
||||||
|
for _, n := range consumer.Needs {
|
||||||
|
if n.Name == s.Name && n.For == s.ConsumerModule && n.From == provider {
|
||||||
|
return n.Identity
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return catalogue.DefaultIdentityBound
|
||||||
}
|
}
|
||||||
|
|
||||||
// listensLines is what a person is told about what this module would open, and why — the same
|
// listensLines is what a person is told about what this module would open, and why — the same
|
||||||
@@ -1082,6 +1111,11 @@ func planCommand(ctx context.Context, args []string) error {
|
|||||||
left := plan.LeftOut(settings, record.Adopted)
|
left := plan.LeftOut(settings, record.Adopted)
|
||||||
reportLeftOut(args[0], sendable{LeftOut: sortedKeysOf(left), leftOutWhy: left})
|
reportLeftOut(args[0], sendable{LeftOut: sortedKeysOf(left), leftOutWhy: left})
|
||||||
}
|
}
|
||||||
|
// And which of its modules no provider will grant, because the identity overflows the bound of
|
||||||
|
// what it requires (novox/hq ADR 0225) — said on the machine the remedy is for.
|
||||||
|
for _, o := range plan.Overflowing() {
|
||||||
|
fmt.Printf("%s: %s\n", args[0], o)
|
||||||
|
}
|
||||||
// And a setting that reaches nothing — refused where it is stored, and said here for one
|
// And a setting that reaches nothing — refused where it is stored, and said here for one
|
||||||
// stored before its definition moved from under it.
|
// stored before its definition moved from under it.
|
||||||
for _, m := range plan.Modules {
|
for _, m := range plan.Modules {
|
||||||
|
|||||||
@@ -0,0 +1,820 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"regexp"
|
||||||
|
"slices"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/micro"
|
||||||
|
"github.com/novox/mesh-host/validate"
|
||||||
|
"golang.org/x/net/dns/dnsmessage"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/artifacts"
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
"github.com/novox/mesh-controller/internal/overlay"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The probes of the self-check's registry (doctor.go), one function each. A probe answers what is
|
||||||
|
// wrong now as observations, or an error when it could not tell — never "nothing" for "could not
|
||||||
|
// look" (ADR 0227 rule 4).
|
||||||
|
|
||||||
|
// probeDeclarations is D1: every machine's declaration composes, and the node-engine's own validator
|
||||||
|
// (mesh-host's `validate`, the package the host runs) takes it.
|
||||||
|
func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
open := d.open
|
||||||
|
nodes, err := open.inventory.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var out []conditions.Observation
|
||||||
|
// The private network every declaration is composed with. Not computable is not "could not
|
||||||
|
// look": it is the finding — no machine's declaration composes without it — and the machines that
|
||||||
|
// do not resolve, which are usually why, are named below.
|
||||||
|
gens, gensErr := generators(ctx, open)
|
||||||
|
if gensErr != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMesh, ID: "private-network",
|
||||||
|
Token: "uncomposable", Severity: conditions.Urgent,
|
||||||
|
Summary: "the private network cannot be computed, so no machine's declaration composes",
|
||||||
|
Said: firstLine(gensErr.Error())})
|
||||||
|
}
|
||||||
|
for _, n := range nodes {
|
||||||
|
plan, settings, err := planFor(ctx, open, n.Name)
|
||||||
|
if err != nil && !unresolvable(err) {
|
||||||
|
return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err)
|
||||||
|
}
|
||||||
|
var problems []string
|
||||||
|
if err == nil && gensErr == nil {
|
||||||
|
var declared sendable
|
||||||
|
if declared, err = declarationWith(ctx, open, n.Name, plan, settings, gens, Reading); err == nil {
|
||||||
|
var body []byte
|
||||||
|
if body, err = declared.Body(); err == nil {
|
||||||
|
problems = validate.Declaration(body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "uncomposable",
|
||||||
|
Machine: n.Name, Severity: conditions.Urgent,
|
||||||
|
Summary: fmt.Sprintf("nothing can be sent to %s: its declaration does not compose — %s", n.Name,
|
||||||
|
oneLine(err.Error())),
|
||||||
|
Said: oneLine(err.Error())})
|
||||||
|
case len(problems) > 0:
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "refused",
|
||||||
|
Machine: n.Name, Severity: conditions.Urgent,
|
||||||
|
Summary: fmt.Sprintf("%s's node-engine would refuse its declaration whole: %d problem(s), the first: %s",
|
||||||
|
n.Name, len(problems), problems[0]),
|
||||||
|
Said: strings.Join(problems, "; ")})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeResolvers is D2: every holder of the mesh's resolver answers each machine's name with its
|
||||||
|
// address for IPv4, and with no address and no error for IPv6 — NODATA, not NXDOMAIN, which musl
|
||||||
|
// takes as final (issue 262).
|
||||||
|
func probeResolvers(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
inv := d.open.inventory
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
holders, err := replicatedHolders(ctx, inv, shelf)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
resolvers := holders["mesh-dns-resolver"]
|
||||||
|
if len(resolvers) == 0 {
|
||||||
|
return nil, nil // the mesh holds no resolver of its own: nothing is asked of one
|
||||||
|
}
|
||||||
|
places, err := onTheNetwork(ctx, inv, shelf)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
suffix := overlay.Suffix()
|
||||||
|
var out []conditions.Observation
|
||||||
|
for holder, at := range resolvers {
|
||||||
|
var wrong []string
|
||||||
|
for _, p := range places {
|
||||||
|
if p.Address == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
name := p.Name + "." + suffix
|
||||||
|
v4, rcode, err := askResolver(ctx, at, name, dnsmessage.TypeA)
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
wrong = append(wrong, fmt.Sprintf("%s for %s: %v", "A", name, err))
|
||||||
|
continue
|
||||||
|
case rcode != dnsmessage.RCodeSuccess:
|
||||||
|
wrong = append(wrong, fmt.Sprintf("%s answered %s for %s (A)", holder, rcode, name))
|
||||||
|
case !slices.Contains(v4, p.Address):
|
||||||
|
wrong = append(wrong, fmt.Sprintf("%s answered %v for %s (A), not %s", holder, v4, name, p.Address))
|
||||||
|
}
|
||||||
|
v6, rcode, err := askResolver(ctx, at, name, dnsmessage.TypeAAAA)
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
wrong = append(wrong, fmt.Sprintf("%s for %s: %v", "AAAA", name, err))
|
||||||
|
case rcode != dnsmessage.RCodeSuccess:
|
||||||
|
wrong = append(wrong, fmt.Sprintf("%s answered %s for %s (AAAA), not NODATA: a musl machine "+
|
||||||
|
"takes that as no such name", holder, rcode, name))
|
||||||
|
case len(v6) > 0:
|
||||||
|
wrong = append(wrong, fmt.Sprintf("%s answered %v for %s (AAAA); the mesh has no IPv6 addresses", holder, v6, name))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(wrong) > 0 {
|
||||||
|
node := strings.TrimSuffix(holder, "."+suffix)
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeSeat, ID: "mesh-dns-resolver." + node,
|
||||||
|
Token: "wrong", Machine: node, Severity: conditions.Urgent,
|
||||||
|
Summary: fmt.Sprintf("the mesh's resolver on %s does not answer machine names as it must: %s",
|
||||||
|
node, wrong[0]),
|
||||||
|
Said: strings.Join(wrong, "; ")})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sortedFound(out), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolverPort is where a resolver answers; a variable so a test can stand one up.
|
||||||
|
var resolverPort = "53"
|
||||||
|
|
||||||
|
// askResolver asks one resolver one question over UDP, and answers the addresses and the code.
|
||||||
|
func askResolver(ctx context.Context, at, name string, kind dnsmessage.Type) ([]string, dnsmessage.RCode, error) {
|
||||||
|
q, err := dnsmessage.NewName(strings.TrimSuffix(name, ".") + ".")
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
msg := dnsmessage.Message{Header: dnsmessage.Header{ID: uint16(time.Now().UnixNano()), RecursionDesired: true},
|
||||||
|
Questions: []dnsmessage.Question{{Name: q, Type: kind, Class: dnsmessage.ClassINET}}}
|
||||||
|
packed, err := msg.Pack()
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
dialer := net.Dialer{Timeout: 3 * time.Second}
|
||||||
|
conn, err := dialer.DialContext(ctx, "udp", net.JoinHostPort(at, resolverPort))
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
_ = conn.SetDeadline(time.Now().Add(3 * time.Second))
|
||||||
|
if _, err := conn.Write(packed); err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
buf := make([]byte, 1500)
|
||||||
|
n, err := conn.Read(buf)
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, fmt.Errorf("no answer from %s: %w", at, err)
|
||||||
|
}
|
||||||
|
var answer dnsmessage.Message
|
||||||
|
if err := answer.Unpack(buf[:n]); err != nil {
|
||||||
|
return nil, 0, fmt.Errorf("an answer from %s that cannot be read: %w", at, err)
|
||||||
|
}
|
||||||
|
var addresses []string
|
||||||
|
for _, a := range answer.Answers {
|
||||||
|
switch r := a.Body.(type) {
|
||||||
|
case *dnsmessage.AResource:
|
||||||
|
addresses = append(addresses, net.IP(r.A[:]).String())
|
||||||
|
case *dnsmessage.AAAAResource:
|
||||||
|
addresses = append(addresses, net.IP(r.AAAA[:]).String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return addresses, answer.RCode, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeHolders is D3: every seat that serves verbs has, on every machine that holds it and is heard
|
||||||
|
// from, a holder answering the bus's discovery for that seat. A machine past its heartbeat's bound is
|
||||||
|
// S1's, and is not asked about here.
|
||||||
|
func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
entries, err := d.open.inventory.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
recorded, err := d.open.inventory.Holdings(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
heard := heardMachines(d)
|
||||||
|
expected := map[string]map[string]bool{} // seat → machine
|
||||||
|
add := func(seat, node string) {
|
||||||
|
if expected[seat] == nil {
|
||||||
|
expected[seat] = map[string]bool{}
|
||||||
|
}
|
||||||
|
expected[seat][node] = true
|
||||||
|
}
|
||||||
|
for _, s := range catalogue.SeatsWithAProtocol() {
|
||||||
|
if len(s.Serves) == 0 || s.Name == catalogue.ControllerSeatName {
|
||||||
|
continue // a seat with no verb has nothing to answer with; this controller is answering now
|
||||||
|
}
|
||||||
|
onRecord := false
|
||||||
|
for _, h := range recorded {
|
||||||
|
if h.Claim == s.Name && heard[h.Node] {
|
||||||
|
add(s.Name, h.Node)
|
||||||
|
onRecord = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if onRecord && s.Scope == catalogue.ScopeMesh {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, e := range entries {
|
||||||
|
for _, c := range e.Manifest.Claims {
|
||||||
|
if c.Name != s.Name {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, node := range e.On {
|
||||||
|
if heard[node] && (s.Scope == catalogue.ScopeNode || !onRecord) {
|
||||||
|
add(s.Name, node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(expected) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
answering, err := discoverHolders(ctx, d.js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var out []conditions.Observation
|
||||||
|
for seat, nodes := range expected {
|
||||||
|
for node := range nodes {
|
||||||
|
if answering[seat][node] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeSeat, ID: seat + "." + node,
|
||||||
|
Token: "silent", Machine: node, Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s's holder on %s does not answer the bus: its verbs reach nothing there", seat, node),
|
||||||
|
Said: fmt.Sprintf("no answer for %s from %s to the bus's discovery", seat, node)})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sortedFound(out), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// heardMachines is every machine within its heartbeat's bound, as the watchdogs last saw.
|
||||||
|
func heardMachines(d *doctor) map[string]bool {
|
||||||
|
out := map[string]bool{}
|
||||||
|
if d.watchdogs == nil {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
f := d.watchdogs.lastFacts()
|
||||||
|
if f == nil {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
for _, m := range f.machines {
|
||||||
|
if !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) <= heartbeatBound(m.every) && !m.asleep() {
|
||||||
|
out[m.name] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// discoveryPatience is how long the discovery's answers are waited for after the last arrived, and at
|
||||||
|
// the most (ADR 0197: a large runtime's answer arrives last).
|
||||||
|
const (
|
||||||
|
discoveryQuiet = 1500 * time.Millisecond
|
||||||
|
discoveryPatience = 8 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
// discoverHolders asks the bus's discovery who serves what, and answers seat → machine for every
|
||||||
|
// endpoint a seat's verb is served on.
|
||||||
|
func discoverHolders(ctx context.Context, conn *nats.Conn) (map[string]map[string]bool, error) {
|
||||||
|
inbox := conn.NewRespInbox()
|
||||||
|
sub, err := conn.SubscribeSync(inbox)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer func() { _ = sub.Unsubscribe() }()
|
||||||
|
if err := conn.PublishRequest("$SRV.INFO", inbox, nil); err != nil {
|
||||||
|
return nil, fmt.Errorf("asking the bus who serves what: %w", err)
|
||||||
|
}
|
||||||
|
out := map[string]map[string]bool{}
|
||||||
|
deadline := time.Now().Add(discoveryPatience)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
wait, cancel := context.WithTimeout(ctx, discoveryQuiet)
|
||||||
|
msg, err := sub.NextMsgWithContext(wait)
|
||||||
|
cancel()
|
||||||
|
if err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
var info micro.Info
|
||||||
|
if json.Unmarshal(msg.Data, &info) != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, e := range info.Endpoints {
|
||||||
|
seat, node := e.Metadata["seat"], e.Metadata["node"]
|
||||||
|
if seat == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if node == "" {
|
||||||
|
node = info.ID
|
||||||
|
}
|
||||||
|
if out[seat] == nil {
|
||||||
|
out[seat] = map[string]bool{}
|
||||||
|
}
|
||||||
|
out[seat][node] = true
|
||||||
|
}
|
||||||
|
// A holder that announces itself as its seat, by name and machine.
|
||||||
|
if out[info.Name] == nil {
|
||||||
|
out[info.Name] = map[string]bool{}
|
||||||
|
}
|
||||||
|
out[info.Name][info.ID] = true
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeArchives is D4: every archive the mesh keeps is held by its manifest in the artifact store.
|
||||||
|
func probeArchives(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
inv := d.open.inventory
|
||||||
|
kept, err := inv.KeptArchives(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(kept) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
store, err := artifactStoreAddress(ctx, inv, shelf, "")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if store == "" {
|
||||||
|
return nil, errors.New("the mesh keeps archives and has no artifact store on its network to ask")
|
||||||
|
}
|
||||||
|
var report collectionReport
|
||||||
|
if unasked, stopped := askHeld(ctx, artifacts.Store{Address: store}, kept, &report); stopped != "" {
|
||||||
|
return nil, fmt.Errorf("%d kept archive(s) could not be asked about: %s", unasked, stopped)
|
||||||
|
}
|
||||||
|
var out []conditions.Observation
|
||||||
|
if n := len(report.Unheld); n > 0 {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMesh, ID: "artifact-store", Token: "archives-unheld",
|
||||||
|
Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%d of %d kept archive(s) are not held by a manifest: the store's collector would "+
|
||||||
|
"delete them", n, len(kept)),
|
||||||
|
Said: "first: " + report.Unheld[0]})
|
||||||
|
}
|
||||||
|
if n := len(report.Missing); n > 0 {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMesh, ID: "artifact-store", Token: "archives-missing",
|
||||||
|
Kind: "archives-missing", Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%d kept archive(s) are not in the artifact store at all", n),
|
||||||
|
Said: "first: " + report.Missing[0]})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// consumerFarBehind is how far a durable consumer may be from its stream's head (D6).
|
||||||
|
const consumerFarBehind = 1000
|
||||||
|
|
||||||
|
// probeConsumers is D6: every durable consumer the mesh expects exists, as the controller defines it,
|
||||||
|
// and is near its stream's head.
|
||||||
|
func probeConsumers(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
_, consumers, err := expectedBusObjects(ctx, d.open.inventory)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
js := d.js.Context()
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, c := range consumers {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
info, err := js.ConsumerInfo(c.Stream, c.Name, nats.Context(ctx))
|
||||||
|
who := c.Stream + "." + c.Name
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, nats.ErrConsumerNotFound) || errors.Is(err, nats.ErrStreamNotFound):
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "missing",
|
||||||
|
Kind: "consumer-lost", Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s is not on the bus: what it delivers reaches nobody", consumerWords(c)),
|
||||||
|
Said: "no consumer " + c.Name + " on " + c.Stream})
|
||||||
|
continue
|
||||||
|
case err != nil:
|
||||||
|
return nil, fmt.Errorf("%s cannot be read: %w", consumerWords(c), err)
|
||||||
|
}
|
||||||
|
if differs := consumerDiffers(c, info.Config); differs != "" {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "redefined",
|
||||||
|
Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s is not as the controller defines it: %s", consumerWords(c), differs),
|
||||||
|
Said: differs})
|
||||||
|
}
|
||||||
|
if c.Stream != "NODES" && info.NumPending > consumerFarBehind {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "behind",
|
||||||
|
Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s is %d message(s) behind its stream's head", consumerWords(c), info.NumPending),
|
||||||
|
Said: fmt.Sprintf("%d pending, %d handed out and not settled", info.NumPending, info.NumAckPending)})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sortedFound(out), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// consumerWords is a consumer as the mesh says it, with its name.
|
||||||
|
func consumerWords(c broker.Consumer) string {
|
||||||
|
return fmt.Sprintf("%s (%s on %s)", link.ConsumerInWords(c.Stream, c.Name), c.Name, c.Stream)
|
||||||
|
}
|
||||||
|
|
||||||
|
// consumerDiffers is what about a consumer on the bus is not as defined; empty when nothing is. The
|
||||||
|
// delivery subject is not compared: one kept as it was while a holder is bound is the assertion's
|
||||||
|
// stated choice (novox/hq issue 156).
|
||||||
|
func consumerDiffers(want broker.Consumer, have nats.ConsumerConfig) string {
|
||||||
|
var differs []string
|
||||||
|
haveFilters := append([]string(nil), have.FilterSubjects...)
|
||||||
|
if have.FilterSubject != "" {
|
||||||
|
haveFilters = append(haveFilters, have.FilterSubject)
|
||||||
|
}
|
||||||
|
wantFilters := append([]string(nil), want.Filters...)
|
||||||
|
sort.Strings(haveFilters)
|
||||||
|
sort.Strings(wantFilters)
|
||||||
|
if !slices.Equal(haveFilters, wantFilters) {
|
||||||
|
differs = append(differs, fmt.Sprintf("filters %v, defined %v", haveFilters, wantFilters))
|
||||||
|
}
|
||||||
|
if have.AckPolicy != nats.AckExplicitPolicy {
|
||||||
|
differs = append(differs, "acknowledges "+have.AckPolicy.String()+", defined explicit")
|
||||||
|
}
|
||||||
|
if wantMax := want.MaxDeliver; wantMax != 0 && have.MaxDeliver != wantMax {
|
||||||
|
differs = append(differs, fmt.Sprintf("hands a message over %d times, defined %d", have.MaxDeliver, wantMax))
|
||||||
|
}
|
||||||
|
if wantWait := time.Duration(want.AckWaitSeconds) * time.Second; wantWait != 0 && have.AckWait != wantWait {
|
||||||
|
differs = append(differs, fmt.Sprintf("waits %s for an acknowledgement, defined %s", have.AckWait, wantWait))
|
||||||
|
}
|
||||||
|
if want.MaxAckPending != 0 && have.MaxAckPending != want.MaxAckPending {
|
||||||
|
differs = append(differs, fmt.Sprintf("hands out %d at once, defined %d", have.MaxAckPending, want.MaxAckPending))
|
||||||
|
}
|
||||||
|
if wantPush, havePush := want.Push || want.Queue != "", have.DeliverSubject != ""; wantPush != havePush {
|
||||||
|
differs = append(differs, "delivers by the other shape (push or pull) than defined")
|
||||||
|
}
|
||||||
|
return strings.Join(differs, "; ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeStreams is D7: every stream the controller defines exists as defined, and its own buckets.
|
||||||
|
func probeStreams(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
streams, _, err := expectedBusObjects(ctx, d.open.inventory)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
js := d.js.Context()
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, s := range streams {
|
||||||
|
info, err := js.StreamInfo(s.Name, nats.Context(ctx))
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, nats.ErrStreamNotFound):
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: s.Name, Token: "stream-missing",
|
||||||
|
Kind: "stream-wrong", Severity: conditions.Urgent,
|
||||||
|
Summary: fmt.Sprintf("the stream %s is not on the bus: %s", s.Name, firstLine(s.Why)),
|
||||||
|
Said: "no stream " + s.Name})
|
||||||
|
continue
|
||||||
|
case err != nil:
|
||||||
|
return nil, fmt.Errorf("the stream %s cannot be read: %w", s.Name, err)
|
||||||
|
}
|
||||||
|
if differs := streamDiffers(s, info.Config); differs != "" {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: s.Name, Token: "stream-redefined",
|
||||||
|
Kind: "stream-wrong", Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("the stream %s is not as the controller defines it: %s", s.Name, differs),
|
||||||
|
Said: differs})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, bucket := range broker.ControllerBuckets() {
|
||||||
|
if _, err := js.StreamInfo("KV_"+bucket, nats.Context(ctx)); errors.Is(err, nats.ErrStreamNotFound) {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: bucket, Token: "bucket-missing",
|
||||||
|
Kind: "stream-wrong", Severity: conditions.Urgent,
|
||||||
|
Summary: fmt.Sprintf("the controller's bucket %s is not on the bus: what it keeps there is not kept", bucket),
|
||||||
|
Said: "no bucket " + bucket})
|
||||||
|
} else if err != nil {
|
||||||
|
return nil, fmt.Errorf("the bucket %s cannot be read: %w", bucket, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sortedFound(out), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// streamDiffers is what about a stream on the bus is not as defined; empty when nothing is.
|
||||||
|
func streamDiffers(want broker.Stream, have nats.StreamConfig) string {
|
||||||
|
var differs []string
|
||||||
|
haveSubjects := append([]string(nil), have.Subjects...)
|
||||||
|
wantSubjects := append([]string(nil), want.Subjects...)
|
||||||
|
sort.Strings(haveSubjects)
|
||||||
|
sort.Strings(wantSubjects)
|
||||||
|
if !slices.Equal(haveSubjects, wantSubjects) {
|
||||||
|
differs = append(differs, fmt.Sprintf("subjects %v, defined %v", haveSubjects, wantSubjects))
|
||||||
|
}
|
||||||
|
retention, perSubject := nats.LimitsPolicy, int64(want.MaxMsgsPerSubject)
|
||||||
|
switch want.Retention {
|
||||||
|
case broker.RetentionWorkQueue:
|
||||||
|
retention = nats.WorkQueuePolicy
|
||||||
|
case broker.RetentionLastPerSubject:
|
||||||
|
perSubject = 1
|
||||||
|
}
|
||||||
|
if have.Retention != retention {
|
||||||
|
differs = append(differs, fmt.Sprintf("keeps by %s, defined %s", have.Retention, retention))
|
||||||
|
}
|
||||||
|
if perSubject != 0 && have.MaxMsgsPerSubject != perSubject {
|
||||||
|
differs = append(differs, fmt.Sprintf("keeps %d per subject, defined %d", have.MaxMsgsPerSubject, perSubject))
|
||||||
|
}
|
||||||
|
return strings.Join(differs, "; ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// ipLike finds the addresses in a ban list, whatever shape its holder answers in.
|
||||||
|
var ipLike = regexp.MustCompile(`\b(?:\d{1,3}\.){3}\d{1,3}\b`)
|
||||||
|
|
||||||
|
// probeBans is D8: no address the mesh owns is in any machine's ban list. The mesh's own addresses are
|
||||||
|
// every machine's private address and the address its endpoint names; the ban lists are asked of each
|
||||||
|
// machine's holder of `node-intrusion-prevention` that is heard from.
|
||||||
|
func probeBans(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
inv := d.open.inventory
|
||||||
|
places, err := inv.Overlays(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
owned := map[string]string{} // address → whose
|
||||||
|
for _, p := range places {
|
||||||
|
if p.Address != "" {
|
||||||
|
owned[p.Address] = p.Name + "'s private address"
|
||||||
|
}
|
||||||
|
if host, _, err := net.SplitHostPort(p.Endpoint); err == nil && host != "" {
|
||||||
|
if ip := net.ParseIP(host); ip != nil {
|
||||||
|
owned[ip.String()] = p.Name + "'s endpoint"
|
||||||
|
} else if addrs, err := net.DefaultResolver.LookupHost(ctx, host); err == nil {
|
||||||
|
for _, a := range addrs {
|
||||||
|
owned[a] = p.Name + "'s endpoint (" + host + ")"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
entries, err := inv.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
heard := heardMachines(d)
|
||||||
|
var holders []string
|
||||||
|
for _, e := range entries {
|
||||||
|
for _, c := range e.Manifest.Claims {
|
||||||
|
if c.Name == "node-intrusion-prevention" {
|
||||||
|
for _, node := range e.On {
|
||||||
|
if heard[node] && !slices.Contains(holders, node) {
|
||||||
|
holders = append(holders, node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(holders)
|
||||||
|
var out []conditions.Observation
|
||||||
|
// Every machine asked at once: one after another, four holders that each wait their bound
|
||||||
|
// outlast the probe's thirty seconds, and the probe says nothing about any of them.
|
||||||
|
answers := make([]json.RawMessage, len(holders))
|
||||||
|
errs := make([]error, len(holders))
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for i, node := range holders {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(i int, node string) {
|
||||||
|
defer wg.Done()
|
||||||
|
answers[i], errs[i] = askSeatTool(ctx, d.js.Conn(), "node-intrusion-prevention", "banned", node)
|
||||||
|
}(i, node)
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
var unasked []string
|
||||||
|
for i, node := range holders {
|
||||||
|
answer, err := answers[i], errs[i]
|
||||||
|
if err != nil {
|
||||||
|
unasked = append(unasked, err.Error())
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var banned []string
|
||||||
|
for _, ip := range ipLike.FindAllString(string(answer), -1) {
|
||||||
|
if whose, ours := owned[ip]; ours && !slices.Contains(banned, ip+" ("+whose+")") {
|
||||||
|
banned = append(banned, ip+" ("+whose+")")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(banned) > 0 {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Token: "bans-the-mesh",
|
||||||
|
Machine: node, Severity: conditions.Urgent,
|
||||||
|
Summary: fmt.Sprintf("%s's ban list holds %d of the mesh's own address(es): %s — the mesh is locked out "+
|
||||||
|
"of itself there (ADR 0186)", node, len(banned), strings.Join(banned, ", ")),
|
||||||
|
Said: strings.Join(banned, ", ")})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(unasked) > 0 {
|
||||||
|
return nil, fmt.Errorf("a ban list could not be read: %s", strings.Join(unasked, "; "))
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeStatus is D9: `status` answers in full within ten seconds, from a summary composed lately.
|
||||||
|
func probeStatus(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
started := time.Now()
|
||||||
|
stale := ""
|
||||||
|
if statusFrom != nil {
|
||||||
|
answer, err := statusFrom.answer(ctx)
|
||||||
|
if err != nil {
|
||||||
|
stale = err.Error()
|
||||||
|
} else if m, ok := answer.(map[string]any); ok {
|
||||||
|
if failed, _ := m["lastAttemptFailed"].(string); failed != "" {
|
||||||
|
stale = "its last composition failed: " + failed
|
||||||
|
}
|
||||||
|
if composed, err := time.Parse(time.RFC3339, fmt.Sprint(m["composed"])); err == nil &&
|
||||||
|
time.Since(composed) > 3*statusEvery+statusComposeWithin {
|
||||||
|
stale = fmt.Sprintf("it answers a summary composed %s ago", time.Since(composed).Round(time.Second))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if _, err := theThreeQuestions(ctx, d.open); err != nil {
|
||||||
|
stale = err.Error()
|
||||||
|
}
|
||||||
|
took := time.Since(started)
|
||||||
|
var out []conditions.Observation
|
||||||
|
if took > 10*time.Second || stale != "" {
|
||||||
|
why := stale
|
||||||
|
if why == "" {
|
||||||
|
why = fmt.Sprintf("it took %s", took.Round(time.Millisecond))
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: "controller", Token: "status-slow",
|
||||||
|
Machine: d.host, Severity: conditions.Warning,
|
||||||
|
Summary: "status does not answer in full within ten seconds: " + why, Said: why})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeCoreBuilds is D10: every machine runs the node-engine and the node tools the mesh holds, or a
|
||||||
|
// plan is rolling one of them out. The node-engine says its build in each report; the node tools' is
|
||||||
|
// the build the machine was last sent, once it reported applying that send.
|
||||||
|
func probeCoreBuilds(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
inv := d.open.inventory
|
||||||
|
current, err := inv.CurrentBuilds(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
plans, err := inv.OpenPlans(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
hostVersions := deliveredVersions(shelf[hostModule])
|
||||||
|
rolling := map[string]bool{}
|
||||||
|
for _, p := range plans {
|
||||||
|
for m := range p.Modules {
|
||||||
|
rolling[m] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
nodes, err := inv.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
reports, err := inv.LastReports(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
applied := map[string]bool{}
|
||||||
|
for _, r := range reports {
|
||||||
|
applied[r.Node] = r.Current
|
||||||
|
}
|
||||||
|
heard := heardMachines(d)
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, n := range nodes {
|
||||||
|
if !heard[n.Name] {
|
||||||
|
continue // a machine not heard from is S1's
|
||||||
|
}
|
||||||
|
var behind []string
|
||||||
|
// **A node-engine says its version as the directory it was delivered into** — its archive's
|
||||||
|
// digest, twelve characters (catalogue.versionOf, ADR 0141) — not the commit it was built
|
||||||
|
// from. Compared as a commit, every machine read as behind right after a push sent it the
|
||||||
|
// current one (2026-10-06). An engine placed by hand reports its link-time stamp instead,
|
||||||
|
// which a commit can match.
|
||||||
|
if !rolling[hostModule] && engineBehind(n.HostVersion, hostVersions, current[hostModule].Commit) {
|
||||||
|
behind = append(behind, fmt.Sprintf("the node-engine %s, the mesh holds %s (built from %s)",
|
||||||
|
n.HostVersion, strings.Join(hostVersions, " or "), short(current[hostModule].Commit)))
|
||||||
|
}
|
||||||
|
assigned, err := inv.Assigned(ctx, n.Name)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
tools := current[broker.RuntimeModule].Commit
|
||||||
|
if tools != "" && slices.Contains(assigned, broker.RuntimeModule) && !rolling[broker.RuntimeModule] {
|
||||||
|
sent, known, err := inv.SentBuilds(ctx, n.Name)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case !known:
|
||||||
|
case !sameCommit(sent[broker.RuntimeModule], tools):
|
||||||
|
behind = append(behind, fmt.Sprintf("the node tools %s, the mesh holds %s",
|
||||||
|
short(orNotKnown(sent[broker.RuntimeModule])), short(tools)))
|
||||||
|
case !applied[n.Name]:
|
||||||
|
behind = append(behind, "the node tools it was last sent, not yet reported applied")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(behind) > 0 {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "core-behind",
|
||||||
|
Machine: n.Name, Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s runs %s, and no plan is rolling them out", n.Name, strings.Join(behind, "; ")),
|
||||||
|
Said: strings.Join(behind, "; ")})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// deliveredVersions are the versions a module's registered build is delivered as: the last element
|
||||||
|
// of every resource path under a `versions/` directory, which registration filled from the artifact's
|
||||||
|
// digest (catalogue `${version}`). The node-engine names itself by that directory.
|
||||||
|
func deliveredVersions(m catalogue.Manifest) []string {
|
||||||
|
var out []string
|
||||||
|
for _, r := range m.Resources {
|
||||||
|
path, _ := r["path"].(string)
|
||||||
|
before, version, found := strings.Cut(path, "/versions/")
|
||||||
|
if !found || before == "" || version == "" || strings.Contains(version, "/") || strings.Contains(version, "$") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !slices.Contains(out, version) {
|
||||||
|
out = append(out, version)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// engineBehind says a node-engine's reported version is not the build the mesh holds: neither the
|
||||||
|
// directory that build is delivered as, nor (for an engine placed by hand) its commit. A machine that
|
||||||
|
// has not said, or a mesh that holds no delivered build, is not behind anything.
|
||||||
|
func engineBehind(reported string, delivered []string, commit string) bool {
|
||||||
|
if reported == "" || len(delivered) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return !slices.Contains(delivered, reported) && !sameCommit(reported, commit)
|
||||||
|
}
|
||||||
|
|
||||||
|
// hostModule is the node-engine's module.
|
||||||
|
const hostModule = "mesh-host"
|
||||||
|
|
||||||
|
// sameCommit says two commits are one, either written short.
|
||||||
|
func sameCommit(a, b string) bool {
|
||||||
|
if a == "" || b == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return strings.HasPrefix(a, b) || strings.HasPrefix(b, a)
|
||||||
|
}
|
||||||
|
|
||||||
|
func orNotKnown(s string) string {
|
||||||
|
if s == "" {
|
||||||
|
return "(not known)"
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeWatchdogs is DW: the watchdogs ran within three of their intervals. The doctor and the
|
||||||
|
// watchdogs watch each other: S10 is the other half.
|
||||||
|
func probeWatchdogs(_ context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
if d.watchdogs == nil {
|
||||||
|
return nil, nil // a self-check run outside the serving controller has none to watch
|
||||||
|
}
|
||||||
|
ticked := d.watchdogs.lastTick()
|
||||||
|
since := ticked
|
||||||
|
if since.IsZero() {
|
||||||
|
since = d.watchdogs.started
|
||||||
|
}
|
||||||
|
if time.Since(since) <= 3*watchEvery {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return []conditions.Observation{{Scope: conditions.ScopeCore, ID: "watchdogs", Token: "silent",
|
||||||
|
Machine: d.host, Severity: conditions.Urgent,
|
||||||
|
Summary: fmt.Sprintf("the watchdogs of the signals table have not run since %s: no late signal is being said",
|
||||||
|
since.UTC().Format("2006-01-02 15:04 MST")),
|
||||||
|
Said: fmt.Sprintf("no tick for %s", time.Since(since).Round(time.Second))}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// askSeatTool asks one machine's holder of a node seat a verb and answers its result; the holder's
|
||||||
|
// own refusal is an error.
|
||||||
|
func askSeatTool(ctx context.Context, conn *nats.Conn, seat, verb, node string) (json.RawMessage, error) {
|
||||||
|
if who, declared := declaredBy(ctx, seat, verb); !declared {
|
||||||
|
return nil, fmt.Errorf("%s asks %s.%s, which it does not declare in the probe registry — and so the "+
|
||||||
|
"controller is not granted it", who, seat, verb)
|
||||||
|
}
|
||||||
|
answer, err := link.AskSeatTool(ctx, conn, seat, verb, node, map[string]any{}, 10*time.Second)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if answer.Error != "" {
|
||||||
|
return nil, fmt.Errorf("%s's %s.%s refused: %s", node, seat, verb, answer.Error)
|
||||||
|
}
|
||||||
|
return answer.Result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// oneLine is a message of several lines said on one, its runs of space made one.
|
||||||
|
func oneLine(s string) string { return strings.Join(strings.Fields(s), " ") }
|
||||||
+80
-41
@@ -8,6 +8,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
"io"
|
"io"
|
||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
@@ -105,7 +106,10 @@ func serve(ctx context.Context) error {
|
|||||||
|
|
||||||
work := link.Enrolment{Inventory: inv, Identity: ident, Broker: known,
|
work := link.Enrolment{Inventory: inv, Identity: ident, Broker: known,
|
||||||
OnNATS: true}
|
OnNATS: true}
|
||||||
server, err := connectLink(ctx, inv, work, work)
|
// `status` from a summary kept current here (novox/hq to-be 45 Phase 0): a machine saying
|
||||||
|
// something new is one thing that moves it, so the listener nudges it.
|
||||||
|
statusFrom = newStatusSummary(composeStatus(open))
|
||||||
|
server, err := connectLink(ctx, inv, work, nudgingListener{work, statusFrom})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -121,6 +125,8 @@ func serve(ctx context.Context) error {
|
|||||||
// Open plans move on a timer as well as on outcomes (novox/hq ADR 0162): a tier waiting for
|
// Open plans move on a timer as well as on outcomes (novox/hq ADR 0162): a tier waiting for
|
||||||
// machines to report moves when they have, and a plan left by a replaced controller resumes.
|
// machines to report moves when they have, and a plan left by a replaced controller resumes.
|
||||||
go planTicker(ctx, open)
|
go planTicker(ctx, open)
|
||||||
|
// The durations the core's bounds are set from are kept a month (novox/hq to-be 45 Phase 0).
|
||||||
|
go forgettingOldDurations(ctx, inv)
|
||||||
// And what the catalogue decided a build meant. The builder's own result is already handled
|
// And what the catalogue decided a build meant. The builder's own result is already handled
|
||||||
// above; this is the other half — the control plane is the only one of the three that knows
|
// above; this is the other half — the control plane is the only one of the three that knows
|
||||||
// which machines run the thing, so it is the one that acts (novox/hq ADR 0072).
|
// which machines run the thing, so it is the one that acts (novox/hq ADR 0072).
|
||||||
@@ -137,7 +143,7 @@ func serve(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
// And what providers say about consumers they keep failing, kept for `status` (novox/hq ADR
|
// And what providers say about consumers they keep failing, kept for `status` (novox/hq ADR
|
||||||
// 0224): a provider's journal must not be the only place that says so.
|
// 0224): a provider's journal must not be the only place that says so.
|
||||||
if err := server.Watches(standings{inv}); err != nil {
|
if err := server.Watches(standings{keeper: func() *conditions.Keeper { return conditionsFrom }}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -158,8 +164,28 @@ func serve(ctx context.Context) error {
|
|||||||
if !isNATS {
|
if !isNATS {
|
||||||
return errors.New("the mesh's verbs are served over the bus, and this control plane is not on it")
|
return errors.New("the mesh's verbs are served over the bus, and this control plane is not on it")
|
||||||
}
|
}
|
||||||
|
// The hand-act log is counted for `status` on this connection rather than a new one a minute.
|
||||||
|
handActConn = bus.Conn
|
||||||
|
// Composed now and kept current, before the verb that answers from it is served.
|
||||||
|
go statusFrom.keep(ctx)
|
||||||
// A call that outlasts its caller's patience is followed by `calls` (novox/hq issue 265).
|
// A call that outlasts its caller's patience is followed by `calls` (novox/hq issue 265).
|
||||||
link.Calls.Follow = catalogue.ControllerSeatName + ".calls"
|
link.Calls.Follow = catalogue.ControllerSeatName + ".calls"
|
||||||
|
// And every call is kept on the bus, so a restart of this process keeps what came of each
|
||||||
|
// (novox/hq to-be 45 §6). A bus without the bucket is said and served from memory, as before:
|
||||||
|
// answering no calls at all would be worse than answering them without the record.
|
||||||
|
said := log.New(os.Stdout, "", log.LstdFlags)
|
||||||
|
if keeper, err := link.CallsOnTheBus(ctx, bus.Conn); err != nil {
|
||||||
|
fmt.Printf("calls are kept in memory only, and lost when this controller stops: %v\n", err)
|
||||||
|
} else if err := link.Calls.Durably(ctx, keeper, controllerProcess(), said); err != nil {
|
||||||
|
fmt.Printf("calls are kept on the bus from now on; the ones kept before could not be read: %v\n", err)
|
||||||
|
}
|
||||||
|
// What is wrong, kept and said (novox/hq to-be 45 §2): the condition store, the watchdogs of the
|
||||||
|
// signals table, what the bus says about itself, and the self-check. A store that cannot be opened
|
||||||
|
// is said and the controller serves on: status then says the conditions cannot be read, and is
|
||||||
|
// not well — louder than not serving at all, and the push that repairs the bus still runs.
|
||||||
|
if stopWatching := watchTheMesh(ctx, open, server, bus); stopWatching != nil {
|
||||||
|
defer stopWatching()
|
||||||
|
}
|
||||||
stopServing, err := bus.ServeSeatTools(catalogue.ControllerSeatName, handlers, log.New(os.Stdout, "", log.LstdFlags))
|
stopServing, err := bus.ServeSeatTools(catalogue.ControllerSeatName, handlers, log.New(os.Stdout, "", log.LstdFlags))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -260,6 +286,9 @@ func pushCommand(ctx context.Context, args []string) error {
|
|||||||
// (novox/hq ADR 0010). 0 waits for nothing, which is the old fire-and-forget.
|
// (novox/hq ADR 0010). 0 waits for nothing, which is the old fire-and-forget.
|
||||||
wait := set.Duration("wait", 0,
|
wait := set.Duration("wait", 0,
|
||||||
"for a named node, how long to wait for it to report applying what it was sent (0: do not wait)")
|
"for a named node, how long to wait for it to report applying what it was sent (0: do not wait)")
|
||||||
|
// A push by hand is a repair, and says why (novox/hq to-be 45 §7): required through the seat,
|
||||||
|
// recorded when given at a shell — see handacts.go for why a shell is not refused.
|
||||||
|
why := addHandActFlags(set)
|
||||||
positionals, err := parseAround(set, args)
|
positionals, err := parseAround(set, args)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -274,6 +303,11 @@ func pushCommand(ctx context.Context, args []string) error {
|
|||||||
return errors.New("push <node> or push --behind, not both: one names a machine and the " +
|
return errors.New("push <node> or push --behind, not both: one names a machine and the " +
|
||||||
"other asks which machines need one")
|
"other asks which machines need one")
|
||||||
}
|
}
|
||||||
|
recorded := append([]string(nil), args...)
|
||||||
|
if *behind {
|
||||||
|
recorded = append(recorded, "--behind")
|
||||||
|
}
|
||||||
|
why.record(ctx, "push", recorded)
|
||||||
open, err := openStores(ctx)
|
open, err := openStores(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -742,6 +776,13 @@ type overTheBus struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (b overTheBus) grant(ctx context.Context, sending []readyNode) error {
|
func (b overTheBus) grant(ctx context.Context, sending []readyNode) error {
|
||||||
|
// The bus's objects first, which every declaration implies (novox/hq issue 208): said and raised
|
||||||
|
// when they cannot be, and never what holds the send back (assertOnSend).
|
||||||
|
if bus, ok := b.server.Bus().(link.OverNATS); ok {
|
||||||
|
js := broker.OnConn(bus.Conn)
|
||||||
|
js.Note = func(format string, args ...any) { fmt.Printf(b.indent+" "+format+"\n", args...) }
|
||||||
|
_ = assertOnSend(ctx, b.open.inventory, js, b.indent)
|
||||||
|
}
|
||||||
return issueMemberships(ctx, b.open, b.server, sending)
|
return issueMemberships(ctx, b.open, b.server, sending)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1039,6 +1080,20 @@ func digestOf(body []byte) string {
|
|||||||
// be worked out" is a different problem with a different remedy, and `plan` is where it is said.
|
// be worked out" is a different problem with a different remedy, and `plan` is where it is said.
|
||||||
func wouldSend(ctx context.Context, open *stores,
|
func wouldSend(ctx context.Context, open *stores,
|
||||||
nodes []inventory.Node) (map[string]string, error) {
|
nodes []inventory.Node) (map[string]string, error) {
|
||||||
|
return wouldSendFrom(ctx, open, nodes, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// planned is one machine's plan as planFor answered it, for a caller that already asked.
|
||||||
|
type planned struct {
|
||||||
|
plan catalogue.Resolution
|
||||||
|
settings catalogue.SettingsBy
|
||||||
|
}
|
||||||
|
|
||||||
|
// wouldSendFrom is wouldSend reusing the plans a caller worked out a moment before: resolving a
|
||||||
|
// machine is most of what `status` costs, and it used to resolve every machine twice (novox/hq
|
||||||
|
// to-be 45 Phase 0). A machine absent from plans is worked out here.
|
||||||
|
func wouldSendFrom(ctx context.Context, open *stores,
|
||||||
|
nodes []inventory.Node, plans map[string]planned) (map[string]string, error) {
|
||||||
|
|
||||||
gens, err := generators(ctx, open)
|
gens, err := generators(ctx, open)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1046,10 +1101,13 @@ func wouldSend(ctx context.Context, open *stores,
|
|||||||
}
|
}
|
||||||
out := map[string]string{}
|
out := map[string]string{}
|
||||||
for _, n := range nodes {
|
for _, n := range nodes {
|
||||||
plan, settings, err := planFor(ctx, open, n.Name)
|
known, have := plans[n.Name]
|
||||||
if err != nil {
|
plan, settings := known.plan, known.settings
|
||||||
|
if !have {
|
||||||
|
if plan, settings, err = planFor(ctx, open, n.Name); err != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
}
|
||||||
declared, err := declarationWith(ctx, open, n.Name, plan, settings, gens, Reading)
|
declared, err := declarationWith(ctx, open, n.Name, plan, settings, gens, Reading)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
continue
|
continue
|
||||||
@@ -1107,35 +1165,22 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
nodes, err := inv.Nodes(ctx)
|
// The mesh's own streams and consumers, the seats' work queues and their workers, and how every
|
||||||
|
// machine hears its declaration — one derivation, which the self-check reads as well (D6, D7).
|
||||||
|
names, err := assertBusObjects(ctx, inv, js)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
names := make([]string, 0, len(nodes))
|
|
||||||
for _, n := range nodes {
|
|
||||||
names = append(names, n.Name)
|
|
||||||
}
|
|
||||||
if err := broker.Raise(js, names); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// The work queues of the mesh's own roles (novox/hq ADR 0121). The queue before the holder,
|
|
||||||
// deliberately: work queues until somebody arrives to do it, so assigning a build machine a week
|
|
||||||
// after something started asking for builds flushes the backlog instead of having lost it.
|
|
||||||
// With the seats' holders, so each role's work queue gets the consumer its holder takes
|
|
||||||
// work from. Passed as nil until the first live raise, which left the build machine bound to a
|
|
||||||
// consumer nothing had created (2026-09-28).
|
|
||||||
holders, err := seatHolders(ctx, inv)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := broker.RaiseSeats(js, inventory.MeshSeats(), holders); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// And each work queue's cancelled set (novox/hq ADR 0219), so a holder taking an ask can ask
|
// And each work queue's cancelled set (novox/hq ADR 0219), so a holder taking an ask can ask
|
||||||
// whether it was cancelled the moment it took it.
|
// whether it was cancelled the moment it took it.
|
||||||
if err := broker.RaiseCancelledSets(js, inventory.MeshSeats()); err != nil {
|
if err := broker.RaiseCancelledSets(js, inventory.MeshSeats()); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
// And the controller's own buckets (novox/hq to-be 45 §1): the calls it serves and the acts done
|
||||||
|
// by hand, kept where a restart of this process does not take them.
|
||||||
|
if err := js.EnsureControllerBuckets(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
// Every module's state (novox/hq ADR 0201), from the catalogue: a bucket exists from
|
// Every module's state (novox/hq ADR 0201), from the catalogue: a bucket exists from
|
||||||
// registration, so a module reading one may watch it before its owner runs anywhere. One that
|
// registration, so a module reading one may watch it before its owner runs anywhere. One that
|
||||||
// nothing declares any more is said and kept — what it holds is data.
|
// nothing declares any more is said and kept — what it holds is data.
|
||||||
@@ -1151,25 +1196,11 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string)
|
|||||||
fmt.Printf("the bus holds state nothing declares any more, kept because it is data: %s — "+
|
fmt.Printf("the bus holds state nothing declares any more, kept because it is data: %s — "+
|
||||||
"removing it is a person's act\n", strings.Join(undeclared, ", "))
|
"removing it is a person's act\n", strings.Join(undeclared, ", "))
|
||||||
}
|
}
|
||||||
// And how every module hears what it consumes. Derived from the same records the user list is
|
// And how every module hears what it consumes: asserted with the rest above, counted here.
|
||||||
// composed from, so a module the mesh grants a consumer's subjects has that consumer waiting.
|
hearing, err := moduleConsumerCount(ctx, inv)
|
||||||
// Done on every raise, not only when a credential is issued: every module moved onto this bus
|
|
||||||
// by the rollout was issued on the old one, and came up with nothing to bind to (2026-09-28).
|
|
||||||
records, err := inv.BusRecords(ctx)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
users, err := broker.Users(records)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
hearing := 0
|
|
||||||
for _, c := range broker.ConsumersOf(users) {
|
|
||||||
if err := js.EnsureConsumer(c.Consumer); err != nil {
|
|
||||||
return fmt.Errorf("how %s on %s hears what it consumes: %w", c.Module, c.Node, err)
|
|
||||||
}
|
|
||||||
hearing++
|
|
||||||
}
|
|
||||||
fmt.Printf("the bus at %s has its streams, %d machine(s) can hear a declaration, %d module(s) "+
|
fmt.Printf("the bus at %s has its streams, %d machine(s) can hear a declaration, %d module(s) "+
|
||||||
"can hear what they consume, and %d bucket(s) of state\n", broker.BareAddress(address), len(names), hearing, len(buckets))
|
"can hear what they consume, and %d bucket(s) of state\n", broker.BareAddress(address), len(names), hearing, len(buckets))
|
||||||
return nil
|
return nil
|
||||||
@@ -1271,3 +1302,11 @@ func reportUnheldPushed(w io.Writer, named bool, asked []string, unheld map[stri
|
|||||||
fmt.Fprintf(w, "%s: %d unmet seat dependenc(ies) — see `status`\n", node, len(lines))
|
fmt.Fprintf(w, "%s: %d unmet seat dependenc(ies) — see `status`\n", node, len(lines))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// controllerProcess names this serving process among controllers: the machine, the process and when
|
||||||
|
// it started — what a call kept on the bus carries, so the next controller can tell a call this one
|
||||||
|
// left running from one it is running itself (novox/hq to-be 45 §6).
|
||||||
|
func controllerProcess() string {
|
||||||
|
host, _ := os.Hostname()
|
||||||
|
return fmt.Sprintf("controller@%s pid %d since %s", host, os.Getpid(), time.Now().UTC().Format(time.RFC3339))
|
||||||
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"sort"
|
"sort"
|
||||||
"time"
|
"time"
|
||||||
@@ -78,11 +79,24 @@ type meshStatus struct {
|
|||||||
// that machine holds, with the modules that could hold it (novox/hq ADR 0207). Absent when every
|
// that machine holds, with the modules that could hold it (novox/hq ADR 0207). Absent when every
|
||||||
// dependency is met. Reported, not refused, until the switch.
|
// dependency is met. Reported, not refused, until the switch.
|
||||||
Unheld []catalogue.Unheld `json:"unheld,omitempty"`
|
Unheld []catalogue.Unheld `json:"unheld,omitempty"`
|
||||||
// Failing is every consumer a provider says it keeps failing, with the class of error, since
|
// HandActsThisWeek is how many acts were done by hand in the last seven days (novox/hq to-be 45
|
||||||
// when, and when it was last said (novox/hq ADR 0224). Absent when no provider says so. A
|
// §7): every one is a repair a healer could have made. Absent where the log is not on hand;
|
||||||
// document without this called the mesh well while the identity provider refused every consumer
|
// HandActsUnread says why when it could not be read, rather than reading as none.
|
||||||
// for a day (04-ISSUES/179).
|
HandActsThisWeek *int `json:"handActsThisWeek,omitempty"`
|
||||||
Failing []inventory.ProviderStanding `json:"failing,omitempty"`
|
HandActsUnread string `json:"handActsUnread,omitempty"`
|
||||||
|
// Conditions is every open condition, urgent first and then oldest first (novox/hq to-be 45 §2):
|
||||||
|
// what is wrong, as the watchdogs, the self-check and the providers say it. Always present — an
|
||||||
|
// empty list is "none open" — unless they could not be read, which ConditionsUnread says.
|
||||||
|
Conditions []conditions.Condition `json:"conditions"`
|
||||||
|
ConditionsUnread string `json:"conditionsUnread,omitempty"`
|
||||||
|
// Failing is every consumer a provider says it keeps failing (novox/hq ADR 0224): the open
|
||||||
|
// conditions of that kind, carried here as well because ADR 0224 names this field. Absent when no
|
||||||
|
// provider says so. A document without it called the mesh well while the identity provider
|
||||||
|
// refused every consumer for a day (04-ISSUES/179).
|
||||||
|
Failing []conditions.Condition `json:"failing,omitempty"`
|
||||||
|
// Overflowing is every module whose identity overflows the bound of a provision it requires, and
|
||||||
|
// so is left out of its provider's grants (novox/hq ADR 0225). Absent when every identity fits.
|
||||||
|
Overflowing []catalogue.Overflow `json:"overflowing,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// machineFiltered is one rule set on a converged machine that the mesh did not write and that
|
// machineFiltered is one rule set on a converged machine that the mesh did not write and that
|
||||||
@@ -215,7 +229,13 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
out.Unheld = asked.unheld
|
out.Unheld = asked.unheld
|
||||||
out.Failing = asked.failing
|
out.HandActsThisWeek, out.HandActsUnread = asked.handActs, asked.handActsUnread
|
||||||
|
out.Conditions, out.ConditionsUnread = asked.conditions, asked.conditionsUnread
|
||||||
|
if out.Conditions == nil {
|
||||||
|
out.Conditions = []conditions.Condition{}
|
||||||
|
}
|
||||||
|
out.Failing = providerStandings(asked.conditions)
|
||||||
|
out.Overflowing = asked.overflowing
|
||||||
for name := range asked.refused {
|
for name := range asked.refused {
|
||||||
out.Unresolved = append(out.Unresolved, machineUnresolved{
|
out.Unresolved = append(out.Unresolved, machineUnresolved{
|
||||||
Node: name, Problem: asked.refused[name]})
|
Node: name, Problem: asked.refused[name]})
|
||||||
|
|||||||
@@ -986,10 +986,18 @@ func plansCommand(ctx context.Context, args []string) error {
|
|||||||
whatIf := set.String("what-if", "", "owner/repository: the plan a merge there would produce, saving nothing — with --paths or --modules")
|
whatIf := set.String("what-if", "", "owner/repository: the plan a merge there would produce, saving nothing — with --paths or --modules")
|
||||||
paths := set.String("paths", "", "the files the merge would change, comma-separated, from the repository's root")
|
paths := set.String("paths", "", "the files the merge would change, comma-separated, from the repository's root")
|
||||||
modules := set.String("modules", "", "or the modules it would change, comma-separated")
|
modules := set.String("modules", "", "or the modules it would change, comma-separated")
|
||||||
|
// Ending a plan by hand is a repair, and says why (novox/hq to-be 45 §7).
|
||||||
|
why := addHandActFlags(set)
|
||||||
positionals, err := parseAround(set, args)
|
positionals, err := parseAround(set, args)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if len(positionals) == 2 && (positionals[0] == "stop" || positionals[0] == "close") {
|
||||||
|
// Refused before anything is opened: a repair by hand says why.
|
||||||
|
if err := why.require("plans " + positionals[0]); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
open, err := openStores(ctx)
|
open, err := openStores(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -1061,8 +1069,9 @@ func plansCommand(ctx context.Context, args []string) error {
|
|||||||
if !p.Open() {
|
if !p.Open() {
|
||||||
return fmt.Errorf("%s is already %s", p.ID, p.State)
|
return fmt.Errorf("%s is already %s", p.ID, p.State)
|
||||||
}
|
}
|
||||||
|
why.record(ctx, "plans "+positionals[0], positionals[1:])
|
||||||
p.State = inventory.PlanFailed
|
p.State = inventory.PlanFailed
|
||||||
p.Note = how + " by hand at tier " + fmt.Sprint(p.Tier)
|
p.Note = how + " by hand at tier " + fmt.Sprint(p.Tier) + ": " + strings.TrimSpace(*why.why)
|
||||||
sayUnsent(&p, func(m string) bool {
|
sayUnsent(&p, func(m string) bool {
|
||||||
u, err := inv.UpgradeOf(ctx, m)
|
u, err := inv.UpgradeOf(ctx, m)
|
||||||
return err == nil && u.RollOut
|
return err == nil && u.RollOut
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ import (
|
|||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"sort"
|
"sort"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
)
|
)
|
||||||
|
|
||||||
// rotateCommand replaces a credential and moves both ends together.
|
// rotateCommand replaces a credential and moves both ends together.
|
||||||
@@ -33,12 +35,16 @@ func rotateCommand(ctx context.Context, args []string) error {
|
|||||||
// One consumer rather than all of them. Ordinary: a credential is suspected on one machine,
|
// One consumer rather than all of them. Ordinary: a credential is suspected on one machine,
|
||||||
// and rotating the other nine would be a great deal of disruption for one suspicion.
|
// and rotating the other nine would be a great deal of disruption for one suspicion.
|
||||||
only := set.String("consumer", "", "only this machine's credential, rather than every holder's")
|
only := set.String("consumer", "", "only this machine's credential, rather than every holder's")
|
||||||
|
// One consuming module rather than every module on the machine. A machine runs many consumers
|
||||||
|
// of one provision, each with its own credential; one module that leaked its credential (novox/hq
|
||||||
|
// issue 268) is no reason to restart every other one on the machine.
|
||||||
|
module := set.String("module", "", "only this consuming module's credential")
|
||||||
positionals, err := parseAround(set, args)
|
positionals, err := parseAround(set, args)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if len(positionals) != 1 {
|
if len(positionals) != 1 {
|
||||||
return errors.New("rotate <provision> [--consumer <machine>]")
|
return errors.New("rotate <provision> [--consumer <machine>] [--module <module>]")
|
||||||
}
|
}
|
||||||
provision := positionals[0]
|
provision := positionals[0]
|
||||||
|
|
||||||
@@ -53,6 +59,12 @@ func rotateCommand(ctx context.Context, args []string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
holders = ofModule(holders, *module)
|
||||||
|
if len(holders) == 0 && *module != "" {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"no module %s%s holds a credential for %q, so there is nothing to rotate. `plan <machine>` "+
|
||||||
|
"says what a machine holds", *module, onMachine(*only), provision)
|
||||||
|
}
|
||||||
if len(holders) == 0 {
|
if len(holders) == 0 {
|
||||||
// Said, not silent. "Nobody holds this" and "this did not run" must never look the same —
|
// Said, not silent. "Nobody holds this" and "this did not run" must never look the same —
|
||||||
// and a rotation somebody believes happened is worse than one they know did not.
|
// and a rotation somebody believes happened is worse than one they know did not.
|
||||||
@@ -116,6 +128,27 @@ func rotateCommand(ctx context.Context, args []string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ofModule is the holders whose consuming module is this one; all of them when none is named.
|
||||||
|
func ofModule(holders []inventory.Holder, module string) []inventory.Holder {
|
||||||
|
if module == "" {
|
||||||
|
return holders
|
||||||
|
}
|
||||||
|
var out []inventory.Holder
|
||||||
|
for _, h := range holders {
|
||||||
|
if h.ConsumerModule == module {
|
||||||
|
out = append(out, h)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func onMachine(machine string) string {
|
||||||
|
if machine == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return " on " + machine
|
||||||
|
}
|
||||||
|
|
||||||
// asLocal names the credential inside the consumer where it holds several (ADR 0094).
|
// asLocal names the credential inside the consumer where it holds several (ADR 0094).
|
||||||
func asLocal(local string) string {
|
func asLocal(local string) string {
|
||||||
if local == "" {
|
if local == "" {
|
||||||
|
|||||||
@@ -0,0 +1,27 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// One consuming module's credential, and not its neighbours' on the same machine (novox/hq issue
|
||||||
|
// 268): a module that leaked its database password is no reason to restart every other consumer.
|
||||||
|
func TestARotationNarrowedToAModuleTouchesOnlyThatModulesCredential(t *testing.T) {
|
||||||
|
holders := []inventory.Holder{
|
||||||
|
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "letta", Provider: "ace"},
|
||||||
|
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "n8n", Provider: "ace"},
|
||||||
|
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "letta", Local: "reader", Provider: "ace"},
|
||||||
|
}
|
||||||
|
got := ofModule(holders, "letta")
|
||||||
|
if len(got) != 2 || got[0].ConsumerModule != "letta" || got[1].Local != "reader" {
|
||||||
|
t.Fatalf("narrowed to letta: %+v", got)
|
||||||
|
}
|
||||||
|
if len(ofModule(holders, "")) != 3 {
|
||||||
|
t.Fatal("no module named narrowed anyway")
|
||||||
|
}
|
||||||
|
if len(ofModule(holders, "absent")) != 0 {
|
||||||
|
t.Fatal("a module holding nothing matched")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -9,9 +9,11 @@ import (
|
|||||||
"github.com/nats-io/nats.go/micro"
|
"github.com/nats-io/nats.go/micro"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
)
|
)
|
||||||
@@ -239,6 +241,12 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
|||||||
if len(argv) == 0 {
|
if len(argv) == 0 {
|
||||||
return nil, errors.New("command names no command")
|
return nil, errors.New("command names no command")
|
||||||
}
|
}
|
||||||
|
// The generic verb is no way round the hand-act log (novox/hq to-be 45 §7): a repair through
|
||||||
|
// it says why, as it would through its own verb.
|
||||||
|
if repair := repairingCommand(argv); repair != "" && !slices.ContainsFunc(argv, isWhyFlag) {
|
||||||
|
return nil, fmt.Errorf("%s is a repair done by hand, and says why: add --why <text> to the command "+
|
||||||
|
"line (recorded in the hand-act log). Nothing was done", repair)
|
||||||
|
}
|
||||||
return argv, nil
|
return argv, nil
|
||||||
case "tools":
|
case "tools":
|
||||||
return nil, errors.New("tools is answered from the records, not by a command")
|
return nil, errors.New("tools is answered from the records, not by a command")
|
||||||
@@ -280,7 +288,18 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
|||||||
}
|
}
|
||||||
for _, act := range []string{"stop", "close", "retry"} {
|
for _, act := range []string{"stop", "close", "retry"} {
|
||||||
if id := str(act); id != "" {
|
if id := str(act); id != "" {
|
||||||
return []string{"plans", act, id}, nil
|
argv := []string{"plans", act, id}
|
||||||
|
if act == "retry" {
|
||||||
|
return argv, nil
|
||||||
|
}
|
||||||
|
// Ending a plan by hand says why (novox/hq to-be 45 §7); the command refuses it without.
|
||||||
|
if w := str("why"); w != "" {
|
||||||
|
argv = append(argv, "--why", w)
|
||||||
|
}
|
||||||
|
if c := str("cause"); c != "" {
|
||||||
|
argv = append(argv, "--cause", c)
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if id := str("id"); id != "" {
|
if id := str("id"); id != "" {
|
||||||
@@ -355,22 +374,105 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
|||||||
// Sent and not waited for: the asker reads `status` for what the machine did, which is
|
// Sent and not waited for: the asker reads `status` for what the machine did, which is
|
||||||
// what a person at a shell does too. A tool call that blocked for a push's whole apply would
|
// what a person at a shell does too. A tool call that blocked for a push's whole apply would
|
||||||
// time out on every machine that takes a minute, and say nothing about the ones that did not.
|
// time out on every machine that takes a minute, and say nothing about the ones that did not.
|
||||||
|
// A push through the seat is a push by hand, and says why (novox/hq to-be 45 §7).
|
||||||
|
if err := need("why"); err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: a push by hand is a repair, recorded in the hand-act log with why", err)
|
||||||
|
}
|
||||||
|
why := []string{"--why", str("why")}
|
||||||
|
if c := str("cause"); c != "" {
|
||||||
|
why = append(why, "--cause", c)
|
||||||
|
}
|
||||||
if n := str("node"); n != "" {
|
if n := str("node"); n != "" {
|
||||||
// behind is not read here: given with a machine, it is refused as passed over — naming
|
// behind is not read here: given with a machine, it is refused as passed over — naming
|
||||||
// a machine and asking for every machine behind are two requests, and guessing one
|
// a machine and asking for every machine behind are two requests, and guessing one
|
||||||
// would push a machine nobody named, or not push one somebody did.
|
// would push a machine nobody named, or not push one somebody did.
|
||||||
return []string{"push", n, "--wait", "0"}, nil
|
return append([]string{"push", n, "--wait", "0"}, why...), nil
|
||||||
}
|
}
|
||||||
// No machine: the whole mesh, whether or not behind said so. The command's answer says it
|
// No machine: the whole mesh, whether or not behind said so. The command's answer says it
|
||||||
// first, so a caller who meant one machine reads that it was not one.
|
// first, so a caller who meant one machine reads that it was not one.
|
||||||
on("behind")
|
on("behind")
|
||||||
return []string{"push", "--behind", "--wait", "0"}, nil
|
return append([]string{"push", "--behind", "--wait", "0"}, why...), nil
|
||||||
|
case "hand-act":
|
||||||
|
if err := need("what", "why", "cause"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
argv := []string{"hand-act", "record", str("what"), "--why", str("why"), "--cause", str("cause")}
|
||||||
|
if c := str("condition"); c != "" {
|
||||||
|
argv = append(argv, "--condition", c)
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
|
case "hand-acts":
|
||||||
|
argv := []string{"hand-acts", "--json"}
|
||||||
|
if d := str("days"); d != "" {
|
||||||
|
argv = append(argv, "--days", d)
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
|
case "durations":
|
||||||
|
argv := []string{"durations", "--json"}
|
||||||
|
if k := str("kind"); k != "" {
|
||||||
|
argv = append(argv, "--kind", k)
|
||||||
|
}
|
||||||
|
if d := str("days"); d != "" {
|
||||||
|
argv = append(argv, "--days", d)
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
|
case "conditions":
|
||||||
|
// One verb, four shapes, as `plans` (novox/hq to-be 45 §2): a silence, one condition, the
|
||||||
|
// history, or the open ones filtered.
|
||||||
|
if key := str("silence"); key != "" {
|
||||||
|
if err := need("for", "why"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
argv := []string{"conditions", "silence", key, "--for", str("for"), "--why", str("why")}
|
||||||
|
if c := str("cause"); c != "" {
|
||||||
|
argv = append(argv, "--cause", c)
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
|
}
|
||||||
|
if on("history") {
|
||||||
|
argv := []string{"conditions", "history", "--json"}
|
||||||
|
if d := str("days"); d != "" {
|
||||||
|
argv = append(argv, "--days", d)
|
||||||
|
}
|
||||||
|
if k := str("key"); k != "" {
|
||||||
|
argv = append(argv, "--key", k)
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
|
}
|
||||||
|
if k := str("key"); k != "" {
|
||||||
|
return []string{"conditions", "show", k, "--json"}, nil
|
||||||
|
}
|
||||||
|
argv := []string{"conditions", "--json"}
|
||||||
|
for _, filter := range []string{"scope", "severity", "machine"} {
|
||||||
|
if v := str(filter); v != "" {
|
||||||
|
argv = append(argv, "--"+filter, v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
|
case "doctor":
|
||||||
|
which := 0
|
||||||
|
argv := []string{"doctor"}
|
||||||
|
for _, sub := range []string{"run", "probes", "signals"} {
|
||||||
|
if on(sub) {
|
||||||
|
which++
|
||||||
|
argv = append(argv, sub)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if which > 1 {
|
||||||
|
return nil, errors.New("doctor answers one of run, probes or signals at a time")
|
||||||
|
}
|
||||||
|
return append(argv, "--json"), nil
|
||||||
case "rotate":
|
case "rotate":
|
||||||
if p := str("provision"); p != "" {
|
if p := str("provision"); p != "" {
|
||||||
argv := []string{"rotate", p}
|
argv := []string{"rotate", p}
|
||||||
if c := str("consumer"); c != "" {
|
if c := str("consumer"); c != "" {
|
||||||
argv = append(argv, "--consumer", c)
|
argv = append(argv, "--consumer", c)
|
||||||
}
|
}
|
||||||
|
// With a provision, module narrows to one consuming module (novox/hq issue 268); node
|
||||||
|
// and secret stay the other shape's, and are refused as passed over.
|
||||||
|
if m := str("module"); m != "" {
|
||||||
|
argv = append(argv, "--module", m)
|
||||||
|
}
|
||||||
return argv, nil
|
return argv, nil
|
||||||
}
|
}
|
||||||
_, node := a.given["node"]
|
_, node := a.given["node"]
|
||||||
@@ -437,7 +539,30 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// jsonVerbs are the verbs whose command speaks JSON, so the answer carries it as data as well.
|
// jsonVerbs are the verbs whose command speaks JSON, so the answer carries it as data as well.
|
||||||
var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true}
|
var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true,
|
||||||
|
"hand-acts": true, "durations": true, "conditions": true, "doctor": true}
|
||||||
|
|
||||||
|
// repairingCommand names a command line that repairs by hand, and so says why: a push, a plan stopped
|
||||||
|
// or closed, a consumer re-made (novox/hq to-be 45 §7). Empty for any other.
|
||||||
|
func repairingCommand(argv []string) string {
|
||||||
|
switch {
|
||||||
|
case argv[0] == "push":
|
||||||
|
return "push"
|
||||||
|
case argv[0] == "plans" && len(argv) > 1 && (argv[1] == "stop" || argv[1] == "close"):
|
||||||
|
return "plans " + argv[1]
|
||||||
|
case argv[0] == "broker" && len(argv) > 1 && argv[1] == "consumer-reset":
|
||||||
|
return "broker consumer-reset"
|
||||||
|
case argv[0] == "hand-act":
|
||||||
|
return "hand-act record"
|
||||||
|
case argv[0] == "conditions" && len(argv) > 1 && argv[1] == "silence":
|
||||||
|
return "conditions silence"
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func isWhyFlag(word string) bool {
|
||||||
|
return word == "--why" || word == "-why" || strings.HasPrefix(word, "--why=") || strings.HasPrefix(word, "-why=")
|
||||||
|
}
|
||||||
|
|
||||||
// runVerb runs this binary with the given command line and gathers what it said.
|
// runVerb runs this binary with the given command line and gathers what it said.
|
||||||
func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
||||||
@@ -449,6 +574,12 @@ func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
|||||||
// The same environment: the stores' credentials, the bus, the broker — everything a command run
|
// The same environment: the stores' credentials, the bus, the broker — everything a command run
|
||||||
// from a shell in this container would have, because it is that.
|
// from a shell in this container would have, because it is that.
|
||||||
cmd.Env = os.Environ()
|
cmd.Env = os.Environ()
|
||||||
|
// And who asked, so an act it does by hand is recorded as theirs (novox/hq to-be 45 §7).
|
||||||
|
caller := link.CallerIn(ctx)
|
||||||
|
if caller == "" {
|
||||||
|
caller = "a seat call whose caller the bus did not name"
|
||||||
|
}
|
||||||
|
cmd.Env = append(cmd.Env, link.CallerVar+"="+caller+", through the "+catalogue.ControllerSeatName+" seat")
|
||||||
// Two buffers, one answer. What the command *says* is both streams, in the order a person at
|
// Two buffers, one answer. What the command *says* is both streams, in the order a person at
|
||||||
// a shell would read them; what it *answers as data* is standard output alone — `status --json`
|
// a shell would read them; what it *answers as data* is standard output alone — `status --json`
|
||||||
// prints its warnings beside the document, and a JSON parsed from the two together parsed
|
// prints its warnings beside the document, and a JSON parsed from the two together parsed
|
||||||
@@ -501,6 +632,19 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
|||||||
if verb == "calls" {
|
if verb == "calls" {
|
||||||
return callsAnswer(link.Calls, a.given["call"])
|
return callsAnswer(link.Calls, a.given["call"])
|
||||||
}
|
}
|
||||||
|
if verb == "doctor" {
|
||||||
|
// From the serving controller, which runs the self-check and hears the signals
|
||||||
|
// (novox/hq to-be 45 §4): the last verdict at once, or a run now.
|
||||||
|
argv, err := a.commandLine()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
sub := ""
|
||||||
|
if len(argv) > 2 {
|
||||||
|
sub = argv[1]
|
||||||
|
}
|
||||||
|
return doctorAnswer(ctx, sub)
|
||||||
|
}
|
||||||
return seatTools(), nil
|
return seatTools(), nil
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
@@ -539,6 +683,14 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
if verb == "status" && statusFrom != nil {
|
||||||
|
// At once, from the summary the serving controller keeps (novox/hq to-be 45 Phase 0).
|
||||||
|
return statusFrom.answer(ctx)
|
||||||
|
}
|
||||||
|
if !readingVerbs[verb] && !(verb == "plans" && !actsOnAPlan(args)) {
|
||||||
|
// Whatever it did, `status` is composed again once it has.
|
||||||
|
defer statusFrom.nudge()
|
||||||
|
}
|
||||||
if answersFirst(argv) {
|
if answersFirst(argv) {
|
||||||
// Before anything is sent: a push sends the bus's own machine first, and a broker
|
// Before anything is sent: a push sends the bus's own machine first, and a broker
|
||||||
// reloading its user list forgets the answer it was about to permit (novox/hq issue 265).
|
// reloading its user list forgets the answer it was about to permit (novox/hq issue 265).
|
||||||
@@ -550,9 +702,19 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
|||||||
return handlers, behind, nil
|
return handlers, behind, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// actsOnAPlan is `plans` asked to stop, close or retry one rather than to show them.
|
||||||
|
func actsOnAPlan(args map[string]any) bool {
|
||||||
|
for _, act := range []string{"stop", "close", "retry"} {
|
||||||
|
if v, _ := args[act].(string); strings.TrimSpace(v) != "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// inProcess are the verbs answered by this process rather than by a command it runs: `tools` from
|
// inProcess are the verbs answered by this process rather than by a command it runs: `tools` from
|
||||||
// the records, `calls` from what this process served.
|
// the records, `calls` from what this process served.
|
||||||
var inProcess = map[string]bool{"tools": true, "calls": true}
|
var inProcess = map[string]bool{"tools": true, "calls": true, "doctor": true}
|
||||||
|
|
||||||
// answersFirst is a command line whose caller is answered before it runs: a push, by its verb or
|
// answersFirst is a command line whose caller is answered before it runs: a push, by its verb or
|
||||||
// through `command`. A push sends the machine holding the bus first when its user list changed, the
|
// through `command`. A push sends the machine holding the bus first when its user list changed, the
|
||||||
@@ -563,22 +725,41 @@ func answersFirst(argv []string) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// callsAnswer is what `calls` answers: the kept calls, newest first, without their answers — or
|
// callsAnswer is what `calls` answers: the kept calls, newest first, without their answers — or
|
||||||
// one call whole.
|
// one call whole. Kept on the bus, so a call a controller before this one served is answered too
|
||||||
|
// (novox/hq to-be 45 §6); where the bus cannot be read, what this process served is answered and
|
||||||
|
// the reason said beside it.
|
||||||
func callsAnswer(log *link.CallLog, id string) (any, error) {
|
func callsAnswer(log *link.CallLog, id string) (any, error) {
|
||||||
if id != "" {
|
if id != "" {
|
||||||
c, ok := log.Get(id)
|
c, ok, err := log.Get(id)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("call %s is not in this controller's memory, and the calls kept on the "+
|
||||||
|
"bus could not be read: %w", id, err)
|
||||||
|
}
|
||||||
if !ok {
|
if !ok {
|
||||||
|
if log.IsDurable() {
|
||||||
|
return nil, fmt.Errorf("no call %s is kept: the bus keeps the last %d calls, or %s, and this "+
|
||||||
|
"is not among them — `calls` lists them", id, broker.KeptCallsDurably, broker.CallsKeptFor)
|
||||||
|
}
|
||||||
return nil, fmt.Errorf("no call %s is kept here: calls are kept by the controller that "+
|
return nil, fmt.Errorf("no call %s is kept here: calls are kept by the controller that "+
|
||||||
"answered them, the last %d, and not across a restart — `calls` lists them", id, link.KeptCalls)
|
"answered them, the last %d, and not across a restart — `calls` lists them", id, link.KeptCalls)
|
||||||
}
|
}
|
||||||
return c, nil
|
return c, nil
|
||||||
}
|
}
|
||||||
recent := log.Recent()
|
recent, err := log.Recent()
|
||||||
for i := range recent {
|
for i := range recent {
|
||||||
recent[i].Answer = nil
|
recent[i].Answer = nil
|
||||||
}
|
}
|
||||||
return map[string]any{"calls": recent, "kept": link.KeptCalls,
|
answer := map[string]any{"calls": recent, "note": "newest first; `calls` with a call's id gives its whole answer"}
|
||||||
"note": "newest first; `calls` with a call's id gives its whole answer"}, nil
|
if log.IsDurable() {
|
||||||
|
answer["kept"] = fmt.Sprintf("the last %d calls, or %s, on the bus — across a restart of the controller",
|
||||||
|
broker.KeptCallsDurably, broker.CallsKeptFor)
|
||||||
|
} else {
|
||||||
|
answer["kept"] = fmt.Sprintf("the last %d calls this controller served, in its memory only", link.KeptCalls)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
answer["unread"] = err.Error()
|
||||||
|
}
|
||||||
|
return answer, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// seatTools is what `tools` answers: every seat with a protocol, and the tools each serves, from the
|
// seatTools is what `tools` answers: every seat with a protocol, and the tools each serves, from the
|
||||||
|
|||||||
@@ -145,17 +145,17 @@ func TestAnArgumentAVerbDoesNotDeclareIsRefused(t *testing.T) {
|
|||||||
// The push that was the cause: a machine named is that machine; none named is the whole mesh, and
|
// The push that was the cause: a machine named is that machine; none named is the whole mesh, and
|
||||||
// naming one beside behind is refused rather than one of the two guessed.
|
// naming one beside behind is refused rather than one of the two guessed.
|
||||||
func TestAPushIsOneMachineOrSaysItIsTheWholeMesh(t *testing.T) {
|
func TestAPushIsOneMachineOrSaysItIsTheWholeMesh(t *testing.T) {
|
||||||
argv, err := argvFor("push", map[string]any{"node": "g1"})
|
argv, err := argvFor("push", map[string]any{"node": "g1", "why": "w"})
|
||||||
if err != nil || strings.Join(argv, " ") != "push g1 --wait 0" {
|
if err != nil || strings.Join(argv, " ") != "push g1 --wait 0 --why w" {
|
||||||
t.Fatalf("a named push: %v %v", argv, err)
|
t.Fatalf("a named push: %v %v", argv, err)
|
||||||
}
|
}
|
||||||
for _, args := range []map[string]any{{}, {"behind": "true"}} {
|
for _, args := range []map[string]any{{"why": "w"}, {"behind": "true", "why": "w"}} {
|
||||||
argv, err := argvFor("push", args)
|
argv, err := argvFor("push", args)
|
||||||
if err != nil || strings.Join(argv, " ") != "push --behind --wait 0" {
|
if err != nil || strings.Join(argv, " ") != "push --behind --wait 0 --why w" {
|
||||||
t.Fatalf("a push of the whole mesh %v: %v %v", args, argv, err)
|
t.Fatalf("a push of the whole mesh %v: %v %v", args, argv, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if _, err := argvFor("push", map[string]any{"node": "g1", "behind": "true"}); err == nil ||
|
if _, err := argvFor("push", map[string]any{"node": "g1", "behind": "true", "why": "w"}); err == nil ||
|
||||||
!strings.Contains(err.Error(), `"behind"`) {
|
!strings.Contains(err.Error(), `"behind"`) {
|
||||||
t.Fatalf("a named push with behind was taken: %v", err)
|
t.Fatalf("a named push with behind was taken: %v", err)
|
||||||
}
|
}
|
||||||
@@ -268,6 +268,14 @@ var accountedFlags = map[string]map[string]string{
|
|||||||
},
|
},
|
||||||
"builds": {"n": "=limit"},
|
"builds": {"n": "=limit"},
|
||||||
"plans": {"n": "=limit", "what-if": "=repository"},
|
"plans": {"n": "=limit", "what-if": "=repository"},
|
||||||
|
"durations": {
|
||||||
|
"json": "set by the verb: the answer is data",
|
||||||
|
"all": "withheld: every measurement of a fortnight is more than a call should carry; `command` reaches it",
|
||||||
|
},
|
||||||
|
"hand-acts": {"json": "set by the verb: the answer is data"},
|
||||||
|
"conditions": {"json": "set by the verb: the answer is data"},
|
||||||
|
"conditions history": {"json": "set by the verb: the answer is data"},
|
||||||
|
"conditions show": {"json": "set by the verb: the answer is data"},
|
||||||
}
|
}
|
||||||
|
|
||||||
// **Every flag of the command a verb runs is in the verb's schema, or accounted for here.** Derived
|
// **Every flag of the command a verb runs is in the verb's schema, or accounted for here.** Derived
|
||||||
|
|||||||
@@ -43,6 +43,10 @@ func TestRotateTakesAProvisionOrAnOwnSecret(t *testing.T) {
|
|||||||
if strings.Join(argv, " ") != "rotate postgres-database --consumer ace" {
|
if strings.Join(argv, " ") != "rotate postgres-database --consumer ace" {
|
||||||
t.Fatalf("a pair credential: %v", argv)
|
t.Fatalf("a pair credential: %v", argv)
|
||||||
}
|
}
|
||||||
|
argv, _ = argvFor("rotate", map[string]any{"provision": "postgres-database", "consumer": "ace", "module": "letta"})
|
||||||
|
if strings.Join(argv, " ") != "rotate postgres-database --consumer ace --module letta" {
|
||||||
|
t.Fatalf("one consuming module's pair credential: %v", argv)
|
||||||
|
}
|
||||||
argv, _ = argvFor("rotate", map[string]any{"node": "ace", "module": "nodered", "secret": "api-token"})
|
argv, _ = argvFor("rotate", map[string]any{"node": "ace", "module": "nodered", "secret": "api-token"})
|
||||||
if strings.Join(argv, " ") != "secret rotate ace nodered api-token" {
|
if strings.Join(argv, " ") != "secret rotate ace nodered api-token" {
|
||||||
t.Fatalf("an own secret: %v", argv)
|
t.Fatalf("an own secret: %v", argv)
|
||||||
@@ -103,8 +107,8 @@ func TestAVerbMissingWhatItNeedsIsRefused(t *testing.T) {
|
|||||||
// A push and a build are sent, not waited for: the asker reads status, or the build's log by its
|
// A push and a build are sent, not waited for: the asker reads status, or the build's log by its
|
||||||
// id, for what happened. A repository given as a forge path is said to be one (issue 176).
|
// id, for what happened. A repository given as a forge path is said to be one (issue 176).
|
||||||
func TestActsDoNotBlockTheCall(t *testing.T) {
|
func TestActsDoNotBlockTheCall(t *testing.T) {
|
||||||
argv, _ := argvFor("push", map[string]any{"node": "one"})
|
argv, _ := argvFor("push", map[string]any{"node": "one", "why": "w"})
|
||||||
if strings.Join(argv, " ") != "push one --wait 0" {
|
if strings.Join(argv, " ") != "push one --wait 0 --why w" {
|
||||||
t.Fatalf("push waits: %v", argv)
|
t.Fatalf("push waits: %v", argv)
|
||||||
}
|
}
|
||||||
argv, _ = argvFor("build", map[string]any{"repository": "novox/x", "path": "modules/x"})
|
argv, _ = argvFor("build", map[string]any{"repository": "novox/x", "path": "modules/x"})
|
||||||
|
|||||||
@@ -43,6 +43,9 @@ type sendable struct {
|
|||||||
LeftOut []string
|
LeftOut []string
|
||||||
// leftOutWhy is why each was, for push and plan to say; never on the wire.
|
// leftOutWhy is why each was, for push and plan to say; never on the wire.
|
||||||
leftOutWhy map[string]string
|
leftOutWhy map[string]string
|
||||||
|
// withheld is every consumer this machine's grants leave out, because its identity overflows the
|
||||||
|
// provision's bound (novox/hq ADR 0225); for push and plan to say, never on the wire.
|
||||||
|
withheld []catalogue.Overflow
|
||||||
// Builds is the build of each module this declaration carries — module to the commit its build
|
// Builds is the build of each module this declaration carries — module to the commit its build
|
||||||
// was made from — recorded with the send and never on the wire (novox/hq issue 259, ADR 0221).
|
// was made from — recorded with the send and never on the wire (novox/hq issue 259, ADR 0221).
|
||||||
// Composed only on the send path; nil records that it is not known.
|
// Composed only on the send path; nil records that it is not known.
|
||||||
|
|||||||
@@ -0,0 +1,485 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The signals table (novox/hq to-be 45 §3, ADR 0227 rule 5), compiled in.
|
||||||
|
//
|
||||||
|
// **Every signal the core expects has a watchdog; its absence is a condition.** A row says what is
|
||||||
|
// expected, from whom, after what, within what bound, and what is raised when it does not come. One
|
||||||
|
// table, read three ways: by the watchdog loop (watchdogs.go), which runs every row's watch over the
|
||||||
|
// facts it gathered; by `doctor signals`, which says the age of every row's newest signal; and by the
|
||||||
|
// test generated from it (signals_test.go), which suppresses each signal in turn and asserts its
|
||||||
|
// condition — so a row added without a watch, or a watch that does not fire, fails the build.
|
||||||
|
//
|
||||||
|
// A row not watched yet says why, and in which phase it will be: a watchdog of a signal nothing emits
|
||||||
|
// would be a condition that can only cry wolf. Bounds marked provisional are set from what Phase 0
|
||||||
|
// measured (`durations`) and corrected in Phase 1's first live week; a corrected bound is a change to
|
||||||
|
// this table, reviewed like code.
|
||||||
|
|
||||||
|
// signalRow is one row of the table.
|
||||||
|
type signalRow struct {
|
||||||
|
Row string
|
||||||
|
Signal string
|
||||||
|
Emitter string
|
||||||
|
Trigger string
|
||||||
|
Bound string
|
||||||
|
Kind string
|
||||||
|
Severity conditions.Severity
|
||||||
|
// Phase is the phase of to-be 45 its watchdog is built in.
|
||||||
|
Phase int
|
||||||
|
// Deferred says why it is not watched yet; empty for a row that is.
|
||||||
|
Deferred string
|
||||||
|
// needs is the part of the facts the row reads: an error there is the row blind, which is
|
||||||
|
// itself said (probe-failed) rather than read as nothing wrong.
|
||||||
|
needs func(f *signalFacts) error
|
||||||
|
// watch is what is wrong now, from the facts.
|
||||||
|
watch func(f *signalFacts) []conditions.Observation
|
||||||
|
// newest is the time of the newest signal of this row, for `doctor signals`; zero when none.
|
||||||
|
newest func(f *signalFacts) time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// The bounds, provisional where to-be 45 says so.
|
||||||
|
const (
|
||||||
|
// heartbeatEvery is the interval a node-engine or node tools that say none have always used.
|
||||||
|
heartbeatEvery = 60 * time.Second
|
||||||
|
// heartbeatsMissed is how many intervals may pass in silence (S1, S11).
|
||||||
|
heartbeatsMissed = 3
|
||||||
|
// controlNodeUrgentAfter is how long the control node may be silent before it is urgent (S1).
|
||||||
|
controlNodeUrgentAfter = 30 * time.Minute
|
||||||
|
// reportAtLeast is the least a machine is given to report a send (S2).
|
||||||
|
reportAtLeast = 2 * time.Minute
|
||||||
|
// tierAtLeast is the least a plan's tier is given (S3), the bound `status` calls a plan late at.
|
||||||
|
tierAtLeast = planWaitBound
|
||||||
|
// loopDeafAfter is how long the event loop may take nothing while its consumers hold some (S4).
|
||||||
|
loopDeafAfter = 2 * time.Minute
|
||||||
|
// askAtLeast is the least a build ask is given, and askDefault the bound while nothing is
|
||||||
|
// measured (S6): the build seat declares no timeout of its own.
|
||||||
|
askAtLeast = 20 * time.Minute
|
||||||
|
askDefault = time.Hour
|
||||||
|
// callDefault is the bound of a verb that declares none (S7); push's and build's are longer.
|
||||||
|
callDefault = 10 * time.Minute
|
||||||
|
// advisoryQuiet is how long the bus must be quiet about a thing before its advisory clears (S9).
|
||||||
|
advisoryQuiet = time.Hour
|
||||||
|
// staleRefusalsAllowed in staleRefusalsWithin are what S13 lets pass from one writer.
|
||||||
|
staleRefusalsAllowed = 5
|
||||||
|
staleRefusalsWithin = 5 * time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
// callBounds are the verbs that may run longer than callDefault, and how long (S7).
|
||||||
|
var callBounds = map[string]time.Duration{
|
||||||
|
"push": 30 * time.Minute, "rotate": 30 * time.Minute, "assign": 15 * time.Minute,
|
||||||
|
"unassign": 15 * time.Minute, "command": 30 * time.Minute, "doctor": 3 * time.Minute,
|
||||||
|
}
|
||||||
|
|
||||||
|
// callBound is a verb's bound.
|
||||||
|
func callBound(verb string) time.Duration {
|
||||||
|
if b, ok := callBounds[verb]; ok {
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
return callDefault
|
||||||
|
}
|
||||||
|
|
||||||
|
// signalsTable is the table, in to-be 45's order.
|
||||||
|
var signalsTable = []signalRow{
|
||||||
|
{Row: "S1", Signal: "machine heartbeat", Emitter: "node-engine", Trigger: "its interval",
|
||||||
|
Bound: "3 × the interval it says (60 s when it says none), provisional; not raised while the machine " +
|
||||||
|
"said it is asleep or shutting down (ADR 0211); urgent after 30 min for a control node",
|
||||||
|
Kind: "silent", Severity: conditions.Warning, Phase: 1,
|
||||||
|
needs: func(f *signalFacts) error { return f.machinesErr }, watch: watchHeartbeats,
|
||||||
|
newest: func(f *signalFacts) time.Time {
|
||||||
|
return newestOf(f.machines, func(m machineFacts) time.Time { return m.lastHeard })
|
||||||
|
}},
|
||||||
|
{Row: "S2", Signal: "report after a send", Emitter: "node-engine", Trigger: "each declaration sent",
|
||||||
|
Bound: "max(2 min, 3 × that machine's last apply duration), provisional; not while the machine is silent or asleep",
|
||||||
|
Kind: "sent-not-reported", Severity: conditions.Warning, Phase: 1,
|
||||||
|
needs: func(f *signalFacts) error { return f.machinesErr }, watch: watchReports,
|
||||||
|
newest: func(f *signalFacts) time.Time {
|
||||||
|
return newestOf(f.machines, func(m machineFacts) time.Time { return m.reportedAt })
|
||||||
|
}},
|
||||||
|
{Row: "S3", Signal: "plan tier progress", Emitter: "controller's plan", Trigger: "each tier entered",
|
||||||
|
Bound: "max(30 min, 3 × the p90 of that repository's measured tiers), provisional; not while the " +
|
||||||
|
"build seat is paused under it",
|
||||||
|
Kind: "stalled", Severity: conditions.Warning, Phase: 1,
|
||||||
|
needs: func(f *signalFacts) error { return f.plansErr }, watch: watchPlans,
|
||||||
|
newest: func(f *signalFacts) time.Time {
|
||||||
|
return newestOf(f.plans, func(p planFacts) time.Time { return p.entered })
|
||||||
|
}},
|
||||||
|
{Row: "S4", Signal: "the controller's event loop takes a message", Emitter: "controller",
|
||||||
|
Trigger: "while its consumers have pending messages", Bound: "2 min",
|
||||||
|
Kind: "controller-deaf", Severity: conditions.Urgent, Phase: 1,
|
||||||
|
needs: func(f *signalFacts) error { return f.loopErr }, watch: watchLoop,
|
||||||
|
newest: func(f *signalFacts) time.Time { return f.loop.took }},
|
||||||
|
{Row: "S5", Signal: "a merge announced becomes a plan, or nothing reads it", Emitter: "announcer → controller",
|
||||||
|
Trigger: "each merge", Bound: "10 min (the catch-up pass of issue 266)",
|
||||||
|
Kind: "merge-not-acted", Severity: conditions.Urgent, Phase: 1,
|
||||||
|
needs: func(f *signalFacts) error { return f.mergesErr }, watch: watchMerges,
|
||||||
|
newest: func(f *signalFacts) time.Time { return f.mergesPassed }},
|
||||||
|
{Row: "S6", Signal: "a build asked → its outcome", Emitter: "build seat", Trigger: "each ask",
|
||||||
|
Bound: "max(20 min, 3 × the p90 of measured builds), 1 h while nothing is measured, provisional — the " +
|
||||||
|
"build seat declares no timeout; and an ask the queue gave up on (dead) at once",
|
||||||
|
Kind: "ask-lost", Severity: conditions.Warning, Phase: 1,
|
||||||
|
needs: func(f *signalFacts) error { return f.asksErr }, watch: watchAsks,
|
||||||
|
newest: func(f *signalFacts) time.Time { return newestOf(f.asks, func(a askFacts) time.Time { return a.since }) }},
|
||||||
|
{Row: "S7", Signal: "a call running → finished", Emitter: "controller", Trigger: "each call",
|
||||||
|
Bound: "the verb's bound: push, rotate and command 30 min, assign and unassign 15 min, doctor 3 min, " +
|
||||||
|
"any other 10 min",
|
||||||
|
Kind: "call-hung", Severity: conditions.Warning, Phase: 1,
|
||||||
|
needs: func(*signalFacts) error { return nil }, watch: watchCalls,
|
||||||
|
newest: func(f *signalFacts) time.Time {
|
||||||
|
return newestOf(f.calls, func(c link.Call) time.Time { return c.Started })
|
||||||
|
}},
|
||||||
|
{Row: "S8", Signal: "a provider's failing word repeated", Emitter: "provider",
|
||||||
|
Trigger: "every 15 min while failing (ADR 0224)", Bound: "30 min",
|
||||||
|
Kind: kindProviderSilent, Severity: conditions.Warning, Phase: 1,
|
||||||
|
needs: func(f *signalFacts) error { return f.standingsErr }, watch: watchProviders,
|
||||||
|
newest: func(f *signalFacts) time.Time {
|
||||||
|
return newestOf(f.standings, func(c conditions.Condition) time.Time { return c.LastObserved })
|
||||||
|
}},
|
||||||
|
{Row: "S9", Signal: "bus advisories: maximum deliveries, consumer deleted; the controller's own slow " +
|
||||||
|
"consumer and refused subjects", Emitter: "bus server's advisory subjects; the controller's connection",
|
||||||
|
Trigger: "any", Bound: "any occurrence; clears after an hour without another, and a deleted consumer " +
|
||||||
|
"once it exists again or the mesh no longer expects it",
|
||||||
|
Kind: "slow-consumer, max-deliveries, refused, consumer-lost", Severity: conditions.Warning, Phase: 1,
|
||||||
|
needs: func(f *signalFacts) error { return f.advisoriesErr }, watch: watchAdvisories,
|
||||||
|
newest: func(f *signalFacts) time.Time {
|
||||||
|
return newestOf(f.advisories, func(a link.Advisory) time.Time { return a.Last })
|
||||||
|
}},
|
||||||
|
{Row: "S10", Signal: "the self-check's heartbeat", Emitter: "controller's doctor", Trigger: "every run",
|
||||||
|
Bound: "2 × its interval; watched from a second machine by mesh-watcher, and here as well",
|
||||||
|
Kind: "self-check-silent", Severity: conditions.Urgent, Phase: 1,
|
||||||
|
needs: func(*signalFacts) error { return nil }, watch: watchSelfCheck,
|
||||||
|
newest: func(f *signalFacts) time.Time { return f.selfCheck.last }},
|
||||||
|
{Row: "S11", Signal: "node tools heartbeat", Emitter: "node tools", Trigger: "its interval",
|
||||||
|
Bound: "3 × the interval it says (60 s when it says none), provisional; only where node-tools is assigned",
|
||||||
|
Kind: "tools-silent", Severity: conditions.Warning, Phase: 1,
|
||||||
|
needs: func(f *signalFacts) error { return f.machinesErr }, watch: watchTools,
|
||||||
|
newest: func(f *signalFacts) time.Time {
|
||||||
|
return newestOf(f.machines, func(m machineFacts) time.Time { return m.toolsHeard })
|
||||||
|
}},
|
||||||
|
{Row: "S12", Signal: "the controller lease renewed", Emitter: "controller", Trigger: "every 5 s",
|
||||||
|
Bound: "15 s", Kind: "lease-lost", Severity: conditions.Urgent, Phase: 2,
|
||||||
|
Deferred: "the lease is built in Phase 2 (to-be 45 §6): there is nothing renewed to watch yet, and a " +
|
||||||
|
"second controller is caught today by its consumers being bound (standingBy)"},
|
||||||
|
{Row: "S13", Signal: "stale refusals", Emitter: "every receiver (rule 2)", Trigger: "each refusal",
|
||||||
|
Bound: "more than 5 from one machine in 5 min", Kind: "stale-writer", Severity: conditions.Warning, Phase: 1,
|
||||||
|
needs: func(*signalFacts) error { return nil }, watch: watchStaleRefusals,
|
||||||
|
newest: func(f *signalFacts) time.Time { return time.Time{} }},
|
||||||
|
{Row: "S14", Signal: "facts snapshot exported", Emitter: "controller", Trigger: "daily",
|
||||||
|
Bound: "2 days", Kind: "facts-stale", Severity: conditions.Warning, Phase: 5,
|
||||||
|
Deferred: "the facts snapshot is built in Phase 5 (to-be 45 §9): nothing exports one yet"},
|
||||||
|
{Row: "S15", Signal: "a hand act with a cause already recorded", Emitter: "hand-act log",
|
||||||
|
Trigger: "each act", Bound: "the second within 14 days", Kind: "healer-wanted",
|
||||||
|
Severity: conditions.Warning, Phase: 3,
|
||||||
|
Deferred: "Phase 3 (to-be 45 §10): `hand-acts` lists repeated causes today; the condition comes with the healers"},
|
||||||
|
}
|
||||||
|
|
||||||
|
// newestOf is the newest time among things.
|
||||||
|
func newestOf[T any](list []T, at func(T) time.Time) time.Time {
|
||||||
|
var newest time.Time
|
||||||
|
for _, x := range list {
|
||||||
|
if t := at(x); t.After(newest) {
|
||||||
|
newest = t
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return newest
|
||||||
|
}
|
||||||
|
|
||||||
|
// ago is a duration as the summaries say it.
|
||||||
|
func ago(d time.Duration) string {
|
||||||
|
if d < time.Minute {
|
||||||
|
return d.Round(time.Second).String()
|
||||||
|
}
|
||||||
|
return d.Round(time.Minute).String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// heartbeatBound is a machine's S1 or S11 bound from the interval it says.
|
||||||
|
func heartbeatBound(every time.Duration) time.Duration {
|
||||||
|
if every <= 0 {
|
||||||
|
every = heartbeatEvery
|
||||||
|
}
|
||||||
|
return heartbeatsMissed * every
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchHeartbeats(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, m := range f.machines {
|
||||||
|
if m.lastHeard.IsZero() || m.asleep() {
|
||||||
|
// Never heard is a machine that has not joined, which status says; asleep is not lost.
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
bound := heartbeatBound(m.every)
|
||||||
|
silent := f.now.Sub(m.lastHeard)
|
||||||
|
if silent <= bound {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
severity := conditions.Warning
|
||||||
|
if m.control && silent > controlNodeUrgentAfter {
|
||||||
|
severity = conditions.Urgent
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name, Kind: "silent",
|
||||||
|
Machine: m.name, Severity: severity,
|
||||||
|
Summary: fmt.Sprintf("%s has not been heard from since %s (bound %s)", m.name,
|
||||||
|
m.lastHeard.UTC().Format("2006-01-02 15:04 MST"), bound),
|
||||||
|
Said: fmt.Sprintf("silent for %s", ago(silent))})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchReports(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, m := range f.machines {
|
||||||
|
if m.sentAt.IsZero() || m.reportedCurrent || m.asleep() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) > heartbeatBound(m.every) {
|
||||||
|
continue // silent: S1 says it, and a silent machine reports nothing
|
||||||
|
}
|
||||||
|
bound := max(reportAtLeast, 3*m.lastApply)
|
||||||
|
waited := f.now.Sub(m.sentAt)
|
||||||
|
if waited <= bound {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name,
|
||||||
|
Kind: "sent-not-reported", Machine: m.name, Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s was sent a declaration at %s and has not reported applying it (bound %s)",
|
||||||
|
m.name, m.sentAt.UTC().Format("2006-01-02 15:04 MST"), ago(bound)),
|
||||||
|
Said: fmt.Sprintf("waiting %s for the report of the declaration sent", ago(waited))})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchPlans(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, p := range f.plans {
|
||||||
|
if p.paused {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
in := f.now.Sub(p.entered)
|
||||||
|
if in <= p.bound {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.id, Kind: "stalled",
|
||||||
|
Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("the plan for %s %s has been at tier %d of %d since %s (bound %s): %s",
|
||||||
|
p.repository, short(p.commit), p.tier+1, p.tiers, p.entered.UTC().Format("2006-01-02 15:04 MST"),
|
||||||
|
ago(p.bound), p.waiting),
|
||||||
|
Said: fmt.Sprintf("at tier %d for %s: %s", p.tier+1, ago(in), p.waiting)})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchLoop(f *signalFacts) []conditions.Observation {
|
||||||
|
if f.loop.pending == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
since := f.loop.took
|
||||||
|
if since.IsZero() {
|
||||||
|
since = f.started
|
||||||
|
}
|
||||||
|
if f.now.Sub(since) <= loopDeafAfter {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return []conditions.Observation{{Scope: conditions.ScopeCore, ID: "controller", Kind: "controller-deaf",
|
||||||
|
Token: "deaf", Machine: f.host, Severity: conditions.Urgent,
|
||||||
|
Summary: fmt.Sprintf("the controller's event loop has taken nothing for %s while its consumers hold %d "+
|
||||||
|
"message(s): reports, builds and merges are not being acted on", ago(f.now.Sub(since)), f.loop.pending),
|
||||||
|
Said: fmt.Sprintf("%d pending (%s), last taken %s", f.loop.pending, f.loop.where, since.UTC().Format(time.RFC3339))}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchMerges(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, m := range f.merges {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMerge, ID: m.Repo + "." + short(m.Commit),
|
||||||
|
Kind: "merge-not-acted", Token: "not-acted", Severity: conditions.Urgent,
|
||||||
|
Summary: fmt.Sprintf("%s/%s merged into %s (%s) was never handed to the controller by the bus; "+
|
||||||
|
"acted on late by the catch-up", m.Owner, m.Repo, m.Base, short(m.Commit)),
|
||||||
|
Said: fmt.Sprintf("announced %s, %s behind it", m.At.UTC().Format(time.RFC3339), readableList(m.Modules))})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchAsks(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, a := range f.asks {
|
||||||
|
var said string
|
||||||
|
switch {
|
||||||
|
case a.state == link.AskDead:
|
||||||
|
said = fmt.Sprintf("the %s queue handed it out as often as it may and it was never settled", a.seat)
|
||||||
|
case a.state == link.AskInFlight && f.now.Sub(a.since) > a.bound:
|
||||||
|
said = fmt.Sprintf("in flight on %s for %s (bound %s)", orSomewhere(a.on), ago(f.now.Sub(a.since)), ago(a.bound))
|
||||||
|
default:
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBuild, ID: a.id, Kind: "ask-lost",
|
||||||
|
Token: "lost", Machine: a.on, Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("the build %s of %s has no outcome: %s", a.id, a.what, said), Said: said})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func orSomewhere(node string) string {
|
||||||
|
if node == "" {
|
||||||
|
return "a machine that did not say which"
|
||||||
|
}
|
||||||
|
return node
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchCalls(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, c := range f.calls {
|
||||||
|
bound := callBound(c.Verb)
|
||||||
|
running := f.now.Sub(c.Started)
|
||||||
|
if running <= bound {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeCall, ID: c.ID, Kind: "call-hung",
|
||||||
|
Token: "hung", Machine: f.host, Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s.%s (call %s) has been running since %s, past its bound of %s", c.Seat, c.Verb,
|
||||||
|
c.ID, c.Started.UTC().Format("2006-01-02 15:04 MST"), ago(bound)),
|
||||||
|
Said: fmt.Sprintf("running %s, asked by %s", ago(running), orSomebody(c.Caller))})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func orSomebody(caller string) string {
|
||||||
|
if caller == "" {
|
||||||
|
return "a caller the bus did not name"
|
||||||
|
}
|
||||||
|
return caller
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchProviders(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, c := range f.standings {
|
||||||
|
quiet := f.now.Sub(c.LastObserved)
|
||||||
|
if quiet <= providerSaysAgainWithin {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
module, node, consumer, ok := providerOf(c)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeProvider, ID: module + "." + node + "." + consumer,
|
||||||
|
Token: "silent", Kind: kindProviderSilent, Machine: node, Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s on %s said it keeps failing %s and has said nothing since %s: it stopped "+
|
||||||
|
"saying anything, so its last word is all the mesh has", module, node, consumer,
|
||||||
|
c.LastObserved.UTC().Format("2006-01-02 15:04 MST")),
|
||||||
|
Said: fmt.Sprintf("not said again for %s", ago(quiet))})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchAdvisories(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, a := range f.advisories {
|
||||||
|
if f.now.Sub(a.Last) > advisoryQuiet {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if a.Kind == link.AdvisoryConsumerLost && !f.lostConsumers[a.Stream+"."+a.Consumer] {
|
||||||
|
continue // it exists again, or the mesh no longer expects it: a removal, not a loss
|
||||||
|
}
|
||||||
|
severity := conditions.Warning
|
||||||
|
times := ""
|
||||||
|
if a.Count > 1 {
|
||||||
|
times = fmt.Sprintf(" (%d times since %s)", a.Count, a.First.UTC().Format("15:04 MST"))
|
||||||
|
}
|
||||||
|
machine := ""
|
||||||
|
if a.ID == "controller" {
|
||||||
|
machine = f.host
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: a.ID, Kind: a.Kind,
|
||||||
|
Machine: machine, Severity: severity, Summary: a.Said + times, Said: a.Said})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchSelfCheck(f *signalFacts) []conditions.Observation {
|
||||||
|
every := f.selfCheck.every
|
||||||
|
if every <= 0 {
|
||||||
|
every = doctorEvery
|
||||||
|
}
|
||||||
|
since := f.selfCheck.last
|
||||||
|
if since.IsZero() {
|
||||||
|
since = f.started
|
||||||
|
}
|
||||||
|
if f.now.Sub(since) <= 2*every {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return []conditions.Observation{{Scope: conditions.ScopeCore, ID: "doctor", Kind: "self-check-silent",
|
||||||
|
Machine: f.host, Severity: conditions.Urgent,
|
||||||
|
Summary: fmt.Sprintf("the self-check has not finished a run since %s (it runs every %s): the mesh's "+
|
||||||
|
"invariants are not being checked", since.UTC().Format("2006-01-02 15:04 MST"), every),
|
||||||
|
Said: fmt.Sprintf("no run for %s", ago(f.now.Sub(since)))}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchTools(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, m := range f.machines {
|
||||||
|
if !m.tools || m.asleep() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
bound := heartbeatBound(m.toolsEvery)
|
||||||
|
since := m.toolsHeard
|
||||||
|
if since.IsZero() {
|
||||||
|
// Not heard since this controller started: silent since then at the most.
|
||||||
|
since = f.toolsHeardFrom
|
||||||
|
}
|
||||||
|
silent := f.now.Sub(since)
|
||||||
|
if silent <= bound {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
heard := "not since this controller started at " + since.UTC().Format("2006-01-02 15:04 MST")
|
||||||
|
if !m.toolsHeard.IsZero() {
|
||||||
|
heard = "since " + m.toolsHeard.UTC().Format("2006-01-02 15:04 MST")
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name, Kind: "tools-silent",
|
||||||
|
Machine: m.name, Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("the node tools on %s have not said they are there %s (bound %s): nothing can "+
|
||||||
|
"ask that machine anything", m.name, heard, bound),
|
||||||
|
Said: fmt.Sprintf("silent for %s", ago(silent))})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchStaleRefusals(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for node, n := range f.staleRefusals {
|
||||||
|
if n <= staleRefusalsAllowed {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: "stale-writer",
|
||||||
|
Machine: node, Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s refused %d declarations in %s as older than the one it holds: a controller "+
|
||||||
|
"is sending what it has moved past (which one is said once declarations carry an epoch, Phase 2)",
|
||||||
|
node, n, staleRefusalsWithin),
|
||||||
|
Said: fmt.Sprintf("%d stale refusals in %s", n, staleRefusalsWithin)})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// watchedRows are the rows a watchdog runs for.
|
||||||
|
func watchedRows() []signalRow {
|
||||||
|
var out []signalRow
|
||||||
|
for _, r := range signalsTable {
|
||||||
|
if r.watch != nil {
|
||||||
|
out = append(out, r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// kindsOf is a row's condition kinds, one or several.
|
||||||
|
func kindsOf(r signalRow) []string {
|
||||||
|
var out []string
|
||||||
|
for _, k := range strings.Split(r.Kind, ",") {
|
||||||
|
out = append(out, strings.TrimSpace(k))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,275 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The test generated from the signals table (novox/hq to-be 45 §3, ADR 0227 rule 5, "how it is
|
||||||
|
// checked"): **every row is walked**. A watched row's signal is suppressed just inside its bound —
|
||||||
|
// nothing raised — and just past it — its condition raised, with its kind and severity — and restored,
|
||||||
|
// and the condition clears. A row that is not watched says why. A row added to the table without a
|
||||||
|
// suppression here fails, so the table cannot grow a watchdog nobody has seen fire.
|
||||||
|
|
||||||
|
// calm is a mesh whose every signal is fresh: one control node heard ten seconds ago, its last send
|
||||||
|
// reported, a plan a minute into its tier, the loop taking, the merges read, nothing asked, no call
|
||||||
|
// running, the self-check a minute old.
|
||||||
|
func calm(now time.Time) *signalFacts {
|
||||||
|
return &signalFacts{now: now, started: now.Add(-time.Hour), host: "anchor", toolsHeardFrom: now.Add(-time.Hour),
|
||||||
|
machines: []machineFacts{{name: "anchor", control: true, lastHeard: now.Add(-10 * time.Second),
|
||||||
|
every: time.Minute, sentAt: now.Add(-time.Hour), reportedCurrent: true, reportedAt: now.Add(-59 * time.Minute),
|
||||||
|
lastApply: 20 * time.Second, tools: true, toolsHeard: now.Add(-10 * time.Second), toolsEvery: time.Minute}},
|
||||||
|
plans: []planFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee00", tier: 0, tiers: 2,
|
||||||
|
entered: now.Add(-time.Minute), bound: 30 * time.Minute, waiting: "building"}},
|
||||||
|
loop: loopFacts{took: now.Add(-time.Second), pending: 1},
|
||||||
|
mergesPassed: now.Add(-time.Minute),
|
||||||
|
selfCheck: selfCheckFacts{last: now.Add(-time.Minute), every: 5 * time.Minute},
|
||||||
|
lostConsumers: map[string]bool{}, staleRefusals: map[string]int{},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// suppression is one row's signal held back: inside its bound, and past it.
|
||||||
|
type suppression struct{ inside, past func(f *signalFacts) }
|
||||||
|
|
||||||
|
// suppressions are every watched row's, by row.
|
||||||
|
var suppressions = map[string]suppression{
|
||||||
|
"S1": {
|
||||||
|
inside: func(f *signalFacts) { f.machines[0].lastHeard = f.now.Add(-3*time.Minute + time.Second) },
|
||||||
|
past: func(f *signalFacts) { f.machines[0].lastHeard = f.now.Add(-3*time.Minute - time.Second) },
|
||||||
|
},
|
||||||
|
"S2": {
|
||||||
|
inside: func(f *signalFacts) {
|
||||||
|
f.machines[0].sentAt, f.machines[0].reportedCurrent = f.now.Add(-110*time.Second), false
|
||||||
|
},
|
||||||
|
past: func(f *signalFacts) {
|
||||||
|
f.machines[0].sentAt, f.machines[0].reportedCurrent = f.now.Add(-121*time.Second), false
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"S3": {
|
||||||
|
inside: func(f *signalFacts) { f.plans[0].entered = f.now.Add(-29 * time.Minute) },
|
||||||
|
past: func(f *signalFacts) { f.plans[0].entered = f.now.Add(-31 * time.Minute) },
|
||||||
|
},
|
||||||
|
"S4": {
|
||||||
|
inside: func(f *signalFacts) { f.loop.took = f.now.Add(-119 * time.Second) },
|
||||||
|
past: func(f *signalFacts) { f.loop.took = f.now.Add(-121 * time.Second) },
|
||||||
|
},
|
||||||
|
"S5": {
|
||||||
|
inside: func(f *signalFacts) {},
|
||||||
|
past: func(f *signalFacts) {
|
||||||
|
f.merges = []missedMerge{{Owner: "novox", Repo: "app", Base: "main", Commit: "c0ffee0011", At: f.now.Add(-11 * time.Minute)}}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"S6": {
|
||||||
|
inside: func(f *signalFacts) {
|
||||||
|
f.asks = []askFacts{{id: "build-1", seat: "node-build-agent", what: "novox/app", state: link.AskInFlight,
|
||||||
|
on: "anchor", since: f.now.Add(-59 * time.Minute), bound: time.Hour}}
|
||||||
|
},
|
||||||
|
past: func(f *signalFacts) {
|
||||||
|
f.asks = []askFacts{{id: "build-1", seat: "node-build-agent", what: "novox/app", state: link.AskInFlight,
|
||||||
|
on: "anchor", since: f.now.Add(-61 * time.Minute), bound: time.Hour}}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"S7": {
|
||||||
|
inside: func(f *signalFacts) {
|
||||||
|
f.calls = []link.Call{{ID: "call-1", Seat: "mesh-controller", Verb: "push", Started: f.now.Add(-29 * time.Minute)}}
|
||||||
|
},
|
||||||
|
past: func(f *signalFacts) {
|
||||||
|
f.calls = []link.Call{{ID: "call-1", Seat: "mesh-controller", Verb: "push", Started: f.now.Add(-31 * time.Minute)}}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"S8": {
|
||||||
|
inside: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-29 * time.Minute))} },
|
||||||
|
past: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-31 * time.Minute))} },
|
||||||
|
},
|
||||||
|
"S9": {
|
||||||
|
inside: func(f *signalFacts) {
|
||||||
|
f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop", Said: "gave up",
|
||||||
|
First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-61 * time.Minute), Count: 1}}
|
||||||
|
},
|
||||||
|
past: func(f *signalFacts) {
|
||||||
|
f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop", Said: "gave up",
|
||||||
|
First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-59 * time.Minute), Count: 1}}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"S10": {
|
||||||
|
inside: func(f *signalFacts) { f.selfCheck.last = f.now.Add(-9 * time.Minute) },
|
||||||
|
past: func(f *signalFacts) { f.selfCheck.last = f.now.Add(-11 * time.Minute) },
|
||||||
|
},
|
||||||
|
"S11": {
|
||||||
|
inside: func(f *signalFacts) { f.machines[0].toolsHeard = f.now.Add(-179 * time.Second) },
|
||||||
|
past: func(f *signalFacts) { f.machines[0].toolsHeard = f.now.Add(-181 * time.Second) },
|
||||||
|
},
|
||||||
|
"S13": {
|
||||||
|
inside: func(f *signalFacts) { f.staleRefusals = map[string]int{"anchor": 5} },
|
||||||
|
past: func(f *signalFacts) { f.staleRefusals = map[string]int{"anchor": 6} },
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// standingSaid is a provider's failing word last said at a moment.
|
||||||
|
func standingSaid(at time.Time) conditions.Condition {
|
||||||
|
return conditions.Condition{Key: "provider.idp.anchor.app.failing", Kind: kindProviderFailing, LastObserved: at}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEveryRowOfTheSignalsTableIsWatchedRaisedAndCleared(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for _, row := range signalsTable {
|
||||||
|
t.Run(row.Row, func(t *testing.T) {
|
||||||
|
if seen[row.Row] {
|
||||||
|
t.Fatalf("%s is in the table twice", row.Row)
|
||||||
|
}
|
||||||
|
seen[row.Row] = true
|
||||||
|
if row.Signal == "" || row.Emitter == "" || row.Trigger == "" || row.Bound == "" || row.Kind == "" ||
|
||||||
|
(row.Severity != conditions.Urgent && row.Severity != conditions.Warning) {
|
||||||
|
t.Fatalf("%s does not say what it expects, from whom, within what, and what it raises: %+v", row.Row, row)
|
||||||
|
}
|
||||||
|
if row.Deferred != "" {
|
||||||
|
if row.watch != nil || row.Phase <= 1 {
|
||||||
|
t.Fatalf("%s is deferred and watched, or deferred out of Phase 1's own rows: %+v", row.Row, row)
|
||||||
|
}
|
||||||
|
if _, has := suppressions[row.Row]; has {
|
||||||
|
t.Fatalf("%s is deferred and has a suppression: one of the two is stale", row.Row)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if row.watch == nil || row.needs == nil || row.newest == nil {
|
||||||
|
t.Fatalf("%s is watched and lacks its watch, its needs or its newest", row.Row)
|
||||||
|
}
|
||||||
|
s, ok := suppressions[row.Row]
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("%s has no suppression in this test: a watchdog nobody has seen fire", row.Row)
|
||||||
|
}
|
||||||
|
if got := row.watch(calm(now)); len(got) != 0 {
|
||||||
|
t.Fatalf("%s raised on a calm mesh: %+v", row.Row, got)
|
||||||
|
}
|
||||||
|
inside := calm(now)
|
||||||
|
s.inside(inside)
|
||||||
|
if got := row.watch(inside); len(got) != 0 {
|
||||||
|
t.Fatalf("%s raised inside its bound: %+v", row.Row, got)
|
||||||
|
}
|
||||||
|
past := calm(now)
|
||||||
|
s.past(past)
|
||||||
|
got := row.watch(past)
|
||||||
|
if len(got) == 0 {
|
||||||
|
t.Fatalf("%s raised nothing past its bound", row.Row)
|
||||||
|
}
|
||||||
|
for _, o := range got {
|
||||||
|
if !slices.Contains(kindsOf(row), o.Kind) {
|
||||||
|
t.Errorf("%s raised %q, which is not its kind %q", row.Row, o.Kind, row.Kind)
|
||||||
|
}
|
||||||
|
if o.Severity != row.Severity {
|
||||||
|
t.Errorf("%s raised %s, the table says %s", row.Row, o.Severity, row.Severity)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(o.Summary) == "" {
|
||||||
|
t.Errorf("%s raised a condition that says nothing", row.Row)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Through the store: raised past the bound, cleared when the signal returns.
|
||||||
|
store := conditions.NewInMemory()
|
||||||
|
told := &conditions.Told{}
|
||||||
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: told,
|
||||||
|
Now: func() time.Time { return now }})
|
||||||
|
defer k.Close(context.Background())
|
||||||
|
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
||||||
|
w.see(t.Context(), past)
|
||||||
|
open, err := k.Open(t.Context())
|
||||||
|
if err != nil || len(open) != len(got) {
|
||||||
|
t.Fatalf("%s past its bound left %d open (%v), want %d", row.Row, len(open), err, len(got))
|
||||||
|
}
|
||||||
|
w.see(t.Context(), calm(now))
|
||||||
|
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||||
|
t.Fatalf("%s's condition stayed open after the signal returned: %+v", row.Row, open)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
for name := range suppressions {
|
||||||
|
if !seen[name] {
|
||||||
|
t.Errorf("a suppression for %s, which the table does not have", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The control node silent for half an hour is urgent** (S1); any other machine stays a warning.
|
||||||
|
func TestTheControlNodeSilentIsUrgentAfterHalfAnHour(t *testing.T) {
|
||||||
|
now := time.Now()
|
||||||
|
f := calm(now)
|
||||||
|
f.machines[0].lastHeard = now.Add(-31 * time.Minute)
|
||||||
|
f.machines = append(f.machines, machineFacts{name: "laptop", lastHeard: now.Add(-31 * time.Minute)})
|
||||||
|
got := watchHeartbeats(f)
|
||||||
|
if len(got) != 2 || got[0].Severity != conditions.Urgent || got[1].Severity != conditions.Warning {
|
||||||
|
t.Fatalf("%+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A machine that said it sleeps is not silent** (ADR 0211), nor late to report; one that woke is.
|
||||||
|
func TestAMachineThatSaidItSleepsIsNotSilent(t *testing.T) {
|
||||||
|
now := time.Now()
|
||||||
|
f := calm(now)
|
||||||
|
f.machines[0].lastHeard = now.Add(-2 * time.Hour)
|
||||||
|
f.machines[0].sentAt, f.machines[0].reportedCurrent = now.Add(-time.Hour), false
|
||||||
|
f.machines[0].power = link.PowerState{State: "sleeping", At: now.Add(-2 * time.Hour)}
|
||||||
|
if got := append(watchHeartbeats(f), append(watchReports(f), watchTools(f)...)...); len(got) != 0 {
|
||||||
|
t.Fatalf("a sleeping machine raised %+v", got)
|
||||||
|
}
|
||||||
|
f.machines[0].power = link.PowerState{State: "woke", At: now.Add(-time.Hour)}
|
||||||
|
if got := watchHeartbeats(f); len(got) != 1 {
|
||||||
|
t.Fatalf("a woken machine silent past its bound raised %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A watchdog that cannot see says so, and clears nothing it raised** (ADR 0227 rule 4): the store
|
||||||
|
// unreadable is a probe-failed of its own, and the machine's silence stays open until it can see again.
|
||||||
|
func TestABlindWatchdogSaysSoAndClearsNothing(t *testing.T) {
|
||||||
|
now := time.Now()
|
||||||
|
store := conditions.NewInMemory()
|
||||||
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||||
|
defer k.Close(context.Background())
|
||||||
|
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
||||||
|
silent := calm(now)
|
||||||
|
silent.machines[0].lastHeard = now.Add(-10 * time.Minute)
|
||||||
|
w.see(t.Context(), silent)
|
||||||
|
blind := calm(now)
|
||||||
|
blind.machines, blind.machinesErr = nil, errors.New("the store is away")
|
||||||
|
w.see(t.Context(), blind)
|
||||||
|
open, err := k.Open(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var keys []string
|
||||||
|
for _, c := range open {
|
||||||
|
keys = append(keys, c.Key)
|
||||||
|
}
|
||||||
|
for _, want := range []string{"machine.anchor.silent", "probe.S1.failed", "probe.S2.failed", "probe.S11.failed"} {
|
||||||
|
if !slices.Contains(keys, want) {
|
||||||
|
t.Errorf("%s is not open while the machines cannot be read: %v", want, keys)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
w.see(t.Context(), calm(now))
|
||||||
|
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||||
|
t.Fatalf("seeing again left open %+v", open)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A controller standing by sees nothing and says nothing**: it hears no heartbeat, and would call
|
||||||
|
// every machine silent.
|
||||||
|
func TestAControllerStandingBySaysNothing(t *testing.T) {
|
||||||
|
store := conditions.NewInMemory()
|
||||||
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||||
|
defer k.Close(context.Background())
|
||||||
|
w := &watchdogs{keeper: k, started: time.Now(), acting: func() bool { return false }}
|
||||||
|
w.tick(t.Context())
|
||||||
|
if w.lastTick().IsZero() {
|
||||||
|
t.Fatal("a tick standing by was not counted")
|
||||||
|
}
|
||||||
|
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||||
|
t.Fatalf("%+v", open)
|
||||||
|
}
|
||||||
|
}
|
||||||
+123
-76
@@ -2,91 +2,153 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
)
|
)
|
||||||
|
|
||||||
// A provider that keeps failing a consumer is a problem the controller reports (novox/hq ADR 0224).
|
// A provider that keeps failing a consumer is a problem the controller reports (novox/hq ADR 0224) —
|
||||||
|
// **the condition store's first kind** (to-be 45 §2, ADR 0227).
|
||||||
//
|
//
|
||||||
// On 2026-10-05 the identity provider's provisioner failed every consumer from shortly after midnight
|
// On 2026-10-05 the identity provider's provisioner failed every consumer from shortly after midnight
|
||||||
// until it was fixed by hand that night — 31,000 refused logins after its database was moved and its
|
// until it was fixed by hand that night — 31,000 refused logins after its database was moved and its
|
||||||
// admin kept an older password — and `status` called the mesh well all day (novox/hq issue 179). A
|
// admin kept an older password — and `status` called the mesh well all day (novox/hq issue 179). A
|
||||||
// provider now announces a consumer it has failed for minutes; the controller keeps it until the
|
// provider announces a consumer it has failed for minutes; the controller keeps it until the provider
|
||||||
// provider says it recovered; and `status`, its JSON and `node show` name it, breaking "all well".
|
// says it recovered; and `status`, its JSON and `node show` name it, breaking "all well". Unchanged in
|
||||||
|
// what it says and when; kept as a condition, `provider.<module>.<node>.<consumer>.failing`, rather
|
||||||
|
// than a row of its own, so it is said outward like every other fault and silenced like one.
|
||||||
|
|
||||||
// standings keeps what providers say, in the inventory.
|
// Kinds of the provider standing.
|
||||||
type standings struct{ inv *inventory.Inventory }
|
const (
|
||||||
|
kindProviderFailing = "provider-failing"
|
||||||
|
kindProviderSilent = "provider-silent"
|
||||||
|
// sourceProvisioner is what raised a standing: the provider's own event.
|
||||||
|
sourceProvisioner = "provisioner.failing"
|
||||||
|
)
|
||||||
|
|
||||||
func (s standings) Stood(ctx context.Context, st link.Standing) (bool, error) {
|
// providerSaysAgainWithin is how long a failing word stays current without being said again: twice
|
||||||
return s.inv.KeepStanding(ctx, st.Failing, inventory.ProviderStanding{
|
// the quarter of an hour a provider repeats it at (ADR 0224). Past it, S8.
|
||||||
Module: st.Module, ProviderNode: st.ProviderNode, Provision: st.Provider,
|
const providerSaysAgainWithin = 30 * time.Minute
|
||||||
Consumer: st.Consumer, ConsumerNode: st.Node,
|
|
||||||
Class: st.Class, Error: st.Error, Since: st.Since, Attempts: st.Attempts,
|
// standings keeps what providers say, as conditions.
|
||||||
})
|
type standings struct {
|
||||||
|
keeper func() *conditions.Keeper
|
||||||
}
|
}
|
||||||
|
|
||||||
// failingProviders is every consumer a provider still assigned where it ran says it keeps failing.
|
// standingObservation is a provider's failing word as an observation: the provider, its machine and
|
||||||
//
|
// the consumer name it, so the same consumer failed again is the same condition.
|
||||||
// **A provider no longer assigned is not asked about.** Its last word stays in the store, and is
|
func standingObservation(st link.Standing) conditions.Observation {
|
||||||
// not a problem: nothing runs there to fail anybody. Assigned again, its first success for each
|
whom := st.Consumer
|
||||||
// consumer clears it.
|
if st.Node != "" {
|
||||||
func failingProviders(ctx context.Context, inv *inventory.Inventory) ([]inventory.ProviderStanding, error) {
|
whom += " on " + st.Node
|
||||||
all, err := inv.FailingProviders(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("what providers say they keep failing cannot be read: %w", err)
|
|
||||||
}
|
}
|
||||||
|
summary := fmt.Sprintf("%s on %s keeps failing %s: %s, %d attempt(s) since %s", st.Module, st.ProviderNode,
|
||||||
|
whom, orUnclassed(st.Class), st.Attempts, st.Since.UTC().Format("2006-01-02 15:04 MST"))
|
||||||
|
said := orUnclassed(st.Class)
|
||||||
|
if e := firstLine(st.Error); e != "" {
|
||||||
|
said += ": " + e
|
||||||
|
}
|
||||||
|
if st.Provider != "" {
|
||||||
|
said += fmt.Sprintf(" (provision %s, %d attempts)", st.Provider, st.Attempts)
|
||||||
|
}
|
||||||
|
return conditions.Observation{Scope: conditions.ScopeProvider,
|
||||||
|
ID: st.Module + "." + st.ProviderNode + "." + st.Consumer,
|
||||||
|
Token: "failing", Kind: kindProviderFailing, Machine: st.ProviderNode, Also: alsoOn(st.Node, st.ProviderNode),
|
||||||
|
Severity: conditions.Warning, Summary: summary, Said: said, Source: sourceProvisioner}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stood keeps a provider's newest word: failing raises or observes its condition, recovered clears
|
||||||
|
// it. An error is the store away, and the link holds the message to be asked again — a recovery is
|
||||||
|
// said once, and dropping it would leave a consumer named failing that is fine.
|
||||||
|
func (s standings) Stood(ctx context.Context, st link.Standing) (bool, error) {
|
||||||
|
k := s.keeper()
|
||||||
|
if k == nil {
|
||||||
|
return false, fmt.Errorf("the condition store is not open in this controller: %w", link.ErrTryAgain)
|
||||||
|
}
|
||||||
|
o := standingObservation(st)
|
||||||
|
if !st.Failing {
|
||||||
|
why := "the provider says it recovered"
|
||||||
|
if st.Why != "" {
|
||||||
|
why += ": " + st.Why
|
||||||
|
}
|
||||||
|
cleared, err := k.Clear(ctx, o.Key(), why)
|
||||||
|
return cleared, storeAway(err)
|
||||||
|
}
|
||||||
|
_, err := k.Observe(ctx, o)
|
||||||
|
return false, storeAway(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// storeAway reads the condition store failing as the bus being away for the moment: the link holds the
|
||||||
|
// message and asks again, as it does for a store restarting (ADR 0083), rather than taking it unkept.
|
||||||
|
func storeAway(err error) error {
|
||||||
|
if err == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return fmt.Errorf("%v: %w", err, link.ErrTryAgain)
|
||||||
|
}
|
||||||
|
|
||||||
|
// providerStandings is every open provider-failing condition, from what is open.
|
||||||
|
func providerStandings(open []conditions.Condition) []conditions.Condition {
|
||||||
|
var out []conditions.Condition
|
||||||
|
for _, c := range open {
|
||||||
|
if c.Kind == kindProviderFailing {
|
||||||
|
out = append(out, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// providerOf reads a standing's provider module and machine back from its key.
|
||||||
|
func providerOf(c conditions.Condition) (module, node, consumer string, ok bool) {
|
||||||
|
parts := strings.Split(c.Key, ".")
|
||||||
|
if len(parts) != 5 || parts[0] != conditions.ScopeProvider {
|
||||||
|
return "", "", "", false
|
||||||
|
}
|
||||||
|
return parts[1], parts[2], parts[3], true
|
||||||
|
}
|
||||||
|
|
||||||
|
// unassignedProviders clears the standing of every provider no longer assigned where it ran.
|
||||||
|
//
|
||||||
|
// **A provider no longer assigned is not asked about** (ADR 0224 §4): nothing runs there to fail
|
||||||
|
// anybody, and nothing there will ever say it recovered. The observation that resolves it is the
|
||||||
|
// assignment. Assigned again, its first failure raises it again.
|
||||||
|
func unassignedProviders(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper,
|
||||||
|
open []conditions.Condition) error {
|
||||||
assigned := map[string]map[string]bool{}
|
assigned := map[string]map[string]bool{}
|
||||||
var out []inventory.ProviderStanding
|
for _, c := range providerStandings(open) {
|
||||||
for _, s := range all {
|
module, node, _, ok := providerOf(c)
|
||||||
on, asked := assigned[s.ProviderNode]
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
on, asked := assigned[node]
|
||||||
if !asked {
|
if !asked {
|
||||||
modules, err := inv.Assigned(ctx, s.ProviderNode)
|
modules, err := inv.Assigned(ctx, node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// A provider on a machine the mesh no longer knows has nothing running to fail anybody.
|
if errors.Is(err, inventory.ErrNoSuchNode) {
|
||||||
modules = nil
|
modules = nil // a machine the mesh no longer knows runs nothing
|
||||||
|
} else {
|
||||||
|
return fmt.Errorf("what %s is assigned cannot be read: %w", node, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
on = map[string]bool{}
|
on = map[string]bool{}
|
||||||
for _, m := range modules {
|
for _, m := range modules {
|
||||||
on[m] = true
|
on[m] = true
|
||||||
}
|
}
|
||||||
assigned[s.ProviderNode] = on
|
assigned[node] = on
|
||||||
}
|
}
|
||||||
if on[s.Module] {
|
if !on[module] {
|
||||||
out = append(out, s)
|
if _, err := k.Clear(ctx, c.Key, module+" is no longer assigned to "+node+
|
||||||
|
": nothing runs there to fail anybody"); err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// failingLines is how status says them: one consumer per entry, the error under it, and a provider
|
|
||||||
// that stopped repeating itself said so.
|
|
||||||
func failingLines(list []inventory.ProviderStanding, now time.Time) []string {
|
|
||||||
var out []string
|
|
||||||
for _, s := range list {
|
|
||||||
where := s.Module
|
|
||||||
if s.ProviderNode != "" {
|
|
||||||
where += " on " + s.ProviderNode
|
|
||||||
}
|
}
|
||||||
whom := s.Consumer
|
return nil
|
||||||
if s.ConsumerNode != "" {
|
|
||||||
whom += " (" + s.ConsumerNode + ")"
|
|
||||||
}
|
|
||||||
out = append(out, fmt.Sprintf(" %-24s fails %s: %s, for %s (%d attempts since %s)",
|
|
||||||
where, whom, orUnclassed(s.Class), roughly(now.Sub(s.Since)), s.Attempts,
|
|
||||||
s.Since.Local().Format("2006-01-02 15:04")))
|
|
||||||
if e := strings.TrimSpace(s.Error); e != "" {
|
|
||||||
out = append(out, fmt.Sprintf(" %-24s %s", "", firstLine(e)))
|
|
||||||
}
|
|
||||||
if s.Quiet(now) {
|
|
||||||
out = append(out, fmt.Sprintf(" %-24s not said again for %s — the provider has stopped "+
|
|
||||||
"saying anything, so this is its last word", "", roughly(now.Sub(s.SaidAt))))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func orUnclassed(class string) string {
|
func orUnclassed(class string) string {
|
||||||
@@ -96,25 +158,10 @@ func orUnclassed(class string) string {
|
|||||||
return class
|
return class
|
||||||
}
|
}
|
||||||
|
|
||||||
// printFailing is the status section, said when there is anything to say.
|
// alsoOn is a consumer's machine, when it is not the provider's.
|
||||||
func printFailing(list []inventory.ProviderStanding, now time.Time) {
|
func alsoOn(consumerNode, providerNode string) []string {
|
||||||
if len(list) == 0 {
|
if consumerNode == "" || consumerNode == providerNode {
|
||||||
return
|
return nil
|
||||||
}
|
}
|
||||||
fmt.Printf("%d consumer(s) a provider keeps failing (ADR 0224):\n\n", len(list))
|
return []string{consumerNode}
|
||||||
for _, line := range failingLines(list, now) {
|
|
||||||
fmt.Println(line)
|
|
||||||
}
|
|
||||||
fmt.Printf("\n the provider's journal has every attempt; it says recovered on its next success\n\n")
|
|
||||||
}
|
|
||||||
|
|
||||||
// failingOn is the standings that concern one machine: a provider running there, or a consumer.
|
|
||||||
func failingOn(list []inventory.ProviderStanding, node string) []inventory.ProviderStanding {
|
|
||||||
var out []inventory.ProviderStanding
|
|
||||||
for _, s := range list {
|
|
||||||
if s.ProviderNode == node || s.ConsumerNode == node {
|
|
||||||
out = append(out, s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,13 +7,14 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
)
|
)
|
||||||
|
|
||||||
// A provider that keeps failing a consumer is a problem `status` names (novox/hq ADR 0224). On
|
// A provider that keeps failing a consumer is a problem `status` names (novox/hq ADR 0224), kept as
|
||||||
// 2026-10-05 the identity provider refused every consumer for a day and status called the mesh well
|
// the condition store's first kind (to-be 45 §2). On 2026-10-05 the identity provider refused every
|
||||||
// (04-ISSUES/179): this is that day, told to the controller the way the provider now tells it.
|
// consumer for a day and status called the mesh well (04-ISSUES/179): this is that day, told to the
|
||||||
|
// controller the way the provider now tells it.
|
||||||
func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
|
func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
@@ -22,7 +23,7 @@ func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
|
|||||||
if _, err := assign(ctx, open, "anchor", "idp"); err != nil {
|
if _, err := assign(ctx, open, "anchor", "idp"); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
kept := standings{open.inventory}
|
kept := standings{keeper: func() *conditions.Keeper { return conditionsFrom }}
|
||||||
since := time.Now().Add(-23 * time.Hour)
|
since := time.Now().Add(-23 * time.Hour)
|
||||||
failing := link.Standing{Module: "idp", Failing: true, Provider: "oidc-client", ProviderNode: "anchor",
|
failing := link.Standing{Module: "idp", Failing: true, Provider: "oidc-client", ProviderNode: "anchor",
|
||||||
Consumer: "mesh_laptop_dashboard", Node: "laptop", Class: "credentials-rejected",
|
Consumer: "mesh_laptop_dashboard", Node: "laptop", Class: "credentials-rejected",
|
||||||
@@ -39,8 +40,8 @@ func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
|
|||||||
t.Fatal("a mesh whose identity provider fails a consumer reads as well")
|
t.Fatal("a mesh whose identity provider fails a consumer reads as well")
|
||||||
}
|
}
|
||||||
said := printed(t, func() error { return printStatus(asked) })
|
said := printed(t, func() error { return printStatus(asked) })
|
||||||
for _, want := range []string{"1 consumer(s) a provider keeps failing", "idp on anchor",
|
for _, want := range []string{"1 open condition(s)", "provider.idp.anchor.mesh_laptop_dashboard.failing",
|
||||||
"mesh_laptop_dashboard (laptop)", "credentials-rejected", "31000 attempts", "invalid_grant"} {
|
"idp on anchor keeps failing mesh_laptop_dashboard on laptop", "credentials-rejected", "31000 attempt(s)"} {
|
||||||
if !strings.Contains(said, want) {
|
if !strings.Contains(said, want) {
|
||||||
t.Fatalf("status does not say %q:\n%s", want, said)
|
t.Fatalf("status does not say %q:\n%s", want, said)
|
||||||
}
|
}
|
||||||
@@ -48,20 +49,25 @@ func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
|
|||||||
if strings.Contains(said, "all doing what they were told") {
|
if strings.Contains(said, "all doing what they were told") {
|
||||||
t.Fatalf("status said all well beside a failing provider:\n%s", said)
|
t.Fatalf("status said all well beside a failing provider:\n%s", said)
|
||||||
}
|
}
|
||||||
|
if !strings.HasPrefix(said, "1 open condition(s)") {
|
||||||
|
t.Fatalf("status does not lead with what is open:\n%s", said)
|
||||||
|
}
|
||||||
body, err := statusAsJSON(asked)
|
body, err := statusAsJSON(asked)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
var doc struct {
|
var doc struct {
|
||||||
Failing []inventory.ProviderStanding `json:"failing"`
|
Conditions []conditions.Condition `json:"conditions"`
|
||||||
|
Failing []conditions.Condition `json:"failing"`
|
||||||
}
|
}
|
||||||
if err := json.Unmarshal(body, &doc); err != nil || len(doc.Failing) != 1 || doc.Failing[0].Consumer != "mesh_laptop_dashboard" {
|
if err := json.Unmarshal(body, &doc); err != nil || len(doc.Failing) != 1 || len(doc.Conditions) != 1 ||
|
||||||
|
!strings.Contains(doc.Failing[0].Evidence[0].Said, "invalid_grant") {
|
||||||
t.Fatalf("the document does not carry it: %v\n%s", err, body)
|
t.Fatalf("the document does not carry it: %v\n%s", err, body)
|
||||||
}
|
}
|
||||||
// Both machines' `node show` name it: where the provider runs, and where the consumer is.
|
// Both machines' `node show` name it: where the provider runs, and where the consumer is.
|
||||||
for _, node := range []string{"anchor", "laptop"} {
|
for _, node := range []string{"anchor", "laptop"} {
|
||||||
shown := printed(t, func() error { return showNode(ctx, open.inventory, node) })
|
shown := printed(t, func() error { return showNode(ctx, open.inventory, node) })
|
||||||
if !strings.Contains(shown, "a provider keeps failing") || !strings.Contains(shown, "mesh_laptop_dashboard") {
|
if !strings.Contains(shown, "open condition(s) about this machine") || !strings.Contains(shown, "mesh_laptop_dashboard") {
|
||||||
t.Fatalf("node show %s does not name it:\n%s", node, shown)
|
t.Fatalf("node show %s does not name it:\n%s", node, shown)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -75,41 +81,38 @@ func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if len(asked.failing) != 0 {
|
if len(asked.conditions) != 0 {
|
||||||
t.Fatalf("a recovered consumer is still named: %+v", asked.failing)
|
t.Fatalf("a recovered consumer is still named: %+v", asked.conditions)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A provider no longer assigned where it ran has nothing running to fail anybody: its last word is
|
// A provider no longer assigned where it ran has nothing running to fail anybody: its last word is
|
||||||
// not a problem.
|
// cleared on the next look, said as resolved by the assignment.
|
||||||
func TestAnUnassignedProvidersLastWordIsNotAProblem(t *testing.T) {
|
func TestAnUnassignedProvidersLastWordIsCleared(t *testing.T) {
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
if _, err := (standings{open.inventory}).Stood(ctx, link.Standing{Module: "gone", Failing: true,
|
kept := standings{keeper: func() *conditions.Keeper { return conditionsFrom }}
|
||||||
|
if _, err := kept.Stood(ctx, link.Standing{Module: "gone", Failing: true,
|
||||||
ProviderNode: "anchor", Consumer: "x", Since: time.Now()}); err != nil {
|
ProviderNode: "anchor", Consumer: "x", Since: time.Now()}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
asked, err := theThreeQuestions(ctx, open)
|
all, err := conditionsFrom.Open(ctx)
|
||||||
if err != nil {
|
if err != nil || len(all) != 1 {
|
||||||
|
t.Fatalf("%+v %v", all, err)
|
||||||
|
}
|
||||||
|
if err := unassignedProviders(ctx, open.inventory, conditionsFrom, all); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if len(asked.failing) != 0 {
|
if all, _ := conditionsFrom.Open(ctx); len(all) != 0 {
|
||||||
t.Fatalf("%+v", asked.failing)
|
t.Fatalf("%+v", all)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAProviderThatStoppedRepeatingItselfIsSaidToHaveGoneQuiet(t *testing.T) {
|
// **The store away holds a recovery** (ADR 0224 §3): a standing that cannot be kept is asked again.
|
||||||
now := time.Now()
|
func TestAStandingTheStoreCannotKeepIsAskedAgain(t *testing.T) {
|
||||||
lines := strings.Join(failingLines([]inventory.ProviderStanding{{
|
kept := standings{keeper: func() *conditions.Keeper { return nil }}
|
||||||
Module: "idp", ProviderNode: "anchor", Consumer: "c", Class: "unreachable", Error: "connection refused\nmore",
|
if _, err := kept.Stood(t.Context(), link.Standing{Module: "idp", ProviderNode: "anchor", Consumer: "x"}); err == nil ||
|
||||||
Since: now.Add(-3 * time.Hour), SaidAt: now.Add(-2 * time.Hour), Attempts: 9,
|
!strings.Contains(err.Error(), link.ErrTryAgain.Error()) {
|
||||||
}}, now), "\n")
|
t.Fatalf("answered %v", err)
|
||||||
for _, want := range []string{"unreachable, for 3h", "connection refused", "not said again for 2h"} {
|
|
||||||
if !strings.Contains(lines, want) {
|
|
||||||
t.Fatalf("%q not in:\n%s", want, lines)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if strings.Contains(lines, "more") {
|
|
||||||
t.Fatalf("more than the first line of an error:\n%s", lines)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/overlay"
|
"github.com/novox/mesh-controller/internal/overlay"
|
||||||
)
|
)
|
||||||
@@ -81,8 +82,12 @@ func printStatus(asked answers) error {
|
|||||||
wrong, nodes, quiet := asked.wrong, asked.nodes, asked.quiet
|
wrong, nodes, quiet := asked.wrong, asked.nodes, asked.quiet
|
||||||
behind, sources := asked.behind, asked.sources
|
behind, sources := asked.behind, asked.sources
|
||||||
|
|
||||||
|
// **What is wrong leads** (novox/hq to-be 45 §2): every open condition, urgent first, oldest
|
||||||
|
// first, silenced ones with when their silence ends.
|
||||||
|
printConditions(asked.conditions, asked.conditionsUnread, time.Now())
|
||||||
|
|
||||||
if len(asked.refused) > 0 {
|
if len(asked.refused) > 0 {
|
||||||
// First, above everything else. A machine that cannot be worked out is not running an old
|
// First of what follows. A machine that cannot be worked out is not running an old
|
||||||
// declaration — it has no declaration, and nothing below this line is about it.
|
// declaration — it has no declaration, and nothing below this line is about it.
|
||||||
var names []string
|
var names []string
|
||||||
for name := range asked.refused {
|
for name := range asked.refused {
|
||||||
@@ -129,10 +134,6 @@ func printStatus(asked answers) error {
|
|||||||
fmt.Println()
|
fmt.Println()
|
||||||
}
|
}
|
||||||
|
|
||||||
// A provider failing a consumer, beside machines failing what they were told: both are something
|
|
||||||
// not working now (novox/hq ADR 0224).
|
|
||||||
printFailing(asked.failing, time.Now())
|
|
||||||
|
|
||||||
if len(quiet) > 0 {
|
if len(quiet) > 0 {
|
||||||
var said []string
|
var said []string
|
||||||
for _, n := range quiet {
|
for _, n := range quiet {
|
||||||
@@ -302,6 +303,29 @@ func printStatus(asked answers) error {
|
|||||||
fmt.Printf("\n `assign <node> <holder>` meets it; reported until every machine has its holders, then refused\n\n")
|
fmt.Printf("\n `assign <node> <holder>` meets it; reported until every machine has its holders, then refused\n\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if len(asked.overflowing) > 0 {
|
||||||
|
// **Reported, and the provider still pushed** (novox/hq ADR 0225, issue 263). Each is left
|
||||||
|
// out of its provider's grants, so the module holds a login nothing created; the provider's
|
||||||
|
// machine is sent everything else rather than refused for one consumer elsewhere.
|
||||||
|
fmt.Printf("%d module(s) identified too long for a provision they require, and not granted it:\n",
|
||||||
|
len(asked.overflowing))
|
||||||
|
for _, o := range asked.overflowing {
|
||||||
|
fmt.Printf(" %-12s %-20s %-22s %q is %d, %s keeps %d\n", o.Consumer, o.Module, o.Provision,
|
||||||
|
o.Identity, len(o.Identity), o.Bound.In, o.Bound.Max)
|
||||||
|
}
|
||||||
|
fmt.Printf("\n a shorter `slug` in the module's definition fits it; `module check` refuses one before merge\n\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Repairs done by hand this week (novox/hq to-be 45 §7). Not a fault, so it does not break "all
|
||||||
|
// well"; each is a healer the mesh does not have yet, and the count is how that is watched.
|
||||||
|
switch {
|
||||||
|
case asked.handActsUnread != "":
|
||||||
|
fmt.Printf("the hand-act log could not be read, so how much was done by hand this week is not known: %s\n\n",
|
||||||
|
asked.handActsUnread)
|
||||||
|
case asked.handActs != nil && *asked.handActs > 0:
|
||||||
|
fmt.Printf("%d act(s) done by hand in the last seven days — `hand-acts` lists them, and why\n\n", *asked.handActs)
|
||||||
|
}
|
||||||
|
|
||||||
if adopted := adoptedNodes(nodes); len(adopted) > 0 {
|
if adopted := adoptedNodes(nodes); len(adopted) > 0 {
|
||||||
// Said, because nothing forces the flip: a node left adopted is visible here rather than
|
// Said, because nothing forces the flip: a node left adopted is visible here rather than
|
||||||
// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
|
// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
|
||||||
@@ -311,8 +335,9 @@ func printStatus(asked answers) error {
|
|||||||
|
|
||||||
if asked.well() {
|
if asked.well() {
|
||||||
// Said plainly. "Nothing to report" and "nothing was checked" must never look the same,
|
// Said plainly. "Nothing to report" and "nothing was checked" must never look the same,
|
||||||
// and getting here means every question was asked and answered.
|
// and getting here means every question was asked and answered. **No open conditions first**
|
||||||
fmt.Printf("%d machine(s), all doing what they were told, all heard from, running what "+
|
// (novox/hq to-be 45 §2): it is what the sentence means now, silenced ones included.
|
||||||
|
fmt.Printf("no open conditions; %d machine(s), all doing what they were told, all heard from, running what "+
|
||||||
"the mesh would send them, and every module current with its source\n", len(nodes))
|
"the mesh would send them, and every module current with its source\n", len(nodes))
|
||||||
// **And what that sentence does not cover**, because for eleven hours it was true of a mesh
|
// **And what that sentence does not cover**, because for eleven hours it was true of a mesh
|
||||||
// in which no module could reach another (novox/hq 04-ISSUES/145). Every question above is
|
// in which no module could reach another (novox/hq 04-ISSUES/145). Every question above is
|
||||||
@@ -410,21 +435,27 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
|||||||
// ADR 0207). Each machine resolved again rather than threaded through whoResolves, whose answer
|
// ADR 0207). Each machine resolved again rather than threaded through whoResolves, whose answer
|
||||||
// the private network is built from and should say nothing else; a machine that does not
|
// the private network is built from and should say nothing else; a machine that does not
|
||||||
// resolve is already in refused, and is passed over here.
|
// resolve is already in refused, and is passed over here.
|
||||||
|
plans := map[string]planned{}
|
||||||
for _, n := range out.nodes {
|
for _, n := range out.nodes {
|
||||||
plan, _, err := planFor(ctx, open, n.Name)
|
plan, settings, err := planFor(ctx, open, n.Name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if unresolvable(err) {
|
if unresolvable(err) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
return answers{}, err
|
return answers{}, err
|
||||||
}
|
}
|
||||||
|
plans[n.Name] = planned{plan, settings}
|
||||||
out.unheld = append(out.unheld, plan.Unheld...)
|
out.unheld = append(out.unheld, plan.Unheld...)
|
||||||
|
// And which of its modules a provider leaves out of its grants, for an identity too long
|
||||||
|
// for what the provision keeps (novox/hq ADR 0225) — judged from the consumer's own
|
||||||
|
// resolution, as the provider's composition judges it.
|
||||||
|
out.overflowing = append(out.overflowing, plan.Overflowing()...)
|
||||||
}
|
}
|
||||||
// And every consumer a provider says it keeps failing (novox/hq ADR 0224). Read from what the
|
// And every open condition (novox/hq to-be 45 §2): what the watchdogs, the self-check and the
|
||||||
// providers announced: nothing else in the mesh knows whether a provision is being made.
|
// providers' own words say is wrong — a provider failing a consumer among them (ADR 0224). Kept on
|
||||||
out.failing, err = failingProviders(ctx, inv)
|
// the bus; a process that cannot read them says so, and the mesh is then not called well.
|
||||||
if err != nil {
|
if out.conditions, err = openConditions(ctx); err != nil {
|
||||||
return answers{}, err
|
out.conditionsUnread = err.Error()
|
||||||
}
|
}
|
||||||
out.plans, err = inv.RecentPlans(ctx, 5)
|
out.plans, err = inv.RecentPlans(ctx, 5)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -432,6 +463,16 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
|||||||
}
|
}
|
||||||
// Whether the build seat takes work, for a plan waiting on it (novox/hq ADR 0219).
|
// Whether the build seat takes work, for a plan waiting on it (novox/hq ADR 0219).
|
||||||
out.paused = buildSeatPause(ctx, inv, out.plans)
|
out.paused = buildSeatPause(ctx, inv, out.plans)
|
||||||
|
// And how many repairs were done by hand this week (novox/hq to-be 45 §7) — where there is a bus
|
||||||
|
// to read the log from; a process with none has no log to count.
|
||||||
|
if _, onBus := broker.BusAddress(); onBus == nil {
|
||||||
|
n, unread := handActsThisWeek(ctx)
|
||||||
|
if unread != "" {
|
||||||
|
out.handActsUnread = unread
|
||||||
|
} else {
|
||||||
|
out.handActs = &n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// And which machines are not running what the mesh would send them. The same question as a
|
// And which machines are not running what the mesh would send them. The same question as a
|
||||||
// module being behind its source, one level down: that one says the catalogue is out of date,
|
// module being behind its source, one level down: that one says the catalogue is out of date,
|
||||||
@@ -443,7 +484,7 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
|||||||
// said nothing at all and `status --json` emitted prose to stderr and no JSON anywhere. The
|
// said nothing at all and `status --json` emitted prose to stderr and no JSON anywhere. The
|
||||||
// reason is kept and reported as data; every question that does not depend on it is still
|
// reason is kept and reported as data; every question that does not depend on it is still
|
||||||
// answered.
|
// answered.
|
||||||
would, err := wouldSend(ctx, open, out.nodes)
|
would, err := wouldSendFrom(ctx, open, out.nodes, plans)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
out.network = err.Error()
|
out.network = err.Error()
|
||||||
would = map[string]string{}
|
would = map[string]string{}
|
||||||
@@ -538,7 +579,8 @@ func untakenModules(ctx context.Context, inv *inventory.Inventory, nodes []inven
|
|||||||
func (a answers) well() bool {
|
func (a answers) well() bool {
|
||||||
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
|
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
|
||||||
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 &&
|
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 &&
|
||||||
len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.failing) == 0
|
len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.overflowing) == 0 &&
|
||||||
|
len(a.conditions) == 0 && a.conditionsUnread == ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// hostSplit is which machines report which host version, for every version more than one machine
|
// hostSplit is which machines report which host version, for every version more than one machine
|
||||||
|
|||||||
@@ -0,0 +1,193 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// `status` answered from a summary the serving controller keeps current (novox/hq to-be 45 Phase 0,
|
||||||
|
// §4's D9 and §8's health of the controller).
|
||||||
|
//
|
||||||
|
// **Asked, it was composed: every machine resolved, twice, while its caller waited.** On 2026-10-06
|
||||||
|
// the verb took eighteen seconds on a mesh of four machines, so its caller read "still running" and
|
||||||
|
// had to ask `calls` for the answer to "is the mesh alright" — the one question that must answer at
|
||||||
|
// once, and the one a self-check and a rollout gate will ask every few minutes. So the serving
|
||||||
|
// controller composes it in the background — at its start, after anything that changes what it says
|
||||||
|
// (a machine's report, a build, a verb that acts), and every minute regardless — and the verb answers
|
||||||
|
// the last composition at once, saying when it was composed and how long that took. A caller who
|
||||||
|
// needs it newer than that reads the time and asks again; nothing is answered as current that is not.
|
||||||
|
|
||||||
|
// statusEvery is how often the summary is composed with nothing having nudged it; statusSettle how
|
||||||
|
// long a nudge waits for the next, so a push answered by four machines is composed once.
|
||||||
|
var (
|
||||||
|
statusEvery = time.Minute
|
||||||
|
statusSettle = 2 * time.Second
|
||||||
|
// statusComposeWithin bounds one composition, so a store that hangs cannot stop the summary for
|
||||||
|
// good; the attempt is said as failed, and the last summary stands with its age.
|
||||||
|
statusComposeWithin = 2 * time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
// statusSummary is the last composed `status --json`, and when.
|
||||||
|
type statusSummary struct {
|
||||||
|
compose func(context.Context) ([]byte, error)
|
||||||
|
// every, settle and within are the clocks above, read once when it is made.
|
||||||
|
every, settle, within time.Duration
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
body []byte
|
||||||
|
composedAt time.Time
|
||||||
|
took time.Duration
|
||||||
|
failed string
|
||||||
|
failedAt time.Time
|
||||||
|
started time.Time
|
||||||
|
first chan struct{} // closed when the first attempt ends, either way
|
||||||
|
|
||||||
|
nudged chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newStatusSummary(compose func(context.Context) ([]byte, error)) *statusSummary {
|
||||||
|
return &statusSummary{compose: compose, started: time.Now(), first: make(chan struct{}),
|
||||||
|
nudged: make(chan struct{}, 1), every: statusEvery, settle: statusSettle, within: statusComposeWithin}
|
||||||
|
}
|
||||||
|
|
||||||
|
// statusFrom is the serving controller's summary; nil in any other process, where `status` is
|
||||||
|
// composed when asked, as at a shell.
|
||||||
|
var statusFrom *statusSummary
|
||||||
|
|
||||||
|
// nudge asks for a composition soon. Never blocks: one pending is as good as many.
|
||||||
|
func (s *statusSummary) nudge() {
|
||||||
|
if s == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case s.nudged <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// keep composes until ctx ends: now, on a nudge once things settle, and every statusEvery.
|
||||||
|
func (s *statusSummary) keep(ctx context.Context) {
|
||||||
|
once := sync.Once{}
|
||||||
|
for {
|
||||||
|
s.composeOnce(ctx)
|
||||||
|
once.Do(func() { close(s.first) })
|
||||||
|
timer := time.NewTimer(s.every)
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
timer.Stop()
|
||||||
|
return
|
||||||
|
case <-timer.C:
|
||||||
|
case <-s.nudged:
|
||||||
|
timer.Stop()
|
||||||
|
// Let what else is arriving arrive, then compose once for all of it.
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-time.After(s.settle):
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-s.nudged:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *statusSummary) composeOnce(ctx context.Context) {
|
||||||
|
start := time.Now()
|
||||||
|
asking, cancel := context.WithTimeout(ctx, s.within)
|
||||||
|
body, err := s.compose(asking)
|
||||||
|
cancel()
|
||||||
|
took := time.Since(start)
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if err != nil {
|
||||||
|
s.failed, s.failedAt = err.Error(), time.Now()
|
||||||
|
fmt.Printf("status could not be composed (after %s): %v — `status` answers the last summary, "+
|
||||||
|
"with its age\n", took.Round(time.Millisecond), err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.body, s.composedAt, s.took, s.failed = body, start, took, ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// answer is what the `status` verb answers: the last summary at once, the same document `status
|
||||||
|
// --json` prints, with when it was composed. Before the first composition has ended it waits for it,
|
||||||
|
// but never past the caller's window; a controller that has none says so and why, rather than
|
||||||
|
// answering an empty mesh as a well one.
|
||||||
|
func (s *statusSummary) answer(ctx context.Context) (any, error) {
|
||||||
|
wait := time.NewTimer(link.AnswerWithin - time.Second)
|
||||||
|
defer wait.Stop()
|
||||||
|
select {
|
||||||
|
case <-s.first:
|
||||||
|
case <-wait.C:
|
||||||
|
case <-ctx.Done():
|
||||||
|
}
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if s.body == nil {
|
||||||
|
why := "its first composition has not finished"
|
||||||
|
if s.failed != "" {
|
||||||
|
why = "it could not be composed: " + s.failed
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("this controller started %s ago and has no status to answer yet — %s. "+
|
||||||
|
"Ask again shortly", time.Since(s.started).Round(time.Second), why)
|
||||||
|
}
|
||||||
|
var parsed any
|
||||||
|
_ = json.Unmarshal(s.body, &parsed)
|
||||||
|
out := map[string]any{
|
||||||
|
"output": string(s.body), "ok": true, "answer": parsed,
|
||||||
|
"composed": s.composedAt.UTC().Format(time.RFC3339),
|
||||||
|
"age": time.Since(s.composedAt).Round(time.Second).String(),
|
||||||
|
"composedIn": s.took.Round(time.Millisecond).String(),
|
||||||
|
"note": "composed by the serving controller at its start, after each report, build or act, and " +
|
||||||
|
"every minute; answered at once from the last composition",
|
||||||
|
}
|
||||||
|
if s.failed != "" && s.failedAt.After(s.composedAt) {
|
||||||
|
out["lastAttemptFailed"] = fmt.Sprintf("%s: %s — this summary is the last that could be composed",
|
||||||
|
s.failedAt.UTC().Format(time.RFC3339), s.failed)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// composeStatus is `status --json`, composed in this process against its stores.
|
||||||
|
func composeStatus(open *stores) func(context.Context) ([]byte, error) {
|
||||||
|
return func(ctx context.Context) ([]byte, error) {
|
||||||
|
asked, err := theThreeQuestions(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return statusAsJSON(asked)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// readingVerbs are the verbs that only read; after any other, what `status` says may have changed, so the summary is
|
||||||
|
// composed again; a verb that only reads leaves it alone, or a console polling `nodes` would keep the
|
||||||
|
// controller composing for ever.
|
||||||
|
var readingVerbs = map[string]bool{
|
||||||
|
"tools": true, "calls": true, "status": true, "nodes": true, "node": true, "modules": true,
|
||||||
|
"seats": true, "builds": true, "plan": true, "queue": true, "durations": true, "hand-acts": true,
|
||||||
|
"doctor": true, "conditions": true,
|
||||||
|
}
|
||||||
|
|
||||||
|
// nudgingListener is the enrolment, nudging the summary when a machine said something new.
|
||||||
|
type nudgingListener struct {
|
||||||
|
link.Enrolment
|
||||||
|
summary *statusSummary
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, error) {
|
||||||
|
// A declaration refused as older than the one the machine holds is counted (novox/hq to-be 45 S13).
|
||||||
|
if link.IsStaleRefusal(report.Refused) {
|
||||||
|
link.StaleRefusals.Refused(report.Node, time.Now())
|
||||||
|
}
|
||||||
|
news, err := l.Enrolment.Heard(ctx, report)
|
||||||
|
if news {
|
||||||
|
l.summary.nudge()
|
||||||
|
}
|
||||||
|
return news, err
|
||||||
|
}
|
||||||
@@ -0,0 +1,152 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// quickly shortens the summary's clocks for one test.
|
||||||
|
func quickly(t *testing.T) {
|
||||||
|
t.Helper()
|
||||||
|
every, settle := statusEvery, statusSettle
|
||||||
|
statusEvery, statusSettle = time.Hour, 10*time.Millisecond
|
||||||
|
t.Cleanup(func() { statusEvery, statusSettle = every, settle })
|
||||||
|
}
|
||||||
|
|
||||||
|
// **`status` answers in full within ten seconds, five times in a row** (novox/hq to-be 45 Phase 0,
|
||||||
|
// D9) — however long composing it takes. On 2026-10-06 composing took eighteen seconds and the
|
||||||
|
// verb answered "still running"; from the summary it answers at once, in full, saying when.
|
||||||
|
func TestStatusAnswersAtOnceHoweverLongComposingTakes(t *testing.T) {
|
||||||
|
quickly(t)
|
||||||
|
var composed atomic.Int32
|
||||||
|
slow := make(chan struct{})
|
||||||
|
s := newStatusSummary(func(ctx context.Context) ([]byte, error) {
|
||||||
|
if composed.Add(1) > 1 {
|
||||||
|
<-slow // every composition after the first outlasts any caller
|
||||||
|
}
|
||||||
|
return []byte(`{"wrong":[],"machines":4}`), nil
|
||||||
|
})
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
defer close(slow)
|
||||||
|
go s.keep(ctx)
|
||||||
|
for i := 0; i < 5; i++ {
|
||||||
|
s.nudge() // a composition is under way and does not finish
|
||||||
|
start := time.Now()
|
||||||
|
got, err := s.answer(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if took := time.Since(start); took > time.Second {
|
||||||
|
t.Fatalf("answer %d took %s", i+1, took)
|
||||||
|
}
|
||||||
|
m := got.(map[string]any)
|
||||||
|
if m["answer"].(map[string]any)["machines"] != float64(4) || m["composed"] == "" || m["ok"] != true {
|
||||||
|
t.Fatalf("answer %d was not in full: %v", i+1, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The first composition is waited for, never past the caller's window; a controller with none yet
|
||||||
|
// says so rather than answering an empty mesh as a well one.
|
||||||
|
func TestStatusBeforeItsFirstCompositionSaysSo(t *testing.T) {
|
||||||
|
quickly(t)
|
||||||
|
was := link.AnswerWithin
|
||||||
|
link.AnswerWithin = 1100 * time.Millisecond
|
||||||
|
t.Cleanup(func() { link.AnswerWithin = was })
|
||||||
|
never := make(chan struct{})
|
||||||
|
defer close(never)
|
||||||
|
s := newStatusSummary(func(context.Context) ([]byte, error) { <-never; return nil, nil })
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
go s.keep(ctx)
|
||||||
|
start := time.Now()
|
||||||
|
_, err := s.answer(ctx)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "has no status to answer yet") {
|
||||||
|
t.Fatalf("answered %v", err)
|
||||||
|
}
|
||||||
|
if took := time.Since(start); took > link.AnswerWithin {
|
||||||
|
t.Fatalf("waited %s, past the caller's window", took)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A nudge composes it again, once for several close together; a failed composition leaves the last
|
||||||
|
// summary standing and says it is the last that could be composed.
|
||||||
|
func TestANudgeComposesAgainAndAFailureKeepsTheLastSummary(t *testing.T) {
|
||||||
|
quickly(t)
|
||||||
|
var composed atomic.Int32
|
||||||
|
fail := atomic.Bool{}
|
||||||
|
s := newStatusSummary(func(context.Context) ([]byte, error) {
|
||||||
|
n := composed.Add(1)
|
||||||
|
if fail.Load() {
|
||||||
|
return nil, errors.New("the store did not answer")
|
||||||
|
}
|
||||||
|
body, _ := json.Marshal(map[string]any{"n": n})
|
||||||
|
return body, nil
|
||||||
|
})
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
go s.keep(ctx)
|
||||||
|
if _, err := s.answer(ctx); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
s.nudge()
|
||||||
|
s.nudge()
|
||||||
|
s.nudge()
|
||||||
|
waitFor(t, func() bool { return composed.Load() == 2 })
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
if n := composed.Load(); n != 2 {
|
||||||
|
t.Fatalf("three nudges together composed %d times after the first", n-1)
|
||||||
|
}
|
||||||
|
fail.Store(true)
|
||||||
|
s.nudge()
|
||||||
|
waitFor(t, func() bool { return composed.Load() == 3 })
|
||||||
|
waitFor(t, func() bool {
|
||||||
|
got, err := s.answer(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m := got.(map[string]any)
|
||||||
|
return m["lastAttemptFailed"] != nil && m["answer"].(map[string]any)["n"] == float64(2)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// The seat's `status` answers from the summary when this process keeps one.
|
||||||
|
func TestTheStatusVerbAnswersFromTheSummary(t *testing.T) {
|
||||||
|
quickly(t)
|
||||||
|
s := newStatusSummary(func(context.Context) ([]byte, error) { return []byte(`{"from":"summary"}`), nil })
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
go s.keep(ctx)
|
||||||
|
statusFrom = s
|
||||||
|
t.Cleanup(func() { statusFrom = nil })
|
||||||
|
handlers, _, err := seatToolHandlers()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, err := handlers["status"](ctx, json.RawMessage(`{}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.(map[string]any)["answer"].(map[string]any)["from"] != "summary" {
|
||||||
|
t.Fatalf("answered %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func waitFor(t *testing.T, ok func() bool) {
|
||||||
|
t.Helper()
|
||||||
|
deadline := time.Now().Add(3 * time.Second)
|
||||||
|
for !ok() {
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatal("never happened")
|
||||||
|
}
|
||||||
|
time.Sleep(5 * time.Millisecond)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -75,21 +75,25 @@ func TestAPersonClosesAStuckPlan(t *testing.T) {
|
|||||||
if err := open.inventory.SavePlan(ctx, stuck); err != nil {
|
if err := open.inventory.SavePlan(ctx, stuck); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := plansCommand(ctx, []string{"close", stuck.ID}); err != nil {
|
if err := plansCommand(ctx, []string{"close", stuck.ID}); err == nil || !strings.Contains(err.Error(), "--why") {
|
||||||
|
t.Fatalf("a plan was closed by hand without saying why: %v", err)
|
||||||
|
}
|
||||||
|
if err := plansCommand(ctx, []string{"close", stuck.ID, "--why", "its report will not come"}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
closed, err := open.inventory.PlanByID(ctx, stuck.ID)
|
closed, err := open.inventory.PlanByID(ctx, stuck.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if closed.State != inventory.PlanFailed || !strings.Contains(closed.Note, "closed by hand") {
|
if closed.State != inventory.PlanFailed || !strings.Contains(closed.Note, "closed by hand") ||
|
||||||
|
!strings.Contains(closed.Note, "its report will not come") {
|
||||||
t.Fatalf("the plan was left %s: %q", closed.State, closed.Note)
|
t.Fatalf("the plan was left %s: %q", closed.State, closed.Note)
|
||||||
}
|
}
|
||||||
if err := plansCommand(ctx, []string{"close", stuck.ID}); err == nil {
|
if err := plansCommand(ctx, []string{"close", stuck.ID, "--why", "again"}); err == nil {
|
||||||
t.Fatal("a plan already closed was closed again")
|
t.Fatal("a plan already closed was closed again")
|
||||||
}
|
}
|
||||||
if argv, err := argvFor("plans", map[string]any{"close": stuck.ID}); err != nil ||
|
if argv, err := argvFor("plans", map[string]any{"close": stuck.ID, "why": "w"}); err != nil ||
|
||||||
!reflect.DeepEqual(argv, []string{"plans", "close", stuck.ID}) {
|
!reflect.DeepEqual(argv, []string{"plans", "close", stuck.ID, "--why", "w"}) {
|
||||||
t.Fatalf("the seat's verb does not close a plan: %v %v", argv, err)
|
t.Fatalf("the seat's verb does not close a plan: %v %v", argv, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,549 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"slices"
|
||||||
|
"sort"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The watchdogs (novox/hq to-be 45 §3): every row of the signals table, run over what the serving
|
||||||
|
// controller knows, every half minute.
|
||||||
|
//
|
||||||
|
// **One loop, one gathering, every row.** The facts are gathered once a tick — the store's machines,
|
||||||
|
// plans and durations, the bus's queue and consumers, what this process heard — and each row's watch
|
||||||
|
// is a pure reading of them, which is what lets the test generated from the table suppress a signal by
|
||||||
|
// changing a fact. A part that cannot be gathered makes the rows that read it blind: each says so as
|
||||||
|
// a condition of its own (`probe.<row>.failed`) and keeps what it raised before, because a watchdog
|
||||||
|
// that cannot see must not read as one that sees nothing wrong (ADR 0227 rule 4).
|
||||||
|
//
|
||||||
|
// **The watchdogs are themselves watched.** Each tick is recorded; the self-check (doctor.go) fails
|
||||||
|
// its probe DW when the ticks stop, and the watchdogs raise S10 when the self-check stops — two loops
|
||||||
|
// watching each other, and mesh-watcher on a second machine watching the self-check's heartbeat for
|
||||||
|
// the case both stop with the process.
|
||||||
|
|
||||||
|
// watchEvery is how often the watchdogs run.
|
||||||
|
var watchEvery = 30 * time.Second
|
||||||
|
|
||||||
|
// signalFacts is what one tick of the watchdogs reads.
|
||||||
|
type signalFacts struct {
|
||||||
|
now time.Time
|
||||||
|
started time.Time
|
||||||
|
// host is the machine this controller runs on, as the mesh names it, for a condition about itself.
|
||||||
|
host string
|
||||||
|
|
||||||
|
machines []machineFacts
|
||||||
|
machinesErr error
|
||||||
|
// toolsHeardFrom is when this process began hearing node tools: one not heard since is silent
|
||||||
|
// since then at the most.
|
||||||
|
toolsHeardFrom time.Time
|
||||||
|
|
||||||
|
plans []planFacts
|
||||||
|
plansErr error
|
||||||
|
|
||||||
|
loop loopFacts
|
||||||
|
loopErr error
|
||||||
|
|
||||||
|
merges []missedMerge
|
||||||
|
mergesPassed time.Time
|
||||||
|
mergesErr error
|
||||||
|
|
||||||
|
asks []askFacts
|
||||||
|
asksErr error
|
||||||
|
|
||||||
|
calls []link.Call
|
||||||
|
|
||||||
|
standings []conditions.Condition
|
||||||
|
standingsErr error
|
||||||
|
|
||||||
|
advisories []link.Advisory
|
||||||
|
lostConsumers map[string]bool
|
||||||
|
advisoriesErr error
|
||||||
|
|
||||||
|
selfCheck selfCheckFacts
|
||||||
|
|
||||||
|
staleRefusals map[string]int
|
||||||
|
}
|
||||||
|
|
||||||
|
type machineFacts struct {
|
||||||
|
name string
|
||||||
|
control bool
|
||||||
|
// lastHeard is the store's last word from it, any word; zero when never.
|
||||||
|
lastHeard time.Time
|
||||||
|
// every is the heartbeat interval it said; zero when it said none.
|
||||||
|
every time.Duration
|
||||||
|
power link.PowerState
|
||||||
|
// sentAt is when it was last sent a declaration; reportedCurrent whether it has reported that one.
|
||||||
|
sentAt time.Time
|
||||||
|
reportedCurrent bool
|
||||||
|
reportedAt time.Time
|
||||||
|
// lastApply is its newest measured apply.
|
||||||
|
lastApply time.Duration
|
||||||
|
// tools is whether node-tools is assigned there; toolsHeard and toolsEvery its heartbeat.
|
||||||
|
tools bool
|
||||||
|
toolsHeard time.Time
|
||||||
|
toolsEvery time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// asleep says the machine said it would be away and has not said it is back (ADR 0211).
|
||||||
|
func (m machineFacts) asleep() bool { return m.power.Away() }
|
||||||
|
|
||||||
|
type planFacts struct {
|
||||||
|
id, repository, commit string
|
||||||
|
tier, tiers int
|
||||||
|
entered time.Time
|
||||||
|
bound time.Duration
|
||||||
|
waiting string
|
||||||
|
paused bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type loopFacts struct {
|
||||||
|
took time.Time
|
||||||
|
pending uint64
|
||||||
|
where string
|
||||||
|
}
|
||||||
|
|
||||||
|
type askFacts struct {
|
||||||
|
id, seat, what, state, on string
|
||||||
|
since time.Time
|
||||||
|
bound time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
type selfCheckFacts struct {
|
||||||
|
last time.Time
|
||||||
|
every time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// watchdogs is the loop and what it needs.
|
||||||
|
type watchdogs struct {
|
||||||
|
open *stores
|
||||||
|
server *link.Server
|
||||||
|
js *broker.JetStream
|
||||||
|
keeper *conditions.Keeper
|
||||||
|
doctor *doctor
|
||||||
|
|
||||||
|
started time.Time
|
||||||
|
// acting says this controller is the one acting, not one standing by (link.Holding): a controller
|
||||||
|
// standing by hears no heartbeat and would call every machine silent.
|
||||||
|
acting func() bool
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
ticked time.Time
|
||||||
|
last *signalFacts
|
||||||
|
failed string
|
||||||
|
}
|
||||||
|
|
||||||
|
// lastTick is when the watchdogs last finished a tick.
|
||||||
|
func (w *watchdogs) lastTick() time.Time {
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
return w.ticked
|
||||||
|
}
|
||||||
|
|
||||||
|
// lastFacts is the facts of the newest tick, for `doctor signals`; nil before the first.
|
||||||
|
func (w *watchdogs) lastFacts() *signalFacts {
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
return w.last
|
||||||
|
}
|
||||||
|
|
||||||
|
// keep runs the watchdogs until ctx ends.
|
||||||
|
func (w *watchdogs) keep(ctx context.Context) {
|
||||||
|
tick := time.NewTicker(watchEvery)
|
||||||
|
defer tick.Stop()
|
||||||
|
for {
|
||||||
|
w.tick(ctx)
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-tick.C:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// tick is one run of every row.
|
||||||
|
func (w *watchdogs) tick(ctx context.Context) {
|
||||||
|
if w.acting != nil && !w.acting() {
|
||||||
|
// Standing by: nothing seen, nothing said, and the tick counted — this process's self-check
|
||||||
|
// is not the one that matters while another acts.
|
||||||
|
w.mu.Lock()
|
||||||
|
w.ticked = time.Now()
|
||||||
|
w.mu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
running, cancel := context.WithTimeout(ctx, watchEvery)
|
||||||
|
defer cancel()
|
||||||
|
w.see(running, w.gather(running))
|
||||||
|
}
|
||||||
|
|
||||||
|
// see runs every watched row over one gathering, keeps what each found, and records the tick.
|
||||||
|
func (w *watchdogs) see(running context.Context, f *signalFacts) {
|
||||||
|
var problems []string
|
||||||
|
var blind []conditions.Observation
|
||||||
|
for _, row := range watchedRows() {
|
||||||
|
if err := row.needs(f); err != nil {
|
||||||
|
blind = append(blind, blindRow(row, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := w.keeper.Reconcile(running, row.Row, row.watch(f)); err != nil {
|
||||||
|
problems = append(problems, row.Row+": "+err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The rows that could not see, said; the ones that see again, cleared.
|
||||||
|
if err := w.keeper.Reconcile(running, sourceWatchdogs, blind); err != nil {
|
||||||
|
problems = append(problems, err.Error())
|
||||||
|
}
|
||||||
|
// A provider no longer assigned where it ran: its standing is resolved by the assignment.
|
||||||
|
if f.standingsErr == nil && w.open != nil {
|
||||||
|
if err := unassignedProviders(running, w.open.inventory, w.keeper, f.standings); err != nil {
|
||||||
|
problems = append(problems, "S8: "+err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := w.keeper.EndSilences(running); err != nil {
|
||||||
|
problems = append(problems, err.Error())
|
||||||
|
}
|
||||||
|
failed := ""
|
||||||
|
if len(problems) > 0 {
|
||||||
|
failed = fmt.Sprintf("%v", problems)
|
||||||
|
}
|
||||||
|
w.mu.Lock()
|
||||||
|
said := w.failed
|
||||||
|
w.ticked, w.last, w.failed = time.Now(), f, failed
|
||||||
|
w.mu.Unlock()
|
||||||
|
if failed != said {
|
||||||
|
if failed != "" {
|
||||||
|
fmt.Printf("the watchdogs could not keep what they saw: %s\n", failed)
|
||||||
|
} else if said != "" {
|
||||||
|
fmt.Println("the watchdogs keep what they see again")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// sourceWatchdogs is what raises a blind row's condition.
|
||||||
|
const sourceWatchdogs = "watchdogs"
|
||||||
|
|
||||||
|
// blindRow is a row whose facts could not be gathered, as a condition of its own.
|
||||||
|
func blindRow(row signalRow, err error) conditions.Observation {
|
||||||
|
return conditions.Observation{Scope: conditions.ScopeProbe, ID: row.Row, Kind: "probe-failed", Token: "failed",
|
||||||
|
Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("the watchdog of %s (%s) cannot see: what it reads could not be read, so nothing "+
|
||||||
|
"it would raise can be — and nothing it raised before is cleared", row.Row, row.Signal),
|
||||||
|
Said: firstLine(err.Error())}
|
||||||
|
}
|
||||||
|
|
||||||
|
// gather reads every fact a tick needs. Each part's failure is kept beside it, never an empty part.
|
||||||
|
func (w *watchdogs) gather(ctx context.Context) *signalFacts {
|
||||||
|
now := time.Now()
|
||||||
|
f := &signalFacts{now: now, started: w.started, toolsHeardFrom: link.ToolsBeats.Started(), calls: link.Calls.Running(),
|
||||||
|
staleRefusals: link.StaleRefusals.Within(now.Add(-staleRefusalsWithin)), lostConsumers: map[string]bool{}}
|
||||||
|
if w.doctor != nil {
|
||||||
|
f.selfCheck = selfCheckFacts{last: w.doctor.lastRunEnded(), every: doctorEvery}
|
||||||
|
}
|
||||||
|
inv := w.open.inventory
|
||||||
|
f.host = controlHost(ctx, inv)
|
||||||
|
f.machines, f.machinesErr = w.gatherMachines(ctx, inv, now)
|
||||||
|
f.plans, f.plansErr = gatherPlans(ctx, inv, now)
|
||||||
|
f.loop, f.loopErr = w.gatherLoop()
|
||||||
|
f.mergesPassed, f.merges, f.mergesErr = watchedMerges.last()
|
||||||
|
if f.mergesErr == nil && !f.mergesPassed.IsZero() && now.Sub(f.mergesPassed) > 3*mergeCatchUpEvery {
|
||||||
|
f.mergesErr = fmt.Errorf("the catch-up of merges has not passed since %s", f.mergesPassed.UTC().Format(time.RFC3339))
|
||||||
|
}
|
||||||
|
f.asks, f.asksErr = w.gatherAsks(ctx, inv)
|
||||||
|
if open, err := w.keeper.Open(ctx); err != nil {
|
||||||
|
f.standingsErr = err
|
||||||
|
} else {
|
||||||
|
f.standings = providerStandings(open)
|
||||||
|
}
|
||||||
|
f.advisories = link.Advisories.Since(now.Add(-advisoryQuiet))
|
||||||
|
f.lostConsumers, f.advisoriesErr = w.lostConsumers(ctx, f.advisories)
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|
||||||
|
// controlHost is the machine running the controller, as the mesh names it: the one the controller
|
||||||
|
// module is assigned to, or this process's host name where that is not one machine.
|
||||||
|
func controlHost(ctx context.Context, inv *inventory.Inventory) string {
|
||||||
|
if on, err := inv.Running(ctx, "mesh-controller"); err == nil && len(on) == 1 {
|
||||||
|
return on[0]
|
||||||
|
}
|
||||||
|
host, _ := os.Hostname()
|
||||||
|
return host
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *watchdogs) gatherMachines(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]machineFacts, error) {
|
||||||
|
nodes, err := inv.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the machines cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
reports, err := inv.LastReports(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("what each machine last reported cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
reported := map[string]inventory.Reported{}
|
||||||
|
for _, r := range reports {
|
||||||
|
reported[r.Node] = r
|
||||||
|
}
|
||||||
|
control, err := inv.Running(ctx, "mesh-controller")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("where the controller runs cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
tooled, err := inv.Running(ctx, broker.RuntimeModule)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("where the node tools run cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
applies, err := inv.Durations(ctx, inventory.DurationApply, now.Add(-7*24*time.Hour))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the measured applies cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
lastApply := map[string]time.Duration{}
|
||||||
|
for _, d := range applies { // oldest first: the last one read is the newest
|
||||||
|
lastApply[d.Node] = d.Took
|
||||||
|
}
|
||||||
|
var out []machineFacts
|
||||||
|
anyPast := false
|
||||||
|
for _, n := range nodes {
|
||||||
|
m := machineFacts{name: n.Name, control: slices.Contains(control, n.Name), lastHeard: n.LastSeen,
|
||||||
|
lastApply: lastApply[n.Name], tools: slices.Contains(tooled, n.Name)}
|
||||||
|
if beat, ok := link.HostBeats.Of(n.Name); ok {
|
||||||
|
m.every = beat.Every
|
||||||
|
}
|
||||||
|
if beat, ok := link.ToolsBeats.Of(n.Name); ok {
|
||||||
|
m.toolsHeard, m.toolsEvery = beat.At, beat.Every
|
||||||
|
}
|
||||||
|
if r, ok := reported[n.Name]; ok {
|
||||||
|
if r.Sent != nil {
|
||||||
|
m.sentAt = *r.Sent
|
||||||
|
}
|
||||||
|
if r.At != nil {
|
||||||
|
m.reportedAt = *r.At
|
||||||
|
}
|
||||||
|
m.reportedCurrent = r.Current
|
||||||
|
}
|
||||||
|
if !m.lastHeard.IsZero() && now.Sub(m.lastHeard) > heartbeatBound(m.every) {
|
||||||
|
anyPast = true
|
||||||
|
}
|
||||||
|
if m.tools && now.Sub(later(m.toolsHeard, link.ToolsBeats.Started())) > heartbeatBound(m.toolsEvery) {
|
||||||
|
anyPast = true
|
||||||
|
}
|
||||||
|
out = append(out, m)
|
||||||
|
}
|
||||||
|
// What a machine past its bound last said of its power, read only then: a machine that said it
|
||||||
|
// is asleep is not lost (ADR 0211). Unreadable is said: a sleeping laptop is then called silent,
|
||||||
|
// which is the louder mistake and the right one to make.
|
||||||
|
if anyPast && w.server != nil {
|
||||||
|
states, err := w.server.PowerStates(ctx, now.Add(-7*24*time.Hour))
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("what machines said of their power cannot be read, so a sleeping one is called silent: %v\n", err)
|
||||||
|
}
|
||||||
|
for i := range out {
|
||||||
|
out[i].power = states[out[i].name]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// gatherPlans is every open plan, its tier's bound from what was measured, and what it waits on.
|
||||||
|
func gatherPlans(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]planFacts, error) {
|
||||||
|
plans, err := inv.OpenPlans(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the open plans cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
if len(plans) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
tiers, err := inv.Durations(ctx, inventory.DurationPlanTier, now.Add(-14*24*time.Hour))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the measured plan tiers cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
measured := map[string][]time.Duration{}
|
||||||
|
for _, d := range tiers {
|
||||||
|
measured[d.Subject] = append(measured[d.Subject], d.Took)
|
||||||
|
}
|
||||||
|
pause := buildSeatPause(ctx, inv, plans)
|
||||||
|
var out []planFacts
|
||||||
|
for _, p := range plans {
|
||||||
|
_, paused := pausedWaiting(p, pause, now)
|
||||||
|
out = append(out, planFacts{id: p.ID, repository: p.Repository, commit: p.Commit, tier: p.Tier,
|
||||||
|
tiers: len(p.Tiers), entered: p.TierEntered, bound: max(tierAtLeast, 3*p90(measured[p.Repository])),
|
||||||
|
waiting: planLineWith(p, now, pause), paused: paused})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// p90 is the ninetieth percentile of measurements; zero for none.
|
||||||
|
func p90(took []time.Duration) time.Duration {
|
||||||
|
if len(took) == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
sorted := append([]time.Duration(nil), took...)
|
||||||
|
sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] })
|
||||||
|
return sorted[int(0.9*float64(len(sorted)-1))]
|
||||||
|
}
|
||||||
|
|
||||||
|
// gatherLoop is when the event loop last took a message and what its consumers hold.
|
||||||
|
func (w *watchdogs) gatherLoop() (loopFacts, error) {
|
||||||
|
f := loopFacts{took: link.Loop.Last()}
|
||||||
|
if w.js == nil {
|
||||||
|
return f, errors.New("this controller is not on the bus")
|
||||||
|
}
|
||||||
|
var where []string
|
||||||
|
for _, c := range broker.MeshConsumers() {
|
||||||
|
info, err := w.js.Context().ConsumerInfo(c.Stream, c.Name)
|
||||||
|
if err != nil {
|
||||||
|
return f, fmt.Errorf("the controller's consumer on %s cannot be read: %w", c.Stream, err)
|
||||||
|
}
|
||||||
|
if held := info.NumPending + uint64(info.NumAckPending); held > 0 {
|
||||||
|
f.pending += held
|
||||||
|
where = append(where, fmt.Sprintf("%d on %s", held, c.Stream))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
f.where = fmt.Sprint(where)
|
||||||
|
return f, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// gatherAsks is every ask in the build seat's queue that is in flight or dead, with its bound.
|
||||||
|
func (w *watchdogs) gatherAsks(ctx context.Context, inv *inventory.Inventory) ([]askFacts, error) {
|
||||||
|
if w.js == nil {
|
||||||
|
return nil, errors.New("this controller is not on the bus")
|
||||||
|
}
|
||||||
|
entries, err := inv.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the catalogue cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
seat := buildSeatAmong(entries)
|
||||||
|
q, err := link.ReadQueue(ctx, w.js, seat)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
builds, err := inv.Durations(ctx, inventory.DurationBuild, time.Now().Add(-14*24*time.Hour))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the measured builds cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
var took []time.Duration
|
||||||
|
for _, d := range builds {
|
||||||
|
took = append(took, d.Took)
|
||||||
|
}
|
||||||
|
bound := askDefault
|
||||||
|
if len(took) > 0 {
|
||||||
|
bound = max(askAtLeast, 3*p90(took))
|
||||||
|
}
|
||||||
|
var out []askFacts
|
||||||
|
for _, a := range q.Asks {
|
||||||
|
if a.State != link.AskInFlight && a.State != link.AskDead {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
since := a.Started
|
||||||
|
if since.IsZero() {
|
||||||
|
since = a.AskedAt
|
||||||
|
}
|
||||||
|
what := a.Repository
|
||||||
|
if a.Path != "" {
|
||||||
|
what += " " + a.Path
|
||||||
|
}
|
||||||
|
out = append(out, askFacts{id: a.ID, seat: seat, what: what, state: a.State, on: a.On, since: since, bound: bound})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// lostConsumers is, of the consumers the bus said were deleted, each that is still missing and that
|
||||||
|
// the mesh expects: a consumer removed because its module was unassigned is not lost.
|
||||||
|
func (w *watchdogs) lostConsumers(ctx context.Context, heard []link.Advisory) (map[string]bool, error) {
|
||||||
|
out := map[string]bool{}
|
||||||
|
var deleted []link.Advisory
|
||||||
|
for _, a := range heard {
|
||||||
|
if a.Kind == link.AdvisoryConsumerLost {
|
||||||
|
deleted = append(deleted, a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(deleted) == 0 {
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
if w.js == nil {
|
||||||
|
return nil, errors.New("this controller is not on the bus")
|
||||||
|
}
|
||||||
|
_, expected, err := expectedBusObjects(ctx, w.open.inventory)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
want := map[string]bool{}
|
||||||
|
for _, c := range expected {
|
||||||
|
want[c.Stream+"."+c.Name] = true
|
||||||
|
}
|
||||||
|
for _, a := range deleted {
|
||||||
|
_, err := w.js.Context().ConsumerInfo(a.Stream, a.Consumer)
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, nats.ErrConsumerNotFound):
|
||||||
|
out[a.Stream+"."+a.Consumer] = want[a.Stream+"."+a.Consumer]
|
||||||
|
case err != nil:
|
||||||
|
return nil, fmt.Errorf("whether %s exists cannot be read: %w", link.ConsumerInWords(a.Stream, a.Consumer), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// later is the later of two moments.
|
||||||
|
func later(a, b time.Time) time.Time {
|
||||||
|
if a.After(b) {
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
// watchTheMesh opens the condition store and starts the watchdogs, the bus's advisories and the
|
||||||
|
// self-check, for the serving controller; the returned function stops them. Nil when the store could
|
||||||
|
// not be opened, which is said.
|
||||||
|
func watchTheMesh(ctx context.Context, open *stores, server *link.Server, bus link.OverNATS) func() {
|
||||||
|
keeper, err := keeperOn(ctx, bus.Conn)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("the condition store could NOT be opened, so nothing that goes wrong is kept or said, and "+
|
||||||
|
"status says the conditions cannot be read: %v\n", err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
conditionsFrom = keeper
|
||||||
|
logf := func(format string, args ...any) { fmt.Printf(format+"\n", args...) }
|
||||||
|
stopHearing, err := server.HearAdvisories(logf)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("what the bus says about itself cannot be heard (S9 is blind): %v\n", err)
|
||||||
|
stopHearing = func() {}
|
||||||
|
}
|
||||||
|
host := controlHost(ctx, open.inventory)
|
||||||
|
w := &watchdogs{open: open, server: server, js: server.JetStream(), keeper: keeper, started: time.Now(),
|
||||||
|
acting: link.Holding}
|
||||||
|
d := &doctor{open: open, js: server.JetStream(), keeper: keeper, teller: bus, watchdogs: w, host: host}
|
||||||
|
w.doctor = d
|
||||||
|
doctorFrom = d
|
||||||
|
watching, stop := context.WithCancel(ctx)
|
||||||
|
go w.keep(watching)
|
||||||
|
go d.keep(watching)
|
||||||
|
fmt.Printf("watching the mesh: %d signal(s) every %s, %d probe(s) every %s; what is wrong is kept in %s "+
|
||||||
|
"and said as %s events\n", len(watchedRows()), watchEvery, len(runnableProbes()), doctorEvery,
|
||||||
|
broker.ConditionsBucket, conditions.Seat)
|
||||||
|
return func() {
|
||||||
|
stop()
|
||||||
|
stopHearing()
|
||||||
|
flushing, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
keeper.Close(flushing)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// runnableProbes are the probes the registry runs.
|
||||||
|
func runnableProbes() []probe {
|
||||||
|
var out []probe
|
||||||
|
for _, p := range probeRegistry {
|
||||||
|
if p.run != nil {
|
||||||
|
out = append(out, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -5,7 +5,9 @@ go 1.26.0
|
|||||||
require (
|
require (
|
||||||
github.com/jackc/pgx/v5 v5.10.0
|
github.com/jackc/pgx/v5 v5.10.0
|
||||||
github.com/nats-io/nats.go v1.54.0
|
github.com/nats-io/nats.go v1.54.0
|
||||||
|
github.com/novox/mesh-host v0.0.0
|
||||||
golang.org/x/crypto v0.57.0
|
golang.org/x/crypto v0.57.0
|
||||||
|
golang.org/x/net v0.58.0
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
@@ -15,8 +17,15 @@ require (
|
|||||||
github.com/klauspost/compress v1.20.0 // indirect
|
github.com/klauspost/compress v1.20.0 // indirect
|
||||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||||
github.com/nats-io/nuid v1.0.1 // indirect
|
github.com/nats-io/nuid v1.0.1 // indirect
|
||||||
golang.org/x/net v0.58.0 // indirect
|
|
||||||
golang.org/x/sync v0.23.0 // indirect
|
golang.org/x/sync v0.23.0 // indirect
|
||||||
golang.org/x/sys v0.48.0 // indirect
|
golang.org/x/sys v0.48.0 // indirect
|
||||||
golang.org/x/text v0.42.0 // indirect
|
golang.org/x/text v0.42.0 // indirect
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// The node-engine's own validator (mesh-host/validate, novox/hq to-be 45 D1): one validator, the host's.
|
||||||
|
// The host's module path names no forge a build can fetch from, so the module is read from the one
|
||||||
|
// that holds it, at the host's commit — **once, by whoever moves the pin, into vendor/**, which is
|
||||||
|
// committed. Every build (the build agent's `go build`, the Dockerfile) compiles from vendor/ and
|
||||||
|
// fetches nothing; go refuses to build when vendor/ and this file disagree, so a pin moved without
|
||||||
|
// `go mod vendor` fails loudly, at once, everywhere.
|
||||||
|
replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261006081854-6953b5bafdb2
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
git.novox.be/novox/mesh-host v0.0.0-20261006081854-6953b5bafdb2 h1:b4+F4tTfrPkuJTST+rkwIHpEb4mkVJR9158hr6nnc4g=
|
||||||
|
git.novox.be/novox/mesh-host v0.0.0-20261006081854-6953b5bafdb2/go.mod h1:VlilMCRZ5yyNXg7SNigNBLr0Gt32jrGw5KSNq5JAVYs=
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
|||||||
@@ -0,0 +1,143 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The controller's own key-value buckets (novox/hq to-be 45 §1, §6, §7).
|
||||||
|
//
|
||||||
|
// **What the controller must remember across its own restart, it keeps on the bus.** A call's
|
||||||
|
// outcome lived in the memory of the process that served it (novox/hq issue 265), so a controller
|
||||||
|
// replaced while a push ran answered "no such call" for the one thing its caller had been told to
|
||||||
|
// ask about. The bus already outlives the controller and is the shape ADR 0201 gives a module's
|
||||||
|
// current state: one value per key, written by one owner, read by anybody granted it. These are the
|
||||||
|
// controller's, written by it alone — the writers table of to-be 45 §1 — and asserted on every start
|
||||||
|
// like the streams, so a bus raised from nothing has them before the first call is served.
|
||||||
|
|
||||||
|
// CallsBucket keeps every call of the mesh's own verbs and what came of it; HandActsBucket every act
|
||||||
|
// a person did by hand, with why; ConditionsBucket every condition open now (to-be 45 §2), one key
|
||||||
|
// each, and ConditionHistoryBucket every transition of one — raised, changed, silenced, cleared —
|
||||||
|
// for ninety days.
|
||||||
|
//
|
||||||
|
// **The history is a bucket of its own** because its keys expire and an open condition's must not:
|
||||||
|
// a bucket has one age for every key, and a condition open longer than the history is kept would
|
||||||
|
// otherwise vanish from the store while still true.
|
||||||
|
var (
|
||||||
|
CallsBucket = BucketName(ControllerSeat, "calls")
|
||||||
|
HandActsBucket = BucketName(ControllerSeat, "hand-acts")
|
||||||
|
ConditionsBucket = BucketName(ControllerSeat, "conditions")
|
||||||
|
ConditionHistoryBucket = BucketName(ControllerSeat, "condition-history")
|
||||||
|
)
|
||||||
|
|
||||||
|
// The bounds to-be 45 §6 sets for calls: the last thousand, or fourteen days, whichever is fewer.
|
||||||
|
// A call is two keys — its record, and its answer apart so a listing does not read every answer —
|
||||||
|
// so the stream holds twice as many messages as it keeps calls.
|
||||||
|
const (
|
||||||
|
KeptCallsDurably = 1000
|
||||||
|
CallsKeptFor = 14 * 24 * time.Hour
|
||||||
|
// CallAnswerBytes is the most of one answer kept: a whole declaration is far smaller, and an
|
||||||
|
// answer larger is cut and says so.
|
||||||
|
CallAnswerBytes = 64 << 10
|
||||||
|
// HandActsKeptFor is as long as a condition's history (to-be 45 §2): an act by hand is read
|
||||||
|
// back beside what it addressed.
|
||||||
|
HandActsKeptFor = 90 * 24 * time.Hour
|
||||||
|
// ConditionHistoryKeptFor is how long a condition's transitions are kept (to-be 45 §2).
|
||||||
|
ConditionHistoryKeptFor = 90 * 24 * time.Hour
|
||||||
|
)
|
||||||
|
|
||||||
|
// IsControllerBucket says a bucket is the controller's own, not a module's state nothing declares.
|
||||||
|
func IsControllerBucket(bucket string) bool {
|
||||||
|
return bucket == CallsBucket || bucket == HandActsBucket || bucket == ConditionsBucket ||
|
||||||
|
bucket == ConditionHistoryBucket
|
||||||
|
}
|
||||||
|
|
||||||
|
// ControllerBuckets are the controller's own buckets, in the order they are asserted.
|
||||||
|
func ControllerBuckets() []string {
|
||||||
|
return []string{CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ControllerBucketsAsserter is what raising the controller's buckets needs of a connection.
|
||||||
|
type ControllerBucketsAsserter interface {
|
||||||
|
EnsureControllerBuckets() error
|
||||||
|
}
|
||||||
|
|
||||||
|
// EnsureControllerBuckets creates the controller's buckets if absent and brings their options to
|
||||||
|
// match. An update, never a delete: what they hold is the record of what the mesh was asked.
|
||||||
|
func (j *JetStream) EnsureControllerBuckets() error {
|
||||||
|
js, err := jetstream.New(j.conn)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||||
|
Bucket: CallsBucket,
|
||||||
|
Description: "the calls of the mesh's own verbs and what came of each (novox/hq to-be 45 §6, issue " +
|
||||||
|
"265): written by the controller alone, read through `calls`; the last thousand, or fourteen days",
|
||||||
|
History: 1,
|
||||||
|
TTL: CallsKeptFor,
|
||||||
|
MaxValueSize: CallAnswerBytes + 4<<10,
|
||||||
|
MaxBytes: 2 * KeptCallsDurably * (CallAnswerBytes + 4<<10),
|
||||||
|
Storage: jetstream.FileStorage,
|
||||||
|
}); err != nil {
|
||||||
|
return fmt.Errorf("asserting bucket %s: %w", CallsBucket, err)
|
||||||
|
}
|
||||||
|
// **The count, on the stream under the bucket.** A bucket has an age and a size and no count;
|
||||||
|
// the stream it is made of does, and with one value per key the oldest message is the oldest
|
||||||
|
// call. Asserted after the bucket, every time, because asserting the bucket writes the stream's
|
||||||
|
// configuration whole and puts the count back to none.
|
||||||
|
stream, err := js.Stream(ctx, "KV_"+CallsBucket)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("reading the stream under %s: %w", CallsBucket, err)
|
||||||
|
}
|
||||||
|
cfg := stream.CachedInfo().Config
|
||||||
|
if cfg.MaxMsgs != 2*KeptCallsDurably {
|
||||||
|
cfg.MaxMsgs = 2 * KeptCallsDurably
|
||||||
|
cfg.Discard = jetstream.DiscardOld
|
||||||
|
if _, err := js.UpdateStream(ctx, cfg); err != nil {
|
||||||
|
return fmt.Errorf("bounding %s to the last %d calls: %w", CallsBucket, KeptCallsDurably, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||||
|
Bucket: HandActsBucket,
|
||||||
|
Description: "every act a person did by hand, with why (novox/hq to-be 45 §7): written by the " +
|
||||||
|
"controller's repairing verbs and `hand-act record`, read through `hand-acts`",
|
||||||
|
History: 1,
|
||||||
|
TTL: HandActsKeptFor,
|
||||||
|
MaxValueSize: 16 << 10,
|
||||||
|
MaxBytes: 64 << 20,
|
||||||
|
Storage: jetstream.FileStorage,
|
||||||
|
}); err != nil {
|
||||||
|
return fmt.Errorf("asserting bucket %s: %w", HandActsBucket, err)
|
||||||
|
}
|
||||||
|
// **No age on the open conditions.** A condition is removed when observation clears it and at no
|
||||||
|
// other moment: one that expired would be a fault the store forgot while it was still true.
|
||||||
|
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||||
|
Bucket: ConditionsBucket,
|
||||||
|
Description: "every condition open now, one key each (novox/hq to-be 45 §2): written by the " +
|
||||||
|
"controller alone, raised and cleared by observation, read through `conditions`",
|
||||||
|
History: 1,
|
||||||
|
MaxValueSize: 64 << 10,
|
||||||
|
MaxBytes: 64 << 20,
|
||||||
|
Storage: jetstream.FileStorage,
|
||||||
|
}); err != nil {
|
||||||
|
return fmt.Errorf("asserting bucket %s: %w", ConditionsBucket, err)
|
||||||
|
}
|
||||||
|
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||||
|
Bucket: ConditionHistoryBucket,
|
||||||
|
Description: "every transition of a condition — raised, changed, silenced, cleared — kept ninety " +
|
||||||
|
"days (novox/hq to-be 45 §2): written by the controller alone, read through `conditions history`",
|
||||||
|
History: 1,
|
||||||
|
TTL: ConditionHistoryKeptFor,
|
||||||
|
MaxValueSize: 64 << 10,
|
||||||
|
MaxBytes: 256 << 20,
|
||||||
|
Storage: jetstream.FileStorage,
|
||||||
|
}); err != nil {
|
||||||
|
return fmt.Errorf("asserting bucket %s: %w", ConditionHistoryBucket, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **The controller may write every bucket it writes** (novox/hq to-be 45 §1, issue 269). Writing a
|
||||||
|
// key is a publish to the bucket's own subject, which the management interface's grant does not
|
||||||
|
// cover: the cancelled sets' writes timed out for want of this, and the controller's own buckets
|
||||||
|
// would have.
|
||||||
|
func TestTheControllerMayWriteEveryBucketItWrites(t *testing.T) {
|
||||||
|
p, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := []string{"$KV." + CallsBucket + ".>", "$KV." + HandActsBucket + ".>"}
|
||||||
|
for _, seat := range seatsTheControllerAsks {
|
||||||
|
if hasCancelledSet(seat) {
|
||||||
|
want = append(want, "$KV."+CancelledSetName(seat)+".>")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, subject := range want {
|
||||||
|
if !slices.Contains(p.Publish, subject) {
|
||||||
|
t.Errorf("the controller may not publish %s, so it cannot write that bucket", subject)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if slices.Contains(p.Publish, "$KV.>") {
|
||||||
|
t.Error("the controller may write any bucket, a module's state included")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A machine's node tools may say they are there, as that machine and no other** (novox/hq to-be 45
|
||||||
|
// S11), and the controller may hear the bus's advisories and ask who answers — read-only, named.
|
||||||
|
func TestTheWatchedSignalsMayBeSaidAndHeard(t *testing.T) {
|
||||||
|
tools, err := PermissionsFor(Principal{Kind: KindNodeTools, Node: "anchor", Module: RuntimeModule, PasswordHash: "x"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !slices.Contains(tools.Publish, "mesh.control.anchor.tools-alive") {
|
||||||
|
t.Error("the node tools may not say they are there")
|
||||||
|
}
|
||||||
|
for _, s := range tools.Publish {
|
||||||
|
if strings.Contains(s, "tools-alive") && s != "mesh.control.anchor.tools-alive" {
|
||||||
|
t.Errorf("the node tools may say %s", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, s := range BusAdvisories {
|
||||||
|
if !slices.Contains(controller.Subscribe, s) {
|
||||||
|
t.Errorf("the controller may not hear %s", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !slices.Contains(controller.Publish, "$SRV.INFO") || slices.Contains(controller.Subscribe, "$JS.EVENT.>") {
|
||||||
|
t.Error("the controller may not ask who answers, or hears every API call")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -125,6 +125,16 @@ type Principal struct {
|
|||||||
// goes with the retired seat row.
|
// goes with the retired seat row.
|
||||||
var seatsTheControllerAsks = []string{"node-build-agent", "mesh-build-machine"}
|
var seatsTheControllerAsks = []string{"node-build-agent", "mesh-build-machine"}
|
||||||
|
|
||||||
|
// SeatVerb is one verb of one seat, on every machine holding it.
|
||||||
|
type SeatVerb struct{ Seat, Verb string }
|
||||||
|
|
||||||
|
// VerbsTheSelfCheckAsks are the seat verbs the controller's self-check and watchdogs call (novox/hq
|
||||||
|
// to-be 45 §4): D8 reads every machine's ban list. **Named one by one, and the test that holds them
|
||||||
|
// to the probe registry is the reason they cannot drift** — a probe that calls a verb its grant does
|
||||||
|
// not name is refused by the bus on every run (found live on 2026-10-06: D8 timed out on each
|
||||||
|
// machine, refused). Asked of any machine (`.*`), read-only verbs, nothing else of the seat.
|
||||||
|
var VerbsTheSelfCheckAsks = []SeatVerb{{Seat: "node-intrusion-prevention", Verb: "banned"}}
|
||||||
|
|
||||||
// perMachineEvents are a node-scoped seat's events about the holder itself, whose last token is the
|
// perMachineEvents are a node-scoped seat's events about the holder itself, whose last token is the
|
||||||
// holder's machine (novox/hq ADR 0219): `paused.<node>`, the build agent saying whether it takes work.
|
// holder's machine (novox/hq ADR 0219): `paused.<node>`, the build agent saying whether it takes work.
|
||||||
var perMachineEvents = map[string]bool{"paused.*": true}
|
var perMachineEvents = map[string]bool{"paused.*": true}
|
||||||
@@ -237,6 +247,12 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
|||||||
// building, kill it, pause it, resume it. The queue is the controller's to show and to
|
// building, kill it, pause it, resume it. The queue is the controller's to show and to
|
||||||
// change, and what one machine is doing with an ask it took only that machine can say.
|
// change, and what one machine is doing with an ask it took only that machine can say.
|
||||||
pub = append(pub, "mesh.seat."+seat+".tool.>")
|
pub = append(pub, "mesh.seat."+seat+".tool.>")
|
||||||
|
// **And its cancelled set** (novox/hq ADR 0219, issue 269): a cancel writes the ask's id
|
||||||
|
// there before it deletes the ask, and a write is a publish to the bucket's subject, which
|
||||||
|
// `$JS.API.>` does not cover — so every cancel timed out, refused by this list.
|
||||||
|
if hasCancelledSet(seat) {
|
||||||
|
pub = append(pub, "$KV."+CancelledSetName(seat)+".>")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// **And what the mesh says it did** (novox/hq ADR 0134). The control plane states its own
|
// **And what the mesh says it did** (novox/hq ADR 0134). The control plane states its own
|
||||||
// facts under the seat it holds, because a role's events belong to the role and keep their
|
// facts under the seat it holds, because a role's events belong to the role and keep their
|
||||||
@@ -260,6 +276,13 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
|||||||
sub = append(sub, "mesh.seat."+ControllerSeat+".tool.>")
|
sub = append(sub, "mesh.seat."+ControllerSeat+".tool.>")
|
||||||
// And says so (novox/hq ADR 0197): it answers discovery for the seat it serves.
|
// And says so (novox/hq ADR 0197): it answers discovery for the seat it serves.
|
||||||
sub = append(sub, announcing(ControllerSeat)...)
|
sub = append(sub, announcing(ControllerSeat)...)
|
||||||
|
// And the verbs the self-check reads with, of any machine's holder (novox/hq to-be 45 §4).
|
||||||
|
for _, v := range VerbsTheSelfCheckAsks {
|
||||||
|
pub = append(pub, "mesh.seat."+v.Seat+".tool."+v.Verb+".*")
|
||||||
|
}
|
||||||
|
// And asks who answers (novox/hq to-be 45 §4, D3): the self-check finds every seat's holder by
|
||||||
|
// the same discovery the console reads. The question only; the answers come to its own inbox.
|
||||||
|
pub = append(pub, "$SRV.INFO")
|
||||||
|
|
||||||
// The events it reacts to, and its ack subject on the stream they arrive from
|
// The events it reacts to, and its ack subject on the stream they arrive from
|
||||||
// (streams.go). **Each named, not a pattern**: `mesh.mod.*.event.>` would make the
|
// (streams.go). **Each named, not a pattern**: `mesh.mod.*.event.>` would make the
|
||||||
@@ -288,6 +311,18 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
|||||||
// enrolments and can reach nothing else.
|
// enrolments and can reach nothing else.
|
||||||
pub = append(pub, "_INBOX."+enrolmentPrefix+".>")
|
pub = append(pub, "_INBOX."+enrolmentPrefix+".>")
|
||||||
|
|
||||||
|
// **And its own buckets** (novox/hq to-be 45 §1): the calls it served and the acts done by
|
||||||
|
// hand, which it alone writes. A put is a publish to the bucket's subject, which `$JS.API.>`
|
||||||
|
// does not cover; each bucket named, not `$KV.>`, which would let it write any module's state.
|
||||||
|
for _, bucket := range ControllerBuckets() {
|
||||||
|
pub = append(pub, "$KV."+bucket+".>")
|
||||||
|
}
|
||||||
|
// **And what the bus says about itself, read-only** (novox/hq to-be 45 §3, S9): a durable
|
||||||
|
// consumer that gave up on a message, or one that was deleted. The server already publishes
|
||||||
|
// both in the mesh's own account; the controller says each as a condition in the mesh's words.
|
||||||
|
// Named, not `$JS.EVENT.>`: the other advisories are every API call the mesh makes.
|
||||||
|
sub = append(sub, BusAdvisories...)
|
||||||
|
|
||||||
case KindPerson:
|
case KindPerson:
|
||||||
// Tools, and nothing else. Every subject a person may publish is a tool call; a person
|
// Tools, and nothing else. Every subject a person may publish is a tool call; a person
|
||||||
// who could publish an event would be able to claim a module said something.
|
// who could publish an event would be able to claim a module said something.
|
||||||
@@ -499,6 +534,9 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
|||||||
// that varies is the module, so the pattern is the machine's own assignments.
|
// that varies is the module, so the pattern is the machine's own assignments.
|
||||||
sub = append(sub, "mesh.assignment."+p.Node+".*")
|
sub = append(sub, "mesh.assignment."+p.Node+".*")
|
||||||
pub = append(pub, "$JS.API.DIRECT.GET."+AssignmentsStream+".mesh.assignment."+p.Node+".*")
|
pub = append(pub, "$JS.API.DIRECT.GET."+AssignmentsStream+".mesh.assignment."+p.Node+".*")
|
||||||
|
// And that it is there (novox/hq to-be 45 §3, S11): its own heartbeat, under its machine's
|
||||||
|
// name and no other's, on core NATS like the host's.
|
||||||
|
pub = append(pub, "mesh.control."+p.Node+".tools-alive")
|
||||||
// And every tool on the mesh (ADR 0175, decision 5): any node may call any tool on any
|
// And every tool on the mesh (ADR 0175, decision 5): any node may call any tool on any
|
||||||
// node, as the console already could — the runtime is the console's serving mode.
|
// node, as the console already could — the runtime is the console's serving mode.
|
||||||
invoked, err := invokedSubjects([]string{"*"})
|
invoked, err := invokedSubjects([]string{"*"})
|
||||||
|
|||||||
@@ -160,8 +160,9 @@ func RaiseBuckets(a BucketAsserter, buckets []Bucket) (undeclared []string, err
|
|||||||
return nil, fmt.Errorf("listing the bus's state: %w", err)
|
return nil, fmt.Errorf("listing the bus's state: %w", err)
|
||||||
}
|
}
|
||||||
for _, n := range names {
|
for _, n := range names {
|
||||||
// A seat's cancelled set is the mesh's own (novox/hq ADR 0219), not a module's state.
|
// A seat's cancelled set is the mesh's own (novox/hq ADR 0219), not a module's state; so are
|
||||||
if !declared[n] && !IsCancelledSet(n) {
|
// the controller's own buckets (novox/hq to-be 45 §1).
|
||||||
|
if !declared[n] && !IsCancelledSet(n) && !IsControllerBucket(n) {
|
||||||
undeclared = append(undeclared, n)
|
undeclared = append(undeclared, n)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -20,12 +21,15 @@ func TestTheFactsTheGrantPermitsAreTheFactsTheMeshStates(t *testing.T) {
|
|||||||
t.Fatalf("the grant is written for the %q seat and the mesh states its facts under %q",
|
t.Fatalf("the grant is written for the %q seat and the mesh states its facts under %q",
|
||||||
broker.ControllerSeat, link.MeshControllerSeat)
|
broker.ControllerSeat, link.MeshControllerSeat)
|
||||||
}
|
}
|
||||||
for _, event := range []string{link.KeyApplied, link.KeyRefused, link.KeyBuiltBefore} {
|
// And what is wrong, as it changes, and the self-check's heartbeat (novox/hq to-be 45 §2, §4).
|
||||||
|
states := append([]string{link.KeyApplied, link.KeyRefused, link.KeyBuiltBefore}, conditions.Events...)
|
||||||
|
states = append(states, conditions.HeartbeatEvent)
|
||||||
|
for _, event := range states {
|
||||||
if !slices.Contains(broker.ControllerStates, event) {
|
if !slices.Contains(broker.ControllerStates, event) {
|
||||||
t.Errorf("the mesh states %q and its account may not publish it", event)
|
t.Errorf("the mesh states %q and its account may not publish it", event)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(broker.ControllerStates) != 3 {
|
if len(broker.ControllerStates) != len(states) {
|
||||||
t.Errorf("the grant permits %v, which is more than the mesh states", broker.ControllerStates)
|
t.Errorf("the grant permits %v, which is more than the mesh states", broker.ControllerStates)
|
||||||
}
|
}
|
||||||
// **And the seat says it.** A seat carries the protocol of its role (novox/hq ADR 0129), so the
|
// **And the seat says it.** A seat carries the protocol of its role (novox/hq ADR 0129), so the
|
||||||
|
|||||||
@@ -201,7 +201,23 @@ const ControllerName = "controller"
|
|||||||
// something to say.
|
// something to say.
|
||||||
const ControllerSeat = "mesh-controller"
|
const ControllerSeat = "mesh-controller"
|
||||||
|
|
||||||
var ControllerStates = []string{"applied", "refused", "built-before"}
|
var ControllerStates = []string{"applied", "refused", "built-before",
|
||||||
|
// What is wrong, said as it changes (novox/hq to-be 45 §2): a condition raised, changed in
|
||||||
|
// severity, resolver or silence, and cleared. The operator-channel's holder and any other surface
|
||||||
|
// consume them; the controller tells nobody itself.
|
||||||
|
"condition-raised", "condition-changed", "condition-cleared",
|
||||||
|
// And the self-check's heartbeat, at the end of every run (to-be 45 §4, S10): watched from a
|
||||||
|
// machine that is not the control node, so the controller going quiet is itself said.
|
||||||
|
"doctor-heartbeat"}
|
||||||
|
|
||||||
|
// BusAdvisories are what the bus server says about the mesh's own account that the controller
|
||||||
|
// reads (novox/hq to-be 45 §3, S9): a durable consumer that handed a message over as often as it
|
||||||
|
// may and gave up on it, and one that was deleted. Read-only: an advisory is the server's to
|
||||||
|
// publish, and the controller's subscription changes nothing on the bus.
|
||||||
|
var BusAdvisories = []string{
|
||||||
|
"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>",
|
||||||
|
"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>",
|
||||||
|
}
|
||||||
|
|
||||||
// ControllerFollows are the events the controller reacts to: the catalogue saying a module's
|
// ControllerFollows are the events the controller reacts to: the catalogue saying a module's
|
||||||
// current version moved, and a catalogue that has just started saying it may have missed builds.
|
// current version moved, and a catalogue that has just started saying it may have missed builds.
|
||||||
|
|||||||
+2
-2
@@ -24,8 +24,8 @@ accounts {
|
|||||||
jetstream: enabled
|
jetstream: enabled
|
||||||
users = [
|
users = [
|
||||||
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
||||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>"] }
|
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] }
|
||||||
subscribe: { allow: ["$JS.API.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
|
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
|
||||||
allow_responses: { max: 1, ttl: "1m" }
|
allow_responses: { max: 1, ttl: "1m" }
|
||||||
} }
|
} }
|
||||||
{ user: "enrol.one", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: {
|
{ user: "enrol.one", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: {
|
||||||
|
|||||||
@@ -119,3 +119,41 @@ func TestEveryCatalogueStoreSaysHowItIsBackedUp(t *testing.T) {
|
|||||||
t.Fatal("no module in the catalogue provides a store, so this proved nothing")
|
t.Fatal("no module in the catalogue provides a store, so this proved nothing")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestEveryCatalogueIdentityFitsWhatItRequires is ADR 0225's check over the real catalogue: every
|
||||||
|
// module's identity, on the longest machine name, fits the bound of every provision it wants. An
|
||||||
|
// overflow fails here, in the pull request that introduces it, rather than on the provider's machine
|
||||||
|
// the first time a real machine's name meets the module's (issue 263).
|
||||||
|
func TestEveryCatalogueIdentityFitsWhatItRequires(t *testing.T) {
|
||||||
|
root := catalogueRoot(t)
|
||||||
|
found, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||||
|
if err != nil || len(found) == 0 {
|
||||||
|
t.Fatalf("no manifests under %s: %v", root, err)
|
||||||
|
}
|
||||||
|
shelf := Shelf{}
|
||||||
|
for _, p := range found {
|
||||||
|
raw, err := os.ReadFile(p)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: %v", p, err)
|
||||||
|
}
|
||||||
|
m, err := ParseManifest(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: %v", p, err)
|
||||||
|
}
|
||||||
|
shelf[m.Module] = m
|
||||||
|
}
|
||||||
|
for _, p := range IdentityProblems(shelf, DefaultLongestMachine) {
|
||||||
|
t.Error(p)
|
||||||
|
}
|
||||||
|
// The night it was found: the resolver provision bounds nothing, and the object store still 20.
|
||||||
|
if dns, ok := shelf["dnsmasq"]; ok {
|
||||||
|
if b := dns.IdentityBoundOf("wildcard-resolution"); b.Bounded() {
|
||||||
|
t.Errorf("the resolver provision bounds its consumers' identities: %+v", b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if store, ok := shelf["minio"]; ok {
|
||||||
|
if b := store.IdentityBoundOf("s3-bucket"); b.Max != 20 {
|
||||||
|
t.Errorf("the object store's access key is not bounded at 20: %+v", b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -114,7 +114,9 @@ func consumerInto(value, as string) (string, error) {
|
|||||||
// asDNSLabel writes a minted identity as a DNS label.
|
// asDNSLabel writes a minted identity as a DNS label.
|
||||||
//
|
//
|
||||||
// The mesh's identities are already lower-case letters, digits and `_` (ConsumerIdentity), and
|
// The mesh's identities are already lower-case letters, digits and `_` (ConsumerIdentity), and
|
||||||
// already short enough for the tightest backend they reach (CheckIdentity, twenty characters). So
|
// already inside the bound of the provision serving them: a provider that serves one as a DNS label
|
||||||
|
// bounds it at 63 or less (CheckServes, novox/hq ADR 0225), and one that serves it at all without
|
||||||
|
// saying is held to twenty (IdentityBoundOf). So
|
||||||
// this is the separator and nothing else — no lower-casing of what is already lower case, no
|
// this is the separator and nothing else — no lower-casing of what is already lower case, no
|
||||||
// truncation to a limit the identity is already inside, no padding of a name that is already long
|
// truncation to a limit the identity is already inside, no padding of a name that is already long
|
||||||
// enough. Each of those would be the mesh guessing at a rule it has not been given.
|
// enough. Each of those would be the mesh guessing at a rule it has not been given.
|
||||||
@@ -141,11 +143,31 @@ func CheckServes(m Manifest) []string {
|
|||||||
problems = append(problems, fmt.Sprintf(
|
problems = append(problems, fmt.Sprintf(
|
||||||
"%s serves %s, and the value it serves as %q %s", m.Module, provision, key, err))
|
"%s serves %s, and the value it serves as %q %s", m.Module, provision, key, err))
|
||||||
}
|
}
|
||||||
|
// A label longer than DNS keeps is not truncated here (asDNSLabel), so the offer's own
|
||||||
|
// bound has to keep the identity inside one (ADR 0225).
|
||||||
|
if strings.Contains(text, "${consumer:as:dns}") {
|
||||||
|
if b := m.IdentityBoundOf(provision); !b.Bounded() || b.Max > dnsLabelLimit {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s serves %s's consumers their identity as a DNS label in %q, and bounds "+
|
||||||
|
"that identity at %s: a label keeps %d — state an `identity` of at most %d",
|
||||||
|
m.Module, provision, key, boundWords(b), dnsLabelLimit, dnsLabelLimit))
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return problems
|
return problems
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// dnsLabelLimit is the longest DNS label (RFC 1035 §2.3.4).
|
||||||
|
const dnsLabelLimit = 63
|
||||||
|
|
||||||
|
func boundWords(b IdentityBound) string {
|
||||||
|
if !b.Bounded() {
|
||||||
|
return "nothing"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%d", b.Max)
|
||||||
|
}
|
||||||
|
|
||||||
func sortedServes(serves map[string]map[string]any) []string {
|
func sortedServes(serves map[string]map[string]any) []string {
|
||||||
out := make([]string, 0, len(serves))
|
out := make([]string, 0, len(serves))
|
||||||
for k := range serves {
|
for k := range serves {
|
||||||
|
|||||||
@@ -170,6 +170,10 @@ type Rendering struct {
|
|||||||
// rather than resolved, because who consumes a node is a fact about the rest of the mesh and
|
// rather than resolved, because who consumes a node is a fact about the rest of the mesh and
|
||||||
// resolution answers questions about one machine.
|
// resolution answers questions about one machine.
|
||||||
Grants []Grant
|
Grants []Grant
|
||||||
|
// Withheld is every consumer left out of Grants because its identity overflows the provision's
|
||||||
|
// bound (novox/hq ADR 0225). Composed into nothing; carried so the machine's declaration can say
|
||||||
|
// whom it does not serve, and why, beside what it does.
|
||||||
|
Withheld []Overflow
|
||||||
|
|
||||||
// Ports is where this machine puts what each module needs reachable, by module and by the
|
// Ports is where this machine puts what each module needs reachable, by module and by the
|
||||||
// port the software itself uses (novox/hq ADR 0038).
|
// port the software itself uses (novox/hq ADR 0038).
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package catalogue
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"regexp"
|
"regexp"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -56,13 +57,36 @@ func ConsumerIdentity(node, module string) string {
|
|||||||
return IdentityPrefix + clean(node) + "_" + clean(module)
|
return IdentityPrefix + clean(node) + "_" + clean(module)
|
||||||
}
|
}
|
||||||
|
|
||||||
// identityLimit is the shortest identifier limit among the systems these names reach: an S3 access
|
// IdentityBound is the longest consumer identity one provision's backend keeps, and what keeps it
|
||||||
// key's 20 (novox/hq 04-ISSUES/010). PostgreSQL keeps 63 and MinIO 20, so 20 is the one that binds —
|
// (novox/hq ADR 0049, refined by ADR 0225). Max zero means no bound: the provision keeps no name
|
||||||
// the comment used to name PostgreSQL and was wrong. A name over it is refused, with the remedy a
|
// derived from its consumer, or keeps one in something with no limit the mesh need respect.
|
||||||
// short slug (ADR 0049), not silently cut to fit.
|
type IdentityBound struct {
|
||||||
const identityLimit = 20
|
// Max is the longest identity that backend keeps, in characters; zero for none.
|
||||||
|
Max int `json:"max,omitempty"`
|
||||||
|
// In is what keeps it, in words a refusal can quote: "an S3 access key", "a PostgreSQL role".
|
||||||
|
In string `json:"in,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
// CheckIdentity refuses an identity that would not fit the tightest backend a consumer reaches.
|
// Bounded is whether this bound refuses anything.
|
||||||
|
func (b IdentityBound) Bounded() bool { return b.Max > 0 }
|
||||||
|
|
||||||
|
// DefaultIdentityLimit is the bound on a provision whose provider receives its consumers and does
|
||||||
|
// not say how long a name it keeps: an S3 access key's 20 (novox/hq 04-ISSUES/010, 034), the
|
||||||
|
// tightest backend the mesh has met. It was the bound on every provision until ADR 0225; it stays
|
||||||
|
// the bound on any that has not said otherwise, because a provider that is told each consumer's
|
||||||
|
// identity may create a name from it in a backend nobody has measured.
|
||||||
|
const DefaultIdentityLimit = 20
|
||||||
|
|
||||||
|
// DefaultIdentityBound is DefaultIdentityLimit, said as a bound.
|
||||||
|
var DefaultIdentityBound = IdentityBound{Max: DefaultIdentityLimit,
|
||||||
|
In: "a backend that has not said its limit (the tightest known, an S3 access key's)"}
|
||||||
|
|
||||||
|
// identityLimit is the bound CheckIdentity applies, for a caller that does not know which provision
|
||||||
|
// the identity is for.
|
||||||
|
const identityLimit = DefaultIdentityLimit
|
||||||
|
|
||||||
|
// CheckIdentity refuses an identity that would not fit the tightest backend the mesh knows. A caller
|
||||||
|
// that knows the provision uses CheckIdentityWithin and that provision's own bound (ADR 0225).
|
||||||
//
|
//
|
||||||
// **Truncation is not an error in most of these systems** — a name past the limit is cut to fit and
|
// **Truncation is not an error in most of these systems** — a name past the limit is cut to fit and
|
||||||
// the statement succeeds, so two consumers agreeing for the first N bytes would become one login
|
// the statement succeeds, so two consumers agreeing for the first N bytes would become one login
|
||||||
@@ -70,12 +94,127 @@ const identityLimit = 20
|
|||||||
// name and is the only thing that can choose another. The remedy is a first-class one: give the
|
// name and is the only thing that can choose another. The remedy is a first-class one: give the
|
||||||
// module a short `slug` (ADR 0049), or shorten the machine's name.
|
// module a short `slug` (ADR 0049), or shorten the machine's name.
|
||||||
func CheckIdentity(node, module string) error {
|
func CheckIdentity(node, module string) error {
|
||||||
|
return CheckIdentityWithin(node, module, IdentityBound{Max: identityLimit, In: "a backend (an S3 access key)"})
|
||||||
|
}
|
||||||
|
|
||||||
|
// CheckIdentityWithin refuses an identity that would not fit one provision's bound, and accepts any
|
||||||
|
// identity for a provision with none (novox/hq ADR 0225).
|
||||||
|
func CheckIdentityWithin(node, module string, bound IdentityBound) error {
|
||||||
|
if !bound.Bounded() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
got := ConsumerIdentity(node, module)
|
got := ConsumerIdentity(node, module)
|
||||||
if len(got) <= identityLimit {
|
if len(got) <= bound.Max {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"%s on %s is identified as %q, %d characters where a backend (an S3 access key) keeps %d — "+
|
"%s on %s is identified as %q, %d characters where %s keeps %d — "+
|
||||||
"give the module a shorter `slug` or shorten the machine's name",
|
"give the module a shorter `slug` or shorten the machine's name",
|
||||||
module, node, got, len(got), identityLimit)
|
module, node, got, len(got), bound.In, bound.Max)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Overflow is one consumer whose identity does not fit the provision it requires: left out of its
|
||||||
|
// provider's grants and reported, never a reason to refuse the provider's machine (ADR 0225).
|
||||||
|
type Overflow struct {
|
||||||
|
// Provision is what was required, Provider the machine answering it.
|
||||||
|
Provision string `json:"provision"`
|
||||||
|
Provider string `json:"provider"`
|
||||||
|
// Consumer is the machine, Module the module on it that required it.
|
||||||
|
Consumer string `json:"consumer"`
|
||||||
|
Module string `json:"module"`
|
||||||
|
// Identity is the name the mesh derived, and Bound what it overflows.
|
||||||
|
Identity string `json:"identity"`
|
||||||
|
Bound IdentityBound `json:"bound"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o Overflow) String() string {
|
||||||
|
return fmt.Sprintf("%s on %s requires %s from %s and is identified as %q, %d characters where %s "+
|
||||||
|
"keeps %d — left out of %s's grants until the module's `slug` is shorter",
|
||||||
|
o.Module, o.Consumer, o.Provision, o.Provider, o.Identity, len(o.Identity), o.Bound.In,
|
||||||
|
o.Bound.Max, o.Provider)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Overflowing is every requirement of this machine's modules whose identity overflows the bound of
|
||||||
|
// the provision answering it. The same judgement the provider's composition makes before it grants
|
||||||
|
// (grantsFor), made from the consumer's side so `status` can say it about every machine.
|
||||||
|
func (r Resolution) Overflowing() []Overflow {
|
||||||
|
slugs := map[string]string{}
|
||||||
|
for _, m := range r.Modules {
|
||||||
|
slugs[m.Module] = m.Slug
|
||||||
|
}
|
||||||
|
var out []Overflow
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for _, n := range r.Needs {
|
||||||
|
if n.ByRecord || !n.Identity.Bounded() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
source := IdentitySource(slugs[n.For], n.For)
|
||||||
|
if CheckIdentityWithin(r.Node, source, n.Identity) == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
key := n.Name + "\x00" + n.From + "\x00" + n.For
|
||||||
|
if seen[key] {
|
||||||
|
continue // one line per requirement, however many local names it has
|
||||||
|
}
|
||||||
|
seen[key] = true
|
||||||
|
out = append(out, Overflow{Provision: n.Name, Provider: n.From, Consumer: r.Node, Module: n.For,
|
||||||
|
Identity: ConsumerIdentity(r.Node, source), Bound: n.Identity})
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(i, j int) bool {
|
||||||
|
if out[i].Module != out[j].Module {
|
||||||
|
return out[i].Module < out[j].Module
|
||||||
|
}
|
||||||
|
return out[i].Provision < out[j].Provision
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultLongestMachine is the machine name the catalogue check judges identities on when it is not
|
||||||
|
// told one: the longest name of the mesh this catalogue is written for, so a catalogue that passes
|
||||||
|
// passes on every machine that mesh has. `module check --longest-machine-name` says another mesh's;
|
||||||
|
// a mesh that names a longer machine raises this in the same change (ADR 0225).
|
||||||
|
const DefaultLongestMachine = 6
|
||||||
|
|
||||||
|
// IdentityProblems is every module whose identity would overflow a provision it wants, on a machine
|
||||||
|
// whose name is `longestMachine` characters — judged before merge, over the catalogue alone, so the
|
||||||
|
// pull request that introduces an overflow is the one refused (novox/hq ADR 0225, issue 263). A
|
||||||
|
// provision no module in the shelf offers is not judged: its bound is not known here.
|
||||||
|
func IdentityProblems(shelf Shelf, longestMachine int) []string {
|
||||||
|
offeredBy := map[string][]string{}
|
||||||
|
for _, name := range shelfOrder(shelf) {
|
||||||
|
for _, o := range shelf[name].Offers() {
|
||||||
|
offeredBy[o] = append(offeredBy[o], name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
machine := strings.Repeat("n", longestMachine)
|
||||||
|
var problems []string
|
||||||
|
for _, name := range shelfOrder(shelf) {
|
||||||
|
m := shelf[name]
|
||||||
|
source := IdentitySource(m.Slug, m.Module)
|
||||||
|
for _, want := range m.Wants() {
|
||||||
|
// The tightest bound among the modules offering it: whichever one answers on a given
|
||||||
|
// machine, the identity has to fit it.
|
||||||
|
tightest, by := IdentityBound{}, ""
|
||||||
|
for _, provider := range offeredBy[want] {
|
||||||
|
if provider == name {
|
||||||
|
continue // a module answering its own requirement is not its own consumer
|
||||||
|
}
|
||||||
|
b := shelf[provider].IdentityBoundOf(want)
|
||||||
|
if b.Bounded() && (!tightest.Bounded() || b.Max < tightest.Max) {
|
||||||
|
tightest, by = b, provider
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if CheckIdentityWithin(machine, source, tightest) == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
got := ConsumerIdentity(machine, source)
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s wants %s, and %s keeps its consumers' identities in %s of at most %d characters: "+
|
||||||
|
"on a machine with a %d-character name it is identified as %q, %d — give %s a "+
|
||||||
|
"`slug` of at most %d characters",
|
||||||
|
name, want, by, tightest.In, tightest.Max, longestMachine, got, len(got), name,
|
||||||
|
tightest.Max-len(IdentityPrefix)-longestMachine-1))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return problems
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,190 @@
|
|||||||
|
package catalogue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Each test names the decision it defends: novox/hq ADR 0225, which refines ADR 0049 after issue 263.
|
||||||
|
|
||||||
|
// The provision a module requires sets the bound on its identity, not the tightest backend anywhere.
|
||||||
|
func TestABoundIsTheProvisionsOwn(t *testing.T) {
|
||||||
|
store := Manifest{Module: "objects", Provides: []Offer{{Name: "s3-bucket", Scope: ScopeMesh,
|
||||||
|
Identity: &OfferIdentity{Max: 20, In: "an S3 access key"}}},
|
||||||
|
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}}
|
||||||
|
database := Manifest{Module: "db", Provides: []Offer{{Name: "postgres-database", Scope: ScopeMesh,
|
||||||
|
Identity: &OfferIdentity{Max: 63, In: "a PostgreSQL role"}}},
|
||||||
|
Receives: map[string]string{"postgres-database": "/var/lib/mesh/db/mesh.json"}}
|
||||||
|
if b := store.IdentityBoundOf("s3-bucket"); b.Max != 20 || b.In != "an S3 access key" {
|
||||||
|
t.Errorf("an object store's stated bound was not taken: %+v", b)
|
||||||
|
}
|
||||||
|
if b := database.IdentityBoundOf("postgres-database"); b.Max != 63 {
|
||||||
|
t.Errorf("a database's stated bound was not taken: %+v", b)
|
||||||
|
}
|
||||||
|
// mesh_workstation_keycloak is 25: refused by the object store, accepted by the database.
|
||||||
|
if CheckIdentityWithin("workstation", "keycloak", store.IdentityBoundOf("s3-bucket")) == nil {
|
||||||
|
t.Error("a 25-character identity fit a 20-character access key")
|
||||||
|
}
|
||||||
|
if err := CheckIdentityWithin("workstation", "keycloak", database.IdentityBoundOf("postgres-database")); err != nil {
|
||||||
|
t.Errorf("a database consumer paid the object store's limit: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// What an offer leaves unsaid follows from whether its provider can keep a name at all.
|
||||||
|
func TestAnUnstatedBoundFollowsWhatTheProviderIsTold(t *testing.T) {
|
||||||
|
// Told nothing about its consumers — no receives, nothing served from their identity: no bound.
|
||||||
|
// The resolver provision is exactly this, and its consumers paid an object store's limit (263).
|
||||||
|
resolver := Manifest{Module: "resolver", Provides: []Offer{{Name: "wildcard-resolution", Scope: ScopeMesh}}}
|
||||||
|
if b := resolver.IdentityBoundOf("wildcard-resolution"); b.Bounded() {
|
||||||
|
t.Errorf("a provision that is told nothing of its consumers bounds them: %+v", b)
|
||||||
|
}
|
||||||
|
// Told each consumer and silent about its backend: the old global bound, not none.
|
||||||
|
told := Manifest{Module: "told", Provides: []Offer{{Name: "thing", Scope: ScopeMesh}},
|
||||||
|
Receives: map[string]string{"thing": "/var/lib/mesh/told/mesh.json"}}
|
||||||
|
if b := told.IdentityBoundOf("thing"); b.Max != DefaultIdentityLimit {
|
||||||
|
t.Errorf("a provider told its consumers and silent about its backend is not held to %d: %+v",
|
||||||
|
DefaultIdentityLimit, b)
|
||||||
|
}
|
||||||
|
// Serving a value built from the identity is being told it, too.
|
||||||
|
serving := Manifest{Module: "serving", Provides: []Offer{{Name: "bucket", Scope: ScopeMesh}},
|
||||||
|
Serves: map[string]map[string]any{"bucket": {"name": "b-${consumer:as:dns}"}}}
|
||||||
|
if b := serving.IdentityBoundOf("bucket"); b.Max != DefaultIdentityLimit {
|
||||||
|
t.Errorf("a provider deriving a name from its consumers is not bounded: %+v", b)
|
||||||
|
}
|
||||||
|
// And `false` says it outright, even for a provider that receives.
|
||||||
|
routes := Manifest{Module: "routes", Provides: []Offer{{Name: "route", Scope: ScopeMesh,
|
||||||
|
Identity: &OfferIdentity{None: true}}},
|
||||||
|
Receives: map[string]string{"route": "/var/lib/mesh/routes/mesh.json"}}
|
||||||
|
if b := routes.IdentityBoundOf("route"); b.Bounded() {
|
||||||
|
t.Errorf("`identity: false` still bounds: %+v", b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The field reads as written and writes back the same, and refuses what says nothing.
|
||||||
|
func TestAnOffersIdentityIsParsedStrictly(t *testing.T) {
|
||||||
|
for _, raw := range []string{
|
||||||
|
`{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}`,
|
||||||
|
`{"name":"wildcard-resolution","scope":"mesh","identity":false}`,
|
||||||
|
`{"name":"redis-cache","scope":"mesh","identity":{"in":"a Redis ACL user"}}`,
|
||||||
|
} {
|
||||||
|
var o Offer
|
||||||
|
if err := json.Unmarshal([]byte(raw), &o); err != nil {
|
||||||
|
t.Fatalf("%s: %v", raw, err)
|
||||||
|
}
|
||||||
|
back, err := json.Marshal(o)
|
||||||
|
if err != nil || string(back) != raw {
|
||||||
|
t.Errorf("did not round-trip:\n%s\n%s (%v)", raw, back, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, raw := range []string{
|
||||||
|
`{"name":"x","identity":true}`,
|
||||||
|
`{"name":"x","identity":{"max":20,"in":"y","most":3}}`,
|
||||||
|
} {
|
||||||
|
var o Offer
|
||||||
|
if err := json.Unmarshal([]byte(raw), &o); err == nil {
|
||||||
|
t.Errorf("accepted %s", raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bad := Manifest{Module: "bad", Version: "1", Provides: []Offer{
|
||||||
|
{Name: "unsaid", Scope: ScopeMesh, Identity: &OfferIdentity{Max: 20}},
|
||||||
|
{Name: "tiny", Scope: ScopeMesh, Identity: &OfferIdentity{Max: 4, In: "nothing usable"}},
|
||||||
|
}}
|
||||||
|
raw, err := json.Marshal(bad)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, err = ParseManifest(raw)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "without saying what keeps them") ||
|
||||||
|
!strings.Contains(err.Error(), "the shortest the mesh makes") {
|
||||||
|
t.Fatalf("a bound with no `in`, or too short for any identity, was accepted: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Refused before merge: the catalogue check judges each module's identity, on the longest machine
|
||||||
|
// name, against the bound of every provision it wants — and names the module and the slug to set.
|
||||||
|
func TestTheCatalogueCheckRefusesAnIdentityThatOverflowsWhatItRequires(t *testing.T) {
|
||||||
|
shelf := Shelf{
|
||||||
|
"objects": {Module: "objects", Provides: []Offer{{Name: "s3-bucket", Scope: ScopeMesh,
|
||||||
|
Identity: &OfferIdentity{Max: 20, In: "an S3 access key"}}},
|
||||||
|
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}},
|
||||||
|
"photoalbum": {Module: "photoalbum", Requires: []string{"s3-bucket"}},
|
||||||
|
"files": {Module: "files", Requires: []string{"s3-bucket"}},
|
||||||
|
}
|
||||||
|
problems := IdentityProblems(shelf, 6)
|
||||||
|
if len(problems) != 1 || !strings.Contains(problems[0], "photoalbum wants s3-bucket") ||
|
||||||
|
!strings.Contains(problems[0], `"mesh_nnnnnn_photoalbum", 22`) ||
|
||||||
|
!strings.Contains(problems[0], "`slug` of at most 8 characters") {
|
||||||
|
t.Fatalf("one overflow, named with its remedy, was expected: %q", problems)
|
||||||
|
}
|
||||||
|
// A longer machine name refuses more: the check is about the mesh's machines, not one.
|
||||||
|
if got := IdentityProblems(shelf, 10); len(got) != 2 {
|
||||||
|
t.Fatalf("on a 10-character name both overflow (mesh_nnnnnnnnnn_files is 21): %q", got)
|
||||||
|
}
|
||||||
|
// And a slug is the remedy it names.
|
||||||
|
album := shelf["photoalbum"]
|
||||||
|
album.Slug = "album"
|
||||||
|
shelf["photoalbum"] = album
|
||||||
|
if got := IdentityProblems(shelf, 6); len(got) != 0 {
|
||||||
|
t.Fatalf("a slug that fits is still refused: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tonight's case (issue 263): networkmanager, no slug, requiring the mesh's resolver provision on a
|
||||||
|
// machine with a six-character name. Under ADR 0049's one bound it was refused, and its provider's
|
||||||
|
// whole machine with it; the resolver keeps no name, so it is not refused at all.
|
||||||
|
func TestARequirementOnAKeylessProvisionComposesWithALongName(t *testing.T) {
|
||||||
|
shelf := Shelf{
|
||||||
|
"resolver": {Module: "resolver", Provides: []Offer{{Name: "wildcard-resolution", Scope: ScopeMesh}}},
|
||||||
|
"networkmanager": {Module: "networkmanager", Requires: []string{"wildcard-resolution"}},
|
||||||
|
}
|
||||||
|
if CheckIdentity("laptop", "networkmanager") == nil {
|
||||||
|
t.Fatal("the regression is not reproduced: mesh_laptop_networkmanager fits the old global bound")
|
||||||
|
}
|
||||||
|
if got := IdentityProblems(shelf, 6); len(got) != 0 {
|
||||||
|
t.Fatalf("a requirement on a keyless provision was refused for its length: %q", got)
|
||||||
|
}
|
||||||
|
r := Resolution{Node: "laptop", Modules: []Manifest{shelf["networkmanager"]},
|
||||||
|
Needs: []Needed{{Name: "wildcard-resolution", From: "anchor", For: "networkmanager",
|
||||||
|
Identity: shelf["resolver"].IdentityBoundOf("wildcard-resolution")}}}
|
||||||
|
if got := r.Overflowing(); len(got) != 0 {
|
||||||
|
t.Fatalf("a keyless requirement is reported as overflowing: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The consumer's side of the same judgement the provider's composition makes: what `status` says.
|
||||||
|
func TestAnOverflowingRequirementIsNamedFromTheConsumersSide(t *testing.T) {
|
||||||
|
bound := IdentityBound{Max: 20, In: "an S3 access key"}
|
||||||
|
r := Resolution{Node: "laptop",
|
||||||
|
Modules: []Manifest{{Module: "photoalbum"}, {Module: "files"}, {Module: "gallery", Slug: "gal"}},
|
||||||
|
Needs: []Needed{
|
||||||
|
{Name: "s3-bucket", From: "anchor", For: "photoalbum", Identity: bound},
|
||||||
|
{Name: "s3-bucket", From: "anchor", For: "photoalbum", Local: "second", Identity: bound},
|
||||||
|
{Name: "s3-bucket", From: "anchor", For: "files", Identity: bound},
|
||||||
|
{Name: "s3-bucket", From: "anchor", For: "gallery", Identity: bound},
|
||||||
|
{Name: "licence", From: "records", For: "photoalbum", ByRecord: true, Identity: bound},
|
||||||
|
}}
|
||||||
|
got := r.Overflowing()
|
||||||
|
if len(got) != 1 || got[0].Module != "photoalbum" || got[0].Identity != "mesh_laptop_photoalbum" ||
|
||||||
|
got[0].Provider != "anchor" {
|
||||||
|
t.Fatalf("one overflow, once, was expected: %+v", got)
|
||||||
|
}
|
||||||
|
if said := got[0].String(); !strings.Contains(said, "an S3 access key keeps 20") ||
|
||||||
|
!strings.Contains(said, "slug") {
|
||||||
|
t.Fatalf("the overflow does not say what keeps it or the remedy: %s", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An identity served as a DNS label is never truncated, so the offer's bound must keep it in one.
|
||||||
|
func TestAnIdentityServedAsADNSLabelIsBoundedToOne(t *testing.T) {
|
||||||
|
serves := map[string]map[string]any{"bucket": {"name": "${consumer:as:dns}"}}
|
||||||
|
wide := Manifest{Module: "wide", Serves: serves, Provides: []Offer{{Name: "bucket", Scope: ScopeMesh,
|
||||||
|
Identity: &OfferIdentity{Max: 255, In: "a client id"}}}}
|
||||||
|
if got := CheckServes(wide); len(got) != 1 || !strings.Contains(got[0], "a label keeps 63") {
|
||||||
|
t.Fatalf("a 255-character bound on a DNS label passed: %q", got)
|
||||||
|
}
|
||||||
|
unsaid := Manifest{Module: "unsaid", Serves: serves, Provides: []Offer{{Name: "bucket", Scope: ScopeMesh}}}
|
||||||
|
if got := CheckServes(unsaid); len(got) != 0 {
|
||||||
|
t.Fatalf("the default bound (20) on a DNS label was refused: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -8,6 +8,7 @@ package catalogue
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"regexp"
|
"regexp"
|
||||||
"sort"
|
"sort"
|
||||||
@@ -153,6 +154,84 @@ type Offer struct {
|
|||||||
// a seat's holder gated by the machine's graphical session, and pulling one in for whatever asked
|
// a seat's holder gated by the machine's graphical session, and pulling one in for whatever asked
|
||||||
// is the misassignment research 026 found. Unmet, the requirement is refused naming who could.
|
// is the misassignment research 026 found. Unmet, the requirement is refused naming who could.
|
||||||
Reach string `json:"reach,omitempty"`
|
Reach string `json:"reach,omitempty"`
|
||||||
|
// Identity is the longest consumer identity this provision's backend keeps (novox/hq ADR 0225):
|
||||||
|
// `{"max": 63, "in": "a PostgreSQL role"}`, or `false` for a provision that keeps no name derived
|
||||||
|
// from its consumer. Unsaid, the mesh assumes the tightest backend it knows when the provider is
|
||||||
|
// told its consumers, and no bound when it is not — see IdentityBoundOf.
|
||||||
|
Identity *OfferIdentity `json:"identity,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// OfferIdentity is what an offer says about the names its backend keeps for its consumers.
|
||||||
|
type OfferIdentity struct {
|
||||||
|
// None is set by `"identity": false`: the provision keeps no name derived from its consumer.
|
||||||
|
None bool
|
||||||
|
// Max is the longest identity kept, in characters; zero with In set means no limit worth stating.
|
||||||
|
Max int
|
||||||
|
// In is what keeps it, for a refusal to quote.
|
||||||
|
In string
|
||||||
|
}
|
||||||
|
|
||||||
|
// UnmarshalJSON accepts `false` or `{"max": N, "in": "..."}`.
|
||||||
|
func (i *OfferIdentity) UnmarshalJSON(raw []byte) error {
|
||||||
|
var flag bool
|
||||||
|
if err := json.Unmarshal(raw, &flag); err == nil {
|
||||||
|
if flag {
|
||||||
|
return errors.New("an offer's identity is false (it keeps no name) or {max, in}; true says nothing")
|
||||||
|
}
|
||||||
|
*i = OfferIdentity{None: true}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var full struct {
|
||||||
|
Max int `json:"max,omitempty"`
|
||||||
|
In string `json:"in"`
|
||||||
|
}
|
||||||
|
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||||
|
dec.DisallowUnknownFields()
|
||||||
|
if err := dec.Decode(&full); err != nil {
|
||||||
|
return fmt.Errorf("an offer's identity is false or {max, in}: %w", err)
|
||||||
|
}
|
||||||
|
*i = OfferIdentity{Max: full.Max, In: full.In}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// MarshalJSON writes it back in the form it was written.
|
||||||
|
func (i OfferIdentity) MarshalJSON() ([]byte, error) {
|
||||||
|
if i.None {
|
||||||
|
return []byte("false"), nil
|
||||||
|
}
|
||||||
|
return json.Marshal(struct {
|
||||||
|
Max int `json:"max,omitempty"`
|
||||||
|
In string `json:"in"`
|
||||||
|
}{i.Max, i.In})
|
||||||
|
}
|
||||||
|
|
||||||
|
// IdentityBoundOf is the bound this module's offer of a provision puts on its consumers' identities
|
||||||
|
// (novox/hq ADR 0225).
|
||||||
|
//
|
||||||
|
// **What an offer says, it gets.** Where it says nothing, the bound follows from whether the
|
||||||
|
// provider can keep a name at all: a provider that receives the provision, or serves its consumers
|
||||||
|
// a value built from their identity, is told who each consumer is and may create a name from it in
|
||||||
|
// a backend nobody measured — so it keeps the old global bound, DefaultIdentityBound. One that does
|
||||||
|
// neither is told nothing about its consumers and keeps nothing of them: no bound. The resolver
|
||||||
|
// provision is that case, and its consumers paid an object store's limit until this (issue 263).
|
||||||
|
func (m Manifest) IdentityBoundOf(provision string) IdentityBound {
|
||||||
|
for _, o := range m.Provides {
|
||||||
|
if o.Name != provision || o.Identity == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if o.Identity.None {
|
||||||
|
return IdentityBound{}
|
||||||
|
}
|
||||||
|
return IdentityBound{Max: o.Identity.Max, In: o.Identity.In}
|
||||||
|
}
|
||||||
|
if _, receives := m.Receives[provision]; receives {
|
||||||
|
return DefaultIdentityBound
|
||||||
|
}
|
||||||
|
if served, err := json.Marshal(m.Serves[provision]); err == nil &&
|
||||||
|
bytes.Contains(served, []byte("${consumer:as")) {
|
||||||
|
return DefaultIdentityBound
|
||||||
|
}
|
||||||
|
return IdentityBound{}
|
||||||
}
|
}
|
||||||
|
|
||||||
// MachineReach is whether a provision is usable only on its provider's own machine.
|
// MachineReach is whether a provision is usable only on its provider's own machine.
|
||||||
@@ -206,20 +285,21 @@ func (o *Offer) UnmarshalJSON(raw []byte) error {
|
|||||||
Scope string `json:"scope,omitempty"`
|
Scope string `json:"scope,omitempty"`
|
||||||
Credential *OfferCredential `json:"credential,omitempty"`
|
Credential *OfferCredential `json:"credential,omitempty"`
|
||||||
Reach string `json:"reach,omitempty"`
|
Reach string `json:"reach,omitempty"`
|
||||||
|
Identity *OfferIdentity `json:"identity,omitempty"`
|
||||||
}
|
}
|
||||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||||
dec.DisallowUnknownFields()
|
dec.DisallowUnknownFields()
|
||||||
if err := dec.Decode(&full); err != nil {
|
if err := dec.Decode(&full); err != nil {
|
||||||
return fmt.Errorf("a provided name is either a string or {name, scope, credential, reach}: %w", err)
|
return fmt.Errorf("a provided name is either a string or {name, scope, credential, reach, identity}: %w", err)
|
||||||
}
|
}
|
||||||
o.Name, o.Scope, o.Credential, o.Reach = full.Name, full.Scope, full.Credential, full.Reach
|
o.Name, o.Scope, o.Credential, o.Reach, o.Identity = full.Name, full.Scope, full.Credential, full.Reach, full.Identity
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// MarshalJSON writes back the short form when there is nothing else to say, so a manifest that
|
// MarshalJSON writes back the short form when there is nothing else to say, so a manifest that
|
||||||
// went through the mesh comes out looking like the one that went in.
|
// went through the mesh comes out looking like the one that went in.
|
||||||
func (o Offer) MarshalJSON() ([]byte, error) {
|
func (o Offer) MarshalJSON() ([]byte, error) {
|
||||||
if o.Scope == "" && o.Credential == nil && o.Reach == "" {
|
if o.Scope == "" && o.Credential == nil && o.Reach == "" && o.Identity == nil {
|
||||||
return json.Marshal(o.Name)
|
return json.Marshal(o.Name)
|
||||||
}
|
}
|
||||||
return json.Marshal(struct {
|
return json.Marshal(struct {
|
||||||
@@ -227,7 +307,8 @@ func (o Offer) MarshalJSON() ([]byte, error) {
|
|||||||
Scope string `json:"scope,omitempty"`
|
Scope string `json:"scope,omitempty"`
|
||||||
Credential *OfferCredential `json:"credential,omitempty"`
|
Credential *OfferCredential `json:"credential,omitempty"`
|
||||||
Reach string `json:"reach,omitempty"`
|
Reach string `json:"reach,omitempty"`
|
||||||
}{o.Name, o.Scope, o.Credential, o.Reach})
|
Identity *OfferIdentity `json:"identity,omitempty"`
|
||||||
|
}{o.Name, o.Scope, o.Credential, o.Reach, o.Identity})
|
||||||
}
|
}
|
||||||
|
|
||||||
// Manifest is everything a module says about itself.
|
// Manifest is everything a module says about itself.
|
||||||
@@ -237,9 +318,10 @@ type Manifest struct {
|
|||||||
|
|
||||||
// Slug is a short identifier the mesh uses in place of the module name when it derives a
|
// Slug is a short identifier the mesh uses in place of the module name when it derives a
|
||||||
// consumer's login (novox/hq ADR 0049). Optional: a module with a short name needs none. It
|
// consumer's login (novox/hq ADR 0049). Optional: a module with a short name needs none. It
|
||||||
// exists because `mesh_<node>_<module>` must fit the tightest backend a consumer reaches — an S3
|
// exists because `mesh_<node>_<module>` must fit the bound of every provision the module
|
||||||
// access key is 20 characters — and a long module name would overflow it. A person choosing
|
// requires — an S3 access key's 20 characters is the tightest — and a long module name would
|
||||||
// `kc` for keycloak keeps the identity legible where a hash would not.
|
// overflow it (ADR 0225: the bound is the provision's own, and a keyless one has none). A person
|
||||||
|
// choosing `kc` for keycloak keeps the identity legible where a hash would not.
|
||||||
Slug string `json:"slug,omitempty"`
|
Slug string `json:"slug,omitempty"`
|
||||||
|
|
||||||
// Provides are the names other modules may require. A module always provides its own name;
|
// Provides are the names other modules may require. A module always provides its own name;
|
||||||
@@ -1265,8 +1347,10 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
|||||||
"%q is not a usable module name: lower-case letters, digits, dashes and dots", m.Module))
|
"%q is not a usable module name: lower-case letters, digits, dashes and dots", m.Module))
|
||||||
}
|
}
|
||||||
// A slug is a short identifier the mesh derives a login from (novox/hq ADR 0049). The same
|
// A slug is a short identifier the mesh derives a login from (novox/hq ADR 0049). The same
|
||||||
// charset as a name; its length is checked against a backend's limit at assignment, where the
|
// charset as a name; its length is judged against the bound of each provision it wants on the
|
||||||
// node it joins is known — a slug that is fine on one machine's short name can overflow another's.
|
// longest machine name by the catalogue check (IdentityProblems, ADR 0225), and against the
|
||||||
|
// machine it is on when its provider grants it — a slug that is fine on one machine's short name
|
||||||
|
// can overflow another's.
|
||||||
if m.Slug != "" && !name.MatchString(m.Slug) {
|
if m.Slug != "" && !name.MatchString(m.Slug) {
|
||||||
problems = append(problems, fmt.Sprintf(
|
problems = append(problems, fmt.Sprintf(
|
||||||
"%q is not a usable slug: lower-case letters, digits, dashes and dots", m.Slug))
|
"%q is not a usable slug: lower-case letters, digits, dashes and dots", m.Slug))
|
||||||
@@ -1303,6 +1387,20 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
|||||||
if !name.MatchString(p) {
|
if !name.MatchString(p) {
|
||||||
problems = append(problems, fmt.Sprintf("%q is not a usable name to provide", p))
|
problems = append(problems, fmt.Sprintf("%q is not a usable name to provide", p))
|
||||||
}
|
}
|
||||||
|
// A bound says what keeps the name, so the refusal it causes can say it (ADR 0225); and it
|
||||||
|
// leaves room for the shortest identity the mesh makes, or it would refuse every consumer.
|
||||||
|
if id := offer.Identity; id != nil && !id.None {
|
||||||
|
if strings.TrimSpace(id.In) == "" {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s bounds the identities of %s's consumers without saying what keeps them: `in`",
|
||||||
|
m.Module, p))
|
||||||
|
}
|
||||||
|
if shortest := len(IdentityPrefix) + 3; id.Max < 0 || id.Max > 0 && id.Max < shortest {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s bounds %s's consumers' identities at %d characters, and the shortest the mesh "+
|
||||||
|
"makes is %d", m.Module, p, id.Max, shortest))
|
||||||
|
}
|
||||||
|
}
|
||||||
if offer.Credential != nil {
|
if offer.Credential != nil {
|
||||||
own, declared := m.OwnSecrets[offer.Credential.Own]
|
own, declared := m.OwnSecrets[offer.Credential.Own]
|
||||||
switch {
|
switch {
|
||||||
|
|||||||
@@ -194,6 +194,11 @@ type Needed struct {
|
|||||||
// state, not a consumer missing its key. Set by the plan, which is the only layer that knows a
|
// state, not a consumer missing its key. Set by the plan, which is the only layer that knows a
|
||||||
// licence's manager; empty for every consumer.
|
// licence's manager; empty for every consumer.
|
||||||
Manager bool
|
Manager bool
|
||||||
|
// Identity is the longest consumer identity the answering provision keeps (novox/hq ADR 0225),
|
||||||
|
// from the provider's own offer: what the mesh judges this consumer's identity against, on the
|
||||||
|
// consumer's side for `status` and on the provider's before it grants. No bound for a provision
|
||||||
|
// answered by a record, which keeps no name of anybody's.
|
||||||
|
Identity IdentityBound
|
||||||
}
|
}
|
||||||
|
|
||||||
// Refusal is why a set of assignments cannot become a declaration.
|
// Refusal is why a set of assignments cannot become a declaration.
|
||||||
@@ -403,7 +408,7 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
|
|||||||
needs = append(needs, Needed{
|
needs = append(needs, Needed{
|
||||||
Name: want, From: node.Name, At: at,
|
Name: want, From: node.Name, At: at,
|
||||||
Serves: servedByOne(by, want), For: because[want],
|
Serves: servedByOne(by, want), For: because[want],
|
||||||
SharedOwn: sharedByOne(by, want)})
|
SharedOwn: sharedByOne(by, want), Identity: by.IdentityBoundOf(want)})
|
||||||
} else if served := servedByOne(by, want); len(served) > 0 {
|
} else if served := servedByOne(by, want); len(served) > 0 {
|
||||||
// Answered here with no credential to mint, but the provider serves facts the
|
// Answered here with no credential to mint, but the provider serves facts the
|
||||||
// consumer cannot guess — a port, a model name — and so still needs a binding.
|
// consumer cannot guess — a port, a model name — and so still needs a binding.
|
||||||
@@ -447,11 +452,15 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
shared := ""
|
shared := ""
|
||||||
|
// A provider whose definition is not in hand is held to the tightest bound the
|
||||||
|
// mesh knows, not to none: what it keeps is not known here (ADR 0225).
|
||||||
|
bound := DefaultIdentityBound
|
||||||
if pm, known := catalogue[p.Module]; known {
|
if pm, known := catalogue[p.Module]; known {
|
||||||
shared, _ = pm.SharedCredentialOf(want)
|
shared, _ = pm.SharedCredentialOf(want)
|
||||||
|
bound = pm.IdentityBoundOf(want)
|
||||||
}
|
}
|
||||||
needs = append(needs, Needed{Name: want, From: p.Node, At: p.At,
|
needs = append(needs, Needed{Name: want, From: p.Node, At: p.At,
|
||||||
Serves: p.Serves, For: because[want], SharedOwn: shared})
|
Serves: p.Serves, For: because[want], SharedOwn: shared, Identity: bound})
|
||||||
}
|
}
|
||||||
switch {
|
switch {
|
||||||
case world.Unchecked:
|
case world.Unchecked:
|
||||||
|
|||||||
@@ -83,7 +83,9 @@ var defaultSeats = append([]Seat{
|
|||||||
// `assign` and the rest are a role's interface, not a container's, and stay addressable while
|
// `assign` and the rest are a role's interface, not a container's, and stay addressable while
|
||||||
// the control plane is replaced.
|
// the control plane is replaced.
|
||||||
{Name: ControllerSeatName, Scope: ScopeMesh, Decision: "novox/hq ADR 0079",
|
{Name: ControllerSeatName, Scope: ScopeMesh, Decision: "novox/hq ADR 0079",
|
||||||
Emits: []string{"applied", "refused", "built-before"},
|
// And what is wrong, as it changes, and the self-check's heartbeat (novox/hq to-be 45 §2, §4).
|
||||||
|
Emits: []string{"applied", "refused", "built-before",
|
||||||
|
"condition-raised", "condition-changed", "condition-cleared", "doctor-heartbeat"},
|
||||||
Serves: ControllerVerbs},
|
Serves: ControllerVerbs},
|
||||||
// The store's first verbs (novox/hq ADR 0159): the smallest set that makes the store askable,
|
// The store's first verbs (novox/hq ADR 0159): the smallest set that makes the store askable,
|
||||||
// served by whichever module holds the seat with tools of these names.
|
// served by whichever module holds the seat with tools of these names.
|
||||||
|
|||||||
@@ -110,6 +110,8 @@ var ControllerVerbs = []Verb{
|
|||||||
"paths": "with repository: the files the merge would change, comma-separated, from the repository's root",
|
"paths": "with repository: the files the merge would change, comma-separated, from the repository's root",
|
||||||
"modules": "with repository: or the modules it would change, comma-separated",
|
"modules": "with repository: or the modules it would change, comma-separated",
|
||||||
"limit": "how many plans to list (default 10); only when listing",
|
"limit": "how many plans to list (default 10); only when listing",
|
||||||
|
"why": "with stop or close: why it is ended by hand — required, and recorded in the hand-act log (novox/hq to-be 45 §7)",
|
||||||
|
"cause": "with stop or close: the cause in a word, or a condition's kind (optional)",
|
||||||
}, nil)},
|
}, nil)},
|
||||||
{Name: "plan", Description: "What one machine would run, and why: the declaration the mesh would send it — " +
|
{Name: "plan", Description: "What one machine would run, and why: the declaration the mesh would send it — " +
|
||||||
"or, with files, the files it would be given.",
|
"or, with files, the files it would be given.",
|
||||||
@@ -137,12 +139,15 @@ var ControllerVerbs = []Verb{
|
|||||||
{Name: "push", Description: "Send one machine everything it should be. With no machine named it is a push of the " +
|
{Name: "push", Description: "Send one machine everything it should be. With no machine named it is a push of the " +
|
||||||
"WHOLE mesh — every machine that is behind — and the answer says so first; behind says that outright. " +
|
"WHOLE mesh — every machine that is behind — and the answer says so first; behind says that outright. " +
|
||||||
"Answers at once that it is running, with a call id: `calls` with that id says what it sent " +
|
"Answers at once that it is running, with a call id: `calls` with that id says what it sent " +
|
||||||
"(a push can reload the bus, which then refuses any answer still to come).",
|
"(a push can reload the bus, which then refuses any answer still to come). A push by hand is a repair, " +
|
||||||
|
"and says why: recorded in the hand-act log (novox/hq to-be 45 §7).",
|
||||||
Input: schema(map[string]string{
|
Input: schema(map[string]string{
|
||||||
"node": "the machine's name; without it, every machine that is behind",
|
"node": "the machine's name; without it, every machine that is behind",
|
||||||
"behind": "\"true\": every machine that is behind, the whole mesh — the same as naming none, said outright; not with node",
|
"behind": "\"true\": every machine that is behind, the whole mesh — the same as naming none, said outright; not with node",
|
||||||
}, nil, "behind")},
|
"why": "why this is pushed by hand: recorded in the hand-act log",
|
||||||
{Name: "rotate", Description: "Replace a credential. A pair credential, by provision (and a consuming machine, " +
|
"cause": "the cause in a word, or a condition's kind — the word a second push for the same reason uses (optional)",
|
||||||
|
}, []string{"why"}, "behind")},
|
||||||
|
{Name: "rotate", Description: "Replace a credential. A pair credential, by provision (and a consuming machine and module, " +
|
||||||
"else every holder): both ends are re-sent together. Or a module's own secret, by machine, module and " +
|
"else every holder): both ends are re-sent together. Or a module's own secret, by machine, module and " +
|
||||||
"name: made anew and the machine sent, so the module starts again on it — only for a secret its " +
|
"name: made anew and the machine sent, so the module starts again on it — only for a secret its " +
|
||||||
"definition says it reads at start; a value given to the mesh, or one the module applies to a backend, is refused with the reason.",
|
"definition says it reads at start; a value given to the mesh, or one the module applies to a backend, is refused with the reason.",
|
||||||
@@ -150,7 +155,7 @@ var ControllerVerbs = []Verb{
|
|||||||
"provision": "a pair credential: the provision whose credential to replace",
|
"provision": "a pair credential: the provision whose credential to replace",
|
||||||
"consumer": "with provision: only the holder on this machine (optional)",
|
"consumer": "with provision: only the holder on this machine (optional)",
|
||||||
"node": "an own secret: the machine",
|
"node": "an own secret: the machine",
|
||||||
"module": "an own secret: the module",
|
"module": "an own secret: the module; with provision: only this consuming module's credential (optional)",
|
||||||
"secret": "an own secret: its name in the module's definition",
|
"secret": "an own secret: its name in the module's definition",
|
||||||
}, nil)},
|
}, nil)},
|
||||||
{Name: "issue", Description: "Give a module on a machine its account on the bus: minted, and sealed to the " +
|
{Name: "issue", Description: "Give a module on a machine its account on the bus: minted, and sealed to the " +
|
||||||
@@ -206,6 +211,53 @@ var ControllerVerbs = []Verb{
|
|||||||
Input: schema(map[string]string{"node": "one machine; every holder when absent"}, nil)},
|
Input: schema(map[string]string{"node": "one machine; every holder when absent"}, nil)},
|
||||||
{Name: "resume", Description: "The build seat's holder on one machine — or every holder — takes builds again.",
|
{Name: "resume", Description: "The build seat's holder on one machine — or every holder — takes builds again.",
|
||||||
Input: schema(map[string]string{"node": "one machine; every holder when absent"}, nil)},
|
Input: schema(map[string]string{"node": "one machine; every holder when absent"}, nil)},
|
||||||
|
// Acts done by hand, and what the bounds are set from (novox/hq to-be 45 §7, Phase 0).
|
||||||
|
{Name: "hand-act", Description: "Record an act done by hand outside the mesh — a container restarted, a file " +
|
||||||
|
"edited, a service started on a machine — with why and its cause, in the hand-act log beside the pushes and " +
|
||||||
|
"plans ended by hand (novox/hq to-be 45 §7). A cause recorded twice in a fortnight is a healer wanted.",
|
||||||
|
Input: schema(map[string]string{
|
||||||
|
"what": "what was done, in a line",
|
||||||
|
"why": "why it had to be done by hand",
|
||||||
|
"cause": "the cause in a word, or a condition's kind — the word a second act for the same reason uses",
|
||||||
|
"condition": "the key of the condition it addressed, if any (optional)",
|
||||||
|
}, []string{"what", "why", "cause"})},
|
||||||
|
{Name: "hand-acts", Description: "What was done by hand lately — pushes, plans ended, consumers re-made, acts " +
|
||||||
|
"recorded — who, why and the cause of each, and which causes repeat: each repeat is a healer the mesh lacks.",
|
||||||
|
Input: schema(map[string]string{"days": "how many days back (default 14)"}, nil)},
|
||||||
|
{Name: "durations", Description: "How long things take, as the controller measured them: a send to its machine's " +
|
||||||
|
"report (apply), a machine's silence between words (heartbeat-gap), a plan's tier, a build — per machine, " +
|
||||||
|
"repository or module, with median, p90 and max. What the core's bounds are set from (novox/hq to-be 45 Phase 0).",
|
||||||
|
Input: schema(map[string]string{
|
||||||
|
"kind": "one kind: apply, heartbeat-gap, plan-tier or build; every kind when absent",
|
||||||
|
"days": "how many days back (default 14)",
|
||||||
|
}, nil)},
|
||||||
|
// What is wrong, and the self-check (novox/hq to-be 45 §2, §4).
|
||||||
|
{Name: "conditions", Description: "What is wrong with the mesh now: every open condition, urgent first, " +
|
||||||
|
"then oldest — raised by the watchdogs of the signals table, the self-check's probes and the providers' " +
|
||||||
|
"own words, and cleared when observation says it is resolved, never by hand. Given a key, that one " +
|
||||||
|
"whole with its evidence; with history, every transition lately; with silence, stop one's messages " +
|
||||||
|
"for a while — a hand act, which says why (novox/hq to-be 45 §2).",
|
||||||
|
Input: schema(map[string]string{
|
||||||
|
"key": "a condition's key: that one whole; with history, only its transitions",
|
||||||
|
"scope": "only this scope: machine, plan, call, build, merge, provider, seat, bus, core, probe or mesh",
|
||||||
|
"severity": "only urgent, or only warning",
|
||||||
|
"machine": "only those about this machine",
|
||||||
|
"history": "\"true\": every raising, change, silence and clearing lately, oldest first",
|
||||||
|
"days": "with history: how many days back (default 7, at most 90)",
|
||||||
|
"silence": "a condition's key: send no message for it for a while; it stays open and in status",
|
||||||
|
"for": "with silence: how long — 30m, 4h, 2d; at most 7d",
|
||||||
|
"why": "with silence: why — required, and recorded in the hand-act log",
|
||||||
|
"cause": "with silence: the cause in a word (the condition's kind when absent)",
|
||||||
|
}, nil, "history")},
|
||||||
|
{Name: "doctor", Description: "The self-check (novox/hq to-be 45 §4): the last run's verdict at once — " +
|
||||||
|
"each probe of the design's live invariants passed, failed or could not run, and how long ago. With " +
|
||||||
|
"run, a run now; with probes, the registry; with signals, every row of the signals table and the age " +
|
||||||
|
"of its newest signal. Runs every five minutes on its own; each failure is an open condition.",
|
||||||
|
Input: schema(map[string]string{
|
||||||
|
"run": "\"true\": run every probe now and answer the verdict",
|
||||||
|
"probes": "\"true\": the registry — what each probe asserts, and the condition it raises",
|
||||||
|
"signals": "\"true\": the signals table, each row with the age of its newest signal",
|
||||||
|
}, nil, "run", "probes", "signals")},
|
||||||
{Name: "build", Description: "Have the build machine build a repository. Answers at once with the build's id: " +
|
{Name: "build", Description: "Have the build machine build a repository. Answers at once with the build's id: " +
|
||||||
"`builds` with that id follows it line by line, and the module is registered when the outcome comes.",
|
"`builds` with that id follows it line by line, and the module is registered when the outcome comes.",
|
||||||
Input: schema(map[string]string{
|
Input: schema(map[string]string{
|
||||||
|
|||||||
@@ -0,0 +1,159 @@
|
|||||||
|
package conditions
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The condition store on the bus (to-be 45 §2, ADR 0201): the controller's two buckets, asserted at
|
||||||
|
// its start like its calls and its hand-act log.
|
||||||
|
|
||||||
|
// OnTheBus opens the store and its history on a connection.
|
||||||
|
func OnTheBus(ctx context.Context, conn *nats.Conn) (Backend, History, error) {
|
||||||
|
api, err := jetstream.New(conn)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
open, err := api.KeyValue(ctx, broker.ConditionsBucket)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("the condition store %s is not on the bus — the controller asserts it at "+
|
||||||
|
"its start, so one older than this has not: %w", broker.ConditionsBucket, err)
|
||||||
|
}
|
||||||
|
history, err := api.KeyValue(ctx, broker.ConditionHistoryBucket)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("the condition history %s is not on the bus — the controller asserts it "+
|
||||||
|
"at its start, so one older than this has not: %w", broker.ConditionHistoryBucket, err)
|
||||||
|
}
|
||||||
|
return busStore{open}, &busHistory{api: api, kv: history}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type busStore struct{ kv jetstream.KeyValue }
|
||||||
|
|
||||||
|
func (b busStore) Get(ctx context.Context, key string) (Entry, bool, error) {
|
||||||
|
e, err := b.kv.Get(ctx, key)
|
||||||
|
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||||
|
return Entry{}, false, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return Entry{}, false, err
|
||||||
|
}
|
||||||
|
return Entry{Value: e.Value(), Revision: e.Revision()}, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b busStore) Create(ctx context.Context, key string, value []byte) error {
|
||||||
|
_, err := b.kv.Create(ctx, key, value)
|
||||||
|
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||||
|
return ErrMoved
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b busStore) Update(ctx context.Context, key string, value []byte, revision uint64) error {
|
||||||
|
_, err := b.kv.Update(ctx, key, value, revision)
|
||||||
|
return moved(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b busStore) Delete(ctx context.Context, key string, revision uint64) error {
|
||||||
|
return moved(b.kv.Delete(ctx, key, jetstream.LastRevision(revision)))
|
||||||
|
}
|
||||||
|
|
||||||
|
// moved reads the server's refusal of a compare-and-set as what it is.
|
||||||
|
func moved(err error) error {
|
||||||
|
var apiErr *jetstream.APIError
|
||||||
|
if errors.As(err, &apiErr) && apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence {
|
||||||
|
return ErrMoved
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// All is every key, read through a watch that hands over each current value and then says it has.
|
||||||
|
func (b busStore) All(ctx context.Context) (map[string]Entry, error) {
|
||||||
|
w, err := b.kv.WatchAll(ctx, jetstream.IgnoreDeletes())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer func() { _ = w.Stop() }()
|
||||||
|
out := map[string]Entry{}
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, fmt.Errorf("reading the condition store: %w", ctx.Err())
|
||||||
|
case e := <-w.Updates():
|
||||||
|
if e == nil {
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
out[e.Key()] = Entry{Value: e.Value(), Revision: e.Revision()}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// busHistory keeps each transition under a key of its time and a sequence, and reads them back
|
||||||
|
// from a moment through the stream under the bucket — by time, so a read of the last ten minutes
|
||||||
|
// does not read ninety days.
|
||||||
|
type busHistory struct {
|
||||||
|
api jetstream.JetStream
|
||||||
|
kv jetstream.KeyValue
|
||||||
|
seq atomic.Uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *busHistory) Append(ctx context.Context, e Event) error {
|
||||||
|
body, err := json.Marshal(e)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
key := strconv.FormatInt(e.At.UnixNano(), 10) + "-" + strconv.FormatUint(h.seq.Add(1), 10)
|
||||||
|
_, err = h.kv.Put(ctx, key, body)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// historyQuiet is how long a read of the history waits for one more transition before it takes the
|
||||||
|
// stream as read to its end; it answers at once while it holds something.
|
||||||
|
const historyQuiet = 2 * time.Second
|
||||||
|
|
||||||
|
func (h *busHistory) Since(ctx context.Context, since time.Time) ([]Event, error) {
|
||||||
|
start := since
|
||||||
|
consumer, err := h.api.OrderedConsumer(ctx, "KV_"+broker.ConditionHistoryBucket, jetstream.OrderedConsumerConfig{
|
||||||
|
DeliverPolicy: jetstream.DeliverByStartTimePolicy, OptStartTime: &start,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("reading the condition history: %w", err)
|
||||||
|
}
|
||||||
|
info, err := consumer.Info(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("reading the condition history: %w", err)
|
||||||
|
}
|
||||||
|
var out []Event
|
||||||
|
pending := info.NumPending
|
||||||
|
for pending > 0 {
|
||||||
|
msg, err := consumer.Next(jetstream.FetchMaxWait(historyQuiet))
|
||||||
|
if err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
// Nothing more within the quiet wait: read to its end.
|
||||||
|
break
|
||||||
|
}
|
||||||
|
meta, err := msg.Metadata()
|
||||||
|
if err != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
pending = meta.NumPending
|
||||||
|
var e Event
|
||||||
|
if len(msg.Data()) > 0 && json.Unmarshal(msg.Data(), &e) == nil && e.Key != "" {
|
||||||
|
out = append(out, e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.SliceStable(out, func(i, j int) bool { return out[i].At.Before(out[j].At) })
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,117 @@
|
|||||||
|
package conditions
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The condition store against a real server: compare-and-set, an unreadable store refused, and the
|
||||||
|
// history read back by time are claims about what the bus does.
|
||||||
|
|
||||||
|
func busStoreForTest(t *testing.T) (*broker.JetStream, Backend, History) {
|
||||||
|
t.Helper()
|
||||||
|
url := os.Getenv("MESH_TEST_NATS")
|
||||||
|
if url == "" {
|
||||||
|
t.Skip("MESH_TEST_NATS unset")
|
||||||
|
}
|
||||||
|
js, err := broker.Dial(url)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(js.Close)
|
||||||
|
api, err := jetstream.New(js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = api.DeleteKeyValue(t.Context(), broker.ConditionsBucket)
|
||||||
|
_ = api.DeleteKeyValue(t.Context(), broker.ConditionHistoryBucket)
|
||||||
|
if err := js.EnsureControllerBuckets(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
store, history, err := OnTheBus(t.Context(), js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return js, store, history
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A condition outlives the controller that raised it**, and two writers on the bus cannot lose each
|
||||||
|
// other's word: a stale revision is refused as moved.
|
||||||
|
func TestNatsTheStoreKeepsConditionsByCompareAndSet(t *testing.T) {
|
||||||
|
_, store, history := busStoreForTest(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
told := &Told{}
|
||||||
|
k := NewKeeper(ctx, Options{Store: store, History: history, Teller: told})
|
||||||
|
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
k.Close(context.Background())
|
||||||
|
|
||||||
|
again := NewKeeper(ctx, Options{Store: store, History: history})
|
||||||
|
defer again.Close(context.Background())
|
||||||
|
open, err := again.Open(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(open) != 1 || open[0].Observations != 2 {
|
||||||
|
t.Fatalf("a new keeper read %+v", open)
|
||||||
|
}
|
||||||
|
e, _, err := store.Get(ctx, "machine.ace.silent")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := store.Update(ctx, "machine.ace.silent", []byte(`{}`), e.Revision-1); err != ErrMoved {
|
||||||
|
t.Fatalf("a write at a stale revision answered %v", err)
|
||||||
|
}
|
||||||
|
if err := store.Create(ctx, "machine.ace.silent", []byte(`{}`)); err != ErrMoved {
|
||||||
|
t.Fatalf("creating an open condition answered %v", err)
|
||||||
|
}
|
||||||
|
if err := store.Delete(ctx, "machine.ace.silent", e.Revision-1); err != ErrMoved {
|
||||||
|
t.Fatalf("a delete at a stale revision answered %v", err)
|
||||||
|
}
|
||||||
|
if cleared, err := again.Clear(ctx, "machine.ace.silent", "heard"); err != nil || !cleared {
|
||||||
|
t.Fatalf("cleared %v: %v", cleared, err)
|
||||||
|
}
|
||||||
|
// Raised again at once: the store takes a key whose last word was a delete.
|
||||||
|
if _, err := again.Observe(ctx, silent("ace")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, _, _ := again.Get(ctx, "machine.ace.silent")
|
||||||
|
if got.Count != 2 {
|
||||||
|
t.Fatalf("raised again after its clearing as %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The history is read back from a moment, oldest first**, through the stream under its bucket.
|
||||||
|
func TestNatsTheHistoryIsReadByTime(t *testing.T) {
|
||||||
|
_, store, history := busStoreForTest(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
k := NewKeeper(ctx, Options{Store: store, History: history})
|
||||||
|
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := k.Clear(ctx, "machine.ace.silent", "heard"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
k.Close(context.Background())
|
||||||
|
all, err := history.Since(ctx, time.Now().Add(-time.Hour))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(all) != 2 || all[0].Change != ChangeRaised || all[1].Change != ChangeCleared {
|
||||||
|
t.Fatalf("history %+v", all)
|
||||||
|
}
|
||||||
|
none, err := history.Since(ctx, time.Now().Add(time.Hour))
|
||||||
|
if err != nil || len(none) != 0 {
|
||||||
|
t.Fatalf("history from the future: %+v %v", none, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,232 @@
|
|||||||
|
// Package conditions is the condition store (novox/hq to-be 45 §2, ADR 0227 rules 5 and 6).
|
||||||
|
//
|
||||||
|
// **A condition is a durable fact about something the mesh owns that is wrong.** Until this, every
|
||||||
|
// one of the forty-eight core failures of research 031 was noticed because a person or an agent
|
||||||
|
// looked: the mesh's own answers carried the fact for whoever asked, and told nobody. A condition is
|
||||||
|
// raised when an observation says something is wrong past its bound, kept with since-when, evidence
|
||||||
|
// and who can resolve it, said on the bus as it changes, and cleared when an observation says it is
|
||||||
|
// resolved — never by hand.
|
||||||
|
//
|
||||||
|
// The controller is the store's only writer (to-be 45 §1). Two of its processes may write at once —
|
||||||
|
// the serving controller's watchdogs, and a command a person runs to silence one — so every write
|
||||||
|
// is a compare-and-set on the key's revision, and a write that lost the race reads again and redoes
|
||||||
|
// itself rather than overwriting what the other said.
|
||||||
|
package conditions
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Severity is how soon the operator is needed: two levels, no more (to-be 45 §2).
|
||||||
|
type Severity string
|
||||||
|
|
||||||
|
const (
|
||||||
|
// Urgent needs the operator now.
|
||||||
|
Urgent Severity = "urgent"
|
||||||
|
// Warning needs the operator when they can.
|
||||||
|
Warning Severity = "warning"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The scopes a condition's key starts with: what kind of thing is wrong.
|
||||||
|
const (
|
||||||
|
ScopeMachine = "machine"
|
||||||
|
ScopePlan = "plan"
|
||||||
|
ScopeCall = "call"
|
||||||
|
ScopeBuild = "build"
|
||||||
|
ScopeMerge = "merge"
|
||||||
|
ScopeProvider = "provider"
|
||||||
|
ScopeSeat = "seat"
|
||||||
|
ScopeBus = "bus"
|
||||||
|
ScopeCore = "core"
|
||||||
|
ScopeProbe = "probe"
|
||||||
|
ScopeMesh = "mesh"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Scopes is every scope, in the order a person reads them.
|
||||||
|
var Scopes = []string{ScopeMachine, ScopePlan, ScopeCall, ScopeBuild, ScopeMerge, ScopeProvider,
|
||||||
|
ScopeSeat, ScopeBus, ScopeCore, ScopeProbe, ScopeMesh}
|
||||||
|
|
||||||
|
// Who resolves a condition.
|
||||||
|
const (
|
||||||
|
// ResolverSelf clears on observation: the signal returns, the probe passes.
|
||||||
|
ResolverSelf = "self"
|
||||||
|
// ResolverOperator needs a person: a healer's budget spent, or a repair that could only destroy.
|
||||||
|
ResolverOperator = "operator"
|
||||||
|
// ResolverAgent is work handed to an agent (research 017; not raised by anything yet).
|
||||||
|
ResolverAgent = "agent"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ResolverHealer is the resolver of a condition a registered healer works on (Phase 3).
|
||||||
|
func ResolverHealer(name string) string { return "healer:" + name }
|
||||||
|
|
||||||
|
// Subject is what the condition is about: its scope, its id within the scope, and the machine it
|
||||||
|
// concerns when there is one.
|
||||||
|
type Subject struct {
|
||||||
|
Scope string `json:"scope"`
|
||||||
|
ID string `json:"id"`
|
||||||
|
Machine string `json:"machine,omitempty"`
|
||||||
|
// Also are the other machines it concerns: a consumer's, for a provider failing it.
|
||||||
|
Also []string `json:"also,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Evidence is one observation, as it was said.
|
||||||
|
type Evidence struct {
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
Said string `json:"said"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Attempt is one healer's try at a condition (to-be 45 §7; written from Phase 3).
|
||||||
|
type Attempt struct {
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
What string `json:"what"`
|
||||||
|
Outcome string `json:"outcome"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Silence is a person saying they know: no messages until it ends (to-be 45 §2). Recorded as a hand
|
||||||
|
// act; the condition stays open, and `status` still says it.
|
||||||
|
type Silence struct {
|
||||||
|
Until time.Time `json:"until"`
|
||||||
|
By string `json:"by"`
|
||||||
|
Why string `json:"why"`
|
||||||
|
Since time.Time `json:"since"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// KeptEvidence is how many observations a condition keeps, newest first.
|
||||||
|
const KeptEvidence = 10
|
||||||
|
|
||||||
|
// MaxSilence is the longest a condition may be silenced at once: past it, a person says so again.
|
||||||
|
const MaxSilence = 7 * 24 * time.Hour
|
||||||
|
|
||||||
|
// ReopenWithin is how soon after it cleared a condition raised again is the same one again, with its
|
||||||
|
// count increased, rather than news (to-be 45 §2).
|
||||||
|
const ReopenWithin = 10 * time.Minute
|
||||||
|
|
||||||
|
// Condition is one open condition, as the store keeps it and its events carry it.
|
||||||
|
type Condition struct {
|
||||||
|
Key string `json:"key"`
|
||||||
|
// Kind is the condition kind: from the signals table, the probe registry or an event kind.
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Subject Subject `json:"subject"`
|
||||||
|
Severity Severity `json:"severity"`
|
||||||
|
// Summary is one line in the mesh's words.
|
||||||
|
Summary string `json:"summary"`
|
||||||
|
// Evidence is the newest observations, at most KeptEvidence, newest first.
|
||||||
|
Evidence []Evidence `json:"evidence"`
|
||||||
|
// Source is the signals-table row, probe or event that raised it: `S1`, `D3`, `provisioner.failing`.
|
||||||
|
Source string `json:"source"`
|
||||||
|
// Raised is when it was first observed this time; LastObserved the newest observation.
|
||||||
|
Raised time.Time `json:"raised"`
|
||||||
|
LastObserved time.Time `json:"last-observed"`
|
||||||
|
// Observations is how many times it was observed since raised.
|
||||||
|
Observations int `json:"observations"`
|
||||||
|
// Count is how many times it has been raised, a reopening within ReopenWithin counted.
|
||||||
|
Count int `json:"count"`
|
||||||
|
Tried []Attempt `json:"tried,omitempty"`
|
||||||
|
Resolver string `json:"resolver"`
|
||||||
|
// Silenced is null when no silence is in force: said, not left out, so a reader need not guess.
|
||||||
|
Silenced *Silence `json:"silenced"`
|
||||||
|
// Epoch is the controller lease epoch that last wrote it. Zero until the lease exists (to-be 45
|
||||||
|
// Phase 2): no controller holds an epoch yet, and a number invented here would be one nobody
|
||||||
|
// could compare.
|
||||||
|
Epoch uint64 `json:"epoch"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// SilencedAt says whether a person's silence is in force at a moment.
|
||||||
|
func (c Condition) SilencedAt(now time.Time) bool {
|
||||||
|
return c.Silenced != nil && now.Before(c.Silenced.Until)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Show is the verb that shows more about a condition, as a message carries it.
|
||||||
|
func (c Condition) Show() string { return "mesh-controller.conditions key=" + c.Key }
|
||||||
|
|
||||||
|
// Observation is one watchdog, probe or event saying something is wrong now.
|
||||||
|
type Observation struct {
|
||||||
|
Scope string
|
||||||
|
// ID is the thing within the scope; several tokens joined by dots where the thing is named by
|
||||||
|
// several (a provider's module, its machine and the consumer).
|
||||||
|
ID string
|
||||||
|
// Token is the last part of the key, short for the kind: `silent` for a machine, `failing` for a
|
||||||
|
// provider. Kind's own word when empty.
|
||||||
|
Token string
|
||||||
|
Kind string
|
||||||
|
Machine string
|
||||||
|
// Also are the other machines it concerns.
|
||||||
|
Also []string
|
||||||
|
Severity Severity
|
||||||
|
Summary string
|
||||||
|
// Said is this observation's evidence, in the mesh's words; Summary when empty.
|
||||||
|
Said string
|
||||||
|
Source string
|
||||||
|
Resolver string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Key is where the observation's condition is kept: `<scope>.<id>.<kind>`, so the same fault said
|
||||||
|
// again is the same condition.
|
||||||
|
func (o Observation) Key() string {
|
||||||
|
token := o.Token
|
||||||
|
if token == "" {
|
||||||
|
token = o.Kind
|
||||||
|
}
|
||||||
|
return Key(o.Scope, o.ID, token)
|
||||||
|
}
|
||||||
|
|
||||||
|
// unsafeKey is anything a key may not hold: the bus takes letters, digits and `-_/=` in a key's
|
||||||
|
// tokens, and a `*` or `>` would make one a wildcard.
|
||||||
|
var unsafeKey = regexp.MustCompile(`[^A-Za-z0-9_=/-]`)
|
||||||
|
|
||||||
|
// Key composes a condition's key from its parts, each token made safe for the bus: a character the
|
||||||
|
// bus would refuse becomes `_`, so a key is never refused for the name of the thing it is about.
|
||||||
|
func Key(scope, id, token string) string {
|
||||||
|
var parts []string
|
||||||
|
for _, p := range append(append([]string{scope}, strings.Split(id, ".")...), token) {
|
||||||
|
p = unsafeKey.ReplaceAllString(strings.TrimSpace(p), "_")
|
||||||
|
if p == "" {
|
||||||
|
p = "_"
|
||||||
|
}
|
||||||
|
parts = append(parts, p)
|
||||||
|
}
|
||||||
|
return strings.Join(parts, ".")
|
||||||
|
}
|
||||||
|
|
||||||
|
// check refuses an observation that could not be said: a condition with no kind, no scope the mesh
|
||||||
|
// knows, or no severity is one nobody could route.
|
||||||
|
func (o Observation) check() error {
|
||||||
|
known := false
|
||||||
|
for _, s := range Scopes {
|
||||||
|
if s == o.Scope {
|
||||||
|
known = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case !known:
|
||||||
|
return fmt.Errorf("a condition's scope is one of %s, not %q", strings.Join(Scopes, ", "), o.Scope)
|
||||||
|
case strings.TrimSpace(o.ID) == "":
|
||||||
|
return fmt.Errorf("a %s condition names what it is about", o.Scope)
|
||||||
|
case strings.TrimSpace(o.Kind) == "":
|
||||||
|
return fmt.Errorf("the condition %s has no kind", o.Key())
|
||||||
|
case o.Severity != Urgent && o.Severity != Warning:
|
||||||
|
return fmt.Errorf("the condition %s is urgent or a warning, not %q", o.Key(), o.Severity)
|
||||||
|
case strings.TrimSpace(o.Summary) == "":
|
||||||
|
return fmt.Errorf("the condition %s says nothing", o.Key())
|
||||||
|
case strings.TrimSpace(o.Source) == "":
|
||||||
|
return fmt.Errorf("the condition %s does not say what raised it", o.Key())
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Order sorts conditions as `status` says them: urgent before warning, then oldest first.
|
||||||
|
func Order(list []Condition) {
|
||||||
|
sort.SliceStable(list, func(i, j int) bool {
|
||||||
|
if list[i].Severity != list[j].Severity {
|
||||||
|
return list[i].Severity == Urgent
|
||||||
|
}
|
||||||
|
if !list[i].Raised.Equal(list[j].Raised) {
|
||||||
|
return list[i].Raised.Before(list[j].Raised)
|
||||||
|
}
|
||||||
|
return list[i].Key < list[j].Key
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
package conditions
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
|
// The events a condition's life emits (to-be 45 §2), as the mesh-controller seat's own: published on
|
||||||
|
// `mesh.seat.mesh-controller.event.<name>`, on the events stream, so a consumer that was away catches
|
||||||
|
// up. **This is a contract**: the operator-channel's holder is written against these names and the
|
||||||
|
// shape of Event, and the controller learns nothing about telling.
|
||||||
|
const (
|
||||||
|
// EventRaised: a condition was raised — new, or the same fault again within ReopenWithin of its
|
||||||
|
// clearing (Change says which). A reopened condition is not news: its key is the one the
|
||||||
|
// first message was about.
|
||||||
|
EventRaised = "condition-raised"
|
||||||
|
// EventChanged: its severity, its resolver or its silence changed. Not every observation: a
|
||||||
|
// condition observed again is written, and says nothing.
|
||||||
|
EventChanged = "condition-changed"
|
||||||
|
// EventCleared: an observation says it is resolved. The condition is removed from the store and
|
||||||
|
// the transition kept in its history.
|
||||||
|
EventCleared = "condition-cleared"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Events is every event a condition's life emits.
|
||||||
|
var Events = []string{EventRaised, EventChanged, EventCleared}
|
||||||
|
|
||||||
|
// HeartbeatEvent is the self-check's heartbeat (to-be 45 §4, S10), said under the same seat at the end
|
||||||
|
// of every run: `{run, at, interval-seconds, counts: {passed, failed, failed-to-run, deferred}, probes,
|
||||||
|
// controller, why}`. mesh-watcher, on a machine that is not the control node, listens for it.
|
||||||
|
const HeartbeatEvent = "doctor-heartbeat"
|
||||||
|
|
||||||
|
// What changed, as an event's Change and a history entry's says it.
|
||||||
|
const (
|
||||||
|
ChangeRaised = "raised"
|
||||||
|
ChangeReopened = "reopened"
|
||||||
|
ChangeSeverity = "severity"
|
||||||
|
ChangeResolver = "resolver"
|
||||||
|
ChangeSilenced = "silenced"
|
||||||
|
ChangeUnsilenced = "silence-ended"
|
||||||
|
ChangeCleared = "cleared"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Event is the body of every condition event, and the shape a history entry keeps: **the condition
|
||||||
|
// itself, at the top level** — key, kind, subject, severity, summary, source, raised, last-observed,
|
||||||
|
// observations, resolver, silenced (null when not), epoch, and the evidence — with what happened to
|
||||||
|
// it beside. One object a consumer reads the same way whichever of the three it is.
|
||||||
|
type Event struct {
|
||||||
|
// Condition is the condition after the transition — as it was last held, for a clearing.
|
||||||
|
Condition
|
||||||
|
// Event is the event's own name, so a body read without its subject still says what it is.
|
||||||
|
Event string `json:"event"`
|
||||||
|
// At is when the transition happened.
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
// Change is what happened: raised, reopened, severity, resolver, silenced, silence-ended, cleared.
|
||||||
|
Change string `json:"change"`
|
||||||
|
// Was is the value before, for a severity or resolver change.
|
||||||
|
Was string `json:"was,omitempty"`
|
||||||
|
// Why says why it cleared, or why it was silenced.
|
||||||
|
Why string `json:"why,omitempty"`
|
||||||
|
// Cleared is when it cleared, on a clearing.
|
||||||
|
Cleared *time.Time `json:"cleared,omitempty"`
|
||||||
|
// Show is the verb that shows more.
|
||||||
|
Show string `json:"show"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// eventFor is the event a change is said under.
|
||||||
|
func eventFor(change string) string {
|
||||||
|
switch change {
|
||||||
|
case ChangeRaised, ChangeReopened:
|
||||||
|
return EventRaised
|
||||||
|
case ChangeCleared:
|
||||||
|
return EventCleared
|
||||||
|
}
|
||||||
|
return EventChanged
|
||||||
|
}
|
||||||
@@ -0,0 +1,147 @@
|
|||||||
|
package conditions
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"sort"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// InMemory is a store and a history held in this process: for tests, and for nothing else — a
|
||||||
|
// condition kept here is forgotten by a restart, which is the fault the store exists to remove.
|
||||||
|
type InMemory struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
values map[string]Entry
|
||||||
|
revision uint64
|
||||||
|
events []Event
|
||||||
|
// Fail, when set, is what every read and write answers: a store that is away.
|
||||||
|
Fail error
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewInMemory is an empty store.
|
||||||
|
func NewInMemory() *InMemory { return &InMemory{values: map[string]Entry{}} }
|
||||||
|
|
||||||
|
func (m *InMemory) Get(_ context.Context, key string) (Entry, bool, error) {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
if m.Fail != nil {
|
||||||
|
return Entry{}, false, m.Fail
|
||||||
|
}
|
||||||
|
e, ok := m.values[key]
|
||||||
|
return e, ok, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *InMemory) Create(_ context.Context, key string, value []byte) error {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
if m.Fail != nil {
|
||||||
|
return m.Fail
|
||||||
|
}
|
||||||
|
if _, ok := m.values[key]; ok {
|
||||||
|
return ErrMoved
|
||||||
|
}
|
||||||
|
m.revision++
|
||||||
|
m.values[key] = Entry{Value: value, Revision: m.revision}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *InMemory) Update(_ context.Context, key string, value []byte, revision uint64) error {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
if m.Fail != nil {
|
||||||
|
return m.Fail
|
||||||
|
}
|
||||||
|
if e, ok := m.values[key]; !ok || e.Revision != revision {
|
||||||
|
return ErrMoved
|
||||||
|
}
|
||||||
|
m.revision++
|
||||||
|
m.values[key] = Entry{Value: value, Revision: m.revision}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *InMemory) Delete(_ context.Context, key string, revision uint64) error {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
if m.Fail != nil {
|
||||||
|
return m.Fail
|
||||||
|
}
|
||||||
|
if e, ok := m.values[key]; !ok || e.Revision != revision {
|
||||||
|
return ErrMoved
|
||||||
|
}
|
||||||
|
delete(m.values, key)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *InMemory) All(context.Context) (map[string]Entry, error) {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
if m.Fail != nil {
|
||||||
|
return nil, m.Fail
|
||||||
|
}
|
||||||
|
out := make(map[string]Entry, len(m.values))
|
||||||
|
for k, v := range m.values {
|
||||||
|
out[k] = v
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *InMemory) Append(_ context.Context, e Event) error {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
if m.Fail != nil {
|
||||||
|
return m.Fail
|
||||||
|
}
|
||||||
|
m.events = append(m.events, e)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *InMemory) Since(_ context.Context, since time.Time) ([]Event, error) {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
if m.Fail != nil {
|
||||||
|
return nil, m.Fail
|
||||||
|
}
|
||||||
|
var out []Event
|
||||||
|
for _, e := range m.events {
|
||||||
|
if !e.At.Before(since) {
|
||||||
|
out = append(out, e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.SliceStable(out, func(i, j int) bool { return out[i].At.Before(out[j].At) })
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Told is a teller that remembers what it was told, for tests.
|
||||||
|
type Told struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
Events []Event
|
||||||
|
Names []string
|
||||||
|
Fail error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *Told) PublishSeatEvent(_ context.Context, seat, event string, body []byte) error {
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
if t.Fail != nil {
|
||||||
|
return t.Fail
|
||||||
|
}
|
||||||
|
if seat != Seat {
|
||||||
|
return errors.New("told under the wrong seat: " + seat)
|
||||||
|
}
|
||||||
|
var e Event
|
||||||
|
if err := json.Unmarshal(body, &e); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
t.Events = append(t.Events, e)
|
||||||
|
t.Names = append(t.Names, event)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Said is a copy of what was told so far.
|
||||||
|
func (t *Told) Said() []Event {
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
return append([]Event(nil), t.Events...)
|
||||||
|
}
|
||||||
@@ -0,0 +1,502 @@
|
|||||||
|
package conditions
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Backend is where the open conditions are kept: one value per key, written by compare-and-set.
|
||||||
|
type Backend interface {
|
||||||
|
// Get is one key's value and revision; false when it holds none.
|
||||||
|
Get(ctx context.Context, key string) (Entry, bool, error)
|
||||||
|
// Create writes a key that holds nothing, and fails with ErrMoved when it holds something.
|
||||||
|
Create(ctx context.Context, key string, value []byte) error
|
||||||
|
// Update writes a key at the revision it was read at, and fails with ErrMoved when it moved.
|
||||||
|
Update(ctx context.Context, key string, value []byte, revision uint64) error
|
||||||
|
// Delete removes a key at the revision it was read at, and fails with ErrMoved when it moved.
|
||||||
|
Delete(ctx context.Context, key string, revision uint64) error
|
||||||
|
// All is every key's value. An error is an error: never an empty store (ADR 0227 rule 4).
|
||||||
|
All(ctx context.Context) (map[string]Entry, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Entry is one key's value, at a revision.
|
||||||
|
type Entry struct {
|
||||||
|
Value []byte
|
||||||
|
Revision uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
// ErrMoved is a compare-and-set that lost: somebody wrote the key since it was read.
|
||||||
|
var ErrMoved = errors.New("the condition was written by somebody else since it was read")
|
||||||
|
|
||||||
|
// History keeps every transition (to-be 45 §2): appended, read back from a moment.
|
||||||
|
type History interface {
|
||||||
|
Append(ctx context.Context, e Event) error
|
||||||
|
// Since is every transition from a moment, oldest first.
|
||||||
|
Since(ctx context.Context, since time.Time) ([]Event, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Teller says a transition on the bus, as the mesh-controller seat's event. The link's bus is one.
|
||||||
|
type Teller interface {
|
||||||
|
PublishSeatEvent(ctx context.Context, seat, event string, body []byte) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Seat is the role the events are said under (novox/hq ADR 0134): the control plane's.
|
||||||
|
const Seat = "mesh-controller"
|
||||||
|
|
||||||
|
// Keeper raises, observes, silences and clears conditions, and says each transition.
|
||||||
|
type Keeper struct {
|
||||||
|
store Backend
|
||||||
|
history History
|
||||||
|
teller Teller
|
||||||
|
now func() time.Time
|
||||||
|
say func(format string, args ...any)
|
||||||
|
changed func()
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
// cleared is when each recently cleared condition cleared and how often it had been raised, so
|
||||||
|
// one raised again within ReopenWithin is the same one again.
|
||||||
|
cleared map[string]clearing
|
||||||
|
|
||||||
|
// out is the transitions still to be said and kept, in order: said by one goroutine, so a
|
||||||
|
// condition's events arrive in the order they happened, and offered again while the bus is away.
|
||||||
|
out chan Event
|
||||||
|
drained chan struct{}
|
||||||
|
closing sync.Once
|
||||||
|
// Unsaid counts the transitions given up on, for the self-check to say.
|
||||||
|
unsaid int
|
||||||
|
}
|
||||||
|
|
||||||
|
type clearing struct {
|
||||||
|
at time.Time
|
||||||
|
count int
|
||||||
|
silenced *Silence
|
||||||
|
}
|
||||||
|
|
||||||
|
// Options are what a Keeper is made with.
|
||||||
|
type Options struct {
|
||||||
|
Store Backend
|
||||||
|
History History
|
||||||
|
// Teller says the transitions; nil says nothing (a test, or a command run with no bus to say on).
|
||||||
|
Teller Teller
|
||||||
|
Now func() time.Time
|
||||||
|
// Say is where a transition that could not be said or kept is said instead.
|
||||||
|
Say func(format string, args ...any)
|
||||||
|
// Changed is told of every transition, at once — for `status`, which leads with what is open.
|
||||||
|
Changed func()
|
||||||
|
}
|
||||||
|
|
||||||
|
// TellFor is how long one transition is offered to the bus before it is said lost.
|
||||||
|
var TellFor = 10 * time.Minute
|
||||||
|
|
||||||
|
// NewKeeper is a keeper over a store. It reads what cleared lately from the history, so a condition
|
||||||
|
// that cleared just before this controller started and is raised again now is a reopening.
|
||||||
|
func NewKeeper(ctx context.Context, o Options) *Keeper {
|
||||||
|
k := &Keeper{store: o.Store, history: o.History, teller: o.Teller, now: o.Now, say: o.Say, changed: o.Changed,
|
||||||
|
cleared: map[string]clearing{}, out: make(chan Event, 1024), drained: make(chan struct{})}
|
||||||
|
if k.now == nil {
|
||||||
|
k.now = time.Now
|
||||||
|
}
|
||||||
|
if k.say == nil {
|
||||||
|
k.say = func(string, ...any) {}
|
||||||
|
}
|
||||||
|
if k.history != nil {
|
||||||
|
if recent, err := k.history.Since(ctx, k.now().Add(-ReopenWithin)); err == nil {
|
||||||
|
for _, e := range recent {
|
||||||
|
if e.Change == ChangeCleared {
|
||||||
|
k.cleared[e.Key] = clearing{at: e.At, count: e.Condition.Count, silenced: e.Condition.Silenced}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
k.say("what cleared lately could not be read from the condition history, so a condition "+
|
||||||
|
"raised again now is said as new rather than reopened: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
go k.telling()
|
||||||
|
return k
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close says what is still to be said, waiting at most until ctx ends.
|
||||||
|
func (k *Keeper) Close(ctx context.Context) {
|
||||||
|
k.closing.Do(func() { close(k.out) })
|
||||||
|
select {
|
||||||
|
case <-k.drained:
|
||||||
|
case <-ctx.Done():
|
||||||
|
k.say("%d condition transition(s) were not yet said when this process ended", len(k.out))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unsaid is how many transitions were given up on since this keeper started.
|
||||||
|
func (k *Keeper) Unsaid() int {
|
||||||
|
k.mu.Lock()
|
||||||
|
defer k.mu.Unlock()
|
||||||
|
return k.unsaid
|
||||||
|
}
|
||||||
|
|
||||||
|
// tries bounds one compare-and-set: two writers rarely race more than once.
|
||||||
|
const tries = 8
|
||||||
|
|
||||||
|
// Observe records one observation: raises the condition if it is not open, and otherwise adds the
|
||||||
|
// evidence. Says a raising, a reopening, and a change of severity or resolver; an observation that
|
||||||
|
// changes neither is written and said nowhere.
|
||||||
|
func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error) {
|
||||||
|
if err := o.check(); err != nil {
|
||||||
|
return Condition{}, err
|
||||||
|
}
|
||||||
|
key := o.Key()
|
||||||
|
for i := 0; i < tries; i++ {
|
||||||
|
now := k.now().UTC()
|
||||||
|
said := o.Said
|
||||||
|
if said == "" {
|
||||||
|
said = o.Summary
|
||||||
|
}
|
||||||
|
entry, found, err := k.store.Get(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
return Condition{}, fmt.Errorf("reading the condition %s: %w", key, err)
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
c := Condition{Key: key, Kind: o.Kind, Subject: Subject{Scope: o.Scope, ID: o.ID, Machine: o.Machine, Also: o.Also},
|
||||||
|
Severity: o.Severity, Summary: o.Summary, Evidence: []Evidence{{At: now, Said: said}},
|
||||||
|
Source: o.Source, Raised: now, LastObserved: now, Observations: 1, Count: 1,
|
||||||
|
Resolver: orSelf(o.Resolver)}
|
||||||
|
change := ChangeRaised
|
||||||
|
k.mu.Lock()
|
||||||
|
if before, ok := k.cleared[key]; ok && now.Sub(before.at) <= ReopenWithin {
|
||||||
|
c.Count, change = before.count+1, ChangeReopened
|
||||||
|
// A silence a person gave the condition before it cleared still holds: they said
|
||||||
|
// they knew, and the same fault again ten minutes later is what they knew about.
|
||||||
|
if before.silenced != nil && now.Before(before.silenced.Until) {
|
||||||
|
c.Silenced = before.silenced
|
||||||
|
}
|
||||||
|
}
|
||||||
|
k.mu.Unlock()
|
||||||
|
body, err := json.Marshal(c)
|
||||||
|
if err != nil {
|
||||||
|
return Condition{}, err
|
||||||
|
}
|
||||||
|
if err := k.store.Create(ctx, key, body); errors.Is(err, ErrMoved) {
|
||||||
|
continue
|
||||||
|
} else if err != nil {
|
||||||
|
return Condition{}, fmt.Errorf("raising the condition %s: %w", key, err)
|
||||||
|
}
|
||||||
|
k.mu.Lock()
|
||||||
|
delete(k.cleared, key)
|
||||||
|
k.mu.Unlock()
|
||||||
|
k.tell(Event{Condition: c, At: now, Change: change})
|
||||||
|
return c, nil
|
||||||
|
}
|
||||||
|
var c Condition
|
||||||
|
if err := json.Unmarshal(entry.Value, &c); err != nil {
|
||||||
|
return Condition{}, fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err)
|
||||||
|
}
|
||||||
|
var changes []Event
|
||||||
|
if o.Severity != c.Severity {
|
||||||
|
changes = append(changes, Event{Change: ChangeSeverity, Was: string(c.Severity)})
|
||||||
|
c.Severity = o.Severity
|
||||||
|
}
|
||||||
|
if r := orSelf(o.Resolver); o.Resolver != "" && r != c.Resolver {
|
||||||
|
changes = append(changes, Event{Change: ChangeResolver, Was: c.Resolver})
|
||||||
|
c.Resolver = r
|
||||||
|
}
|
||||||
|
c.Summary, c.Source, c.LastObserved = o.Summary, o.Source, now
|
||||||
|
if o.Machine != "" {
|
||||||
|
c.Subject.Machine = o.Machine
|
||||||
|
}
|
||||||
|
if len(o.Also) > 0 {
|
||||||
|
c.Subject.Also = o.Also
|
||||||
|
}
|
||||||
|
c.Observations++
|
||||||
|
c.Evidence = append([]Evidence{{At: now, Said: said}}, c.Evidence...)
|
||||||
|
if len(c.Evidence) > KeptEvidence {
|
||||||
|
c.Evidence = c.Evidence[:KeptEvidence]
|
||||||
|
}
|
||||||
|
body, err := json.Marshal(c)
|
||||||
|
if err != nil {
|
||||||
|
return Condition{}, err
|
||||||
|
}
|
||||||
|
if err := k.store.Update(ctx, key, body, entry.Revision); errors.Is(err, ErrMoved) {
|
||||||
|
continue
|
||||||
|
} else if err != nil {
|
||||||
|
return Condition{}, fmt.Errorf("observing the condition %s: %w", key, err)
|
||||||
|
}
|
||||||
|
for _, e := range changes {
|
||||||
|
e.At, e.Condition = now, c
|
||||||
|
k.tell(e)
|
||||||
|
}
|
||||||
|
return c, nil
|
||||||
|
}
|
||||||
|
return Condition{}, fmt.Errorf("the condition %s kept moving under this write; %d tries", key, tries)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clear removes a condition an observation says is resolved, and says so. False when none was open.
|
||||||
|
func (k *Keeper) Clear(ctx context.Context, key, why string) (bool, error) {
|
||||||
|
for i := 0; i < tries; i++ {
|
||||||
|
entry, found, err := k.store.Get(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("reading the condition %s: %w", key, err)
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
var c Condition
|
||||||
|
if err := json.Unmarshal(entry.Value, &c); err != nil {
|
||||||
|
// Unreadable is not resolved: kept, and said, rather than removed unread.
|
||||||
|
return false, fmt.Errorf("the condition %s on the bus cannot be read, so it is not cleared: %w", key, err)
|
||||||
|
}
|
||||||
|
if err := k.store.Delete(ctx, key, entry.Revision); errors.Is(err, ErrMoved) {
|
||||||
|
continue
|
||||||
|
} else if err != nil {
|
||||||
|
return false, fmt.Errorf("clearing the condition %s: %w", key, err)
|
||||||
|
}
|
||||||
|
now := k.now().UTC()
|
||||||
|
k.mu.Lock()
|
||||||
|
k.cleared[key] = clearing{at: now, count: c.Count, silenced: c.Silenced}
|
||||||
|
k.mu.Unlock()
|
||||||
|
k.tell(Event{Condition: c, At: now, Change: ChangeCleared, Why: why, Cleared: &now})
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
return false, fmt.Errorf("the condition %s kept moving under this clearing; %d tries", key, tries)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reconcile is one source's whole observation: every condition it observes is observed, and every
|
||||||
|
// condition it raised before and no longer observes is cleared — the observation says it is
|
||||||
|
// resolved. A source that could not observe must not call this: an empty observation clears all it
|
||||||
|
// raised, which is exactly the fault of saying "none" for "I could not tell" (ADR 0227 rule 4).
|
||||||
|
func (k *Keeper) Reconcile(ctx context.Context, source string, observed []Observation) error {
|
||||||
|
all, err := k.Open(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
seen := map[string]bool{}
|
||||||
|
var problems []string
|
||||||
|
for _, o := range observed {
|
||||||
|
o.Source = source
|
||||||
|
seen[o.Key()] = true
|
||||||
|
if _, err := k.Observe(ctx, o); err != nil {
|
||||||
|
problems = append(problems, err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, c := range all {
|
||||||
|
if c.Source != source || seen[c.Key] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := k.Clear(ctx, c.Key, source+" no longer observes it"); err != nil {
|
||||||
|
problems = append(problems, err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(problems) > 0 {
|
||||||
|
return errors.New(strings.Join(problems, "; "))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Silence stops a condition's messages for a while, with a reason, by somebody (to-be 45 §2). The
|
||||||
|
// condition stays open and `status` still says it; recording the act in the hand-act log is the
|
||||||
|
// caller's, which knows who acted.
|
||||||
|
func (k *Keeper) Silence(ctx context.Context, key string, d time.Duration, by, why string) (Condition, error) {
|
||||||
|
if strings.TrimSpace(why) == "" {
|
||||||
|
return Condition{}, errors.New("a silence says why: --why <text>")
|
||||||
|
}
|
||||||
|
if d <= 0 || d > MaxSilence {
|
||||||
|
return Condition{}, fmt.Errorf("a condition is silenced for a while, at most %s — not %s", MaxSilence, d)
|
||||||
|
}
|
||||||
|
for i := 0; i < tries; i++ {
|
||||||
|
entry, found, err := k.store.Get(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
return Condition{}, fmt.Errorf("reading the condition %s: %w", key, err)
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
return Condition{}, fmt.Errorf("no condition %s is open — `conditions` lists them", key)
|
||||||
|
}
|
||||||
|
var c Condition
|
||||||
|
if err := json.Unmarshal(entry.Value, &c); err != nil {
|
||||||
|
return Condition{}, fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err)
|
||||||
|
}
|
||||||
|
now := k.now().UTC()
|
||||||
|
c.Silenced = &Silence{Until: now.Add(d), By: by, Why: strings.TrimSpace(why), Since: now}
|
||||||
|
body, err := json.Marshal(c)
|
||||||
|
if err != nil {
|
||||||
|
return Condition{}, err
|
||||||
|
}
|
||||||
|
if err := k.store.Update(ctx, key, body, entry.Revision); errors.Is(err, ErrMoved) {
|
||||||
|
continue
|
||||||
|
} else if err != nil {
|
||||||
|
return Condition{}, fmt.Errorf("silencing the condition %s: %w", key, err)
|
||||||
|
}
|
||||||
|
k.tell(Event{Condition: c, At: now, Change: ChangeSilenced, Why: c.Silenced.Why})
|
||||||
|
return c, nil
|
||||||
|
}
|
||||||
|
return Condition{}, fmt.Errorf("the condition %s kept moving under this silence; %d tries", key, tries)
|
||||||
|
}
|
||||||
|
|
||||||
|
// EndSilences ends every silence that has run out, and says each: the condition is still open, and
|
||||||
|
// its messages start again.
|
||||||
|
func (k *Keeper) EndSilences(ctx context.Context) error {
|
||||||
|
all, err := k.Open(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
now := k.now().UTC()
|
||||||
|
for _, c := range all {
|
||||||
|
if c.Silenced == nil || now.Before(c.Silenced.Until) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for i := 0; i < tries; i++ {
|
||||||
|
entry, found, err := k.store.Get(ctx, c.Key)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
var held Condition
|
||||||
|
if err := json.Unmarshal(entry.Value, &held); err != nil {
|
||||||
|
return fmt.Errorf("the condition %s on the bus cannot be read: %w", c.Key, err)
|
||||||
|
}
|
||||||
|
if held.Silenced == nil || now.Before(held.Silenced.Until) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
was := held.Silenced.Why
|
||||||
|
held.Silenced = nil
|
||||||
|
body, err := json.Marshal(held)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := k.store.Update(ctx, c.Key, body, entry.Revision); errors.Is(err, ErrMoved) {
|
||||||
|
continue
|
||||||
|
} else if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
k.tell(Event{Condition: held, At: now, Change: ChangeUnsilenced, Why: was})
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Open is every open condition, urgent first and then oldest first.
|
||||||
|
func (k *Keeper) Open(ctx context.Context) ([]Condition, error) {
|
||||||
|
return Read(ctx, k.store)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get is one open condition.
|
||||||
|
func (k *Keeper) Get(ctx context.Context, key string) (Condition, bool, error) {
|
||||||
|
return ReadOne(ctx, k.store, key)
|
||||||
|
}
|
||||||
|
|
||||||
|
// HistorySince is every transition from a moment, oldest first.
|
||||||
|
func (k *Keeper) HistorySince(ctx context.Context, since time.Time) ([]Event, error) {
|
||||||
|
if k.history == nil {
|
||||||
|
return nil, errors.New("this keeper has no history to read")
|
||||||
|
}
|
||||||
|
return k.history.Since(ctx, since)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read is every open condition in a store, in the order status says them. A value that cannot be
|
||||||
|
// read is an error naming its key, never a condition left out (ADR 0227 rule 4).
|
||||||
|
func Read(ctx context.Context, store Backend) ([]Condition, error) {
|
||||||
|
all, err := store.All(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the open conditions cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
out := make([]Condition, 0, len(all))
|
||||||
|
for key, e := range all {
|
||||||
|
var c Condition
|
||||||
|
if err := json.Unmarshal(e.Value, &c); err != nil {
|
||||||
|
return nil, fmt.Errorf("the condition %s cannot be read: %w", key, err)
|
||||||
|
}
|
||||||
|
out = append(out, c)
|
||||||
|
}
|
||||||
|
Order(out)
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadOne is one open condition from a store.
|
||||||
|
func ReadOne(ctx context.Context, store Backend, key string) (Condition, bool, error) {
|
||||||
|
e, found, err := store.Get(ctx, key)
|
||||||
|
if err != nil || !found {
|
||||||
|
return Condition{}, found, err
|
||||||
|
}
|
||||||
|
var c Condition
|
||||||
|
if err := json.Unmarshal(e.Value, &c); err != nil {
|
||||||
|
return Condition{}, false, fmt.Errorf("the condition %s cannot be read: %w", key, err)
|
||||||
|
}
|
||||||
|
return c, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// tell queues a transition to be kept and said. Never blocks the caller for long: a queue that is
|
||||||
|
// full is a bus away for a long time, and the transition is said lost rather than holding a watchdog.
|
||||||
|
func (k *Keeper) tell(e Event) {
|
||||||
|
e.Event = eventFor(e.Change)
|
||||||
|
e.Show = e.Condition.Show()
|
||||||
|
if k.changed != nil {
|
||||||
|
k.changed()
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
// A keeper closed while a write was in flight: said, not a panic.
|
||||||
|
if recover() != nil {
|
||||||
|
k.lost(e, errors.New("the keeper was closed"))
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case k.out <- e:
|
||||||
|
default:
|
||||||
|
k.lost(e, errors.New("too many transitions are waiting to be said"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (k *Keeper) lost(e Event, err error) {
|
||||||
|
k.mu.Lock()
|
||||||
|
k.unsaid++
|
||||||
|
k.mu.Unlock()
|
||||||
|
k.say("the condition %s was %s and that could NOT be said or kept: %v", e.Key, e.Change, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// telling keeps and says every transition in order, offering each again while the bus is away.
|
||||||
|
func (k *Keeper) telling() {
|
||||||
|
defer close(k.drained)
|
||||||
|
for e := range k.out {
|
||||||
|
body, err := json.Marshal(e)
|
||||||
|
if err != nil {
|
||||||
|
k.lost(e, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
deadline := time.Now().Add(TellFor)
|
||||||
|
wait := 200 * time.Millisecond
|
||||||
|
kept, said := k.history == nil, k.teller == nil
|
||||||
|
for {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
if !kept {
|
||||||
|
kept = k.history.Append(ctx, e) == nil
|
||||||
|
}
|
||||||
|
if !said {
|
||||||
|
said = k.teller.PublishSeatEvent(ctx, Seat, e.Event, body) == nil
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
if kept && said {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
what := "said"
|
||||||
|
if !kept {
|
||||||
|
what = "kept in the history"
|
||||||
|
}
|
||||||
|
k.lost(e, fmt.Errorf("not %s within %s", what, TellFor))
|
||||||
|
break
|
||||||
|
}
|
||||||
|
time.Sleep(wait)
|
||||||
|
wait = min(2*wait, 10*time.Second)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func orSelf(resolver string) string {
|
||||||
|
if resolver == "" {
|
||||||
|
return ResolverSelf
|
||||||
|
}
|
||||||
|
return resolver
|
||||||
|
}
|
||||||
@@ -0,0 +1,379 @@
|
|||||||
|
package conditions
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// clock is a time a test moves by hand.
|
||||||
|
type clock struct{ at time.Time }
|
||||||
|
|
||||||
|
func (c *clock) now() time.Time { return c.at }
|
||||||
|
func (c *clock) pass(d time.Duration) { c.at = c.at.Add(d) }
|
||||||
|
func newClock() *clock { return &clock{at: time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)} }
|
||||||
|
func keeper(t *testing.T) (*Keeper, *InMemory, *Told, *clock) {
|
||||||
|
t.Helper()
|
||||||
|
store, told, c := NewInMemory(), &Told{}, newClock()
|
||||||
|
k := NewKeeper(t.Context(), Options{Store: store, History: store, Teller: told, Now: c.now,
|
||||||
|
Say: func(f string, a ...any) { t.Logf(f, a...) }})
|
||||||
|
t.Cleanup(func() { k.Close(context.Background()) })
|
||||||
|
return k, store, told, c
|
||||||
|
}
|
||||||
|
|
||||||
|
// settled waits until the teller has been told n events.
|
||||||
|
func settled(t *testing.T, told *Told, n int) []Event {
|
||||||
|
t.Helper()
|
||||||
|
deadline := time.Now().Add(5 * time.Second)
|
||||||
|
for {
|
||||||
|
said := told.Said()
|
||||||
|
if len(said) >= n {
|
||||||
|
return said
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatalf("told %d event(s), want %d: %+v", len(said), n, said)
|
||||||
|
}
|
||||||
|
time.Sleep(5 * time.Millisecond)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func silent(node string) Observation {
|
||||||
|
return Observation{Scope: ScopeMachine, ID: node, Kind: "silent", Machine: node, Severity: Warning,
|
||||||
|
Summary: node + " has not been heard from", Source: "S1"}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A condition is raised once, observed many times, and said on the bus only when it changes**
|
||||||
|
// (to-be 45 §2): an observation that changes nothing is written and said nowhere, or the operator's
|
||||||
|
// channel would hear the same fault every thirty seconds.
|
||||||
|
func TestAConditionIsSaidWhenItChangesNotWhenItIsSeenAgain(t *testing.T) {
|
||||||
|
k, _, told, c := keeper(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
c.pass(time.Minute)
|
||||||
|
}
|
||||||
|
urgent := silent("ace")
|
||||||
|
urgent.Severity = Urgent
|
||||||
|
got, err := k.Observe(ctx, urgent)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.Key != "machine.ace.silent" || got.Observations != 4 || got.Count != 1 || got.Severity != Urgent {
|
||||||
|
t.Fatalf("held %+v", got)
|
||||||
|
}
|
||||||
|
if len(got.Evidence) != 4 || !got.Evidence[0].At.Equal(c.at) {
|
||||||
|
t.Fatalf("evidence is not newest first: %+v", got.Evidence)
|
||||||
|
}
|
||||||
|
said := settled(t, told, 2)
|
||||||
|
if said[0].Event != EventRaised || said[0].Change != ChangeRaised || said[1].Event != EventChanged ||
|
||||||
|
said[1].Change != ChangeSeverity || said[1].Was != string(Warning) {
|
||||||
|
t.Fatalf("said %+v", said)
|
||||||
|
}
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
if n := len(told.Said()); n != 2 {
|
||||||
|
t.Fatalf("said %d events for one raising and one change", n)
|
||||||
|
}
|
||||||
|
for i, name := range told.Names {
|
||||||
|
if name != told.Events[i].Event {
|
||||||
|
t.Errorf("event %d published as %s and says it is %s", i, name, told.Events[i].Event)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Evidence is bounded**: a condition open for a week keeps its newest ten observations, not all.
|
||||||
|
func TestEvidenceKeepsTheNewestTen(t *testing.T) {
|
||||||
|
k, _, _, c := keeper(t)
|
||||||
|
var got Condition
|
||||||
|
for i := 0; i < 25; i++ {
|
||||||
|
var err error
|
||||||
|
if got, err = k.Observe(t.Context(), silent("ace")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
c.pass(time.Minute)
|
||||||
|
}
|
||||||
|
if len(got.Evidence) != KeptEvidence || got.Observations != 25 {
|
||||||
|
t.Fatalf("kept %d evidence of %d observations", len(got.Evidence), got.Observations)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Cleared and raised again within ten minutes is the same condition again** (to-be 45 §2): its
|
||||||
|
// count goes up and it is said as reopened, not as news; a person's silence of it still holds.
|
||||||
|
func TestRaisedAgainSoonAfterClearingReopens(t *testing.T) {
|
||||||
|
k, store, told, c := keeper(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := k.Silence(ctx, "machine.ace.silent", time.Hour, "jochen", "the laptop is on the train"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if cleared, err := k.Clear(ctx, "machine.ace.silent", "heard again"); err != nil || !cleared {
|
||||||
|
t.Fatalf("cleared %v: %v", cleared, err)
|
||||||
|
}
|
||||||
|
c.pass(5 * time.Minute)
|
||||||
|
again, err := k.Observe(ctx, silent("ace"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if again.Count != 2 || again.Silenced == nil {
|
||||||
|
t.Fatalf("reopened as %+v", again)
|
||||||
|
}
|
||||||
|
said := settled(t, told, 4)
|
||||||
|
if said[3].Event != EventRaised || said[3].Change != ChangeReopened {
|
||||||
|
t.Fatalf("the reopening was said as %+v", said[3])
|
||||||
|
}
|
||||||
|
// And from a new keeper — the controller restarted between — reading what cleared from history.
|
||||||
|
if _, err := k.Clear(ctx, "machine.ace.silent", "heard again"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
settled(t, told, 5)
|
||||||
|
k.Close(context.Background())
|
||||||
|
next := NewKeeper(ctx, Options{Store: store, History: store, Now: c.now})
|
||||||
|
defer next.Close(context.Background())
|
||||||
|
c.pass(time.Minute)
|
||||||
|
third, err := next.Observe(ctx, silent("ace"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if third.Count != 3 {
|
||||||
|
t.Fatalf("a controller restarted between cleared and raised said it as new: %+v", third)
|
||||||
|
}
|
||||||
|
// Past the window it is news.
|
||||||
|
if _, err := next.Clear(ctx, "machine.ace.silent", "heard again"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
c.pass(ReopenWithin + time.Minute)
|
||||||
|
fourth, err := next.Observe(ctx, silent("ace"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if fourth.Count != 1 || fourth.Silenced != nil {
|
||||||
|
t.Fatalf("raised past the window as %+v", fourth)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A source's whole observation clears what it no longer observes, and only its own.** A watchdog
|
||||||
|
// that stops seeing a fault says it is resolved; it does not clear what another raised.
|
||||||
|
func TestReconcileClearsOnlyTheSourcesOwn(t *testing.T) {
|
||||||
|
k, _, told, _ := keeper(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
other := Observation{Scope: ScopeProbe, ID: "D3", Kind: "probe-failed", Token: "failed", Severity: Warning,
|
||||||
|
Summary: "D3 did not answer", Source: "doctor"}
|
||||||
|
if _, err := k.Observe(ctx, other); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := k.Reconcile(ctx, "S1", []Observation{silent("ace"), silent("g14")}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := k.Reconcile(ctx, "S1", []Observation{silent("g14")}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
open, err := k.Open(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var keys []string
|
||||||
|
for _, c := range open {
|
||||||
|
keys = append(keys, c.Key)
|
||||||
|
}
|
||||||
|
if strings.Join(keys, ",") != "machine.g14.silent,probe.D3.failed" {
|
||||||
|
t.Fatalf("open after the second observation: %v", keys)
|
||||||
|
}
|
||||||
|
said := settled(t, told, 4)
|
||||||
|
last := said[3]
|
||||||
|
if last.Event != EventCleared || last.Key != "machine.ace.silent" || last.Why == "" {
|
||||||
|
t.Fatalf("the clearing was said as %+v", last)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A store that cannot be read is never an empty one** (ADR 0227 rule 4): reconciling against it
|
||||||
|
// clears nothing and says why.
|
||||||
|
func TestAnUnreadableStoreClearsNothing(t *testing.T) {
|
||||||
|
k, store, _, _ := keeper(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
store.Fail = errors.New("the bus is away")
|
||||||
|
if err := k.Reconcile(ctx, "S1", nil); err == nil {
|
||||||
|
t.Fatal("reconciled against a store it could not read")
|
||||||
|
}
|
||||||
|
if _, err := k.Open(ctx); err == nil {
|
||||||
|
t.Fatal("an unreadable store answered as read")
|
||||||
|
}
|
||||||
|
store.Fail = nil
|
||||||
|
store.values["machine.g14.silent"] = Entry{Value: []byte("{not a condition"), Revision: 99}
|
||||||
|
if _, err := k.Open(ctx); err == nil || !strings.Contains(err.Error(), "machine.g14.silent") {
|
||||||
|
t.Fatalf("an unreadable condition was left out rather than said: %v", err)
|
||||||
|
}
|
||||||
|
if cleared, err := k.Clear(ctx, "machine.g14.silent", "x"); err == nil || cleared {
|
||||||
|
t.Fatal("an unreadable condition was cleared unread")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A silence is bounded, says why, and ends on its own** (to-be 45 §2): the condition stays open
|
||||||
|
// through it, and its messages start again when it ends.
|
||||||
|
func TestASilenceIsBoundedAndEnds(t *testing.T) {
|
||||||
|
k, _, told, c := keeper(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := k.Silence(ctx, "machine.ace.silent", 8*24*time.Hour, "jochen", "away"); err == nil {
|
||||||
|
t.Fatal("silenced for longer than a week")
|
||||||
|
}
|
||||||
|
if _, err := k.Silence(ctx, "machine.ace.silent", time.Hour, "jochen", " "); err == nil {
|
||||||
|
t.Fatal("silenced without a reason")
|
||||||
|
}
|
||||||
|
if _, err := k.Silence(ctx, "machine.nothing.silent", time.Hour, "jochen", "x"); err == nil {
|
||||||
|
t.Fatal("silenced a condition that is not open")
|
||||||
|
}
|
||||||
|
held, err := k.Silence(ctx, "machine.ace.silent", time.Hour, "jochen", "on the train")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !held.SilencedAt(c.at) || held.Silenced.By != "jochen" {
|
||||||
|
t.Fatalf("silenced as %+v", held.Silenced)
|
||||||
|
}
|
||||||
|
c.pass(30 * time.Minute)
|
||||||
|
if err := k.EndSilences(ctx); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
c.pass(31 * time.Minute)
|
||||||
|
if err := k.EndSilences(ctx); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, _, _ := k.Get(ctx, "machine.ace.silent")
|
||||||
|
if got.Silenced != nil {
|
||||||
|
t.Fatalf("a silence past its end still held: %+v", got.Silenced)
|
||||||
|
}
|
||||||
|
said := settled(t, told, 3)
|
||||||
|
if said[1].Change != ChangeSilenced || said[2].Change != ChangeUnsilenced || said[2].Event != EventChanged {
|
||||||
|
t.Fatalf("said %+v", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Two writers never lose each other's word.** The serving controller observes while a person's
|
||||||
|
// command silences: the write that lost the compare-and-set reads again and redoes itself.
|
||||||
|
func TestAWriteThatLostTheRaceRedoesItself(t *testing.T) {
|
||||||
|
k, store, _, _ := keeper(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
racing := &racingStore{InMemory: store, before: func() {
|
||||||
|
// Another process silences between this keeper's read and its write.
|
||||||
|
other := NewKeeper(ctx, Options{Store: store})
|
||||||
|
defer other.Close(context.Background())
|
||||||
|
if _, err := other.Silence(ctx, "machine.ace.silent", time.Hour, "jochen", "known"); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
k.store = racing
|
||||||
|
got, err := k.Observe(ctx, silent("ace"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.Silenced == nil || got.Observations != 2 {
|
||||||
|
t.Fatalf("the observation overwrote the silence: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// racingStore lets another writer in once, between a read and the write after it.
|
||||||
|
type racingStore struct {
|
||||||
|
*InMemory
|
||||||
|
before func()
|
||||||
|
done bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *racingStore) Update(ctx context.Context, key string, value []byte, revision uint64) error {
|
||||||
|
if !r.done {
|
||||||
|
r.done = true
|
||||||
|
r.before()
|
||||||
|
}
|
||||||
|
return r.InMemory.Update(ctx, key, value, revision)
|
||||||
|
}
|
||||||
|
|
||||||
|
// **An observation that could not be routed is refused**, naming what it lacks.
|
||||||
|
func TestAnObservationSaysWhatItIs(t *testing.T) {
|
||||||
|
k, _, _, _ := keeper(t)
|
||||||
|
for _, o := range []Observation{
|
||||||
|
{Scope: "elsewhere", ID: "x", Kind: "k", Severity: Warning, Summary: "s", Source: "S1"},
|
||||||
|
{Scope: ScopeMachine, ID: "x", Kind: "k", Severity: "loud", Summary: "s", Source: "S1"},
|
||||||
|
{Scope: ScopeMachine, ID: "x", Kind: "k", Severity: Warning, Source: "S1"},
|
||||||
|
{Scope: ScopeMachine, ID: "x", Kind: "k", Severity: Warning, Summary: "s"},
|
||||||
|
} {
|
||||||
|
if _, err := k.Observe(t.Context(), o); err == nil {
|
||||||
|
t.Errorf("observed %+v", o)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A key holds nothing the bus would refuse or read as a wildcard**, whatever the thing is called.
|
||||||
|
func TestAKeyIsSafeForTheBus(t *testing.T) {
|
||||||
|
if got := Key(ScopeProvider, "keycloak.novox.my app*", "failing"); got != "provider.keycloak.novox.my_app_.failing" {
|
||||||
|
t.Fatalf("key %q", got)
|
||||||
|
}
|
||||||
|
if got := Key(ScopeBus, "EVENTS.>", "consumer-lost"); got != "bus.EVENTS._.consumer-lost" {
|
||||||
|
t.Fatalf("key %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The event's shape is a contract** (to-be 45 §2): the operator-channel's holder is written against
|
||||||
|
// these field names. A rename here is a channel that reads nothing, so they are held still.
|
||||||
|
func TestTheEventShapeIsTheContract(t *testing.T) {
|
||||||
|
k, _, told, _ := keeper(t)
|
||||||
|
if _, err := k.Observe(t.Context(), silent("ace")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
said := settled(t, told, 1)
|
||||||
|
body, err := json.Marshal(said[0])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var shape map[string]any
|
||||||
|
if err := json.Unmarshal(body, &shape); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// The condition at the top level, kebab-case, beside what happened to it.
|
||||||
|
for _, field := range []string{"event", "at", "change", "show", "key", "kind", "subject", "severity",
|
||||||
|
"summary", "evidence", "source", "raised", "last-observed", "observations", "count", "resolver",
|
||||||
|
"silenced", "epoch"} {
|
||||||
|
if _, ok := shape[field]; !ok {
|
||||||
|
t.Errorf("the event carries no %q: %s", field, body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if shape["silenced"] != nil {
|
||||||
|
t.Errorf("an unsilenced condition says silenced %v, not null", shape["silenced"])
|
||||||
|
}
|
||||||
|
subject, _ := shape["subject"].(map[string]any)
|
||||||
|
if subject["scope"] != "machine" || subject["id"] != "ace" || subject["machine"] != "ace" {
|
||||||
|
t.Errorf("subject %v", shape["subject"])
|
||||||
|
}
|
||||||
|
if said[0].Show != "mesh-controller.conditions key=machine.ace.silent" {
|
||||||
|
t.Errorf("show is %q", said[0].Show)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A transition the bus will not take is offered again**, and said lost only after TellFor.
|
||||||
|
func TestATransitionIsOfferedAgainWhileTheBusIsAway(t *testing.T) {
|
||||||
|
store, told, c := NewInMemory(), &Told{Fail: errors.New("no responders")}, newClock()
|
||||||
|
k := NewKeeper(t.Context(), Options{Store: store, History: store, Teller: told, Now: c.now})
|
||||||
|
defer k.Close(context.Background())
|
||||||
|
if _, err := k.Observe(t.Context(), silent("ace")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
time.Sleep(300 * time.Millisecond)
|
||||||
|
told.mu.Lock()
|
||||||
|
told.Fail = nil
|
||||||
|
told.mu.Unlock()
|
||||||
|
said := settled(t, told, 1)
|
||||||
|
if said[0].Key != "machine.ace.silent" || k.Unsaid() != 0 {
|
||||||
|
t.Fatalf("said %+v, unsaid %d", said, k.Unsaid())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,146 @@
|
|||||||
|
package inventory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The durations the core's bounds are set from (novox/hq to-be 45 Phase 0): see migration 0066.
|
||||||
|
|
||||||
|
// The kinds of duration recorded.
|
||||||
|
const (
|
||||||
|
DurationApply = "apply"
|
||||||
|
DurationHeartbeatGap = "heartbeat-gap"
|
||||||
|
DurationPlanTier = "plan-tier"
|
||||||
|
DurationBuild = "build"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DurationKinds are every kind, in the order `durations` shows them.
|
||||||
|
var DurationKinds = []string{DurationApply, DurationHeartbeatGap, DurationPlanTier, DurationBuild}
|
||||||
|
|
||||||
|
// DurationsKeptFor is how long a duration is kept: long enough to set a bound from, and to correct it
|
||||||
|
// in Phase 1's first live week.
|
||||||
|
const DurationsKeptFor = 30 * 24 * time.Hour
|
||||||
|
|
||||||
|
// Duration is one measurement.
|
||||||
|
type Duration struct {
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Subject string `json:"subject"`
|
||||||
|
Node string `json:"node,omitempty"`
|
||||||
|
Ref string `json:"ref"`
|
||||||
|
Started time.Time `json:"started"`
|
||||||
|
Took time.Duration `json:"took"`
|
||||||
|
Detail string `json:"detail,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordDuration keeps one measurement, once: the same thing measured again is not a second row.
|
||||||
|
func (i *Inventory) RecordDuration(ctx context.Context, d Duration) error {
|
||||||
|
if d.Took < 0 {
|
||||||
|
return nil // a clock that went back measures nothing
|
||||||
|
}
|
||||||
|
_, err := i.store.Pool().Exec(ctx,
|
||||||
|
`insert into duration (kind, subject, node, ref, started, took_ms, detail)
|
||||||
|
values ($1, $2, $3, $4, $5, $6, $7) on conflict do nothing`,
|
||||||
|
d.Kind, d.Subject, d.Node, d.Ref, d.Started, d.Took.Milliseconds(), d.Detail)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordApplyDuration measures a machine's report of the declaration it was last sent: from the send
|
||||||
|
// to the first report of it. A report of anything else, or of a send already measured, measures
|
||||||
|
// nothing — a machine reconciling reports the same declaration every few minutes.
|
||||||
|
func (i *Inventory) RecordApplyDuration(ctx context.Context, node, declared, outcome string) error {
|
||||||
|
if declared == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
_, err := i.store.Pool().Exec(ctx,
|
||||||
|
`insert into duration (kind, subject, node, ref, started, took_ms, detail)
|
||||||
|
select $1, n.name, n.name, n.sent || '@' || to_char(n.sent_at at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US'),
|
||||||
|
n.sent_at, (extract(epoch from (now() - n.sent_at)) * 1000)::bigint, $4
|
||||||
|
from node n
|
||||||
|
where n.name = $2 and n.sent = $3 and n.sent_at is not null
|
||||||
|
on conflict do nothing`,
|
||||||
|
DurationApply, node, declared, outcome)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordHeartbeatGap measures the silence before a machine's word: from the last word heard before
|
||||||
|
// it, which the caller read before recording this one.
|
||||||
|
func (i *Inventory) RecordHeartbeatGap(ctx context.Context, node string, before time.Time) error {
|
||||||
|
if before.IsZero() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
return i.RecordDuration(ctx, Duration{Kind: DurationHeartbeatGap, Subject: node, Node: node,
|
||||||
|
Ref: before.UTC().Format(time.RFC3339Nano), Started: before, Took: now.Sub(before)})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Durations is every measurement of a kind since a moment, oldest first; every kind when kind is empty.
|
||||||
|
func (i *Inventory) Durations(ctx context.Context, kind string, since time.Time) ([]Duration, error) {
|
||||||
|
rows, err := i.store.Pool().Query(ctx,
|
||||||
|
`select kind, subject, node, ref, started, took_ms, detail from duration
|
||||||
|
where ($1 = '' or kind = $1) and recorded >= $2 order by recorded`, kind, since)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var out []Duration
|
||||||
|
for rows.Next() {
|
||||||
|
var d Duration
|
||||||
|
var ms int64
|
||||||
|
if err := rows.Scan(&d.Kind, &d.Subject, &d.Node, &d.Ref, &d.Started, &ms, &d.Detail); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
d.Took = time.Duration(ms) * time.Millisecond
|
||||||
|
out = append(out, d)
|
||||||
|
}
|
||||||
|
return out, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// ForgetOldDurations removes what is older than DurationsKeptFor, and says how many.
|
||||||
|
func (i *Inventory) ForgetOldDurations(ctx context.Context) (int64, error) {
|
||||||
|
tag, err := i.store.Pool().Exec(ctx, `delete from duration where recorded < $1`,
|
||||||
|
time.Now().Add(-DurationsKeptFor))
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return tag.RowsAffected(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// planTierLeft is the measurement a plan's save makes when it leaves a tier: the tier moved on, or
|
||||||
|
// the plan ended. Read in the save's own transaction, so two saves cannot both measure one tier.
|
||||||
|
func planTierLeft(ctx context.Context, tx pgx.Tx, p Plan, now time.Time) (entered time.Time, err error) {
|
||||||
|
var oldTier int
|
||||||
|
var oldState string
|
||||||
|
var since time.Time
|
||||||
|
err = tx.QueryRow(ctx,
|
||||||
|
`select tier, state, coalesce(tier_entered, created) from release_plan where id = $1 for update`,
|
||||||
|
p.ID).Scan(&oldTier, &oldState, &since)
|
||||||
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
return now, nil // a new plan enters its first tier now
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return time.Time{}, err
|
||||||
|
}
|
||||||
|
wasOpen := oldState == PlanBuilding || oldState == PlanRolling
|
||||||
|
if !wasOpen || (oldTier == p.Tier && p.Open()) {
|
||||||
|
return since, nil // still in the tier, or already ended
|
||||||
|
}
|
||||||
|
var modules []string
|
||||||
|
if oldTier >= 0 && oldTier < len(p.Tiers) {
|
||||||
|
modules = p.Tiers[oldTier]
|
||||||
|
}
|
||||||
|
detail := fmt.Sprintf("plan %s, tier %d of %d (%v), left %s", p.ID, oldTier, len(p.Tiers), modules, p.State)
|
||||||
|
if p.Open() {
|
||||||
|
detail = fmt.Sprintf("plan %s, tier %d of %d (%v), moved on to tier %d", p.ID, oldTier, len(p.Tiers), modules, p.Tier)
|
||||||
|
}
|
||||||
|
_, err = tx.Exec(ctx,
|
||||||
|
`insert into duration (kind, subject, node, ref, started, took_ms, detail)
|
||||||
|
values ($1, $2, '', $3, $4, $5, $6) on conflict do nothing`,
|
||||||
|
DurationPlanTier, p.Repository, fmt.Sprintf("%s/tier-%d", p.ID, oldTier), since,
|
||||||
|
now.Sub(since).Milliseconds(), detail)
|
||||||
|
return now, err
|
||||||
|
}
|
||||||
@@ -0,0 +1,97 @@
|
|||||||
|
package inventory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **The durations the bounds are set from are recorded once each** (novox/hq to-be 45 Phase 0): a
|
||||||
|
// send measured at its first report and not again at every reconcile that repeats it; a send of
|
||||||
|
// something else measures nothing; a new send is a new measurement.
|
||||||
|
func TestAnApplyIsMeasuredOncePerSend(t *testing.T) {
|
||||||
|
inv := ForTest(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
node, err := inv.AddNode(ctx, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.RecordSent(ctx, node.ID, "d1", nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for range 3 { // the report, then two reconciles saying the same
|
||||||
|
if err := inv.RecordApplyDuration(ctx, "anchor", "d1", OutcomeApplied); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := inv.RecordApplyDuration(ctx, "anchor", "d0", OutcomeApplied); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ds, err := inv.Durations(ctx, DurationApply, time.Now().Add(-time.Hour))
|
||||||
|
if err != nil || len(ds) != 1 || ds[0].Subject != "anchor" || ds[0].Detail != OutcomeApplied || ds[0].Took < 0 {
|
||||||
|
t.Fatalf("%v %+v", err, ds)
|
||||||
|
}
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
if err := inv.RecordSent(ctx, node.ID, "d2", nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.RecordApplyDuration(ctx, "anchor", "d2", OutcomeFailed); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if ds, _ = inv.Durations(ctx, "", time.Now().Add(-time.Hour)); len(ds) != 2 {
|
||||||
|
t.Fatalf("a second send was not measured: %+v", ds)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A machine's silence is the time since its last word, once per word.
|
||||||
|
func TestASilenceIsMeasuredFromTheLastWord(t *testing.T) {
|
||||||
|
inv := ForTest(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
before := time.Now().Add(-90 * time.Second)
|
||||||
|
for range 2 {
|
||||||
|
if err := inv.RecordHeartbeatGap(ctx, "anchor", before); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := inv.RecordHeartbeatGap(ctx, "anchor", time.Time{}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ds, err := inv.Durations(ctx, DurationHeartbeatGap, time.Now().Add(-time.Hour))
|
||||||
|
if err != nil || len(ds) != 1 || ds[0].Took < 90*time.Second || ds[0].Took > 2*time.Minute {
|
||||||
|
t.Fatalf("%v %+v", err, ds)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A plan's tier is measured when the plan leaves it — moving on, or ending — and only then.
|
||||||
|
func TestAPlansTierIsMeasuredWhenItIsLeft(t *testing.T) {
|
||||||
|
inv := ForTest(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
p := Plan{ID: "plan-1", Repository: "novox/mesh-tools", Commit: "abc", Created: time.Now().UTC(),
|
||||||
|
State: PlanBuilding, Tiers: [][]string{{"mesh-tools"}, {"builder"}},
|
||||||
|
Modules: map[string]*PlanModule{"mesh-tools": {}, "builder": {}}}
|
||||||
|
if err := inv.SavePlan(ctx, p); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
p.State = PlanRolling // the same tier, saved again
|
||||||
|
if err := inv.SavePlan(ctx, p); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if ds, _ := inv.Durations(ctx, DurationPlanTier, time.Now().Add(-time.Hour)); len(ds) != 0 {
|
||||||
|
t.Fatalf("a tier not left was measured: %+v", ds)
|
||||||
|
}
|
||||||
|
p.Tier = 1
|
||||||
|
if err := inv.SavePlan(ctx, p); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
p.State = PlanDone
|
||||||
|
if err := inv.SavePlan(ctx, p); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.SavePlan(ctx, p); err != nil { // saved again once ended: nothing more to measure
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ds, err := inv.Durations(ctx, DurationPlanTier, time.Now().Add(-time.Hour))
|
||||||
|
if err != nil || len(ds) != 2 || ds[0].Subject != "novox/mesh-tools" || ds[0].Ref != "plan-1/tier-0" ||
|
||||||
|
ds[1].Ref != "plan-1/tier-1" {
|
||||||
|
t.Fatalf("%v %+v", err, ds)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
-- The durations the core's bounds are set from (novox/hq to-be 45 Phase 0).
|
||||||
|
--
|
||||||
|
-- Every watchdog of the core's signals table has a bound — how long a machine may take to report
|
||||||
|
-- after a send, how long it may be silent, how long a plan's tier or a build may take — and a bound
|
||||||
|
-- guessed is a condition that cries wolf or one that never fires. So the controller records each
|
||||||
|
-- duration as it is observed, and Phase 1 sets the bounds from what was recorded:
|
||||||
|
--
|
||||||
|
-- apply a declaration sent → the machine's first report of that declaration, per machine
|
||||||
|
-- heartbeat-gap one word from a machine → the next, per machine
|
||||||
|
-- plan-tier a plan entering a tier → leaving it, per repository
|
||||||
|
-- build a build asked → its outcome heard, per module
|
||||||
|
--
|
||||||
|
-- One row per thing measured: `ref` names it (the send, the earlier word, the plan's tier, the
|
||||||
|
-- build), so a report repeated by a reconcile is not a second measurement. Kept a month; read
|
||||||
|
-- through `durations`.
|
||||||
|
create table duration (
|
||||||
|
kind text not null,
|
||||||
|
subject text not null,
|
||||||
|
node text not null default '',
|
||||||
|
ref text not null,
|
||||||
|
started timestamptz not null,
|
||||||
|
took_ms bigint not null,
|
||||||
|
detail text not null default '',
|
||||||
|
recorded timestamptz not null default now(),
|
||||||
|
primary key (kind, subject, ref)
|
||||||
|
);
|
||||||
|
|
||||||
|
create index duration_by_kind on duration (kind, recorded);
|
||||||
|
|
||||||
|
-- When a plan entered the tier it is at, so leaving it measures the tier.
|
||||||
|
alter table release_plan add column tier_entered timestamptz;
|
||||||
@@ -28,6 +28,10 @@ type Plan struct {
|
|||||||
Tiers [][]string `json:"tiers"`
|
Tiers [][]string `json:"tiers"`
|
||||||
Modules map[string]*PlanModule `json:"modules"`
|
Modules map[string]*PlanModule `json:"modules"`
|
||||||
Note string `json:"note,omitempty"`
|
Note string `json:"note,omitempty"`
|
||||||
|
// TierEntered is when the plan entered the tier it is at (novox/hq to-be 45 Phase 0), read and
|
||||||
|
// never written from here: a save measures the tier it leaves and stamps the next. What the
|
||||||
|
// watchdog of a plan's progress (S3) reads.
|
||||||
|
TierEntered time.Time `json:"tier_entered,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// PlanModule is one module's state within a plan.
|
// PlanModule is one module's state within a plan.
|
||||||
@@ -80,13 +84,29 @@ func (i *Inventory) SavePlan(ctx context.Context, p Plan) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
_, err = i.store.Pool().Exec(ctx,
|
// **And how long the tier it left took** (novox/hq to-be 45 Phase 0): measured here, where the
|
||||||
`insert into release_plan (id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch)
|
// plan moves, in the same transaction as the move, so no save can move a tier unmeasured or
|
||||||
values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9, $10)
|
// measure one twice.
|
||||||
on conflict (id) do update set updated = now(), state = excluded.state, tier = excluded.tier,
|
tx, err := i.store.Pool().Begin(ctx)
|
||||||
tiers = excluded.tiers, modules = excluded.modules, note = excluded.note, branch = excluded.branch`,
|
if err != nil {
|
||||||
p.ID, p.Repository, p.Commit, p.Created, p.State, p.Tier, tiers, modules, p.Note, p.Branch)
|
|
||||||
return err
|
return err
|
||||||
|
}
|
||||||
|
defer func() { _ = tx.Rollback(ctx) }()
|
||||||
|
entered, err := planTierLeft(ctx, tx, p, time.Now())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = tx.Exec(ctx,
|
||||||
|
`insert into release_plan (id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch, tier_entered)
|
||||||
|
values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9, $10, $11)
|
||||||
|
on conflict (id) do update set updated = now(), state = excluded.state, tier = excluded.tier,
|
||||||
|
tiers = excluded.tiers, modules = excluded.modules, note = excluded.note, branch = excluded.branch,
|
||||||
|
tier_entered = excluded.tier_entered`,
|
||||||
|
p.ID, p.Repository, p.Commit, p.Created, p.State, p.Tier, tiers, modules, p.Note, p.Branch, entered)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return tx.Commit(ctx)
|
||||||
}
|
}
|
||||||
|
|
||||||
// OpenPlans is every plan still being worked, oldest first.
|
// OpenPlans is every plan still being worked, oldest first.
|
||||||
@@ -113,7 +133,8 @@ func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) {
|
|||||||
|
|
||||||
func (i *Inventory) plans(ctx context.Context, tail string) ([]Plan, error) {
|
func (i *Inventory) plans(ctx context.Context, tail string) ([]Plan, error) {
|
||||||
rows, err := i.store.Pool().Query(ctx,
|
rows, err := i.store.Pool().Query(ctx,
|
||||||
`select id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch
|
`select id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch,
|
||||||
|
coalesce(tier_entered, created)
|
||||||
from release_plan `+tail)
|
from release_plan `+tail)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -124,7 +145,7 @@ func (i *Inventory) plans(ctx context.Context, tail string) ([]Plan, error) {
|
|||||||
var p Plan
|
var p Plan
|
||||||
var tiers, modules []byte
|
var tiers, modules []byte
|
||||||
if err := rows.Scan(&p.ID, &p.Repository, &p.Commit, &p.Created, &p.Updated, &p.State,
|
if err := rows.Scan(&p.ID, &p.Repository, &p.Commit, &p.Created, &p.Updated, &p.State,
|
||||||
&p.Tier, &tiers, &modules, &p.Note, &p.Branch); err != nil {
|
&p.Tier, &tiers, &modules, &p.Note, &p.Branch, &p.TierEntered); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := json.Unmarshal(tiers, &p.Tiers); err != nil {
|
if err := json.Unmarshal(tiers, &p.Tiers); err != nil {
|
||||||
|
|||||||
@@ -105,14 +105,27 @@ func (i *Inventory) widenProtocol(ctx context.Context, s catalogue.Seat) error {
|
|||||||
changed := false
|
changed := false
|
||||||
row.Accepts, changed = union(row.Accepts, s.Accepts, changed)
|
row.Accepts, changed = union(row.Accepts, s.Accepts, changed)
|
||||||
row.Emits, changed = union(row.Emits, s.Emits, changed)
|
row.Emits, changed = union(row.Emits, s.Emits, changed)
|
||||||
have := map[string]bool{}
|
have := map[string]int{}
|
||||||
for _, v := range row.Serves {
|
for n, v := range row.Serves {
|
||||||
have[v.Name] = true
|
have[v.Name] = n
|
||||||
}
|
}
|
||||||
for _, v := range s.Serves {
|
for _, v := range s.Serves {
|
||||||
if !have[v.Name] {
|
at, kept := have[v.Name]
|
||||||
|
if !kept {
|
||||||
row.Serves = append(row.Serves, v)
|
row.Serves = append(row.Serves, v)
|
||||||
changed = true
|
changed = true
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// **The controller's own verbs are described by the binary that runs them** (novox/hq
|
||||||
|
// issue 244, to-be 45 §7). Its seat's verbs are not an operator's to reshape: each is a
|
||||||
|
// command line this binary composes from the arguments its own table declares, and the
|
||||||
|
// console judges a call against the row. A row kept from an older build described `push`
|
||||||
|
// without the `behind` and `why` the binary takes, so the console refused an argument the verb
|
||||||
|
// needs. So a verb this binary defines takes this binary's definition; a verb only the row has
|
||||||
|
// — a newer build's, during a roll-out (ADR 0185) — is left as it is.
|
||||||
|
if s.Name == catalogue.ControllerSeatName && !sameVerb(row.Serves[at], v) {
|
||||||
|
row.Serves[at] = v
|
||||||
|
changed = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !changed {
|
if !changed {
|
||||||
@@ -282,3 +295,18 @@ func (i *Inventory) Holdings(ctx context.Context) ([]catalogue.Held, error) {
|
|||||||
}
|
}
|
||||||
return out, rows.Err()
|
return out, rows.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// sameVerb is whether two definitions of a verb say the same, read as the row stores them.
|
||||||
|
func sameVerb(a, b catalogue.Verb) bool {
|
||||||
|
ja, errA := json.Marshal(a)
|
||||||
|
jb, errB := json.Marshal(b)
|
||||||
|
if errA != nil || errB != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
var ra, rb any
|
||||||
|
_ = json.Unmarshal(ja, &ra)
|
||||||
|
_ = json.Unmarshal(jb, &rb)
|
||||||
|
ca, _ := json.Marshal(ra)
|
||||||
|
cb, _ := json.Marshal(rb)
|
||||||
|
return string(ca) == string(cb)
|
||||||
|
}
|
||||||
|
|||||||
@@ -113,3 +113,46 @@ func TestRenameSeatKeepsTheFormerNameAsAnAlias(t *testing.T) {
|
|||||||
t.Fatal("renaming a seat to its own name was accepted")
|
t.Fatal("renaming a seat to its own name was accepted")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **The controller's verbs in the row are the binary's** (novox/hq issue 244, to-be 45 §7): a row
|
||||||
|
// seeded by an older build describes `push` without the arguments this one takes, and the console
|
||||||
|
// judges a call against the row — so re-seeding brings the controller's verbs to this binary's
|
||||||
|
// definition, and keeps a verb only the row has, which a newer build added (ADR 0185).
|
||||||
|
func TestTheControllersVerbsInTheRowAreTheBinarys(t *testing.T) {
|
||||||
|
inv := ForTest(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
old := `[{"name":"push","description":"an older push","input":{"type":"object","properties":{"node":{"type":"string"}}}},
|
||||||
|
{"name":"newer","description":"a verb of a newer build"}]`
|
||||||
|
if _, err := inv.store.Pool().Exec(ctx, `update seat set serves = $1 where name = $2`,
|
||||||
|
[]byte(old), catalogue.ControllerSeatName); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
seats, err := inv.Seats(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
verbs := map[string]catalogue.Verb{}
|
||||||
|
for _, s := range seats {
|
||||||
|
if s.Name == catalogue.ControllerSeatName {
|
||||||
|
for _, v := range s.Serves {
|
||||||
|
verbs[v.Name] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
props, _ := verbs["push"].Input["properties"].(map[string]any)
|
||||||
|
if _, takesWhy := props["why"]; !takesWhy || verbs["push"].Description == "an older push" {
|
||||||
|
t.Fatalf("push in the row is still the older build's: %+v", verbs["push"])
|
||||||
|
}
|
||||||
|
if _, kept := verbs["newer"]; !kept {
|
||||||
|
t.Fatal("a verb only the row has was dropped")
|
||||||
|
}
|
||||||
|
if _, added := verbs["durations"]; !added {
|
||||||
|
t.Fatal("a verb this binary adds was not added")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,76 +0,0 @@
|
|||||||
package inventory
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ProviderStanding is a consumer a provider says it keeps failing (novox/hq ADR 0224).
|
|
||||||
type ProviderStanding struct {
|
|
||||||
// Module is the provider's module, and ProviderNode the machine it runs on.
|
|
||||||
Module string `json:"module"`
|
|
||||||
ProviderNode string `json:"provider-node"`
|
|
||||||
// Provision is the interface it provides, e.g. `oidc-client`.
|
|
||||||
Provision string `json:"provision"`
|
|
||||||
// Consumer is the identity the mesh derived for the consumer, ConsumerNode its machine.
|
|
||||||
Consumer string `json:"consumer"`
|
|
||||||
ConsumerNode string `json:"consumer-node"`
|
|
||||||
// Class is what kind of failure: credentials-rejected, unreachable, secret-unreadable, refused.
|
|
||||||
Class string `json:"class"`
|
|
||||||
Error string `json:"error"`
|
|
||||||
// Since is when the unbroken run of failures began; Attempts how many it has been.
|
|
||||||
Since time.Time `json:"since"`
|
|
||||||
Attempts int `json:"attempts"`
|
|
||||||
// SaidAt is when the controller last heard it. A provider says it again every quarter of an hour
|
|
||||||
// while it lasts, so an old one is a provider that stopped saying anything.
|
|
||||||
SaidAt time.Time `json:"said-at"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// SayAgainWithin is how long a failing standing stays current without being said again: twice the
|
|
||||||
// quarter of an hour a provider repeats it at. Older, and status says the provider has gone quiet.
|
|
||||||
const SayAgainWithin = 30 * time.Minute
|
|
||||||
|
|
||||||
// Quiet says the provider has not repeated this standing for longer than it would while it lasts.
|
|
||||||
func (s ProviderStanding) Quiet(now time.Time) bool { return now.Sub(s.SaidAt) > SayAgainWithin }
|
|
||||||
|
|
||||||
// KeepStanding records a provider's newest word: failing keeps it, recovered removes it, and says
|
|
||||||
// whether a recovery removed anything.
|
|
||||||
func (i *Inventory) KeepStanding(ctx context.Context, failing bool, s ProviderStanding) (bool, error) {
|
|
||||||
if !failing {
|
|
||||||
tag, err := i.store.Pool().Exec(ctx,
|
|
||||||
`delete from provider_standing where module = $1 and provider_node = $2 and consumer = $3`,
|
|
||||||
s.Module, s.ProviderNode, s.Consumer)
|
|
||||||
return err == nil && tag.RowsAffected() > 0, err
|
|
||||||
}
|
|
||||||
_, err := i.store.Pool().Exec(ctx, `
|
|
||||||
insert into provider_standing
|
|
||||||
(module, provider_node, consumer, consumer_node, provision, class, error, since, attempts, said_at)
|
|
||||||
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, now())
|
|
||||||
on conflict (module, provider_node, consumer) do update set
|
|
||||||
consumer_node = excluded.consumer_node, provision = excluded.provision,
|
|
||||||
class = excluded.class, error = excluded.error, since = excluded.since,
|
|
||||||
attempts = excluded.attempts, said_at = excluded.said_at`,
|
|
||||||
s.Module, s.ProviderNode, s.Consumer, s.ConsumerNode, s.Provision, s.Class, s.Error, s.Since, s.Attempts)
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// FailingProviders is every consumer a provider last said it keeps failing, oldest run first.
|
|
||||||
func (i *Inventory) FailingProviders(ctx context.Context) ([]ProviderStanding, error) {
|
|
||||||
rows, err := i.store.Pool().Query(ctx, `
|
|
||||||
select module, provider_node, provision, consumer, consumer_node, class, error, since, attempts, said_at
|
|
||||||
from provider_standing order by since, module, consumer`)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer rows.Close()
|
|
||||||
var out []ProviderStanding
|
|
||||||
for rows.Next() {
|
|
||||||
var s ProviderStanding
|
|
||||||
if err := rows.Scan(&s.Module, &s.ProviderNode, &s.Provision, &s.Consumer, &s.ConsumerNode,
|
|
||||||
&s.Class, &s.Error, &s.Since, &s.Attempts, &s.SaidAt); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
out = append(out, s)
|
|
||||||
}
|
|
||||||
return out, rows.Err()
|
|
||||||
}
|
|
||||||
@@ -1,130 +0,0 @@
|
|||||||
package inventory
|
|
||||||
|
|
||||||
import (
|
|
||||||
"slices"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A provider's standing (novox/hq ADR 0224), from the grant that lets it say so to the row status
|
|
||||||
// reads.
|
|
||||||
|
|
||||||
// The broker spells the events itself because it cannot import the catalogue; the two agree.
|
|
||||||
func TestTheBrokerAndTheCatalogueNameTheSameStandingEvents(t *testing.T) {
|
|
||||||
if broker.ProvisionerFailing != catalogue.ProvisionerFailing ||
|
|
||||||
broker.ProvisionerRecovered != catalogue.ProvisionerRecovered {
|
|
||||||
t.Fatal("the broker and the catalogue disagree about what a provider's standing is called")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **Every provider may say it, whatever its manifest lists**: a provider whose manifest forgot the
|
|
||||||
// events would have its announcement refused by the bus, and fail its consumers as silently as on
|
|
||||||
// 2026-10-05 (issue 179). A module that receives no contributions provides nothing and is given
|
|
||||||
// nothing.
|
|
||||||
func TestEveryProviderIsGrantedItsStandingAndNothingElseIs(t *testing.T) {
|
|
||||||
provider := catalogue.Manifest{Module: "keycloak", Version: "1",
|
|
||||||
Emits: []string{"client.created"}, Receives: map[string]string{"oidc-client": "/x/mesh.json"}}
|
|
||||||
consumer := catalogue.Manifest{Module: "grafana", Version: "1", Emits: []string{"dashboard.saved"}}
|
|
||||||
|
|
||||||
d := declaredFor(provider, nil)
|
|
||||||
for _, e := range []string{"client.created", catalogue.ProvisionerFailing, catalogue.ProvisionerRecovered} {
|
|
||||||
if !slices.Contains(d.Emits, e) {
|
|
||||||
t.Fatalf("a provider is not granted %s: %v", e, d.Emits)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindModule, Node: "anchor",
|
|
||||||
Module: "keycloak", Emits: d.Emits})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !slices.Contains(perms.Publish, "mesh.mod.keycloak.event.provisioner.failing") {
|
|
||||||
t.Fatalf("the bus would refuse a provider's standing: %v", perms.Publish)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got := declaredFor(consumer, nil).Emits; slices.Contains(got, catalogue.ProvisionerFailing) {
|
|
||||||
t.Fatalf("a module that provides nothing was granted a provider's standing: %v", got)
|
|
||||||
}
|
|
||||||
// Declared by hand as well: said once.
|
|
||||||
provider.Emits = append(provider.Emits, catalogue.ProvisionerFailing)
|
|
||||||
n := 0
|
|
||||||
for _, e := range provider.EmitsAll() {
|
|
||||||
if e == catalogue.ProvisionerFailing {
|
|
||||||
n++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if n != 1 {
|
|
||||||
t.Fatalf("%v", provider.EmitsAll())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// And the controller may hear it from every provider, and only those two events.
|
|
||||||
func TestTheControllerHearsEveryProvidersStanding(t *testing.T) {
|
|
||||||
perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, want := range []string{"mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered"} {
|
|
||||||
if !slices.Contains(perms.Subscribe, want) {
|
|
||||||
t.Fatalf("the controller may not hear %s: %v", want, perms.Subscribe)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if slices.Contains(perms.Subscribe, "mesh.mod.*.event.>") {
|
|
||||||
t.Fatal("the controller hears every event in the mesh")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAFailingStandingIsKeptUntilItRecovers(t *testing.T) {
|
|
||||||
inv := ForTest(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
since := time.Date(2026, 10, 5, 0, 49, 0, 0, time.UTC)
|
|
||||||
s := ProviderStanding{Module: "keycloak", ProviderNode: "anchor", Provision: "oidc-client",
|
|
||||||
Consumer: "mesh_home_grafana", ConsumerNode: "home-server", Class: "credentials-rejected",
|
|
||||||
Error: "401 invalid_grant", Since: since, Attempts: 60}
|
|
||||||
if _, err := inv.KeepStanding(ctx, true, s); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
// Said again: one row, the newest word.
|
|
||||||
s.Attempts = 31000
|
|
||||||
if _, err := inv.KeepStanding(ctx, true, s); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
got, err := inv.FailingProviders(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(got) != 1 || got[0].Attempts != 31000 || !got[0].Since.Equal(since) || got[0].ConsumerNode != "home-server" ||
|
|
||||||
got[0].Class != "credentials-rejected" || got[0].SaidAt.IsZero() {
|
|
||||||
t.Fatalf("%+v", got)
|
|
||||||
}
|
|
||||||
if got[0].Quiet(time.Now()) {
|
|
||||||
t.Fatal("a standing just said reads as quiet")
|
|
||||||
}
|
|
||||||
if !got[0].Quiet(time.Now().Add(SayAgainWithin + time.Minute)) {
|
|
||||||
t.Fatal("a standing not said again for longer than a provider repeats it does not read as quiet")
|
|
||||||
}
|
|
||||||
|
|
||||||
// The same consumer from another machine's provider is its own row.
|
|
||||||
other := s
|
|
||||||
other.ProviderNode = "laptop"
|
|
||||||
if _, err := inv.KeepStanding(ctx, true, other); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
cleared, err := inv.KeepStanding(ctx, false, s)
|
|
||||||
if err != nil || !cleared {
|
|
||||||
t.Fatalf("recovered cleared nothing: %v %v", cleared, err)
|
|
||||||
}
|
|
||||||
cleared, err = inv.KeepStanding(ctx, false, s)
|
|
||||||
if err != nil || cleared {
|
|
||||||
t.Fatalf("a recovery for nothing kept said it cleared something: %v %v", cleared, err)
|
|
||||||
}
|
|
||||||
got, err = inv.FailingProviders(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(got) != 1 || got[0].ProviderNode != "laptop" {
|
|
||||||
t.Fatalf("%+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,283 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What the bus says about itself, in the mesh's words (novox/hq to-be 45 §3, S9).
|
||||||
|
//
|
||||||
|
// **The server already says it; nothing listened.** A durable consumer that hands a message over as
|
||||||
|
// often as it may gives up on it and says so on `$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES`; one
|
||||||
|
// deleted says so on `…CONSUMER.DELETED`. The controller's own connection is told when it falls behind
|
||||||
|
// (a slow consumer: the client library dropped messages — issue 184, the controller deaf for 24
|
||||||
|
// minutes) and when the bus refuses it a subject (a permissions violation — issues 183, 217, 265,
|
||||||
|
// days each as a line in a client library's output). Each is recorded here, named in the mesh's words —
|
||||||
|
// which consumer of whose, which subject — for the watchdog to say as a condition, as the refused
|
||||||
|
// reply of issue 265 is said today.
|
||||||
|
//
|
||||||
|
// What the bus says about **other** principals' connections — a module's slow consumer, a module
|
||||||
|
// refused a subject — the server publishes only to a system account, which the mesh's bus does not
|
||||||
|
// have; that half is not heard yet (to-be 45 S9, recorded as deferred).
|
||||||
|
|
||||||
|
// The advisory kinds, as conditions name them.
|
||||||
|
const (
|
||||||
|
AdvisorySlowConsumer = "slow-consumer"
|
||||||
|
AdvisoryMaxDeliveries = "max-deliveries"
|
||||||
|
AdvisoryRefused = "refused"
|
||||||
|
AdvisoryConsumerLost = "consumer-lost"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Advisory is one thing the bus said, kept as its newest word and how often it was said.
|
||||||
|
type Advisory struct {
|
||||||
|
Kind string
|
||||||
|
// ID names what it is about, for the condition's key: `<stream>.<consumer>`, or `controller`.
|
||||||
|
ID string
|
||||||
|
// Stream and Consumer are the consumer it is about, when it is about one.
|
||||||
|
Stream, Consumer string
|
||||||
|
// Said is the newest saying, in the mesh's words.
|
||||||
|
Said string
|
||||||
|
First, Last time.Time
|
||||||
|
Count int
|
||||||
|
}
|
||||||
|
|
||||||
|
// AdvisoryLog keeps what the bus said lately.
|
||||||
|
type AdvisoryLog struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
seen map[string]*Advisory
|
||||||
|
}
|
||||||
|
|
||||||
|
// Advisories is this process's log.
|
||||||
|
var Advisories = &AdvisoryLog{seen: map[string]*Advisory{}}
|
||||||
|
|
||||||
|
// Heard records one advisory.
|
||||||
|
func (l *AdvisoryLog) Heard(a Advisory, at time.Time) {
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
key := a.Kind + "/" + a.ID
|
||||||
|
if had, ok := l.seen[key]; ok {
|
||||||
|
had.Last, had.Said, had.Count = at, a.Said, had.Count+1
|
||||||
|
return
|
||||||
|
}
|
||||||
|
a.First, a.Last, a.Count = at, at, 1
|
||||||
|
l.seen[key] = &a
|
||||||
|
}
|
||||||
|
|
||||||
|
// Since is every advisory said at or after a moment, and forgets the older ones.
|
||||||
|
func (l *AdvisoryLog) Since(since time.Time) []Advisory {
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
var out []Advisory
|
||||||
|
for key, a := range l.seen {
|
||||||
|
if a.Last.Before(since) {
|
||||||
|
delete(l.seen, key)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, *a)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// jsAdvisory is the part of a JetStream advisory the mesh reads.
|
||||||
|
type jsAdvisory struct {
|
||||||
|
Type string `json:"type"`
|
||||||
|
Stream string `json:"stream"`
|
||||||
|
Consumer string `json:"consumer"`
|
||||||
|
StreamSeq uint64 `json:"stream_seq"`
|
||||||
|
Deliveries uint64 `json:"deliveries"`
|
||||||
|
Action string `json:"action"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadAdvisory is one JetStream advisory as the mesh says it; false for one it does not watch.
|
||||||
|
func ReadAdvisory(subject string, body []byte) (Advisory, bool) {
|
||||||
|
var a jsAdvisory
|
||||||
|
if err := json.Unmarshal(body, &a); err != nil || a.Stream == "" || a.Consumer == "" {
|
||||||
|
return Advisory{}, false
|
||||||
|
}
|
||||||
|
who := ConsumerInWords(a.Stream, a.Consumer)
|
||||||
|
id := a.Stream + "." + a.Consumer
|
||||||
|
switch {
|
||||||
|
case strings.HasPrefix(subject, "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES."):
|
||||||
|
return Advisory{Kind: AdvisoryMaxDeliveries, ID: id, Stream: a.Stream, Consumer: a.Consumer,
|
||||||
|
Said: fmt.Sprintf("%s handed message %d over %d times and gave up on it: it will not be delivered "+
|
||||||
|
"again, and what it asked for was not done", who, a.StreamSeq, a.Deliveries)}, true
|
||||||
|
case strings.HasPrefix(subject, "$JS.EVENT.ADVISORY.CONSUMER.DELETED."):
|
||||||
|
if !MeshNamed(a.Stream, a.Consumer) {
|
||||||
|
// A reader's own consumer, gone when it finished — every watch of a bucket and every
|
||||||
|
// read-back of a stream makes one, many a minute: not something the mesh defines.
|
||||||
|
return Advisory{}, false
|
||||||
|
}
|
||||||
|
return Advisory{Kind: AdvisoryConsumerLost, ID: id, Stream: a.Stream, Consumer: a.Consumer,
|
||||||
|
Said: fmt.Sprintf("%s was deleted from the bus", who)}, true
|
||||||
|
}
|
||||||
|
return Advisory{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// MeshNamed says a consumer is one of the durable consumers the mesh defines, by its name's shape:
|
||||||
|
// the controller's own, a machine's declaration consumer, a module's (`<node>_<module>`), a seat's
|
||||||
|
// worker. Every other consumer is a reader's own — an ordered consumer, a bucket's watcher — named at
|
||||||
|
// random by the client library, deleted when the read is done, and not the mesh's to say anything of.
|
||||||
|
func MeshNamed(stream, name string) bool {
|
||||||
|
switch {
|
||||||
|
case strings.HasPrefix(stream, "KV_") || stream == broker.AssignmentsStream:
|
||||||
|
return false // the mesh defines no durable consumer on a bucket's stream, or the memberships'
|
||||||
|
case stream == "NODES":
|
||||||
|
return true
|
||||||
|
case strings.HasPrefix(stream, "SEAT_"):
|
||||||
|
return strings.HasSuffix(name, "_worker")
|
||||||
|
case name == broker.ControllerName:
|
||||||
|
return true
|
||||||
|
case stream == broker.EventsStream:
|
||||||
|
return strings.Contains(name, "_")
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// ConsumerInWords is a durable consumer as the mesh says it: whose, and for what.
|
||||||
|
func ConsumerInWords(stream, name string) string {
|
||||||
|
switch {
|
||||||
|
case name == broker.ControllerName:
|
||||||
|
return "the controller's consumer on " + stream
|
||||||
|
case stream == "NODES":
|
||||||
|
return "how " + name + " hears what it should be (its declaration consumer)"
|
||||||
|
case strings.HasPrefix(stream, "SEAT_") && strings.HasSuffix(name, "_worker"):
|
||||||
|
seat := strings.ToLower(strings.ReplaceAll(strings.TrimPrefix(stream, "SEAT_"), "_", "-"))
|
||||||
|
return "the worker every holder of " + seat + " takes its asks from"
|
||||||
|
case stream == broker.EventsStream:
|
||||||
|
if node, module, ok := strings.Cut(name, "_"); ok {
|
||||||
|
return "how " + module + " on " + node + " hears what it consumes"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "the consumer " + name + " on " + stream
|
||||||
|
}
|
||||||
|
|
||||||
|
// HearAdvisories subscribes what the bus says about the mesh's account, and listens for what it
|
||||||
|
// tells this connection, until the returned function is called. Read-only: nothing is published.
|
||||||
|
func (s *Server) HearAdvisories(logf func(string, ...any)) (func(), error) {
|
||||||
|
if s.js == nil {
|
||||||
|
return nil, errors.New("this control plane is not on the bus, so it cannot hear what the bus says")
|
||||||
|
}
|
||||||
|
conn := s.js.Conn()
|
||||||
|
var subs []*nats.Subscription
|
||||||
|
stop := func() {
|
||||||
|
for _, sub := range subs {
|
||||||
|
_ = sub.Unsubscribe()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, subject := range broker.BusAdvisories {
|
||||||
|
sub, err := conn.Subscribe(subject, func(m *nats.Msg) {
|
||||||
|
if a, ok := ReadAdvisory(m.Subject, m.Data); ok {
|
||||||
|
Advisories.Heard(a, time.Now())
|
||||||
|
logf("the bus says: %s", a.Said)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
stop()
|
||||||
|
return nil, fmt.Errorf("listening to what the bus says (%s): %w", subject, err)
|
||||||
|
}
|
||||||
|
subs = append(subs, sub)
|
||||||
|
}
|
||||||
|
WatchConnection(conn, logf)
|
||||||
|
return stop, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// WatchConnection records what the bus tells this connection about itself: it fell behind, or a
|
||||||
|
// subject was refused it. Chained before whatever handler the connection had, which still runs. A
|
||||||
|
// refused answer to a call is the call log's to say (issue 265), and is not said twice.
|
||||||
|
func WatchConnection(conn *nats.Conn, logf func(string, ...any)) {
|
||||||
|
before := conn.ErrorHandler()
|
||||||
|
conn.SetErrorHandler(func(c *nats.Conn, sub *nats.Subscription, err error) {
|
||||||
|
if a, ok := connectionAdvisory(sub, err); ok {
|
||||||
|
Advisories.Heard(a, time.Now())
|
||||||
|
logf("the bus says: %s", a.Said)
|
||||||
|
}
|
||||||
|
if before != nil {
|
||||||
|
before(c, sub, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// connectionAdvisory is an error the bus handed the controller's connection, as an advisory.
|
||||||
|
func connectionAdvisory(sub *nats.Subscription, err error) (Advisory, bool) {
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, nats.ErrSlowConsumer):
|
||||||
|
subject := "a subscription"
|
||||||
|
if sub != nil {
|
||||||
|
subject = sub.Subject
|
||||||
|
}
|
||||||
|
return Advisory{Kind: AdvisorySlowConsumer, ID: "controller",
|
||||||
|
Said: fmt.Sprintf("the controller fell behind on %s and the client dropped messages it was sent", subject)}, true
|
||||||
|
case errors.Is(err, nats.ErrPermissionViolation):
|
||||||
|
if m := refusedPublish.FindStringSubmatch(err.Error()); m != nil && strings.HasPrefix(m[1], "_INBOX.") &&
|
||||||
|
!strings.HasPrefix(m[1], "_INBOX.enrol.") {
|
||||||
|
return Advisory{}, false // a refused answer to a call, which the call log says against its call
|
||||||
|
}
|
||||||
|
return Advisory{Kind: AdvisoryRefused, ID: "controller",
|
||||||
|
Said: "the bus refused the controller: " + err.Error()}, true
|
||||||
|
}
|
||||||
|
return Advisory{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Refusals of a publish, by subject, for a caller waiting on an answer to it.
|
||||||
|
var refusalWaiters = struct {
|
||||||
|
sync.Mutex
|
||||||
|
hooked map[*nats.Conn]bool
|
||||||
|
by map[string][]chan error
|
||||||
|
}{hooked: map[*nats.Conn]bool{}, by: map[string][]chan error{}}
|
||||||
|
|
||||||
|
// refusalsOf is told when the bus refuses this connection a publish to subject, until stop is called.
|
||||||
|
// The connection's error handler is chained once, before whatever it had, which still runs.
|
||||||
|
func refusalsOf(conn *nats.Conn, subject string) (<-chan error, func()) {
|
||||||
|
ch := make(chan error, 1)
|
||||||
|
refusalWaiters.Lock()
|
||||||
|
defer refusalWaiters.Unlock()
|
||||||
|
if !refusalWaiters.hooked[conn] {
|
||||||
|
refusalWaiters.hooked[conn] = true
|
||||||
|
before := conn.ErrorHandler()
|
||||||
|
conn.SetErrorHandler(func(c *nats.Conn, sub *nats.Subscription, err error) {
|
||||||
|
if m := refusedPublish.FindStringSubmatch(errString(err)); m != nil {
|
||||||
|
refusalWaiters.Lock()
|
||||||
|
for _, w := range refusalWaiters.by[m[1]] {
|
||||||
|
select {
|
||||||
|
case w <- err:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
refusalWaiters.Unlock()
|
||||||
|
}
|
||||||
|
if before != nil {
|
||||||
|
before(c, sub, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
refusalWaiters.by[subject] = append(refusalWaiters.by[subject], ch)
|
||||||
|
return ch, func() {
|
||||||
|
refusalWaiters.Lock()
|
||||||
|
defer refusalWaiters.Unlock()
|
||||||
|
waiting := refusalWaiters.by[subject]
|
||||||
|
for i, w := range waiting {
|
||||||
|
if w == ch {
|
||||||
|
refusalWaiters.by[subject] = append(waiting[:i], waiting[i+1:]...)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(refusalWaiters.by[subject]) == 0 {
|
||||||
|
delete(refusalWaiters.by, subject)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func errString(err error) string {
|
||||||
|
if err == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return err.Error()
|
||||||
|
}
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **A reader's own consumer gone is not a consumer lost**: every bucket watch and stream read-back
|
||||||
|
// makes and deletes one, many a minute, and the bus says so each time (found running the controller
|
||||||
|
// against a real bus: five a tick).
|
||||||
|
func TestOnlyTheMeshsOwnConsumersAreSaidLost(t *testing.T) {
|
||||||
|
deleted := func(stream, consumer string) bool {
|
||||||
|
_, ok := ReadAdvisory("$JS.EVENT.ADVISORY.CONSUMER.DELETED."+stream+"."+consumer,
|
||||||
|
[]byte(`{"type":"io.nats.jetstream.advisory.v1.consumer_action","stream":"`+stream+`","consumer":"`+consumer+`","action":"delete"}`))
|
||||||
|
return ok
|
||||||
|
}
|
||||||
|
for _, c := range [][2]string{{"EVENTS", "controller"}, {"CONTROL", "controller"}, {"NODES", "anchor"},
|
||||||
|
{"EVENTS", "anchor_shop"}, {"SEAT_NODE_BUILD_AGENT", "SEAT_NODE_BUILD_AGENT_worker"}} {
|
||||||
|
if !deleted(c[0], c[1]) {
|
||||||
|
t.Errorf("%s on %s deleted is not said", c[1], c[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, c := range [][2]string{{"KV_mesh-controller_conditions", "381UWW5Y"}, {"EVENTS", "E7vVYoe6"},
|
||||||
|
{"SEAT_NODE_BUILD_AGENT", "Rcnt6jla"}, {"ASSIGNMENTS", "x"}} {
|
||||||
|
if deleted(c[0], c[1]) {
|
||||||
|
t.Errorf("a reader's own consumer %s on %s is said lost", c[1], c[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a, ok := ReadAdvisory("$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.EVENTS.anchor_shop",
|
||||||
|
[]byte(`{"stream":"EVENTS","consumer":"anchor_shop","stream_seq":7,"deliveries":5}`))
|
||||||
|
if !ok || a.Kind != AdvisoryMaxDeliveries || a.ID != "EVENTS.anchor_shop" ||
|
||||||
|
a.Said != "how shop on anchor hears what it consumes handed message 7 over 5 times and gave up on it: "+
|
||||||
|
"it will not be delivered again, and what it asked for was not done" {
|
||||||
|
t.Fatalf("%+v", a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A question the bus refuses is answered as refused at once**, not after its timeout: against a
|
||||||
|
// server whose only user may not publish where it asks.
|
||||||
|
func TestNatsARefusedQuestionIsSaidAtOnce(t *testing.T) {
|
||||||
|
url := os.Getenv("MESH_TEST_NATS_REFUSING")
|
||||||
|
if url == "" {
|
||||||
|
t.Skip("MESH_TEST_NATS_REFUSING unset: a server whose user may not publish mesh.seat.>")
|
||||||
|
}
|
||||||
|
conn, err := nats.Connect(url)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
started := time.Now()
|
||||||
|
_, err = AskSeatTool(t.Context(), conn, "node-intrusion-prevention", "banned", "anchor", map[string]any{}, 10*time.Second)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "the bus refused") {
|
||||||
|
t.Fatalf("answered %v", err)
|
||||||
|
}
|
||||||
|
if took := time.Since(started); took > 3*time.Second {
|
||||||
|
t.Fatalf("a refusal took %s to be known", took)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The refusal reaches whoever waits on that subject, and only them.
|
||||||
|
func TestNatsARefusalReachesItsWaiter(t *testing.T) {
|
||||||
|
url := os.Getenv("MESH_TEST_NATS")
|
||||||
|
if url == "" {
|
||||||
|
t.Skip("MESH_TEST_NATS unset")
|
||||||
|
}
|
||||||
|
conn, err := nats.Connect(url)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
mine, stop := refusalsOf(conn, "mesh.seat.s.tool.v.anchor")
|
||||||
|
defer stop()
|
||||||
|
other, stopOther := refusalsOf(conn, "mesh.seat.s.tool.v.laptop")
|
||||||
|
defer stopOther()
|
||||||
|
conn.ErrorHandler()(conn, nil, fmt.Errorf("%w: Permissions Violation for Publish to %q",
|
||||||
|
nats.ErrPermissionViolation, "mesh.seat.s.tool.v.anchor"))
|
||||||
|
select {
|
||||||
|
case <-mine:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("the waiter was not told")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-other:
|
||||||
|
t.Fatal("another subject's waiter was told")
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -71,6 +71,10 @@ const (
|
|||||||
// next heartbeat, and a stream of them is the mesh's least valuable message competing for
|
// next heartbeat, and a stream of them is the mesh's least valuable message competing for
|
||||||
// retention with its most valuable (design 25 §3).
|
// retention with its most valuable (design 25 §3).
|
||||||
AliveSubjects = "mesh.control.*.alive"
|
AliveSubjects = "mesh.control.*.alive"
|
||||||
|
|
||||||
|
// ToolsAliveSubjects is every machine's node tools saying they are there (novox/hq to-be 45 §3,
|
||||||
|
// S11): core NATS like the host's, for the same reason.
|
||||||
|
ToolsAliveSubjects = "mesh.control.*.tools-alive"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ReportSubject is where one node says what it did. On the CONTROL stream, because it is the
|
// ReportSubject is where one node says what it did. On the CONTROL stream, because it is the
|
||||||
@@ -80,6 +84,9 @@ func ReportSubject(node string) string { return "mesh.control." + node + ".repor
|
|||||||
// AliveSubject is one node's heartbeat.
|
// AliveSubject is one node's heartbeat.
|
||||||
func AliveSubject(node string) string { return "mesh.control." + node + ".alive" }
|
func AliveSubject(node string) string { return "mesh.control." + node + ".alive" }
|
||||||
|
|
||||||
|
// ToolsAliveSubject is one machine's node tools' heartbeat.
|
||||||
|
func ToolsAliveSubject(node string) string { return "mesh.control." + node + ".tools-alive" }
|
||||||
|
|
||||||
// EventSubject is where a module's event lands. Derived from the emitter, never taken from the
|
// EventSubject is where a module's event lands. Derived from the emitter, never taken from the
|
||||||
// caller: a source that could differ from the subject is an envelope that can lie about its
|
// caller: a source that could differ from the subject is an envelope that can lie about its
|
||||||
// origin, and on NATS the account's permissions make the subject the authority (design 29 §2).
|
// origin, and on NATS the account's permissions make the subject the authority (design 29 §2).
|
||||||
|
|||||||
+218
-13
@@ -7,7 +7,9 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"regexp"
|
"regexp"
|
||||||
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -33,8 +35,9 @@ import (
|
|||||||
// either. A variable so a test need not wait.
|
// either. A variable so a test need not wait.
|
||||||
var AnswerWithin = 10 * time.Second
|
var AnswerWithin = 10 * time.Second
|
||||||
|
|
||||||
// KeptCalls is how many calls are kept, newest first; keptAnswer the largest answer kept of a call
|
// KeptCalls is how many calls this process keeps in memory, newest first — to match a refusal to its
|
||||||
// whose caller was sent it. One its caller never had is kept whole.
|
// call, and to answer at once; the bus keeps the last thousand (Durably). keptAnswer is the largest
|
||||||
|
// answer kept of a call whose caller was sent it. One its caller never had is kept whole.
|
||||||
const (
|
const (
|
||||||
KeptCalls = 100
|
KeptCalls = 100
|
||||||
keptAnswer = 64 << 10
|
keptAnswer = 64 << 10
|
||||||
@@ -47,6 +50,10 @@ const (
|
|||||||
// CallFinishedAfter is a call that finished after its caller was told it was still running: its
|
// CallFinishedAfter is a call that finished after its caller was told it was still running: its
|
||||||
// answer is here and nowhere else.
|
// answer is here and nowhere else.
|
||||||
CallFinishedAfter = "finished after its caller was answered"
|
CallFinishedAfter = "finished after its caller was answered"
|
||||||
|
// CallAbandoned is a call whose controller stopped before it finished (novox/hq to-be 45 §6): a
|
||||||
|
// controller starting finds it running under another and says so, rather than leaving it running
|
||||||
|
// for ever in the record.
|
||||||
|
CallAbandoned = "abandoned: the controller running it stopped before it finished"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Call is one call of a role's tool, as `calls` shows it.
|
// Call is one call of a role's tool, as `calls` shows it.
|
||||||
@@ -65,10 +72,27 @@ type Call struct {
|
|||||||
// Refused is the bus refusing the answer this holder sent: the caller got nothing, and this is
|
// Refused is the bus refusing the answer this holder sent: the caller got nothing, and this is
|
||||||
// the only place that says what it would have.
|
// the only place that says what it would have.
|
||||||
Refused string `json:"answer refused by the bus,omitempty"`
|
Refused string `json:"answer refused by the bus,omitempty"`
|
||||||
|
// Caller is the bus principal that asked, read from the inbox its answer went to — every principal
|
||||||
|
// is granted only its own (novox/hq to-be 45 §7: a hand act says who).
|
||||||
|
Caller string `json:"caller,omitempty"`
|
||||||
|
// Holder is the controller process that served it, so one starting can tell its own running
|
||||||
|
// calls from those a stopped one left.
|
||||||
|
Holder string `json:"holder,omitempty"`
|
||||||
|
|
||||||
reply string // the subject the answer went to, which the bus names when it refuses it
|
reply string // the subject the answer went to, which the bus names when it refuses it
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CallKeeper keeps calls where the process serving them does not: the controller's bucket on the
|
||||||
|
// bus (novox/hq to-be 45 §6). A call is kept whole on every change — begun, finished, refused — so a
|
||||||
|
// controller replaced at any moment leaves the last word on each.
|
||||||
|
type CallKeeper interface {
|
||||||
|
Keep(ctx context.Context, c Call) error
|
||||||
|
// Kept is one call with its whole answer.
|
||||||
|
Kept(ctx context.Context, id string) (Call, bool, error)
|
||||||
|
// Recent is the kept calls, newest first, without their answers.
|
||||||
|
Recent(ctx context.Context) ([]Call, error)
|
||||||
|
}
|
||||||
|
|
||||||
// CallLog keeps the latest calls a holder served.
|
// CallLog keeps the latest calls a holder served.
|
||||||
type CallLog struct {
|
type CallLog struct {
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
@@ -78,6 +102,15 @@ type CallLog struct {
|
|||||||
// Follow is the tool that reads this log back, named in a running answer — set by a holder that
|
// Follow is the tool that reads this log back, named in a running answer — set by a holder that
|
||||||
// serves one (the controller's `calls`); without it, the answer points at the holder's journal.
|
// serves one (the controller's `calls`); without it, the answer points at the holder's journal.
|
||||||
Follow string
|
Follow string
|
||||||
|
|
||||||
|
// keeper keeps every call beyond this process, when Durably was given one; writes go through
|
||||||
|
// one goroutine, in order, so a call's last state is the one kept.
|
||||||
|
keeper CallKeeper
|
||||||
|
holder string
|
||||||
|
writes chan Call
|
||||||
|
logger *log.Logger
|
||||||
|
lost int // writes the keeper could not take, said once each
|
||||||
|
keepErr error
|
||||||
}
|
}
|
||||||
|
|
||||||
// Calls is this process's log: one holder process serves its seats on one connection.
|
// Calls is this process's log: one holder process serves its seats on one connection.
|
||||||
@@ -85,16 +118,130 @@ var Calls = NewCallLog()
|
|||||||
|
|
||||||
func NewCallLog() *CallLog { return &CallLog{now: time.Now} }
|
func NewCallLog() *CallLog { return &CallLog{now: time.Now} }
|
||||||
|
|
||||||
|
// keepTries is how many times one call's state is offered to the keeper before it is said lost: the
|
||||||
|
// bus reloading its user list refuses for a moment, and that is exactly when a push runs.
|
||||||
|
const keepTries = 5
|
||||||
|
|
||||||
|
// Durably keeps every call from now on with keeper as well as in memory, under this process's name,
|
||||||
|
// and marks running the calls a controller before this one left running: it stopped, so they cannot
|
||||||
|
// finish (novox/hq to-be 45 §6). Said, naming each.
|
||||||
|
func (l *CallLog) Durably(ctx context.Context, keeper CallKeeper, holder string, logger *log.Logger) error {
|
||||||
|
l.mu.Lock()
|
||||||
|
l.keeper, l.holder, l.logger = keeper, holder, logger
|
||||||
|
if l.writes == nil {
|
||||||
|
l.writes = make(chan Call, 256)
|
||||||
|
go l.keepWrites()
|
||||||
|
}
|
||||||
|
l.mu.Unlock()
|
||||||
|
kept, err := keeper.Recent(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("reading the calls kept on the bus: %w", err)
|
||||||
|
}
|
||||||
|
for _, c := range kept {
|
||||||
|
if c.State != CallRunning || c.Holder == holder {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
whole, found, err := keeper.Kept(ctx, c.ID)
|
||||||
|
if err != nil || !found {
|
||||||
|
whole = c
|
||||||
|
}
|
||||||
|
whole.State = CallAbandoned
|
||||||
|
if err := keeper.Keep(ctx, whole); err != nil {
|
||||||
|
return fmt.Errorf("marking %s abandoned: %w", c.ID, err)
|
||||||
|
}
|
||||||
|
if logger != nil {
|
||||||
|
logger.Printf("%s (%s.%s, asked %s by %s) was running under %s, which stopped: marked abandoned — "+
|
||||||
|
"it may have done part of what it was asked, and nothing will finish it", c.ID, c.Seat, c.Verb,
|
||||||
|
c.Started.Format(time.RFC3339), orSomebody(c.Caller), orSomebody(c.Holder))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func orSomebody(s string) string {
|
||||||
|
if s == "" {
|
||||||
|
return "an unnamed caller"
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// keep queues one call's state for the keeper. Never blocks a call: a queue that is full is a keeper
|
||||||
|
// that is not taking writes, and that is said rather than waited on.
|
||||||
|
func (l *CallLog) keep(c Call) {
|
||||||
|
if l.writes == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case l.writes <- c:
|
||||||
|
default:
|
||||||
|
l.lost++
|
||||||
|
if l.logger != nil {
|
||||||
|
l.logger.Printf("%s (%s.%s) is kept in memory only: the bus is not taking calls' records (%d not kept)",
|
||||||
|
c.ID, c.Seat, c.Verb, l.lost)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *CallLog) keepWrites() {
|
||||||
|
for c := range l.writes {
|
||||||
|
var err error
|
||||||
|
for try := 0; try < keepTries; try++ {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
err = l.keeper.Keep(ctx, c)
|
||||||
|
cancel()
|
||||||
|
if err == nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
time.Sleep(time.Duration(try+1) * time.Second)
|
||||||
|
}
|
||||||
|
if err != nil && l.logger != nil {
|
||||||
|
l.logger.Printf("%s (%s.%s, %s) could not be kept on the bus after %d tries: %v — `calls` "+
|
||||||
|
"answers it from memory until this controller stops", c.ID, c.Seat, c.Verb, c.State, keepTries, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// callerOf is the bus principal an answer goes to: every principal's inbox is `_INBOX.<its user>.`
|
||||||
|
// followed by the client's own random token, and a user may itself hold dots.
|
||||||
|
func callerOf(reply string) string {
|
||||||
|
rest, ok := strings.CutPrefix(reply, "_INBOX.")
|
||||||
|
if !ok {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
tokens := strings.Split(rest, ".")
|
||||||
|
for i, t := range tokens {
|
||||||
|
if i > 0 && isNUID(t) {
|
||||||
|
return strings.Join(tokens[:i], ".")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// isNUID is the client library's random inbox token: twenty-two letters and digits.
|
||||||
|
func isNUID(t string) bool {
|
||||||
|
if len(t) != 22 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, r := range t {
|
||||||
|
if !(r >= '0' && r <= '9' || r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z') {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
func (l *CallLog) begin(seat, verb string, args json.RawMessage, reply string) *Call {
|
func (l *CallLog) begin(seat, verb string, args json.RawMessage, reply string) *Call {
|
||||||
l.mu.Lock()
|
l.mu.Lock()
|
||||||
defer l.mu.Unlock()
|
defer l.mu.Unlock()
|
||||||
l.next++
|
l.next++
|
||||||
c := &Call{ID: "call-" + strconv.FormatInt(l.now().UnixNano(), 10) + "-" + strconv.FormatUint(l.next, 10),
|
c := &Call{ID: "call-" + strconv.FormatInt(l.now().UnixNano(), 10) + "-" + strconv.FormatUint(l.next, 10),
|
||||||
Seat: seat, Verb: verb, Args: args, Started: l.now(), State: CallRunning, reply: reply}
|
Seat: seat, Verb: verb, Args: args, Started: l.now(), State: CallRunning, reply: reply,
|
||||||
|
Caller: callerOf(reply), Holder: l.holder}
|
||||||
l.calls = append(l.calls, c)
|
l.calls = append(l.calls, c)
|
||||||
if len(l.calls) > KeptCalls {
|
if len(l.calls) > KeptCalls {
|
||||||
l.calls = l.calls[len(l.calls)-KeptCalls:]
|
l.calls = l.calls[len(l.calls)-KeptCalls:]
|
||||||
}
|
}
|
||||||
|
l.keep(*c)
|
||||||
return c
|
return c
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -117,29 +264,84 @@ func (l *CallLog) finish(c *Call, answer []byte, failed, answeredAlready bool) {
|
|||||||
} else {
|
} else {
|
||||||
c.State = CallAnswered
|
c.State = CallAnswered
|
||||||
}
|
}
|
||||||
|
l.keep(*c)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Recent is the kept calls, newest first, as copies.
|
// Recent is the kept calls, newest first, as copies: this process's from memory, and, when they are
|
||||||
func (l *CallLog) Recent() []Call {
|
// kept durably, every other the bus holds — a call a controller before this one served included.
|
||||||
|
// Memory wins for a call in both, being the newer word on it. A bus that cannot be read is said in
|
||||||
|
// the error beside what memory holds, never answered as no calls.
|
||||||
|
// Running is every call this process is serving that has not finished, oldest first, without
|
||||||
|
// answers: what the watchdog of a call's bound (novox/hq to-be 45 S7) reads. This process's own,
|
||||||
|
// because a call another controller left running is said abandoned when this one starts.
|
||||||
|
func (l *CallLog) Running() []Call {
|
||||||
l.mu.Lock()
|
l.mu.Lock()
|
||||||
defer l.mu.Unlock()
|
defer l.mu.Unlock()
|
||||||
out := make([]Call, 0, len(l.calls))
|
var out []Call
|
||||||
for i := len(l.calls) - 1; i >= 0; i-- {
|
for _, c := range l.calls {
|
||||||
out = append(out, *l.calls[i])
|
if c.State == CallRunning {
|
||||||
|
running := *c
|
||||||
|
running.Answer = nil
|
||||||
|
out = append(out, running)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get is one kept call.
|
func (l *CallLog) Recent() ([]Call, error) {
|
||||||
func (l *CallLog) Get(id string) (Call, bool) {
|
l.mu.Lock()
|
||||||
|
out := make([]Call, 0, len(l.calls))
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for i := len(l.calls) - 1; i >= 0; i-- {
|
||||||
|
out = append(out, *l.calls[i])
|
||||||
|
seen[l.calls[i].ID] = true
|
||||||
|
}
|
||||||
|
keeper := l.keeper
|
||||||
|
l.mu.Unlock()
|
||||||
|
if keeper == nil {
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
kept, err := keeper.Recent(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return out, fmt.Errorf("the calls kept on the bus could not be read, so only this controller's own "+
|
||||||
|
"are listed: %w", err)
|
||||||
|
}
|
||||||
|
for _, c := range kept {
|
||||||
|
if !seen[c.ID] {
|
||||||
|
out = append(out, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.SliceStable(out, func(i, j int) bool { return out[i].Started.After(out[j].Started) })
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get is one kept call: from memory, or from the bus when this process did not serve it.
|
||||||
|
func (l *CallLog) Get(id string) (Call, bool, error) {
|
||||||
l.mu.Lock()
|
l.mu.Lock()
|
||||||
defer l.mu.Unlock()
|
|
||||||
for _, c := range l.calls {
|
for _, c := range l.calls {
|
||||||
if c.ID == id {
|
if c.ID == id {
|
||||||
return *c, true
|
found := *c
|
||||||
|
l.mu.Unlock()
|
||||||
|
return found, true, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return Call{}, false
|
keeper := l.keeper
|
||||||
|
l.mu.Unlock()
|
||||||
|
if keeper == nil {
|
||||||
|
return Call{}, false, nil
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
return keeper.Kept(ctx, id)
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsDurable says whether calls outlive this process.
|
||||||
|
func (l *CallLog) IsDurable() bool {
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
return l.keeper != nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// kept is what a call's arguments are kept as: each argument by name, a value only when it is short
|
// kept is what a call's arguments are kept as: each argument by name, a value only when it is short
|
||||||
@@ -195,6 +397,7 @@ func (l *CallLog) Refusal(err error, logger *log.Logger) bool {
|
|||||||
at := l.now()
|
at := l.now()
|
||||||
hit.Refused = fmt.Sprintf("%s: %s", at.Format(time.RFC3339), err)
|
hit.Refused = fmt.Sprintf("%s: %s", at.Format(time.RFC3339), err)
|
||||||
id, verb, took := hit.ID, hit.Seat+"."+hit.Verb, at.Sub(hit.Started).Round(time.Second)
|
id, verb, took := hit.ID, hit.Seat+"."+hit.Verb, at.Sub(hit.Started).Round(time.Second)
|
||||||
|
l.keep(*hit)
|
||||||
l.mu.Unlock()
|
l.mu.Unlock()
|
||||||
if logger != nil {
|
if logger != nil {
|
||||||
// Said in the mesh's words, beside the library's own line: which call, and where its answer is.
|
// Said in the mesh's words, beside the library's own line: which call, and where its answer is.
|
||||||
@@ -277,6 +480,8 @@ func (l *CallLog) serveCall(seat, verb string, args json.RawMessage, reply strin
|
|||||||
args = json.RawMessage(`{}`)
|
args = json.RawMessage(`{}`)
|
||||||
}
|
}
|
||||||
c := l.begin(seat, verb, kept(args), reply)
|
c := l.begin(seat, verb, kept(args), reply)
|
||||||
|
// Who asked travels with the call, so an act it does by hand says so (novox/hq to-be 45 §7).
|
||||||
|
ctx = context.WithValue(ctx, callerKey{}, c.Caller)
|
||||||
acknowledged := make(chan struct{})
|
acknowledged := make(chan struct{})
|
||||||
var once sync.Once
|
var once sync.Once
|
||||||
ctx = context.WithValue(ctx, answerNowKey{}, func() { once.Do(func() { close(acknowledged) }) })
|
ctx = context.WithValue(ctx, answerNowKey{}, func() { once.Do(func() { close(acknowledged) }) })
|
||||||
|
|||||||
@@ -0,0 +1,115 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Calls kept on the bus (novox/hq to-be 45 §6).
|
||||||
|
//
|
||||||
|
// **Two keys a call**: `<id>` holds the record — verb, arguments as kept, caller, state, times — and
|
||||||
|
// `<id>.answer` the answer, bounded. A listing watches the records alone, so reading the last thousand
|
||||||
|
// calls reads the last thousand small records and not a thousand answers; one call asked by id reads
|
||||||
|
// both. A call's id is one token, so the record's key never holds a dot and `*` matches records only.
|
||||||
|
|
||||||
|
// BusCalls keeps calls in the controller's calls bucket.
|
||||||
|
type BusCalls struct {
|
||||||
|
kv jetstream.KeyValue
|
||||||
|
}
|
||||||
|
|
||||||
|
// CallsOnTheBus opens the calls bucket the controller asserts at its start.
|
||||||
|
func CallsOnTheBus(ctx context.Context, conn *nats.Conn) (*BusCalls, error) {
|
||||||
|
api, err := jetstream.New(conn)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
kv, err := api.KeyValue(ctx, broker.CallsBucket)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the calls bucket %s is not on the bus — the controller asserts it at its "+
|
||||||
|
"start, so one older than this has not: %w", broker.CallsBucket, err)
|
||||||
|
}
|
||||||
|
return &BusCalls{kv: kv}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const answerKey = ".answer"
|
||||||
|
|
||||||
|
// Keep writes a call's record, and its answer when it has one.
|
||||||
|
func (b *BusCalls) Keep(ctx context.Context, c Call) error {
|
||||||
|
answer := c.Answer
|
||||||
|
c.Answer = nil
|
||||||
|
if len(answer) > 0 {
|
||||||
|
if len(answer) > broker.CallAnswerBytes {
|
||||||
|
answer, _ = json.Marshal(map[string]any{"cut": fmt.Sprintf("an answer of %d bytes; the first %d "+
|
||||||
|
"are kept", len(answer), broker.CallAnswerBytes), "start": string(answer[:broker.CallAnswerBytes])})
|
||||||
|
}
|
||||||
|
// The answer before the record, so a record saying a call finished never points at an answer
|
||||||
|
// not yet written.
|
||||||
|
if _, err := b.kv.Put(ctx, c.ID+answerKey, answer); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
record, err := json.Marshal(c)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = b.kv.Put(ctx, c.ID, record)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Kept is one call, with its answer.
|
||||||
|
func (b *BusCalls) Kept(ctx context.Context, id string) (Call, bool, error) {
|
||||||
|
entry, err := b.kv.Get(ctx, id)
|
||||||
|
if errors.Is(err, jetstream.ErrKeyNotFound) || errors.Is(err, jetstream.ErrInvalidKey) {
|
||||||
|
return Call{}, false, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return Call{}, false, err
|
||||||
|
}
|
||||||
|
var c Call
|
||||||
|
if err := json.Unmarshal(entry.Value(), &c); err != nil {
|
||||||
|
return Call{}, false, fmt.Errorf("the record of %s on the bus is not a call: %w", id, err)
|
||||||
|
}
|
||||||
|
answer, err := b.kv.Get(ctx, id+answerKey)
|
||||||
|
switch {
|
||||||
|
case err == nil:
|
||||||
|
c.Answer = json.RawMessage(answer.Value())
|
||||||
|
case !errors.Is(err, jetstream.ErrKeyNotFound):
|
||||||
|
return Call{}, false, err
|
||||||
|
}
|
||||||
|
return c, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recent is every call the bucket holds, newest first, without answers: read once through a watch
|
||||||
|
// of the records, which hands over the current value of each and then says it has.
|
||||||
|
func (b *BusCalls) Recent(ctx context.Context) ([]Call, error) {
|
||||||
|
w, err := b.kv.Watch(ctx, "*", jetstream.IgnoreDeletes())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer func() { _ = w.Stop() }()
|
||||||
|
var out []Call
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, fmt.Errorf("reading the calls bucket: %w", ctx.Err())
|
||||||
|
case entry := <-w.Updates():
|
||||||
|
if entry == nil {
|
||||||
|
// Every current value handed over.
|
||||||
|
sort.SliceStable(out, func(i, j int) bool { return out[i].Started.After(out[j].Started) })
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
var c Call
|
||||||
|
if json.Unmarshal(entry.Value(), &c) == nil && c.ID != "" {
|
||||||
|
out = append(out, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,152 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"log"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The calls bucket against a real server (novox/hq to-be 45 §6): whether a call's outcome is
|
||||||
|
// readable by id from a controller that did not serve it is a claim about what the bus keeps.
|
||||||
|
|
||||||
|
func callsBucket(t *testing.T) *BusCalls {
|
||||||
|
t.Helper()
|
||||||
|
js := aBus(t)
|
||||||
|
api, err := jetstream.New(js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = api.DeleteKeyValue(t.Context(), broker.CallsBucket)
|
||||||
|
if err := js.EnsureControllerBuckets(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
keeper, err := CallsOnTheBus(t.Context(), js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return keeper
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitKept waits until the keeper holds a call in a state, the writes being queued.
|
||||||
|
func waitKept(t *testing.T, keeper CallKeeper, id, state string) Call {
|
||||||
|
t.Helper()
|
||||||
|
deadline := time.Now().Add(5 * time.Second)
|
||||||
|
for {
|
||||||
|
c, found, err := keeper.Kept(context.Background(), id)
|
||||||
|
if err == nil && found && c.State == state {
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatalf("%s never kept as %q: %+v %v %v", id, state, c, found, err)
|
||||||
|
}
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A controller restart keeps every call's outcome** (to-be 45 Phase 0): a call one controller
|
||||||
|
// answered is read whole by id from the next, and a call it left running is said abandoned rather
|
||||||
|
// than running for ever.
|
||||||
|
func TestNatsACallsOutcomeOutlivesItsController(t *testing.T) {
|
||||||
|
keeper := callsBucket(t)
|
||||||
|
first := NewCallLog()
|
||||||
|
if err := first.Durably(t.Context(), keeper, "controller@one", nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
a := newAnswers(t)
|
||||||
|
first.serveCall("mesh-controller", "push", json.RawMessage(`{"node":"anchor","values":"secret"}`),
|
||||||
|
"_INBOX.node-tools.g14.Pe3sGzAtv8jUBYQ6sKSvKz.1",
|
||||||
|
func(context.Context, json.RawMessage) (any, error) { return "anchor told", nil }, a.respond, nil)
|
||||||
|
recent, _ := first.Recent()
|
||||||
|
finished := waitKept(t, keeper, recent[0].ID, CallAnswered)
|
||||||
|
// A second call, still running when its controller stops.
|
||||||
|
left := first.begin("mesh-controller", "plans", json.RawMessage(`{}`), "")
|
||||||
|
waitKept(t, keeper, left.ID, CallRunning)
|
||||||
|
|
||||||
|
var said bytes.Buffer
|
||||||
|
second := NewCallLog()
|
||||||
|
if err := second.Durably(t.Context(), keeper, "controller@two", log.New(&said, "", 0)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
c, found, err := second.Get(finished.ID)
|
||||||
|
if err != nil || !found {
|
||||||
|
t.Fatalf("the next controller cannot read %s: %v %v", finished.ID, found, err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(c.Answer), "anchor told") || c.Caller != "node-tools.g14" || c.Holder != "controller@one" {
|
||||||
|
t.Fatalf("kept %+v", c)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(c.Args), "secret") {
|
||||||
|
t.Fatalf("a setting was kept: %s", c.Args)
|
||||||
|
}
|
||||||
|
abandoned, _, _ := second.Get(left.ID)
|
||||||
|
if abandoned.State != CallAbandoned || !strings.Contains(said.String(), left.ID) {
|
||||||
|
t.Fatalf("a call left running reads %q, and was said: %q", abandoned.State, said.String())
|
||||||
|
}
|
||||||
|
listed, err := second.Recent()
|
||||||
|
if err != nil || len(listed) != 2 {
|
||||||
|
t.Fatalf("the next controller lists %d calls: %v", len(listed), err)
|
||||||
|
}
|
||||||
|
// Its own running calls are not its predecessor's: a controller starting again under the same
|
||||||
|
// name leaves them alone.
|
||||||
|
mine := second.begin("mesh-controller", "status", nil, "")
|
||||||
|
waitKept(t, keeper, mine.ID, CallRunning)
|
||||||
|
if err := second.Durably(t.Context(), keeper, "controller@two", nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if c, _, _ := keeper.Kept(t.Context(), mine.ID); c.State != CallRunning {
|
||||||
|
t.Fatalf("a controller marked its own running call %q", c.State)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The bucket holds the last thousand calls or fourteen days: a call is two keys, so the stream under
|
||||||
|
// it holds twice as many messages, and asserting it again keeps the count.
|
||||||
|
func TestNatsTheCallsBucketIsBounded(t *testing.T) {
|
||||||
|
callsBucket(t)
|
||||||
|
js := aBus(t)
|
||||||
|
if err := js.EnsureControllerBuckets(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
info, err := js.Context().StreamInfo("KV_" + broker.CallsBucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if info.Config.MaxMsgs != 2*broker.KeptCallsDurably || info.Config.MaxAge != broker.CallsKeptFor {
|
||||||
|
t.Fatalf("kept %d messages for %s", info.Config.MaxMsgs, info.Config.MaxAge)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An answer larger than the bound is cut and says so, and the record is still written.
|
||||||
|
func TestNatsAnAnswerLargerThanTheBoundIsCut(t *testing.T) {
|
||||||
|
keeper := callsBucket(t)
|
||||||
|
big, _ := json.Marshal(map[string]string{"result": strings.Repeat("x", broker.CallAnswerBytes+10)})
|
||||||
|
c := Call{ID: "call-1-1", Seat: "mesh-controller", Verb: "plan", State: CallAnswered, Answer: big}
|
||||||
|
if err := keeper.Keep(t.Context(), c); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, found, err := keeper.Kept(t.Context(), "call-1-1")
|
||||||
|
if err != nil || !found || !strings.Contains(string(got.Answer), "the first") {
|
||||||
|
t.Fatalf("%v %v %.200s", found, err, got.Answer)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Who asked is read from the inbox the answer goes to.
|
||||||
|
func TestTheCallerIsTheInboxsPrincipal(t *testing.T) {
|
||||||
|
for reply, want := range map[string]string{
|
||||||
|
"_INBOX.node-tools.g14.Pe3sGzAtv8jUBYQ6sKSvKz.1": "node-tools.g14",
|
||||||
|
"_INBOX.jochen.Pe3sGzAtv8jUBYQ6sKSvKz": "jochen",
|
||||||
|
"_INBOX.x.1": "",
|
||||||
|
"mesh.control.one": "",
|
||||||
|
"": "",
|
||||||
|
} {
|
||||||
|
if got := callerOf(reply); got != want {
|
||||||
|
t.Errorf("%q: %q, want %q", reply, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -62,7 +62,7 @@ func TestACallThatFinishesInTimeAnswersInFull(t *testing.T) {
|
|||||||
if got := a.only()["result"]; got != "all well" {
|
if got := a.only()["result"]; got != "all well" {
|
||||||
t.Fatalf("answered %v", got)
|
t.Fatalf("answered %v", got)
|
||||||
}
|
}
|
||||||
recent := l.Recent()
|
recent, _ := l.Recent()
|
||||||
if len(recent) != 1 || recent[0].State != CallAnswered || string(recent[0].Args) != "{}" {
|
if len(recent) != 1 || recent[0].State != CallAnswered || string(recent[0].Args) != "{}" {
|
||||||
t.Fatalf("kept %+v", recent)
|
t.Fatalf("kept %+v", recent)
|
||||||
}
|
}
|
||||||
@@ -90,12 +90,12 @@ func TestACallThatOutlastsTheWindowSaysItIsRunningAndKeepsItsAnswer(t *testing.T
|
|||||||
if got["running"] != true || id == "" || !strings.Contains(got["output"].(string), id) {
|
if got["running"] != true || id == "" || !strings.Contains(got["output"].(string), id) {
|
||||||
t.Fatalf("the running answer does not name its call: %v", got)
|
t.Fatalf("the running answer does not name its call: %v", got)
|
||||||
}
|
}
|
||||||
if c, _ := l.Get(id); c.State != CallRunning {
|
if c, _, _ := l.Get(id); c.State != CallRunning {
|
||||||
t.Fatalf("while it runs it is kept as %q", c.State)
|
t.Fatalf("while it runs it is kept as %q", c.State)
|
||||||
}
|
}
|
||||||
close(release)
|
close(release)
|
||||||
<-finished
|
<-finished
|
||||||
c, ok := l.Get(id)
|
c, ok, _ := l.Get(id)
|
||||||
if !ok || c.State != CallFinishedAfter || !strings.Contains(string(c.Answer), "anchor told") {
|
if !ok || c.State != CallFinishedAfter || !strings.Contains(string(c.Answer), "anchor told") {
|
||||||
t.Fatalf("its answer was not kept: %+v", c)
|
t.Fatalf("its answer was not kept: %+v", c)
|
||||||
}
|
}
|
||||||
@@ -125,7 +125,7 @@ func TestAnAcknowledgedCallIsAnsweredBeforeItGoesOn(t *testing.T) {
|
|||||||
if got := a.only()["result"].(map[string]any); got["running"] != true {
|
if got := a.only()["result"].(map[string]any); got["running"] != true {
|
||||||
t.Fatalf("an acknowledged call answered %v", got)
|
t.Fatalf("an acknowledged call answered %v", got)
|
||||||
}
|
}
|
||||||
if c := l.Recent()[0]; c.State != CallFinishedAfter || !strings.Contains(string(c.Answer), "sent") {
|
if c := recentOf(l)[0]; c.State != CallFinishedAfter || !strings.Contains(string(c.Answer), "sent") {
|
||||||
t.Fatalf("kept %+v", c)
|
t.Fatalf("kept %+v", c)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -143,7 +143,7 @@ func TestARefusedAnswerIsKeptAgainstItsCall(t *testing.T) {
|
|||||||
if !l.Refusal(refusal, log.New(&logged, "", 0)) {
|
if !l.Refusal(refusal, log.New(&logged, "", 0)) {
|
||||||
t.Fatal("the refusal of a kept call's answer was not recognised")
|
t.Fatal("the refusal of a kept call's answer was not recognised")
|
||||||
}
|
}
|
||||||
c := l.Recent()[0]
|
c := recentOf(l)[0]
|
||||||
if c.Refused == "" || !strings.Contains(logged.String(), c.ID) {
|
if c.Refused == "" || !strings.Contains(logged.String(), c.ID) {
|
||||||
t.Fatalf("the refusal is not kept or not said: %+v / %q", c, logged.String())
|
t.Fatalf("the refusal is not kept or not said: %+v / %q", c, logged.String())
|
||||||
}
|
}
|
||||||
@@ -167,8 +167,13 @@ func TestTheLogKeepsTheNewest(t *testing.T) {
|
|||||||
for i := 0; i < KeptCalls+5; i++ {
|
for i := 0; i < KeptCalls+5; i++ {
|
||||||
l.begin("s", "v", nil, "")
|
l.begin("s", "v", nil, "")
|
||||||
}
|
}
|
||||||
recent := l.Recent()
|
recent, _ := l.Recent()
|
||||||
if len(recent) != KeptCalls || !strings.HasSuffix(recent[0].ID, "-105") || !strings.HasSuffix(recent[KeptCalls-1].ID, "-6") {
|
if len(recent) != KeptCalls || !strings.HasSuffix(recent[0].ID, "-105") || !strings.HasSuffix(recent[KeptCalls-1].ID, "-6") {
|
||||||
t.Fatalf("kept %d, newest %s, oldest %s", len(recent), recent[0].ID, recent[len(recent)-1].ID)
|
t.Fatalf("kept %d, newest %s, oldest %s", len(recent), recent[0].ID, recent[len(recent)-1].ID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func recentOf(l *CallLog) []Call {
|
||||||
|
out, _ := l.Recent()
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|||||||
@@ -380,6 +380,10 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err
|
|||||||
if report.Applied == nil && report.Refused == "" && len(report.Failed) == 0 {
|
if report.Applied == nil && report.Refused == "" && len(report.Failed) == 0 {
|
||||||
if report.Superseded != "" {
|
if report.Superseded != "" {
|
||||||
log.Printf("%s set aside declaration %s for the newer %s", report.Node, report.Declared, report.Superseded)
|
log.Printf("%s set aside declaration %s for the newer %s", report.Node, report.Declared, report.Superseded)
|
||||||
|
} else if err := e.Inventory.RecordHeartbeatGap(ctx, report.Node, node.LastSeen); err != nil {
|
||||||
|
// The silence before this word, which the machine-silent bound will be set from (novox/hq
|
||||||
|
// to-be 45 Phase 0). A measurement lost is said and costs the report nothing.
|
||||||
|
log.Printf("the silence before %s's word could not be recorded: %v", report.Node, err)
|
||||||
}
|
}
|
||||||
return false, e.Inventory.Seen(ctx, node.ID)
|
return false, e.Inventory.Seen(ctx, node.ID)
|
||||||
}
|
}
|
||||||
@@ -435,6 +439,11 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
|
// And how long the machine took, from the send to this first account of it (novox/hq to-be 45
|
||||||
|
// Phase 0): what the sent-not-reported bound will be set from. Said if lost, never a failure.
|
||||||
|
if err := e.Inventory.RecordApplyDuration(ctx, report.Node, report.Declared, doing.Outcome); err != nil {
|
||||||
|
log.Printf("how long %s took to apply could not be recorded: %v", report.Node, err)
|
||||||
|
}
|
||||||
|
|
||||||
// A refusal, a failure, or a bare word that the node is there — none of them is an account of
|
// A refusal, a failure, or a bare word that the node is there — none of them is an account of
|
||||||
// what the machine holds, so each moves last_seen and nothing else. Recording a partial list
|
// what the machine holds, so each moves last_seen and nothing else. Recording a partial list
|
||||||
|
|||||||
@@ -0,0 +1,162 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The hand-act log (novox/hq to-be 45 §7).
|
||||||
|
//
|
||||||
|
// **A repair a person makes by hand is the record of a healer the mesh does not have yet.** Tonight's
|
||||||
|
// pushes of one machine, plans closed because a report never came, a consumer re-made because it fell
|
||||||
|
// a week behind — each was done, and the only trace was a chat. So every verb that repairs by hand
|
||||||
|
// asks why, and writes one entry: who, which verb and arguments, why, when, the condition it
|
||||||
|
// addresses if one is named, and a cause — a word that, recorded twice in a fortnight, says a healer
|
||||||
|
// is wanted (S15, from Phase 3). `hand-act record` is the same entry for an act done outside the mesh.
|
||||||
|
// The controller is the bucket's only writer; its verbs are the way in.
|
||||||
|
|
||||||
|
// HandAct is one entry.
|
||||||
|
type HandAct struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
// By is who: the bus principal a seat call came from, or the account and machine at a shell.
|
||||||
|
By string `json:"by"`
|
||||||
|
// Verb and Args are the act as given: `push`, `plans close`, `broker consumer-reset`, or
|
||||||
|
// `hand-act record` with what was done outside the mesh.
|
||||||
|
Verb string `json:"verb"`
|
||||||
|
Args []string `json:"arguments,omitempty"`
|
||||||
|
Why string `json:"why"`
|
||||||
|
// Cause is the condition kind, or a word the person gives; the verb's own name when neither.
|
||||||
|
Cause string `json:"cause"`
|
||||||
|
// Condition is the condition's key the act addresses, when it names one.
|
||||||
|
Condition string `json:"condition,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// CallerVar carries a seat call's caller to the command the controller runs for it, so an act done
|
||||||
|
// through the console says who asked rather than "the controller".
|
||||||
|
const CallerVar = "MESH_CALLER"
|
||||||
|
|
||||||
|
// Caller is who is acting in this process: the seat call's caller when the controller ran it for
|
||||||
|
// one, otherwise the account and machine at the shell.
|
||||||
|
func Caller() string {
|
||||||
|
if c := strings.TrimSpace(os.Getenv(CallerVar)); c != "" {
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
user := os.Getenv("USER")
|
||||||
|
if user == "" {
|
||||||
|
user = "an unnamed account"
|
||||||
|
}
|
||||||
|
host, _ := os.Hostname()
|
||||||
|
return fmt.Sprintf("%s at a shell on %s", user, host)
|
||||||
|
}
|
||||||
|
|
||||||
|
type callerKey struct{}
|
||||||
|
|
||||||
|
// CallerIn is the caller of the seat call ctx belongs to, empty outside one.
|
||||||
|
func CallerIn(ctx context.Context) string {
|
||||||
|
c, _ := ctx.Value(callerKey{}).(string)
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
var handActSeq atomic.Uint64
|
||||||
|
|
||||||
|
// RecordHandAct writes one entry. Its key is its time and a sequence, so the bucket lists in order.
|
||||||
|
func RecordHandAct(ctx context.Context, conn *nats.Conn, act HandAct) (HandAct, error) {
|
||||||
|
if strings.TrimSpace(act.Why) == "" {
|
||||||
|
return act, errors.New("an act by hand says why: --why <text>")
|
||||||
|
}
|
||||||
|
if act.At.IsZero() {
|
||||||
|
act.At = time.Now().UTC()
|
||||||
|
}
|
||||||
|
if act.ID == "" {
|
||||||
|
act.ID = "act-" + strconv.FormatInt(act.At.UnixNano(), 10) + "-" + strconv.FormatUint(handActSeq.Add(1), 10)
|
||||||
|
}
|
||||||
|
if act.By == "" {
|
||||||
|
act.By = Caller()
|
||||||
|
}
|
||||||
|
if act.Cause == "" {
|
||||||
|
act.Cause = act.Verb
|
||||||
|
}
|
||||||
|
kv, err := handActs(ctx, conn)
|
||||||
|
if err != nil {
|
||||||
|
return act, err
|
||||||
|
}
|
||||||
|
body, err := json.Marshal(act)
|
||||||
|
if err != nil {
|
||||||
|
return act, err
|
||||||
|
}
|
||||||
|
_, err = kv.Put(ctx, act.ID, body)
|
||||||
|
return act, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// HandActs is every entry since a moment, oldest first.
|
||||||
|
func HandActs(ctx context.Context, conn *nats.Conn, since time.Time) ([]HandAct, error) {
|
||||||
|
kv, err := handActs(ctx, conn)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
w, err := kv.WatchAll(ctx, jetstream.IgnoreDeletes())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer func() { _ = w.Stop() }()
|
||||||
|
var out []HandAct
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, fmt.Errorf("reading the hand-act log: %w", ctx.Err())
|
||||||
|
case entry := <-w.Updates():
|
||||||
|
if entry == nil {
|
||||||
|
sort.SliceStable(out, func(i, j int) bool { return out[i].At.Before(out[j].At) })
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
var a HandAct
|
||||||
|
if json.Unmarshal(entry.Value(), &a) == nil && !a.At.Before(since) {
|
||||||
|
out = append(out, a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// RepeatedCauses are the causes recorded more than once within the fortnight before now, with how
|
||||||
|
// often: each is a repair done by hand again, which is what S15 will raise as a healer wanted.
|
||||||
|
func RepeatedCauses(acts []HandAct, now time.Time) map[string]int {
|
||||||
|
counts := map[string]int{}
|
||||||
|
for _, a := range acts {
|
||||||
|
if now.Sub(a.At) <= 14*24*time.Hour {
|
||||||
|
counts[a.Cause]++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for c, n := range counts {
|
||||||
|
if n < 2 {
|
||||||
|
delete(counts, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return counts
|
||||||
|
}
|
||||||
|
|
||||||
|
func handActs(ctx context.Context, conn *nats.Conn) (jetstream.KeyValue, error) {
|
||||||
|
api, err := jetstream.New(conn)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
kv, err := api.KeyValue(ctx, broker.HandActsBucket)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the hand-act log %s is not on the bus — the controller asserts it at its "+
|
||||||
|
"start, so one older than this has not: %w", broker.HandActsBucket, err)
|
||||||
|
}
|
||||||
|
return kv, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The hand-act log against a real server (novox/hq to-be 45 §7): an act is written with who, why and
|
||||||
|
// its cause, read back in order, and a cause recorded twice within a fortnight is found.
|
||||||
|
func TestNatsAnActByHandIsKeptWithWhyAndARepeatIsFound(t *testing.T) {
|
||||||
|
js := aBus(t)
|
||||||
|
api, err := jetstream.New(js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = api.DeleteKeyValue(t.Context(), broker.HandActsBucket)
|
||||||
|
if err := js.EnsureControllerBuckets(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Setenv(CallerVar, "node-tools.g14, through the mesh-controller seat")
|
||||||
|
if _, err := RecordHandAct(t.Context(), js.Conn(), HandAct{Verb: "push", Args: []string{"anchor"}}); err == nil {
|
||||||
|
t.Fatal("an act without why was written")
|
||||||
|
}
|
||||||
|
first, err := RecordHandAct(t.Context(), js.Conn(), HandAct{Verb: "push", Args: []string{"anchor"},
|
||||||
|
Why: "it never reported the send", Cause: "sent-not-reported"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if first.By != "node-tools.g14, through the mesh-controller seat" || first.ID == "" {
|
||||||
|
t.Fatalf("written as %+v", first)
|
||||||
|
}
|
||||||
|
if _, err := RecordHandAct(t.Context(), js.Conn(), HandAct{Verb: "plans close", Args: []string{"plan-1"},
|
||||||
|
Why: "waiting on the same report"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := RecordHandAct(t.Context(), js.Conn(), HandAct{Verb: "push", Args: []string{"ace"},
|
||||||
|
Why: "again", Cause: "sent-not-reported", At: time.Now().UTC().Add(time.Second)}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
acts, err := HandActs(t.Context(), js.Conn(), time.Now().Add(-time.Hour))
|
||||||
|
if err != nil || len(acts) != 3 || acts[0].ID != first.ID || acts[1].Cause != "plans close" {
|
||||||
|
t.Fatalf("%v %+v", err, acts)
|
||||||
|
}
|
||||||
|
repeated := RepeatedCauses(acts, time.Now())
|
||||||
|
if len(repeated) != 1 || repeated["sent-not-reported"] != 2 {
|
||||||
|
t.Fatalf("repeated %v", repeated)
|
||||||
|
}
|
||||||
|
if old, _ := HandActs(t.Context(), js.Conn(), time.Now().Add(time.Hour)); len(old) != 0 {
|
||||||
|
t.Fatalf("acts before the moment asked were listed: %+v", old)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(acts[2].ID, "act-") {
|
||||||
|
t.Fatalf("%q", acts[2].ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -138,6 +138,20 @@ type Signed struct {
|
|||||||
// is current.
|
// is current.
|
||||||
type Alive struct {
|
type Alive struct {
|
||||||
Node string `json:"node"`
|
Node string `json:"node"`
|
||||||
|
// IntervalSeconds is how often the node says it is there (novox/hq to-be 45 §3, S1): the
|
||||||
|
// watchdog's bound is three of them. Zero from a host older than that, which is read as the
|
||||||
|
// interval hosts have always used.
|
||||||
|
IntervalSeconds int `json:"interval_seconds,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ToolsAlive is a machine's node tools saying they are there (novox/hq to-be 45 §3, S11): the
|
||||||
|
// runtime every module's tools and every held seat's verbs are served by. Its own word, apart from the
|
||||||
|
// host's, because a host heard and a runtime gone is a machine nobody can ask anything.
|
||||||
|
type ToolsAlive struct {
|
||||||
|
Node string `json:"node"`
|
||||||
|
IntervalSeconds int `json:"interval_seconds,omitempty"`
|
||||||
|
// Version is the runtime's build, as it says it.
|
||||||
|
Version string `json:"version,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Report is what a node states after applying. It states; the owning context writes.
|
// Report is what a node states after applying. It states; the owning context writes.
|
||||||
|
|||||||
+24
-1
@@ -413,7 +413,30 @@ func AskSeatTool(ctx context.Context, conn *nats.Conn, seat, verb, node string,
|
|||||||
}
|
}
|
||||||
asking, cancel := context.WithTimeout(ctx, timeout)
|
asking, cancel := context.WithTimeout(ctx, timeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
reply, err := conn.RequestWithContext(asking, NodeSeatToolSubject(seat, verb, node), body)
|
subject := NodeSeatToolSubject(seat, verb, node)
|
||||||
|
// **A refused question is known at once** (novox/hq to-be 45, D8 live on 2026-10-06): the bus
|
||||||
|
// says it to the connection and the request would otherwise wait out its whole timeout, reading
|
||||||
|
// as a machine that did not answer.
|
||||||
|
refused, stop := refusalsOf(conn, subject)
|
||||||
|
defer stop()
|
||||||
|
type replied struct {
|
||||||
|
msg *nats.Msg
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
done := make(chan replied, 1)
|
||||||
|
go func() {
|
||||||
|
msg, err := conn.RequestWithContext(asking, subject, body)
|
||||||
|
done <- replied{msg, err}
|
||||||
|
}()
|
||||||
|
var reply *nats.Msg
|
||||||
|
select {
|
||||||
|
case r := <-done:
|
||||||
|
reply, err = r.msg, r.err
|
||||||
|
case why := <-refused:
|
||||||
|
cancel()
|
||||||
|
return Answer{}, fmt.Errorf("the bus refused the controller asking %s.%s of %s — its grants do not "+
|
||||||
|
"name %s: %v", seat, verb, node, subject, why)
|
||||||
|
}
|
||||||
switch {
|
switch {
|
||||||
case errors.Is(err, nats.ErrNoResponders):
|
case errors.Is(err, nats.ErrNoResponders):
|
||||||
return Answer{}, fmt.Errorf("nothing on %s answers %s.%s: its holder is not running, or is "+
|
return Answer{}, fmt.Errorf("nothing on %s answers %s.%s: its holder is not running, or is "+
|
||||||
|
|||||||
@@ -28,6 +28,8 @@ const (
|
|||||||
KindEnrolment = "enrolment"
|
KindEnrolment = "enrolment"
|
||||||
KindReport = "report"
|
KindReport = "report"
|
||||||
KindHeartbeat = "heartbeat"
|
KindHeartbeat = "heartbeat"
|
||||||
|
// KindToolsHeartbeat is a machine's node tools saying they are there (novox/hq to-be 45 S11).
|
||||||
|
KindToolsHeartbeat = "tools-heartbeat"
|
||||||
KindBuilt = "built"
|
KindBuilt = "built"
|
||||||
KindModuleMoved = "module-moved"
|
KindModuleMoved = "module-moved"
|
||||||
// KindSourceMoved is the forge announcing a merge: a source moved, and what it produces is
|
// KindSourceMoved is the forge announcing a merge: a source moved, and what it produces is
|
||||||
|
|||||||
@@ -89,6 +89,10 @@ func (n *natsInbound) Receive(ctx context.Context, act func(context.Context, Con
|
|||||||
return nil // stopped while standing by
|
return nil // stopped while standing by
|
||||||
}
|
}
|
||||||
defer func() { _ = said.Unsubscribe() }()
|
defer func() { _ = said.Unsubscribe() }()
|
||||||
|
// This controller is the one acting now: its watchdogs and self-check may say what they see
|
||||||
|
// (novox/hq to-be 45 §3). One standing by hears nothing, and would call every machine silent.
|
||||||
|
holding.Store(true)
|
||||||
|
defer holding.Store(false)
|
||||||
|
|
||||||
// Heartbeats, on core NATS and off any stream (design 25 §3). Their own subscription because
|
// Heartbeats, on core NATS and off any stream (design 25 §3). Their own subscription because
|
||||||
// they are their own guarantee: a lost one is the next one.
|
// they are their own guarantee: a lost one is the next one.
|
||||||
@@ -98,6 +102,12 @@ func (n *natsInbound) Receive(ctx context.Context, act func(context.Context, Con
|
|||||||
return fmt.Errorf("subscribing to heartbeats: %w", err)
|
return fmt.Errorf("subscribing to heartbeats: %w", err)
|
||||||
}
|
}
|
||||||
defer func() { _ = alive.Unsubscribe() }()
|
defer func() { _ = alive.Unsubscribe() }()
|
||||||
|
// And each machine's node tools, on the same channel: as cheap, and lost the same way.
|
||||||
|
toolsAlive, err := conn.ChanSubscribe(ToolsAliveSubjects, beats)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("subscribing to the node tools' heartbeats: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = toolsAlive.Unsubscribe() }()
|
||||||
|
|
||||||
// The events the controller follows, when something is listening for them.
|
// The events the controller follows, when something is listening for them.
|
||||||
var events chan *nats.Msg
|
var events chan *nats.Msg
|
||||||
@@ -159,6 +169,8 @@ func (n *natsInbound) deliver(ctx context.Context, act func(context.Context, Con
|
|||||||
}
|
}
|
||||||
m := &natsControl{kind: kind, msg: msg, on: n}
|
m := &natsControl{kind: kind, msg: msg, on: n}
|
||||||
if streamed {
|
if streamed {
|
||||||
|
// The event loop took one: what S4 watches (novox/hq to-be 45 §3).
|
||||||
|
Loop.Took(time.Now())
|
||||||
// A message with no metadata is not from a stream, whatever it was delivered on, and the
|
// A message with no metadata is not from a stream, whatever it was delivered on, and the
|
||||||
// window has nothing to hold it by. Said by leaving the sequence at zero.
|
// window has nothing to hold it by. Said by leaving the sequence at zero.
|
||||||
if meta, err := msg.Metadata(); err == nil {
|
if meta, err := msg.Metadata(); err == nil {
|
||||||
@@ -225,6 +237,8 @@ func kindOfSubject(subject string) (string, bool) {
|
|||||||
return KindReport, true
|
return KindReport, true
|
||||||
case "alive":
|
case "alive":
|
||||||
return KindHeartbeat, true
|
return KindHeartbeat, true
|
||||||
|
case "tools-alive":
|
||||||
|
return KindToolsHeartbeat, true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
switch subject {
|
switch subject {
|
||||||
|
|||||||
@@ -170,6 +170,8 @@ func (s *Server) act(ctx context.Context, m Control) {
|
|||||||
s.reported(ctx, m)
|
s.reported(ctx, m)
|
||||||
case KindHeartbeat:
|
case KindHeartbeat:
|
||||||
s.heartbeat(m)
|
s.heartbeat(m)
|
||||||
|
case KindToolsHeartbeat:
|
||||||
|
s.toolsHeartbeat(m)
|
||||||
case KindBuilt:
|
case KindBuilt:
|
||||||
s.wasBuilt(ctx, m)
|
s.wasBuilt(ctx, m)
|
||||||
case KindModuleMoved:
|
case KindModuleMoved:
|
||||||
@@ -268,6 +270,8 @@ func (s *Server) heartbeat(m Control) {
|
|||||||
_ = m.Drop()
|
_ = m.Drop()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
// The interval it says, for the watchdog's bound (novox/hq to-be 45 S1).
|
||||||
|
HostBeats.Heard(alive.Node, time.Now(), time.Duration(alive.IntervalSeconds)*time.Second, "")
|
||||||
if s.listener != nil {
|
if s.listener != nil {
|
||||||
if _, err := s.listener.Heard(context.Background(), Report{Node: alive.Node}); err != nil {
|
if _, err := s.listener.Heard(context.Background(), Report{Node: alive.Node}); err != nil {
|
||||||
s.log.Printf("could not record that %s is here: %v", alive.Node, err)
|
s.log.Printf("could not record that %s is here: %v", alive.Node, err)
|
||||||
@@ -276,6 +280,22 @@ func (s *Server) heartbeat(m Control) {
|
|||||||
_ = m.Took()
|
_ = m.Took()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// toolsHeartbeat records that a machine's node tools were heard from (novox/hq to-be 45 S11), and
|
||||||
|
// nothing else: in this process's memory, for the watchdog — the next one is a minute away.
|
||||||
|
func (s *Server) toolsHeartbeat(m Control) {
|
||||||
|
var alive ToolsAlive
|
||||||
|
if err := json.Unmarshal(m.Body(), &alive); err != nil || alive.Node == "" {
|
||||||
|
_ = m.Drop()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// The machine is the one in the subject the bus let the runtime publish on, never the body's.
|
||||||
|
if node, ok := nodeOfToolsAlive(m.Subject()); ok {
|
||||||
|
alive.Node = node
|
||||||
|
}
|
||||||
|
ToolsBeats.Heard(alive.Node, time.Now(), time.Duration(alive.IntervalSeconds)*time.Second, alive.Version)
|
||||||
|
_ = m.Took()
|
||||||
|
}
|
||||||
|
|
||||||
// reported records what a node says it did.
|
// reported records what a node says it did.
|
||||||
//
|
//
|
||||||
// A node states; nothing here writes anything the node claimed about itself beyond that it was
|
// A node states; nothing here writes anything the node claimed about itself beyond that it was
|
||||||
|
|||||||
@@ -0,0 +1,227 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What the serving controller hears that its watchdogs read (novox/hq to-be 45 §3).
|
||||||
|
//
|
||||||
|
// **In memory, on purpose.** A heartbeat is the least valuable message the mesh sends — the next one
|
||||||
|
// is a minute away — and what a watchdog needs of it is the newest moment and the interval it said.
|
||||||
|
// A machine's last word is kept in the store as well (`last_seen`), which is what S1 reads; these are
|
||||||
|
// what the store does not keep: the interval, the node tools' word, and when the event loop last took
|
||||||
|
// a message.
|
||||||
|
|
||||||
|
// Beat is one emitter's newest heartbeat.
|
||||||
|
type Beat struct {
|
||||||
|
At time.Time
|
||||||
|
// Every is the interval it said; zero when it said none.
|
||||||
|
Every time.Duration
|
||||||
|
Version string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Beats is the newest heartbeat of each machine, for one kind of emitter.
|
||||||
|
type Beats struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
started time.Time
|
||||||
|
heard map[string]Beat
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewBeats is an empty record, started now.
|
||||||
|
func NewBeats() *Beats { return &Beats{started: time.Now(), heard: map[string]Beat{}} }
|
||||||
|
|
||||||
|
// HostBeats are the node-engines' heartbeats (S1); ToolsBeats the node tools' (S11).
|
||||||
|
var (
|
||||||
|
HostBeats = NewBeats()
|
||||||
|
ToolsBeats = NewBeats()
|
||||||
|
)
|
||||||
|
|
||||||
|
// Heard records one heartbeat.
|
||||||
|
func (b *Beats) Heard(node string, at time.Time, every time.Duration, version string) {
|
||||||
|
b.mu.Lock()
|
||||||
|
defer b.mu.Unlock()
|
||||||
|
b.heard[node] = Beat{At: at, Every: every, Version: version}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Of is one machine's newest heartbeat; false when none was heard since this process started.
|
||||||
|
func (b *Beats) Of(node string) (Beat, bool) {
|
||||||
|
b.mu.Lock()
|
||||||
|
defer b.mu.Unlock()
|
||||||
|
beat, ok := b.heard[node]
|
||||||
|
return beat, ok
|
||||||
|
}
|
||||||
|
|
||||||
|
// Started is when this record began: a machine not heard since is silent since then at the most.
|
||||||
|
func (b *Beats) Started() time.Time {
|
||||||
|
b.mu.Lock()
|
||||||
|
defer b.mu.Unlock()
|
||||||
|
return b.started
|
||||||
|
}
|
||||||
|
|
||||||
|
// nodeOfToolsAlive is the machine a node tools' heartbeat names in its subject.
|
||||||
|
func nodeOfToolsAlive(subject string) (string, bool) {
|
||||||
|
rest, ok := strings.CutPrefix(subject, "mesh.control.")
|
||||||
|
if !ok {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
node, kind, ok := strings.Cut(rest, ".")
|
||||||
|
if !ok || kind != "tools-alive" || node == "" {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
return node, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// LoopActivity is when the controller's event loop last took a message from a stream (S4).
|
||||||
|
type LoopActivity struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
took time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// Loop is this process's event loop.
|
||||||
|
var Loop = &LoopActivity{}
|
||||||
|
|
||||||
|
// Took records that the loop was handed a message.
|
||||||
|
func (l *LoopActivity) Took(at time.Time) {
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
l.took = at
|
||||||
|
}
|
||||||
|
|
||||||
|
// Last is when the loop last took one; zero when it has not since this process started.
|
||||||
|
func (l *LoopActivity) Last() time.Time {
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
return l.took
|
||||||
|
}
|
||||||
|
|
||||||
|
// PowerState is what a machine last said about its power (novox/hq ADR 0211): `sleeping` or
|
||||||
|
// `shutting-down` until it says `booted` or `woke`.
|
||||||
|
type PowerState struct {
|
||||||
|
State string
|
||||||
|
At time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// PowerModule is the module whose events say a machine's power (ADR 0211), and PowerEvents its
|
||||||
|
// events that say whether the machine is about to be away or is back.
|
||||||
|
const PowerModule = "power"
|
||||||
|
|
||||||
|
var PowerEvents = map[string]bool{"sleeping": true, "shutting-down": true, "booted": true, "woke": true}
|
||||||
|
|
||||||
|
// PowerStates is each machine's newest word about its power since a moment, read back from the
|
||||||
|
// events stream on a consumer of its own that acknowledges nothing (as AnnouncedMerges reads). The
|
||||||
|
// machine is the one the runtime stamped on the event (`x-node`); an event without one names none.
|
||||||
|
func (s *Server) PowerStates(ctx context.Context, since time.Time) (map[string]PowerState, error) {
|
||||||
|
if s.js == nil {
|
||||||
|
return nil, errors.New("this control plane is not on the bus, so it cannot read what machines said of their power")
|
||||||
|
}
|
||||||
|
sub, err := s.js.Context().SubscribeSync(EventSubject(PowerModule, ">"), nats.OrderedConsumer(), nats.StartTime(since))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("reading what machines said of their power: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = sub.Unsubscribe() }()
|
||||||
|
out := map[string]PowerState{}
|
||||||
|
for {
|
||||||
|
wait, cancel := context.WithTimeout(ctx, readQuiet)
|
||||||
|
msg, err := sub.NextMsgWithContext(wait)
|
||||||
|
cancel()
|
||||||
|
if err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
meta, err := msg.Metadata()
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
state := strings.TrimPrefix(msg.Subject, "mesh.mod."+PowerModule+".event.")
|
||||||
|
node := msg.Header.Get("x-node")
|
||||||
|
if PowerEvents[state] && node != "" {
|
||||||
|
at := meta.Timestamp
|
||||||
|
var body struct {
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
}
|
||||||
|
if json.Unmarshal(msg.Data, &body) == nil && !body.At.IsZero() {
|
||||||
|
at = body.At
|
||||||
|
}
|
||||||
|
if before, ok := out[node]; !ok || !at.Before(before.At) {
|
||||||
|
out[node] = PowerState{State: state, At: at}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if meta.NumPending == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Away says whether a power state is a machine that said it would be away.
|
||||||
|
func (p PowerState) Away() bool { return p.State == "sleeping" || p.State == "shutting-down" }
|
||||||
|
|
||||||
|
// StaleRefusal is the node-engine's words for a declaration it refused because it is older than the
|
||||||
|
// one it holds (mesh-host `refuseOlder`, novox/hq issue 107): the receiver's refusal S13 counts.
|
||||||
|
const StaleRefusal = "is older than what the mesh last said to this node"
|
||||||
|
|
||||||
|
// IsStaleRefusal says a report's refusal is the node-engine refusing a declaration older than it holds.
|
||||||
|
func IsStaleRefusal(refused string) bool { return strings.Contains(refused, StaleRefusal) }
|
||||||
|
|
||||||
|
// RefusalCount keeps when each machine refused a stale declaration, for S13 (novox/hq to-be 45 §3):
|
||||||
|
// more than five from one writer in five minutes is a writer sending what it has moved past.
|
||||||
|
type RefusalCount struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
per map[string][]time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// StaleRefusals is this process's count.
|
||||||
|
var StaleRefusals = &RefusalCount{per: map[string][]time.Time{}}
|
||||||
|
|
||||||
|
// Refused records one refusal by a machine.
|
||||||
|
func (r *RefusalCount) Refused(node string, at time.Time) {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
r.per[node] = append(r.per[node], at)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Within is how many refusals each machine made since a moment; older ones are forgotten.
|
||||||
|
func (r *RefusalCount) Within(since time.Time) map[string]int {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
out := map[string]int{}
|
||||||
|
for node, times := range r.per {
|
||||||
|
var kept []time.Time
|
||||||
|
for _, t := range times {
|
||||||
|
if !t.Before(since) {
|
||||||
|
kept = append(kept, t)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(kept) == 0 {
|
||||||
|
delete(r.per, node)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
r.per[node] = kept
|
||||||
|
out[node] = len(kept)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// holding is whether this process holds the controller's consumer of what nodes say: the controller
|
||||||
|
// acting, not one standing by for another (issue 213). Until the lease (to-be 45 §6) it is how a
|
||||||
|
// watchdog knows it is the one that hears.
|
||||||
|
var holding atomic.Bool
|
||||||
|
|
||||||
|
// Holding says this process is the controller acting now.
|
||||||
|
func Holding() bool { return holding.Load() }
|
||||||
|
|
||||||
|
// JetStream is the connection the server consumes on.
|
||||||
|
func (s *Server) JetStream() *broker.JetStream { return s.js }
|
||||||
+9
@@ -0,0 +1,9 @@
|
|||||||
|
language: go
|
||||||
|
|
||||||
|
go:
|
||||||
|
- 1.x
|
||||||
|
- tip
|
||||||
|
|
||||||
|
matrix:
|
||||||
|
allow_failures:
|
||||||
|
- go: tip
|
||||||
+22
@@ -0,0 +1,22 @@
|
|||||||
|
Copyright (c) 2019 Jack Christensen
|
||||||
|
|
||||||
|
MIT License
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining
|
||||||
|
a copy of this software and associated documentation files (the
|
||||||
|
"Software"), to deal in the Software without restriction, including
|
||||||
|
without limitation the rights to use, copy, modify, merge, publish,
|
||||||
|
distribute, sublicense, and/or sell copies of the Software, and to
|
||||||
|
permit persons to whom the Software is furnished to do so, subject to
|
||||||
|
the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be
|
||||||
|
included in all copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||||
|
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||||
|
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||||
|
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
||||||
|
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
||||||
|
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||||
|
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||||
+8
@@ -0,0 +1,8 @@
|
|||||||
|
[](https://godoc.org/github.com/jackc/pgpassfile)
|
||||||
|
[](https://travis-ci.org/jackc/pgpassfile)
|
||||||
|
|
||||||
|
# pgpassfile
|
||||||
|
|
||||||
|
Package pgpassfile is a parser PostgreSQL .pgpass files.
|
||||||
|
|
||||||
|
Extracted and rewritten from original implementation in https://github.com/jackc/pgx.
|
||||||
+110
@@ -0,0 +1,110 @@
|
|||||||
|
// Package pgpassfile is a parser PostgreSQL .pgpass files.
|
||||||
|
package pgpassfile
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Entry represents a line in a PG passfile.
|
||||||
|
type Entry struct {
|
||||||
|
Hostname string
|
||||||
|
Port string
|
||||||
|
Database string
|
||||||
|
Username string
|
||||||
|
Password string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Passfile is the in memory data structure representing a PG passfile.
|
||||||
|
type Passfile struct {
|
||||||
|
Entries []*Entry
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadPassfile reads the file at path and parses it into a Passfile.
|
||||||
|
func ReadPassfile(path string) (*Passfile, error) {
|
||||||
|
f, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
|
||||||
|
return ParsePassfile(f)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParsePassfile reads r and parses it into a Passfile.
|
||||||
|
func ParsePassfile(r io.Reader) (*Passfile, error) {
|
||||||
|
passfile := &Passfile{}
|
||||||
|
|
||||||
|
scanner := bufio.NewScanner(r)
|
||||||
|
for scanner.Scan() {
|
||||||
|
entry := parseLine(scanner.Text())
|
||||||
|
if entry != nil {
|
||||||
|
passfile.Entries = append(passfile.Entries, entry)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return passfile, scanner.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Match (not colons or escaped colon or escaped backslash)+. Essentially gives a split on unescaped
|
||||||
|
// colon.
|
||||||
|
var colonSplitterRegexp = regexp.MustCompile("(([^:]|(\\:)))+")
|
||||||
|
|
||||||
|
// var colonSplitterRegexp = regexp.MustCompile("((?:[^:]|(?:\\:)|(?:\\\\))+)")
|
||||||
|
|
||||||
|
// parseLine parses a line into an *Entry. It returns nil on comment lines or any other unparsable
|
||||||
|
// line.
|
||||||
|
func parseLine(line string) *Entry {
|
||||||
|
const (
|
||||||
|
tmpBackslash = "\r"
|
||||||
|
tmpColon = "\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
|
||||||
|
if strings.HasPrefix(line, "#") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
line = strings.Replace(line, `\\`, tmpBackslash, -1)
|
||||||
|
line = strings.Replace(line, `\:`, tmpColon, -1)
|
||||||
|
|
||||||
|
parts := strings.Split(line, ":")
|
||||||
|
if len(parts) != 5 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unescape escaped colons and backslashes
|
||||||
|
for i := range parts {
|
||||||
|
parts[i] = strings.Replace(parts[i], tmpBackslash, `\`, -1)
|
||||||
|
parts[i] = strings.Replace(parts[i], tmpColon, `:`, -1)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &Entry{
|
||||||
|
Hostname: parts[0],
|
||||||
|
Port: parts[1],
|
||||||
|
Database: parts[2],
|
||||||
|
Username: parts[3],
|
||||||
|
Password: parts[4],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// FindPassword finds the password for the provided hostname, port, database, and username. For a
|
||||||
|
// Unix domain socket hostname must be set to "localhost". An empty string will be returned if no
|
||||||
|
// match is found.
|
||||||
|
//
|
||||||
|
// See https://www.postgresql.org/docs/current/libpq-pgpass.html for more password file information.
|
||||||
|
func (pf *Passfile) FindPassword(hostname, port, database, username string) (password string) {
|
||||||
|
for _, e := range pf.Entries {
|
||||||
|
if (e.Hostname == "*" || e.Hostname == hostname) &&
|
||||||
|
(e.Port == "*" || e.Port == port) &&
|
||||||
|
(e.Database == "*" || e.Database == database) &&
|
||||||
|
(e.Username == "*" || e.Username == username) {
|
||||||
|
return e.Password
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
+22
@@ -0,0 +1,22 @@
|
|||||||
|
Copyright (c) 2020 Jack Christensen
|
||||||
|
|
||||||
|
MIT License
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining
|
||||||
|
a copy of this software and associated documentation files (the
|
||||||
|
"Software"), to deal in the Software without restriction, including
|
||||||
|
without limitation the rights to use, copy, modify, merge, publish,
|
||||||
|
distribute, sublicense, and/or sell copies of the Software, and to
|
||||||
|
permit persons to whom the Software is furnished to do so, subject to
|
||||||
|
the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be
|
||||||
|
included in all copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||||
|
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||||
|
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||||
|
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
||||||
|
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
||||||
|
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||||
|
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||||
+7
@@ -0,0 +1,7 @@
|
|||||||
|
[](https://pkg.go.dev/github.com/jackc/pgservicefile)
|
||||||
|
[](https://github.com/jackc/pgservicefile/actions/workflows/ci.yml)
|
||||||
|
|
||||||
|
|
||||||
|
# pgservicefile
|
||||||
|
|
||||||
|
Package pgservicefile is a parser for PostgreSQL service files (e.g. `.pg_service.conf`).
|
||||||
+81
@@ -0,0 +1,81 @@
|
|||||||
|
// Package pgservicefile is a parser for PostgreSQL service files (e.g. .pg_service.conf).
|
||||||
|
package pgservicefile
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Service struct {
|
||||||
|
Name string
|
||||||
|
Settings map[string]string
|
||||||
|
}
|
||||||
|
|
||||||
|
type Servicefile struct {
|
||||||
|
Services []*Service
|
||||||
|
servicesByName map[string]*Service
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetService returns the named service.
|
||||||
|
func (sf *Servicefile) GetService(name string) (*Service, error) {
|
||||||
|
service, present := sf.servicesByName[name]
|
||||||
|
if !present {
|
||||||
|
return nil, errors.New("not found")
|
||||||
|
}
|
||||||
|
return service, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadServicefile reads the file at path and parses it into a Servicefile.
|
||||||
|
func ReadServicefile(path string) (*Servicefile, error) {
|
||||||
|
f, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
|
||||||
|
return ParseServicefile(f)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseServicefile reads r and parses it into a Servicefile.
|
||||||
|
func ParseServicefile(r io.Reader) (*Servicefile, error) {
|
||||||
|
servicefile := &Servicefile{}
|
||||||
|
|
||||||
|
var service *Service
|
||||||
|
scanner := bufio.NewScanner(r)
|
||||||
|
lineNum := 0
|
||||||
|
for scanner.Scan() {
|
||||||
|
lineNum += 1
|
||||||
|
line := scanner.Text()
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
|
||||||
|
if line == "" || strings.HasPrefix(line, "#") {
|
||||||
|
// ignore comments and empty lines
|
||||||
|
} else if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") {
|
||||||
|
service = &Service{Name: line[1 : len(line)-1], Settings: make(map[string]string)}
|
||||||
|
servicefile.Services = append(servicefile.Services, service)
|
||||||
|
} else if service != nil {
|
||||||
|
parts := strings.SplitN(line, "=", 2)
|
||||||
|
if len(parts) != 2 {
|
||||||
|
return nil, fmt.Errorf("unable to parse line %d", lineNum)
|
||||||
|
}
|
||||||
|
|
||||||
|
key := strings.TrimSpace(parts[0])
|
||||||
|
value := strings.TrimSpace(parts[1])
|
||||||
|
|
||||||
|
service.Settings[key] = value
|
||||||
|
} else {
|
||||||
|
return nil, fmt.Errorf("line %d is not in a section", lineNum)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
servicefile.servicesByName = make(map[string]*Service, len(servicefile.Services))
|
||||||
|
for _, service := range servicefile.Services {
|
||||||
|
servicefile.servicesByName[service.Name] = service
|
||||||
|
}
|
||||||
|
|
||||||
|
return servicefile, scanner.Err()
|
||||||
|
}
|
||||||
+27
@@ -0,0 +1,27 @@
|
|||||||
|
# Compiled Object files, Static and Dynamic libs (Shared Objects)
|
||||||
|
*.o
|
||||||
|
*.a
|
||||||
|
*.so
|
||||||
|
|
||||||
|
# Folders
|
||||||
|
_obj
|
||||||
|
_test
|
||||||
|
|
||||||
|
# Architecture specific extensions/prefixes
|
||||||
|
*.[568vq]
|
||||||
|
[568vq].out
|
||||||
|
|
||||||
|
*.cgo1.go
|
||||||
|
*.cgo2.c
|
||||||
|
_cgo_defun.c
|
||||||
|
_cgo_gotypes.go
|
||||||
|
_cgo_export.*
|
||||||
|
|
||||||
|
_testmain.go
|
||||||
|
|
||||||
|
*.exe
|
||||||
|
|
||||||
|
.envrc
|
||||||
|
/.testdb
|
||||||
|
|
||||||
|
.DS_Store
|
||||||
+28
@@ -0,0 +1,28 @@
|
|||||||
|
# See for configurations: https://golangci-lint.run/usage/configuration/
|
||||||
|
version: "2"
|
||||||
|
|
||||||
|
linters:
|
||||||
|
default: none
|
||||||
|
enable:
|
||||||
|
- govet
|
||||||
|
- ineffassign
|
||||||
|
- unconvert
|
||||||
|
- gocritic
|
||||||
|
|
||||||
|
# See: https://golangci-lint.run/usage/formatters/
|
||||||
|
formatters:
|
||||||
|
enable:
|
||||||
|
- gofmt # https://pkg.go.dev/cmd/gofmt
|
||||||
|
- gofumpt # https://github.com/mvdan/gofumpt
|
||||||
|
|
||||||
|
settings:
|
||||||
|
gofmt:
|
||||||
|
simplify: true # Simplify code: gofmt with `-s` option.
|
||||||
|
|
||||||
|
gofumpt:
|
||||||
|
# Module path which contains the source code being formatted.
|
||||||
|
# Default: ""
|
||||||
|
module-path: github.com/jackc/pgx/v5 # Should match with module in go.mod
|
||||||
|
# Choose whether to use the extra rules.
|
||||||
|
# Default: false
|
||||||
|
extra-rules: true
|
||||||
+605
@@ -0,0 +1,605 @@
|
|||||||
|
# 5.10.0 (June 3, 2026)
|
||||||
|
|
||||||
|
This release includes a significant amount of hardening against malicious or compromised PostgreSQL servers,
|
||||||
|
contributed by Sean Chittenden at CrowdStrike, Inc. This work bounds binary decoders against attacker-controlled
|
||||||
|
message sizes, caps server-supplied SCRAM iteration counts, adds `require_auth` to restrict which authentication
|
||||||
|
methods a server may use (mitigating downgrade attacks under `sslmode=prefer`), and ensures cancellation requests are
|
||||||
|
sent over TLS when the original connection used TLS.
|
||||||
|
|
||||||
|
## Features
|
||||||
|
|
||||||
|
* Add `require_auth` to restrict accepted server authentication methods (Sean Chittenden at CrowdStrike, Inc.)
|
||||||
|
* Add `ParseConfigOptions.ConnStringAllowedKeys` to restrict allowed connection string keys (Sean Chittenden at CrowdStrike, Inc.)
|
||||||
|
* Add `StructArgs` and `StrictStructArgs` for `@`-named queries (Tubelight30)
|
||||||
|
* Add `ErrConnClosed` sentinel error and unwrap it from `connLockError` (Charlie Tonneslan)
|
||||||
|
* pgxpool: check if connection is expired before acquire (arthurdotwork)
|
||||||
|
|
||||||
|
## Security Hardening
|
||||||
|
|
||||||
|
* Encrypt `CancelRequest` connection when the primary connection used TLS (Sean Chittenden at CrowdStrike, Inc.)
|
||||||
|
* Cap server-supplied SCRAM iteration count (Sean Chittenden at CrowdStrike, Inc.)
|
||||||
|
* Default Frontend max message body length to ~1 GiB (Sean Chittenden at CrowdStrike, Inc.)
|
||||||
|
* Bound hstore binary decode against malicious server input (Sean Chittenden at CrowdStrike, Inc.)
|
||||||
|
* Bound array binary decode element length against remaining message bytes (Sean Chittenden at CrowdStrike, Inc.)
|
||||||
|
* Bound array element count against remaining message bytes (Sean Chittenden at CrowdStrike, Inc.)
|
||||||
|
* Bound range, multirange, and tsvector binary decoders (Sean Chittenden at CrowdStrike, Inc.)
|
||||||
|
* Document secure connection configuration (Sean Chittenden at CrowdStrike, Inc.)
|
||||||
|
* Fix panic on malformed geometric text; return an error instead (MaIII)
|
||||||
|
|
||||||
|
## Fixes
|
||||||
|
|
||||||
|
* Fix scanning `"char"` (OID 18) into `*string` in binary format (luongs3)
|
||||||
|
* Fix handling of typed-nil `driver.Valuer` in array and composite codecs (Donncha Fahy)
|
||||||
|
* Fix `CopyData.Data` hex decoding in `UnmarshalJSON` (Charlie Tonneslan)
|
||||||
|
* Fix data race when context is cancelled during connect
|
||||||
|
* Fix `parseKeywordValueSettings` rejecting trailing whitespace (alliasgher)
|
||||||
|
* pgconn: preserve full error chain in `normalizeTimeoutError` (Charlie Tonneslan)
|
||||||
|
* pgconn: use a fresh context for the fallback connection in `connectPreferred` (Charlie Tonneslan)
|
||||||
|
* pgxpool: fix `MaxLifetimeDestroyCount` and ping order for acquire-time expiry check
|
||||||
|
* Add missing error check of `rows.Err` to load types (Jen Altavilla)
|
||||||
|
|
||||||
|
# 5.9.2 (April 18, 2026)
|
||||||
|
|
||||||
|
Fix SQL Injection via placeholder confusion with dollar quoted string literals (GHSA-j88v-2chj-qfwx)
|
||||||
|
|
||||||
|
SQL injection can occur when:
|
||||||
|
|
||||||
|
1. The non-default simple protocol is used.
|
||||||
|
2. A dollar quoted string literal is used in the SQL query.
|
||||||
|
3. That query contains text that would be would be interpreted outside as a placeholder outside of a string literal.
|
||||||
|
4. The value of that placeholder is controllable by the attacker.
|
||||||
|
|
||||||
|
e.g.
|
||||||
|
|
||||||
|
```go
|
||||||
|
attackValue := `$tag$; drop table canary; --`
|
||||||
|
_, err = tx.Exec(ctx, `select $tag$ $1 $tag$, $1`, pgx.QueryExecModeSimpleProtocol, attackValue)
|
||||||
|
```
|
||||||
|
|
||||||
|
This is unlikely to occur outside of a contrived scenario.
|
||||||
|
|
||||||
|
# 5.9.1 (March 22, 2026)
|
||||||
|
|
||||||
|
* Fix: batch result format corruption when using cached prepared statements (reported by Dirkjan Bussink)
|
||||||
|
|
||||||
|
# 5.9.0 (March 21, 2026)
|
||||||
|
|
||||||
|
This release includes a number of new features such as SCRAM-SHA-256-PLUS support, OAuth authentication support, and
|
||||||
|
PostgreSQL protocol 3.2 support.
|
||||||
|
|
||||||
|
It significantly reduces the amount of network traffic when using prepared statements (which are used automatically by
|
||||||
|
default) by avoiding unnecessary Describe Portal messages. This also reduces local memory usage.
|
||||||
|
|
||||||
|
It also includes multiple fixes for potential DoS due to panic or OOM if connected to a malicious server that sends
|
||||||
|
deliberately malformed messages.
|
||||||
|
|
||||||
|
* Require Go 1.25+
|
||||||
|
* Add SCRAM-SHA-256-PLUS support (Adam Brightwell)
|
||||||
|
* Add OAuth authentication support for PostgreSQL 18 (David Schneider)
|
||||||
|
* Add PostgreSQL protocol 3.2 support (Dirkjan Bussink)
|
||||||
|
* Add tsvector type support (Adam Brightwell)
|
||||||
|
* Skip Describe Portal for cached prepared statements reducing network round trips
|
||||||
|
* Make LoadTypes query easier to support on "postgres-like" servers (Jelte Fennema-Nio)
|
||||||
|
* Default empty user to current OS user matching libpq behavior (ShivangSrivastava)
|
||||||
|
* Optimize LRU statement cache with custom linked list and node pooling (Mathias Bogaert)
|
||||||
|
* Optimize date scanning by replacing regex with manual parsing (Mathias Bogaert)
|
||||||
|
* Optimize pgio append/set functions with direct byte shifts (Mathias Bogaert)
|
||||||
|
* Make RowsAffected faster (Abhishek Chanda)
|
||||||
|
* Fix: Pipeline.Close panic when server sends multiple FATAL errors (Varun Chawla)
|
||||||
|
* Fix: ContextWatcher goroutine leak (Hank Donnay)
|
||||||
|
* Fix: stdlib discard connections with open transactions in ResetSession (Jeremy Schneider)
|
||||||
|
* Fix: pipelineBatchResults.Exec silently swallowing lastRows error
|
||||||
|
* Fix: ColumnTypeLength using BPCharArrayOID instead of BPCharOID
|
||||||
|
* Fix: TSVector text encoding returning nil for valid empty tsvector
|
||||||
|
* Fix: wrong error messages for Int2 and Int4 underflow
|
||||||
|
* Fix: Numeric nil Int pointer dereference with Valid: true
|
||||||
|
* Fix: reversed strings.ContainsAny arguments in Numeric.ScanScientific
|
||||||
|
* Fix: message length parsing on 32-bit platforms
|
||||||
|
* Fix: FunctionCallResponse.Decode mishandling of signed result size
|
||||||
|
* Fix: returning wrong error in configTLS when DecryptPEMBlock fails (Maxim Motyshen)
|
||||||
|
* Fix: misleading ParseConfig error when default_query_exec_mode is invalid (Skarm)
|
||||||
|
* Fix: missed Unwatch in Pipeline error paths
|
||||||
|
* Clarify too many failed acquire attempts error message
|
||||||
|
* Better error wrapping with context and SQL statement (Aneesh Makala)
|
||||||
|
* Enable govet and ineffassign linters (Federico Guerinoni)
|
||||||
|
* Guard against various malformed binary messages (arrays, hstore, multirange, protocol messages)
|
||||||
|
* Fix various godoc comments (ferhat elmas)
|
||||||
|
* Fix typos in comments (Oleksandr Redko)
|
||||||
|
|
||||||
|
# 5.8.0 (December 26, 2025)
|
||||||
|
|
||||||
|
* Require Go 1.24+
|
||||||
|
* Remove golang.org/x/crypto dependency
|
||||||
|
* Add OptionShouldPing to control ResetSession ping behavior (ilyam8)
|
||||||
|
* Fix: Avoid overflow when MaxConns is set to MaxInt32
|
||||||
|
* Fix: Close batch pipeline after a query error (Anthonin Bonnefoy)
|
||||||
|
* Faster shutdown of pgxpool.Pool background goroutines (Blake Gentry)
|
||||||
|
* Add pgxpool ping timeout (Amirsalar Safaei)
|
||||||
|
* Fix: Rows.FieldDescriptions for empty query
|
||||||
|
* Scan unknown types into *any as string or []byte based on format code
|
||||||
|
* Optimize pgtype.Numeric (Philip Dubé)
|
||||||
|
* Add AfterNetConnect hook to pgconn.Config
|
||||||
|
* Fix: Handle for preparing statements that fail during the Describe phase
|
||||||
|
* Fix overflow in numeric scanning (Ilia Demianenko)
|
||||||
|
* Fix: json/jsonb sql.Scanner source type is []byte
|
||||||
|
* Migrate from math/rand to math/rand/v2 (Mathias Bogaert)
|
||||||
|
* Optimize internal iobufpool (Mathias Bogaert)
|
||||||
|
* Optimize stmtcache invalidation (Mathias Bogaert)
|
||||||
|
* Fix: missing error case in interval parsing (Maxime Soulé)
|
||||||
|
* Fix: invalidate statement/description cache in Exec (James Hartig)
|
||||||
|
* ColumnTypeLength method return the type length for varbit type (DengChan)
|
||||||
|
* Array and Composite codecs handle typed nils
|
||||||
|
|
||||||
|
# 5.7.6 (September 8, 2025)
|
||||||
|
|
||||||
|
* Use ParseConfigError in pgx.ParseConfig and pgxpool.ParseConfig (Yurasov Ilia)
|
||||||
|
* Add PrepareConn hook to pgxpool (Jonathan Hall)
|
||||||
|
* Reduce allocations in QueryContext (Dominique Lefevre)
|
||||||
|
* Add MarshalJSON and UnmarshalJSON for pgtype.Uint32 (Panos Koutsovasilis)
|
||||||
|
* Configure ping behavior on pgxpool with ShouldPing (Christian Kiely)
|
||||||
|
* zeronull int types implement Int64Valuer and Int64Scanner (Li Zeghong)
|
||||||
|
* Fix panic when receiving terminate connection message during CopyFrom (Michal Drausowski)
|
||||||
|
* Fix statement cache not being invalidated on error during batch (Muhammadali Nazarov)
|
||||||
|
|
||||||
|
# 5.7.5 (May 17, 2025)
|
||||||
|
|
||||||
|
* Support sslnegotiation connection option (divyam234)
|
||||||
|
* Update golang.org/x/crypto to v0.37.0. This placates security scanners that were unable to see that pgx did not use the behavior affected by https://pkg.go.dev/vuln/GO-2025-3487.
|
||||||
|
* TraceLog now logs Acquire and Release at the debug level (dave sinclair)
|
||||||
|
* Add support for PGTZ environment variable
|
||||||
|
* Add support for PGOPTIONS environment variable
|
||||||
|
* Unpin memory used by Rows quicker
|
||||||
|
* Remove PlanScan memoization. This resolves a rare issue where scanning could be broken for one type by first scanning another. The problem was in the memoization system and benchmarking revealed that memoization was not providing any meaningful benefit.
|
||||||
|
|
||||||
|
# 5.7.4 (March 24, 2025)
|
||||||
|
|
||||||
|
* Fix / revert change to scanning JSON `null` (Felix Röhrich)
|
||||||
|
|
||||||
|
# 5.7.3 (March 21, 2025)
|
||||||
|
|
||||||
|
* Expose EmptyAcquireWaitTime in pgxpool.Stat (vamshiaruru32)
|
||||||
|
* Improve SQL sanitizer performance (ninedraft)
|
||||||
|
* Fix Scan confusion with json(b), sql.Scanner, and automatic dereferencing (moukoublen, felix-roehrich)
|
||||||
|
* Fix Values() for xml type always returning nil instead of []byte
|
||||||
|
* Add ability to send Flush message in pipeline mode (zenkovev)
|
||||||
|
* Fix pgtype.Timestamp's JSON behavior to match PostgreSQL (pconstantinou)
|
||||||
|
* Better error messages when scanning structs (logicbomb)
|
||||||
|
* Fix handling of error on batch write (bonnefoa)
|
||||||
|
* Match libpq's connection fallback behavior more closely (felix-roehrich)
|
||||||
|
* Add MinIdleConns to pgxpool (djahandarie)
|
||||||
|
|
||||||
|
# 5.7.2 (December 21, 2024)
|
||||||
|
|
||||||
|
* Fix prepared statement already exists on batch prepare failure
|
||||||
|
* Add commit query to tx options (Lucas Hild)
|
||||||
|
* Fix pgtype.Timestamp json unmarshal (Shean de Montigny-Desautels)
|
||||||
|
* Add message body size limits in frontend and backend (zene)
|
||||||
|
* Add xid8 type
|
||||||
|
* Ensure planning encodes and scans cannot infinitely recurse
|
||||||
|
* Implement pgtype.UUID.String() (Konstantin Grachev)
|
||||||
|
* Switch from ExecParams to Exec in ValidateConnectTargetSessionAttrs functions (Alexander Rumyantsev)
|
||||||
|
* Update golang.org/x/crypto
|
||||||
|
* Fix json(b) columns prefer sql.Scanner interface like database/sql (Ludovico Russo)
|
||||||
|
|
||||||
|
# 5.7.1 (September 10, 2024)
|
||||||
|
|
||||||
|
* Fix data race in tracelog.TraceLog
|
||||||
|
* Update puddle to v2.2.2. This removes the import of nanotime via linkname.
|
||||||
|
* Update golang.org/x/crypto and golang.org/x/text
|
||||||
|
|
||||||
|
# 5.7.0 (September 7, 2024)
|
||||||
|
|
||||||
|
* Add support for sslrootcert=system (Yann Soubeyrand)
|
||||||
|
* Add LoadTypes to load multiple types in a single SQL query (Nick Farrell)
|
||||||
|
* Add XMLCodec supports encoding + scanning XML column type like json (nickcruess-soda)
|
||||||
|
* Add MultiTrace (Stepan Rabotkin)
|
||||||
|
* Add TraceLogConfig with customizable TimeKey (stringintech)
|
||||||
|
* pgx.ErrNoRows wraps sql.ErrNoRows to aid in database/sql compatibility with native pgx functions (merlin)
|
||||||
|
* Support scanning binary formatted uint32 into string / TextScanner (jennifersp)
|
||||||
|
* Fix interval encoding to allow 0s and avoid extra spaces (Carlos Pérez-Aradros Herce)
|
||||||
|
* Update pgservicefile - fixes panic when parsing invalid file
|
||||||
|
* Better error message when reading past end of batch
|
||||||
|
* Don't print url when url.Parse returns an error (Kevin Biju)
|
||||||
|
* Fix snake case name normalization collision in RowToStructByName with db tag (nolandseigler)
|
||||||
|
* Fix: Scan and encode types with underlying types of arrays
|
||||||
|
|
||||||
|
# 5.6.0 (May 25, 2024)
|
||||||
|
|
||||||
|
* Add StrictNamedArgs (Tomas Zahradnicek)
|
||||||
|
* Add support for macaddr8 type (Carlos Pérez-Aradros Herce)
|
||||||
|
* Add SeverityUnlocalized field to PgError / Notice
|
||||||
|
* Performance optimization of RowToStructByPos/Name (Zach Olstein)
|
||||||
|
* Allow customizing context canceled behavior for pgconn
|
||||||
|
* Add ScanLocation to pgtype.Timestamp[tz]Codec
|
||||||
|
* Add custom data to pgconn.PgConn
|
||||||
|
* Fix ResultReader.Read() to handle nil values
|
||||||
|
* Do not encode interval microseconds when they are 0 (Carlos Pérez-Aradros Herce)
|
||||||
|
* pgconn.SafeToRetry checks for wrapped errors (tjasko)
|
||||||
|
* Failed connection attempts include all errors
|
||||||
|
* Optimize LargeObject.Read (Mitar)
|
||||||
|
* Add tracing for connection acquire and release from pool (ngavinsir)
|
||||||
|
* Fix encode driver.Valuer not called when nil
|
||||||
|
* Add support for custom JSON marshal and unmarshal (Mitar)
|
||||||
|
* Use Go default keepalive for TCP connections (Hans-Joachim Kliemeck)
|
||||||
|
|
||||||
|
# 5.5.5 (March 9, 2024)
|
||||||
|
|
||||||
|
Use spaces instead of parentheses for SQL sanitization.
|
||||||
|
|
||||||
|
This still solves the problem of negative numbers creating a line comment, but this avoids breaking edge cases such as
|
||||||
|
`set foo to $1` where the substitution is taking place in a location where an arbitrary expression is not allowed.
|
||||||
|
|
||||||
|
# 5.5.4 (March 4, 2024)
|
||||||
|
|
||||||
|
Fix CVE-2024-27304
|
||||||
|
|
||||||
|
SQL injection can occur if an attacker can cause a single query or bind message to exceed 4 GB in size. An integer
|
||||||
|
overflow in the calculated message size can cause the one large message to be sent as multiple messages under the
|
||||||
|
attacker's control.
|
||||||
|
|
||||||
|
Thanks to Paul Gerste for reporting this issue.
|
||||||
|
|
||||||
|
* Fix behavior of CollectRows to return empty slice if Rows are empty (Felix)
|
||||||
|
* Fix simple protocol encoding of json.RawMessage
|
||||||
|
* Fix *Pipeline.getResults should close pipeline on error
|
||||||
|
* Fix panic in TryFindUnderlyingTypeScanPlan (David Kurman)
|
||||||
|
* Fix deallocation of invalidated cached statements in a transaction
|
||||||
|
* Handle invalid sslkey file
|
||||||
|
* Fix scan float4 into sql.Scanner
|
||||||
|
* Fix pgtype.Bits not making copy of data from read buffer. This would cause the data to be corrupted by future reads.
|
||||||
|
|
||||||
|
# 5.5.3 (February 3, 2024)
|
||||||
|
|
||||||
|
* Fix: prepared statement already exists
|
||||||
|
* Improve CopyFrom auto-conversion of text-ish values
|
||||||
|
* Add ltree type support (Florent Viel)
|
||||||
|
* Make some properties of Batch and QueuedQuery public (Pavlo Golub)
|
||||||
|
* Add AppendRows function (Edoardo Spadolini)
|
||||||
|
* Optimize convert UUID [16]byte to string (Kirill Malikov)
|
||||||
|
* Fix: LargeObject Read and Write of more than ~1GB at a time (Mitar)
|
||||||
|
|
||||||
|
# 5.5.2 (January 13, 2024)
|
||||||
|
|
||||||
|
* Allow NamedArgs to start with underscore
|
||||||
|
* pgproto3: Maximum message body length support (jeremy.spriet)
|
||||||
|
* Upgrade golang.org/x/crypto to v0.17.0
|
||||||
|
* Add snake_case support to RowToStructByName (Tikhon Fedulov)
|
||||||
|
* Fix: update description cache after exec prepare (James Hartig)
|
||||||
|
* Fix: pipeline checks if it is closed (James Hartig and Ryan Fowler)
|
||||||
|
* Fix: normalize timeout / context errors during TLS startup (Samuel Stauffer)
|
||||||
|
* Add OnPgError for easier centralized error handling (James Hartig)
|
||||||
|
|
||||||
|
# 5.5.1 (December 9, 2023)
|
||||||
|
|
||||||
|
* Add CopyFromFunc helper function. (robford)
|
||||||
|
* Add PgConn.Deallocate method that uses PostgreSQL protocol Close message.
|
||||||
|
* pgx uses new PgConn.Deallocate method. This allows deallocating statements to work in a failed transaction. This fixes a case where the prepared statement map could become invalid.
|
||||||
|
* Fix: Prefer driver.Valuer over json.Marshaler for json fields. (Jacopo)
|
||||||
|
* Fix: simple protocol SQL sanitizer previously panicked if an invalid $0 placeholder was used. This now returns an error instead. (maksymnevajdev)
|
||||||
|
* Add pgtype.Numeric.ScanScientific (Eshton Robateau)
|
||||||
|
|
||||||
|
# 5.5.0 (November 4, 2023)
|
||||||
|
|
||||||
|
* Add CollectExactlyOneRow. (Julien GOTTELAND)
|
||||||
|
* Add OpenDBFromPool to create *database/sql.DB from *pgxpool.Pool. (Lev Zakharov)
|
||||||
|
* Prepare can automatically choose statement name based on sql. This makes it easier to explicitly manage prepared statements.
|
||||||
|
* Statement cache now uses deterministic, stable statement names.
|
||||||
|
* database/sql prepared statement names are deterministically generated.
|
||||||
|
* Fix: SendBatch wasn't respecting context cancellation.
|
||||||
|
* Fix: Timeout error from pipeline is now normalized.
|
||||||
|
* Fix: database/sql encoding json.RawMessage to []byte.
|
||||||
|
* CancelRequest: Wait for the cancel request to be acknowledged by the server. This should improve PgBouncer compatibility. (Anton Levakin)
|
||||||
|
* stdlib: Use Ping instead of CheckConn in ResetSession
|
||||||
|
* Add json.Marshaler and json.Unmarshaler for Float4, Float8 (Kirill Mironov)
|
||||||
|
|
||||||
|
# 5.4.3 (August 5, 2023)
|
||||||
|
|
||||||
|
* Fix: QCharArrayOID was defined with the wrong OID (Christoph Engelbert)
|
||||||
|
* Fix: connect_timeout for sslmode=allow|prefer (smaher-edb)
|
||||||
|
* Fix: pgxpool: background health check cannot overflow pool
|
||||||
|
* Fix: Check for nil in defer when sending batch (recover properly from panic)
|
||||||
|
* Fix: json scan of non-string pointer to pointer
|
||||||
|
* Fix: zeronull.Timestamptz should use pgtype.Timestamptz
|
||||||
|
* Fix: NewConnsCount was not correctly counting connections created by Acquire directly. (James Hartig)
|
||||||
|
* RowTo(AddrOf)StructByPos ignores fields with "-" db tag
|
||||||
|
* Optimization: improve text format numeric parsing (horpto)
|
||||||
|
|
||||||
|
# 5.4.2 (July 11, 2023)
|
||||||
|
|
||||||
|
* Fix: RowScanner errors are fatal to Rows
|
||||||
|
* Fix: Enable failover efforts when pg_hba.conf disallows non-ssl connections (Brandon Kauffman)
|
||||||
|
* Hstore text codec internal improvements (Evan Jones)
|
||||||
|
* Fix: Stop timers for background reader when not in use. Fixes memory leak when closing connections (Adrian-Stefan Mares)
|
||||||
|
* Fix: Stop background reader as soon as possible.
|
||||||
|
* Add PgConn.SyncConn(). This combined with the above fix makes it safe to directly use the underlying net.Conn.
|
||||||
|
|
||||||
|
# 5.4.1 (June 18, 2023)
|
||||||
|
|
||||||
|
* Fix: concurrency bug with pgtypeDefaultMap and simple protocol (Lev Zakharov)
|
||||||
|
* Add TxOptions.BeginQuery to allow overriding the default BEGIN query
|
||||||
|
|
||||||
|
# 5.4.0 (June 14, 2023)
|
||||||
|
|
||||||
|
* Replace platform specific syscalls for non-blocking IO with more traditional goroutines and deadlines. This returns to the v4 approach with some additional improvements and fixes. This restores the ability to use a pgx.Conn over an ssh.Conn as well as other non-TCP or Unix socket connections. In addition, it is a significantly simpler implementation that is less likely to have cross platform issues.
|
||||||
|
* Optimization: The default type registrations are now shared among all connections. This saves about 100KB of memory per connection. `pgtype.Type` and `pgtype.Codec` values are now required to be immutable after registration. This was already necessary in most cases but wasn't documented until now. (Lev Zakharov)
|
||||||
|
* Fix: Ensure pgxpool.Pool.QueryRow.Scan releases connection on panic
|
||||||
|
* CancelRequest: don't try to read the reply (Nicola Murino)
|
||||||
|
* Fix: correctly handle bool type aliases (Wichert Akkerman)
|
||||||
|
* Fix: pgconn.CancelRequest: Fix unix sockets: don't use RemoteAddr()
|
||||||
|
* Fix: pgx.Conn memory leak with prepared statement caching (Evan Jones)
|
||||||
|
* Add BeforeClose to pgxpool.Pool (Evan Cordell)
|
||||||
|
* Fix: various hstore fixes and optimizations (Evan Jones)
|
||||||
|
* Fix: RowToStructByPos with embedded unexported struct
|
||||||
|
* Support different bool string representations (Lev Zakharov)
|
||||||
|
* Fix: error when using BatchResults.Exec on a select that returns an error after some rows.
|
||||||
|
* Fix: pipelineBatchResults.Exec() not returning error from ResultReader
|
||||||
|
* Fix: pipeline batch results not closing pipeline when error occurs while reading directly from results instead of using
|
||||||
|
a callback.
|
||||||
|
* Fix: scanning a table type into a struct
|
||||||
|
* Fix: scan array of record to pointer to slice of struct
|
||||||
|
* Fix: handle null for json (Cemre Mengu)
|
||||||
|
* Batch Query callback is called even when there is an error
|
||||||
|
* Add RowTo(AddrOf)StructByNameLax (Audi P. Risa P)
|
||||||
|
|
||||||
|
# 5.3.1 (February 27, 2023)
|
||||||
|
|
||||||
|
* Fix: Support v4 and v5 stdlib in same program (Tomáš Procházka)
|
||||||
|
* Fix: sql.Scanner not being used in certain cases
|
||||||
|
* Add text format jsonpath support
|
||||||
|
* Fix: fake non-blocking read adaptive wait time
|
||||||
|
|
||||||
|
# 5.3.0 (February 11, 2023)
|
||||||
|
|
||||||
|
* Fix: json values work with sql.Scanner
|
||||||
|
* Fixed / improved error messages (Mark Chambers and Yevgeny Pats)
|
||||||
|
* Fix: support scan into single dimensional arrays
|
||||||
|
* Fix: MaxConnLifetimeJitter setting actually jitter (Ben Weintraub)
|
||||||
|
* Fix: driver.Value representation of bytea should be []byte not string
|
||||||
|
* Fix: better handling of unregistered OIDs
|
||||||
|
* CopyFrom can use query cache to avoid extra round trip to get OIDs (Alejandro Do Nascimento Mora)
|
||||||
|
* Fix: encode to json ignoring driver.Valuer
|
||||||
|
* Support sql.Scanner on renamed base type
|
||||||
|
* Fix: pgtype.Numeric text encoding of negative numbers (Mark Chambers)
|
||||||
|
* Fix: connect with multiple hostnames when one can't be resolved
|
||||||
|
* Upgrade puddle to remove dependency on uber/atomic and fix alignment issue on 32-bit platform
|
||||||
|
* Fix: scanning json column into **string
|
||||||
|
* Multiple reductions in memory allocations
|
||||||
|
* Fake non-blocking read adapts its max wait time
|
||||||
|
* Improve CopyFrom performance and reduce memory usage
|
||||||
|
* Fix: encode []any to array
|
||||||
|
* Fix: LoadType for composite with dropped attributes (Felix Röhrich)
|
||||||
|
* Support v4 and v5 stdlib in same program
|
||||||
|
* Fix: text format array decoding with string of "NULL"
|
||||||
|
* Prefer binary format for arrays
|
||||||
|
|
||||||
|
# 5.2.0 (December 5, 2022)
|
||||||
|
|
||||||
|
* `tracelog.TraceLog` implements the pgx.PrepareTracer interface. (Vitalii Solodilov)
|
||||||
|
* Optimize creating begin transaction SQL string (Petr Evdokimov and ksco)
|
||||||
|
* `Conn.LoadType` supports range and multirange types (Vitalii Solodilov)
|
||||||
|
* Fix scan `uint` and `uint64` `ScanNumeric`. This resolves a PostgreSQL `numeric` being incorrectly scanned into `uint` and `uint64`.
|
||||||
|
|
||||||
|
# 5.1.1 (November 17, 2022)
|
||||||
|
|
||||||
|
* Fix simple query sanitizer where query text contains a Unicode replacement character.
|
||||||
|
* Remove erroneous `name` argument from `DeallocateAll()`. Technically, this is a breaking change, but given that method was only added 5 days ago this change was accepted. (Bodo Kaiser)
|
||||||
|
|
||||||
|
# 5.1.0 (November 12, 2022)
|
||||||
|
|
||||||
|
* Update puddle to v2.1.2. This resolves a race condition and a deadlock in pgxpool.
|
||||||
|
* `QueryRewriter.RewriteQuery` now returns an error. Technically, this is a breaking change for any external implementers, but given the minimal likelihood that there are actually any external implementers this change was accepted.
|
||||||
|
* Expose `GetSSLPassword` support to pgx.
|
||||||
|
* Fix encode `ErrorResponse` unknown field handling. This would only affect pgproto3 being used directly as a proxy with a non-PostgreSQL server that included additional error fields.
|
||||||
|
* Fix date text format encoding with 5 digit years.
|
||||||
|
* Fix date values passed to a `sql.Scanner` as `string` instead of `time.Time`.
|
||||||
|
* DateCodec.DecodeValue can return `pgtype.InfinityModifier` instead of `string` for infinite values. This now matches the behavior of the timestamp types.
|
||||||
|
* Add domain type support to `Conn.LoadType()`.
|
||||||
|
* Add `RowToStructByName` and `RowToAddrOfStructByName`. (Pavlo Golub)
|
||||||
|
* Add `Conn.DeallocateAll()` to clear all prepared statements including the statement cache. (Bodo Kaiser)
|
||||||
|
|
||||||
|
# 5.0.4 (October 24, 2022)
|
||||||
|
|
||||||
|
* Fix: CollectOneRow prefers PostgreSQL error over pgx.ErrorNoRows
|
||||||
|
* Fix: some reflect Kind checks to first check for nil
|
||||||
|
* Bump golang.org/x/text dependency to placate snyk
|
||||||
|
* Fix: RowToStructByPos on structs with multiple anonymous sub-structs (Baptiste Fontaine)
|
||||||
|
* Fix: Exec checks if tx is closed
|
||||||
|
|
||||||
|
# 5.0.3 (October 14, 2022)
|
||||||
|
|
||||||
|
* Fix `driver.Valuer` handling edge cases that could cause infinite loop or crash
|
||||||
|
|
||||||
|
# v5.0.2 (October 8, 2022)
|
||||||
|
|
||||||
|
* Fix date encoding in text format to always use 2 digits for month and day
|
||||||
|
* Prefer driver.Valuer over wrap plans when encoding
|
||||||
|
* Fix scan to pointer to pointer to renamed type
|
||||||
|
* Allow scanning NULL even if PG and Go types are incompatible
|
||||||
|
|
||||||
|
# v5.0.1 (September 24, 2022)
|
||||||
|
|
||||||
|
* Fix 32-bit atomic usage
|
||||||
|
* Add MarshalJSON for Float8 (yogipristiawan)
|
||||||
|
* Add `[` and `]` to text encoding of `Lseg`
|
||||||
|
* Fix sqlScannerWrapper NULL handling
|
||||||
|
|
||||||
|
# v5.0.0 (September 17, 2022)
|
||||||
|
|
||||||
|
## Merged Packages
|
||||||
|
|
||||||
|
`github.com/jackc/pgtype`, `github.com/jackc/pgconn`, and `github.com/jackc/pgproto3` are now included in the main
|
||||||
|
`github.com/jackc/pgx` repository. Previously there was confusion as to where issues should be reported, additional
|
||||||
|
release work due to releasing multiple packages, and less clear changelogs.
|
||||||
|
|
||||||
|
## pgconn
|
||||||
|
|
||||||
|
`CommandTag` is now an opaque type instead of directly exposing an underlying `[]byte`.
|
||||||
|
|
||||||
|
The return value `ResultReader.Values()` is no longer safe to retain a reference to after a subsequent call to `NextRow()` or `Close()`.
|
||||||
|
|
||||||
|
`Trace()` method adds low level message tracing similar to the `PQtrace` function in `libpq`.
|
||||||
|
|
||||||
|
pgconn now uses non-blocking IO. This is a significant internal restructuring, but it should not cause any visible changes on its own. However, it is important in implementing other new features.
|
||||||
|
|
||||||
|
`CheckConn()` checks a connection's liveness by doing a non-blocking read. This can be used to detect database restarts or network interruptions without executing a query or a ping.
|
||||||
|
|
||||||
|
pgconn now supports pipeline mode.
|
||||||
|
|
||||||
|
`*PgConn.ReceiveResults` removed. Use pipeline mode instead.
|
||||||
|
|
||||||
|
`Timeout()` no longer considers `context.Canceled` as a timeout error. `context.DeadlineExceeded` still is considered a timeout error.
|
||||||
|
|
||||||
|
## pgxpool
|
||||||
|
|
||||||
|
`Connect` and `ConnectConfig` have been renamed to `New` and `NewWithConfig` respectively. The `LazyConnect` option has been removed. Pools always lazily connect.
|
||||||
|
|
||||||
|
## pgtype
|
||||||
|
|
||||||
|
The `pgtype` package has been significantly changed.
|
||||||
|
|
||||||
|
### NULL Representation
|
||||||
|
|
||||||
|
Previously, types had a `Status` field that could be `Undefined`, `Null`, or `Present`. This has been changed to a
|
||||||
|
`Valid` `bool` field to harmonize with how `database/sql` represents `NULL` and to make the zero value useable.
|
||||||
|
|
||||||
|
Previously, a type that implemented `driver.Valuer` would have the `Value` method called even on a nil pointer. All nils
|
||||||
|
whether typed or untyped now represent `NULL`.
|
||||||
|
|
||||||
|
### Codec and Value Split
|
||||||
|
|
||||||
|
Previously, the type system combined decoding and encoding values with the value types. e.g. Type `Int8` both handled
|
||||||
|
encoding and decoding the PostgreSQL representation and acted as a value object. This caused some difficulties when
|
||||||
|
there was not an exact 1 to 1 relationship between the Go types and the PostgreSQL types For example, scanning a
|
||||||
|
PostgreSQL binary `numeric` into a Go `float64` was awkward (see https://github.com/jackc/pgtype/issues/147). This
|
||||||
|
concepts have been separated. A `Codec` only has responsibility for encoding and decoding values. Value types are
|
||||||
|
generally defined by implementing an interface that a particular `Codec` understands (e.g. `PointScanner` and
|
||||||
|
`PointValuer` for the PostgreSQL `point` type).
|
||||||
|
|
||||||
|
### Array Types
|
||||||
|
|
||||||
|
All array types are now handled by `ArrayCodec` instead of using code generation for each new array type. This also
|
||||||
|
means that less common array types such as `point[]` are now supported. `Array[T]` supports PostgreSQL multi-dimensional
|
||||||
|
arrays.
|
||||||
|
|
||||||
|
### Composite Types
|
||||||
|
|
||||||
|
Composite types must be registered before use. `CompositeFields` may still be used to construct and destruct composite
|
||||||
|
values, but any type may now implement `CompositeIndexGetter` and `CompositeIndexScanner` to be used as a composite.
|
||||||
|
|
||||||
|
### Range Types
|
||||||
|
|
||||||
|
Range types are now handled with types `RangeCodec` and `Range[T]`. This allows additional user defined range types to
|
||||||
|
easily be handled. Multirange types are handled similarly with `MultirangeCodec` and `Multirange[T]`.
|
||||||
|
|
||||||
|
### pgxtype
|
||||||
|
|
||||||
|
`LoadDataType` moved to `*Conn` as `LoadType`.
|
||||||
|
|
||||||
|
### Bytea
|
||||||
|
|
||||||
|
The `Bytea` and `GenericBinary` types have been replaced. Use the following instead:
|
||||||
|
|
||||||
|
* `[]byte` - For normal usage directly use `[]byte`.
|
||||||
|
* `DriverBytes` - Uses driver memory only available until next database method call. Avoids a copy and an allocation.
|
||||||
|
* `PreallocBytes` - Uses preallocated byte slice to avoid an allocation.
|
||||||
|
* `UndecodedBytes` - Avoids any decoding. Allows working with raw bytes.
|
||||||
|
|
||||||
|
### Dropped lib/pq Support
|
||||||
|
|
||||||
|
`pgtype` previously supported and was tested against [lib/pq](https://github.com/lib/pq). While it will continue to work
|
||||||
|
in most cases this is no longer supported.
|
||||||
|
|
||||||
|
### database/sql Scan
|
||||||
|
|
||||||
|
Previously, most `Scan` implementations would convert `[]byte` to `string` automatically to decode a text value. Now
|
||||||
|
only `string` is handled. This is to allow the possibility of future binary support in `database/sql` mode by
|
||||||
|
considering `[]byte` to be binary format and `string` text format. This change should have no effect for any use with
|
||||||
|
`pgx`. The previous behavior was only necessary for `lib/pq` compatibility.
|
||||||
|
|
||||||
|
Added `*Map.SQLScanner` to create a `sql.Scanner` for types such as `[]int32` and `Range[T]` that do not implement
|
||||||
|
`sql.Scanner` directly.
|
||||||
|
|
||||||
|
### Number Type Fields Include Bit size
|
||||||
|
|
||||||
|
`Int2`, `Int4`, `Int8`, `Float4`, `Float8`, and `Uint32` fields now include bit size. e.g. `Int` is renamed to `Int64`.
|
||||||
|
This matches the convention set by `database/sql`. In addition, for comparable types like `pgtype.Int8` and
|
||||||
|
`sql.NullInt64` the structures are identical. This means they can be directly converted one to another.
|
||||||
|
|
||||||
|
### 3rd Party Type Integrations
|
||||||
|
|
||||||
|
* Extracted integrations with https://github.com/shopspring/decimal and https://github.com/gofrs/uuid to
|
||||||
|
https://github.com/jackc/pgx-shopspring-decimal and https://github.com/jackc/pgx-gofrs-uuid respectively. This trims
|
||||||
|
the pgx dependency tree.
|
||||||
|
|
||||||
|
### Other Changes
|
||||||
|
|
||||||
|
* `Bit` and `Varbit` are both replaced by the `Bits` type.
|
||||||
|
* `CID`, `OID`, `OIDValue`, and `XID` are replaced by the `Uint32` type.
|
||||||
|
* `Hstore` is now defined as `map[string]*string`.
|
||||||
|
* `JSON` and `JSONB` types removed. Use `[]byte` or `string` directly.
|
||||||
|
* `QChar` type removed. Use `rune` or `byte` directly.
|
||||||
|
* `Inet` and `Cidr` types removed. Use `netip.Addr` and `netip.Prefix` directly. These types are more memory efficient than the previous `net.IPNet`.
|
||||||
|
* `Macaddr` type removed. Use `net.HardwareAddr` directly.
|
||||||
|
* Renamed `pgtype.ConnInfo` to `pgtype.Map`.
|
||||||
|
* Renamed `pgtype.DataType` to `pgtype.Type`.
|
||||||
|
* Renamed `pgtype.None` to `pgtype.Finite`.
|
||||||
|
* `RegisterType` now accepts a `*Type` instead of `Type`.
|
||||||
|
* Assorted array helper methods and types made private.
|
||||||
|
|
||||||
|
## stdlib
|
||||||
|
|
||||||
|
* Removed `AcquireConn` and `ReleaseConn` as that functionality has been built in since Go 1.13.
|
||||||
|
|
||||||
|
## Reduced Memory Usage by Reusing Read Buffers
|
||||||
|
|
||||||
|
Previously, the connection read buffer would allocate large chunks of memory and never reuse them. This allowed
|
||||||
|
transferring ownership to anything such as scanned values without incurring an additional allocation and memory copy.
|
||||||
|
However, this came at the cost of overall increased memory allocation size. But worse it was also possible to pin large
|
||||||
|
chunks of memory by retaining a reference to a small value that originally came directly from the read buffer. Now
|
||||||
|
ownership remains with the read buffer and anything needing to retain a value must make a copy.
|
||||||
|
|
||||||
|
## Query Execution Modes
|
||||||
|
|
||||||
|
Control over automatic prepared statement caching and simple protocol use are now combined into query execution mode.
|
||||||
|
See documentation for `QueryExecMode`.
|
||||||
|
|
||||||
|
## QueryRewriter Interface and NamedArgs
|
||||||
|
|
||||||
|
pgx now supports named arguments with the `NamedArgs` type. This is implemented via the new `QueryRewriter` interface which
|
||||||
|
allows arbitrary rewriting of query SQL and arguments.
|
||||||
|
|
||||||
|
## RowScanner Interface
|
||||||
|
|
||||||
|
The `RowScanner` interface allows a single argument to Rows.Scan to scan the entire row.
|
||||||
|
|
||||||
|
## Rows Result Helpers
|
||||||
|
|
||||||
|
* `CollectRows` and `RowTo*` functions simplify collecting results into a slice.
|
||||||
|
* `CollectOneRow` collects one row using `RowTo*` functions.
|
||||||
|
* `ForEachRow` simplifies scanning each row and executing code using the scanned values. `ForEachRow` replaces `QueryFunc`.
|
||||||
|
|
||||||
|
## Tx Helpers
|
||||||
|
|
||||||
|
Rather than every type that implemented `Begin` or `BeginTx` methods also needing to implement `BeginFunc` and
|
||||||
|
`BeginTxFunc` these methods have been converted to functions that take a db that implements `Begin` or `BeginTx`.
|
||||||
|
|
||||||
|
## Improved Batch Query Ergonomics
|
||||||
|
|
||||||
|
Previously, the code for building a batch went in one place before the call to `SendBatch`, and the code for reading the
|
||||||
|
results went in one place after the call to `SendBatch`. This could make it difficult to match up the query and the code
|
||||||
|
to handle the results. Now `Queue` returns a `QueuedQuery` which has methods `Query`, `QueryRow`, and `Exec` which can
|
||||||
|
be used to register a callback function that will handle the result. Callback functions are called automatically when
|
||||||
|
`BatchResults.Close` is called.
|
||||||
|
|
||||||
|
## SendBatch Uses Pipeline Mode When Appropriate
|
||||||
|
|
||||||
|
Previously, a batch with 10 unique parameterized statements executed 100 times would entail 11 network round trips. 1
|
||||||
|
for each prepare / describe and 1 for executing them all. Now pipeline mode is used to prepare / describe all statements
|
||||||
|
in a single network round trip. So it would only take 2 round trips.
|
||||||
|
|
||||||
|
## Tracing and Logging
|
||||||
|
|
||||||
|
Internal logging support has been replaced with tracing hooks. This allows custom tracing integration with tools like OpenTelemetry. Package tracelog provides an adapter for pgx v4 loggers to act as a tracer.
|
||||||
|
|
||||||
|
All integrations with 3rd party loggers have been extracted to separate repositories. This trims the pgx dependency
|
||||||
|
tree.
|
||||||
+73
@@ -0,0 +1,73 @@
|
|||||||
|
# CLAUDE.md
|
||||||
|
|
||||||
|
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||||
|
|
||||||
|
## Project Overview
|
||||||
|
|
||||||
|
pgx is a PostgreSQL driver and toolkit for Go (`github.com/jackc/pgx/v5`). It provides both a native PostgreSQL interface and a `database/sql` compatible driver. Requires Go 1.25+ and supports PostgreSQL 14+ and CockroachDB.
|
||||||
|
|
||||||
|
## Build & Test Commands
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Run all tests (requires PGX_TEST_DATABASE to be set)
|
||||||
|
go test ./...
|
||||||
|
|
||||||
|
# Run a specific test
|
||||||
|
go test -run TestFunctionName ./...
|
||||||
|
|
||||||
|
# Run tests for a specific package
|
||||||
|
go test ./pgconn/...
|
||||||
|
|
||||||
|
# Run tests with race detector
|
||||||
|
go test -race ./...
|
||||||
|
|
||||||
|
# DevContainer: run tests against specific PostgreSQL versions
|
||||||
|
./test.sh pg18 # Default: PostgreSQL 18
|
||||||
|
./test.sh pg16 -run TestConnect # Specific test against PG16
|
||||||
|
./test.sh crdb # CockroachDB
|
||||||
|
./test.sh all # All targets (pg14-18 + crdb)
|
||||||
|
|
||||||
|
# Format (always run after making changes)
|
||||||
|
goimports -w .
|
||||||
|
|
||||||
|
# Lint
|
||||||
|
golangci-lint run ./...
|
||||||
|
```
|
||||||
|
|
||||||
|
## Test Database Setup
|
||||||
|
|
||||||
|
Tests require `PGX_TEST_DATABASE` environment variable. In the devcontainer, `test.sh` handles this. For local development:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export PGX_TEST_DATABASE="host=localhost user=postgres password=postgres dbname=pgx_test"
|
||||||
|
```
|
||||||
|
|
||||||
|
The test database needs extensions: `hstore`, `ltree`, and a `uint64` domain. See `testsetup/postgresql_setup.sql` for full setup. Many tests are skipped unless additional `PGX_TEST_*` env vars are set (for TLS, SCRAM, MD5, unix socket, PgBouncer testing).
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
The codebase is a layered architecture, bottom-up:
|
||||||
|
|
||||||
|
- **pgproto3/** — PostgreSQL wire protocol v3 encoder/decoder. Defines `FrontendMessage` and `BackendMessage` types for every protocol message.
|
||||||
|
- **pgconn/** — Low-level connection layer (roughly libpq-equivalent). Handles authentication, TLS, query execution, COPY protocol, and notifications. `PgConn` is the core type.
|
||||||
|
- **pgx** (root package) — High-level query interface built on `pgconn`. Provides `Conn`, `Rows`, `Tx`, `Batch`, `CopyFrom`, and generic helpers like `CollectRows`/`ForEachRow`. Includes automatic statement caching (LRU).
|
||||||
|
- **pgtype/** — Type system mapping between Go and PostgreSQL types (70+ types). Key interfaces: `Codec`, `Type`, `TypeMap`. Custom types (enums, composites, domains) are registered through `TypeMap`.
|
||||||
|
- **pgxpool/** — Concurrency-safe connection pool built on `puddle/v2`. `Pool` is the main type; wraps `pgx.Conn`.
|
||||||
|
- **stdlib/** — `database/sql` compatibility adapter.
|
||||||
|
|
||||||
|
Supporting packages:
|
||||||
|
- **internal/stmtcache/** — Prepared statement cache with LRU eviction
|
||||||
|
- **internal/sanitize/** — SQL query sanitization
|
||||||
|
- **tracelog/** — Logging adapter that implements tracer interfaces
|
||||||
|
- **multitracer/** — Composes multiple tracers into one
|
||||||
|
- **pgxtest/** — Test helpers for running tests across connection types
|
||||||
|
|
||||||
|
## Key Design Conventions
|
||||||
|
|
||||||
|
- **Semantic versioning** — strictly followed. Do not break the public API (no removing or renaming exported types, functions, methods, or fields; no changing function signatures).
|
||||||
|
- **Minimal dependencies** — adding new dependencies is strongly discouraged (see CONTRIBUTING.md).
|
||||||
|
- **Context-based** — all blocking operations take `context.Context`.
|
||||||
|
- **Tracer interfaces** — observability via `QueryTracer`, `BatchTracer`, `CopyFromTracer`, `PrepareTracer` on `ConnConfig.Tracer`.
|
||||||
|
- **Formatting** — always run `goimports -w .` after making changes to ensure code is properly formatted. CI checks formatting via `gofmt -l -s -w . && git diff --exit-code`. `gofumpt` with extra rules is also enforced via `golangci-lint`.
|
||||||
|
- **Linters** — `govet`, `ineffassign`, and `unconvert` only (configured in `.golangci.yml`).
|
||||||
|
- **CI matrix** — tests run against Go 1.25/1.26 × PostgreSQL 14-18 + CockroachDB, on Linux and Windows. Race detector enabled on Linux only.
|
||||||
+139
@@ -0,0 +1,139 @@
|
|||||||
|
# Contributing
|
||||||
|
|
||||||
|
## Discuss Significant Changes
|
||||||
|
|
||||||
|
Before you invest a significant amount of time on a change, please create a discussion or issue describing your
|
||||||
|
proposal. This will help to ensure your proposed change has a reasonable chance of being merged.
|
||||||
|
|
||||||
|
## Avoid Dependencies
|
||||||
|
|
||||||
|
Adding a dependency is a big deal. While on occasion a new dependency may be accepted, the default answer to any change
|
||||||
|
that adds a dependency is no.
|
||||||
|
|
||||||
|
## AI
|
||||||
|
|
||||||
|
Using AI is acceptable (not that it can really be stopped) under one the following conditions.
|
||||||
|
|
||||||
|
* AI was used, but you deeply understand the code and you can answer questions regarding your change. You are not going
|
||||||
|
to answer questions with "I don't know", AI did it. You are not going to "answer" questions by relaying them to your
|
||||||
|
agent. This is wasteful of the code reviewer's time.
|
||||||
|
* AI was used to solve a problem without your deep understanding. This can still be a good starting point for a fix or
|
||||||
|
feature. But you need to clearly state that this is an AI proposal. You should include additional information such as
|
||||||
|
the AI used and what prompts were used. You should also be aware that large, complicated, or subtle changes may be
|
||||||
|
rejected simply because the reviewer is not confident in a change that no human understands.
|
||||||
|
|
||||||
|
## Development Environment Setup
|
||||||
|
|
||||||
|
pgx tests naturally require a PostgreSQL database. It will connect to the database specified in the `PGX_TEST_DATABASE`
|
||||||
|
environment variable. The `PGX_TEST_DATABASE` environment variable can either be a URL or key-value pairs. In addition,
|
||||||
|
the standard `PG*` environment variables will be respected. Consider using [direnv](https://github.com/direnv/direnv) to
|
||||||
|
simplify environment variable handling.
|
||||||
|
|
||||||
|
### Devcontainer
|
||||||
|
|
||||||
|
The easiest way to start development is with the included devcontainer. It includes containers for each supported
|
||||||
|
PostgreSQL version as well as CockroachDB. `./test.sh all` will run the tests against all database types.
|
||||||
|
|
||||||
|
### Using an Existing PostgreSQL Cluster Outside of a Devcontainer
|
||||||
|
|
||||||
|
If you already have a PostgreSQL development server this is the quickest way to start and run the majority of the pgx
|
||||||
|
test suite. Some tests will be skipped that require server configuration changes (e.g. those testing different
|
||||||
|
authentication methods).
|
||||||
|
|
||||||
|
Create and setup a test database:
|
||||||
|
|
||||||
|
```
|
||||||
|
export PGDATABASE=pgx_test
|
||||||
|
createdb
|
||||||
|
psql -c 'create extension hstore;'
|
||||||
|
psql -c 'create extension ltree;'
|
||||||
|
psql -c 'create domain uint64 as numeric(20,0);'
|
||||||
|
```
|
||||||
|
|
||||||
|
Ensure a `postgres` user exists. This happens by default in normal PostgreSQL installs, but some installation methods
|
||||||
|
such as Homebrew do not.
|
||||||
|
|
||||||
|
```
|
||||||
|
createuser -s postgres
|
||||||
|
```
|
||||||
|
|
||||||
|
Ensure your `PGX_TEST_DATABASE` environment variable points to the database you just created and run the tests.
|
||||||
|
|
||||||
|
```
|
||||||
|
export PGX_TEST_DATABASE="host=/private/tmp database=pgx_test"
|
||||||
|
go test ./...
|
||||||
|
```
|
||||||
|
|
||||||
|
This will run the vast majority of the tests, but some tests will be skipped (e.g. those testing different connection methods).
|
||||||
|
|
||||||
|
### Creating a New PostgreSQL Cluster Exclusively for Testing Outside of a Devcontainer
|
||||||
|
|
||||||
|
The following environment variables need to be set both for initial setup and whenever the tests are run. (direnv is
|
||||||
|
highly recommended). Depending on your platform, you may need to change the host for `PGX_TEST_UNIX_SOCKET_CONN_STRING`.
|
||||||
|
|
||||||
|
```
|
||||||
|
export PGPORT=5015
|
||||||
|
export PGUSER=postgres
|
||||||
|
export PGDATABASE=pgx_test
|
||||||
|
export POSTGRESQL_DATA_DIR=postgresql
|
||||||
|
|
||||||
|
export PGX_TEST_DATABASE="host=127.0.0.1 database=pgx_test user=pgx_md5 password=secret"
|
||||||
|
export PGX_TEST_UNIX_SOCKET_CONN_STRING="host=/private/tmp database=pgx_test"
|
||||||
|
export PGX_TEST_TCP_CONN_STRING="host=127.0.0.1 database=pgx_test user=pgx_md5 password=secret"
|
||||||
|
export PGX_TEST_SCRAM_PASSWORD_CONN_STRING="host=127.0.0.1 user=pgx_scram password=secret database=pgx_test channel_binding=disable"
|
||||||
|
export PGX_TEST_SCRAM_PLUS_CONN_STRING="host=localhost user=pgx_ssl password=secret sslmode=verify-full sslrootcert=`pwd`/.testdb/ca.pem database=pgx_test channel_binding=require"
|
||||||
|
export PGX_TEST_MD5_PASSWORD_CONN_STRING="host=127.0.0.1 database=pgx_test user=pgx_md5 password=secret"
|
||||||
|
export PGX_TEST_PLAIN_PASSWORD_CONN_STRING="host=127.0.0.1 user=pgx_pw password=secret"
|
||||||
|
export PGX_TEST_TLS_CONN_STRING="host=localhost user=pgx_ssl password=secret sslmode=verify-full sslrootcert=`pwd`/.testdb/ca.pem channel_binding=disable"
|
||||||
|
export PGX_SSL_PASSWORD=certpw
|
||||||
|
export PGX_TEST_TLS_CLIENT_CONN_STRING="host=localhost user=pgx_sslcert sslmode=verify-full sslrootcert=`pwd`/.testdb/ca.pem database=pgx_test sslcert=`pwd`/.testdb/pgx_sslcert.crt sslkey=`pwd`/.testdb/pgx_sslcert.key"
|
||||||
|
```
|
||||||
|
|
||||||
|
Create a new database cluster.
|
||||||
|
|
||||||
|
```
|
||||||
|
initdb --locale=en_US -E UTF-8 --username=postgres .testdb/$POSTGRESQL_DATA_DIR
|
||||||
|
|
||||||
|
echo "listen_addresses = '127.0.0.1'" >> .testdb/$POSTGRESQL_DATA_DIR/postgresql.conf
|
||||||
|
echo "port = $PGPORT" >> .testdb/$POSTGRESQL_DATA_DIR/postgresql.conf
|
||||||
|
cat testsetup/postgresql_ssl.conf >> .testdb/$POSTGRESQL_DATA_DIR/postgresql.conf
|
||||||
|
cp testsetup/pg_hba.conf .testdb/$POSTGRESQL_DATA_DIR/pg_hba.conf
|
||||||
|
|
||||||
|
cd .testdb
|
||||||
|
|
||||||
|
# Generate CA, server, and encrypted client certificates.
|
||||||
|
go run ../testsetup/generate_certs.go
|
||||||
|
|
||||||
|
# Copy certificates to server directory and set permissions.
|
||||||
|
cp ca.pem $POSTGRESQL_DATA_DIR/root.crt
|
||||||
|
cp localhost.key $POSTGRESQL_DATA_DIR/server.key
|
||||||
|
chmod 600 $POSTGRESQL_DATA_DIR/server.key
|
||||||
|
cp localhost.crt $POSTGRESQL_DATA_DIR/server.crt
|
||||||
|
|
||||||
|
cd ..
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
Start the new cluster. This will be necessary whenever you are running pgx tests.
|
||||||
|
|
||||||
|
```
|
||||||
|
postgres -D .testdb/$POSTGRESQL_DATA_DIR
|
||||||
|
```
|
||||||
|
|
||||||
|
Setup the test database in the new cluster.
|
||||||
|
|
||||||
|
```
|
||||||
|
createdb
|
||||||
|
psql --no-psqlrc -f testsetup/postgresql_setup.sql
|
||||||
|
```
|
||||||
|
|
||||||
|
### PgBouncer
|
||||||
|
|
||||||
|
There are tests specific for PgBouncer that will be executed if `PGX_TEST_PGBOUNCER_CONN_STRING` is set.
|
||||||
|
|
||||||
|
### Optional Tests
|
||||||
|
|
||||||
|
pgx supports multiple connection types and means of authentication. These tests are optional. They will only run if the
|
||||||
|
appropriate environment variables are set. In addition, there may be tests specific to particular PostgreSQL versions,
|
||||||
|
non-PostgreSQL servers (e.g. CockroachDB), or connection poolers (e.g. PgBouncer). `go test ./... -v | grep SKIP` to see
|
||||||
|
if any tests are being skipped.
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user