Compare commits
12
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9cf47f4429 | ||
|
|
8ddc019cd2 | ||
|
|
fab6b0059e | ||
|
|
e1f5d4fdf0 | ||
|
|
bb1607e424 | ||
|
|
cf4834a36c | ||
|
|
9d8cbe7b81 | ||
|
|
e74c32ed50 | ||
|
|
146c48fd96 | ||
|
|
1f3abd3e0e | ||
|
|
6d620f77c3 | ||
|
|
f8286c063d |
+5
-3
@@ -21,13 +21,15 @@ ARG GO_BASE=golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7
|
|||||||
FROM ${GO_BASE} AS build
|
FROM ${GO_BASE} AS build
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
|
|
||||||
# Dependencies first, so a change to the source does not refetch them.
|
# **Nothing is fetched** (novox/hq to-be 45 Phase 1): every dependency is in vendor/, committed, so
|
||||||
|
# the image builds from this repository alone — the host's validator among them, whose module no
|
||||||
|
# public proxy is asked for. Dependencies first, so a change to the source does not re-copy them.
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
COPY vendor/ vendor/
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
ARG VERSION=development
|
ARG VERSION=development
|
||||||
RUN CGO_ENABLED=0 go build -trimpath \
|
RUN CGO_ENABLED=0 GOFLAGS=-mod=vendor GOPROXY=off go build -trimpath \
|
||||||
-ldflags "-s -w -X main.version=${VERSION}" \
|
-ldflags "-s -w -X main.version=${VERSION}" \
|
||||||
-o /mesh-controller ./cmd/mesh-controller
|
-o /mesh-controller ./cmd/mesh-controller
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@@ -676,9 +677,20 @@ type answers struct {
|
|||||||
// refused, until the switch — and while there is any, the mesh is not all well: the order the
|
// refused, until the switch — and while there is any, the mesh is not all well: the order the
|
||||||
// machines' modules are built in is the mesh's to keep, and this is where it says it is not kept.
|
// machines' modules are built in is the mesh's to keep, and this is where it says it is not kept.
|
||||||
unheld []catalogue.Unheld
|
unheld []catalogue.Unheld
|
||||||
// failing is every consumer a provider says it keeps failing (novox/hq ADR 0224): a provider's
|
// conditions is every open condition (novox/hq to-be 45 §2), urgent first and then oldest first:
|
||||||
// journal was the only place that said so for a day (04-ISSUES/179).
|
// what leads status, and what its all-well sentence needs to be none of, silenced ones included.
|
||||||
failing []inventory.ProviderStanding
|
// A provider failing a consumer is one of them (ADR 0224). conditionsUnread says why they could
|
||||||
|
// not be read when they could not — never read as none.
|
||||||
|
conditions []conditions.Condition
|
||||||
|
conditionsUnread string
|
||||||
|
// overflowing is every module whose identity overflows the bound of a provision it requires
|
||||||
|
// (novox/hq ADR 0225): its provider leaves it out of the grants and composes everything else, so
|
||||||
|
// this is the one place it is said across the mesh. Not well while there is any.
|
||||||
|
overflowing []catalogue.Overflow
|
||||||
|
// handActs is how many acts were done by hand in the last seven days (novox/hq to-be 45 §7), nil
|
||||||
|
// where the log is not on hand; handActsUnread why it could not be read when it could not.
|
||||||
|
handActs *int
|
||||||
|
handActsUnread string
|
||||||
}
|
}
|
||||||
|
|
||||||
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
|
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
|
||||||
|
|||||||
@@ -0,0 +1,104 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The streams and durable consumers the mesh's own traffic needs, derived once (novox/hq to-be 45 §4,
|
||||||
|
// D6 and D7).
|
||||||
|
//
|
||||||
|
// **What the controller asserts at its start and what the self-check expects to find are one
|
||||||
|
// derivation**, run against the bus to make them and against a recorder to list them. Two lists would
|
||||||
|
// drift, and a self-check comparing the bus with a second opinion of what should be there would find
|
||||||
|
// the drift rather than the fault.
|
||||||
|
|
||||||
|
// assertBusObjects brings every stream and consumer into being on r, and answers the machines that
|
||||||
|
// can now hear a declaration.
|
||||||
|
func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Raiser) ([]string, error) {
|
||||||
|
nodes, err := inv.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(nodes))
|
||||||
|
for _, n := range nodes {
|
||||||
|
names = append(names, n.Name)
|
||||||
|
}
|
||||||
|
if err := broker.Raise(r, names); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// The work queues of the mesh's own roles (novox/hq ADR 0121). The queue before the holder,
|
||||||
|
// deliberately: work queues until somebody arrives to do it, so assigning a build machine a week
|
||||||
|
// after something started asking for builds flushes the backlog instead of having lost it.
|
||||||
|
// With the seats' holders, so each role's work queue gets the consumer its holder takes
|
||||||
|
// work from. Passed as nil until the first live raise, which left the build machine bound to a
|
||||||
|
// consumer nothing had created (2026-09-28).
|
||||||
|
holders, err := seatHolders(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := broker.RaiseSeats(r, inventory.MeshSeats(), holders); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// And how every module hears what it consumes. Derived from the same records the user list is
|
||||||
|
// composed from, so a module the mesh grants a consumer's subjects has that consumer waiting.
|
||||||
|
// Done on every raise, not only when a credential is issued: every module moved onto this bus
|
||||||
|
// by the rollout was issued on the old one, and came up with nothing to bind to (2026-09-28).
|
||||||
|
consumers, err := moduleConsumers(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, c := range consumers {
|
||||||
|
if err := r.EnsureConsumer(c.Consumer); err != nil {
|
||||||
|
return nil, fmt.Errorf("how %s on %s hears what it consumes: %w", c.Module, c.Node, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return names, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// moduleConsumers is every module's durable consumer, from the records the user list is composed from.
|
||||||
|
func moduleConsumers(ctx context.Context, inv *inventory.Inventory) ([]broker.ModuleConsumer, error) {
|
||||||
|
records, err := inv.BusRecords(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
users, err := broker.Users(records)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return broker.ConsumersOf(users), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// moduleConsumerCount is how many modules hear what they consume, for the raise's one line.
|
||||||
|
func moduleConsumerCount(ctx context.Context, inv *inventory.Inventory) (int, error) {
|
||||||
|
consumers, err := moduleConsumers(ctx, inv)
|
||||||
|
return len(consumers), err
|
||||||
|
}
|
||||||
|
|
||||||
|
// expectedBusObjects is every stream and consumer assertBusObjects would make, made nowhere.
|
||||||
|
func expectedBusObjects(ctx context.Context, inv *inventory.Inventory) ([]broker.Stream, []broker.Consumer, error) {
|
||||||
|
var rec recordingRaiser
|
||||||
|
if _, err := assertBusObjects(ctx, inv, &rec); err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("what the bus should hold cannot be worked out: %w", err)
|
||||||
|
}
|
||||||
|
return rec.streams, rec.consumers, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// recordingRaiser keeps what it was asked to assert and asserts nothing.
|
||||||
|
type recordingRaiser struct {
|
||||||
|
streams []broker.Stream
|
||||||
|
consumers []broker.Consumer
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *recordingRaiser) EnsureStream(s broker.Stream) error {
|
||||||
|
r.streams = append(r.streams, s)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *recordingRaiser) EnsureConsumer(c broker.Consumer) error {
|
||||||
|
r.consumers = append(r.consumers, c)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -35,10 +35,10 @@ func TestAPushAnswersBeforeItSends(t *testing.T) {
|
|||||||
args map[string]any
|
args map[string]any
|
||||||
want bool
|
want bool
|
||||||
}{
|
}{
|
||||||
{"push", map[string]any{"node": "anchor"}, true},
|
{"push", map[string]any{"node": "anchor", "why": "w"}, true},
|
||||||
{"push", map[string]any{}, true},
|
{"push", map[string]any{"why": "w"}, true},
|
||||||
{"command", map[string]any{"command": "push anchor"}, true},
|
{"command", map[string]any{"command": "push anchor --why w"}, true},
|
||||||
{"command", map[string]any{"command": "push --behind"}, true},
|
{"command", map[string]any{"command": "push --behind --why=w"}, true},
|
||||||
{"command", map[string]any{"command": "builds"}, false},
|
{"command", map[string]any{"command": "builds"}, false},
|
||||||
{"status", map[string]any{}, false},
|
{"status", map[string]any{}, false},
|
||||||
{"assign", map[string]any{"node": "anchor", "module": "m"}, false},
|
{"assign", map[string]any{"node": "anchor", "module": "m"}, false},
|
||||||
|
|||||||
@@ -25,7 +25,17 @@ import (
|
|||||||
// mesh seat is judged fully only at registration. A seat another module declares is unknown unless
|
// mesh seat is judged fully only at registration. A seat another module declares is unknown unless
|
||||||
// that module's manifest is passed too. Both are printed as a note, not as a problem — a check that
|
// that module's manifest is passed too. Both are printed as a note, not as a problem — a check that
|
||||||
// refused what it could not see would teach people to ignore it.
|
// refused what it could not see would teach people to ignore it.
|
||||||
|
//
|
||||||
|
// **And every identity against every bound it meets** (novox/hq ADR 0225, issue 263): each module's
|
||||||
|
// identity, on a machine whose name is `longestMachine` characters, against the bound of every
|
||||||
|
// provision it wants that a manifest given here offers. An overflow is refused in the pull request
|
||||||
|
// that introduces it — a new requirement, a lowered bound, a longer slug — instead of on the
|
||||||
|
// provider's machine when a real machine's name first meets the module's.
|
||||||
func moduleCheck(paths []string, out io.Writer) error {
|
func moduleCheck(paths []string, out io.Writer) error {
|
||||||
|
return moduleCheckFor(paths, catalogue.DefaultLongestMachine, out)
|
||||||
|
}
|
||||||
|
|
||||||
|
func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||||
if len(paths) == 0 {
|
if len(paths) == 0 {
|
||||||
return errors.New("module check <manifest.json>... — one file per module; pass every " +
|
return errors.New("module check <manifest.json>... — one file per module; pass every " +
|
||||||
"manifest of a repository together so the rules between them are checked too")
|
"manifest of a repository together so the rules between them are checked too")
|
||||||
@@ -74,6 +84,15 @@ func moduleCheck(paths []string, out io.Writer) error {
|
|||||||
}
|
}
|
||||||
failed += len(problems)
|
failed += len(problems)
|
||||||
|
|
||||||
|
// Between the manifests too: an identity against the bounds of the provisions it wants, which
|
||||||
|
// only the provider's manifest states.
|
||||||
|
identities := catalogue.IdentityProblems(shelf, longestMachine)
|
||||||
|
sort.Strings(identities)
|
||||||
|
for _, p := range identities {
|
||||||
|
fmt.Fprintln(out, p)
|
||||||
|
}
|
||||||
|
failed += len(identities)
|
||||||
|
|
||||||
var names []string
|
var names []string
|
||||||
for name := range shelf {
|
for name := range shelf {
|
||||||
names = append(names, name)
|
names = append(names, name)
|
||||||
@@ -109,7 +128,8 @@ func moduleCheck(paths []string, out io.Writer) error {
|
|||||||
}
|
}
|
||||||
fmt.Fprintf(out, "%d manifest(s) checked. Judged against the seats this binary carries; a claim on "+
|
fmt.Fprintf(out, "%d manifest(s) checked. Judged against the seats this binary carries; a claim on "+
|
||||||
"one of the mesh's own seats is judged fully at registration, and a seat declared by a "+
|
"one of the mesh's own seats is judged fully at registration, and a seat declared by a "+
|
||||||
"module not given here reads as unknown\n", len(paths))
|
"module not given here reads as unknown. Identities judged on a %d-character machine name, "+
|
||||||
|
"against the bounds of the providers given here\n", len(paths), longestMachine)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,431 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"slices"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What is wrong, kept until observation says it is not (novox/hq to-be 45 §2, ADR 0227).
|
||||||
|
//
|
||||||
|
// **`status` used to be the only place the mesh said it was wrong, and only to whoever asked.** Every
|
||||||
|
// core failure of research 031 was found by a person looking. A condition is the mesh saying it: raised
|
||||||
|
// by a watchdog when a signal is late (signals.go), by a probe when an invariant does not hold
|
||||||
|
// (doctor.go), or by an event a provider sends (standing.go); kept on the bus with since-when and
|
||||||
|
// evidence; said on the bus as it changes, for the operator's channel to carry; and cleared when an
|
||||||
|
// observation says it is resolved. Nobody resolves one by hand. A person who knows silences it, for a
|
||||||
|
// while, with a reason, and that is recorded as a hand act.
|
||||||
|
|
||||||
|
// conditionsFrom is the serving controller's keeper; nil in any other process, which opens its own.
|
||||||
|
var conditionsFrom *conditions.Keeper
|
||||||
|
|
||||||
|
// keeperOn is a keeper over the store on a connection, saying its transitions on that connection.
|
||||||
|
func keeperOn(ctx context.Context, conn *nats.Conn) (*conditions.Keeper, error) {
|
||||||
|
store, history, err := conditions.OnTheBus(ctx, conn)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
js, err := conn.JetStream()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history,
|
||||||
|
Teller: link.OverNATS{Conn: conn, JS: js},
|
||||||
|
Say: func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) },
|
||||||
|
// What status leads with changed: composed again soon (a nudge outside the serving controller
|
||||||
|
// does nothing).
|
||||||
|
Changed: statusFrom.nudge}), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// withKeeper runs f with the serving controller's keeper, or one of its own that says everything
|
||||||
|
// it was given before it returns.
|
||||||
|
func withKeeper(ctx context.Context, f func(*conditions.Keeper) error) error {
|
||||||
|
if conditionsFrom != nil {
|
||||||
|
return f(conditionsFrom)
|
||||||
|
}
|
||||||
|
return onTheBus(func(conn *nats.Conn) error {
|
||||||
|
k, err := keeperOn(ctx, conn)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
flushing, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
k.Close(flushing)
|
||||||
|
}()
|
||||||
|
return f(k)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// openConditions is every open condition, for `status` and `node show`: from the serving keeper, or
|
||||||
|
// read from the bus. Where there is no bus to read it from, it says so — never "none open".
|
||||||
|
func openConditions(ctx context.Context) ([]conditions.Condition, error) {
|
||||||
|
if conditionsFrom != nil {
|
||||||
|
return conditionsFrom.Open(ctx)
|
||||||
|
}
|
||||||
|
if _, err := broker.BusAddress(); err != nil {
|
||||||
|
return nil, fmt.Errorf("this process has no bus to read the conditions from: %w", err)
|
||||||
|
}
|
||||||
|
var out []conditions.Condition
|
||||||
|
err := onTheBus(func(conn *nats.Conn) error {
|
||||||
|
store, _, err := conditions.OnTheBus(ctx, conn)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
reading, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
out, err = conditions.Read(reading, store)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// conditionsUsage is how the verb is typed.
|
||||||
|
const conditionsUsage = "conditions [--scope S] [--severity urgent|warning] [--machine M] [--json] | " +
|
||||||
|
"conditions show <key> | conditions silence <key> --for <duration> --why <text> | " +
|
||||||
|
"conditions history [--days N] [--key K] [--json]"
|
||||||
|
|
||||||
|
// conditionsCommand is `conditions`, `conditions show`, `conditions silence` and `conditions history`.
|
||||||
|
func conditionsCommand(ctx context.Context, args []string) error {
|
||||||
|
sub := "list"
|
||||||
|
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||||
|
sub, args = args[0], args[1:]
|
||||||
|
}
|
||||||
|
switch sub {
|
||||||
|
case "list":
|
||||||
|
return listConditions(ctx, args)
|
||||||
|
case "show":
|
||||||
|
return showCondition(ctx, args)
|
||||||
|
case "silence":
|
||||||
|
return silenceCondition(ctx, args)
|
||||||
|
case "history":
|
||||||
|
return conditionHistory(ctx, args)
|
||||||
|
}
|
||||||
|
return errors.New(conditionsUsage)
|
||||||
|
}
|
||||||
|
|
||||||
|
func listConditions(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("conditions", flag.ContinueOnError)
|
||||||
|
scope := set.String("scope", "", "only this scope: "+strings.Join(conditions.Scopes, ", "))
|
||||||
|
severity := set.String("severity", "", "only urgent, or only warning")
|
||||||
|
machine := set.String("machine", "", "only those about this machine")
|
||||||
|
asJSON := set.Bool("json", false, "as data")
|
||||||
|
if rest, err := parseAround(set, args); err != nil {
|
||||||
|
return err
|
||||||
|
} else if len(rest) > 0 {
|
||||||
|
return errors.New(conditionsUsage)
|
||||||
|
}
|
||||||
|
if *severity != "" && *severity != string(conditions.Urgent) && *severity != string(conditions.Warning) {
|
||||||
|
return fmt.Errorf("a severity is urgent or warning, not %q", *severity)
|
||||||
|
}
|
||||||
|
open, err := openConditions(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var out []conditions.Condition
|
||||||
|
for _, c := range open {
|
||||||
|
if (*scope == "" || c.Subject.Scope == *scope) && (*severity == "" || string(c.Severity) == *severity) &&
|
||||||
|
(*machine == "" || concerns(c, *machine)) {
|
||||||
|
out = append(out, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if *asJSON {
|
||||||
|
if out == nil {
|
||||||
|
out = []conditions.Condition{}
|
||||||
|
}
|
||||||
|
return printJSON(map[string]any{"conditions": out, "open": len(open),
|
||||||
|
"note": "urgent first, then oldest first; a condition clears when observation says so, never by hand"})
|
||||||
|
}
|
||||||
|
if len(out) == 0 {
|
||||||
|
if len(open) == 0 {
|
||||||
|
fmt.Println("no open conditions")
|
||||||
|
} else {
|
||||||
|
fmt.Printf("none of the %d open condition(s) is about that\n", len(open))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, line := range conditionLines(out, time.Now()) {
|
||||||
|
fmt.Println(line)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// concerns says whether a condition is about a machine: it names it, or its key does.
|
||||||
|
func concerns(c conditions.Condition, machine string) bool {
|
||||||
|
if c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
for _, part := range strings.Split(c.Key, ".") {
|
||||||
|
if part == machine {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// conditionLines is how a list of conditions reads: one line each, its silence under it.
|
||||||
|
func conditionLines(list []conditions.Condition, now time.Time) []string {
|
||||||
|
var out []string
|
||||||
|
for _, c := range list {
|
||||||
|
times := ""
|
||||||
|
if c.Count > 1 {
|
||||||
|
times = fmt.Sprintf(", raised %d times", c.Count)
|
||||||
|
}
|
||||||
|
out = append(out, fmt.Sprintf(" %-7s %s — %s (since %s%s)", strings.ToUpper(string(c.Severity)),
|
||||||
|
c.Key, c.Summary, c.Raised.Local().Format("2006-01-02 15:04"), times))
|
||||||
|
if c.SilencedAt(now) {
|
||||||
|
out = append(out, fmt.Sprintf(" silenced until %s by %s: %s",
|
||||||
|
c.Silenced.Until.Local().Format("2006-01-02 15:04"), c.Silenced.By, c.Silenced.Why))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func showCondition(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("conditions show", flag.ContinueOnError)
|
||||||
|
asJSON := set.Bool("json", false, "as data")
|
||||||
|
rest, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(rest) != 1 {
|
||||||
|
return errors.New("conditions show <key>")
|
||||||
|
}
|
||||||
|
key := rest[0]
|
||||||
|
var c conditions.Condition
|
||||||
|
var found bool
|
||||||
|
if conditionsFrom != nil {
|
||||||
|
c, found, err = conditionsFrom.Get(ctx, key)
|
||||||
|
} else {
|
||||||
|
err = onTheBus(func(conn *nats.Conn) error {
|
||||||
|
store, _, err := conditions.OnTheBus(ctx, conn)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
c, found, err = conditions.ReadOne(ctx, store, key)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
return fmt.Errorf("no condition %s is open — `conditions` lists those that are, and `conditions "+
|
||||||
|
"history --key %s` what became of it", key, key)
|
||||||
|
}
|
||||||
|
if *asJSON {
|
||||||
|
return printJSON(c)
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
fmt.Printf("%s %s\n %s\n\n", strings.ToUpper(string(c.Severity)), c.Key, c.Summary)
|
||||||
|
fmt.Printf(" kind %s\n about %s %s", c.Kind, c.Subject.Scope, c.Subject.ID)
|
||||||
|
if c.Subject.Machine != "" {
|
||||||
|
fmt.Printf(", on %s", c.Subject.Machine)
|
||||||
|
}
|
||||||
|
fmt.Printf("\n raised by %s\n since %s (%s ago), observed %d time(s), last %s ago\n",
|
||||||
|
c.Source, c.Raised.Local().Format("2006-01-02 15:04:05"), roughly(now.Sub(c.Raised)), c.Observations,
|
||||||
|
now.Sub(c.LastObserved).Round(time.Second))
|
||||||
|
if c.Count > 1 {
|
||||||
|
fmt.Printf(" raised %d times, each within ten minutes of clearing\n", c.Count)
|
||||||
|
}
|
||||||
|
fmt.Printf(" resolved by %s\n", resolverWords(c.Resolver))
|
||||||
|
if c.Silenced != nil {
|
||||||
|
fmt.Printf(" silenced until %s by %s: %s\n", c.Silenced.Until.Local().Format("2006-01-02 15:04"),
|
||||||
|
c.Silenced.By, c.Silenced.Why)
|
||||||
|
}
|
||||||
|
if len(c.Tried) > 0 {
|
||||||
|
fmt.Println("\n tried:")
|
||||||
|
for _, t := range c.Tried {
|
||||||
|
fmt.Printf(" %s %s: %s\n", t.At.Local().Format("2006-01-02 15:04"), t.What, t.Outcome)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Println("\n evidence, newest first:")
|
||||||
|
for _, e := range c.Evidence {
|
||||||
|
fmt.Printf(" %s %s\n", e.At.Local().Format("2006-01-02 15:04:05"), e.Said)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolverWords(r string) string {
|
||||||
|
switch r {
|
||||||
|
case conditions.ResolverSelf:
|
||||||
|
return "itself: it clears when observation says it is resolved"
|
||||||
|
case conditions.ResolverOperator:
|
||||||
|
return "the operator: nothing in the mesh will repair it"
|
||||||
|
case conditions.ResolverAgent:
|
||||||
|
return "an agent"
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// silenceCondition stops a condition's messages for a while (to-be 45 §2). A hand act: recorded with
|
||||||
|
// who and why before it is done, its cause the condition's kind unless one is given.
|
||||||
|
func silenceCondition(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("conditions silence", flag.ContinueOnError)
|
||||||
|
forFlag := set.String("for", "", "how long: 30m, 4h, 2d — at most 7d")
|
||||||
|
acts := addHandActFlags(set)
|
||||||
|
rest, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(rest) != 1 {
|
||||||
|
return errors.New("conditions silence <key> --for <duration> --why <text>")
|
||||||
|
}
|
||||||
|
key := rest[0]
|
||||||
|
if err := acts.require("conditions silence"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
d, err := parseFor(*forFlag)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if d > conditions.MaxSilence {
|
||||||
|
return fmt.Errorf("a condition is silenced for at most %s at once; past it, say so again", conditions.MaxSilence)
|
||||||
|
}
|
||||||
|
return withKeeper(ctx, func(k *conditions.Keeper) error {
|
||||||
|
c, found, err := k.Get(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
return fmt.Errorf("no condition %s is open — `conditions` lists them. Nothing was silenced", key)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(*acts.cause) == "" {
|
||||||
|
*acts.cause = c.Kind
|
||||||
|
}
|
||||||
|
*acts.condition = key
|
||||||
|
acts.record(ctx, "conditions silence", []string{key, "--for", *forFlag})
|
||||||
|
held, err := k.Silence(ctx, key, d, link.Caller(), *acts.why)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s is silenced until %s: no message is sent for it until then. It is still open, and "+
|
||||||
|
"`status` still says it; it clears when observation says it is resolved\n",
|
||||||
|
held.Key, held.Silenced.Until.Local().Format("2006-01-02 15:04"))
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseFor reads a duration, days included.
|
||||||
|
func parseFor(s string) (time.Duration, error) {
|
||||||
|
s = strings.TrimSpace(s)
|
||||||
|
if s == "" {
|
||||||
|
return 0, errors.New("say for how long: --for 30m, 4h or 2d")
|
||||||
|
}
|
||||||
|
if days, ok := strings.CutSuffix(s, "d"); ok {
|
||||||
|
n, err := strconv.Atoi(days)
|
||||||
|
if err != nil || n <= 0 {
|
||||||
|
return 0, fmt.Errorf("%q is not a number of days", s)
|
||||||
|
}
|
||||||
|
return time.Duration(n) * 24 * time.Hour, nil
|
||||||
|
}
|
||||||
|
d, err := time.ParseDuration(s)
|
||||||
|
if err != nil || d <= 0 {
|
||||||
|
return 0, fmt.Errorf("%q is not a duration: 30m, 4h or 2d", s)
|
||||||
|
}
|
||||||
|
return d, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func conditionHistory(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("conditions history", flag.ContinueOnError)
|
||||||
|
days := set.Int("days", 7, "how many days back, at most 90")
|
||||||
|
key := set.String("key", "", "only this condition")
|
||||||
|
asJSON := set.Bool("json", false, "as data")
|
||||||
|
if rest, err := parseAround(set, args); err != nil {
|
||||||
|
return err
|
||||||
|
} else if len(rest) > 0 {
|
||||||
|
return errors.New("conditions history [--days N] [--key K] [--json]")
|
||||||
|
}
|
||||||
|
since := time.Now().Add(-time.Duration(*days) * 24 * time.Hour)
|
||||||
|
var events []conditions.Event
|
||||||
|
read := func(h conditions.History) error {
|
||||||
|
var err error
|
||||||
|
events, err = h.Since(ctx, since)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var err error
|
||||||
|
if conditionsFrom != nil {
|
||||||
|
events, err = conditionsFrom.HistorySince(ctx, since)
|
||||||
|
} else {
|
||||||
|
err = onTheBus(func(conn *nats.Conn) error {
|
||||||
|
_, history, err := conditions.OnTheBus(ctx, conn)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return read(history)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var out []conditions.Event
|
||||||
|
for _, e := range events {
|
||||||
|
if *key == "" || e.Key == *key {
|
||||||
|
out = append(out, e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if *asJSON {
|
||||||
|
if out == nil {
|
||||||
|
out = []conditions.Event{}
|
||||||
|
}
|
||||||
|
return printJSON(map[string]any{"history": out, "days": *days})
|
||||||
|
}
|
||||||
|
if len(out) == 0 {
|
||||||
|
fmt.Printf("nothing was raised, changed or cleared in the last %d day(s)\n", *days)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, e := range out {
|
||||||
|
line := fmt.Sprintf("%s %-13s %s", e.At.Local().Format("2006-01-02 15:04:05"), e.Change, e.Key)
|
||||||
|
switch e.Change {
|
||||||
|
case conditions.ChangeRaised, conditions.ChangeReopened:
|
||||||
|
line += " — " + e.Summary
|
||||||
|
case conditions.ChangeSeverity:
|
||||||
|
line += fmt.Sprintf(" — %s, was %s", e.Severity, e.Was)
|
||||||
|
case conditions.ChangeResolver:
|
||||||
|
line += fmt.Sprintf(" — %s, was %s", e.Resolver, e.Was)
|
||||||
|
default:
|
||||||
|
if e.Why != "" {
|
||||||
|
line += " — " + e.Why
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Println(line)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// printConditions is the status section that leads it: every open condition, urgent first, oldest
|
||||||
|
// first, silenced ones with their expiry (to-be 45 §2). A store that could not be read is said, and
|
||||||
|
// is not "none open".
|
||||||
|
func printConditions(list []conditions.Condition, unread string, now time.Time) {
|
||||||
|
if unread != "" {
|
||||||
|
fmt.Printf("the open conditions could NOT be read, so whether anything is wrong is not known: %s\n\n", unread)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if len(list) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
urgent := 0
|
||||||
|
for _, c := range list {
|
||||||
|
if c.Severity == conditions.Urgent {
|
||||||
|
urgent++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Printf("%d open condition(s), %d urgent:\n\n", len(list), urgent)
|
||||||
|
for _, line := range conditionLines(list, now) {
|
||||||
|
fmt.Println(line)
|
||||||
|
}
|
||||||
|
fmt.Printf("\n `conditions show <key>` says more; each clears when observation says it is resolved, " +
|
||||||
|
"never by hand — `conditions silence <key> --for <d> --why <text>` stops its messages\n\n")
|
||||||
|
}
|
||||||
@@ -0,0 +1,107 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The verbs of the condition store (novox/hq to-be 45 §2) as the console reaches them, and status led
|
||||||
|
// by what is open.
|
||||||
|
|
||||||
|
func TestTheConditionsVerbComposesEachShape(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
args map[string]any
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{map[string]any{}, "conditions --json"},
|
||||||
|
{map[string]any{"severity": "urgent", "machine": "ace"}, "conditions --json --severity urgent --machine ace"},
|
||||||
|
{map[string]any{"key": "machine.ace.silent"}, "conditions show machine.ace.silent --json"},
|
||||||
|
{map[string]any{"history": "true", "days": "3", "key": "machine.ace.silent"},
|
||||||
|
"conditions history --json --days 3 --key machine.ace.silent"},
|
||||||
|
{map[string]any{"silence": "machine.ace.silent", "for": "2h", "why": "on the train"},
|
||||||
|
"conditions silence machine.ace.silent --for 2h --why on the train"},
|
||||||
|
{map[string]any{"run": "true"}, "doctor run --json"},
|
||||||
|
{map[string]any{}, "doctor --json"},
|
||||||
|
} {
|
||||||
|
verb := "conditions"
|
||||||
|
if strings.HasPrefix(c.want, "doctor") {
|
||||||
|
verb = "doctor"
|
||||||
|
}
|
||||||
|
argv, err := argvFor(verb, c.args)
|
||||||
|
if err != nil || strings.Join(argv, " ") != c.want {
|
||||||
|
t.Errorf("%s %v composed %q (%v), want %q", verb, c.args, strings.Join(argv, " "), err, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, c := range []struct {
|
||||||
|
verb string
|
||||||
|
args map[string]any
|
||||||
|
}{
|
||||||
|
{"conditions", map[string]any{"silence": "machine.ace.silent", "why": "x"}}, // no for
|
||||||
|
{"doctor", map[string]any{"run": "true", "signals": "true"}}, // two at once
|
||||||
|
{"conditions", map[string]any{"silence": "machine.ace.silent", "for": "1h", "why": "x", "days": "3"}}, // passed over
|
||||||
|
} {
|
||||||
|
if argv, err := argvFor(c.verb, c.args); err == nil {
|
||||||
|
t.Errorf("%s %v composed %v", c.verb, c.args, argv)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if repairingCommand([]string{"conditions", "silence", "k"}) != "conditions silence" {
|
||||||
|
t.Error("a silence is not a hand act")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A silence through the verb is recorded and bounded**; the condition stays open.
|
||||||
|
func TestASilenceThroughTheVerbHoldsAndTheConditionStaysOpen(t *testing.T) {
|
||||||
|
k, _ := withConditionsInMemory(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if _, err := k.Observe(ctx, conditions.Observation{Scope: conditions.ScopeMachine, ID: "ace", Kind: "silent",
|
||||||
|
Severity: conditions.Warning, Summary: "ace is silent", Source: "S1"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := conditionsCommand(ctx, []string{"silence", "machine.ace.silent", "--for", "2d"}); err == nil {
|
||||||
|
t.Fatal("silenced without saying why")
|
||||||
|
}
|
||||||
|
if err := conditionsCommand(ctx, []string{"silence", "machine.ace.silent", "--for", "8d", "--why", "x"}); err == nil {
|
||||||
|
t.Fatal("silenced for more than a week")
|
||||||
|
}
|
||||||
|
said := printed(t, func() error {
|
||||||
|
return conditionsCommand(ctx, []string{"silence", "machine.ace.silent", "--for", "2d", "--why", "on the train"})
|
||||||
|
})
|
||||||
|
if !strings.Contains(said, "is silenced until") {
|
||||||
|
t.Fatalf("%s", said)
|
||||||
|
}
|
||||||
|
c, found, _ := k.Get(ctx, "machine.ace.silent")
|
||||||
|
if !found || c.Silenced == nil || c.Silenced.Why != "on the train" {
|
||||||
|
t.Fatalf("%+v", c)
|
||||||
|
}
|
||||||
|
listed := printed(t, func() error { return conditionsCommand(ctx, nil) })
|
||||||
|
if !strings.Contains(listed, "machine.ace.silent") || !strings.Contains(listed, "silenced until") {
|
||||||
|
t.Fatalf("%s", listed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Conditions that cannot be read are not none open**: status says so, and is not well.
|
||||||
|
func TestUnreadableConditionsAreNotAWellMesh(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
_, store := withConditionsInMemory(t)
|
||||||
|
store.Fail = errors.New("the bus is away")
|
||||||
|
asked, err := theThreeQuestions(t.Context(), open)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if asked.well() || asked.conditionsUnread == "" {
|
||||||
|
t.Fatalf("well with its conditions unread: %+v", asked.conditionsUnread)
|
||||||
|
}
|
||||||
|
said := printed(t, func() error { return printStatus(asked) })
|
||||||
|
if !strings.HasPrefix(said, "the open conditions could NOT be read") || strings.Contains(said, "no open conditions") {
|
||||||
|
t.Fatalf("%s", said)
|
||||||
|
}
|
||||||
|
store.Fail = nil
|
||||||
|
asked, _ = theThreeQuestions(t.Context(), open)
|
||||||
|
said = printed(t, func() error { return printStatus(asked) })
|
||||||
|
if asked.well() && !strings.Contains(said, "no open conditions;") {
|
||||||
|
t.Fatalf("the all-well sentence does not say no conditions are open:\n%s", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,536 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The self-check: `doctor` (novox/hq to-be 45 §4, ADR 0227 rule 6).
|
||||||
|
//
|
||||||
|
// **The design's invariants, run against the running mesh.** A probe is one live invariant of a
|
||||||
|
// design — every machine's declaration composes and validates, every resolver answers, every seat's
|
||||||
|
// holder answers, every stream and consumer is there as defined — with an id, a bound, and the
|
||||||
|
// condition it raises when the invariant does not hold. The serving controller runs the registry every
|
||||||
|
// five minutes, each probe given thirty seconds; a probe that errors or does not finish raises
|
||||||
|
// `probe-failed` for itself, because an unanswered probe is never a pass. Every run ends with a
|
||||||
|
// heartbeat on the bus (`doctor-heartbeat`, S10), which mesh-watcher listens for from a second
|
||||||
|
// machine: a controller that stops checking is itself said, through a channel that does not pass
|
||||||
|
// through it.
|
||||||
|
//
|
||||||
|
// `doctor` answers the last run's verdict at once; `doctor run` runs now; `doctor probes` lists the
|
||||||
|
// registry; `doctor signals` says, for every row of the signals table, the age of its newest signal.
|
||||||
|
|
||||||
|
// The self-check's clocks.
|
||||||
|
var (
|
||||||
|
// doctorEvery is how often the registry runs; doctorFirstAfter how long after the controller
|
||||||
|
// starts the first run waits, so what the controller hears at its start has arrived.
|
||||||
|
doctorEvery = 5 * time.Minute
|
||||||
|
doctorFirstAfter = time.Minute
|
||||||
|
// probeWithin is each probe's bound.
|
||||||
|
probeWithin = 30 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
// The verdicts a probe can have.
|
||||||
|
const (
|
||||||
|
verdictPass = "pass"
|
||||||
|
verdictFail = "fail"
|
||||||
|
verdictFailedToRun = "failed-to-run"
|
||||||
|
verdictDeferred = "deferred"
|
||||||
|
)
|
||||||
|
|
||||||
|
// probe is one live invariant.
|
||||||
|
type probe struct {
|
||||||
|
ID string
|
||||||
|
Asserts string
|
||||||
|
From string
|
||||||
|
// Kind is the condition kind raised when the invariant does not hold.
|
||||||
|
Kind string
|
||||||
|
Phase int
|
||||||
|
// Deferred says why it is not run yet; empty for one that is.
|
||||||
|
Deferred string
|
||||||
|
// Asks are the seat verbs it calls. A probe may call no other (askSeatTool refuses), and the
|
||||||
|
// controller's grant names every one (a test over this registry): a probe whose question the bus
|
||||||
|
// refuses checks nothing (D8, 2026-10-06).
|
||||||
|
Asks []broker.SeatVerb
|
||||||
|
run func(ctx context.Context, d *doctor) ([]conditions.Observation, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeRegistry is the registry, in to-be 45's order. **The registry is the design's live form**: a
|
||||||
|
// probe added to a design is a row added here.
|
||||||
|
var probeRegistry = []probe{
|
||||||
|
{ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation",
|
||||||
|
From: "issues 236, 263", Kind: "declaration-refused", Phase: 1, run: probeDeclarations},
|
||||||
|
{ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6",
|
||||||
|
From: "issue 262", Kind: "resolver-wrong", Phase: 1, run: probeResolvers},
|
||||||
|
{ID: "D3", Asserts: "every seat on record that serves verbs has a live holder that answers, on every " +
|
||||||
|
"machine that is heard from", From: "issues 208, 218", Kind: "holder-silent", Phase: 1, run: probeHolders},
|
||||||
|
{ID: "D4", Asserts: "every kept archive is held by a manifest", From: "issue 253",
|
||||||
|
Kind: "archives-unheld", Phase: 1, run: probeArchives},
|
||||||
|
{ID: "D5", Asserts: "exactly one lease holder; no message from a stale epoch in the last interval",
|
||||||
|
From: "issue 204", Kind: "lease-split", Phase: 2,
|
||||||
|
Deferred: "the lease and epoch are built in Phase 2 (to-be 45 §6): nothing holds one yet"},
|
||||||
|
{ID: "D6", Asserts: "every durable consumer the mesh expects exists with its definition, and is near its " +
|
||||||
|
"stream's head", From: "issues 248, 266", Kind: "consumer-wrong", Phase: 1, run: probeConsumers},
|
||||||
|
{ID: "D7", Asserts: "every stream the controller defines exists with its definition, and its own buckets",
|
||||||
|
From: "issue 208", Kind: "stream-wrong", Phase: 1, run: probeStreams},
|
||||||
|
{ID: "D8", Asserts: "no address the mesh owns — a machine's private address or its endpoint — is in a ban list",
|
||||||
|
From: "issue 238", Kind: "own-address-banned", Phase: 1, run: probeBans,
|
||||||
|
Asks: []broker.SeatVerb{{Seat: "node-intrusion-prevention", Verb: "banned"}}},
|
||||||
|
{ID: "D9", Asserts: "status answers in full within ten seconds, from a summary composed lately",
|
||||||
|
From: "issue 265", Kind: "status-slow", Phase: 1, run: probeStatus},
|
||||||
|
{ID: "D10", Asserts: "every machine runs the node-engine and node tools builds the mesh holds, or is inside " +
|
||||||
|
"a plan's window", From: "the version split", Kind: "core-behind", Phase: 1, run: probeCoreBuilds},
|
||||||
|
{ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals",
|
||||||
|
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs},
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeVerdict is one probe's outcome in a run.
|
||||||
|
type probeVerdict struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Verdict string `json:"verdict"`
|
||||||
|
Found []string `json:"found,omitempty"`
|
||||||
|
Error string `json:"error,omitempty"`
|
||||||
|
Took string `json:"took,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// doctorCounts are a run's verdicts, counted.
|
||||||
|
type doctorCounts struct {
|
||||||
|
Passed int `json:"passed"`
|
||||||
|
Failed int `json:"failed"`
|
||||||
|
FailedToRun int `json:"failed-to-run"`
|
||||||
|
Deferred int `json:"deferred"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// doctorRun is one run of the registry, and the body of its heartbeat.
|
||||||
|
type doctorRun struct {
|
||||||
|
Run string `json:"run"`
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
Started time.Time `json:"started"`
|
||||||
|
Took string `json:"took"`
|
||||||
|
IntervalSeconds int `json:"interval-seconds"`
|
||||||
|
Counts doctorCounts `json:"counts"`
|
||||||
|
Probes []probeVerdict `json:"probes"`
|
||||||
|
// Controller is the machine that ran it, and Why what started it: the schedule, or a person.
|
||||||
|
Controller string `json:"controller"`
|
||||||
|
Why string `json:"why"`
|
||||||
|
// Unsaid is how many condition transitions this controller could not say, since it started.
|
||||||
|
Unsaid int `json:"unsaid,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// doctor is the registry and what its probes need.
|
||||||
|
type doctor struct {
|
||||||
|
open *stores
|
||||||
|
js *broker.JetStream
|
||||||
|
keeper *conditions.Keeper
|
||||||
|
teller conditions.Teller
|
||||||
|
watchdogs *watchdogs
|
||||||
|
host string
|
||||||
|
|
||||||
|
running sync.Mutex
|
||||||
|
mu sync.Mutex
|
||||||
|
last *doctorRun
|
||||||
|
ended time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// lastRunEnded is when the last run ended; zero before the first.
|
||||||
|
func (d *doctor) lastRunEnded() time.Time {
|
||||||
|
d.mu.Lock()
|
||||||
|
defer d.mu.Unlock()
|
||||||
|
return d.ended
|
||||||
|
}
|
||||||
|
|
||||||
|
// lastRun is the last run's verdict; nil before the first.
|
||||||
|
func (d *doctor) lastRun() *doctorRun {
|
||||||
|
d.mu.Lock()
|
||||||
|
defer d.mu.Unlock()
|
||||||
|
return d.last
|
||||||
|
}
|
||||||
|
|
||||||
|
// keep runs the registry on its schedule until ctx ends.
|
||||||
|
func (d *doctor) keep(ctx context.Context) {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-time.After(doctorFirstAfter):
|
||||||
|
}
|
||||||
|
tick := time.NewTicker(doctorEvery)
|
||||||
|
defer tick.Stop()
|
||||||
|
for {
|
||||||
|
// Only the controller acting checks the mesh on a schedule: one standing by would say a
|
||||||
|
// heartbeat for a self-check that is not the mesh's.
|
||||||
|
if d.watchdogs == nil || d.watchdogs.acting == nil || d.watchdogs.acting() {
|
||||||
|
d.runOnce(ctx, "the schedule")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-tick.C:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var doctorRuns struct {
|
||||||
|
sync.Mutex
|
||||||
|
n uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
// runOnce runs every probe the registry runs, keeps what each found, and says the heartbeat. One run
|
||||||
|
// at a time: a person's `doctor run` during a scheduled one waits for it.
|
||||||
|
func (d *doctor) runOnce(ctx context.Context, why string) doctorRun {
|
||||||
|
d.running.Lock()
|
||||||
|
defer d.running.Unlock()
|
||||||
|
doctorRuns.Lock()
|
||||||
|
doctorRuns.n++
|
||||||
|
n := doctorRuns.n
|
||||||
|
doctorRuns.Unlock()
|
||||||
|
started := time.Now()
|
||||||
|
run := doctorRun{Run: fmt.Sprintf("doctor-%d-%d", started.Unix(), n), Started: started.UTC(),
|
||||||
|
IntervalSeconds: int(doctorEvery / time.Second), Controller: d.host, Why: why}
|
||||||
|
|
||||||
|
type result struct {
|
||||||
|
obs []conditions.Observation
|
||||||
|
err error
|
||||||
|
took time.Duration
|
||||||
|
}
|
||||||
|
results := make([]result, len(probeRegistry))
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for i, p := range probeRegistry {
|
||||||
|
if p.run == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
wg.Add(1)
|
||||||
|
go func(i int, p probe) {
|
||||||
|
defer wg.Done()
|
||||||
|
probing, cancel := context.WithTimeout(context.WithValue(ctx, probeAsksKey{}, p), probeWithin)
|
||||||
|
defer cancel()
|
||||||
|
began := time.Now()
|
||||||
|
done := make(chan result, 1)
|
||||||
|
go func() {
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
done <- result{err: fmt.Errorf("the probe panicked: %v", r)}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
obs, err := p.run(probing, d)
|
||||||
|
done <- result{obs: obs, err: err}
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case r := <-done:
|
||||||
|
r.took = time.Since(began)
|
||||||
|
results[i] = r
|
||||||
|
case <-probing.Done():
|
||||||
|
results[i] = result{err: fmt.Errorf("it did not finish within %s", probeWithin), took: time.Since(began)}
|
||||||
|
}
|
||||||
|
}(i, p)
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
var blind []conditions.Observation
|
||||||
|
for i, p := range probeRegistry {
|
||||||
|
v := probeVerdict{ID: p.ID}
|
||||||
|
r := results[i]
|
||||||
|
switch {
|
||||||
|
case p.run == nil:
|
||||||
|
v.Verdict = verdictDeferred
|
||||||
|
run.Counts.Deferred++
|
||||||
|
case r.err != nil:
|
||||||
|
v.Verdict, v.Error = verdictFailedToRun, r.err.Error()
|
||||||
|
run.Counts.FailedToRun++
|
||||||
|
blind = append(blind, conditions.Observation{Scope: conditions.ScopeProbe, ID: p.ID, Kind: "probe-failed",
|
||||||
|
Token: "failed", Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("the probe %s (%s) could not run: what it checks is not known — never a pass", p.ID, p.Asserts),
|
||||||
|
Said: firstLine(r.err.Error())})
|
||||||
|
default:
|
||||||
|
if err := d.keeper.Reconcile(ctx, p.ID, kindedAs(r.obs, p.Kind)); err != nil {
|
||||||
|
v.Error = "what it found could not be kept: " + err.Error()
|
||||||
|
}
|
||||||
|
if len(r.obs) == 0 {
|
||||||
|
v.Verdict = verdictPass
|
||||||
|
run.Counts.Passed++
|
||||||
|
} else {
|
||||||
|
v.Verdict = verdictFail
|
||||||
|
run.Counts.Failed++
|
||||||
|
for _, o := range r.obs {
|
||||||
|
v.Found = append(v.Found, o.Summary)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if p.run != nil {
|
||||||
|
v.Took = r.took.Round(time.Millisecond).String()
|
||||||
|
}
|
||||||
|
run.Probes = append(run.Probes, v)
|
||||||
|
}
|
||||||
|
if err := d.keeper.Reconcile(ctx, sourceDoctor, blind); err != nil {
|
||||||
|
fmt.Printf("the self-check's own failures could not be kept: %v\n", err)
|
||||||
|
}
|
||||||
|
ended := time.Now()
|
||||||
|
run.At, run.Took, run.Unsaid = ended.UTC(), ended.Sub(started).Round(time.Millisecond).String(), d.keeper.Unsaid()
|
||||||
|
d.mu.Lock()
|
||||||
|
d.last, d.ended = &run, ended
|
||||||
|
d.mu.Unlock()
|
||||||
|
d.sayHeartbeat(ctx, run)
|
||||||
|
return run
|
||||||
|
}
|
||||||
|
|
||||||
|
// sourceDoctor is what raises a probe's own failure to run.
|
||||||
|
const sourceDoctor = "doctor"
|
||||||
|
|
||||||
|
// kindedAs gives each observation of a probe the probe's kind where it named none.
|
||||||
|
func kindedAs(obs []conditions.Observation, kind string) []conditions.Observation {
|
||||||
|
out := make([]conditions.Observation, 0, len(obs))
|
||||||
|
for _, o := range obs {
|
||||||
|
if o.Kind == "" {
|
||||||
|
o.Kind = kind
|
||||||
|
}
|
||||||
|
out = append(out, o)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// sayHeartbeat publishes the run's heartbeat. Not said is said here, and S10 on the second machine
|
||||||
|
// says it outward: the watcher hears nothing.
|
||||||
|
func (d *doctor) sayHeartbeat(ctx context.Context, run doctorRun) {
|
||||||
|
if d.teller == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
body, err := json.Marshal(run)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("the self-check's heartbeat could not be written: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
saying, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if err := d.teller.PublishSeatEvent(saying, conditions.Seat, conditions.HeartbeatEvent, body); err != nil {
|
||||||
|
fmt.Printf("the self-check's heartbeat (%s) could NOT be said, so the watcher on the second machine "+
|
||||||
|
"will say the self-check is silent: %v\n", run.Run, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// doctorFrom is the serving controller's self-check; nil in any other process.
|
||||||
|
var doctorFrom *doctor
|
||||||
|
|
||||||
|
// doctorCommand is `doctor`, `doctor run`, `doctor probes` and `doctor signals`.
|
||||||
|
func doctorCommand(ctx context.Context, args []string) error {
|
||||||
|
sub := ""
|
||||||
|
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||||
|
sub, args = args[0], args[1:]
|
||||||
|
}
|
||||||
|
set := flag.NewFlagSet("doctor", flag.ContinueOnError)
|
||||||
|
asJSON := set.Bool("json", false, "as data")
|
||||||
|
if rest, err := parseAround(set, args); err != nil {
|
||||||
|
return err
|
||||||
|
} else if len(rest) > 0 {
|
||||||
|
return errors.New("doctor [run|probes|signals] [--json]")
|
||||||
|
}
|
||||||
|
answer, err := doctorAnswer(ctx, sub)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if *asJSON {
|
||||||
|
return printJSON(answer)
|
||||||
|
}
|
||||||
|
fmt.Print(doctorText(answer))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// doctorAnswer is what the verb answers, as data.
|
||||||
|
func doctorAnswer(ctx context.Context, sub string) (any, error) {
|
||||||
|
switch sub {
|
||||||
|
case "":
|
||||||
|
if doctorFrom != nil {
|
||||||
|
if run := doctorFrom.lastRun(); run != nil {
|
||||||
|
return verdictAnswer(*run, time.Now()), nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("the self-check has not finished its first run yet: it runs %s after the "+
|
||||||
|
"controller starts, then every %s — `doctor run` runs it now", doctorFirstAfter, doctorEvery)
|
||||||
|
}
|
||||||
|
run, err := lastHeartbeat(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return verdictAnswer(run, time.Now()), nil
|
||||||
|
case "run":
|
||||||
|
d := doctorFrom
|
||||||
|
if d == nil {
|
||||||
|
local, closeIt, err := localDoctor(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer closeIt()
|
||||||
|
d = local
|
||||||
|
}
|
||||||
|
return verdictAnswer(d.runOnce(ctx, "asked by "+link.Caller()), time.Now()), nil
|
||||||
|
case "probes":
|
||||||
|
return probesAnswer(), nil
|
||||||
|
case "signals":
|
||||||
|
if doctorFrom == nil || doctorFrom.watchdogs == nil {
|
||||||
|
return nil, errors.New("the age of each signal is known to the serving controller alone, which " +
|
||||||
|
"hears them: ask it through the mesh-controller seat's doctor verb")
|
||||||
|
}
|
||||||
|
return signalsAnswer(doctorFrom.watchdogs.lastFacts(), doctorFrom.watchdogs.lastTick()), nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("doctor answers the last run, or `run`, `probes` or `signals` — not %q", sub)
|
||||||
|
}
|
||||||
|
|
||||||
|
// verdictAnswer is a run as the verb answers it, with its age.
|
||||||
|
func verdictAnswer(run doctorRun, now time.Time) map[string]any {
|
||||||
|
return map[string]any{"run": run, "age": now.Sub(run.At).Round(time.Second).String(),
|
||||||
|
"note": "a probe that could not run is never a pass; each failure is an open condition until a run passes it"}
|
||||||
|
}
|
||||||
|
|
||||||
|
// probesAnswer is the registry.
|
||||||
|
func probesAnswer() map[string]any {
|
||||||
|
var out []map[string]any
|
||||||
|
for _, p := range probeRegistry {
|
||||||
|
row := map[string]any{"id": p.ID, "asserts": p.Asserts, "from": p.From, "kind": p.Kind, "phase": p.Phase}
|
||||||
|
if p.Deferred != "" {
|
||||||
|
row["deferred"] = p.Deferred
|
||||||
|
}
|
||||||
|
out = append(out, row)
|
||||||
|
}
|
||||||
|
return map[string]any{"probes": out, "every": doctorEvery.String(), "each within": probeWithin.String()}
|
||||||
|
}
|
||||||
|
|
||||||
|
// signalsAnswer is every row of the signals table with the age of its newest signal.
|
||||||
|
func signalsAnswer(f *signalFacts, ticked time.Time) map[string]any {
|
||||||
|
var rows []map[string]any
|
||||||
|
for _, r := range signalsTable {
|
||||||
|
row := map[string]any{"row": r.Row, "signal": r.Signal, "emitter": r.Emitter, "bound": r.Bound,
|
||||||
|
"kind": r.Kind, "severity": r.Severity, "phase": r.Phase}
|
||||||
|
switch {
|
||||||
|
case r.Deferred != "":
|
||||||
|
row["deferred"] = r.Deferred
|
||||||
|
case f == nil:
|
||||||
|
row["newest"] = "not yet looked at"
|
||||||
|
default:
|
||||||
|
if err := r.needs(f); err != nil {
|
||||||
|
row["blind"] = err.Error()
|
||||||
|
} else if newest := r.newest(f); newest.IsZero() {
|
||||||
|
row["newest"] = "none heard"
|
||||||
|
} else {
|
||||||
|
row["newest"] = newest.UTC().Format(time.RFC3339)
|
||||||
|
row["age"] = f.now.Sub(newest).Round(time.Second).String()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rows = append(rows, row)
|
||||||
|
}
|
||||||
|
out := map[string]any{"signals": rows, "every": watchEvery.String()}
|
||||||
|
if !ticked.IsZero() {
|
||||||
|
out["looked"] = ticked.UTC().Format(time.RFC3339)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// doctorText is an answer as a person reads it.
|
||||||
|
func doctorText(answer any) string {
|
||||||
|
body, _ := json.Marshal(answer)
|
||||||
|
var b strings.Builder
|
||||||
|
var verdict struct {
|
||||||
|
Run doctorRun `json:"run"`
|
||||||
|
Age string `json:"age"`
|
||||||
|
}
|
||||||
|
if json.Unmarshal(body, &verdict) == nil && verdict.Run.Run != "" {
|
||||||
|
r := verdict.Run
|
||||||
|
fmt.Fprintf(&b, "%s, %s ago (took %s, %s): %d passed, %d failed, %d could not run, %d not built yet\n\n",
|
||||||
|
r.Run, verdict.Age, r.Took, r.Why, r.Counts.Passed, r.Counts.Failed, r.Counts.FailedToRun, r.Counts.Deferred)
|
||||||
|
for _, p := range r.Probes {
|
||||||
|
fmt.Fprintf(&b, " %-4s %-14s %s\n", p.ID, p.Verdict, p.Took)
|
||||||
|
for _, f := range p.Found {
|
||||||
|
fmt.Fprintf(&b, " %s\n", f)
|
||||||
|
}
|
||||||
|
if p.Error != "" {
|
||||||
|
fmt.Fprintf(&b, " %s\n", p.Error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
pretty, _ := json.MarshalIndent(answer, "", " ")
|
||||||
|
return string(pretty) + "\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
// lastHeartbeat is the newest run's heartbeat, read from the events stream: what a process other than
|
||||||
|
// the serving controller answers `doctor` from.
|
||||||
|
func lastHeartbeat(ctx context.Context) (doctorRun, error) {
|
||||||
|
var run doctorRun
|
||||||
|
err := onTheBus(func(conn *nats.Conn) error {
|
||||||
|
js, err := conn.JetStream(nats.Context(ctx))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
msg, err := js.GetLastMsg(broker.EventsStream, link.SeatEventSubject(conditions.Seat, conditions.HeartbeatEvent))
|
||||||
|
if errors.Is(err, nats.ErrMsgNotFound) {
|
||||||
|
return errors.New("the self-check has said no heartbeat on the bus in the last week: it is not running")
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the self-check's last heartbeat cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
return json.Unmarshal(msg.Data, &run)
|
||||||
|
})
|
||||||
|
return run, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// localDoctor is a self-check run by a process other than the serving controller: its own stores,
|
||||||
|
// its own connection, and its own keeper, closed after.
|
||||||
|
func localDoctor(ctx context.Context) (*doctor, func(), error) {
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
js, err := dialTheBus()
|
||||||
|
if err != nil {
|
||||||
|
open.Close()
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
k, err := keeperOn(ctx, js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
js.Close()
|
||||||
|
open.Close()
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
jsCtx := js.Context()
|
||||||
|
d := &doctor{open: open, js: js, keeper: k, teller: link.OverNATS{Conn: js.Conn(), JS: jsCtx},
|
||||||
|
host: controlHost(ctx, open.inventory)}
|
||||||
|
return d, func() {
|
||||||
|
flushing, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
k.Close(flushing)
|
||||||
|
js.Close()
|
||||||
|
open.Close()
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// sortedFound is a probe's findings in a stated order, so two runs over one mesh say the same.
|
||||||
|
func sortedFound(obs []conditions.Observation) []conditions.Observation {
|
||||||
|
sort.Slice(obs, func(i, j int) bool { return obs[i].Key() < obs[j].Key() })
|
||||||
|
return obs
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeAsksKey carries the running probe, so a seat verb it calls is checked against what it declares.
|
||||||
|
type probeAsksKey struct{}
|
||||||
|
|
||||||
|
// declaredBy says whether the probe running in ctx declared a seat verb; outside a probe, false.
|
||||||
|
func declaredBy(ctx context.Context, seat, verb string) (string, bool) {
|
||||||
|
p, ok := ctx.Value(probeAsksKey{}).(probe)
|
||||||
|
if !ok {
|
||||||
|
return "a caller outside the self-check", false
|
||||||
|
}
|
||||||
|
for _, v := range p.Asks {
|
||||||
|
if v.Seat == seat && v.Verb == verb {
|
||||||
|
return p.ID, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return p.ID, false
|
||||||
|
}
|
||||||
@@ -0,0 +1,425 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
"golang.org/x/net/dns/dnsmessage"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The self-check (novox/hq to-be 45 §4): a probe that fails raises its condition, one that cannot run
|
||||||
|
// raises probe-failed for itself and is never a pass, and every run ends with its heartbeat.
|
||||||
|
|
||||||
|
// withProbes runs the test with a registry of its own.
|
||||||
|
func withProbes(t *testing.T, probes ...probe) {
|
||||||
|
t.Helper()
|
||||||
|
before := probeRegistry
|
||||||
|
probeRegistry = probes
|
||||||
|
t.Cleanup(func() { probeRegistry = before })
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestARunKeepsWhatEachProbeFoundAndSaysItsHeartbeat(t *testing.T) {
|
||||||
|
failing := conditions.Observation{Scope: conditions.ScopeMachine, ID: "anchor", Token: "refused",
|
||||||
|
Severity: conditions.Urgent, Summary: "anchor's node-engine would refuse its declaration"}
|
||||||
|
var broken atomic.Bool
|
||||||
|
broken.Store(true)
|
||||||
|
withProbes(t,
|
||||||
|
probe{ID: "P1", Asserts: "passes", Kind: "never", Phase: 1,
|
||||||
|
run: func(context.Context, *doctor) ([]conditions.Observation, error) { return nil, nil }},
|
||||||
|
probe{ID: "P2", Asserts: "finds a fault", Kind: "declaration-refused", Phase: 1,
|
||||||
|
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
|
||||||
|
if broken.Load() {
|
||||||
|
return []conditions.Observation{failing}, nil
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}},
|
||||||
|
probe{ID: "P3", Asserts: "cannot run", Kind: "x", Phase: 1,
|
||||||
|
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
|
||||||
|
if broken.Load() {
|
||||||
|
return nil, errors.New("the store is away")
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}},
|
||||||
|
probe{ID: "P4", Asserts: "hangs", Kind: "x", Phase: 1,
|
||||||
|
run: func(ctx context.Context, _ *doctor) ([]conditions.Observation, error) {
|
||||||
|
if broken.Load() {
|
||||||
|
<-ctx.Done()
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}},
|
||||||
|
probe{ID: "P5", Asserts: "later", Kind: "x", Phase: 2, Deferred: "not yet"},
|
||||||
|
)
|
||||||
|
before := probeWithin
|
||||||
|
probeWithin = 200 * time.Millisecond
|
||||||
|
t.Cleanup(func() { probeWithin = before })
|
||||||
|
|
||||||
|
store := conditions.NewInMemory()
|
||||||
|
told := &conditions.Told{}
|
||||||
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||||
|
defer k.Close(context.Background())
|
||||||
|
d := &doctor{keeper: k, teller: told, host: "anchor"}
|
||||||
|
run := d.runOnce(t.Context(), "a test")
|
||||||
|
if run.Counts != (doctorCounts{Passed: 1, Failed: 1, FailedToRun: 2, Deferred: 1}) {
|
||||||
|
t.Fatalf("counted %+v", run.Counts)
|
||||||
|
}
|
||||||
|
open, err := k.Open(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var keys []string
|
||||||
|
for _, c := range open {
|
||||||
|
keys = append(keys, c.Key+"="+c.Kind)
|
||||||
|
}
|
||||||
|
for _, want := range []string{"machine.anchor.refused=declaration-refused", "probe.P3.failed=probe-failed",
|
||||||
|
"probe.P4.failed=probe-failed"} {
|
||||||
|
if !slices.Contains(keys, want) {
|
||||||
|
t.Errorf("%s is not open: %v", want, keys)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The heartbeat, in the shape mesh-watcher reads (the contract with the operator's channel).
|
||||||
|
if len(told.Names) != 1 || told.Names[0] != conditions.HeartbeatEvent {
|
||||||
|
t.Fatalf("said %v", told.Names)
|
||||||
|
}
|
||||||
|
if d.lastRunEnded().IsZero() || d.lastRun().Run != run.Run {
|
||||||
|
t.Fatal("the run is not the last verdict")
|
||||||
|
}
|
||||||
|
body, _ := json.Marshal(run)
|
||||||
|
var shape map[string]any
|
||||||
|
_ = json.Unmarshal(body, &shape)
|
||||||
|
for _, field := range []string{"run", "at", "interval-seconds", "counts", "probes", "controller"} {
|
||||||
|
if _, ok := shape[field]; !ok {
|
||||||
|
t.Errorf("the heartbeat carries no %q: %s", field, body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mended: the next run clears every one of them.
|
||||||
|
broken.Store(false)
|
||||||
|
d.runOnce(t.Context(), "a test")
|
||||||
|
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||||
|
t.Fatalf("a passing run left open %+v", open)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The registry says what each probe asserts**, and a probe not built says why and when.
|
||||||
|
func TestTheRegistryIsTheDesignsLiveForm(t *testing.T) {
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for _, p := range probeRegistry {
|
||||||
|
if seen[p.ID] {
|
||||||
|
t.Errorf("%s twice", p.ID)
|
||||||
|
}
|
||||||
|
seen[p.ID] = true
|
||||||
|
if p.Asserts == "" || p.From == "" || p.Kind == "" {
|
||||||
|
t.Errorf("%s does not say what it asserts, where from, or what it raises", p.ID)
|
||||||
|
}
|
||||||
|
if (p.run == nil) != (p.Deferred != "") || (p.Deferred != "" && p.Phase <= 1) {
|
||||||
|
t.Errorf("%s is run and deferred, or neither, or deferred out of Phase 1: %+v", p.ID, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, id := range []string{"D1", "D2", "D3", "D4", "D5", "D6", "D7", "D8", "D9", "D10"} {
|
||||||
|
if !seen[id] {
|
||||||
|
t.Errorf("to-be 45 §4 has %s and the registry does not", id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The doctor and the watchdogs watch each other**: watchdogs that stopped are DW; a self-check that
|
||||||
|
// stopped is S10 (signals_test.go).
|
||||||
|
func TestWatchdogsThatStoppedAreSaid(t *testing.T) {
|
||||||
|
w := &watchdogs{started: time.Now().Add(-time.Hour)}
|
||||||
|
d := &doctor{watchdogs: w, host: "anchor"}
|
||||||
|
got, err := probeWatchdogs(t.Context(), d)
|
||||||
|
if err != nil || len(got) != 1 || got[0].Severity != conditions.Urgent {
|
||||||
|
t.Fatalf("%+v %v", got, err)
|
||||||
|
}
|
||||||
|
w.ticked = time.Now()
|
||||||
|
if got, _ := probeWatchdogs(t.Context(), d); len(got) != 0 {
|
||||||
|
t.Fatalf("%+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **D1 composes every machine of a healthy mesh and the host's own validator takes each.**
|
||||||
|
func TestEveryMachineOfAHealthyMeshComposesAndValidates(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
got, err := probeDeclarations(t.Context(), &doctor{open: open})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(got) != 0 {
|
||||||
|
t.Fatalf("a healthy mesh failed D1: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **D1 names a machine nothing can be sent to**, and the network that cannot be computed for it.
|
||||||
|
func TestAMachineWhoseDeclarationDoesNotComposeIsSaid(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
one, two := rivals()
|
||||||
|
register(t, open, one)
|
||||||
|
register(t, open, two)
|
||||||
|
for _, m := range []string{"rival-one", "rival-two"} {
|
||||||
|
if _, err := assign(ctx, open, "laptop", m); err != nil && m == "rival-one" {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
got, err := probeDeclarations(ctx, &doctor{open: open})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "the-seat") {
|
||||||
|
t.Fatalf("%+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **D2: a resolver answering NXDOMAIN for IPv6 is wrong** — musl takes it as no such name (issue 262).
|
||||||
|
func TestAResolverAnsweringNoSuchNameForIPv6IsWrong(t *testing.T) {
|
||||||
|
answerAs := func(rcode dnsmessage.RCode) string {
|
||||||
|
conn, err := net.ListenPacket("udp", "127.0.0.1:0")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = conn.Close() })
|
||||||
|
go func() {
|
||||||
|
buf := make([]byte, 1500)
|
||||||
|
for {
|
||||||
|
n, from, err := conn.ReadFrom(buf)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var q dnsmessage.Message
|
||||||
|
if q.Unpack(buf[:n]) != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
reply := dnsmessage.Message{Header: dnsmessage.Header{ID: q.ID, Response: true}, Questions: q.Questions}
|
||||||
|
if q.Questions[0].Type == dnsmessage.TypeA {
|
||||||
|
reply.Answers = []dnsmessage.Resource{{Header: dnsmessage.ResourceHeader{Name: q.Questions[0].Name,
|
||||||
|
Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET}, Body: &dnsmessage.AResource{A: [4]byte{10, 77, 0, 1}}}}
|
||||||
|
} else {
|
||||||
|
reply.RCode = rcode
|
||||||
|
}
|
||||||
|
packed, _ := reply.Pack()
|
||||||
|
_, _ = conn.WriteTo(packed, from)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
_, port, _ := net.SplitHostPort(conn.LocalAddr().String())
|
||||||
|
return port
|
||||||
|
}
|
||||||
|
before := resolverPort
|
||||||
|
t.Cleanup(func() { resolverPort = before })
|
||||||
|
|
||||||
|
resolverPort = answerAs(dnsmessage.RCodeSuccess)
|
||||||
|
v4, rcode, err := askResolver(t.Context(), "127.0.0.1", "anchor.internal", dnsmessage.TypeA)
|
||||||
|
if err != nil || rcode != dnsmessage.RCodeSuccess || !slices.Equal(v4, []string{"10.77.0.1"}) {
|
||||||
|
t.Fatalf("%v %v %v", v4, rcode, err)
|
||||||
|
}
|
||||||
|
v6, rcode, err := askResolver(t.Context(), "127.0.0.1", "anchor.internal", dnsmessage.TypeAAAA)
|
||||||
|
if err != nil || rcode != dnsmessage.RCodeSuccess || len(v6) != 0 {
|
||||||
|
t.Fatalf("NODATA read as %v %v %v", v6, rcode, err)
|
||||||
|
}
|
||||||
|
resolverPort = answerAs(dnsmessage.RCodeNameError)
|
||||||
|
if _, rcode, _ := askResolver(t.Context(), "127.0.0.1", "anchor.internal", dnsmessage.TypeAAAA); rcode != dnsmessage.RCodeNameError {
|
||||||
|
t.Fatalf("NXDOMAIN read as %v", rcode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **D6, D7: what the controller defines is what it finds**, and a consumer deleted or a stream
|
||||||
|
// redefined is said — against a real bus, raised by the same derivation the controller starts with.
|
||||||
|
func TestNatsTheBusIsWhatTheControllerDefines(t *testing.T) {
|
||||||
|
url := os.Getenv("MESH_TEST_NATS")
|
||||||
|
if url == "" {
|
||||||
|
t.Skip("MESH_TEST_NATS unset")
|
||||||
|
}
|
||||||
|
open := aMesh(t)
|
||||||
|
js, err := broker.Dial(url)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(js.Close)
|
||||||
|
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
|
||||||
|
_ = js.Context().DeleteStream(s)
|
||||||
|
}
|
||||||
|
if _, err := assertBusObjects(t.Context(), open.inventory, js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := js.EnsureControllerBuckets(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
d := &doctor{open: open, js: js}
|
||||||
|
for _, p := range []func(context.Context, *doctor) ([]conditions.Observation, error){probeConsumers, probeStreams} {
|
||||||
|
got, err := p(t.Context(), d)
|
||||||
|
if err != nil || len(got) != 0 {
|
||||||
|
t.Fatalf("a bus just raised fails: %+v %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
info, err := js.Context().StreamInfo("EVENTS")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cfg := info.Config
|
||||||
|
cfg.MaxMsgsPerSubject = 3
|
||||||
|
if _, err := js.Context().UpdateStream(&cfg); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
consumers, err := probeConsumers(t.Context(), d)
|
||||||
|
if err != nil || len(consumers) != 1 || consumers[0].Key() != "bus.NODES.laptop.missing" {
|
||||||
|
t.Fatalf("the deleted consumer: %+v %v", consumers, err)
|
||||||
|
}
|
||||||
|
streams, err := probeStreams(t.Context(), d)
|
||||||
|
if err != nil || len(streams) != 1 || !strings.Contains(streams[0].Summary, "per subject") {
|
||||||
|
t.Fatalf("the redefined stream: %+v %v", streams, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **S9 hears the bus**: a consumer that gives up on a message, and one deleted, as the server says.
|
||||||
|
func TestNatsTheBusSaysAConsumerGaveUpAndOneWasDeleted(t *testing.T) {
|
||||||
|
url := os.Getenv("MESH_TEST_NATS")
|
||||||
|
if url == "" {
|
||||||
|
t.Skip("MESH_TEST_NATS unset")
|
||||||
|
}
|
||||||
|
conn, err := nats.Connect(url)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
heard := make(chan *nats.Msg, 16)
|
||||||
|
for _, subject := range broker.BusAdvisories {
|
||||||
|
if _, err := conn.ChanSubscribe(subject, heard); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
api, _ := jetstream.New(conn)
|
||||||
|
_ = api.DeleteStream(t.Context(), "SEAT_ADVISED")
|
||||||
|
stream, err := api.CreateStream(t.Context(), jetstream.StreamConfig{Name: "SEAT_ADVISED", Subjects: []string{"advised.>"}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() { _ = api.DeleteStream(context.Background(), "SEAT_ADVISED") }()
|
||||||
|
consumer, err := stream.CreateConsumer(t.Context(), jetstream.ConsumerConfig{Durable: "SEAT_ADVISED_worker",
|
||||||
|
AckPolicy: jetstream.AckExplicitPolicy, MaxDeliver: 1, AckWait: 100 * time.Millisecond})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := api.Publish(t.Context(), "advised.x", []byte("x")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := consumer.Fetch(1, jetstream.FetchMaxWait(time.Second)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// A seat's worker, so the deletion is of a consumer the mesh names (link.MeshNamed).
|
||||||
|
// Not acknowledged: after its one delivery the consumer gives up on it — on the next fetch.
|
||||||
|
time.Sleep(300 * time.Millisecond)
|
||||||
|
_, _ = consumer.Fetch(1, jetstream.FetchMaxWait(300*time.Millisecond))
|
||||||
|
if err := stream.DeleteConsumer(t.Context(), "SEAT_ADVISED_worker"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
kinds := map[string]string{}
|
||||||
|
deadline := time.After(5 * time.Second)
|
||||||
|
for len(kinds) < 2 {
|
||||||
|
select {
|
||||||
|
case m := <-heard:
|
||||||
|
if a, ok := link.ReadAdvisory(m.Subject, m.Data); ok {
|
||||||
|
kinds[a.Kind] = a.Said
|
||||||
|
}
|
||||||
|
case <-deadline:
|
||||||
|
t.Fatalf("the bus said only %v", kinds)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !strings.Contains(kinds["max-deliveries"], "gave up") || !strings.Contains(kinds["consumer-lost"], "was deleted") {
|
||||||
|
t.Fatalf("%v", kinds)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **D10 compares a node-engine with what it is delivered as, not with its commit** (2026-10-06: every
|
||||||
|
// machine read as behind right after a push sent it the current build — it says the digest-named
|
||||||
|
// directory it runs from, and the mesh holds a commit).
|
||||||
|
func TestANodeEngineIsJudgedByTheVersionItIsDeliveredAs(t *testing.T) {
|
||||||
|
m := catalogue.Manifest{Module: "mesh-host", Resources: []map[string]any{
|
||||||
|
{"id": "launcher", "type": "file", "path": "/usr/lib/nox-mesh-host/launch"},
|
||||||
|
{"id": "host", "type": "archive", "path": "/usr/lib/nox-mesh-host/versions/31045596c83a"},
|
||||||
|
{"id": "unfilled", "type": "archive", "path": "/usr/lib/x/versions/${version}"},
|
||||||
|
}}
|
||||||
|
delivered := deliveredVersions(m)
|
||||||
|
if !slices.Equal(delivered, []string{"31045596c83a"}) {
|
||||||
|
t.Fatalf("%v", delivered)
|
||||||
|
}
|
||||||
|
commit := "1545b00a9f0c"
|
||||||
|
for _, c := range []struct {
|
||||||
|
reported string
|
||||||
|
behind bool
|
||||||
|
}{
|
||||||
|
{"31045596c83a", false}, // the live case: current, and was called behind
|
||||||
|
{"0123456789ab", true}, // another delivery
|
||||||
|
{"1545b00a", false}, // placed by hand, stamped with the commit
|
||||||
|
{"", false}, // not said
|
||||||
|
} {
|
||||||
|
if got := engineBehind(c.reported, delivered, commit); got != c.behind {
|
||||||
|
t.Errorf("%q behind = %v, want %v", c.reported, got, c.behind)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if engineBehind("31045596c83a", nil, commit) {
|
||||||
|
t.Error("behind a mesh that holds no delivered build")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Every seat verb a probe calls is one it declares, and one the controller is granted** — derived
|
||||||
|
// from the registry, so a probe added with a question the bus would refuse fails here, not live.
|
||||||
|
func TestEverySeatVerbAProbeAsksIsGranted(t *testing.T) {
|
||||||
|
granted, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController, PasswordHash: "x"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
asked := 0
|
||||||
|
for _, p := range probeRegistry {
|
||||||
|
for _, v := range p.Asks {
|
||||||
|
asked++
|
||||||
|
subject := link.NodeSeatToolSubject(v.Seat, v.Verb, "anchor")
|
||||||
|
if !slices.ContainsFunc(granted.Publish, func(pattern string) bool { return subjectMatches(pattern, subject) }) {
|
||||||
|
t.Errorf("%s asks %s.%s and the controller may not publish %s", p.ID, v.Seat, v.Verb, subject)
|
||||||
|
}
|
||||||
|
if !slices.Contains(broker.VerbsTheSelfCheckAsks, v) {
|
||||||
|
t.Errorf("%s asks %s.%s, which broker.VerbsTheSelfCheckAsks does not name", p.ID, v.Seat, v.Verb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if asked == 0 {
|
||||||
|
t.Fatal("no probe asks a seat verb: D8 lost its declaration")
|
||||||
|
}
|
||||||
|
// And a probe asking what it did not declare is refused before anything is sent.
|
||||||
|
ctx := context.WithValue(t.Context(), probeAsksKey{}, probe{ID: "DX"})
|
||||||
|
if _, err := askSeatTool(ctx, nil, "node-intrusion-prevention", "banned", "anchor"); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "does not declare") {
|
||||||
|
t.Fatalf("an undeclared question was asked: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// subjectMatches is the bus's matching of a permission pattern against a subject.
|
||||||
|
func subjectMatches(pattern, subject string) bool {
|
||||||
|
p, s := strings.Split(pattern, "."), strings.Split(subject, ".")
|
||||||
|
for i, tok := range p {
|
||||||
|
if tok == ">" {
|
||||||
|
return len(s) > i
|
||||||
|
}
|
||||||
|
if i >= len(s) || (tok != "*" && tok != s[i]) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return len(p) == len(s)
|
||||||
|
}
|
||||||
@@ -0,0 +1,170 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"slices"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What the core's bounds are set from (novox/hq to-be 45 Phase 0).
|
||||||
|
//
|
||||||
|
// **A bound is set from what was measured, not from what seemed reasonable.** Phase 1 puts a watchdog
|
||||||
|
// on each row of the signals table, and each has a bound: S1 three heartbeat intervals, S2 three times
|
||||||
|
// a machine's last apply, S3 a tier's build and apply time, S6 a build's timeout. Marked provisional
|
||||||
|
// in the design until a fortnight of these says what the mesh actually takes. Recorded by the serving
|
||||||
|
// controller as it hears each — a send's first report, a machine's next word, a plan leaving a tier, a
|
||||||
|
// build's outcome — and summarised here per machine, repository or module.
|
||||||
|
|
||||||
|
// recordBuildDuration measures one build from its ask to its outcome heard.
|
||||||
|
func recordBuildDuration(ctx context.Context, inv *inventory.Inventory, result link.BuildResult, asked time.Time) {
|
||||||
|
if asked.IsZero() || result.ID == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
subject := result.Module
|
||||||
|
if subject == "" {
|
||||||
|
subject = result.Repository
|
||||||
|
}
|
||||||
|
detail := "built"
|
||||||
|
if result.Failed != "" {
|
||||||
|
detail = "failed: " + firstLine(result.Failed)
|
||||||
|
}
|
||||||
|
if err := inv.RecordDuration(ctx, inventory.Duration{Kind: inventory.DurationBuild, Subject: subject,
|
||||||
|
Node: result.On, Ref: result.ID, Started: asked, Took: time.Since(asked), Detail: detail}); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "%s: how long it took could not be recorded: %v\n", result.ID, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// durationSummary is one subject's measurements of one kind.
|
||||||
|
type durationSummary struct {
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Subject string `json:"subject"`
|
||||||
|
Count int `json:"count"`
|
||||||
|
Median string `json:"median"`
|
||||||
|
P90 string `json:"p90"`
|
||||||
|
Max string `json:"max"`
|
||||||
|
// Bound is what to-be 45's rule would make of these, where the rule is a multiple of a measured
|
||||||
|
// time: three times the slowest apply (S2), three times the median word interval (S1).
|
||||||
|
Suggests string `json:"suggests,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func summarise(ds []inventory.Duration) []durationSummary {
|
||||||
|
type key struct{ kind, subject string }
|
||||||
|
by := map[key][]time.Duration{}
|
||||||
|
for _, d := range ds {
|
||||||
|
k := key{d.Kind, d.Subject}
|
||||||
|
by[k] = append(by[k], d.Took)
|
||||||
|
}
|
||||||
|
var out []durationSummary
|
||||||
|
for k, took := range by {
|
||||||
|
slices.Sort(took)
|
||||||
|
at := func(q float64) time.Duration { return took[int(q*float64(len(took)-1))] }
|
||||||
|
s := durationSummary{Kind: k.kind, Subject: k.subject, Count: len(took),
|
||||||
|
Median: round(at(0.5)), P90: round(at(0.9)), Max: round(took[len(took)-1])}
|
||||||
|
switch k.kind {
|
||||||
|
case inventory.DurationApply:
|
||||||
|
s.Suggests = "S2 bound max(2m, 3×last apply) ≈ " + round(max(2*time.Minute, 3*at(0.9))) + " at the p90"
|
||||||
|
case inventory.DurationHeartbeatGap:
|
||||||
|
s.Suggests = "S1 bound 3×interval ≈ " + round(3*at(0.5))
|
||||||
|
}
|
||||||
|
out = append(out, s)
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(i, j int) bool {
|
||||||
|
ki, kj := slices.Index(inventory.DurationKinds, out[i].Kind), slices.Index(inventory.DurationKinds, out[j].Kind)
|
||||||
|
if ki != kj {
|
||||||
|
return ki < kj
|
||||||
|
}
|
||||||
|
return out[i].Subject < out[j].Subject
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func round(d time.Duration) string {
|
||||||
|
switch {
|
||||||
|
case d < time.Second:
|
||||||
|
return d.Round(time.Millisecond).String()
|
||||||
|
case d < time.Minute:
|
||||||
|
return d.Round(100 * time.Millisecond).String()
|
||||||
|
default:
|
||||||
|
return d.Round(time.Second).String()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// durationsCommand is `durations`: the summary per kind and subject, or every measurement as data.
|
||||||
|
func durationsCommand(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("durations", flag.ContinueOnError)
|
||||||
|
kind := set.String("kind", "", "one kind: "+strings.Join(inventory.DurationKinds, ", "))
|
||||||
|
days := set.Int("days", 14, "how many days back")
|
||||||
|
asJSON := set.Bool("json", false, "the summary as data")
|
||||||
|
all := set.Bool("all", false, "every measurement rather than the summary")
|
||||||
|
if _, err := parseAround(set, args); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if *kind != "" && !slices.Contains(inventory.DurationKinds, *kind) {
|
||||||
|
return fmt.Errorf("%q is not a kind of duration: %s", *kind, strings.Join(inventory.DurationKinds, ", "))
|
||||||
|
}
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
ds, err := open.inventory.Durations(ctx, *kind, time.Now().Add(-time.Duration(*days)*24*time.Hour))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if *all {
|
||||||
|
body, err := json.MarshalIndent(ds, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(string(body))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
summary := summarise(ds)
|
||||||
|
if *asJSON {
|
||||||
|
body, err := json.MarshalIndent(map[string]any{"days": *days, "durations": summary}, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(string(body))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if len(summary) == 0 {
|
||||||
|
fmt.Printf("nothing measured in the last %d day(s): the serving controller records apply, heartbeat-gap, "+
|
||||||
|
"plan-tier and build durations as it hears them\n", *days)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
fmt.Printf("durations over the last %d day(s) — what the core's bounds are set from (to-be 45 Phase 0)\n\n", *days)
|
||||||
|
fmt.Printf(" %-14s %-28s %6s %10s %10s %10s\n", "kind", "of", "count", "median", "p90", "max")
|
||||||
|
for _, s := range summary {
|
||||||
|
fmt.Printf(" %-14s %-28s %6d %10s %10s %10s\n", s.Kind, s.Subject, s.Count, s.Median, s.P90, s.Max)
|
||||||
|
if s.Suggests != "" {
|
||||||
|
fmt.Printf(" %-14s %-28s %s\n", "", "", s.Suggests)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// forgettingOldDurations removes what is older than a month, at start and daily after.
|
||||||
|
func forgettingOldDurations(ctx context.Context, inv *inventory.Inventory) {
|
||||||
|
for {
|
||||||
|
if n, err := inv.ForgetOldDurations(ctx); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "durations older than %s could not be removed: %v\n", inventory.DurationsKeptFor, err)
|
||||||
|
} else if n > 0 {
|
||||||
|
fmt.Printf("removed %d duration(s) older than %s\n", n, inventory.DurationsKeptFor)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-time.After(24 * time.Hour):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,197 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Acts done by hand, and why (novox/hq to-be 45 §7).
|
||||||
|
//
|
||||||
|
// **Which verbs ask why.** `plans close` and `plans stop`, `broker consumer-reset` and `hand-act
|
||||||
|
// record` refuse without it everywhere: nothing automated runs them, so a call without a reason is a
|
||||||
|
// person who has not given one. A named `push` asks for it through the mesh-controller seat, which is
|
||||||
|
// how a person or an agent acts by hand on the mesh; at a shell `--why` is recorded when given and not
|
||||||
|
// required, because the installer and the lab push by command line as a step of what they do, and a
|
||||||
|
// step of a procedure is not a repair. `conditions silence` joins them when the condition store does
|
||||||
|
// (Phase 1).
|
||||||
|
|
||||||
|
// handActFlags are the flags every repairing verb takes.
|
||||||
|
type handActFlags struct {
|
||||||
|
why, cause, condition *string
|
||||||
|
}
|
||||||
|
|
||||||
|
func addHandActFlags(set *flag.FlagSet) handActFlags {
|
||||||
|
return handActFlags{
|
||||||
|
why: set.String("why", "", "why this is done by hand — recorded in the hand-act log (novox/hq to-be 45 §7)"),
|
||||||
|
cause: set.String("cause", "", "the cause, in a word or a condition's kind; the verb's own name when not given"),
|
||||||
|
condition: set.String("condition", "", "the key of the condition this act addresses, if any"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// given is whether a reason was given.
|
||||||
|
func (f handActFlags) given() bool { return strings.TrimSpace(*f.why) != "" }
|
||||||
|
|
||||||
|
// require refuses an act without a reason, before anything is done.
|
||||||
|
func (f handActFlags) require(verb string) error {
|
||||||
|
if f.given() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return fmt.Errorf("%s is a repair done by hand, and says why: --why <text> (recorded in the hand-act "+
|
||||||
|
"log, novox/hq to-be 45 §7). Nothing was done", verb)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handActConn is the serving controller's connection, for what it reads of the log itself; a
|
||||||
|
// command dials its own.
|
||||||
|
var handActConn *nats.Conn
|
||||||
|
|
||||||
|
// onTheBus runs f with a connection to the bus: the serving controller's, or one of its own.
|
||||||
|
func onTheBus(f func(*nats.Conn) error) error {
|
||||||
|
if handActConn != nil {
|
||||||
|
return f(handActConn)
|
||||||
|
}
|
||||||
|
address, err := broker.BusAddress()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
js, err := broker.Dial(address)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot reach the bus: %w", err)
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
return f(js.Conn())
|
||||||
|
}
|
||||||
|
|
||||||
|
// record writes the entry for an act about to be done. **Before the act, and never instead of it**:
|
||||||
|
// a log that cannot be written is said loudly, and the repair it was about still happens — a mesh
|
||||||
|
// whose bus is down is exactly the mesh somebody is repairing by hand.
|
||||||
|
func (f handActFlags) record(ctx context.Context, verb string, args []string) {
|
||||||
|
if !f.given() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
act := link.HandAct{Verb: verb, Args: args, Why: strings.TrimSpace(*f.why),
|
||||||
|
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
|
||||||
|
err := onTheBus(func(conn *nats.Conn) error {
|
||||||
|
written, err := link.RecordHandAct(ctx, conn, act)
|
||||||
|
act = written
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "this act by hand could NOT be recorded in the hand-act log, and is done anyway: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Printf("recorded as %s in the hand-act log: %s, because %q (cause: %s)\n", act.ID, act.By, act.Why, act.Cause)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handActCommand is `hand-act record` and `hand-acts`.
|
||||||
|
func handActCommand(ctx context.Context, args []string) error {
|
||||||
|
if len(args) > 0 && args[0] == "record" {
|
||||||
|
set := flag.NewFlagSet("hand-act record", flag.ContinueOnError)
|
||||||
|
f := addHandActFlags(set)
|
||||||
|
positionals, err := parseAround(set, args[1:])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
what := strings.TrimSpace(strings.Join(positionals, " "))
|
||||||
|
if what == "" {
|
||||||
|
return errors.New("hand-act record <what was done> --why <text> [--cause <word>] [--condition <key>]")
|
||||||
|
}
|
||||||
|
if err := f.require("hand-act record"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(*f.cause) == "" {
|
||||||
|
return errors.New("hand-act record says the cause too: --cause <word>, the word a second " +
|
||||||
|
"act for the same reason will use — it is how a repair done twice is found")
|
||||||
|
}
|
||||||
|
act := link.HandAct{Verb: "hand-act record", Args: []string{what}, Why: strings.TrimSpace(*f.why),
|
||||||
|
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
|
||||||
|
return onTheBus(func(conn *nats.Conn) error {
|
||||||
|
written, err := link.RecordHandAct(ctx, conn, act)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the act could not be recorded: %w", err)
|
||||||
|
}
|
||||||
|
fmt.Printf("recorded as %s: %s did %q, because %q (cause: %s)\n", written.ID, written.By, what,
|
||||||
|
written.Why, written.Cause)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if len(args) > 0 && args[0] != "list" && !strings.HasPrefix(args[0], "-") {
|
||||||
|
return errors.New("hand-act record <what> --why <text> --cause <word> | hand-acts [--days N] [--json]")
|
||||||
|
}
|
||||||
|
if len(args) > 0 && args[0] == "list" {
|
||||||
|
args = args[1:]
|
||||||
|
}
|
||||||
|
set := flag.NewFlagSet("hand-acts", flag.ContinueOnError)
|
||||||
|
days := set.Int("days", 14, "how many days back")
|
||||||
|
asJSON := set.Bool("json", false, "as data")
|
||||||
|
if _, err := parseAround(set, args); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return onTheBus(func(conn *nats.Conn) error {
|
||||||
|
now := time.Now()
|
||||||
|
acts, err := link.HandActs(ctx, conn, now.Add(-time.Duration(*days)*24*time.Hour))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
repeated := link.RepeatedCauses(acts, now)
|
||||||
|
if *asJSON {
|
||||||
|
body, err := json.MarshalIndent(map[string]any{"acts": acts, "repeated": repeated}, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(string(body))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if len(acts) == 0 {
|
||||||
|
fmt.Printf("nothing was done by hand in the last %d day(s)\n", *days)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for i := len(acts) - 1; i >= 0; i-- {
|
||||||
|
a := acts[i]
|
||||||
|
fmt.Printf("%s %s %s %s\n by %s — %s (cause: %s", a.At.Local().Format("2006-01-02 15:04"), a.ID,
|
||||||
|
a.Verb, strings.Join(a.Args, " "), a.By, a.Why, a.Cause)
|
||||||
|
if a.Condition != "" {
|
||||||
|
fmt.Printf(", condition %s", a.Condition)
|
||||||
|
}
|
||||||
|
fmt.Println(")")
|
||||||
|
}
|
||||||
|
if len(repeated) > 0 {
|
||||||
|
causes := make([]string, 0, len(repeated))
|
||||||
|
for c, n := range repeated {
|
||||||
|
causes = append(causes, fmt.Sprintf("%s ×%d", c, n))
|
||||||
|
}
|
||||||
|
sort.Strings(causes)
|
||||||
|
fmt.Printf("\ndone by hand more than once in a fortnight — a healer is wanted (to-be 45 S15): %s\n",
|
||||||
|
strings.Join(causes, ", "))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// handActsThisWeek is how many acts were done by hand in the last seven days, for `status`; -1 when
|
||||||
|
// the log could not be read, which status says rather than reading as none.
|
||||||
|
func handActsThisWeek(ctx context.Context) (int, string) {
|
||||||
|
n := -1
|
||||||
|
err := onTheBus(func(conn *nats.Conn) error {
|
||||||
|
reading, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
acts, err := link.HandActs(reading, conn, time.Now().Add(-7*24*time.Hour))
|
||||||
|
n = len(acts)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return -1, err.Error()
|
||||||
|
}
|
||||||
|
return n, ""
|
||||||
|
}
|
||||||
@@ -0,0 +1,94 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **Every verb that repairs by hand takes a required why** (novox/hq to-be 45 §7): refused before
|
||||||
|
// anything is done, through the seat, through `command`, and at a shell where nothing automated runs
|
||||||
|
// the verb.
|
||||||
|
func TestARepairByHandWithoutAReasonIsRefused(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
verb string
|
||||||
|
args map[string]any
|
||||||
|
}{
|
||||||
|
{"push", map[string]any{"node": "anchor"}},
|
||||||
|
{"push", map[string]any{}},
|
||||||
|
{"plans", map[string]any{"close": "plan-1"}},
|
||||||
|
{"plans", map[string]any{"stop": "plan-1"}},
|
||||||
|
{"hand-act", map[string]any{"what": "restarted the proxy", "cause": "proxy-stuck"}},
|
||||||
|
{"command", map[string]any{"command": "push anchor"}},
|
||||||
|
{"command", map[string]any{"command": "plans close plan-1"}},
|
||||||
|
{"command", map[string]any{"command": "broker consumer-reset EVENTS controller"}},
|
||||||
|
{"command", map[string]any{"command": "hand-act record restarted --cause x"}},
|
||||||
|
} {
|
||||||
|
argv, err := argvFor(c.verb, c.args)
|
||||||
|
if c.verb == "plans" && err == nil {
|
||||||
|
// The seat composes the command line; the command refuses it, before opening anything.
|
||||||
|
err = plansCommand(context.Background(), argv[1:])
|
||||||
|
}
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "why") {
|
||||||
|
t.Errorf("%s %v was not refused for want of why: %v %v", c.verb, c.args, argv, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, args := range [][]string{{"EVENTS", "controller"}} {
|
||||||
|
if err := consumerReset(context.Background(), args); err == nil || !strings.Contains(err.Error(), "--why") {
|
||||||
|
t.Errorf("consumer-reset without why: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := handActCommand(context.Background(), []string{"record", "restarted the proxy", "--cause", "x"}); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "--why") {
|
||||||
|
t.Errorf("hand-act record without why: %v", err)
|
||||||
|
}
|
||||||
|
if err := handActCommand(context.Background(), []string{"record", "restarted the proxy", "--why", "it hung"}); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "--cause") {
|
||||||
|
t.Errorf("hand-act record without a cause: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// With a reason, the seat passes it to the command, and a verb that only reads is not held to one.
|
||||||
|
func TestARepairByHandCarriesItsReason(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
verb string
|
||||||
|
args map[string]any
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"push", map[string]any{"node": "anchor", "why": "stuck", "cause": "sent-not-reported"},
|
||||||
|
"push anchor --wait 0 --why stuck --cause sent-not-reported"},
|
||||||
|
{"plans", map[string]any{"close": "plan-1", "why": "the report will not come"},
|
||||||
|
"plans close plan-1 --why the report will not come"},
|
||||||
|
{"plans", map[string]any{"retry": "plan-1"}, "plans retry plan-1"},
|
||||||
|
{"hand-act", map[string]any{"what": "restarted", "why": "hung", "cause": "proxy", "condition": "machine.a.silent"},
|
||||||
|
"hand-act record restarted --why hung --cause proxy --condition machine.a.silent"},
|
||||||
|
{"command", map[string]any{"command": "push anchor --why stuck"}, "push anchor --why stuck"},
|
||||||
|
{"command", map[string]any{"command": "plans plan-1"}, "plans plan-1"},
|
||||||
|
} {
|
||||||
|
argv, err := argvFor(c.verb, c.args)
|
||||||
|
if err != nil || strings.Join(argv, " ") != c.want {
|
||||||
|
t.Errorf("%s %v: %v %v, want %q", c.verb, c.args, argv, err, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The summary of durations says, per kind and subject, what a bound would be set from.
|
||||||
|
func TestDurationsAreSummarisedPerSubject(t *testing.T) {
|
||||||
|
var ds []inventory.Duration
|
||||||
|
for i := 1; i <= 10; i++ {
|
||||||
|
ds = append(ds, inventory.Duration{Kind: inventory.DurationApply, Subject: "anchor",
|
||||||
|
Took: time.Duration(i) * time.Second})
|
||||||
|
}
|
||||||
|
ds = append(ds, inventory.Duration{Kind: inventory.DurationHeartbeatGap, Subject: "anchor", Took: time.Minute})
|
||||||
|
got := summarise(ds)
|
||||||
|
if len(got) != 2 || got[0].Kind != inventory.DurationApply || got[0].Count != 10 ||
|
||||||
|
got[0].Max != "10s" || got[0].Median != "5s" || got[0].P90 != "9s" {
|
||||||
|
t.Fatalf("%+v", got)
|
||||||
|
}
|
||||||
|
if !strings.Contains(got[1].Suggests, "3m0s") {
|
||||||
|
t.Fatalf("a minute between words suggests %q", got[1].Suggests)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,150 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq ADR 0225, issue 263: a consumer's identity is bounded by the provision it requires, an
|
||||||
|
// overflow is refused before merge by `module check`, and a provider's machine is never refused for
|
||||||
|
// one consumer's identity.
|
||||||
|
|
||||||
|
// `module check` refuses the pull request that introduces an overflow, naming the module.
|
||||||
|
func TestModuleCheckRefusesAnIdentityThatOverflowsWhatItRequires(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
write := func(name, body string) string {
|
||||||
|
p := filepath.Join(dir, name+".json")
|
||||||
|
if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
objects := write("objects", `{"module":"objects","version":"1",
|
||||||
|
"provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}],
|
||||||
|
"receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`)
|
||||||
|
resolver := write("resolver", `{"module":"resolver","version":"1",
|
||||||
|
"provides":[{"name":"wildcard-resolution","scope":"mesh","identity":false}]}`)
|
||||||
|
album := write("photoalbum", `{"module":"photoalbum","version":"1","requires":["s3-bucket"]}`)
|
||||||
|
nm := write("networkmanager", `{"module":"networkmanager","version":"1","requires":["wildcard-resolution"]}`)
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
if err := moduleCheckFor([]string{resolver, nm}, 6, &out); err != nil {
|
||||||
|
t.Fatalf("a long name requiring a keyless provision was refused (issue 263): %v\n%s", err, out.String())
|
||||||
|
}
|
||||||
|
out.Reset()
|
||||||
|
err := moduleCheckFor([]string{objects, album, resolver, nm}, 6, &out)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("an identity overflowing an S3 access key passed:\n%s", out.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(out.String(), "photoalbum wants s3-bucket") ||
|
||||||
|
!strings.Contains(out.String(), "`slug` of at most 8 characters") ||
|
||||||
|
strings.Contains(out.String(), "networkmanager wants") {
|
||||||
|
t.Fatalf("the refusal does not name the one overflowing module and its remedy:\n%s", out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tonight's case, through the commands: networkmanager on a six-character machine requires the
|
||||||
|
// resolver provision, and a second consumer there overflows an object store's access key. The
|
||||||
|
// provider's machine still composes; the overflowing consumer is left out of its grants and named,
|
||||||
|
// by push and by `status`, and the keyless consumer is granted with its long name.
|
||||||
|
func TestAnOverflowingConsumerNeverRefusesItsProvidersMachine(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, catalogue.Manifest{Module: "objects", Version: "1",
|
||||||
|
Provides: []catalogue.Offer{{Name: "s3-bucket", Scope: catalogue.ScopeMesh,
|
||||||
|
Identity: &catalogue.OfferIdentity{Max: 20, In: "an S3 access key"}}},
|
||||||
|
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}})
|
||||||
|
register(t, open, catalogue.Manifest{Module: "resolver", Version: "1",
|
||||||
|
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}})
|
||||||
|
register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1",
|
||||||
|
Requires: []string{"wildcard-resolution"}})
|
||||||
|
register(t, open, catalogue.Manifest{Module: "photoalbum", Version: "1", Requires: []string{"s3-bucket"}})
|
||||||
|
register(t, open, catalogue.Manifest{Module: "files", Version: "1", Requires: []string{"s3-bucket"}})
|
||||||
|
for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "resolver"},
|
||||||
|
{"laptop", "networkmanager"}, {"laptop", "photoalbum"}, {"laptop", "files"}} {
|
||||||
|
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||||
|
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The consumer's machine resolves, and says which of its modules no provider will grant.
|
||||||
|
consumer, _, err := planFor(ctx, open, "laptop")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
over := consumer.Overflowing()
|
||||||
|
if len(over) != 1 || over[0].Module != "photoalbum" || over[0].Provision != "s3-bucket" {
|
||||||
|
t.Fatalf("the consumer's side does not name exactly photoalbum: %+v", over)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The provider's machine composes. Under ADR 0049's one bound this was a refusal naming
|
||||||
|
// networkmanager, and no push to the provider could go through.
|
||||||
|
plan, settings, err := planFor(ctx, open, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
declared, err := declarationFor(ctx, open, "anchor", plan, settings)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("one consumer's identity refused its provider's whole machine: %v", err)
|
||||||
|
}
|
||||||
|
if len(declared.withheld) != 1 || declared.withheld[0].Identity != "mesh_laptop_photoalbum" {
|
||||||
|
t.Fatalf("the overflowing consumer is not the one withheld: %+v", declared.withheld)
|
||||||
|
}
|
||||||
|
grants, _, err := grantsFor(ctx, open, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var keyless bool
|
||||||
|
for _, g := range grants {
|
||||||
|
keyless = keyless || g.Provision == "wildcard-resolution" && g.From == "networkmanager"
|
||||||
|
}
|
||||||
|
if !keyless {
|
||||||
|
t.Fatalf("networkmanager, 26 characters, is not granted the keyless resolver provision: %+v", grants)
|
||||||
|
}
|
||||||
|
var granted []string
|
||||||
|
for _, c := range declared.Received["objects"]["s3-bucket"] {
|
||||||
|
granted = append(granted, c.From)
|
||||||
|
}
|
||||||
|
if strings.Join(granted, ",") != "files" {
|
||||||
|
t.Fatalf("the object store grants %v; files and only files fit", granted)
|
||||||
|
}
|
||||||
|
said := printed(t, func() error { reportLeftOut("anchor", declared); return nil })
|
||||||
|
if !strings.Contains(said, `photoalbum on laptop requires s3-bucket from anchor`) ||
|
||||||
|
!strings.Contains(said, "left out of anchor's grants") {
|
||||||
|
t.Fatalf("the push does not say whom it leaves out:\n%s", said)
|
||||||
|
}
|
||||||
|
|
||||||
|
// And `status` names it, and does not call the mesh well while it stands.
|
||||||
|
asked, err := theThreeQuestions(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(asked.overflowing) != 1 || asked.overflowing[0].Module != "photoalbum" {
|
||||||
|
t.Fatalf("status does not carry the overflow: %+v", asked.overflowing)
|
||||||
|
}
|
||||||
|
if asked.well() {
|
||||||
|
t.Fatal("a mesh with a consumer left out of its grants reads as well")
|
||||||
|
}
|
||||||
|
shown := printed(t, func() error { return printStatus(asked) })
|
||||||
|
if !strings.Contains(shown, "identified too long for a provision they require") ||
|
||||||
|
!strings.Contains(shown, "mesh_laptop_photoalbum") {
|
||||||
|
t.Fatalf("status does not say it:\n%s", shown)
|
||||||
|
}
|
||||||
|
body, err := statusAsJSON(asked)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var doc struct {
|
||||||
|
Overflowing []catalogue.Overflow `json:"overflowing"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(body, &doc); err != nil || len(doc.Overflowing) != 1 ||
|
||||||
|
doc.Overflowing[0].Bound.Max != 20 {
|
||||||
|
t.Fatalf("the document does not carry it: %v\n%s", err, body)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -145,6 +145,19 @@ func run() error {
|
|||||||
return seatCommand(ctx, args[1:])
|
return seatCommand(ctx, args[1:])
|
||||||
case "status":
|
case "status":
|
||||||
return statusCommand(ctx, args[1:])
|
return statusCommand(ctx, args[1:])
|
||||||
|
// Acts done by hand, and why (novox/hq to-be 45 §7).
|
||||||
|
case "hand-act":
|
||||||
|
return handActCommand(ctx, args[1:])
|
||||||
|
case "hand-acts":
|
||||||
|
return handActCommand(ctx, append([]string{"list"}, args[1:]...))
|
||||||
|
// What the mesh's bounds will be set from (novox/hq to-be 45 Phase 0).
|
||||||
|
case "durations":
|
||||||
|
return durationsCommand(ctx, args[1:])
|
||||||
|
// What is wrong, and the self-check (novox/hq to-be 45 §2, §4).
|
||||||
|
case "conditions":
|
||||||
|
return conditionsCommand(ctx, args[1:])
|
||||||
|
case "doctor":
|
||||||
|
return doctorCommand(ctx, args[1:])
|
||||||
case "version":
|
case "version":
|
||||||
fmt.Println(version)
|
fmt.Println(version)
|
||||||
return nil
|
return nil
|
||||||
@@ -226,19 +239,33 @@ func usage() {
|
|||||||
kill <id> end a build where it runs; recorded failed, killed by hand
|
kill <id> end a build where it runs; recorded failed, killed by hand
|
||||||
pause [<node>] / resume [<node>] the build seat's holder there, or every holder, takes nothing new / again
|
pause [<node>] / resume [<node>] the build seat's holder there, or every holder, takes nothing new / again
|
||||||
plans retry <id> ask a failed plan's failed builds again, and carry the plan on
|
plans retry <id> ask a failed plan's failed builds again, and carry the plan on
|
||||||
|
plans stop|close <id> --why <text> end a plan by hand; recorded in the hand-act log
|
||||||
|
hand-act record <what> --why <text> --cause <word> [--condition <key>]
|
||||||
|
record an act done by hand outside the mesh (to-be 45 §7)
|
||||||
|
hand-acts [--days N] [--json] what was done by hand lately, why, and which causes repeat
|
||||||
|
conditions [--scope S] [--severity S] [--machine M] [--json]
|
||||||
|
what is wrong now: every open condition, urgent first (to-be 45 §2)
|
||||||
|
conditions show <key> one condition whole, with its evidence
|
||||||
|
conditions silence <key> --for <d> --why <text> send no message for it a while; a hand act
|
||||||
|
conditions history [--days N] [--key K] every raising, change and clearing lately
|
||||||
|
doctor [run|probes|signals] [--json]
|
||||||
|
the self-check: the last verdict, a run now, the probes, the signals' ages
|
||||||
|
durations [--kind K] [--days N] [--json]
|
||||||
|
apply, heartbeat, plan-tier and build durations, per machine or module
|
||||||
collection [--json] kept archives held/unheld by a manifest, and what the sweep may let go
|
collection [--json] kept archives held/unheld by a manifest, and what the sweep may let go
|
||||||
builder issue <name> a broker account for a build machine, scoped to build work,
|
builder issue <name> a broker account for a build machine, scoped to build work,
|
||||||
delivered as the builder module's broker secret (module add it first)
|
delivered as the builder module's broker secret (module add it first)
|
||||||
licence add|list|use|key model access, under the name a person calls it
|
licence add|list|use|key model access, under the name a person calls it
|
||||||
licence manager <name> <node> the node that holds a refreshable licence's refresh token
|
licence manager <name> <node> the node that holds a refreshable licence's refresh token
|
||||||
licence refresh <name> mint a new access token and seal it to every holder
|
licence refresh <name> mint a new access token and seal it to every holder
|
||||||
rotate <provision> [--consumer <n>] a new credential for every holder, both ends at once
|
rotate <provision> [--consumer <n>] [--module <m>] a new credential for every holder, both ends at once
|
||||||
ask <module> <tool> [json] call one of a module's tools over the broker, and print its answer
|
ask <module> <tool> [json] call one of a module's tools over the broker, and print its answer
|
||||||
pin <node> <provision> <from-node> <module>
|
pin <node> <provision> <from-node> <module>
|
||||||
which provider this one gets a provision from: the module, and its node
|
which provider this one gets a provision from: the module, and its node
|
||||||
unpin <node> <provision> put that question back
|
unpin <node> <provision> put that question back
|
||||||
plan <node> [--files|--json] what that node would run, and why
|
plan <node> [--files|--json] what that node would run, and why
|
||||||
push [<node>] [--behind] send a node everything it should be, or only those that need it
|
push [<node>] [--behind] [--why <text>] send a node everything it should be, or only those
|
||||||
|
that need it; --why records it in the hand-act log
|
||||||
version what this binary is
|
version what this binary is
|
||||||
|
|
||||||
Each context reaches its own store through its own credential (novox/hq ADR 0008), named
|
Each context reaches its own store through its own credential (novox/hq ADR 0008), named
|
||||||
@@ -291,6 +318,9 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
|||||||
// When it was asked, so a plan takes as its outcome only a build asked for it or after it
|
// When it was asked, so a plan takes as its outcome only a build asked for it or after it
|
||||||
// (novox/hq 04-ISSUES/219). Zero when the id does not say.
|
// (novox/hq 04-ISSUES/219). Zero when the id does not say.
|
||||||
asked, _ := link.BuildAskedAt(result.ID)
|
asked, _ := link.BuildAskedAt(result.ID)
|
||||||
|
// And how long it took, asked to heard, which a build's bound will be set from (novox/hq to-be 45
|
||||||
|
// Phase 0). Said if lost; never a reason not to take the build in.
|
||||||
|
recordBuildDuration(ctx, b.inv, result, asked)
|
||||||
switch {
|
switch {
|
||||||
case err != nil && result.Failed != "":
|
case err != nil && result.Failed != "":
|
||||||
fmt.Printf("%s: %v\n", result.ID, err)
|
fmt.Printf("%s: %v\n", result.ID, err)
|
||||||
@@ -317,5 +347,7 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
|||||||
result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit))
|
result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit))
|
||||||
saysWhenThePolicyActs(ctx, b.inv, manifest.Module)
|
saysWhenThePolicyActs(ctx, b.inv, manifest.Module)
|
||||||
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
|
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
|
||||||
|
// A module registered may be one a machine is now behind: `status` is composed again.
|
||||||
|
statusFrom.nudge()
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,11 +1,14 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
"crypto/ecdh"
|
"crypto/ecdh"
|
||||||
"crypto/rand"
|
"crypto/rand"
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
@@ -37,6 +40,9 @@ func aMesh(t *testing.T) *stores {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
t.Cleanup(open.Close)
|
t.Cleanup(open.Close)
|
||||||
|
// A condition store of its own, held in memory (novox/hq to-be 45 §2): status leads with what is
|
||||||
|
// open, and a mesh with no bus would otherwise read as one whose conditions cannot be read.
|
||||||
|
withConditionsInMemory(t)
|
||||||
for _, m := range provided {
|
for _, m := range provided {
|
||||||
if err := open.inventory.Provide(t.Context(), m); err != nil {
|
if err := open.inventory.Provide(t.Context(), m); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -106,3 +112,17 @@ func rivals() (catalogue.Manifest, catalogue.Manifest) {
|
|||||||
return catalogue.Manifest{Module: "rival-one", Version: "1", Claims: claim},
|
return catalogue.Manifest{Module: "rival-one", Version: "1", Claims: claim},
|
||||||
catalogue.Manifest{Module: "rival-two", Version: "1", Claims: claim}
|
catalogue.Manifest{Module: "rival-two", Version: "1", Claims: claim}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// withConditionsInMemory gives the test a condition store in memory, as the serving controller's.
|
||||||
|
func withConditionsInMemory(t *testing.T) (*conditions.Keeper, *conditions.InMemory) {
|
||||||
|
t.Helper()
|
||||||
|
store := conditions.NewInMemory()
|
||||||
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||||
|
before := conditionsFrom
|
||||||
|
conditionsFrom = k
|
||||||
|
t.Cleanup(func() {
|
||||||
|
conditionsFrom = before
|
||||||
|
k.Close(context.Background())
|
||||||
|
})
|
||||||
|
return k, store
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package main
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
@@ -59,9 +60,13 @@ func catchingUpOnMerges(ctx context.Context, open *stores, announced merges) {
|
|||||||
return
|
return
|
||||||
case <-tick.C:
|
case <-tick.C:
|
||||||
}
|
}
|
||||||
|
watchedMerges.begin()
|
||||||
err := catchUpOnMerges(ctx, time.Now(), announced, catalogued, f.SourceMoved, func(format string, args ...any) {
|
err := catchUpOnMerges(ctx, time.Now(), announced, catalogued, f.SourceMoved, func(format string, args ...any) {
|
||||||
fmt.Printf(format+"\n", args...)
|
fmt.Printf(format+"\n", args...)
|
||||||
})
|
})
|
||||||
|
// What the pass found is what S5 says (novox/hq to-be 45 §3); a pass that could not read
|
||||||
|
// says that instead, and leaves what the last one found standing.
|
||||||
|
watchedMerges.end(time.Now(), err)
|
||||||
// A pass that cannot read says so once, not every five minutes, and says when it reads again.
|
// A pass that cannot read says so once, not every five minutes, and says when it reads again.
|
||||||
why := ""
|
why := ""
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -114,6 +119,8 @@ func catchUpOnMerges(ctx context.Context, now time.Time, announced merges,
|
|||||||
for _, e := range moves {
|
for _, e := range moves {
|
||||||
names = append(names, e.Manifest.Module)
|
names = append(names, e.Manifest.Module)
|
||||||
}
|
}
|
||||||
|
watchedMerges.found(missedMerge{Owner: a.Owner, Repo: a.Repo, Base: a.Base, Commit: a.Commit,
|
||||||
|
At: a.At, Modules: names})
|
||||||
say("%s/%s merged into %s (%.8s), announced %s ago, and the controller never acted on it: the bus "+
|
say("%s/%s merged into %s (%.8s), announced %s ago, and the controller never acted on it: the bus "+
|
||||||
"did not hand the announcement over (novox/hq issue 266). %s %s behind it; acting on it now",
|
"did not hand the announcement over (novox/hq issue 266). %s %s behind it; acting on it now",
|
||||||
a.Owner, a.Repo, a.Base, a.Commit, now.Sub(a.At).Round(time.Minute), readableList(names),
|
a.Owner, a.Repo, a.Base, a.Commit, now.Sub(a.At).Round(time.Minute), readableList(names),
|
||||||
@@ -129,3 +136,54 @@ func catchUpOnMerges(ctx context.Context, now time.Time, announced merges,
|
|||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// missedMerge is one merge the bus announced and never handed over, as a pass found it.
|
||||||
|
type missedMerge struct {
|
||||||
|
Owner, Repo, Base, Commit string
|
||||||
|
// At is when the bus took the announcement.
|
||||||
|
At time.Time
|
||||||
|
Modules []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// mergeWatch is what the passes found, for S5 (novox/hq to-be 45 §3): **a merge nothing read is
|
||||||
|
// said**, urgent, even though the pass acts on it at once — the bus skipping a message is a fault of
|
||||||
|
// the transport the mesh's every change rides on, and acting late is the repair, not the absence of
|
||||||
|
// the fault. The next pass, finding it acted on, clears it.
|
||||||
|
type mergeWatch struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
passed time.Time
|
||||||
|
err error
|
||||||
|
finding []missedMerge
|
||||||
|
missed []missedMerge
|
||||||
|
}
|
||||||
|
|
||||||
|
var watchedMerges = &mergeWatch{}
|
||||||
|
|
||||||
|
func (w *mergeWatch) begin() {
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
w.finding = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *mergeWatch) found(m missedMerge) {
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
w.finding = append(w.finding, m)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *mergeWatch) end(at time.Time, err error) {
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
w.passed, w.err = at, err
|
||||||
|
if err == nil {
|
||||||
|
w.missed = w.finding
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// last is when the last pass ended, what the last pass that read found, and what the last pass
|
||||||
|
// could not read.
|
||||||
|
func (w *mergeWatch) last() (time.Time, []missedMerge, error) {
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
return w.passed, append([]missedMerge(nil), w.missed...), w.err
|
||||||
|
}
|
||||||
|
|||||||
@@ -40,12 +40,7 @@ func providedModules() []catalogue.Manifest {
|
|||||||
// and neither could be swapped for anything, which is the test of whether a thing is a
|
// and neither could be swapped for anything, which is the test of whether a thing is a
|
||||||
// module at all (novox/hq ADR 0040). They existed because computed output needed somewhere
|
// module at all (novox/hq ADR 0040). They existed because computed output needed somewhere
|
||||||
// to live, and now a module says where it wants it — `facts` in its own manifest.
|
// to live, and now a module says where it wants it — `facts` in its own manifest.
|
||||||
//
|
overlay.Manifest(), overlay.DomainManifest(),
|
||||||
// **And the bundle that required it is gone** (novox/hq ADR 0226): `networking` named this
|
|
||||||
// module's requirement and nothing else, so every machine carried two modules for one
|
|
||||||
// network. A machine is assigned the private network itself; what a release stops shipping
|
|
||||||
// is retired at the next start (stores.go, Inventory.RetireUnshipped).
|
|
||||||
overlay.Manifest(),
|
|
||||||
} {
|
} {
|
||||||
var m catalogue.Manifest
|
var m catalogue.Manifest
|
||||||
b, _ := json.Marshal(raw)
|
b, _ := json.Marshal(raw)
|
||||||
@@ -64,8 +59,19 @@ func moduleCommand(ctx context.Context, args []string) error {
|
|||||||
// `check` needs no mesh, and must not: it is what somebody runs in their own repository before
|
// `check` needs no mesh, and must not: it is what somebody runs in their own repository before
|
||||||
// there is a mesh in reach (novox/hq issue 148). A directory expands to every manifest under it.
|
// there is a mesh in reach (novox/hq issue 148). A directory expands to every manifest under it.
|
||||||
if args[0] == "check" {
|
if args[0] == "check" {
|
||||||
|
set := flag.NewFlagSet("module check", flag.ContinueOnError)
|
||||||
|
// The longest machine name an identity must fit on (novox/hq ADR 0225): a mesh passes its own.
|
||||||
|
longest := set.Int("longest-machine-name", catalogue.DefaultLongestMachine,
|
||||||
|
"judge each module's identity on a machine name this many characters long")
|
||||||
|
given, err := parseAround(set, args[1:])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if *longest < 1 {
|
||||||
|
return errors.New("--longest-machine-name is a length, at least 1")
|
||||||
|
}
|
||||||
var paths []string
|
var paths []string
|
||||||
for _, a := range args[1:] {
|
for _, a := range given {
|
||||||
if info, err := os.Stat(a); err == nil && info.IsDir() {
|
if info, err := os.Stat(a); err == nil && info.IsDir() {
|
||||||
under, err := manifestsUnder(a)
|
under, err := manifestsUnder(a)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -76,7 +82,7 @@ func moduleCommand(ctx context.Context, args []string) error {
|
|||||||
}
|
}
|
||||||
paths = append(paths, a)
|
paths = append(paths, a)
|
||||||
}
|
}
|
||||||
return moduleCheck(paths, os.Stdout)
|
return moduleCheckFor(paths, *longest, os.Stdout)
|
||||||
}
|
}
|
||||||
open, err := openStores(ctx)
|
open, err := openStores(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -499,8 +505,8 @@ const theBrokerSeat = "mesh-broker"
|
|||||||
// says. Only when nothing holds the seat yet (genesis raised the broker as plumbing and no module
|
// says. Only when nothing holds the seat yet (genesis raised the broker as plumbing and no module
|
||||||
// has adopted it) does the hub stand in, which is where the foundation is by convention.
|
// has adopted it) does the hub stand in, which is where the foundation is by convention.
|
||||||
//
|
//
|
||||||
// "On the overlay" is what `whereEveryoneIs` answers — a machine that RESOLVED the private network
|
// "On the overlay" is what `whereEveryoneIs` answers — a machine that RESOLVED the networking
|
||||||
// — not "has an address", which is true of every placed machine and says nothing about
|
// module — not "has an address", which is true of every placed machine and says nothing about
|
||||||
// whether anything can reach it (novox/hq issue 059). A node not on the overlay — at genesis,
|
// whether anything can reach it (novox/hq issue 059). A node not on the overlay — at genesis,
|
||||||
// before any `overlay place`, which is when the builder's account is issued — keeps the genesis
|
// before any `overlay place`, which is when the builder's account is issued — keeps the genesis
|
||||||
// address, so nothing about bring-up changes. This is issue 055, corrected by 059.
|
// address, so nothing about bring-up changes. This is issue 055, corrected by 059.
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -387,7 +388,7 @@ func brokerCommand(ctx context.Context, args []string) error {
|
|||||||
return busAccounts(ctx, args[1:])
|
return busAccounts(ctx, args[1:])
|
||||||
}
|
}
|
||||||
if len(args) > 0 && args[0] == "consumer-reset" {
|
if len(args) > 0 && args[0] == "consumer-reset" {
|
||||||
return consumerReset(args[1:])
|
return consumerReset(ctx, args[1:])
|
||||||
}
|
}
|
||||||
if len(args) == 0 || args[0] != "show" {
|
if len(args) == 0 || args[0] != "show" {
|
||||||
return errors.New("broker show | broker certificate [--check] --into <directory> | broker accounts --into <file> | " +
|
return errors.New("broker show | broker certificate [--check] --into <directory> | broker accounts --into <file> | " +
|
||||||
@@ -414,10 +415,21 @@ func brokerCommand(ctx context.Context, args []string) error {
|
|||||||
// consumerReset re-makes one consumer on a stream that keeps history to start from now (novox/hq issue
|
// consumerReset re-makes one consumer on a stream that keeps history to start from now (novox/hq issue
|
||||||
// 248): the way out of a consumer replaying a week of announcements, said rather than done by hand. A
|
// 248): the way out of a consumer replaying a week of announcements, said rather than done by hand. A
|
||||||
// person's act — what was pending is dropped — so it is a command, and nothing calls it on its own.
|
// person's act — what was pending is dropped — so it is a command, and nothing calls it on its own.
|
||||||
func consumerReset(args []string) error {
|
func consumerReset(ctx context.Context, args []string) error {
|
||||||
if len(args) != 2 {
|
set := flag.NewFlagSet("broker consumer-reset", flag.ContinueOnError)
|
||||||
return errors.New("broker consumer-reset <stream> <consumer>, e.g. broker consumer-reset EVENTS controller")
|
// A repair by hand, which says why (novox/hq to-be 45 §7).
|
||||||
|
why := addHandActFlags(set)
|
||||||
|
args, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
|
if len(args) != 2 {
|
||||||
|
return errors.New("broker consumer-reset <stream> <consumer> --why <text>, e.g. broker consumer-reset EVENTS controller --why ...")
|
||||||
|
}
|
||||||
|
if err := why.require("broker consumer-reset"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
why.record(ctx, "broker consumer-reset", args)
|
||||||
address, err := broker.BusAddress()
|
address, err := broker.BusAddress()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -505,15 +517,22 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
|
|||||||
fmt.Printf(" public domain %s\n", domain)
|
fmt.Printf(" public domain %s\n", domain)
|
||||||
}
|
}
|
||||||
|
|
||||||
// A provider here failing a consumer, or a consumer here failed (novox/hq ADR 0224). Before the
|
// Every open condition about this machine (novox/hq to-be 45 §2) — a provider here failing a
|
||||||
// capabilities, because it is something not working now and they are a description.
|
// consumer, or a consumer here failed, among them (ADR 0224). Before the capabilities, because it
|
||||||
failing, err := failingProviders(ctx, inv)
|
// is something not working now and they are a description. Unreadable is said, not passed over.
|
||||||
|
open, err := openConditions(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
fmt.Printf("\n the open conditions could NOT be read, so whether anything here is wrong is not known: %v\n", err)
|
||||||
}
|
}
|
||||||
if here := failingOn(failing, name); len(here) > 0 {
|
var here []conditions.Condition
|
||||||
fmt.Printf("\n %d consumer(s) a provider keeps failing, here or for a module here:\n", len(here))
|
for _, c := range open {
|
||||||
for _, line := range failingLines(here, time.Now()) {
|
if concerns(c, name) {
|
||||||
|
here = append(here, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(here) > 0 {
|
||||||
|
fmt.Printf("\n %d open condition(s) about this machine:\n", len(here))
|
||||||
|
for _, line := range conditionLines(here, time.Now()) {
|
||||||
fmt.Printf(" %s\n", line)
|
fmt.Printf(" %s\n", line)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+50
-16
@@ -399,7 +399,7 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
|||||||
}
|
}
|
||||||
out := sendable{Resources: composed.Resources, Adoption: adoption,
|
out := sendable{Resources: composed.Resources, Adoption: adoption,
|
||||||
Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
|
Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
|
||||||
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut}
|
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld}
|
||||||
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
|
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
|
||||||
// 0221). Read only on the send path: a question about what would be sent records nothing.
|
// 0221). Read only on the send path: a question about what would be sent records nothing.
|
||||||
if choosing == Allocating {
|
if choosing == Allocating {
|
||||||
@@ -464,6 +464,11 @@ func reportLeftOut(node string, declared sendable) {
|
|||||||
"what the machine holds for it is kept and its containers are untouched. %s\n",
|
"what the machine holds for it is kept and its containers are untouched. %s\n",
|
||||||
node, m, declared.leftOutWhy[m])
|
node, m, declared.leftOutWhy[m])
|
||||||
}
|
}
|
||||||
|
// And whom it serves nothing, because their identity overflows what the provision keeps (ADR
|
||||||
|
// 0225): the machine is sent everything else, and the consumer is named.
|
||||||
|
for _, o := range declared.withheld {
|
||||||
|
fmt.Printf("%s: %s\n", node, o)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// renderingFor is everything a node's declaration is composed with, and the node's record.
|
// renderingFor is everything a node's declaration is composed with, and the node's record.
|
||||||
@@ -471,7 +476,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
|||||||
plan catalogue.Resolution, settings catalogue.SettingsBy,
|
plan catalogue.Resolution, settings catalogue.SettingsBy,
|
||||||
gens map[string]catalogue.Generator, choosing Choosing) (catalogue.Rendering, inventory.Node, error) {
|
gens map[string]catalogue.Generator, choosing Choosing) (catalogue.Rendering, inventory.Node, error) {
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
grants, err := grantsFor(ctx, open, node)
|
grants, withheld, err := grantsFor(ctx, open, node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return catalogue.Rendering{}, inventory.Node{}, err
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
}
|
}
|
||||||
@@ -794,7 +799,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
|||||||
Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
|
Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
|
||||||
Kept: kept, Adopted: record.Adopted, OutwardLinks: outwardLinks,
|
Kept: kept, Adopted: record.Adopted, OutwardLinks: outwardLinks,
|
||||||
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, SeatReach: reach, Built: built,
|
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, SeatReach: reach, Built: built,
|
||||||
BusUsers: busUsers,
|
BusUsers: busUsers, Withheld: withheld,
|
||||||
}, record, nil
|
}, record, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -930,28 +935,34 @@ func certificateFor(ctx context.Context, open *stores, node string) (string, str
|
|||||||
// The mirror of what a consumer is given, and the half that makes the credential real: a password
|
// The mirror of what a consumer is given, and the half that makes the credential real: a password
|
||||||
// nothing was told to create is a password that authenticates nowhere. Sealed to this node, so
|
// nothing was told to create is a password that authenticates nowhere. Sealed to this node, so
|
||||||
// the mesh hands over something it cannot itself use.
|
// the mesh hands over something it cannot itself use.
|
||||||
func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Grant, error) {
|
//
|
||||||
|
// **One consumer's identity never refuses the provider's machine** (novox/hq ADR 0225, issue 263).
|
||||||
|
// A consumer whose identity overflows the provision's bound is left out of the grants and returned
|
||||||
|
// beside them, for push, plan and `status` to say; every other consumer is granted and the provider's
|
||||||
|
// declaration composes. Refusing here once made a whole machine unpushable for one module elsewhere.
|
||||||
|
func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Grant, []catalogue.Overflow, error) {
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
issued, err := inv.SecretsFrom(ctx, node)
|
issued, err := inv.SecretsFrom(ctx, node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Where each consumer is, so a provider that must reach back to one does not have to know how
|
// Where each consumer is, so a provider that must reach back to one does not have to know how
|
||||||
// the mesh names machines.
|
// the mesh names machines.
|
||||||
shelf, err := inv.Catalogue(ctx)
|
shelf, err := inv.Catalogue(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
|
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// What each consumer actually asked for, taken from that machine's own resolution rather than
|
// What each consumer actually asked for, taken from that machine's own resolution rather than
|
||||||
// from a record beside it. A provider told to create a password and not what to create it for
|
// from a record beside it. A provider told to create a password and not what to create it for
|
||||||
// can do nothing with it, and the name a consumer wants is the consumer's to say.
|
// can do nothing with it, and the name a consumer wants is the consumer's to say.
|
||||||
out := make([]catalogue.Grant, 0, len(issued))
|
out := make([]catalogue.Grant, 0, len(issued))
|
||||||
|
var withheld []catalogue.Overflow
|
||||||
for _, s := range issued {
|
for _, s := range issued {
|
||||||
plan, settings, err := planFor(ctx, open, s.Consumer)
|
plan, settings, err := planFor(ctx, open, s.Consumer)
|
||||||
switch {
|
switch {
|
||||||
@@ -964,11 +975,11 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
|||||||
// The mesh could not be asked what they wanted, which is not the same as their wanting
|
// The mesh could not be asked what they wanted, which is not the same as their wanting
|
||||||
// nothing — and withholding a grant on that reading takes a consumer's access away
|
// nothing — and withholding a grant on that reading takes a consumer's access away
|
||||||
// (novox/hq 04-ISSUES/152).
|
// (novox/hq 04-ISSUES/152).
|
||||||
return nil, fmt.Errorf("what %s asked of %s cannot be read: %w", s.Consumer, s.Name, err)
|
return nil, nil, fmt.Errorf("what %s asked of %s cannot be read: %w", s.Consumer, s.Name, err)
|
||||||
}
|
}
|
||||||
values, asks, err := plan.ContributionsFrom(s.Name, s.ConsumerModule, settings)
|
values, asks, err := plan.ContributionsFrom(s.Name, s.ConsumerModule, settings)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
// A port in there is the consumer's software port until this. The consumer is on another
|
// A port in there is the consumer's software port until this. The consumer is on another
|
||||||
// machine, so the assignment that moved it is that machine's — fetched here rather than
|
// machine, so the assignment that moved it is that machine's — fetched here rather than
|
||||||
@@ -976,7 +987,7 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
|||||||
// this case (novox/hq 04-ISSUES/038, the cross-node half).
|
// this case (novox/hq 04-ISSUES/038, the cross-node half).
|
||||||
published, err := portsOn(ctx, inv, s.Consumer, s.ConsumerModule)
|
published, err := portsOn(ctx, inv, s.Consumer, s.ConsumerModule)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
values = catalogue.AtPublishedPort(values, s.ConsumerModule, published)
|
values = catalogue.AtPublishedPort(values, s.ConsumerModule, published)
|
||||||
from := s.ConsumerModule
|
from := s.ConsumerModule
|
||||||
@@ -987,9 +998,10 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
|||||||
from = ""
|
from = ""
|
||||||
}
|
}
|
||||||
// The consumer's identity slug, from its own manifest, carried on the grant so the provider
|
// The consumer's identity slug, from its own manifest, carried on the grant so the provider
|
||||||
// derives the same login the consumer does (novox/hq ADR 0049). Refused here if it still would
|
// derives the same login the consumer does (novox/hq ADR 0049). Judged against the bound of
|
||||||
// not fit the tightest backend — the mesh chose the name, so the mesh refuses it, with the
|
// this provision, as the consumer's resolution states it from the provider's offer (ADR
|
||||||
// remedy a short slug rather than a login a provider silently shortened.
|
// 0225): a consumer it would not fit is left out of the grants and said, rather than a login a
|
||||||
|
// provider silently shortened — and rather than this whole machine refused for it.
|
||||||
slug := ""
|
slug := ""
|
||||||
for _, mm := range plan.Modules {
|
for _, mm := range plan.Modules {
|
||||||
if mm.Module == s.ConsumerModule {
|
if mm.Module == s.ConsumerModule {
|
||||||
@@ -998,15 +1010,32 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if from != "" {
|
if from != "" {
|
||||||
if err := catalogue.CheckIdentity(s.Consumer, catalogue.IdentitySource(slug, s.ConsumerModule)); err != nil {
|
bound := boundOfGrant(plan, s, node)
|
||||||
return nil, err
|
source := catalogue.IdentitySource(slug, s.ConsumerModule)
|
||||||
|
if catalogue.CheckIdentityWithin(s.Consumer, source, bound) != nil {
|
||||||
|
withheld = append(withheld, catalogue.Overflow{Provision: s.Name, Provider: node,
|
||||||
|
Consumer: s.Consumer, Module: s.ConsumerModule,
|
||||||
|
Identity: catalogue.ConsumerIdentity(s.Consumer, source), Bound: bound})
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
out = append(out, catalogue.Grant{
|
out = append(out, catalogue.Grant{
|
||||||
Provision: s.Name, Consumer: s.Consumer, At: onNetwork[s.Consumer],
|
Provision: s.Name, Consumer: s.Consumer, At: onNetwork[s.Consumer],
|
||||||
From: from, Values: values, Slug: slug, Sealed: s.ForProvider, Local: s.Local})
|
From: from, Values: values, Slug: slug, Sealed: s.ForProvider, Local: s.Local})
|
||||||
}
|
}
|
||||||
return out, nil
|
return out, withheld, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// boundOfGrant is the identity bound the consumer's own resolution states for the requirement this
|
||||||
|
// grant answers. A requirement not found there is held to the tightest bound the mesh knows rather
|
||||||
|
// than to none: what the provider keeps of it is not known here.
|
||||||
|
func boundOfGrant(consumer catalogue.Resolution, s inventory.Secret, provider string) catalogue.IdentityBound {
|
||||||
|
for _, n := range consumer.Needs {
|
||||||
|
if n.Name == s.Name && n.For == s.ConsumerModule && n.From == provider {
|
||||||
|
return n.Identity
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return catalogue.DefaultIdentityBound
|
||||||
}
|
}
|
||||||
|
|
||||||
// listensLines is what a person is told about what this module would open, and why — the same
|
// listensLines is what a person is told about what this module would open, and why — the same
|
||||||
@@ -1082,6 +1111,11 @@ func planCommand(ctx context.Context, args []string) error {
|
|||||||
left := plan.LeftOut(settings, record.Adopted)
|
left := plan.LeftOut(settings, record.Adopted)
|
||||||
reportLeftOut(args[0], sendable{LeftOut: sortedKeysOf(left), leftOutWhy: left})
|
reportLeftOut(args[0], sendable{LeftOut: sortedKeysOf(left), leftOutWhy: left})
|
||||||
}
|
}
|
||||||
|
// And which of its modules no provider will grant, because the identity overflows the bound of
|
||||||
|
// what it requires (novox/hq ADR 0225) — said on the machine the remedy is for.
|
||||||
|
for _, o := range plan.Overflowing() {
|
||||||
|
fmt.Printf("%s: %s\n", args[0], o)
|
||||||
|
}
|
||||||
// And a setting that reaches nothing — refused where it is stored, and said here for one
|
// And a setting that reaches nothing — refused where it is stored, and said here for one
|
||||||
// stored before its definition moved from under it.
|
// stored before its definition moved from under it.
|
||||||
for _, m := range plan.Modules {
|
for _, m := range plan.Modules {
|
||||||
|
|||||||
@@ -0,0 +1,820 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"regexp"
|
||||||
|
"slices"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/micro"
|
||||||
|
"github.com/novox/mesh-host/validate"
|
||||||
|
"golang.org/x/net/dns/dnsmessage"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/artifacts"
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
"github.com/novox/mesh-controller/internal/overlay"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The probes of the self-check's registry (doctor.go), one function each. A probe answers what is
|
||||||
|
// wrong now as observations, or an error when it could not tell — never "nothing" for "could not
|
||||||
|
// look" (ADR 0227 rule 4).
|
||||||
|
|
||||||
|
// probeDeclarations is D1: every machine's declaration composes, and the node-engine's own validator
|
||||||
|
// (mesh-host's `validate`, the package the host runs) takes it.
|
||||||
|
func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
open := d.open
|
||||||
|
nodes, err := open.inventory.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var out []conditions.Observation
|
||||||
|
// The private network every declaration is composed with. Not computable is not "could not
|
||||||
|
// look": it is the finding — no machine's declaration composes without it — and the machines that
|
||||||
|
// do not resolve, which are usually why, are named below.
|
||||||
|
gens, gensErr := generators(ctx, open)
|
||||||
|
if gensErr != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMesh, ID: "private-network",
|
||||||
|
Token: "uncomposable", Severity: conditions.Urgent,
|
||||||
|
Summary: "the private network cannot be computed, so no machine's declaration composes",
|
||||||
|
Said: firstLine(gensErr.Error())})
|
||||||
|
}
|
||||||
|
for _, n := range nodes {
|
||||||
|
plan, settings, err := planFor(ctx, open, n.Name)
|
||||||
|
if err != nil && !unresolvable(err) {
|
||||||
|
return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err)
|
||||||
|
}
|
||||||
|
var problems []string
|
||||||
|
if err == nil && gensErr == nil {
|
||||||
|
var declared sendable
|
||||||
|
if declared, err = declarationWith(ctx, open, n.Name, plan, settings, gens, Reading); err == nil {
|
||||||
|
var body []byte
|
||||||
|
if body, err = declared.Body(); err == nil {
|
||||||
|
problems = validate.Declaration(body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "uncomposable",
|
||||||
|
Machine: n.Name, Severity: conditions.Urgent,
|
||||||
|
Summary: fmt.Sprintf("nothing can be sent to %s: its declaration does not compose — %s", n.Name,
|
||||||
|
oneLine(err.Error())),
|
||||||
|
Said: oneLine(err.Error())})
|
||||||
|
case len(problems) > 0:
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "refused",
|
||||||
|
Machine: n.Name, Severity: conditions.Urgent,
|
||||||
|
Summary: fmt.Sprintf("%s's node-engine would refuse its declaration whole: %d problem(s), the first: %s",
|
||||||
|
n.Name, len(problems), problems[0]),
|
||||||
|
Said: strings.Join(problems, "; ")})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeResolvers is D2: every holder of the mesh's resolver answers each machine's name with its
|
||||||
|
// address for IPv4, and with no address and no error for IPv6 — NODATA, not NXDOMAIN, which musl
|
||||||
|
// takes as final (issue 262).
|
||||||
|
func probeResolvers(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
inv := d.open.inventory
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
holders, err := replicatedHolders(ctx, inv, shelf)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
resolvers := holders["mesh-dns-resolver"]
|
||||||
|
if len(resolvers) == 0 {
|
||||||
|
return nil, nil // the mesh holds no resolver of its own: nothing is asked of one
|
||||||
|
}
|
||||||
|
places, err := onTheNetwork(ctx, inv, shelf)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
suffix := overlay.Suffix()
|
||||||
|
var out []conditions.Observation
|
||||||
|
for holder, at := range resolvers {
|
||||||
|
var wrong []string
|
||||||
|
for _, p := range places {
|
||||||
|
if p.Address == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
name := p.Name + "." + suffix
|
||||||
|
v4, rcode, err := askResolver(ctx, at, name, dnsmessage.TypeA)
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
wrong = append(wrong, fmt.Sprintf("%s for %s: %v", "A", name, err))
|
||||||
|
continue
|
||||||
|
case rcode != dnsmessage.RCodeSuccess:
|
||||||
|
wrong = append(wrong, fmt.Sprintf("%s answered %s for %s (A)", holder, rcode, name))
|
||||||
|
case !slices.Contains(v4, p.Address):
|
||||||
|
wrong = append(wrong, fmt.Sprintf("%s answered %v for %s (A), not %s", holder, v4, name, p.Address))
|
||||||
|
}
|
||||||
|
v6, rcode, err := askResolver(ctx, at, name, dnsmessage.TypeAAAA)
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
wrong = append(wrong, fmt.Sprintf("%s for %s: %v", "AAAA", name, err))
|
||||||
|
case rcode != dnsmessage.RCodeSuccess:
|
||||||
|
wrong = append(wrong, fmt.Sprintf("%s answered %s for %s (AAAA), not NODATA: a musl machine "+
|
||||||
|
"takes that as no such name", holder, rcode, name))
|
||||||
|
case len(v6) > 0:
|
||||||
|
wrong = append(wrong, fmt.Sprintf("%s answered %v for %s (AAAA); the mesh has no IPv6 addresses", holder, v6, name))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(wrong) > 0 {
|
||||||
|
node := strings.TrimSuffix(holder, "."+suffix)
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeSeat, ID: "mesh-dns-resolver." + node,
|
||||||
|
Token: "wrong", Machine: node, Severity: conditions.Urgent,
|
||||||
|
Summary: fmt.Sprintf("the mesh's resolver on %s does not answer machine names as it must: %s",
|
||||||
|
node, wrong[0]),
|
||||||
|
Said: strings.Join(wrong, "; ")})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sortedFound(out), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolverPort is where a resolver answers; a variable so a test can stand one up.
|
||||||
|
var resolverPort = "53"
|
||||||
|
|
||||||
|
// askResolver asks one resolver one question over UDP, and answers the addresses and the code.
|
||||||
|
func askResolver(ctx context.Context, at, name string, kind dnsmessage.Type) ([]string, dnsmessage.RCode, error) {
|
||||||
|
q, err := dnsmessage.NewName(strings.TrimSuffix(name, ".") + ".")
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
msg := dnsmessage.Message{Header: dnsmessage.Header{ID: uint16(time.Now().UnixNano()), RecursionDesired: true},
|
||||||
|
Questions: []dnsmessage.Question{{Name: q, Type: kind, Class: dnsmessage.ClassINET}}}
|
||||||
|
packed, err := msg.Pack()
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
dialer := net.Dialer{Timeout: 3 * time.Second}
|
||||||
|
conn, err := dialer.DialContext(ctx, "udp", net.JoinHostPort(at, resolverPort))
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
_ = conn.SetDeadline(time.Now().Add(3 * time.Second))
|
||||||
|
if _, err := conn.Write(packed); err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
buf := make([]byte, 1500)
|
||||||
|
n, err := conn.Read(buf)
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, fmt.Errorf("no answer from %s: %w", at, err)
|
||||||
|
}
|
||||||
|
var answer dnsmessage.Message
|
||||||
|
if err := answer.Unpack(buf[:n]); err != nil {
|
||||||
|
return nil, 0, fmt.Errorf("an answer from %s that cannot be read: %w", at, err)
|
||||||
|
}
|
||||||
|
var addresses []string
|
||||||
|
for _, a := range answer.Answers {
|
||||||
|
switch r := a.Body.(type) {
|
||||||
|
case *dnsmessage.AResource:
|
||||||
|
addresses = append(addresses, net.IP(r.A[:]).String())
|
||||||
|
case *dnsmessage.AAAAResource:
|
||||||
|
addresses = append(addresses, net.IP(r.AAAA[:]).String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return addresses, answer.RCode, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeHolders is D3: every seat that serves verbs has, on every machine that holds it and is heard
|
||||||
|
// from, a holder answering the bus's discovery for that seat. A machine past its heartbeat's bound is
|
||||||
|
// S1's, and is not asked about here.
|
||||||
|
func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
entries, err := d.open.inventory.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
recorded, err := d.open.inventory.Holdings(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
heard := heardMachines(d)
|
||||||
|
expected := map[string]map[string]bool{} // seat → machine
|
||||||
|
add := func(seat, node string) {
|
||||||
|
if expected[seat] == nil {
|
||||||
|
expected[seat] = map[string]bool{}
|
||||||
|
}
|
||||||
|
expected[seat][node] = true
|
||||||
|
}
|
||||||
|
for _, s := range catalogue.SeatsWithAProtocol() {
|
||||||
|
if len(s.Serves) == 0 || s.Name == catalogue.ControllerSeatName {
|
||||||
|
continue // a seat with no verb has nothing to answer with; this controller is answering now
|
||||||
|
}
|
||||||
|
onRecord := false
|
||||||
|
for _, h := range recorded {
|
||||||
|
if h.Claim == s.Name && heard[h.Node] {
|
||||||
|
add(s.Name, h.Node)
|
||||||
|
onRecord = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if onRecord && s.Scope == catalogue.ScopeMesh {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, e := range entries {
|
||||||
|
for _, c := range e.Manifest.Claims {
|
||||||
|
if c.Name != s.Name {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, node := range e.On {
|
||||||
|
if heard[node] && (s.Scope == catalogue.ScopeNode || !onRecord) {
|
||||||
|
add(s.Name, node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(expected) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
answering, err := discoverHolders(ctx, d.js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var out []conditions.Observation
|
||||||
|
for seat, nodes := range expected {
|
||||||
|
for node := range nodes {
|
||||||
|
if answering[seat][node] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeSeat, ID: seat + "." + node,
|
||||||
|
Token: "silent", Machine: node, Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s's holder on %s does not answer the bus: its verbs reach nothing there", seat, node),
|
||||||
|
Said: fmt.Sprintf("no answer for %s from %s to the bus's discovery", seat, node)})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sortedFound(out), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// heardMachines is every machine within its heartbeat's bound, as the watchdogs last saw.
|
||||||
|
func heardMachines(d *doctor) map[string]bool {
|
||||||
|
out := map[string]bool{}
|
||||||
|
if d.watchdogs == nil {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
f := d.watchdogs.lastFacts()
|
||||||
|
if f == nil {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
for _, m := range f.machines {
|
||||||
|
if !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) <= heartbeatBound(m.every) && !m.asleep() {
|
||||||
|
out[m.name] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// discoveryPatience is how long the discovery's answers are waited for after the last arrived, and at
|
||||||
|
// the most (ADR 0197: a large runtime's answer arrives last).
|
||||||
|
const (
|
||||||
|
discoveryQuiet = 1500 * time.Millisecond
|
||||||
|
discoveryPatience = 8 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
// discoverHolders asks the bus's discovery who serves what, and answers seat → machine for every
|
||||||
|
// endpoint a seat's verb is served on.
|
||||||
|
func discoverHolders(ctx context.Context, conn *nats.Conn) (map[string]map[string]bool, error) {
|
||||||
|
inbox := conn.NewRespInbox()
|
||||||
|
sub, err := conn.SubscribeSync(inbox)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer func() { _ = sub.Unsubscribe() }()
|
||||||
|
if err := conn.PublishRequest("$SRV.INFO", inbox, nil); err != nil {
|
||||||
|
return nil, fmt.Errorf("asking the bus who serves what: %w", err)
|
||||||
|
}
|
||||||
|
out := map[string]map[string]bool{}
|
||||||
|
deadline := time.Now().Add(discoveryPatience)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
wait, cancel := context.WithTimeout(ctx, discoveryQuiet)
|
||||||
|
msg, err := sub.NextMsgWithContext(wait)
|
||||||
|
cancel()
|
||||||
|
if err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
var info micro.Info
|
||||||
|
if json.Unmarshal(msg.Data, &info) != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, e := range info.Endpoints {
|
||||||
|
seat, node := e.Metadata["seat"], e.Metadata["node"]
|
||||||
|
if seat == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if node == "" {
|
||||||
|
node = info.ID
|
||||||
|
}
|
||||||
|
if out[seat] == nil {
|
||||||
|
out[seat] = map[string]bool{}
|
||||||
|
}
|
||||||
|
out[seat][node] = true
|
||||||
|
}
|
||||||
|
// A holder that announces itself as its seat, by name and machine.
|
||||||
|
if out[info.Name] == nil {
|
||||||
|
out[info.Name] = map[string]bool{}
|
||||||
|
}
|
||||||
|
out[info.Name][info.ID] = true
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeArchives is D4: every archive the mesh keeps is held by its manifest in the artifact store.
|
||||||
|
func probeArchives(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
inv := d.open.inventory
|
||||||
|
kept, err := inv.KeptArchives(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(kept) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
store, err := artifactStoreAddress(ctx, inv, shelf, "")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if store == "" {
|
||||||
|
return nil, errors.New("the mesh keeps archives and has no artifact store on its network to ask")
|
||||||
|
}
|
||||||
|
var report collectionReport
|
||||||
|
if unasked, stopped := askHeld(ctx, artifacts.Store{Address: store}, kept, &report); stopped != "" {
|
||||||
|
return nil, fmt.Errorf("%d kept archive(s) could not be asked about: %s", unasked, stopped)
|
||||||
|
}
|
||||||
|
var out []conditions.Observation
|
||||||
|
if n := len(report.Unheld); n > 0 {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMesh, ID: "artifact-store", Token: "archives-unheld",
|
||||||
|
Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%d of %d kept archive(s) are not held by a manifest: the store's collector would "+
|
||||||
|
"delete them", n, len(kept)),
|
||||||
|
Said: "first: " + report.Unheld[0]})
|
||||||
|
}
|
||||||
|
if n := len(report.Missing); n > 0 {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMesh, ID: "artifact-store", Token: "archives-missing",
|
||||||
|
Kind: "archives-missing", Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%d kept archive(s) are not in the artifact store at all", n),
|
||||||
|
Said: "first: " + report.Missing[0]})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// consumerFarBehind is how far a durable consumer may be from its stream's head (D6).
|
||||||
|
const consumerFarBehind = 1000
|
||||||
|
|
||||||
|
// probeConsumers is D6: every durable consumer the mesh expects exists, as the controller defines it,
|
||||||
|
// and is near its stream's head.
|
||||||
|
func probeConsumers(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
_, consumers, err := expectedBusObjects(ctx, d.open.inventory)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
js := d.js.Context()
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, c := range consumers {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
info, err := js.ConsumerInfo(c.Stream, c.Name, nats.Context(ctx))
|
||||||
|
who := c.Stream + "." + c.Name
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, nats.ErrConsumerNotFound) || errors.Is(err, nats.ErrStreamNotFound):
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "missing",
|
||||||
|
Kind: "consumer-lost", Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s is not on the bus: what it delivers reaches nobody", consumerWords(c)),
|
||||||
|
Said: "no consumer " + c.Name + " on " + c.Stream})
|
||||||
|
continue
|
||||||
|
case err != nil:
|
||||||
|
return nil, fmt.Errorf("%s cannot be read: %w", consumerWords(c), err)
|
||||||
|
}
|
||||||
|
if differs := consumerDiffers(c, info.Config); differs != "" {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "redefined",
|
||||||
|
Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s is not as the controller defines it: %s", consumerWords(c), differs),
|
||||||
|
Said: differs})
|
||||||
|
}
|
||||||
|
if c.Stream != "NODES" && info.NumPending > consumerFarBehind {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "behind",
|
||||||
|
Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s is %d message(s) behind its stream's head", consumerWords(c), info.NumPending),
|
||||||
|
Said: fmt.Sprintf("%d pending, %d handed out and not settled", info.NumPending, info.NumAckPending)})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sortedFound(out), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// consumerWords is a consumer as the mesh says it, with its name.
|
||||||
|
func consumerWords(c broker.Consumer) string {
|
||||||
|
return fmt.Sprintf("%s (%s on %s)", link.ConsumerInWords(c.Stream, c.Name), c.Name, c.Stream)
|
||||||
|
}
|
||||||
|
|
||||||
|
// consumerDiffers is what about a consumer on the bus is not as defined; empty when nothing is. The
|
||||||
|
// delivery subject is not compared: one kept as it was while a holder is bound is the assertion's
|
||||||
|
// stated choice (novox/hq issue 156).
|
||||||
|
func consumerDiffers(want broker.Consumer, have nats.ConsumerConfig) string {
|
||||||
|
var differs []string
|
||||||
|
haveFilters := append([]string(nil), have.FilterSubjects...)
|
||||||
|
if have.FilterSubject != "" {
|
||||||
|
haveFilters = append(haveFilters, have.FilterSubject)
|
||||||
|
}
|
||||||
|
wantFilters := append([]string(nil), want.Filters...)
|
||||||
|
sort.Strings(haveFilters)
|
||||||
|
sort.Strings(wantFilters)
|
||||||
|
if !slices.Equal(haveFilters, wantFilters) {
|
||||||
|
differs = append(differs, fmt.Sprintf("filters %v, defined %v", haveFilters, wantFilters))
|
||||||
|
}
|
||||||
|
if have.AckPolicy != nats.AckExplicitPolicy {
|
||||||
|
differs = append(differs, "acknowledges "+have.AckPolicy.String()+", defined explicit")
|
||||||
|
}
|
||||||
|
if wantMax := want.MaxDeliver; wantMax != 0 && have.MaxDeliver != wantMax {
|
||||||
|
differs = append(differs, fmt.Sprintf("hands a message over %d times, defined %d", have.MaxDeliver, wantMax))
|
||||||
|
}
|
||||||
|
if wantWait := time.Duration(want.AckWaitSeconds) * time.Second; wantWait != 0 && have.AckWait != wantWait {
|
||||||
|
differs = append(differs, fmt.Sprintf("waits %s for an acknowledgement, defined %s", have.AckWait, wantWait))
|
||||||
|
}
|
||||||
|
if want.MaxAckPending != 0 && have.MaxAckPending != want.MaxAckPending {
|
||||||
|
differs = append(differs, fmt.Sprintf("hands out %d at once, defined %d", have.MaxAckPending, want.MaxAckPending))
|
||||||
|
}
|
||||||
|
if wantPush, havePush := want.Push || want.Queue != "", have.DeliverSubject != ""; wantPush != havePush {
|
||||||
|
differs = append(differs, "delivers by the other shape (push or pull) than defined")
|
||||||
|
}
|
||||||
|
return strings.Join(differs, "; ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeStreams is D7: every stream the controller defines exists as defined, and its own buckets.
|
||||||
|
func probeStreams(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
streams, _, err := expectedBusObjects(ctx, d.open.inventory)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
js := d.js.Context()
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, s := range streams {
|
||||||
|
info, err := js.StreamInfo(s.Name, nats.Context(ctx))
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, nats.ErrStreamNotFound):
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: s.Name, Token: "stream-missing",
|
||||||
|
Kind: "stream-wrong", Severity: conditions.Urgent,
|
||||||
|
Summary: fmt.Sprintf("the stream %s is not on the bus: %s", s.Name, firstLine(s.Why)),
|
||||||
|
Said: "no stream " + s.Name})
|
||||||
|
continue
|
||||||
|
case err != nil:
|
||||||
|
return nil, fmt.Errorf("the stream %s cannot be read: %w", s.Name, err)
|
||||||
|
}
|
||||||
|
if differs := streamDiffers(s, info.Config); differs != "" {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: s.Name, Token: "stream-redefined",
|
||||||
|
Kind: "stream-wrong", Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("the stream %s is not as the controller defines it: %s", s.Name, differs),
|
||||||
|
Said: differs})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, bucket := range broker.ControllerBuckets() {
|
||||||
|
if _, err := js.StreamInfo("KV_"+bucket, nats.Context(ctx)); errors.Is(err, nats.ErrStreamNotFound) {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: bucket, Token: "bucket-missing",
|
||||||
|
Kind: "stream-wrong", Severity: conditions.Urgent,
|
||||||
|
Summary: fmt.Sprintf("the controller's bucket %s is not on the bus: what it keeps there is not kept", bucket),
|
||||||
|
Said: "no bucket " + bucket})
|
||||||
|
} else if err != nil {
|
||||||
|
return nil, fmt.Errorf("the bucket %s cannot be read: %w", bucket, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sortedFound(out), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// streamDiffers is what about a stream on the bus is not as defined; empty when nothing is.
|
||||||
|
func streamDiffers(want broker.Stream, have nats.StreamConfig) string {
|
||||||
|
var differs []string
|
||||||
|
haveSubjects := append([]string(nil), have.Subjects...)
|
||||||
|
wantSubjects := append([]string(nil), want.Subjects...)
|
||||||
|
sort.Strings(haveSubjects)
|
||||||
|
sort.Strings(wantSubjects)
|
||||||
|
if !slices.Equal(haveSubjects, wantSubjects) {
|
||||||
|
differs = append(differs, fmt.Sprintf("subjects %v, defined %v", haveSubjects, wantSubjects))
|
||||||
|
}
|
||||||
|
retention, perSubject := nats.LimitsPolicy, int64(want.MaxMsgsPerSubject)
|
||||||
|
switch want.Retention {
|
||||||
|
case broker.RetentionWorkQueue:
|
||||||
|
retention = nats.WorkQueuePolicy
|
||||||
|
case broker.RetentionLastPerSubject:
|
||||||
|
perSubject = 1
|
||||||
|
}
|
||||||
|
if have.Retention != retention {
|
||||||
|
differs = append(differs, fmt.Sprintf("keeps by %s, defined %s", have.Retention, retention))
|
||||||
|
}
|
||||||
|
if perSubject != 0 && have.MaxMsgsPerSubject != perSubject {
|
||||||
|
differs = append(differs, fmt.Sprintf("keeps %d per subject, defined %d", have.MaxMsgsPerSubject, perSubject))
|
||||||
|
}
|
||||||
|
return strings.Join(differs, "; ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// ipLike finds the addresses in a ban list, whatever shape its holder answers in.
|
||||||
|
var ipLike = regexp.MustCompile(`\b(?:\d{1,3}\.){3}\d{1,3}\b`)
|
||||||
|
|
||||||
|
// probeBans is D8: no address the mesh owns is in any machine's ban list. The mesh's own addresses are
|
||||||
|
// every machine's private address and the address its endpoint names; the ban lists are asked of each
|
||||||
|
// machine's holder of `node-intrusion-prevention` that is heard from.
|
||||||
|
func probeBans(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
inv := d.open.inventory
|
||||||
|
places, err := inv.Overlays(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
owned := map[string]string{} // address → whose
|
||||||
|
for _, p := range places {
|
||||||
|
if p.Address != "" {
|
||||||
|
owned[p.Address] = p.Name + "'s private address"
|
||||||
|
}
|
||||||
|
if host, _, err := net.SplitHostPort(p.Endpoint); err == nil && host != "" {
|
||||||
|
if ip := net.ParseIP(host); ip != nil {
|
||||||
|
owned[ip.String()] = p.Name + "'s endpoint"
|
||||||
|
} else if addrs, err := net.DefaultResolver.LookupHost(ctx, host); err == nil {
|
||||||
|
for _, a := range addrs {
|
||||||
|
owned[a] = p.Name + "'s endpoint (" + host + ")"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
entries, err := inv.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
heard := heardMachines(d)
|
||||||
|
var holders []string
|
||||||
|
for _, e := range entries {
|
||||||
|
for _, c := range e.Manifest.Claims {
|
||||||
|
if c.Name == "node-intrusion-prevention" {
|
||||||
|
for _, node := range e.On {
|
||||||
|
if heard[node] && !slices.Contains(holders, node) {
|
||||||
|
holders = append(holders, node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(holders)
|
||||||
|
var out []conditions.Observation
|
||||||
|
// Every machine asked at once: one after another, four holders that each wait their bound
|
||||||
|
// outlast the probe's thirty seconds, and the probe says nothing about any of them.
|
||||||
|
answers := make([]json.RawMessage, len(holders))
|
||||||
|
errs := make([]error, len(holders))
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for i, node := range holders {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(i int, node string) {
|
||||||
|
defer wg.Done()
|
||||||
|
answers[i], errs[i] = askSeatTool(ctx, d.js.Conn(), "node-intrusion-prevention", "banned", node)
|
||||||
|
}(i, node)
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
var unasked []string
|
||||||
|
for i, node := range holders {
|
||||||
|
answer, err := answers[i], errs[i]
|
||||||
|
if err != nil {
|
||||||
|
unasked = append(unasked, err.Error())
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var banned []string
|
||||||
|
for _, ip := range ipLike.FindAllString(string(answer), -1) {
|
||||||
|
if whose, ours := owned[ip]; ours && !slices.Contains(banned, ip+" ("+whose+")") {
|
||||||
|
banned = append(banned, ip+" ("+whose+")")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(banned) > 0 {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Token: "bans-the-mesh",
|
||||||
|
Machine: node, Severity: conditions.Urgent,
|
||||||
|
Summary: fmt.Sprintf("%s's ban list holds %d of the mesh's own address(es): %s — the mesh is locked out "+
|
||||||
|
"of itself there (ADR 0186)", node, len(banned), strings.Join(banned, ", ")),
|
||||||
|
Said: strings.Join(banned, ", ")})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(unasked) > 0 {
|
||||||
|
return nil, fmt.Errorf("a ban list could not be read: %s", strings.Join(unasked, "; "))
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeStatus is D9: `status` answers in full within ten seconds, from a summary composed lately.
|
||||||
|
func probeStatus(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
started := time.Now()
|
||||||
|
stale := ""
|
||||||
|
if statusFrom != nil {
|
||||||
|
answer, err := statusFrom.answer(ctx)
|
||||||
|
if err != nil {
|
||||||
|
stale = err.Error()
|
||||||
|
} else if m, ok := answer.(map[string]any); ok {
|
||||||
|
if failed, _ := m["lastAttemptFailed"].(string); failed != "" {
|
||||||
|
stale = "its last composition failed: " + failed
|
||||||
|
}
|
||||||
|
if composed, err := time.Parse(time.RFC3339, fmt.Sprint(m["composed"])); err == nil &&
|
||||||
|
time.Since(composed) > 3*statusEvery+statusComposeWithin {
|
||||||
|
stale = fmt.Sprintf("it answers a summary composed %s ago", time.Since(composed).Round(time.Second))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if _, err := theThreeQuestions(ctx, d.open); err != nil {
|
||||||
|
stale = err.Error()
|
||||||
|
}
|
||||||
|
took := time.Since(started)
|
||||||
|
var out []conditions.Observation
|
||||||
|
if took > 10*time.Second || stale != "" {
|
||||||
|
why := stale
|
||||||
|
if why == "" {
|
||||||
|
why = fmt.Sprintf("it took %s", took.Round(time.Millisecond))
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: "controller", Token: "status-slow",
|
||||||
|
Machine: d.host, Severity: conditions.Warning,
|
||||||
|
Summary: "status does not answer in full within ten seconds: " + why, Said: why})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeCoreBuilds is D10: every machine runs the node-engine and the node tools the mesh holds, or a
|
||||||
|
// plan is rolling one of them out. The node-engine says its build in each report; the node tools' is
|
||||||
|
// the build the machine was last sent, once it reported applying that send.
|
||||||
|
func probeCoreBuilds(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
inv := d.open.inventory
|
||||||
|
current, err := inv.CurrentBuilds(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
plans, err := inv.OpenPlans(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
hostVersions := deliveredVersions(shelf[hostModule])
|
||||||
|
rolling := map[string]bool{}
|
||||||
|
for _, p := range plans {
|
||||||
|
for m := range p.Modules {
|
||||||
|
rolling[m] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
nodes, err := inv.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
reports, err := inv.LastReports(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
applied := map[string]bool{}
|
||||||
|
for _, r := range reports {
|
||||||
|
applied[r.Node] = r.Current
|
||||||
|
}
|
||||||
|
heard := heardMachines(d)
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, n := range nodes {
|
||||||
|
if !heard[n.Name] {
|
||||||
|
continue // a machine not heard from is S1's
|
||||||
|
}
|
||||||
|
var behind []string
|
||||||
|
// **A node-engine says its version as the directory it was delivered into** — its archive's
|
||||||
|
// digest, twelve characters (catalogue.versionOf, ADR 0141) — not the commit it was built
|
||||||
|
// from. Compared as a commit, every machine read as behind right after a push sent it the
|
||||||
|
// current one (2026-10-06). An engine placed by hand reports its link-time stamp instead,
|
||||||
|
// which a commit can match.
|
||||||
|
if !rolling[hostModule] && engineBehind(n.HostVersion, hostVersions, current[hostModule].Commit) {
|
||||||
|
behind = append(behind, fmt.Sprintf("the node-engine %s, the mesh holds %s (built from %s)",
|
||||||
|
n.HostVersion, strings.Join(hostVersions, " or "), short(current[hostModule].Commit)))
|
||||||
|
}
|
||||||
|
assigned, err := inv.Assigned(ctx, n.Name)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
tools := current[broker.RuntimeModule].Commit
|
||||||
|
if tools != "" && slices.Contains(assigned, broker.RuntimeModule) && !rolling[broker.RuntimeModule] {
|
||||||
|
sent, known, err := inv.SentBuilds(ctx, n.Name)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case !known:
|
||||||
|
case !sameCommit(sent[broker.RuntimeModule], tools):
|
||||||
|
behind = append(behind, fmt.Sprintf("the node tools %s, the mesh holds %s",
|
||||||
|
short(orNotKnown(sent[broker.RuntimeModule])), short(tools)))
|
||||||
|
case !applied[n.Name]:
|
||||||
|
behind = append(behind, "the node tools it was last sent, not yet reported applied")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(behind) > 0 {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "core-behind",
|
||||||
|
Machine: n.Name, Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s runs %s, and no plan is rolling them out", n.Name, strings.Join(behind, "; ")),
|
||||||
|
Said: strings.Join(behind, "; ")})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// deliveredVersions are the versions a module's registered build is delivered as: the last element
|
||||||
|
// of every resource path under a `versions/` directory, which registration filled from the artifact's
|
||||||
|
// digest (catalogue `${version}`). The node-engine names itself by that directory.
|
||||||
|
func deliveredVersions(m catalogue.Manifest) []string {
|
||||||
|
var out []string
|
||||||
|
for _, r := range m.Resources {
|
||||||
|
path, _ := r["path"].(string)
|
||||||
|
before, version, found := strings.Cut(path, "/versions/")
|
||||||
|
if !found || before == "" || version == "" || strings.Contains(version, "/") || strings.Contains(version, "$") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !slices.Contains(out, version) {
|
||||||
|
out = append(out, version)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// engineBehind says a node-engine's reported version is not the build the mesh holds: neither the
|
||||||
|
// directory that build is delivered as, nor (for an engine placed by hand) its commit. A machine that
|
||||||
|
// has not said, or a mesh that holds no delivered build, is not behind anything.
|
||||||
|
func engineBehind(reported string, delivered []string, commit string) bool {
|
||||||
|
if reported == "" || len(delivered) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return !slices.Contains(delivered, reported) && !sameCommit(reported, commit)
|
||||||
|
}
|
||||||
|
|
||||||
|
// hostModule is the node-engine's module.
|
||||||
|
const hostModule = "mesh-host"
|
||||||
|
|
||||||
|
// sameCommit says two commits are one, either written short.
|
||||||
|
func sameCommit(a, b string) bool {
|
||||||
|
if a == "" || b == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return strings.HasPrefix(a, b) || strings.HasPrefix(b, a)
|
||||||
|
}
|
||||||
|
|
||||||
|
func orNotKnown(s string) string {
|
||||||
|
if s == "" {
|
||||||
|
return "(not known)"
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeWatchdogs is DW: the watchdogs ran within three of their intervals. The doctor and the
|
||||||
|
// watchdogs watch each other: S10 is the other half.
|
||||||
|
func probeWatchdogs(_ context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
if d.watchdogs == nil {
|
||||||
|
return nil, nil // a self-check run outside the serving controller has none to watch
|
||||||
|
}
|
||||||
|
ticked := d.watchdogs.lastTick()
|
||||||
|
since := ticked
|
||||||
|
if since.IsZero() {
|
||||||
|
since = d.watchdogs.started
|
||||||
|
}
|
||||||
|
if time.Since(since) <= 3*watchEvery {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return []conditions.Observation{{Scope: conditions.ScopeCore, ID: "watchdogs", Token: "silent",
|
||||||
|
Machine: d.host, Severity: conditions.Urgent,
|
||||||
|
Summary: fmt.Sprintf("the watchdogs of the signals table have not run since %s: no late signal is being said",
|
||||||
|
since.UTC().Format("2006-01-02 15:04 MST")),
|
||||||
|
Said: fmt.Sprintf("no tick for %s", time.Since(since).Round(time.Second))}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// askSeatTool asks one machine's holder of a node seat a verb and answers its result; the holder's
|
||||||
|
// own refusal is an error.
|
||||||
|
func askSeatTool(ctx context.Context, conn *nats.Conn, seat, verb, node string) (json.RawMessage, error) {
|
||||||
|
if who, declared := declaredBy(ctx, seat, verb); !declared {
|
||||||
|
return nil, fmt.Errorf("%s asks %s.%s, which it does not declare in the probe registry — and so the "+
|
||||||
|
"controller is not granted it", who, seat, verb)
|
||||||
|
}
|
||||||
|
answer, err := link.AskSeatTool(ctx, conn, seat, verb, node, map[string]any{}, 10*time.Second)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if answer.Error != "" {
|
||||||
|
return nil, fmt.Errorf("%s's %s.%s refused: %s", node, seat, verb, answer.Error)
|
||||||
|
}
|
||||||
|
return answer.Result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// oneLine is a message of several lines said on one, its runs of space made one.
|
||||||
|
func oneLine(s string) string { return strings.Join(strings.Fields(s), " ") }
|
||||||
+74
-42
@@ -8,6 +8,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
"io"
|
"io"
|
||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
@@ -105,7 +106,10 @@ func serve(ctx context.Context) error {
|
|||||||
|
|
||||||
work := link.Enrolment{Inventory: inv, Identity: ident, Broker: known,
|
work := link.Enrolment{Inventory: inv, Identity: ident, Broker: known,
|
||||||
OnNATS: true}
|
OnNATS: true}
|
||||||
server, err := connectLink(ctx, inv, work, work)
|
// `status` from a summary kept current here (novox/hq to-be 45 Phase 0): a machine saying
|
||||||
|
// something new is one thing that moves it, so the listener nudges it.
|
||||||
|
statusFrom = newStatusSummary(composeStatus(open))
|
||||||
|
server, err := connectLink(ctx, inv, work, nudgingListener{work, statusFrom})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -121,6 +125,8 @@ func serve(ctx context.Context) error {
|
|||||||
// Open plans move on a timer as well as on outcomes (novox/hq ADR 0162): a tier waiting for
|
// Open plans move on a timer as well as on outcomes (novox/hq ADR 0162): a tier waiting for
|
||||||
// machines to report moves when they have, and a plan left by a replaced controller resumes.
|
// machines to report moves when they have, and a plan left by a replaced controller resumes.
|
||||||
go planTicker(ctx, open)
|
go planTicker(ctx, open)
|
||||||
|
// The durations the core's bounds are set from are kept a month (novox/hq to-be 45 Phase 0).
|
||||||
|
go forgettingOldDurations(ctx, inv)
|
||||||
// And what the catalogue decided a build meant. The builder's own result is already handled
|
// And what the catalogue decided a build meant. The builder's own result is already handled
|
||||||
// above; this is the other half — the control plane is the only one of the three that knows
|
// above; this is the other half — the control plane is the only one of the three that knows
|
||||||
// which machines run the thing, so it is the one that acts (novox/hq ADR 0072).
|
// which machines run the thing, so it is the one that acts (novox/hq ADR 0072).
|
||||||
@@ -137,7 +143,7 @@ func serve(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
// And what providers say about consumers they keep failing, kept for `status` (novox/hq ADR
|
// And what providers say about consumers they keep failing, kept for `status` (novox/hq ADR
|
||||||
// 0224): a provider's journal must not be the only place that says so.
|
// 0224): a provider's journal must not be the only place that says so.
|
||||||
if err := server.Watches(standings{inv}); err != nil {
|
if err := server.Watches(standings{keeper: func() *conditions.Keeper { return conditionsFrom }}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -158,8 +164,28 @@ func serve(ctx context.Context) error {
|
|||||||
if !isNATS {
|
if !isNATS {
|
||||||
return errors.New("the mesh's verbs are served over the bus, and this control plane is not on it")
|
return errors.New("the mesh's verbs are served over the bus, and this control plane is not on it")
|
||||||
}
|
}
|
||||||
|
// The hand-act log is counted for `status` on this connection rather than a new one a minute.
|
||||||
|
handActConn = bus.Conn
|
||||||
|
// Composed now and kept current, before the verb that answers from it is served.
|
||||||
|
go statusFrom.keep(ctx)
|
||||||
// A call that outlasts its caller's patience is followed by `calls` (novox/hq issue 265).
|
// A call that outlasts its caller's patience is followed by `calls` (novox/hq issue 265).
|
||||||
link.Calls.Follow = catalogue.ControllerSeatName + ".calls"
|
link.Calls.Follow = catalogue.ControllerSeatName + ".calls"
|
||||||
|
// And every call is kept on the bus, so a restart of this process keeps what came of each
|
||||||
|
// (novox/hq to-be 45 §6). A bus without the bucket is said and served from memory, as before:
|
||||||
|
// answering no calls at all would be worse than answering them without the record.
|
||||||
|
said := log.New(os.Stdout, "", log.LstdFlags)
|
||||||
|
if keeper, err := link.CallsOnTheBus(ctx, bus.Conn); err != nil {
|
||||||
|
fmt.Printf("calls are kept in memory only, and lost when this controller stops: %v\n", err)
|
||||||
|
} else if err := link.Calls.Durably(ctx, keeper, controllerProcess(), said); err != nil {
|
||||||
|
fmt.Printf("calls are kept on the bus from now on; the ones kept before could not be read: %v\n", err)
|
||||||
|
}
|
||||||
|
// What is wrong, kept and said (novox/hq to-be 45 §2): the condition store, the watchdogs of the
|
||||||
|
// signals table, what the bus says about itself, and the self-check. A store that cannot be opened
|
||||||
|
// is said and the controller serves on: status then says the conditions cannot be read, and is
|
||||||
|
// not well — louder than not serving at all, and the push that repairs the bus still runs.
|
||||||
|
if stopWatching := watchTheMesh(ctx, open, server, bus); stopWatching != nil {
|
||||||
|
defer stopWatching()
|
||||||
|
}
|
||||||
stopServing, err := bus.ServeSeatTools(catalogue.ControllerSeatName, handlers, log.New(os.Stdout, "", log.LstdFlags))
|
stopServing, err := bus.ServeSeatTools(catalogue.ControllerSeatName, handlers, log.New(os.Stdout, "", log.LstdFlags))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -260,6 +286,9 @@ func pushCommand(ctx context.Context, args []string) error {
|
|||||||
// (novox/hq ADR 0010). 0 waits for nothing, which is the old fire-and-forget.
|
// (novox/hq ADR 0010). 0 waits for nothing, which is the old fire-and-forget.
|
||||||
wait := set.Duration("wait", 0,
|
wait := set.Duration("wait", 0,
|
||||||
"for a named node, how long to wait for it to report applying what it was sent (0: do not wait)")
|
"for a named node, how long to wait for it to report applying what it was sent (0: do not wait)")
|
||||||
|
// A push by hand is a repair, and says why (novox/hq to-be 45 §7): required through the seat,
|
||||||
|
// recorded when given at a shell — see handacts.go for why a shell is not refused.
|
||||||
|
why := addHandActFlags(set)
|
||||||
positionals, err := parseAround(set, args)
|
positionals, err := parseAround(set, args)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -274,6 +303,11 @@ func pushCommand(ctx context.Context, args []string) error {
|
|||||||
return errors.New("push <node> or push --behind, not both: one names a machine and the " +
|
return errors.New("push <node> or push --behind, not both: one names a machine and the " +
|
||||||
"other asks which machines need one")
|
"other asks which machines need one")
|
||||||
}
|
}
|
||||||
|
recorded := append([]string(nil), args...)
|
||||||
|
if *behind {
|
||||||
|
recorded = append(recorded, "--behind")
|
||||||
|
}
|
||||||
|
why.record(ctx, "push", recorded)
|
||||||
open, err := openStores(ctx)
|
open, err := openStores(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -1039,6 +1073,20 @@ func digestOf(body []byte) string {
|
|||||||
// be worked out" is a different problem with a different remedy, and `plan` is where it is said.
|
// be worked out" is a different problem with a different remedy, and `plan` is where it is said.
|
||||||
func wouldSend(ctx context.Context, open *stores,
|
func wouldSend(ctx context.Context, open *stores,
|
||||||
nodes []inventory.Node) (map[string]string, error) {
|
nodes []inventory.Node) (map[string]string, error) {
|
||||||
|
return wouldSendFrom(ctx, open, nodes, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// planned is one machine's plan as planFor answered it, for a caller that already asked.
|
||||||
|
type planned struct {
|
||||||
|
plan catalogue.Resolution
|
||||||
|
settings catalogue.SettingsBy
|
||||||
|
}
|
||||||
|
|
||||||
|
// wouldSendFrom is wouldSend reusing the plans a caller worked out a moment before: resolving a
|
||||||
|
// machine is most of what `status` costs, and it used to resolve every machine twice (novox/hq
|
||||||
|
// to-be 45 Phase 0). A machine absent from plans is worked out here.
|
||||||
|
func wouldSendFrom(ctx context.Context, open *stores,
|
||||||
|
nodes []inventory.Node, plans map[string]planned) (map[string]string, error) {
|
||||||
|
|
||||||
gens, err := generators(ctx, open)
|
gens, err := generators(ctx, open)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1046,9 +1094,12 @@ func wouldSend(ctx context.Context, open *stores,
|
|||||||
}
|
}
|
||||||
out := map[string]string{}
|
out := map[string]string{}
|
||||||
for _, n := range nodes {
|
for _, n := range nodes {
|
||||||
plan, settings, err := planFor(ctx, open, n.Name)
|
known, have := plans[n.Name]
|
||||||
if err != nil {
|
plan, settings := known.plan, known.settings
|
||||||
continue
|
if !have {
|
||||||
|
if plan, settings, err = planFor(ctx, open, n.Name); err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
}
|
}
|
||||||
declared, err := declarationWith(ctx, open, n.Name, plan, settings, gens, Reading)
|
declared, err := declarationWith(ctx, open, n.Name, plan, settings, gens, Reading)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1107,35 +1158,22 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
nodes, err := inv.Nodes(ctx)
|
// The mesh's own streams and consumers, the seats' work queues and their workers, and how every
|
||||||
|
// machine hears its declaration — one derivation, which the self-check reads as well (D6, D7).
|
||||||
|
names, err := assertBusObjects(ctx, inv, js)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
names := make([]string, 0, len(nodes))
|
|
||||||
for _, n := range nodes {
|
|
||||||
names = append(names, n.Name)
|
|
||||||
}
|
|
||||||
if err := broker.Raise(js, names); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// The work queues of the mesh's own roles (novox/hq ADR 0121). The queue before the holder,
|
|
||||||
// deliberately: work queues until somebody arrives to do it, so assigning a build machine a week
|
|
||||||
// after something started asking for builds flushes the backlog instead of having lost it.
|
|
||||||
// With the seats' holders, so each role's work queue gets the consumer its holder takes
|
|
||||||
// work from. Passed as nil until the first live raise, which left the build machine bound to a
|
|
||||||
// consumer nothing had created (2026-09-28).
|
|
||||||
holders, err := seatHolders(ctx, inv)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := broker.RaiseSeats(js, inventory.MeshSeats(), holders); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// And each work queue's cancelled set (novox/hq ADR 0219), so a holder taking an ask can ask
|
// And each work queue's cancelled set (novox/hq ADR 0219), so a holder taking an ask can ask
|
||||||
// whether it was cancelled the moment it took it.
|
// whether it was cancelled the moment it took it.
|
||||||
if err := broker.RaiseCancelledSets(js, inventory.MeshSeats()); err != nil {
|
if err := broker.RaiseCancelledSets(js, inventory.MeshSeats()); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
// And the controller's own buckets (novox/hq to-be 45 §1): the calls it serves and the acts done
|
||||||
|
// by hand, kept where a restart of this process does not take them.
|
||||||
|
if err := js.EnsureControllerBuckets(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
// Every module's state (novox/hq ADR 0201), from the catalogue: a bucket exists from
|
// Every module's state (novox/hq ADR 0201), from the catalogue: a bucket exists from
|
||||||
// registration, so a module reading one may watch it before its owner runs anywhere. One that
|
// registration, so a module reading one may watch it before its owner runs anywhere. One that
|
||||||
// nothing declares any more is said and kept — what it holds is data.
|
// nothing declares any more is said and kept — what it holds is data.
|
||||||
@@ -1151,25 +1189,11 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string)
|
|||||||
fmt.Printf("the bus holds state nothing declares any more, kept because it is data: %s — "+
|
fmt.Printf("the bus holds state nothing declares any more, kept because it is data: %s — "+
|
||||||
"removing it is a person's act\n", strings.Join(undeclared, ", "))
|
"removing it is a person's act\n", strings.Join(undeclared, ", "))
|
||||||
}
|
}
|
||||||
// And how every module hears what it consumes. Derived from the same records the user list is
|
// And how every module hears what it consumes: asserted with the rest above, counted here.
|
||||||
// composed from, so a module the mesh grants a consumer's subjects has that consumer waiting.
|
hearing, err := moduleConsumerCount(ctx, inv)
|
||||||
// Done on every raise, not only when a credential is issued: every module moved onto this bus
|
|
||||||
// by the rollout was issued on the old one, and came up with nothing to bind to (2026-09-28).
|
|
||||||
records, err := inv.BusRecords(ctx)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
users, err := broker.Users(records)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
hearing := 0
|
|
||||||
for _, c := range broker.ConsumersOf(users) {
|
|
||||||
if err := js.EnsureConsumer(c.Consumer); err != nil {
|
|
||||||
return fmt.Errorf("how %s on %s hears what it consumes: %w", c.Module, c.Node, err)
|
|
||||||
}
|
|
||||||
hearing++
|
|
||||||
}
|
|
||||||
fmt.Printf("the bus at %s has its streams, %d machine(s) can hear a declaration, %d module(s) "+
|
fmt.Printf("the bus at %s has its streams, %d machine(s) can hear a declaration, %d module(s) "+
|
||||||
"can hear what they consume, and %d bucket(s) of state\n", broker.BareAddress(address), len(names), hearing, len(buckets))
|
"can hear what they consume, and %d bucket(s) of state\n", broker.BareAddress(address), len(names), hearing, len(buckets))
|
||||||
return nil
|
return nil
|
||||||
@@ -1271,3 +1295,11 @@ func reportUnheldPushed(w io.Writer, named bool, asked []string, unheld map[stri
|
|||||||
fmt.Fprintf(w, "%s: %d unmet seat dependenc(ies) — see `status`\n", node, len(lines))
|
fmt.Fprintf(w, "%s: %d unmet seat dependenc(ies) — see `status`\n", node, len(lines))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// controllerProcess names this serving process among controllers: the machine, the process and when
|
||||||
|
// it started — what a call kept on the bus carries, so the next controller can tell a call this one
|
||||||
|
// left running from one it is running itself (novox/hq to-be 45 §6).
|
||||||
|
func controllerProcess() string {
|
||||||
|
host, _ := os.Hostname()
|
||||||
|
return fmt.Sprintf("controller@%s pid %d since %s", host, os.Getpid(), time.Now().UTC().Format(time.RFC3339))
|
||||||
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"sort"
|
"sort"
|
||||||
"time"
|
"time"
|
||||||
@@ -78,11 +79,24 @@ type meshStatus struct {
|
|||||||
// that machine holds, with the modules that could hold it (novox/hq ADR 0207). Absent when every
|
// that machine holds, with the modules that could hold it (novox/hq ADR 0207). Absent when every
|
||||||
// dependency is met. Reported, not refused, until the switch.
|
// dependency is met. Reported, not refused, until the switch.
|
||||||
Unheld []catalogue.Unheld `json:"unheld,omitempty"`
|
Unheld []catalogue.Unheld `json:"unheld,omitempty"`
|
||||||
// Failing is every consumer a provider says it keeps failing, with the class of error, since
|
// HandActsThisWeek is how many acts were done by hand in the last seven days (novox/hq to-be 45
|
||||||
// when, and when it was last said (novox/hq ADR 0224). Absent when no provider says so. A
|
// §7): every one is a repair a healer could have made. Absent where the log is not on hand;
|
||||||
// document without this called the mesh well while the identity provider refused every consumer
|
// HandActsUnread says why when it could not be read, rather than reading as none.
|
||||||
// for a day (04-ISSUES/179).
|
HandActsThisWeek *int `json:"handActsThisWeek,omitempty"`
|
||||||
Failing []inventory.ProviderStanding `json:"failing,omitempty"`
|
HandActsUnread string `json:"handActsUnread,omitempty"`
|
||||||
|
// Conditions is every open condition, urgent first and then oldest first (novox/hq to-be 45 §2):
|
||||||
|
// what is wrong, as the watchdogs, the self-check and the providers say it. Always present — an
|
||||||
|
// empty list is "none open" — unless they could not be read, which ConditionsUnread says.
|
||||||
|
Conditions []conditions.Condition `json:"conditions"`
|
||||||
|
ConditionsUnread string `json:"conditionsUnread,omitempty"`
|
||||||
|
// Failing is every consumer a provider says it keeps failing (novox/hq ADR 0224): the open
|
||||||
|
// conditions of that kind, carried here as well because ADR 0224 names this field. Absent when no
|
||||||
|
// provider says so. A document without it called the mesh well while the identity provider
|
||||||
|
// refused every consumer for a day (04-ISSUES/179).
|
||||||
|
Failing []conditions.Condition `json:"failing,omitempty"`
|
||||||
|
// Overflowing is every module whose identity overflows the bound of a provision it requires, and
|
||||||
|
// so is left out of its provider's grants (novox/hq ADR 0225). Absent when every identity fits.
|
||||||
|
Overflowing []catalogue.Overflow `json:"overflowing,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// machineFiltered is one rule set on a converged machine that the mesh did not write and that
|
// machineFiltered is one rule set on a converged machine that the mesh did not write and that
|
||||||
@@ -215,7 +229,13 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
out.Unheld = asked.unheld
|
out.Unheld = asked.unheld
|
||||||
out.Failing = asked.failing
|
out.HandActsThisWeek, out.HandActsUnread = asked.handActs, asked.handActsUnread
|
||||||
|
out.Conditions, out.ConditionsUnread = asked.conditions, asked.conditionsUnread
|
||||||
|
if out.Conditions == nil {
|
||||||
|
out.Conditions = []conditions.Condition{}
|
||||||
|
}
|
||||||
|
out.Failing = providerStandings(asked.conditions)
|
||||||
|
out.Overflowing = asked.overflowing
|
||||||
for name := range asked.refused {
|
for name := range asked.refused {
|
||||||
out.Unresolved = append(out.Unresolved, machineUnresolved{
|
out.Unresolved = append(out.Unresolved, machineUnresolved{
|
||||||
Node: name, Problem: asked.refused[name]})
|
Node: name, Problem: asked.refused[name]})
|
||||||
|
|||||||
@@ -986,10 +986,18 @@ func plansCommand(ctx context.Context, args []string) error {
|
|||||||
whatIf := set.String("what-if", "", "owner/repository: the plan a merge there would produce, saving nothing — with --paths or --modules")
|
whatIf := set.String("what-if", "", "owner/repository: the plan a merge there would produce, saving nothing — with --paths or --modules")
|
||||||
paths := set.String("paths", "", "the files the merge would change, comma-separated, from the repository's root")
|
paths := set.String("paths", "", "the files the merge would change, comma-separated, from the repository's root")
|
||||||
modules := set.String("modules", "", "or the modules it would change, comma-separated")
|
modules := set.String("modules", "", "or the modules it would change, comma-separated")
|
||||||
|
// Ending a plan by hand is a repair, and says why (novox/hq to-be 45 §7).
|
||||||
|
why := addHandActFlags(set)
|
||||||
positionals, err := parseAround(set, args)
|
positionals, err := parseAround(set, args)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if len(positionals) == 2 && (positionals[0] == "stop" || positionals[0] == "close") {
|
||||||
|
// Refused before anything is opened: a repair by hand says why.
|
||||||
|
if err := why.require("plans " + positionals[0]); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
open, err := openStores(ctx)
|
open, err := openStores(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -1061,8 +1069,9 @@ func plansCommand(ctx context.Context, args []string) error {
|
|||||||
if !p.Open() {
|
if !p.Open() {
|
||||||
return fmt.Errorf("%s is already %s", p.ID, p.State)
|
return fmt.Errorf("%s is already %s", p.ID, p.State)
|
||||||
}
|
}
|
||||||
|
why.record(ctx, "plans "+positionals[0], positionals[1:])
|
||||||
p.State = inventory.PlanFailed
|
p.State = inventory.PlanFailed
|
||||||
p.Note = how + " by hand at tier " + fmt.Sprint(p.Tier)
|
p.Note = how + " by hand at tier " + fmt.Sprint(p.Tier) + ": " + strings.TrimSpace(*why.why)
|
||||||
sayUnsent(&p, func(m string) bool {
|
sayUnsent(&p, func(m string) bool {
|
||||||
u, err := inv.UpgradeOf(ctx, m)
|
u, err := inv.UpgradeOf(ctx, m)
|
||||||
return err == nil && u.RollOut
|
return err == nil && u.RollOut
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ import (
|
|||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"sort"
|
"sort"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
)
|
)
|
||||||
|
|
||||||
// rotateCommand replaces a credential and moves both ends together.
|
// rotateCommand replaces a credential and moves both ends together.
|
||||||
@@ -33,12 +35,16 @@ func rotateCommand(ctx context.Context, args []string) error {
|
|||||||
// One consumer rather than all of them. Ordinary: a credential is suspected on one machine,
|
// One consumer rather than all of them. Ordinary: a credential is suspected on one machine,
|
||||||
// and rotating the other nine would be a great deal of disruption for one suspicion.
|
// and rotating the other nine would be a great deal of disruption for one suspicion.
|
||||||
only := set.String("consumer", "", "only this machine's credential, rather than every holder's")
|
only := set.String("consumer", "", "only this machine's credential, rather than every holder's")
|
||||||
|
// One consuming module rather than every module on the machine. A machine runs many consumers
|
||||||
|
// of one provision, each with its own credential; one module that leaked its credential (novox/hq
|
||||||
|
// issue 268) is no reason to restart every other one on the machine.
|
||||||
|
module := set.String("module", "", "only this consuming module's credential")
|
||||||
positionals, err := parseAround(set, args)
|
positionals, err := parseAround(set, args)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if len(positionals) != 1 {
|
if len(positionals) != 1 {
|
||||||
return errors.New("rotate <provision> [--consumer <machine>]")
|
return errors.New("rotate <provision> [--consumer <machine>] [--module <module>]")
|
||||||
}
|
}
|
||||||
provision := positionals[0]
|
provision := positionals[0]
|
||||||
|
|
||||||
@@ -53,6 +59,12 @@ func rotateCommand(ctx context.Context, args []string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
holders = ofModule(holders, *module)
|
||||||
|
if len(holders) == 0 && *module != "" {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"no module %s%s holds a credential for %q, so there is nothing to rotate. `plan <machine>` "+
|
||||||
|
"says what a machine holds", *module, onMachine(*only), provision)
|
||||||
|
}
|
||||||
if len(holders) == 0 {
|
if len(holders) == 0 {
|
||||||
// Said, not silent. "Nobody holds this" and "this did not run" must never look the same —
|
// Said, not silent. "Nobody holds this" and "this did not run" must never look the same —
|
||||||
// and a rotation somebody believes happened is worse than one they know did not.
|
// and a rotation somebody believes happened is worse than one they know did not.
|
||||||
@@ -116,6 +128,27 @@ func rotateCommand(ctx context.Context, args []string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ofModule is the holders whose consuming module is this one; all of them when none is named.
|
||||||
|
func ofModule(holders []inventory.Holder, module string) []inventory.Holder {
|
||||||
|
if module == "" {
|
||||||
|
return holders
|
||||||
|
}
|
||||||
|
var out []inventory.Holder
|
||||||
|
for _, h := range holders {
|
||||||
|
if h.ConsumerModule == module {
|
||||||
|
out = append(out, h)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func onMachine(machine string) string {
|
||||||
|
if machine == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return " on " + machine
|
||||||
|
}
|
||||||
|
|
||||||
// asLocal names the credential inside the consumer where it holds several (ADR 0094).
|
// asLocal names the credential inside the consumer where it holds several (ADR 0094).
|
||||||
func asLocal(local string) string {
|
func asLocal(local string) string {
|
||||||
if local == "" {
|
if local == "" {
|
||||||
|
|||||||
@@ -0,0 +1,27 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// One consuming module's credential, and not its neighbours' on the same machine (novox/hq issue
|
||||||
|
// 268): a module that leaked its database password is no reason to restart every other consumer.
|
||||||
|
func TestARotationNarrowedToAModuleTouchesOnlyThatModulesCredential(t *testing.T) {
|
||||||
|
holders := []inventory.Holder{
|
||||||
|
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "letta", Provider: "ace"},
|
||||||
|
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "n8n", Provider: "ace"},
|
||||||
|
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "letta", Local: "reader", Provider: "ace"},
|
||||||
|
}
|
||||||
|
got := ofModule(holders, "letta")
|
||||||
|
if len(got) != 2 || got[0].ConsumerModule != "letta" || got[1].Local != "reader" {
|
||||||
|
t.Fatalf("narrowed to letta: %+v", got)
|
||||||
|
}
|
||||||
|
if len(ofModule(holders, "")) != 3 {
|
||||||
|
t.Fatal("no module named narrowed anyway")
|
||||||
|
}
|
||||||
|
if len(ofModule(holders, "absent")) != 0 {
|
||||||
|
t.Fatal("a module holding nothing matched")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -9,9 +9,11 @@ import (
|
|||||||
"github.com/nats-io/nats.go/micro"
|
"github.com/nats-io/nats.go/micro"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
)
|
)
|
||||||
@@ -239,6 +241,12 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
|||||||
if len(argv) == 0 {
|
if len(argv) == 0 {
|
||||||
return nil, errors.New("command names no command")
|
return nil, errors.New("command names no command")
|
||||||
}
|
}
|
||||||
|
// The generic verb is no way round the hand-act log (novox/hq to-be 45 §7): a repair through
|
||||||
|
// it says why, as it would through its own verb.
|
||||||
|
if repair := repairingCommand(argv); repair != "" && !slices.ContainsFunc(argv, isWhyFlag) {
|
||||||
|
return nil, fmt.Errorf("%s is a repair done by hand, and says why: add --why <text> to the command "+
|
||||||
|
"line (recorded in the hand-act log). Nothing was done", repair)
|
||||||
|
}
|
||||||
return argv, nil
|
return argv, nil
|
||||||
case "tools":
|
case "tools":
|
||||||
return nil, errors.New("tools is answered from the records, not by a command")
|
return nil, errors.New("tools is answered from the records, not by a command")
|
||||||
@@ -280,7 +288,18 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
|||||||
}
|
}
|
||||||
for _, act := range []string{"stop", "close", "retry"} {
|
for _, act := range []string{"stop", "close", "retry"} {
|
||||||
if id := str(act); id != "" {
|
if id := str(act); id != "" {
|
||||||
return []string{"plans", act, id}, nil
|
argv := []string{"plans", act, id}
|
||||||
|
if act == "retry" {
|
||||||
|
return argv, nil
|
||||||
|
}
|
||||||
|
// Ending a plan by hand says why (novox/hq to-be 45 §7); the command refuses it without.
|
||||||
|
if w := str("why"); w != "" {
|
||||||
|
argv = append(argv, "--why", w)
|
||||||
|
}
|
||||||
|
if c := str("cause"); c != "" {
|
||||||
|
argv = append(argv, "--cause", c)
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if id := str("id"); id != "" {
|
if id := str("id"); id != "" {
|
||||||
@@ -355,22 +374,105 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
|||||||
// Sent and not waited for: the asker reads `status` for what the machine did, which is
|
// Sent and not waited for: the asker reads `status` for what the machine did, which is
|
||||||
// what a person at a shell does too. A tool call that blocked for a push's whole apply would
|
// what a person at a shell does too. A tool call that blocked for a push's whole apply would
|
||||||
// time out on every machine that takes a minute, and say nothing about the ones that did not.
|
// time out on every machine that takes a minute, and say nothing about the ones that did not.
|
||||||
|
// A push through the seat is a push by hand, and says why (novox/hq to-be 45 §7).
|
||||||
|
if err := need("why"); err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: a push by hand is a repair, recorded in the hand-act log with why", err)
|
||||||
|
}
|
||||||
|
why := []string{"--why", str("why")}
|
||||||
|
if c := str("cause"); c != "" {
|
||||||
|
why = append(why, "--cause", c)
|
||||||
|
}
|
||||||
if n := str("node"); n != "" {
|
if n := str("node"); n != "" {
|
||||||
// behind is not read here: given with a machine, it is refused as passed over — naming
|
// behind is not read here: given with a machine, it is refused as passed over — naming
|
||||||
// a machine and asking for every machine behind are two requests, and guessing one
|
// a machine and asking for every machine behind are two requests, and guessing one
|
||||||
// would push a machine nobody named, or not push one somebody did.
|
// would push a machine nobody named, or not push one somebody did.
|
||||||
return []string{"push", n, "--wait", "0"}, nil
|
return append([]string{"push", n, "--wait", "0"}, why...), nil
|
||||||
}
|
}
|
||||||
// No machine: the whole mesh, whether or not behind said so. The command's answer says it
|
// No machine: the whole mesh, whether or not behind said so. The command's answer says it
|
||||||
// first, so a caller who meant one machine reads that it was not one.
|
// first, so a caller who meant one machine reads that it was not one.
|
||||||
on("behind")
|
on("behind")
|
||||||
return []string{"push", "--behind", "--wait", "0"}, nil
|
return append([]string{"push", "--behind", "--wait", "0"}, why...), nil
|
||||||
|
case "hand-act":
|
||||||
|
if err := need("what", "why", "cause"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
argv := []string{"hand-act", "record", str("what"), "--why", str("why"), "--cause", str("cause")}
|
||||||
|
if c := str("condition"); c != "" {
|
||||||
|
argv = append(argv, "--condition", c)
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
|
case "hand-acts":
|
||||||
|
argv := []string{"hand-acts", "--json"}
|
||||||
|
if d := str("days"); d != "" {
|
||||||
|
argv = append(argv, "--days", d)
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
|
case "durations":
|
||||||
|
argv := []string{"durations", "--json"}
|
||||||
|
if k := str("kind"); k != "" {
|
||||||
|
argv = append(argv, "--kind", k)
|
||||||
|
}
|
||||||
|
if d := str("days"); d != "" {
|
||||||
|
argv = append(argv, "--days", d)
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
|
case "conditions":
|
||||||
|
// One verb, four shapes, as `plans` (novox/hq to-be 45 §2): a silence, one condition, the
|
||||||
|
// history, or the open ones filtered.
|
||||||
|
if key := str("silence"); key != "" {
|
||||||
|
if err := need("for", "why"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
argv := []string{"conditions", "silence", key, "--for", str("for"), "--why", str("why")}
|
||||||
|
if c := str("cause"); c != "" {
|
||||||
|
argv = append(argv, "--cause", c)
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
|
}
|
||||||
|
if on("history") {
|
||||||
|
argv := []string{"conditions", "history", "--json"}
|
||||||
|
if d := str("days"); d != "" {
|
||||||
|
argv = append(argv, "--days", d)
|
||||||
|
}
|
||||||
|
if k := str("key"); k != "" {
|
||||||
|
argv = append(argv, "--key", k)
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
|
}
|
||||||
|
if k := str("key"); k != "" {
|
||||||
|
return []string{"conditions", "show", k, "--json"}, nil
|
||||||
|
}
|
||||||
|
argv := []string{"conditions", "--json"}
|
||||||
|
for _, filter := range []string{"scope", "severity", "machine"} {
|
||||||
|
if v := str(filter); v != "" {
|
||||||
|
argv = append(argv, "--"+filter, v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
|
case "doctor":
|
||||||
|
which := 0
|
||||||
|
argv := []string{"doctor"}
|
||||||
|
for _, sub := range []string{"run", "probes", "signals"} {
|
||||||
|
if on(sub) {
|
||||||
|
which++
|
||||||
|
argv = append(argv, sub)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if which > 1 {
|
||||||
|
return nil, errors.New("doctor answers one of run, probes or signals at a time")
|
||||||
|
}
|
||||||
|
return append(argv, "--json"), nil
|
||||||
case "rotate":
|
case "rotate":
|
||||||
if p := str("provision"); p != "" {
|
if p := str("provision"); p != "" {
|
||||||
argv := []string{"rotate", p}
|
argv := []string{"rotate", p}
|
||||||
if c := str("consumer"); c != "" {
|
if c := str("consumer"); c != "" {
|
||||||
argv = append(argv, "--consumer", c)
|
argv = append(argv, "--consumer", c)
|
||||||
}
|
}
|
||||||
|
// With a provision, module narrows to one consuming module (novox/hq issue 268); node
|
||||||
|
// and secret stay the other shape's, and are refused as passed over.
|
||||||
|
if m := str("module"); m != "" {
|
||||||
|
argv = append(argv, "--module", m)
|
||||||
|
}
|
||||||
return argv, nil
|
return argv, nil
|
||||||
}
|
}
|
||||||
_, node := a.given["node"]
|
_, node := a.given["node"]
|
||||||
@@ -437,7 +539,30 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// jsonVerbs are the verbs whose command speaks JSON, so the answer carries it as data as well.
|
// jsonVerbs are the verbs whose command speaks JSON, so the answer carries it as data as well.
|
||||||
var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true}
|
var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true,
|
||||||
|
"hand-acts": true, "durations": true, "conditions": true, "doctor": true}
|
||||||
|
|
||||||
|
// repairingCommand names a command line that repairs by hand, and so says why: a push, a plan stopped
|
||||||
|
// or closed, a consumer re-made (novox/hq to-be 45 §7). Empty for any other.
|
||||||
|
func repairingCommand(argv []string) string {
|
||||||
|
switch {
|
||||||
|
case argv[0] == "push":
|
||||||
|
return "push"
|
||||||
|
case argv[0] == "plans" && len(argv) > 1 && (argv[1] == "stop" || argv[1] == "close"):
|
||||||
|
return "plans " + argv[1]
|
||||||
|
case argv[0] == "broker" && len(argv) > 1 && argv[1] == "consumer-reset":
|
||||||
|
return "broker consumer-reset"
|
||||||
|
case argv[0] == "hand-act":
|
||||||
|
return "hand-act record"
|
||||||
|
case argv[0] == "conditions" && len(argv) > 1 && argv[1] == "silence":
|
||||||
|
return "conditions silence"
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func isWhyFlag(word string) bool {
|
||||||
|
return word == "--why" || word == "-why" || strings.HasPrefix(word, "--why=") || strings.HasPrefix(word, "-why=")
|
||||||
|
}
|
||||||
|
|
||||||
// runVerb runs this binary with the given command line and gathers what it said.
|
// runVerb runs this binary with the given command line and gathers what it said.
|
||||||
func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
||||||
@@ -449,6 +574,12 @@ func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
|||||||
// The same environment: the stores' credentials, the bus, the broker — everything a command run
|
// The same environment: the stores' credentials, the bus, the broker — everything a command run
|
||||||
// from a shell in this container would have, because it is that.
|
// from a shell in this container would have, because it is that.
|
||||||
cmd.Env = os.Environ()
|
cmd.Env = os.Environ()
|
||||||
|
// And who asked, so an act it does by hand is recorded as theirs (novox/hq to-be 45 §7).
|
||||||
|
caller := link.CallerIn(ctx)
|
||||||
|
if caller == "" {
|
||||||
|
caller = "a seat call whose caller the bus did not name"
|
||||||
|
}
|
||||||
|
cmd.Env = append(cmd.Env, link.CallerVar+"="+caller+", through the "+catalogue.ControllerSeatName+" seat")
|
||||||
// Two buffers, one answer. What the command *says* is both streams, in the order a person at
|
// Two buffers, one answer. What the command *says* is both streams, in the order a person at
|
||||||
// a shell would read them; what it *answers as data* is standard output alone — `status --json`
|
// a shell would read them; what it *answers as data* is standard output alone — `status --json`
|
||||||
// prints its warnings beside the document, and a JSON parsed from the two together parsed
|
// prints its warnings beside the document, and a JSON parsed from the two together parsed
|
||||||
@@ -501,6 +632,19 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
|||||||
if verb == "calls" {
|
if verb == "calls" {
|
||||||
return callsAnswer(link.Calls, a.given["call"])
|
return callsAnswer(link.Calls, a.given["call"])
|
||||||
}
|
}
|
||||||
|
if verb == "doctor" {
|
||||||
|
// From the serving controller, which runs the self-check and hears the signals
|
||||||
|
// (novox/hq to-be 45 §4): the last verdict at once, or a run now.
|
||||||
|
argv, err := a.commandLine()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
sub := ""
|
||||||
|
if len(argv) > 2 {
|
||||||
|
sub = argv[1]
|
||||||
|
}
|
||||||
|
return doctorAnswer(ctx, sub)
|
||||||
|
}
|
||||||
return seatTools(), nil
|
return seatTools(), nil
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
@@ -539,6 +683,14 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
if verb == "status" && statusFrom != nil {
|
||||||
|
// At once, from the summary the serving controller keeps (novox/hq to-be 45 Phase 0).
|
||||||
|
return statusFrom.answer(ctx)
|
||||||
|
}
|
||||||
|
if !readingVerbs[verb] && !(verb == "plans" && !actsOnAPlan(args)) {
|
||||||
|
// Whatever it did, `status` is composed again once it has.
|
||||||
|
defer statusFrom.nudge()
|
||||||
|
}
|
||||||
if answersFirst(argv) {
|
if answersFirst(argv) {
|
||||||
// Before anything is sent: a push sends the bus's own machine first, and a broker
|
// Before anything is sent: a push sends the bus's own machine first, and a broker
|
||||||
// reloading its user list forgets the answer it was about to permit (novox/hq issue 265).
|
// reloading its user list forgets the answer it was about to permit (novox/hq issue 265).
|
||||||
@@ -550,9 +702,19 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
|||||||
return handlers, behind, nil
|
return handlers, behind, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// actsOnAPlan is `plans` asked to stop, close or retry one rather than to show them.
|
||||||
|
func actsOnAPlan(args map[string]any) bool {
|
||||||
|
for _, act := range []string{"stop", "close", "retry"} {
|
||||||
|
if v, _ := args[act].(string); strings.TrimSpace(v) != "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// inProcess are the verbs answered by this process rather than by a command it runs: `tools` from
|
// inProcess are the verbs answered by this process rather than by a command it runs: `tools` from
|
||||||
// the records, `calls` from what this process served.
|
// the records, `calls` from what this process served.
|
||||||
var inProcess = map[string]bool{"tools": true, "calls": true}
|
var inProcess = map[string]bool{"tools": true, "calls": true, "doctor": true}
|
||||||
|
|
||||||
// answersFirst is a command line whose caller is answered before it runs: a push, by its verb or
|
// answersFirst is a command line whose caller is answered before it runs: a push, by its verb or
|
||||||
// through `command`. A push sends the machine holding the bus first when its user list changed, the
|
// through `command`. A push sends the machine holding the bus first when its user list changed, the
|
||||||
@@ -563,22 +725,41 @@ func answersFirst(argv []string) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// callsAnswer is what `calls` answers: the kept calls, newest first, without their answers — or
|
// callsAnswer is what `calls` answers: the kept calls, newest first, without their answers — or
|
||||||
// one call whole.
|
// one call whole. Kept on the bus, so a call a controller before this one served is answered too
|
||||||
|
// (novox/hq to-be 45 §6); where the bus cannot be read, what this process served is answered and
|
||||||
|
// the reason said beside it.
|
||||||
func callsAnswer(log *link.CallLog, id string) (any, error) {
|
func callsAnswer(log *link.CallLog, id string) (any, error) {
|
||||||
if id != "" {
|
if id != "" {
|
||||||
c, ok := log.Get(id)
|
c, ok, err := log.Get(id)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("call %s is not in this controller's memory, and the calls kept on the "+
|
||||||
|
"bus could not be read: %w", id, err)
|
||||||
|
}
|
||||||
if !ok {
|
if !ok {
|
||||||
|
if log.IsDurable() {
|
||||||
|
return nil, fmt.Errorf("no call %s is kept: the bus keeps the last %d calls, or %s, and this "+
|
||||||
|
"is not among them — `calls` lists them", id, broker.KeptCallsDurably, broker.CallsKeptFor)
|
||||||
|
}
|
||||||
return nil, fmt.Errorf("no call %s is kept here: calls are kept by the controller that "+
|
return nil, fmt.Errorf("no call %s is kept here: calls are kept by the controller that "+
|
||||||
"answered them, the last %d, and not across a restart — `calls` lists them", id, link.KeptCalls)
|
"answered them, the last %d, and not across a restart — `calls` lists them", id, link.KeptCalls)
|
||||||
}
|
}
|
||||||
return c, nil
|
return c, nil
|
||||||
}
|
}
|
||||||
recent := log.Recent()
|
recent, err := log.Recent()
|
||||||
for i := range recent {
|
for i := range recent {
|
||||||
recent[i].Answer = nil
|
recent[i].Answer = nil
|
||||||
}
|
}
|
||||||
return map[string]any{"calls": recent, "kept": link.KeptCalls,
|
answer := map[string]any{"calls": recent, "note": "newest first; `calls` with a call's id gives its whole answer"}
|
||||||
"note": "newest first; `calls` with a call's id gives its whole answer"}, nil
|
if log.IsDurable() {
|
||||||
|
answer["kept"] = fmt.Sprintf("the last %d calls, or %s, on the bus — across a restart of the controller",
|
||||||
|
broker.KeptCallsDurably, broker.CallsKeptFor)
|
||||||
|
} else {
|
||||||
|
answer["kept"] = fmt.Sprintf("the last %d calls this controller served, in its memory only", link.KeptCalls)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
answer["unread"] = err.Error()
|
||||||
|
}
|
||||||
|
return answer, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// seatTools is what `tools` answers: every seat with a protocol, and the tools each serves, from the
|
// seatTools is what `tools` answers: every seat with a protocol, and the tools each serves, from the
|
||||||
|
|||||||
@@ -145,17 +145,17 @@ func TestAnArgumentAVerbDoesNotDeclareIsRefused(t *testing.T) {
|
|||||||
// The push that was the cause: a machine named is that machine; none named is the whole mesh, and
|
// The push that was the cause: a machine named is that machine; none named is the whole mesh, and
|
||||||
// naming one beside behind is refused rather than one of the two guessed.
|
// naming one beside behind is refused rather than one of the two guessed.
|
||||||
func TestAPushIsOneMachineOrSaysItIsTheWholeMesh(t *testing.T) {
|
func TestAPushIsOneMachineOrSaysItIsTheWholeMesh(t *testing.T) {
|
||||||
argv, err := argvFor("push", map[string]any{"node": "g1"})
|
argv, err := argvFor("push", map[string]any{"node": "g1", "why": "w"})
|
||||||
if err != nil || strings.Join(argv, " ") != "push g1 --wait 0" {
|
if err != nil || strings.Join(argv, " ") != "push g1 --wait 0 --why w" {
|
||||||
t.Fatalf("a named push: %v %v", argv, err)
|
t.Fatalf("a named push: %v %v", argv, err)
|
||||||
}
|
}
|
||||||
for _, args := range []map[string]any{{}, {"behind": "true"}} {
|
for _, args := range []map[string]any{{"why": "w"}, {"behind": "true", "why": "w"}} {
|
||||||
argv, err := argvFor("push", args)
|
argv, err := argvFor("push", args)
|
||||||
if err != nil || strings.Join(argv, " ") != "push --behind --wait 0" {
|
if err != nil || strings.Join(argv, " ") != "push --behind --wait 0 --why w" {
|
||||||
t.Fatalf("a push of the whole mesh %v: %v %v", args, argv, err)
|
t.Fatalf("a push of the whole mesh %v: %v %v", args, argv, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if _, err := argvFor("push", map[string]any{"node": "g1", "behind": "true"}); err == nil ||
|
if _, err := argvFor("push", map[string]any{"node": "g1", "behind": "true", "why": "w"}); err == nil ||
|
||||||
!strings.Contains(err.Error(), `"behind"`) {
|
!strings.Contains(err.Error(), `"behind"`) {
|
||||||
t.Fatalf("a named push with behind was taken: %v", err)
|
t.Fatalf("a named push with behind was taken: %v", err)
|
||||||
}
|
}
|
||||||
@@ -268,6 +268,14 @@ var accountedFlags = map[string]map[string]string{
|
|||||||
},
|
},
|
||||||
"builds": {"n": "=limit"},
|
"builds": {"n": "=limit"},
|
||||||
"plans": {"n": "=limit", "what-if": "=repository"},
|
"plans": {"n": "=limit", "what-if": "=repository"},
|
||||||
|
"durations": {
|
||||||
|
"json": "set by the verb: the answer is data",
|
||||||
|
"all": "withheld: every measurement of a fortnight is more than a call should carry; `command` reaches it",
|
||||||
|
},
|
||||||
|
"hand-acts": {"json": "set by the verb: the answer is data"},
|
||||||
|
"conditions": {"json": "set by the verb: the answer is data"},
|
||||||
|
"conditions history": {"json": "set by the verb: the answer is data"},
|
||||||
|
"conditions show": {"json": "set by the verb: the answer is data"},
|
||||||
}
|
}
|
||||||
|
|
||||||
// **Every flag of the command a verb runs is in the verb's schema, or accounted for here.** Derived
|
// **Every flag of the command a verb runs is in the verb's schema, or accounted for here.** Derived
|
||||||
|
|||||||
@@ -43,6 +43,10 @@ func TestRotateTakesAProvisionOrAnOwnSecret(t *testing.T) {
|
|||||||
if strings.Join(argv, " ") != "rotate postgres-database --consumer ace" {
|
if strings.Join(argv, " ") != "rotate postgres-database --consumer ace" {
|
||||||
t.Fatalf("a pair credential: %v", argv)
|
t.Fatalf("a pair credential: %v", argv)
|
||||||
}
|
}
|
||||||
|
argv, _ = argvFor("rotate", map[string]any{"provision": "postgres-database", "consumer": "ace", "module": "letta"})
|
||||||
|
if strings.Join(argv, " ") != "rotate postgres-database --consumer ace --module letta" {
|
||||||
|
t.Fatalf("one consuming module's pair credential: %v", argv)
|
||||||
|
}
|
||||||
argv, _ = argvFor("rotate", map[string]any{"node": "ace", "module": "nodered", "secret": "api-token"})
|
argv, _ = argvFor("rotate", map[string]any{"node": "ace", "module": "nodered", "secret": "api-token"})
|
||||||
if strings.Join(argv, " ") != "secret rotate ace nodered api-token" {
|
if strings.Join(argv, " ") != "secret rotate ace nodered api-token" {
|
||||||
t.Fatalf("an own secret: %v", argv)
|
t.Fatalf("an own secret: %v", argv)
|
||||||
@@ -103,8 +107,8 @@ func TestAVerbMissingWhatItNeedsIsRefused(t *testing.T) {
|
|||||||
// A push and a build are sent, not waited for: the asker reads status, or the build's log by its
|
// A push and a build are sent, not waited for: the asker reads status, or the build's log by its
|
||||||
// id, for what happened. A repository given as a forge path is said to be one (issue 176).
|
// id, for what happened. A repository given as a forge path is said to be one (issue 176).
|
||||||
func TestActsDoNotBlockTheCall(t *testing.T) {
|
func TestActsDoNotBlockTheCall(t *testing.T) {
|
||||||
argv, _ := argvFor("push", map[string]any{"node": "one"})
|
argv, _ := argvFor("push", map[string]any{"node": "one", "why": "w"})
|
||||||
if strings.Join(argv, " ") != "push one --wait 0" {
|
if strings.Join(argv, " ") != "push one --wait 0 --why w" {
|
||||||
t.Fatalf("push waits: %v", argv)
|
t.Fatalf("push waits: %v", argv)
|
||||||
}
|
}
|
||||||
argv, _ = argvFor("build", map[string]any{"repository": "novox/x", "path": "modules/x"})
|
argv, _ = argvFor("build", map[string]any{"repository": "novox/x", "path": "modules/x"})
|
||||||
|
|||||||
@@ -43,6 +43,9 @@ type sendable struct {
|
|||||||
LeftOut []string
|
LeftOut []string
|
||||||
// leftOutWhy is why each was, for push and plan to say; never on the wire.
|
// leftOutWhy is why each was, for push and plan to say; never on the wire.
|
||||||
leftOutWhy map[string]string
|
leftOutWhy map[string]string
|
||||||
|
// withheld is every consumer this machine's grants leave out, because its identity overflows the
|
||||||
|
// provision's bound (novox/hq ADR 0225); for push and plan to say, never on the wire.
|
||||||
|
withheld []catalogue.Overflow
|
||||||
// Builds is the build of each module this declaration carries — module to the commit its build
|
// Builds is the build of each module this declaration carries — module to the commit its build
|
||||||
// was made from — recorded with the send and never on the wire (novox/hq issue 259, ADR 0221).
|
// was made from — recorded with the send and never on the wire (novox/hq issue 259, ADR 0221).
|
||||||
// Composed only on the send path; nil records that it is not known.
|
// Composed only on the send path; nil records that it is not known.
|
||||||
|
|||||||
@@ -0,0 +1,485 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The signals table (novox/hq to-be 45 §3, ADR 0227 rule 5), compiled in.
|
||||||
|
//
|
||||||
|
// **Every signal the core expects has a watchdog; its absence is a condition.** A row says what is
|
||||||
|
// expected, from whom, after what, within what bound, and what is raised when it does not come. One
|
||||||
|
// table, read three ways: by the watchdog loop (watchdogs.go), which runs every row's watch over the
|
||||||
|
// facts it gathered; by `doctor signals`, which says the age of every row's newest signal; and by the
|
||||||
|
// test generated from it (signals_test.go), which suppresses each signal in turn and asserts its
|
||||||
|
// condition — so a row added without a watch, or a watch that does not fire, fails the build.
|
||||||
|
//
|
||||||
|
// A row not watched yet says why, and in which phase it will be: a watchdog of a signal nothing emits
|
||||||
|
// would be a condition that can only cry wolf. Bounds marked provisional are set from what Phase 0
|
||||||
|
// measured (`durations`) and corrected in Phase 1's first live week; a corrected bound is a change to
|
||||||
|
// this table, reviewed like code.
|
||||||
|
|
||||||
|
// signalRow is one row of the table.
|
||||||
|
type signalRow struct {
|
||||||
|
Row string
|
||||||
|
Signal string
|
||||||
|
Emitter string
|
||||||
|
Trigger string
|
||||||
|
Bound string
|
||||||
|
Kind string
|
||||||
|
Severity conditions.Severity
|
||||||
|
// Phase is the phase of to-be 45 its watchdog is built in.
|
||||||
|
Phase int
|
||||||
|
// Deferred says why it is not watched yet; empty for a row that is.
|
||||||
|
Deferred string
|
||||||
|
// needs is the part of the facts the row reads: an error there is the row blind, which is
|
||||||
|
// itself said (probe-failed) rather than read as nothing wrong.
|
||||||
|
needs func(f *signalFacts) error
|
||||||
|
// watch is what is wrong now, from the facts.
|
||||||
|
watch func(f *signalFacts) []conditions.Observation
|
||||||
|
// newest is the time of the newest signal of this row, for `doctor signals`; zero when none.
|
||||||
|
newest func(f *signalFacts) time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// The bounds, provisional where to-be 45 says so.
|
||||||
|
const (
|
||||||
|
// heartbeatEvery is the interval a node-engine or node tools that say none have always used.
|
||||||
|
heartbeatEvery = 60 * time.Second
|
||||||
|
// heartbeatsMissed is how many intervals may pass in silence (S1, S11).
|
||||||
|
heartbeatsMissed = 3
|
||||||
|
// controlNodeUrgentAfter is how long the control node may be silent before it is urgent (S1).
|
||||||
|
controlNodeUrgentAfter = 30 * time.Minute
|
||||||
|
// reportAtLeast is the least a machine is given to report a send (S2).
|
||||||
|
reportAtLeast = 2 * time.Minute
|
||||||
|
// tierAtLeast is the least a plan's tier is given (S3), the bound `status` calls a plan late at.
|
||||||
|
tierAtLeast = planWaitBound
|
||||||
|
// loopDeafAfter is how long the event loop may take nothing while its consumers hold some (S4).
|
||||||
|
loopDeafAfter = 2 * time.Minute
|
||||||
|
// askAtLeast is the least a build ask is given, and askDefault the bound while nothing is
|
||||||
|
// measured (S6): the build seat declares no timeout of its own.
|
||||||
|
askAtLeast = 20 * time.Minute
|
||||||
|
askDefault = time.Hour
|
||||||
|
// callDefault is the bound of a verb that declares none (S7); push's and build's are longer.
|
||||||
|
callDefault = 10 * time.Minute
|
||||||
|
// advisoryQuiet is how long the bus must be quiet about a thing before its advisory clears (S9).
|
||||||
|
advisoryQuiet = time.Hour
|
||||||
|
// staleRefusalsAllowed in staleRefusalsWithin are what S13 lets pass from one writer.
|
||||||
|
staleRefusalsAllowed = 5
|
||||||
|
staleRefusalsWithin = 5 * time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
// callBounds are the verbs that may run longer than callDefault, and how long (S7).
|
||||||
|
var callBounds = map[string]time.Duration{
|
||||||
|
"push": 30 * time.Minute, "rotate": 30 * time.Minute, "assign": 15 * time.Minute,
|
||||||
|
"unassign": 15 * time.Minute, "command": 30 * time.Minute, "doctor": 3 * time.Minute,
|
||||||
|
}
|
||||||
|
|
||||||
|
// callBound is a verb's bound.
|
||||||
|
func callBound(verb string) time.Duration {
|
||||||
|
if b, ok := callBounds[verb]; ok {
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
return callDefault
|
||||||
|
}
|
||||||
|
|
||||||
|
// signalsTable is the table, in to-be 45's order.
|
||||||
|
var signalsTable = []signalRow{
|
||||||
|
{Row: "S1", Signal: "machine heartbeat", Emitter: "node-engine", Trigger: "its interval",
|
||||||
|
Bound: "3 × the interval it says (60 s when it says none), provisional; not raised while the machine " +
|
||||||
|
"said it is asleep or shutting down (ADR 0211); urgent after 30 min for a control node",
|
||||||
|
Kind: "silent", Severity: conditions.Warning, Phase: 1,
|
||||||
|
needs: func(f *signalFacts) error { return f.machinesErr }, watch: watchHeartbeats,
|
||||||
|
newest: func(f *signalFacts) time.Time {
|
||||||
|
return newestOf(f.machines, func(m machineFacts) time.Time { return m.lastHeard })
|
||||||
|
}},
|
||||||
|
{Row: "S2", Signal: "report after a send", Emitter: "node-engine", Trigger: "each declaration sent",
|
||||||
|
Bound: "max(2 min, 3 × that machine's last apply duration), provisional; not while the machine is silent or asleep",
|
||||||
|
Kind: "sent-not-reported", Severity: conditions.Warning, Phase: 1,
|
||||||
|
needs: func(f *signalFacts) error { return f.machinesErr }, watch: watchReports,
|
||||||
|
newest: func(f *signalFacts) time.Time {
|
||||||
|
return newestOf(f.machines, func(m machineFacts) time.Time { return m.reportedAt })
|
||||||
|
}},
|
||||||
|
{Row: "S3", Signal: "plan tier progress", Emitter: "controller's plan", Trigger: "each tier entered",
|
||||||
|
Bound: "max(30 min, 3 × the p90 of that repository's measured tiers), provisional; not while the " +
|
||||||
|
"build seat is paused under it",
|
||||||
|
Kind: "stalled", Severity: conditions.Warning, Phase: 1,
|
||||||
|
needs: func(f *signalFacts) error { return f.plansErr }, watch: watchPlans,
|
||||||
|
newest: func(f *signalFacts) time.Time {
|
||||||
|
return newestOf(f.plans, func(p planFacts) time.Time { return p.entered })
|
||||||
|
}},
|
||||||
|
{Row: "S4", Signal: "the controller's event loop takes a message", Emitter: "controller",
|
||||||
|
Trigger: "while its consumers have pending messages", Bound: "2 min",
|
||||||
|
Kind: "controller-deaf", Severity: conditions.Urgent, Phase: 1,
|
||||||
|
needs: func(f *signalFacts) error { return f.loopErr }, watch: watchLoop,
|
||||||
|
newest: func(f *signalFacts) time.Time { return f.loop.took }},
|
||||||
|
{Row: "S5", Signal: "a merge announced becomes a plan, or nothing reads it", Emitter: "announcer → controller",
|
||||||
|
Trigger: "each merge", Bound: "10 min (the catch-up pass of issue 266)",
|
||||||
|
Kind: "merge-not-acted", Severity: conditions.Urgent, Phase: 1,
|
||||||
|
needs: func(f *signalFacts) error { return f.mergesErr }, watch: watchMerges,
|
||||||
|
newest: func(f *signalFacts) time.Time { return f.mergesPassed }},
|
||||||
|
{Row: "S6", Signal: "a build asked → its outcome", Emitter: "build seat", Trigger: "each ask",
|
||||||
|
Bound: "max(20 min, 3 × the p90 of measured builds), 1 h while nothing is measured, provisional — the " +
|
||||||
|
"build seat declares no timeout; and an ask the queue gave up on (dead) at once",
|
||||||
|
Kind: "ask-lost", Severity: conditions.Warning, Phase: 1,
|
||||||
|
needs: func(f *signalFacts) error { return f.asksErr }, watch: watchAsks,
|
||||||
|
newest: func(f *signalFacts) time.Time { return newestOf(f.asks, func(a askFacts) time.Time { return a.since }) }},
|
||||||
|
{Row: "S7", Signal: "a call running → finished", Emitter: "controller", Trigger: "each call",
|
||||||
|
Bound: "the verb's bound: push, rotate and command 30 min, assign and unassign 15 min, doctor 3 min, " +
|
||||||
|
"any other 10 min",
|
||||||
|
Kind: "call-hung", Severity: conditions.Warning, Phase: 1,
|
||||||
|
needs: func(*signalFacts) error { return nil }, watch: watchCalls,
|
||||||
|
newest: func(f *signalFacts) time.Time {
|
||||||
|
return newestOf(f.calls, func(c link.Call) time.Time { return c.Started })
|
||||||
|
}},
|
||||||
|
{Row: "S8", Signal: "a provider's failing word repeated", Emitter: "provider",
|
||||||
|
Trigger: "every 15 min while failing (ADR 0224)", Bound: "30 min",
|
||||||
|
Kind: kindProviderSilent, Severity: conditions.Warning, Phase: 1,
|
||||||
|
needs: func(f *signalFacts) error { return f.standingsErr }, watch: watchProviders,
|
||||||
|
newest: func(f *signalFacts) time.Time {
|
||||||
|
return newestOf(f.standings, func(c conditions.Condition) time.Time { return c.LastObserved })
|
||||||
|
}},
|
||||||
|
{Row: "S9", Signal: "bus advisories: maximum deliveries, consumer deleted; the controller's own slow " +
|
||||||
|
"consumer and refused subjects", Emitter: "bus server's advisory subjects; the controller's connection",
|
||||||
|
Trigger: "any", Bound: "any occurrence; clears after an hour without another, and a deleted consumer " +
|
||||||
|
"once it exists again or the mesh no longer expects it",
|
||||||
|
Kind: "slow-consumer, max-deliveries, refused, consumer-lost", Severity: conditions.Warning, Phase: 1,
|
||||||
|
needs: func(f *signalFacts) error { return f.advisoriesErr }, watch: watchAdvisories,
|
||||||
|
newest: func(f *signalFacts) time.Time {
|
||||||
|
return newestOf(f.advisories, func(a link.Advisory) time.Time { return a.Last })
|
||||||
|
}},
|
||||||
|
{Row: "S10", Signal: "the self-check's heartbeat", Emitter: "controller's doctor", Trigger: "every run",
|
||||||
|
Bound: "2 × its interval; watched from a second machine by mesh-watcher, and here as well",
|
||||||
|
Kind: "self-check-silent", Severity: conditions.Urgent, Phase: 1,
|
||||||
|
needs: func(*signalFacts) error { return nil }, watch: watchSelfCheck,
|
||||||
|
newest: func(f *signalFacts) time.Time { return f.selfCheck.last }},
|
||||||
|
{Row: "S11", Signal: "node tools heartbeat", Emitter: "node tools", Trigger: "its interval",
|
||||||
|
Bound: "3 × the interval it says (60 s when it says none), provisional; only where node-tools is assigned",
|
||||||
|
Kind: "tools-silent", Severity: conditions.Warning, Phase: 1,
|
||||||
|
needs: func(f *signalFacts) error { return f.machinesErr }, watch: watchTools,
|
||||||
|
newest: func(f *signalFacts) time.Time {
|
||||||
|
return newestOf(f.machines, func(m machineFacts) time.Time { return m.toolsHeard })
|
||||||
|
}},
|
||||||
|
{Row: "S12", Signal: "the controller lease renewed", Emitter: "controller", Trigger: "every 5 s",
|
||||||
|
Bound: "15 s", Kind: "lease-lost", Severity: conditions.Urgent, Phase: 2,
|
||||||
|
Deferred: "the lease is built in Phase 2 (to-be 45 §6): there is nothing renewed to watch yet, and a " +
|
||||||
|
"second controller is caught today by its consumers being bound (standingBy)"},
|
||||||
|
{Row: "S13", Signal: "stale refusals", Emitter: "every receiver (rule 2)", Trigger: "each refusal",
|
||||||
|
Bound: "more than 5 from one machine in 5 min", Kind: "stale-writer", Severity: conditions.Warning, Phase: 1,
|
||||||
|
needs: func(*signalFacts) error { return nil }, watch: watchStaleRefusals,
|
||||||
|
newest: func(f *signalFacts) time.Time { return time.Time{} }},
|
||||||
|
{Row: "S14", Signal: "facts snapshot exported", Emitter: "controller", Trigger: "daily",
|
||||||
|
Bound: "2 days", Kind: "facts-stale", Severity: conditions.Warning, Phase: 5,
|
||||||
|
Deferred: "the facts snapshot is built in Phase 5 (to-be 45 §9): nothing exports one yet"},
|
||||||
|
{Row: "S15", Signal: "a hand act with a cause already recorded", Emitter: "hand-act log",
|
||||||
|
Trigger: "each act", Bound: "the second within 14 days", Kind: "healer-wanted",
|
||||||
|
Severity: conditions.Warning, Phase: 3,
|
||||||
|
Deferred: "Phase 3 (to-be 45 §10): `hand-acts` lists repeated causes today; the condition comes with the healers"},
|
||||||
|
}
|
||||||
|
|
||||||
|
// newestOf is the newest time among things.
|
||||||
|
func newestOf[T any](list []T, at func(T) time.Time) time.Time {
|
||||||
|
var newest time.Time
|
||||||
|
for _, x := range list {
|
||||||
|
if t := at(x); t.After(newest) {
|
||||||
|
newest = t
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return newest
|
||||||
|
}
|
||||||
|
|
||||||
|
// ago is a duration as the summaries say it.
|
||||||
|
func ago(d time.Duration) string {
|
||||||
|
if d < time.Minute {
|
||||||
|
return d.Round(time.Second).String()
|
||||||
|
}
|
||||||
|
return d.Round(time.Minute).String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// heartbeatBound is a machine's S1 or S11 bound from the interval it says.
|
||||||
|
func heartbeatBound(every time.Duration) time.Duration {
|
||||||
|
if every <= 0 {
|
||||||
|
every = heartbeatEvery
|
||||||
|
}
|
||||||
|
return heartbeatsMissed * every
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchHeartbeats(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, m := range f.machines {
|
||||||
|
if m.lastHeard.IsZero() || m.asleep() {
|
||||||
|
// Never heard is a machine that has not joined, which status says; asleep is not lost.
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
bound := heartbeatBound(m.every)
|
||||||
|
silent := f.now.Sub(m.lastHeard)
|
||||||
|
if silent <= bound {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
severity := conditions.Warning
|
||||||
|
if m.control && silent > controlNodeUrgentAfter {
|
||||||
|
severity = conditions.Urgent
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name, Kind: "silent",
|
||||||
|
Machine: m.name, Severity: severity,
|
||||||
|
Summary: fmt.Sprintf("%s has not been heard from since %s (bound %s)", m.name,
|
||||||
|
m.lastHeard.UTC().Format("2006-01-02 15:04 MST"), bound),
|
||||||
|
Said: fmt.Sprintf("silent for %s", ago(silent))})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchReports(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, m := range f.machines {
|
||||||
|
if m.sentAt.IsZero() || m.reportedCurrent || m.asleep() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) > heartbeatBound(m.every) {
|
||||||
|
continue // silent: S1 says it, and a silent machine reports nothing
|
||||||
|
}
|
||||||
|
bound := max(reportAtLeast, 3*m.lastApply)
|
||||||
|
waited := f.now.Sub(m.sentAt)
|
||||||
|
if waited <= bound {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name,
|
||||||
|
Kind: "sent-not-reported", Machine: m.name, Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s was sent a declaration at %s and has not reported applying it (bound %s)",
|
||||||
|
m.name, m.sentAt.UTC().Format("2006-01-02 15:04 MST"), ago(bound)),
|
||||||
|
Said: fmt.Sprintf("waiting %s for the report of the declaration sent", ago(waited))})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchPlans(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, p := range f.plans {
|
||||||
|
if p.paused {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
in := f.now.Sub(p.entered)
|
||||||
|
if in <= p.bound {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.id, Kind: "stalled",
|
||||||
|
Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("the plan for %s %s has been at tier %d of %d since %s (bound %s): %s",
|
||||||
|
p.repository, short(p.commit), p.tier+1, p.tiers, p.entered.UTC().Format("2006-01-02 15:04 MST"),
|
||||||
|
ago(p.bound), p.waiting),
|
||||||
|
Said: fmt.Sprintf("at tier %d for %s: %s", p.tier+1, ago(in), p.waiting)})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchLoop(f *signalFacts) []conditions.Observation {
|
||||||
|
if f.loop.pending == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
since := f.loop.took
|
||||||
|
if since.IsZero() {
|
||||||
|
since = f.started
|
||||||
|
}
|
||||||
|
if f.now.Sub(since) <= loopDeafAfter {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return []conditions.Observation{{Scope: conditions.ScopeCore, ID: "controller", Kind: "controller-deaf",
|
||||||
|
Token: "deaf", Machine: f.host, Severity: conditions.Urgent,
|
||||||
|
Summary: fmt.Sprintf("the controller's event loop has taken nothing for %s while its consumers hold %d "+
|
||||||
|
"message(s): reports, builds and merges are not being acted on", ago(f.now.Sub(since)), f.loop.pending),
|
||||||
|
Said: fmt.Sprintf("%d pending (%s), last taken %s", f.loop.pending, f.loop.where, since.UTC().Format(time.RFC3339))}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchMerges(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, m := range f.merges {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMerge, ID: m.Repo + "." + short(m.Commit),
|
||||||
|
Kind: "merge-not-acted", Token: "not-acted", Severity: conditions.Urgent,
|
||||||
|
Summary: fmt.Sprintf("%s/%s merged into %s (%s) was never handed to the controller by the bus; "+
|
||||||
|
"acted on late by the catch-up", m.Owner, m.Repo, m.Base, short(m.Commit)),
|
||||||
|
Said: fmt.Sprintf("announced %s, %s behind it", m.At.UTC().Format(time.RFC3339), readableList(m.Modules))})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchAsks(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, a := range f.asks {
|
||||||
|
var said string
|
||||||
|
switch {
|
||||||
|
case a.state == link.AskDead:
|
||||||
|
said = fmt.Sprintf("the %s queue handed it out as often as it may and it was never settled", a.seat)
|
||||||
|
case a.state == link.AskInFlight && f.now.Sub(a.since) > a.bound:
|
||||||
|
said = fmt.Sprintf("in flight on %s for %s (bound %s)", orSomewhere(a.on), ago(f.now.Sub(a.since)), ago(a.bound))
|
||||||
|
default:
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBuild, ID: a.id, Kind: "ask-lost",
|
||||||
|
Token: "lost", Machine: a.on, Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("the build %s of %s has no outcome: %s", a.id, a.what, said), Said: said})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func orSomewhere(node string) string {
|
||||||
|
if node == "" {
|
||||||
|
return "a machine that did not say which"
|
||||||
|
}
|
||||||
|
return node
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchCalls(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, c := range f.calls {
|
||||||
|
bound := callBound(c.Verb)
|
||||||
|
running := f.now.Sub(c.Started)
|
||||||
|
if running <= bound {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeCall, ID: c.ID, Kind: "call-hung",
|
||||||
|
Token: "hung", Machine: f.host, Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s.%s (call %s) has been running since %s, past its bound of %s", c.Seat, c.Verb,
|
||||||
|
c.ID, c.Started.UTC().Format("2006-01-02 15:04 MST"), ago(bound)),
|
||||||
|
Said: fmt.Sprintf("running %s, asked by %s", ago(running), orSomebody(c.Caller))})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func orSomebody(caller string) string {
|
||||||
|
if caller == "" {
|
||||||
|
return "a caller the bus did not name"
|
||||||
|
}
|
||||||
|
return caller
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchProviders(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, c := range f.standings {
|
||||||
|
quiet := f.now.Sub(c.LastObserved)
|
||||||
|
if quiet <= providerSaysAgainWithin {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
module, node, consumer, ok := providerOf(c)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeProvider, ID: module + "." + node + "." + consumer,
|
||||||
|
Token: "silent", Kind: kindProviderSilent, Machine: node, Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s on %s said it keeps failing %s and has said nothing since %s: it stopped "+
|
||||||
|
"saying anything, so its last word is all the mesh has", module, node, consumer,
|
||||||
|
c.LastObserved.UTC().Format("2006-01-02 15:04 MST")),
|
||||||
|
Said: fmt.Sprintf("not said again for %s", ago(quiet))})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchAdvisories(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, a := range f.advisories {
|
||||||
|
if f.now.Sub(a.Last) > advisoryQuiet {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if a.Kind == link.AdvisoryConsumerLost && !f.lostConsumers[a.Stream+"."+a.Consumer] {
|
||||||
|
continue // it exists again, or the mesh no longer expects it: a removal, not a loss
|
||||||
|
}
|
||||||
|
severity := conditions.Warning
|
||||||
|
times := ""
|
||||||
|
if a.Count > 1 {
|
||||||
|
times = fmt.Sprintf(" (%d times since %s)", a.Count, a.First.UTC().Format("15:04 MST"))
|
||||||
|
}
|
||||||
|
machine := ""
|
||||||
|
if a.ID == "controller" {
|
||||||
|
machine = f.host
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: a.ID, Kind: a.Kind,
|
||||||
|
Machine: machine, Severity: severity, Summary: a.Said + times, Said: a.Said})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchSelfCheck(f *signalFacts) []conditions.Observation {
|
||||||
|
every := f.selfCheck.every
|
||||||
|
if every <= 0 {
|
||||||
|
every = doctorEvery
|
||||||
|
}
|
||||||
|
since := f.selfCheck.last
|
||||||
|
if since.IsZero() {
|
||||||
|
since = f.started
|
||||||
|
}
|
||||||
|
if f.now.Sub(since) <= 2*every {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return []conditions.Observation{{Scope: conditions.ScopeCore, ID: "doctor", Kind: "self-check-silent",
|
||||||
|
Machine: f.host, Severity: conditions.Urgent,
|
||||||
|
Summary: fmt.Sprintf("the self-check has not finished a run since %s (it runs every %s): the mesh's "+
|
||||||
|
"invariants are not being checked", since.UTC().Format("2006-01-02 15:04 MST"), every),
|
||||||
|
Said: fmt.Sprintf("no run for %s", ago(f.now.Sub(since)))}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchTools(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, m := range f.machines {
|
||||||
|
if !m.tools || m.asleep() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
bound := heartbeatBound(m.toolsEvery)
|
||||||
|
since := m.toolsHeard
|
||||||
|
if since.IsZero() {
|
||||||
|
// Not heard since this controller started: silent since then at the most.
|
||||||
|
since = f.toolsHeardFrom
|
||||||
|
}
|
||||||
|
silent := f.now.Sub(since)
|
||||||
|
if silent <= bound {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
heard := "not since this controller started at " + since.UTC().Format("2006-01-02 15:04 MST")
|
||||||
|
if !m.toolsHeard.IsZero() {
|
||||||
|
heard = "since " + m.toolsHeard.UTC().Format("2006-01-02 15:04 MST")
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name, Kind: "tools-silent",
|
||||||
|
Machine: m.name, Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("the node tools on %s have not said they are there %s (bound %s): nothing can "+
|
||||||
|
"ask that machine anything", m.name, heard, bound),
|
||||||
|
Said: fmt.Sprintf("silent for %s", ago(silent))})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func watchStaleRefusals(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for node, n := range f.staleRefusals {
|
||||||
|
if n <= staleRefusalsAllowed {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: "stale-writer",
|
||||||
|
Machine: node, Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s refused %d declarations in %s as older than the one it holds: a controller "+
|
||||||
|
"is sending what it has moved past (which one is said once declarations carry an epoch, Phase 2)",
|
||||||
|
node, n, staleRefusalsWithin),
|
||||||
|
Said: fmt.Sprintf("%d stale refusals in %s", n, staleRefusalsWithin)})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// watchedRows are the rows a watchdog runs for.
|
||||||
|
func watchedRows() []signalRow {
|
||||||
|
var out []signalRow
|
||||||
|
for _, r := range signalsTable {
|
||||||
|
if r.watch != nil {
|
||||||
|
out = append(out, r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// kindsOf is a row's condition kinds, one or several.
|
||||||
|
func kindsOf(r signalRow) []string {
|
||||||
|
var out []string
|
||||||
|
for _, k := range strings.Split(r.Kind, ",") {
|
||||||
|
out = append(out, strings.TrimSpace(k))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,275 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The test generated from the signals table (novox/hq to-be 45 §3, ADR 0227 rule 5, "how it is
|
||||||
|
// checked"): **every row is walked**. A watched row's signal is suppressed just inside its bound —
|
||||||
|
// nothing raised — and just past it — its condition raised, with its kind and severity — and restored,
|
||||||
|
// and the condition clears. A row that is not watched says why. A row added to the table without a
|
||||||
|
// suppression here fails, so the table cannot grow a watchdog nobody has seen fire.
|
||||||
|
|
||||||
|
// calm is a mesh whose every signal is fresh: one control node heard ten seconds ago, its last send
|
||||||
|
// reported, a plan a minute into its tier, the loop taking, the merges read, nothing asked, no call
|
||||||
|
// running, the self-check a minute old.
|
||||||
|
func calm(now time.Time) *signalFacts {
|
||||||
|
return &signalFacts{now: now, started: now.Add(-time.Hour), host: "anchor", toolsHeardFrom: now.Add(-time.Hour),
|
||||||
|
machines: []machineFacts{{name: "anchor", control: true, lastHeard: now.Add(-10 * time.Second),
|
||||||
|
every: time.Minute, sentAt: now.Add(-time.Hour), reportedCurrent: true, reportedAt: now.Add(-59 * time.Minute),
|
||||||
|
lastApply: 20 * time.Second, tools: true, toolsHeard: now.Add(-10 * time.Second), toolsEvery: time.Minute}},
|
||||||
|
plans: []planFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee00", tier: 0, tiers: 2,
|
||||||
|
entered: now.Add(-time.Minute), bound: 30 * time.Minute, waiting: "building"}},
|
||||||
|
loop: loopFacts{took: now.Add(-time.Second), pending: 1},
|
||||||
|
mergesPassed: now.Add(-time.Minute),
|
||||||
|
selfCheck: selfCheckFacts{last: now.Add(-time.Minute), every: 5 * time.Minute},
|
||||||
|
lostConsumers: map[string]bool{}, staleRefusals: map[string]int{},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// suppression is one row's signal held back: inside its bound, and past it.
|
||||||
|
type suppression struct{ inside, past func(f *signalFacts) }
|
||||||
|
|
||||||
|
// suppressions are every watched row's, by row.
|
||||||
|
var suppressions = map[string]suppression{
|
||||||
|
"S1": {
|
||||||
|
inside: func(f *signalFacts) { f.machines[0].lastHeard = f.now.Add(-3*time.Minute + time.Second) },
|
||||||
|
past: func(f *signalFacts) { f.machines[0].lastHeard = f.now.Add(-3*time.Minute - time.Second) },
|
||||||
|
},
|
||||||
|
"S2": {
|
||||||
|
inside: func(f *signalFacts) {
|
||||||
|
f.machines[0].sentAt, f.machines[0].reportedCurrent = f.now.Add(-110*time.Second), false
|
||||||
|
},
|
||||||
|
past: func(f *signalFacts) {
|
||||||
|
f.machines[0].sentAt, f.machines[0].reportedCurrent = f.now.Add(-121*time.Second), false
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"S3": {
|
||||||
|
inside: func(f *signalFacts) { f.plans[0].entered = f.now.Add(-29 * time.Minute) },
|
||||||
|
past: func(f *signalFacts) { f.plans[0].entered = f.now.Add(-31 * time.Minute) },
|
||||||
|
},
|
||||||
|
"S4": {
|
||||||
|
inside: func(f *signalFacts) { f.loop.took = f.now.Add(-119 * time.Second) },
|
||||||
|
past: func(f *signalFacts) { f.loop.took = f.now.Add(-121 * time.Second) },
|
||||||
|
},
|
||||||
|
"S5": {
|
||||||
|
inside: func(f *signalFacts) {},
|
||||||
|
past: func(f *signalFacts) {
|
||||||
|
f.merges = []missedMerge{{Owner: "novox", Repo: "app", Base: "main", Commit: "c0ffee0011", At: f.now.Add(-11 * time.Minute)}}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"S6": {
|
||||||
|
inside: func(f *signalFacts) {
|
||||||
|
f.asks = []askFacts{{id: "build-1", seat: "node-build-agent", what: "novox/app", state: link.AskInFlight,
|
||||||
|
on: "anchor", since: f.now.Add(-59 * time.Minute), bound: time.Hour}}
|
||||||
|
},
|
||||||
|
past: func(f *signalFacts) {
|
||||||
|
f.asks = []askFacts{{id: "build-1", seat: "node-build-agent", what: "novox/app", state: link.AskInFlight,
|
||||||
|
on: "anchor", since: f.now.Add(-61 * time.Minute), bound: time.Hour}}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"S7": {
|
||||||
|
inside: func(f *signalFacts) {
|
||||||
|
f.calls = []link.Call{{ID: "call-1", Seat: "mesh-controller", Verb: "push", Started: f.now.Add(-29 * time.Minute)}}
|
||||||
|
},
|
||||||
|
past: func(f *signalFacts) {
|
||||||
|
f.calls = []link.Call{{ID: "call-1", Seat: "mesh-controller", Verb: "push", Started: f.now.Add(-31 * time.Minute)}}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"S8": {
|
||||||
|
inside: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-29 * time.Minute))} },
|
||||||
|
past: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-31 * time.Minute))} },
|
||||||
|
},
|
||||||
|
"S9": {
|
||||||
|
inside: func(f *signalFacts) {
|
||||||
|
f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop", Said: "gave up",
|
||||||
|
First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-61 * time.Minute), Count: 1}}
|
||||||
|
},
|
||||||
|
past: func(f *signalFacts) {
|
||||||
|
f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop", Said: "gave up",
|
||||||
|
First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-59 * time.Minute), Count: 1}}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"S10": {
|
||||||
|
inside: func(f *signalFacts) { f.selfCheck.last = f.now.Add(-9 * time.Minute) },
|
||||||
|
past: func(f *signalFacts) { f.selfCheck.last = f.now.Add(-11 * time.Minute) },
|
||||||
|
},
|
||||||
|
"S11": {
|
||||||
|
inside: func(f *signalFacts) { f.machines[0].toolsHeard = f.now.Add(-179 * time.Second) },
|
||||||
|
past: func(f *signalFacts) { f.machines[0].toolsHeard = f.now.Add(-181 * time.Second) },
|
||||||
|
},
|
||||||
|
"S13": {
|
||||||
|
inside: func(f *signalFacts) { f.staleRefusals = map[string]int{"anchor": 5} },
|
||||||
|
past: func(f *signalFacts) { f.staleRefusals = map[string]int{"anchor": 6} },
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// standingSaid is a provider's failing word last said at a moment.
|
||||||
|
func standingSaid(at time.Time) conditions.Condition {
|
||||||
|
return conditions.Condition{Key: "provider.idp.anchor.app.failing", Kind: kindProviderFailing, LastObserved: at}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEveryRowOfTheSignalsTableIsWatchedRaisedAndCleared(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for _, row := range signalsTable {
|
||||||
|
t.Run(row.Row, func(t *testing.T) {
|
||||||
|
if seen[row.Row] {
|
||||||
|
t.Fatalf("%s is in the table twice", row.Row)
|
||||||
|
}
|
||||||
|
seen[row.Row] = true
|
||||||
|
if row.Signal == "" || row.Emitter == "" || row.Trigger == "" || row.Bound == "" || row.Kind == "" ||
|
||||||
|
(row.Severity != conditions.Urgent && row.Severity != conditions.Warning) {
|
||||||
|
t.Fatalf("%s does not say what it expects, from whom, within what, and what it raises: %+v", row.Row, row)
|
||||||
|
}
|
||||||
|
if row.Deferred != "" {
|
||||||
|
if row.watch != nil || row.Phase <= 1 {
|
||||||
|
t.Fatalf("%s is deferred and watched, or deferred out of Phase 1's own rows: %+v", row.Row, row)
|
||||||
|
}
|
||||||
|
if _, has := suppressions[row.Row]; has {
|
||||||
|
t.Fatalf("%s is deferred and has a suppression: one of the two is stale", row.Row)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if row.watch == nil || row.needs == nil || row.newest == nil {
|
||||||
|
t.Fatalf("%s is watched and lacks its watch, its needs or its newest", row.Row)
|
||||||
|
}
|
||||||
|
s, ok := suppressions[row.Row]
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("%s has no suppression in this test: a watchdog nobody has seen fire", row.Row)
|
||||||
|
}
|
||||||
|
if got := row.watch(calm(now)); len(got) != 0 {
|
||||||
|
t.Fatalf("%s raised on a calm mesh: %+v", row.Row, got)
|
||||||
|
}
|
||||||
|
inside := calm(now)
|
||||||
|
s.inside(inside)
|
||||||
|
if got := row.watch(inside); len(got) != 0 {
|
||||||
|
t.Fatalf("%s raised inside its bound: %+v", row.Row, got)
|
||||||
|
}
|
||||||
|
past := calm(now)
|
||||||
|
s.past(past)
|
||||||
|
got := row.watch(past)
|
||||||
|
if len(got) == 0 {
|
||||||
|
t.Fatalf("%s raised nothing past its bound", row.Row)
|
||||||
|
}
|
||||||
|
for _, o := range got {
|
||||||
|
if !slices.Contains(kindsOf(row), o.Kind) {
|
||||||
|
t.Errorf("%s raised %q, which is not its kind %q", row.Row, o.Kind, row.Kind)
|
||||||
|
}
|
||||||
|
if o.Severity != row.Severity {
|
||||||
|
t.Errorf("%s raised %s, the table says %s", row.Row, o.Severity, row.Severity)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(o.Summary) == "" {
|
||||||
|
t.Errorf("%s raised a condition that says nothing", row.Row)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Through the store: raised past the bound, cleared when the signal returns.
|
||||||
|
store := conditions.NewInMemory()
|
||||||
|
told := &conditions.Told{}
|
||||||
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: told,
|
||||||
|
Now: func() time.Time { return now }})
|
||||||
|
defer k.Close(context.Background())
|
||||||
|
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
||||||
|
w.see(t.Context(), past)
|
||||||
|
open, err := k.Open(t.Context())
|
||||||
|
if err != nil || len(open) != len(got) {
|
||||||
|
t.Fatalf("%s past its bound left %d open (%v), want %d", row.Row, len(open), err, len(got))
|
||||||
|
}
|
||||||
|
w.see(t.Context(), calm(now))
|
||||||
|
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||||
|
t.Fatalf("%s's condition stayed open after the signal returned: %+v", row.Row, open)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
for name := range suppressions {
|
||||||
|
if !seen[name] {
|
||||||
|
t.Errorf("a suppression for %s, which the table does not have", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The control node silent for half an hour is urgent** (S1); any other machine stays a warning.
|
||||||
|
func TestTheControlNodeSilentIsUrgentAfterHalfAnHour(t *testing.T) {
|
||||||
|
now := time.Now()
|
||||||
|
f := calm(now)
|
||||||
|
f.machines[0].lastHeard = now.Add(-31 * time.Minute)
|
||||||
|
f.machines = append(f.machines, machineFacts{name: "laptop", lastHeard: now.Add(-31 * time.Minute)})
|
||||||
|
got := watchHeartbeats(f)
|
||||||
|
if len(got) != 2 || got[0].Severity != conditions.Urgent || got[1].Severity != conditions.Warning {
|
||||||
|
t.Fatalf("%+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A machine that said it sleeps is not silent** (ADR 0211), nor late to report; one that woke is.
|
||||||
|
func TestAMachineThatSaidItSleepsIsNotSilent(t *testing.T) {
|
||||||
|
now := time.Now()
|
||||||
|
f := calm(now)
|
||||||
|
f.machines[0].lastHeard = now.Add(-2 * time.Hour)
|
||||||
|
f.machines[0].sentAt, f.machines[0].reportedCurrent = now.Add(-time.Hour), false
|
||||||
|
f.machines[0].power = link.PowerState{State: "sleeping", At: now.Add(-2 * time.Hour)}
|
||||||
|
if got := append(watchHeartbeats(f), append(watchReports(f), watchTools(f)...)...); len(got) != 0 {
|
||||||
|
t.Fatalf("a sleeping machine raised %+v", got)
|
||||||
|
}
|
||||||
|
f.machines[0].power = link.PowerState{State: "woke", At: now.Add(-time.Hour)}
|
||||||
|
if got := watchHeartbeats(f); len(got) != 1 {
|
||||||
|
t.Fatalf("a woken machine silent past its bound raised %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A watchdog that cannot see says so, and clears nothing it raised** (ADR 0227 rule 4): the store
|
||||||
|
// unreadable is a probe-failed of its own, and the machine's silence stays open until it can see again.
|
||||||
|
func TestABlindWatchdogSaysSoAndClearsNothing(t *testing.T) {
|
||||||
|
now := time.Now()
|
||||||
|
store := conditions.NewInMemory()
|
||||||
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||||
|
defer k.Close(context.Background())
|
||||||
|
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
||||||
|
silent := calm(now)
|
||||||
|
silent.machines[0].lastHeard = now.Add(-10 * time.Minute)
|
||||||
|
w.see(t.Context(), silent)
|
||||||
|
blind := calm(now)
|
||||||
|
blind.machines, blind.machinesErr = nil, errors.New("the store is away")
|
||||||
|
w.see(t.Context(), blind)
|
||||||
|
open, err := k.Open(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var keys []string
|
||||||
|
for _, c := range open {
|
||||||
|
keys = append(keys, c.Key)
|
||||||
|
}
|
||||||
|
for _, want := range []string{"machine.anchor.silent", "probe.S1.failed", "probe.S2.failed", "probe.S11.failed"} {
|
||||||
|
if !slices.Contains(keys, want) {
|
||||||
|
t.Errorf("%s is not open while the machines cannot be read: %v", want, keys)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
w.see(t.Context(), calm(now))
|
||||||
|
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||||
|
t.Fatalf("seeing again left open %+v", open)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A controller standing by sees nothing and says nothing**: it hears no heartbeat, and would call
|
||||||
|
// every machine silent.
|
||||||
|
func TestAControllerStandingBySaysNothing(t *testing.T) {
|
||||||
|
store := conditions.NewInMemory()
|
||||||
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
||||||
|
defer k.Close(context.Background())
|
||||||
|
w := &watchdogs{keeper: k, started: time.Now(), acting: func() bool { return false }}
|
||||||
|
w.tick(t.Context())
|
||||||
|
if w.lastTick().IsZero() {
|
||||||
|
t.Fatal("a tick standing by was not counted")
|
||||||
|
}
|
||||||
|
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||||
|
t.Fatalf("%+v", open)
|
||||||
|
}
|
||||||
|
}
|
||||||
+124
-77
@@ -2,91 +2,153 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
)
|
)
|
||||||
|
|
||||||
// A provider that keeps failing a consumer is a problem the controller reports (novox/hq ADR 0224).
|
// A provider that keeps failing a consumer is a problem the controller reports (novox/hq ADR 0224) —
|
||||||
|
// **the condition store's first kind** (to-be 45 §2, ADR 0227).
|
||||||
//
|
//
|
||||||
// On 2026-10-05 the identity provider's provisioner failed every consumer from shortly after midnight
|
// On 2026-10-05 the identity provider's provisioner failed every consumer from shortly after midnight
|
||||||
// until it was fixed by hand that night — 31,000 refused logins after its database was moved and its
|
// until it was fixed by hand that night — 31,000 refused logins after its database was moved and its
|
||||||
// admin kept an older password — and `status` called the mesh well all day (novox/hq issue 179). A
|
// admin kept an older password — and `status` called the mesh well all day (novox/hq issue 179). A
|
||||||
// provider now announces a consumer it has failed for minutes; the controller keeps it until the
|
// provider announces a consumer it has failed for minutes; the controller keeps it until the provider
|
||||||
// provider says it recovered; and `status`, its JSON and `node show` name it, breaking "all well".
|
// says it recovered; and `status`, its JSON and `node show` name it, breaking "all well". Unchanged in
|
||||||
|
// what it says and when; kept as a condition, `provider.<module>.<node>.<consumer>.failing`, rather
|
||||||
|
// than a row of its own, so it is said outward like every other fault and silenced like one.
|
||||||
|
|
||||||
// standings keeps what providers say, in the inventory.
|
// Kinds of the provider standing.
|
||||||
type standings struct{ inv *inventory.Inventory }
|
const (
|
||||||
|
kindProviderFailing = "provider-failing"
|
||||||
|
kindProviderSilent = "provider-silent"
|
||||||
|
// sourceProvisioner is what raised a standing: the provider's own event.
|
||||||
|
sourceProvisioner = "provisioner.failing"
|
||||||
|
)
|
||||||
|
|
||||||
func (s standings) Stood(ctx context.Context, st link.Standing) (bool, error) {
|
// providerSaysAgainWithin is how long a failing word stays current without being said again: twice
|
||||||
return s.inv.KeepStanding(ctx, st.Failing, inventory.ProviderStanding{
|
// the quarter of an hour a provider repeats it at (ADR 0224). Past it, S8.
|
||||||
Module: st.Module, ProviderNode: st.ProviderNode, Provision: st.Provider,
|
const providerSaysAgainWithin = 30 * time.Minute
|
||||||
Consumer: st.Consumer, ConsumerNode: st.Node,
|
|
||||||
Class: st.Class, Error: st.Error, Since: st.Since, Attempts: st.Attempts,
|
// standings keeps what providers say, as conditions.
|
||||||
})
|
type standings struct {
|
||||||
|
keeper func() *conditions.Keeper
|
||||||
}
|
}
|
||||||
|
|
||||||
// failingProviders is every consumer a provider still assigned where it ran says it keeps failing.
|
// standingObservation is a provider's failing word as an observation: the provider, its machine and
|
||||||
//
|
// the consumer name it, so the same consumer failed again is the same condition.
|
||||||
// **A provider no longer assigned is not asked about.** Its last word stays in the store, and is
|
func standingObservation(st link.Standing) conditions.Observation {
|
||||||
// not a problem: nothing runs there to fail anybody. Assigned again, its first success for each
|
whom := st.Consumer
|
||||||
// consumer clears it.
|
if st.Node != "" {
|
||||||
func failingProviders(ctx context.Context, inv *inventory.Inventory) ([]inventory.ProviderStanding, error) {
|
whom += " on " + st.Node
|
||||||
all, err := inv.FailingProviders(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("what providers say they keep failing cannot be read: %w", err)
|
|
||||||
}
|
}
|
||||||
|
summary := fmt.Sprintf("%s on %s keeps failing %s: %s, %d attempt(s) since %s", st.Module, st.ProviderNode,
|
||||||
|
whom, orUnclassed(st.Class), st.Attempts, st.Since.UTC().Format("2006-01-02 15:04 MST"))
|
||||||
|
said := orUnclassed(st.Class)
|
||||||
|
if e := firstLine(st.Error); e != "" {
|
||||||
|
said += ": " + e
|
||||||
|
}
|
||||||
|
if st.Provider != "" {
|
||||||
|
said += fmt.Sprintf(" (provision %s, %d attempts)", st.Provider, st.Attempts)
|
||||||
|
}
|
||||||
|
return conditions.Observation{Scope: conditions.ScopeProvider,
|
||||||
|
ID: st.Module + "." + st.ProviderNode + "." + st.Consumer,
|
||||||
|
Token: "failing", Kind: kindProviderFailing, Machine: st.ProviderNode, Also: alsoOn(st.Node, st.ProviderNode),
|
||||||
|
Severity: conditions.Warning, Summary: summary, Said: said, Source: sourceProvisioner}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stood keeps a provider's newest word: failing raises or observes its condition, recovered clears
|
||||||
|
// it. An error is the store away, and the link holds the message to be asked again — a recovery is
|
||||||
|
// said once, and dropping it would leave a consumer named failing that is fine.
|
||||||
|
func (s standings) Stood(ctx context.Context, st link.Standing) (bool, error) {
|
||||||
|
k := s.keeper()
|
||||||
|
if k == nil {
|
||||||
|
return false, fmt.Errorf("the condition store is not open in this controller: %w", link.ErrTryAgain)
|
||||||
|
}
|
||||||
|
o := standingObservation(st)
|
||||||
|
if !st.Failing {
|
||||||
|
why := "the provider says it recovered"
|
||||||
|
if st.Why != "" {
|
||||||
|
why += ": " + st.Why
|
||||||
|
}
|
||||||
|
cleared, err := k.Clear(ctx, o.Key(), why)
|
||||||
|
return cleared, storeAway(err)
|
||||||
|
}
|
||||||
|
_, err := k.Observe(ctx, o)
|
||||||
|
return false, storeAway(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// storeAway reads the condition store failing as the bus being away for the moment: the link holds the
|
||||||
|
// message and asks again, as it does for a store restarting (ADR 0083), rather than taking it unkept.
|
||||||
|
func storeAway(err error) error {
|
||||||
|
if err == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return fmt.Errorf("%v: %w", err, link.ErrTryAgain)
|
||||||
|
}
|
||||||
|
|
||||||
|
// providerStandings is every open provider-failing condition, from what is open.
|
||||||
|
func providerStandings(open []conditions.Condition) []conditions.Condition {
|
||||||
|
var out []conditions.Condition
|
||||||
|
for _, c := range open {
|
||||||
|
if c.Kind == kindProviderFailing {
|
||||||
|
out = append(out, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// providerOf reads a standing's provider module and machine back from its key.
|
||||||
|
func providerOf(c conditions.Condition) (module, node, consumer string, ok bool) {
|
||||||
|
parts := strings.Split(c.Key, ".")
|
||||||
|
if len(parts) != 5 || parts[0] != conditions.ScopeProvider {
|
||||||
|
return "", "", "", false
|
||||||
|
}
|
||||||
|
return parts[1], parts[2], parts[3], true
|
||||||
|
}
|
||||||
|
|
||||||
|
// unassignedProviders clears the standing of every provider no longer assigned where it ran.
|
||||||
|
//
|
||||||
|
// **A provider no longer assigned is not asked about** (ADR 0224 §4): nothing runs there to fail
|
||||||
|
// anybody, and nothing there will ever say it recovered. The observation that resolves it is the
|
||||||
|
// assignment. Assigned again, its first failure raises it again.
|
||||||
|
func unassignedProviders(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper,
|
||||||
|
open []conditions.Condition) error {
|
||||||
assigned := map[string]map[string]bool{}
|
assigned := map[string]map[string]bool{}
|
||||||
var out []inventory.ProviderStanding
|
for _, c := range providerStandings(open) {
|
||||||
for _, s := range all {
|
module, node, _, ok := providerOf(c)
|
||||||
on, asked := assigned[s.ProviderNode]
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
on, asked := assigned[node]
|
||||||
if !asked {
|
if !asked {
|
||||||
modules, err := inv.Assigned(ctx, s.ProviderNode)
|
modules, err := inv.Assigned(ctx, node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// A provider on a machine the mesh no longer knows has nothing running to fail anybody.
|
if errors.Is(err, inventory.ErrNoSuchNode) {
|
||||||
modules = nil
|
modules = nil // a machine the mesh no longer knows runs nothing
|
||||||
|
} else {
|
||||||
|
return fmt.Errorf("what %s is assigned cannot be read: %w", node, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
on = map[string]bool{}
|
on = map[string]bool{}
|
||||||
for _, m := range modules {
|
for _, m := range modules {
|
||||||
on[m] = true
|
on[m] = true
|
||||||
}
|
}
|
||||||
assigned[s.ProviderNode] = on
|
assigned[node] = on
|
||||||
}
|
}
|
||||||
if on[s.Module] {
|
if !on[module] {
|
||||||
out = append(out, s)
|
if _, err := k.Clear(ctx, c.Key, module+" is no longer assigned to "+node+
|
||||||
|
": nothing runs there to fail anybody"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return out, nil
|
return nil
|
||||||
}
|
|
||||||
|
|
||||||
// failingLines is how status says them: one consumer per entry, the error under it, and a provider
|
|
||||||
// that stopped repeating itself said so.
|
|
||||||
func failingLines(list []inventory.ProviderStanding, now time.Time) []string {
|
|
||||||
var out []string
|
|
||||||
for _, s := range list {
|
|
||||||
where := s.Module
|
|
||||||
if s.ProviderNode != "" {
|
|
||||||
where += " on " + s.ProviderNode
|
|
||||||
}
|
|
||||||
whom := s.Consumer
|
|
||||||
if s.ConsumerNode != "" {
|
|
||||||
whom += " (" + s.ConsumerNode + ")"
|
|
||||||
}
|
|
||||||
out = append(out, fmt.Sprintf(" %-24s fails %s: %s, for %s (%d attempts since %s)",
|
|
||||||
where, whom, orUnclassed(s.Class), roughly(now.Sub(s.Since)), s.Attempts,
|
|
||||||
s.Since.Local().Format("2006-01-02 15:04")))
|
|
||||||
if e := strings.TrimSpace(s.Error); e != "" {
|
|
||||||
out = append(out, fmt.Sprintf(" %-24s %s", "", firstLine(e)))
|
|
||||||
}
|
|
||||||
if s.Quiet(now) {
|
|
||||||
out = append(out, fmt.Sprintf(" %-24s not said again for %s — the provider has stopped "+
|
|
||||||
"saying anything, so this is its last word", "", roughly(now.Sub(s.SaidAt))))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func orUnclassed(class string) string {
|
func orUnclassed(class string) string {
|
||||||
@@ -96,25 +158,10 @@ func orUnclassed(class string) string {
|
|||||||
return class
|
return class
|
||||||
}
|
}
|
||||||
|
|
||||||
// printFailing is the status section, said when there is anything to say.
|
// alsoOn is a consumer's machine, when it is not the provider's.
|
||||||
func printFailing(list []inventory.ProviderStanding, now time.Time) {
|
func alsoOn(consumerNode, providerNode string) []string {
|
||||||
if len(list) == 0 {
|
if consumerNode == "" || consumerNode == providerNode {
|
||||||
return
|
return nil
|
||||||
}
|
}
|
||||||
fmt.Printf("%d consumer(s) a provider keeps failing (ADR 0224):\n\n", len(list))
|
return []string{consumerNode}
|
||||||
for _, line := range failingLines(list, now) {
|
|
||||||
fmt.Println(line)
|
|
||||||
}
|
|
||||||
fmt.Printf("\n the provider's journal has every attempt; it says recovered on its next success\n\n")
|
|
||||||
}
|
|
||||||
|
|
||||||
// failingOn is the standings that concern one machine: a provider running there, or a consumer.
|
|
||||||
func failingOn(list []inventory.ProviderStanding, node string) []inventory.ProviderStanding {
|
|
||||||
var out []inventory.ProviderStanding
|
|
||||||
for _, s := range list {
|
|
||||||
if s.ProviderNode == node || s.ConsumerNode == node {
|
|
||||||
out = append(out, s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,13 +7,14 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
)
|
)
|
||||||
|
|
||||||
// A provider that keeps failing a consumer is a problem `status` names (novox/hq ADR 0224). On
|
// A provider that keeps failing a consumer is a problem `status` names (novox/hq ADR 0224), kept as
|
||||||
// 2026-10-05 the identity provider refused every consumer for a day and status called the mesh well
|
// the condition store's first kind (to-be 45 §2). On 2026-10-05 the identity provider refused every
|
||||||
// (04-ISSUES/179): this is that day, told to the controller the way the provider now tells it.
|
// consumer for a day and status called the mesh well (04-ISSUES/179): this is that day, told to the
|
||||||
|
// controller the way the provider now tells it.
|
||||||
func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
|
func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
@@ -22,7 +23,7 @@ func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
|
|||||||
if _, err := assign(ctx, open, "anchor", "idp"); err != nil {
|
if _, err := assign(ctx, open, "anchor", "idp"); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
kept := standings{open.inventory}
|
kept := standings{keeper: func() *conditions.Keeper { return conditionsFrom }}
|
||||||
since := time.Now().Add(-23 * time.Hour)
|
since := time.Now().Add(-23 * time.Hour)
|
||||||
failing := link.Standing{Module: "idp", Failing: true, Provider: "oidc-client", ProviderNode: "anchor",
|
failing := link.Standing{Module: "idp", Failing: true, Provider: "oidc-client", ProviderNode: "anchor",
|
||||||
Consumer: "mesh_laptop_dashboard", Node: "laptop", Class: "credentials-rejected",
|
Consumer: "mesh_laptop_dashboard", Node: "laptop", Class: "credentials-rejected",
|
||||||
@@ -39,8 +40,8 @@ func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
|
|||||||
t.Fatal("a mesh whose identity provider fails a consumer reads as well")
|
t.Fatal("a mesh whose identity provider fails a consumer reads as well")
|
||||||
}
|
}
|
||||||
said := printed(t, func() error { return printStatus(asked) })
|
said := printed(t, func() error { return printStatus(asked) })
|
||||||
for _, want := range []string{"1 consumer(s) a provider keeps failing", "idp on anchor",
|
for _, want := range []string{"1 open condition(s)", "provider.idp.anchor.mesh_laptop_dashboard.failing",
|
||||||
"mesh_laptop_dashboard (laptop)", "credentials-rejected", "31000 attempts", "invalid_grant"} {
|
"idp on anchor keeps failing mesh_laptop_dashboard on laptop", "credentials-rejected", "31000 attempt(s)"} {
|
||||||
if !strings.Contains(said, want) {
|
if !strings.Contains(said, want) {
|
||||||
t.Fatalf("status does not say %q:\n%s", want, said)
|
t.Fatalf("status does not say %q:\n%s", want, said)
|
||||||
}
|
}
|
||||||
@@ -48,20 +49,25 @@ func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
|
|||||||
if strings.Contains(said, "all doing what they were told") {
|
if strings.Contains(said, "all doing what they were told") {
|
||||||
t.Fatalf("status said all well beside a failing provider:\n%s", said)
|
t.Fatalf("status said all well beside a failing provider:\n%s", said)
|
||||||
}
|
}
|
||||||
|
if !strings.HasPrefix(said, "1 open condition(s)") {
|
||||||
|
t.Fatalf("status does not lead with what is open:\n%s", said)
|
||||||
|
}
|
||||||
body, err := statusAsJSON(asked)
|
body, err := statusAsJSON(asked)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
var doc struct {
|
var doc struct {
|
||||||
Failing []inventory.ProviderStanding `json:"failing"`
|
Conditions []conditions.Condition `json:"conditions"`
|
||||||
|
Failing []conditions.Condition `json:"failing"`
|
||||||
}
|
}
|
||||||
if err := json.Unmarshal(body, &doc); err != nil || len(doc.Failing) != 1 || doc.Failing[0].Consumer != "mesh_laptop_dashboard" {
|
if err := json.Unmarshal(body, &doc); err != nil || len(doc.Failing) != 1 || len(doc.Conditions) != 1 ||
|
||||||
|
!strings.Contains(doc.Failing[0].Evidence[0].Said, "invalid_grant") {
|
||||||
t.Fatalf("the document does not carry it: %v\n%s", err, body)
|
t.Fatalf("the document does not carry it: %v\n%s", err, body)
|
||||||
}
|
}
|
||||||
// Both machines' `node show` name it: where the provider runs, and where the consumer is.
|
// Both machines' `node show` name it: where the provider runs, and where the consumer is.
|
||||||
for _, node := range []string{"anchor", "laptop"} {
|
for _, node := range []string{"anchor", "laptop"} {
|
||||||
shown := printed(t, func() error { return showNode(ctx, open.inventory, node) })
|
shown := printed(t, func() error { return showNode(ctx, open.inventory, node) })
|
||||||
if !strings.Contains(shown, "a provider keeps failing") || !strings.Contains(shown, "mesh_laptop_dashboard") {
|
if !strings.Contains(shown, "open condition(s) about this machine") || !strings.Contains(shown, "mesh_laptop_dashboard") {
|
||||||
t.Fatalf("node show %s does not name it:\n%s", node, shown)
|
t.Fatalf("node show %s does not name it:\n%s", node, shown)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -75,41 +81,38 @@ func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if len(asked.failing) != 0 {
|
if len(asked.conditions) != 0 {
|
||||||
t.Fatalf("a recovered consumer is still named: %+v", asked.failing)
|
t.Fatalf("a recovered consumer is still named: %+v", asked.conditions)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A provider no longer assigned where it ran has nothing running to fail anybody: its last word is
|
// A provider no longer assigned where it ran has nothing running to fail anybody: its last word is
|
||||||
// not a problem.
|
// cleared on the next look, said as resolved by the assignment.
|
||||||
func TestAnUnassignedProvidersLastWordIsNotAProblem(t *testing.T) {
|
func TestAnUnassignedProvidersLastWordIsCleared(t *testing.T) {
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
if _, err := (standings{open.inventory}).Stood(ctx, link.Standing{Module: "gone", Failing: true,
|
kept := standings{keeper: func() *conditions.Keeper { return conditionsFrom }}
|
||||||
|
if _, err := kept.Stood(ctx, link.Standing{Module: "gone", Failing: true,
|
||||||
ProviderNode: "anchor", Consumer: "x", Since: time.Now()}); err != nil {
|
ProviderNode: "anchor", Consumer: "x", Since: time.Now()}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
asked, err := theThreeQuestions(ctx, open)
|
all, err := conditionsFrom.Open(ctx)
|
||||||
if err != nil {
|
if err != nil || len(all) != 1 {
|
||||||
|
t.Fatalf("%+v %v", all, err)
|
||||||
|
}
|
||||||
|
if err := unassignedProviders(ctx, open.inventory, conditionsFrom, all); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if len(asked.failing) != 0 {
|
if all, _ := conditionsFrom.Open(ctx); len(all) != 0 {
|
||||||
t.Fatalf("%+v", asked.failing)
|
t.Fatalf("%+v", all)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAProviderThatStoppedRepeatingItselfIsSaidToHaveGoneQuiet(t *testing.T) {
|
// **The store away holds a recovery** (ADR 0224 §3): a standing that cannot be kept is asked again.
|
||||||
now := time.Now()
|
func TestAStandingTheStoreCannotKeepIsAskedAgain(t *testing.T) {
|
||||||
lines := strings.Join(failingLines([]inventory.ProviderStanding{{
|
kept := standings{keeper: func() *conditions.Keeper { return nil }}
|
||||||
Module: "idp", ProviderNode: "anchor", Consumer: "c", Class: "unreachable", Error: "connection refused\nmore",
|
if _, err := kept.Stood(t.Context(), link.Standing{Module: "idp", ProviderNode: "anchor", Consumer: "x"}); err == nil ||
|
||||||
Since: now.Add(-3 * time.Hour), SaidAt: now.Add(-2 * time.Hour), Attempts: 9,
|
!strings.Contains(err.Error(), link.ErrTryAgain.Error()) {
|
||||||
}}, now), "\n")
|
t.Fatalf("answered %v", err)
|
||||||
for _, want := range []string{"unreachable, for 3h", "connection refused", "not said again for 2h"} {
|
|
||||||
if !strings.Contains(lines, want) {
|
|
||||||
t.Fatalf("%q not in:\n%s", want, lines)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if strings.Contains(lines, "more") {
|
|
||||||
t.Fatalf("more than the first line of an error:\n%s", lines)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/overlay"
|
"github.com/novox/mesh-controller/internal/overlay"
|
||||||
)
|
)
|
||||||
@@ -81,8 +82,12 @@ func printStatus(asked answers) error {
|
|||||||
wrong, nodes, quiet := asked.wrong, asked.nodes, asked.quiet
|
wrong, nodes, quiet := asked.wrong, asked.nodes, asked.quiet
|
||||||
behind, sources := asked.behind, asked.sources
|
behind, sources := asked.behind, asked.sources
|
||||||
|
|
||||||
|
// **What is wrong leads** (novox/hq to-be 45 §2): every open condition, urgent first, oldest
|
||||||
|
// first, silenced ones with when their silence ends.
|
||||||
|
printConditions(asked.conditions, asked.conditionsUnread, time.Now())
|
||||||
|
|
||||||
if len(asked.refused) > 0 {
|
if len(asked.refused) > 0 {
|
||||||
// First, above everything else. A machine that cannot be worked out is not running an old
|
// First of what follows. A machine that cannot be worked out is not running an old
|
||||||
// declaration — it has no declaration, and nothing below this line is about it.
|
// declaration — it has no declaration, and nothing below this line is about it.
|
||||||
var names []string
|
var names []string
|
||||||
for name := range asked.refused {
|
for name := range asked.refused {
|
||||||
@@ -129,10 +134,6 @@ func printStatus(asked answers) error {
|
|||||||
fmt.Println()
|
fmt.Println()
|
||||||
}
|
}
|
||||||
|
|
||||||
// A provider failing a consumer, beside machines failing what they were told: both are something
|
|
||||||
// not working now (novox/hq ADR 0224).
|
|
||||||
printFailing(asked.failing, time.Now())
|
|
||||||
|
|
||||||
if len(quiet) > 0 {
|
if len(quiet) > 0 {
|
||||||
var said []string
|
var said []string
|
||||||
for _, n := range quiet {
|
for _, n := range quiet {
|
||||||
@@ -302,6 +303,29 @@ func printStatus(asked answers) error {
|
|||||||
fmt.Printf("\n `assign <node> <holder>` meets it; reported until every machine has its holders, then refused\n\n")
|
fmt.Printf("\n `assign <node> <holder>` meets it; reported until every machine has its holders, then refused\n\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if len(asked.overflowing) > 0 {
|
||||||
|
// **Reported, and the provider still pushed** (novox/hq ADR 0225, issue 263). Each is left
|
||||||
|
// out of its provider's grants, so the module holds a login nothing created; the provider's
|
||||||
|
// machine is sent everything else rather than refused for one consumer elsewhere.
|
||||||
|
fmt.Printf("%d module(s) identified too long for a provision they require, and not granted it:\n",
|
||||||
|
len(asked.overflowing))
|
||||||
|
for _, o := range asked.overflowing {
|
||||||
|
fmt.Printf(" %-12s %-20s %-22s %q is %d, %s keeps %d\n", o.Consumer, o.Module, o.Provision,
|
||||||
|
o.Identity, len(o.Identity), o.Bound.In, o.Bound.Max)
|
||||||
|
}
|
||||||
|
fmt.Printf("\n a shorter `slug` in the module's definition fits it; `module check` refuses one before merge\n\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Repairs done by hand this week (novox/hq to-be 45 §7). Not a fault, so it does not break "all
|
||||||
|
// well"; each is a healer the mesh does not have yet, and the count is how that is watched.
|
||||||
|
switch {
|
||||||
|
case asked.handActsUnread != "":
|
||||||
|
fmt.Printf("the hand-act log could not be read, so how much was done by hand this week is not known: %s\n\n",
|
||||||
|
asked.handActsUnread)
|
||||||
|
case asked.handActs != nil && *asked.handActs > 0:
|
||||||
|
fmt.Printf("%d act(s) done by hand in the last seven days — `hand-acts` lists them, and why\n\n", *asked.handActs)
|
||||||
|
}
|
||||||
|
|
||||||
if adopted := adoptedNodes(nodes); len(adopted) > 0 {
|
if adopted := adoptedNodes(nodes); len(adopted) > 0 {
|
||||||
// Said, because nothing forces the flip: a node left adopted is visible here rather than
|
// Said, because nothing forces the flip: a node left adopted is visible here rather than
|
||||||
// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
|
// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
|
||||||
@@ -311,8 +335,9 @@ func printStatus(asked answers) error {
|
|||||||
|
|
||||||
if asked.well() {
|
if asked.well() {
|
||||||
// Said plainly. "Nothing to report" and "nothing was checked" must never look the same,
|
// Said plainly. "Nothing to report" and "nothing was checked" must never look the same,
|
||||||
// and getting here means every question was asked and answered.
|
// and getting here means every question was asked and answered. **No open conditions first**
|
||||||
fmt.Printf("%d machine(s), all doing what they were told, all heard from, running what "+
|
// (novox/hq to-be 45 §2): it is what the sentence means now, silenced ones included.
|
||||||
|
fmt.Printf("no open conditions; %d machine(s), all doing what they were told, all heard from, running what "+
|
||||||
"the mesh would send them, and every module current with its source\n", len(nodes))
|
"the mesh would send them, and every module current with its source\n", len(nodes))
|
||||||
// **And what that sentence does not cover**, because for eleven hours it was true of a mesh
|
// **And what that sentence does not cover**, because for eleven hours it was true of a mesh
|
||||||
// in which no module could reach another (novox/hq 04-ISSUES/145). Every question above is
|
// in which no module could reach another (novox/hq 04-ISSUES/145). Every question above is
|
||||||
@@ -410,21 +435,27 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
|||||||
// ADR 0207). Each machine resolved again rather than threaded through whoResolves, whose answer
|
// ADR 0207). Each machine resolved again rather than threaded through whoResolves, whose answer
|
||||||
// the private network is built from and should say nothing else; a machine that does not
|
// the private network is built from and should say nothing else; a machine that does not
|
||||||
// resolve is already in refused, and is passed over here.
|
// resolve is already in refused, and is passed over here.
|
||||||
|
plans := map[string]planned{}
|
||||||
for _, n := range out.nodes {
|
for _, n := range out.nodes {
|
||||||
plan, _, err := planFor(ctx, open, n.Name)
|
plan, settings, err := planFor(ctx, open, n.Name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if unresolvable(err) {
|
if unresolvable(err) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
return answers{}, err
|
return answers{}, err
|
||||||
}
|
}
|
||||||
|
plans[n.Name] = planned{plan, settings}
|
||||||
out.unheld = append(out.unheld, plan.Unheld...)
|
out.unheld = append(out.unheld, plan.Unheld...)
|
||||||
|
// And which of its modules a provider leaves out of its grants, for an identity too long
|
||||||
|
// for what the provision keeps (novox/hq ADR 0225) — judged from the consumer's own
|
||||||
|
// resolution, as the provider's composition judges it.
|
||||||
|
out.overflowing = append(out.overflowing, plan.Overflowing()...)
|
||||||
}
|
}
|
||||||
// And every consumer a provider says it keeps failing (novox/hq ADR 0224). Read from what the
|
// And every open condition (novox/hq to-be 45 §2): what the watchdogs, the self-check and the
|
||||||
// providers announced: nothing else in the mesh knows whether a provision is being made.
|
// providers' own words say is wrong — a provider failing a consumer among them (ADR 0224). Kept on
|
||||||
out.failing, err = failingProviders(ctx, inv)
|
// the bus; a process that cannot read them says so, and the mesh is then not called well.
|
||||||
if err != nil {
|
if out.conditions, err = openConditions(ctx); err != nil {
|
||||||
return answers{}, err
|
out.conditionsUnread = err.Error()
|
||||||
}
|
}
|
||||||
out.plans, err = inv.RecentPlans(ctx, 5)
|
out.plans, err = inv.RecentPlans(ctx, 5)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -432,6 +463,16 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
|||||||
}
|
}
|
||||||
// Whether the build seat takes work, for a plan waiting on it (novox/hq ADR 0219).
|
// Whether the build seat takes work, for a plan waiting on it (novox/hq ADR 0219).
|
||||||
out.paused = buildSeatPause(ctx, inv, out.plans)
|
out.paused = buildSeatPause(ctx, inv, out.plans)
|
||||||
|
// And how many repairs were done by hand this week (novox/hq to-be 45 §7) — where there is a bus
|
||||||
|
// to read the log from; a process with none has no log to count.
|
||||||
|
if _, onBus := broker.BusAddress(); onBus == nil {
|
||||||
|
n, unread := handActsThisWeek(ctx)
|
||||||
|
if unread != "" {
|
||||||
|
out.handActsUnread = unread
|
||||||
|
} else {
|
||||||
|
out.handActs = &n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// And which machines are not running what the mesh would send them. The same question as a
|
// And which machines are not running what the mesh would send them. The same question as a
|
||||||
// module being behind its source, one level down: that one says the catalogue is out of date,
|
// module being behind its source, one level down: that one says the catalogue is out of date,
|
||||||
@@ -443,7 +484,7 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
|||||||
// said nothing at all and `status --json` emitted prose to stderr and no JSON anywhere. The
|
// said nothing at all and `status --json` emitted prose to stderr and no JSON anywhere. The
|
||||||
// reason is kept and reported as data; every question that does not depend on it is still
|
// reason is kept and reported as data; every question that does not depend on it is still
|
||||||
// answered.
|
// answered.
|
||||||
would, err := wouldSend(ctx, open, out.nodes)
|
would, err := wouldSendFrom(ctx, open, out.nodes, plans)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
out.network = err.Error()
|
out.network = err.Error()
|
||||||
would = map[string]string{}
|
would = map[string]string{}
|
||||||
@@ -538,7 +579,8 @@ func untakenModules(ctx context.Context, inv *inventory.Inventory, nodes []inven
|
|||||||
func (a answers) well() bool {
|
func (a answers) well() bool {
|
||||||
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
|
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
|
||||||
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 &&
|
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 &&
|
||||||
len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.failing) == 0
|
len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.overflowing) == 0 &&
|
||||||
|
len(a.conditions) == 0 && a.conditionsUnread == ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// hostSplit is which machines report which host version, for every version more than one machine
|
// hostSplit is which machines report which host version, for every version more than one machine
|
||||||
|
|||||||
@@ -0,0 +1,193 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// `status` answered from a summary the serving controller keeps current (novox/hq to-be 45 Phase 0,
|
||||||
|
// §4's D9 and §8's health of the controller).
|
||||||
|
//
|
||||||
|
// **Asked, it was composed: every machine resolved, twice, while its caller waited.** On 2026-10-06
|
||||||
|
// the verb took eighteen seconds on a mesh of four machines, so its caller read "still running" and
|
||||||
|
// had to ask `calls` for the answer to "is the mesh alright" — the one question that must answer at
|
||||||
|
// once, and the one a self-check and a rollout gate will ask every few minutes. So the serving
|
||||||
|
// controller composes it in the background — at its start, after anything that changes what it says
|
||||||
|
// (a machine's report, a build, a verb that acts), and every minute regardless — and the verb answers
|
||||||
|
// the last composition at once, saying when it was composed and how long that took. A caller who
|
||||||
|
// needs it newer than that reads the time and asks again; nothing is answered as current that is not.
|
||||||
|
|
||||||
|
// statusEvery is how often the summary is composed with nothing having nudged it; statusSettle how
|
||||||
|
// long a nudge waits for the next, so a push answered by four machines is composed once.
|
||||||
|
var (
|
||||||
|
statusEvery = time.Minute
|
||||||
|
statusSettle = 2 * time.Second
|
||||||
|
// statusComposeWithin bounds one composition, so a store that hangs cannot stop the summary for
|
||||||
|
// good; the attempt is said as failed, and the last summary stands with its age.
|
||||||
|
statusComposeWithin = 2 * time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
// statusSummary is the last composed `status --json`, and when.
|
||||||
|
type statusSummary struct {
|
||||||
|
compose func(context.Context) ([]byte, error)
|
||||||
|
// every, settle and within are the clocks above, read once when it is made.
|
||||||
|
every, settle, within time.Duration
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
body []byte
|
||||||
|
composedAt time.Time
|
||||||
|
took time.Duration
|
||||||
|
failed string
|
||||||
|
failedAt time.Time
|
||||||
|
started time.Time
|
||||||
|
first chan struct{} // closed when the first attempt ends, either way
|
||||||
|
|
||||||
|
nudged chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newStatusSummary(compose func(context.Context) ([]byte, error)) *statusSummary {
|
||||||
|
return &statusSummary{compose: compose, started: time.Now(), first: make(chan struct{}),
|
||||||
|
nudged: make(chan struct{}, 1), every: statusEvery, settle: statusSettle, within: statusComposeWithin}
|
||||||
|
}
|
||||||
|
|
||||||
|
// statusFrom is the serving controller's summary; nil in any other process, where `status` is
|
||||||
|
// composed when asked, as at a shell.
|
||||||
|
var statusFrom *statusSummary
|
||||||
|
|
||||||
|
// nudge asks for a composition soon. Never blocks: one pending is as good as many.
|
||||||
|
func (s *statusSummary) nudge() {
|
||||||
|
if s == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case s.nudged <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// keep composes until ctx ends: now, on a nudge once things settle, and every statusEvery.
|
||||||
|
func (s *statusSummary) keep(ctx context.Context) {
|
||||||
|
once := sync.Once{}
|
||||||
|
for {
|
||||||
|
s.composeOnce(ctx)
|
||||||
|
once.Do(func() { close(s.first) })
|
||||||
|
timer := time.NewTimer(s.every)
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
timer.Stop()
|
||||||
|
return
|
||||||
|
case <-timer.C:
|
||||||
|
case <-s.nudged:
|
||||||
|
timer.Stop()
|
||||||
|
// Let what else is arriving arrive, then compose once for all of it.
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-time.After(s.settle):
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-s.nudged:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *statusSummary) composeOnce(ctx context.Context) {
|
||||||
|
start := time.Now()
|
||||||
|
asking, cancel := context.WithTimeout(ctx, s.within)
|
||||||
|
body, err := s.compose(asking)
|
||||||
|
cancel()
|
||||||
|
took := time.Since(start)
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if err != nil {
|
||||||
|
s.failed, s.failedAt = err.Error(), time.Now()
|
||||||
|
fmt.Printf("status could not be composed (after %s): %v — `status` answers the last summary, "+
|
||||||
|
"with its age\n", took.Round(time.Millisecond), err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.body, s.composedAt, s.took, s.failed = body, start, took, ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// answer is what the `status` verb answers: the last summary at once, the same document `status
|
||||||
|
// --json` prints, with when it was composed. Before the first composition has ended it waits for it,
|
||||||
|
// but never past the caller's window; a controller that has none says so and why, rather than
|
||||||
|
// answering an empty mesh as a well one.
|
||||||
|
func (s *statusSummary) answer(ctx context.Context) (any, error) {
|
||||||
|
wait := time.NewTimer(link.AnswerWithin - time.Second)
|
||||||
|
defer wait.Stop()
|
||||||
|
select {
|
||||||
|
case <-s.first:
|
||||||
|
case <-wait.C:
|
||||||
|
case <-ctx.Done():
|
||||||
|
}
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if s.body == nil {
|
||||||
|
why := "its first composition has not finished"
|
||||||
|
if s.failed != "" {
|
||||||
|
why = "it could not be composed: " + s.failed
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("this controller started %s ago and has no status to answer yet — %s. "+
|
||||||
|
"Ask again shortly", time.Since(s.started).Round(time.Second), why)
|
||||||
|
}
|
||||||
|
var parsed any
|
||||||
|
_ = json.Unmarshal(s.body, &parsed)
|
||||||
|
out := map[string]any{
|
||||||
|
"output": string(s.body), "ok": true, "answer": parsed,
|
||||||
|
"composed": s.composedAt.UTC().Format(time.RFC3339),
|
||||||
|
"age": time.Since(s.composedAt).Round(time.Second).String(),
|
||||||
|
"composedIn": s.took.Round(time.Millisecond).String(),
|
||||||
|
"note": "composed by the serving controller at its start, after each report, build or act, and " +
|
||||||
|
"every minute; answered at once from the last composition",
|
||||||
|
}
|
||||||
|
if s.failed != "" && s.failedAt.After(s.composedAt) {
|
||||||
|
out["lastAttemptFailed"] = fmt.Sprintf("%s: %s — this summary is the last that could be composed",
|
||||||
|
s.failedAt.UTC().Format(time.RFC3339), s.failed)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// composeStatus is `status --json`, composed in this process against its stores.
|
||||||
|
func composeStatus(open *stores) func(context.Context) ([]byte, error) {
|
||||||
|
return func(ctx context.Context) ([]byte, error) {
|
||||||
|
asked, err := theThreeQuestions(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return statusAsJSON(asked)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// readingVerbs are the verbs that only read; after any other, what `status` says may have changed, so the summary is
|
||||||
|
// composed again; a verb that only reads leaves it alone, or a console polling `nodes` would keep the
|
||||||
|
// controller composing for ever.
|
||||||
|
var readingVerbs = map[string]bool{
|
||||||
|
"tools": true, "calls": true, "status": true, "nodes": true, "node": true, "modules": true,
|
||||||
|
"seats": true, "builds": true, "plan": true, "queue": true, "durations": true, "hand-acts": true,
|
||||||
|
"doctor": true, "conditions": true,
|
||||||
|
}
|
||||||
|
|
||||||
|
// nudgingListener is the enrolment, nudging the summary when a machine said something new.
|
||||||
|
type nudgingListener struct {
|
||||||
|
link.Enrolment
|
||||||
|
summary *statusSummary
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, error) {
|
||||||
|
// A declaration refused as older than the one the machine holds is counted (novox/hq to-be 45 S13).
|
||||||
|
if link.IsStaleRefusal(report.Refused) {
|
||||||
|
link.StaleRefusals.Refused(report.Node, time.Now())
|
||||||
|
}
|
||||||
|
news, err := l.Enrolment.Heard(ctx, report)
|
||||||
|
if news {
|
||||||
|
l.summary.nudge()
|
||||||
|
}
|
||||||
|
return news, err
|
||||||
|
}
|
||||||
@@ -0,0 +1,152 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// quickly shortens the summary's clocks for one test.
|
||||||
|
func quickly(t *testing.T) {
|
||||||
|
t.Helper()
|
||||||
|
every, settle := statusEvery, statusSettle
|
||||||
|
statusEvery, statusSettle = time.Hour, 10*time.Millisecond
|
||||||
|
t.Cleanup(func() { statusEvery, statusSettle = every, settle })
|
||||||
|
}
|
||||||
|
|
||||||
|
// **`status` answers in full within ten seconds, five times in a row** (novox/hq to-be 45 Phase 0,
|
||||||
|
// D9) — however long composing it takes. On 2026-10-06 composing took eighteen seconds and the
|
||||||
|
// verb answered "still running"; from the summary it answers at once, in full, saying when.
|
||||||
|
func TestStatusAnswersAtOnceHoweverLongComposingTakes(t *testing.T) {
|
||||||
|
quickly(t)
|
||||||
|
var composed atomic.Int32
|
||||||
|
slow := make(chan struct{})
|
||||||
|
s := newStatusSummary(func(ctx context.Context) ([]byte, error) {
|
||||||
|
if composed.Add(1) > 1 {
|
||||||
|
<-slow // every composition after the first outlasts any caller
|
||||||
|
}
|
||||||
|
return []byte(`{"wrong":[],"machines":4}`), nil
|
||||||
|
})
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
defer close(slow)
|
||||||
|
go s.keep(ctx)
|
||||||
|
for i := 0; i < 5; i++ {
|
||||||
|
s.nudge() // a composition is under way and does not finish
|
||||||
|
start := time.Now()
|
||||||
|
got, err := s.answer(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if took := time.Since(start); took > time.Second {
|
||||||
|
t.Fatalf("answer %d took %s", i+1, took)
|
||||||
|
}
|
||||||
|
m := got.(map[string]any)
|
||||||
|
if m["answer"].(map[string]any)["machines"] != float64(4) || m["composed"] == "" || m["ok"] != true {
|
||||||
|
t.Fatalf("answer %d was not in full: %v", i+1, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The first composition is waited for, never past the caller's window; a controller with none yet
|
||||||
|
// says so rather than answering an empty mesh as a well one.
|
||||||
|
func TestStatusBeforeItsFirstCompositionSaysSo(t *testing.T) {
|
||||||
|
quickly(t)
|
||||||
|
was := link.AnswerWithin
|
||||||
|
link.AnswerWithin = 1100 * time.Millisecond
|
||||||
|
t.Cleanup(func() { link.AnswerWithin = was })
|
||||||
|
never := make(chan struct{})
|
||||||
|
defer close(never)
|
||||||
|
s := newStatusSummary(func(context.Context) ([]byte, error) { <-never; return nil, nil })
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
go s.keep(ctx)
|
||||||
|
start := time.Now()
|
||||||
|
_, err := s.answer(ctx)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "has no status to answer yet") {
|
||||||
|
t.Fatalf("answered %v", err)
|
||||||
|
}
|
||||||
|
if took := time.Since(start); took > link.AnswerWithin {
|
||||||
|
t.Fatalf("waited %s, past the caller's window", took)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A nudge composes it again, once for several close together; a failed composition leaves the last
|
||||||
|
// summary standing and says it is the last that could be composed.
|
||||||
|
func TestANudgeComposesAgainAndAFailureKeepsTheLastSummary(t *testing.T) {
|
||||||
|
quickly(t)
|
||||||
|
var composed atomic.Int32
|
||||||
|
fail := atomic.Bool{}
|
||||||
|
s := newStatusSummary(func(context.Context) ([]byte, error) {
|
||||||
|
n := composed.Add(1)
|
||||||
|
if fail.Load() {
|
||||||
|
return nil, errors.New("the store did not answer")
|
||||||
|
}
|
||||||
|
body, _ := json.Marshal(map[string]any{"n": n})
|
||||||
|
return body, nil
|
||||||
|
})
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
go s.keep(ctx)
|
||||||
|
if _, err := s.answer(ctx); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
s.nudge()
|
||||||
|
s.nudge()
|
||||||
|
s.nudge()
|
||||||
|
waitFor(t, func() bool { return composed.Load() == 2 })
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
if n := composed.Load(); n != 2 {
|
||||||
|
t.Fatalf("three nudges together composed %d times after the first", n-1)
|
||||||
|
}
|
||||||
|
fail.Store(true)
|
||||||
|
s.nudge()
|
||||||
|
waitFor(t, func() bool { return composed.Load() == 3 })
|
||||||
|
waitFor(t, func() bool {
|
||||||
|
got, err := s.answer(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m := got.(map[string]any)
|
||||||
|
return m["lastAttemptFailed"] != nil && m["answer"].(map[string]any)["n"] == float64(2)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// The seat's `status` answers from the summary when this process keeps one.
|
||||||
|
func TestTheStatusVerbAnswersFromTheSummary(t *testing.T) {
|
||||||
|
quickly(t)
|
||||||
|
s := newStatusSummary(func(context.Context) ([]byte, error) { return []byte(`{"from":"summary"}`), nil })
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
go s.keep(ctx)
|
||||||
|
statusFrom = s
|
||||||
|
t.Cleanup(func() { statusFrom = nil })
|
||||||
|
handlers, _, err := seatToolHandlers()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, err := handlers["status"](ctx, json.RawMessage(`{}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.(map[string]any)["answer"].(map[string]any)["from"] != "summary" {
|
||||||
|
t.Fatalf("answered %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func waitFor(t *testing.T, ok func() bool) {
|
||||||
|
t.Helper()
|
||||||
|
deadline := time.Now().Add(3 * time.Second)
|
||||||
|
for !ok() {
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatal("never happened")
|
||||||
|
}
|
||||||
|
time.Sleep(5 * time.Millisecond)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -73,22 +73,6 @@ func migrate(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
fmt.Printf("provided %s\n", m.Module)
|
fmt.Printf("provided %s\n", m.Module)
|
||||||
}
|
}
|
||||||
// And what an earlier release shipped and this one does not goes (novox/hq ADR 0226) — unless a
|
|
||||||
// machine still has it, which is said rather than overridden.
|
|
||||||
var shipped []string
|
|
||||||
for _, m := range provided {
|
|
||||||
shipped = append(shipped, m.Module)
|
|
||||||
}
|
|
||||||
retired, kept, err := inv.RetireUnshipped(ctx, shipped)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
for _, name := range retired {
|
|
||||||
fmt.Printf("retired %s: the control plane no longer ships it\n", name)
|
|
||||||
}
|
|
||||||
for name, why := range kept {
|
|
||||||
fmt.Printf("kept %s, which the control plane no longer ships: %s\n", name, why)
|
|
||||||
}
|
|
||||||
// The seats the mesh ships with, into the table that now holds the set (novox/hq ADR 0122).
|
// The seats the mesh ships with, into the table that now holds the set (novox/hq ADR 0122).
|
||||||
// Idempotent: fills an empty table on first boot, adds a seat a release ships, and leaves an
|
// Idempotent: fills an empty table on first boot, adds a seat a release ships, and leaves an
|
||||||
// operator's changes in the table as they are.
|
// operator's changes in the table as they are.
|
||||||
|
|||||||
@@ -75,21 +75,25 @@ func TestAPersonClosesAStuckPlan(t *testing.T) {
|
|||||||
if err := open.inventory.SavePlan(ctx, stuck); err != nil {
|
if err := open.inventory.SavePlan(ctx, stuck); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := plansCommand(ctx, []string{"close", stuck.ID}); err != nil {
|
if err := plansCommand(ctx, []string{"close", stuck.ID}); err == nil || !strings.Contains(err.Error(), "--why") {
|
||||||
|
t.Fatalf("a plan was closed by hand without saying why: %v", err)
|
||||||
|
}
|
||||||
|
if err := plansCommand(ctx, []string{"close", stuck.ID, "--why", "its report will not come"}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
closed, err := open.inventory.PlanByID(ctx, stuck.ID)
|
closed, err := open.inventory.PlanByID(ctx, stuck.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if closed.State != inventory.PlanFailed || !strings.Contains(closed.Note, "closed by hand") {
|
if closed.State != inventory.PlanFailed || !strings.Contains(closed.Note, "closed by hand") ||
|
||||||
|
!strings.Contains(closed.Note, "its report will not come") {
|
||||||
t.Fatalf("the plan was left %s: %q", closed.State, closed.Note)
|
t.Fatalf("the plan was left %s: %q", closed.State, closed.Note)
|
||||||
}
|
}
|
||||||
if err := plansCommand(ctx, []string{"close", stuck.ID}); err == nil {
|
if err := plansCommand(ctx, []string{"close", stuck.ID, "--why", "again"}); err == nil {
|
||||||
t.Fatal("a plan already closed was closed again")
|
t.Fatal("a plan already closed was closed again")
|
||||||
}
|
}
|
||||||
if argv, err := argvFor("plans", map[string]any{"close": stuck.ID}); err != nil ||
|
if argv, err := argvFor("plans", map[string]any{"close": stuck.ID, "why": "w"}); err != nil ||
|
||||||
!reflect.DeepEqual(argv, []string{"plans", "close", stuck.ID}) {
|
!reflect.DeepEqual(argv, []string{"plans", "close", stuck.ID, "--why", "w"}) {
|
||||||
t.Fatalf("the seat's verb does not close a plan: %v %v", argv, err)
|
t.Fatalf("the seat's verb does not close a plan: %v %v", argv, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,549 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"slices"
|
||||||
|
"sort"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The watchdogs (novox/hq to-be 45 §3): every row of the signals table, run over what the serving
|
||||||
|
// controller knows, every half minute.
|
||||||
|
//
|
||||||
|
// **One loop, one gathering, every row.** The facts are gathered once a tick — the store's machines,
|
||||||
|
// plans and durations, the bus's queue and consumers, what this process heard — and each row's watch
|
||||||
|
// is a pure reading of them, which is what lets the test generated from the table suppress a signal by
|
||||||
|
// changing a fact. A part that cannot be gathered makes the rows that read it blind: each says so as
|
||||||
|
// a condition of its own (`probe.<row>.failed`) and keeps what it raised before, because a watchdog
|
||||||
|
// that cannot see must not read as one that sees nothing wrong (ADR 0227 rule 4).
|
||||||
|
//
|
||||||
|
// **The watchdogs are themselves watched.** Each tick is recorded; the self-check (doctor.go) fails
|
||||||
|
// its probe DW when the ticks stop, and the watchdogs raise S10 when the self-check stops — two loops
|
||||||
|
// watching each other, and mesh-watcher on a second machine watching the self-check's heartbeat for
|
||||||
|
// the case both stop with the process.
|
||||||
|
|
||||||
|
// watchEvery is how often the watchdogs run.
|
||||||
|
var watchEvery = 30 * time.Second
|
||||||
|
|
||||||
|
// signalFacts is what one tick of the watchdogs reads.
|
||||||
|
type signalFacts struct {
|
||||||
|
now time.Time
|
||||||
|
started time.Time
|
||||||
|
// host is the machine this controller runs on, as the mesh names it, for a condition about itself.
|
||||||
|
host string
|
||||||
|
|
||||||
|
machines []machineFacts
|
||||||
|
machinesErr error
|
||||||
|
// toolsHeardFrom is when this process began hearing node tools: one not heard since is silent
|
||||||
|
// since then at the most.
|
||||||
|
toolsHeardFrom time.Time
|
||||||
|
|
||||||
|
plans []planFacts
|
||||||
|
plansErr error
|
||||||
|
|
||||||
|
loop loopFacts
|
||||||
|
loopErr error
|
||||||
|
|
||||||
|
merges []missedMerge
|
||||||
|
mergesPassed time.Time
|
||||||
|
mergesErr error
|
||||||
|
|
||||||
|
asks []askFacts
|
||||||
|
asksErr error
|
||||||
|
|
||||||
|
calls []link.Call
|
||||||
|
|
||||||
|
standings []conditions.Condition
|
||||||
|
standingsErr error
|
||||||
|
|
||||||
|
advisories []link.Advisory
|
||||||
|
lostConsumers map[string]bool
|
||||||
|
advisoriesErr error
|
||||||
|
|
||||||
|
selfCheck selfCheckFacts
|
||||||
|
|
||||||
|
staleRefusals map[string]int
|
||||||
|
}
|
||||||
|
|
||||||
|
type machineFacts struct {
|
||||||
|
name string
|
||||||
|
control bool
|
||||||
|
// lastHeard is the store's last word from it, any word; zero when never.
|
||||||
|
lastHeard time.Time
|
||||||
|
// every is the heartbeat interval it said; zero when it said none.
|
||||||
|
every time.Duration
|
||||||
|
power link.PowerState
|
||||||
|
// sentAt is when it was last sent a declaration; reportedCurrent whether it has reported that one.
|
||||||
|
sentAt time.Time
|
||||||
|
reportedCurrent bool
|
||||||
|
reportedAt time.Time
|
||||||
|
// lastApply is its newest measured apply.
|
||||||
|
lastApply time.Duration
|
||||||
|
// tools is whether node-tools is assigned there; toolsHeard and toolsEvery its heartbeat.
|
||||||
|
tools bool
|
||||||
|
toolsHeard time.Time
|
||||||
|
toolsEvery time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// asleep says the machine said it would be away and has not said it is back (ADR 0211).
|
||||||
|
func (m machineFacts) asleep() bool { return m.power.Away() }
|
||||||
|
|
||||||
|
type planFacts struct {
|
||||||
|
id, repository, commit string
|
||||||
|
tier, tiers int
|
||||||
|
entered time.Time
|
||||||
|
bound time.Duration
|
||||||
|
waiting string
|
||||||
|
paused bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type loopFacts struct {
|
||||||
|
took time.Time
|
||||||
|
pending uint64
|
||||||
|
where string
|
||||||
|
}
|
||||||
|
|
||||||
|
type askFacts struct {
|
||||||
|
id, seat, what, state, on string
|
||||||
|
since time.Time
|
||||||
|
bound time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
type selfCheckFacts struct {
|
||||||
|
last time.Time
|
||||||
|
every time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// watchdogs is the loop and what it needs.
|
||||||
|
type watchdogs struct {
|
||||||
|
open *stores
|
||||||
|
server *link.Server
|
||||||
|
js *broker.JetStream
|
||||||
|
keeper *conditions.Keeper
|
||||||
|
doctor *doctor
|
||||||
|
|
||||||
|
started time.Time
|
||||||
|
// acting says this controller is the one acting, not one standing by (link.Holding): a controller
|
||||||
|
// standing by hears no heartbeat and would call every machine silent.
|
||||||
|
acting func() bool
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
ticked time.Time
|
||||||
|
last *signalFacts
|
||||||
|
failed string
|
||||||
|
}
|
||||||
|
|
||||||
|
// lastTick is when the watchdogs last finished a tick.
|
||||||
|
func (w *watchdogs) lastTick() time.Time {
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
return w.ticked
|
||||||
|
}
|
||||||
|
|
||||||
|
// lastFacts is the facts of the newest tick, for `doctor signals`; nil before the first.
|
||||||
|
func (w *watchdogs) lastFacts() *signalFacts {
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
return w.last
|
||||||
|
}
|
||||||
|
|
||||||
|
// keep runs the watchdogs until ctx ends.
|
||||||
|
func (w *watchdogs) keep(ctx context.Context) {
|
||||||
|
tick := time.NewTicker(watchEvery)
|
||||||
|
defer tick.Stop()
|
||||||
|
for {
|
||||||
|
w.tick(ctx)
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-tick.C:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// tick is one run of every row.
|
||||||
|
func (w *watchdogs) tick(ctx context.Context) {
|
||||||
|
if w.acting != nil && !w.acting() {
|
||||||
|
// Standing by: nothing seen, nothing said, and the tick counted — this process's self-check
|
||||||
|
// is not the one that matters while another acts.
|
||||||
|
w.mu.Lock()
|
||||||
|
w.ticked = time.Now()
|
||||||
|
w.mu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
running, cancel := context.WithTimeout(ctx, watchEvery)
|
||||||
|
defer cancel()
|
||||||
|
w.see(running, w.gather(running))
|
||||||
|
}
|
||||||
|
|
||||||
|
// see runs every watched row over one gathering, keeps what each found, and records the tick.
|
||||||
|
func (w *watchdogs) see(running context.Context, f *signalFacts) {
|
||||||
|
var problems []string
|
||||||
|
var blind []conditions.Observation
|
||||||
|
for _, row := range watchedRows() {
|
||||||
|
if err := row.needs(f); err != nil {
|
||||||
|
blind = append(blind, blindRow(row, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := w.keeper.Reconcile(running, row.Row, row.watch(f)); err != nil {
|
||||||
|
problems = append(problems, row.Row+": "+err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The rows that could not see, said; the ones that see again, cleared.
|
||||||
|
if err := w.keeper.Reconcile(running, sourceWatchdogs, blind); err != nil {
|
||||||
|
problems = append(problems, err.Error())
|
||||||
|
}
|
||||||
|
// A provider no longer assigned where it ran: its standing is resolved by the assignment.
|
||||||
|
if f.standingsErr == nil && w.open != nil {
|
||||||
|
if err := unassignedProviders(running, w.open.inventory, w.keeper, f.standings); err != nil {
|
||||||
|
problems = append(problems, "S8: "+err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := w.keeper.EndSilences(running); err != nil {
|
||||||
|
problems = append(problems, err.Error())
|
||||||
|
}
|
||||||
|
failed := ""
|
||||||
|
if len(problems) > 0 {
|
||||||
|
failed = fmt.Sprintf("%v", problems)
|
||||||
|
}
|
||||||
|
w.mu.Lock()
|
||||||
|
said := w.failed
|
||||||
|
w.ticked, w.last, w.failed = time.Now(), f, failed
|
||||||
|
w.mu.Unlock()
|
||||||
|
if failed != said {
|
||||||
|
if failed != "" {
|
||||||
|
fmt.Printf("the watchdogs could not keep what they saw: %s\n", failed)
|
||||||
|
} else if said != "" {
|
||||||
|
fmt.Println("the watchdogs keep what they see again")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// sourceWatchdogs is what raises a blind row's condition.
|
||||||
|
const sourceWatchdogs = "watchdogs"
|
||||||
|
|
||||||
|
// blindRow is a row whose facts could not be gathered, as a condition of its own.
|
||||||
|
func blindRow(row signalRow, err error) conditions.Observation {
|
||||||
|
return conditions.Observation{Scope: conditions.ScopeProbe, ID: row.Row, Kind: "probe-failed", Token: "failed",
|
||||||
|
Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("the watchdog of %s (%s) cannot see: what it reads could not be read, so nothing "+
|
||||||
|
"it would raise can be — and nothing it raised before is cleared", row.Row, row.Signal),
|
||||||
|
Said: firstLine(err.Error())}
|
||||||
|
}
|
||||||
|
|
||||||
|
// gather reads every fact a tick needs. Each part's failure is kept beside it, never an empty part.
|
||||||
|
func (w *watchdogs) gather(ctx context.Context) *signalFacts {
|
||||||
|
now := time.Now()
|
||||||
|
f := &signalFacts{now: now, started: w.started, toolsHeardFrom: link.ToolsBeats.Started(), calls: link.Calls.Running(),
|
||||||
|
staleRefusals: link.StaleRefusals.Within(now.Add(-staleRefusalsWithin)), lostConsumers: map[string]bool{}}
|
||||||
|
if w.doctor != nil {
|
||||||
|
f.selfCheck = selfCheckFacts{last: w.doctor.lastRunEnded(), every: doctorEvery}
|
||||||
|
}
|
||||||
|
inv := w.open.inventory
|
||||||
|
f.host = controlHost(ctx, inv)
|
||||||
|
f.machines, f.machinesErr = w.gatherMachines(ctx, inv, now)
|
||||||
|
f.plans, f.plansErr = gatherPlans(ctx, inv, now)
|
||||||
|
f.loop, f.loopErr = w.gatherLoop()
|
||||||
|
f.mergesPassed, f.merges, f.mergesErr = watchedMerges.last()
|
||||||
|
if f.mergesErr == nil && !f.mergesPassed.IsZero() && now.Sub(f.mergesPassed) > 3*mergeCatchUpEvery {
|
||||||
|
f.mergesErr = fmt.Errorf("the catch-up of merges has not passed since %s", f.mergesPassed.UTC().Format(time.RFC3339))
|
||||||
|
}
|
||||||
|
f.asks, f.asksErr = w.gatherAsks(ctx, inv)
|
||||||
|
if open, err := w.keeper.Open(ctx); err != nil {
|
||||||
|
f.standingsErr = err
|
||||||
|
} else {
|
||||||
|
f.standings = providerStandings(open)
|
||||||
|
}
|
||||||
|
f.advisories = link.Advisories.Since(now.Add(-advisoryQuiet))
|
||||||
|
f.lostConsumers, f.advisoriesErr = w.lostConsumers(ctx, f.advisories)
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|
||||||
|
// controlHost is the machine running the controller, as the mesh names it: the one the controller
|
||||||
|
// module is assigned to, or this process's host name where that is not one machine.
|
||||||
|
func controlHost(ctx context.Context, inv *inventory.Inventory) string {
|
||||||
|
if on, err := inv.Running(ctx, "mesh-controller"); err == nil && len(on) == 1 {
|
||||||
|
return on[0]
|
||||||
|
}
|
||||||
|
host, _ := os.Hostname()
|
||||||
|
return host
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *watchdogs) gatherMachines(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]machineFacts, error) {
|
||||||
|
nodes, err := inv.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the machines cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
reports, err := inv.LastReports(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("what each machine last reported cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
reported := map[string]inventory.Reported{}
|
||||||
|
for _, r := range reports {
|
||||||
|
reported[r.Node] = r
|
||||||
|
}
|
||||||
|
control, err := inv.Running(ctx, "mesh-controller")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("where the controller runs cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
tooled, err := inv.Running(ctx, broker.RuntimeModule)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("where the node tools run cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
applies, err := inv.Durations(ctx, inventory.DurationApply, now.Add(-7*24*time.Hour))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the measured applies cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
lastApply := map[string]time.Duration{}
|
||||||
|
for _, d := range applies { // oldest first: the last one read is the newest
|
||||||
|
lastApply[d.Node] = d.Took
|
||||||
|
}
|
||||||
|
var out []machineFacts
|
||||||
|
anyPast := false
|
||||||
|
for _, n := range nodes {
|
||||||
|
m := machineFacts{name: n.Name, control: slices.Contains(control, n.Name), lastHeard: n.LastSeen,
|
||||||
|
lastApply: lastApply[n.Name], tools: slices.Contains(tooled, n.Name)}
|
||||||
|
if beat, ok := link.HostBeats.Of(n.Name); ok {
|
||||||
|
m.every = beat.Every
|
||||||
|
}
|
||||||
|
if beat, ok := link.ToolsBeats.Of(n.Name); ok {
|
||||||
|
m.toolsHeard, m.toolsEvery = beat.At, beat.Every
|
||||||
|
}
|
||||||
|
if r, ok := reported[n.Name]; ok {
|
||||||
|
if r.Sent != nil {
|
||||||
|
m.sentAt = *r.Sent
|
||||||
|
}
|
||||||
|
if r.At != nil {
|
||||||
|
m.reportedAt = *r.At
|
||||||
|
}
|
||||||
|
m.reportedCurrent = r.Current
|
||||||
|
}
|
||||||
|
if !m.lastHeard.IsZero() && now.Sub(m.lastHeard) > heartbeatBound(m.every) {
|
||||||
|
anyPast = true
|
||||||
|
}
|
||||||
|
if m.tools && now.Sub(later(m.toolsHeard, link.ToolsBeats.Started())) > heartbeatBound(m.toolsEvery) {
|
||||||
|
anyPast = true
|
||||||
|
}
|
||||||
|
out = append(out, m)
|
||||||
|
}
|
||||||
|
// What a machine past its bound last said of its power, read only then: a machine that said it
|
||||||
|
// is asleep is not lost (ADR 0211). Unreadable is said: a sleeping laptop is then called silent,
|
||||||
|
// which is the louder mistake and the right one to make.
|
||||||
|
if anyPast && w.server != nil {
|
||||||
|
states, err := w.server.PowerStates(ctx, now.Add(-7*24*time.Hour))
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("what machines said of their power cannot be read, so a sleeping one is called silent: %v\n", err)
|
||||||
|
}
|
||||||
|
for i := range out {
|
||||||
|
out[i].power = states[out[i].name]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// gatherPlans is every open plan, its tier's bound from what was measured, and what it waits on.
|
||||||
|
func gatherPlans(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]planFacts, error) {
|
||||||
|
plans, err := inv.OpenPlans(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the open plans cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
if len(plans) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
tiers, err := inv.Durations(ctx, inventory.DurationPlanTier, now.Add(-14*24*time.Hour))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the measured plan tiers cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
measured := map[string][]time.Duration{}
|
||||||
|
for _, d := range tiers {
|
||||||
|
measured[d.Subject] = append(measured[d.Subject], d.Took)
|
||||||
|
}
|
||||||
|
pause := buildSeatPause(ctx, inv, plans)
|
||||||
|
var out []planFacts
|
||||||
|
for _, p := range plans {
|
||||||
|
_, paused := pausedWaiting(p, pause, now)
|
||||||
|
out = append(out, planFacts{id: p.ID, repository: p.Repository, commit: p.Commit, tier: p.Tier,
|
||||||
|
tiers: len(p.Tiers), entered: p.TierEntered, bound: max(tierAtLeast, 3*p90(measured[p.Repository])),
|
||||||
|
waiting: planLineWith(p, now, pause), paused: paused})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// p90 is the ninetieth percentile of measurements; zero for none.
|
||||||
|
func p90(took []time.Duration) time.Duration {
|
||||||
|
if len(took) == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
sorted := append([]time.Duration(nil), took...)
|
||||||
|
sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] })
|
||||||
|
return sorted[int(0.9*float64(len(sorted)-1))]
|
||||||
|
}
|
||||||
|
|
||||||
|
// gatherLoop is when the event loop last took a message and what its consumers hold.
|
||||||
|
func (w *watchdogs) gatherLoop() (loopFacts, error) {
|
||||||
|
f := loopFacts{took: link.Loop.Last()}
|
||||||
|
if w.js == nil {
|
||||||
|
return f, errors.New("this controller is not on the bus")
|
||||||
|
}
|
||||||
|
var where []string
|
||||||
|
for _, c := range broker.MeshConsumers() {
|
||||||
|
info, err := w.js.Context().ConsumerInfo(c.Stream, c.Name)
|
||||||
|
if err != nil {
|
||||||
|
return f, fmt.Errorf("the controller's consumer on %s cannot be read: %w", c.Stream, err)
|
||||||
|
}
|
||||||
|
if held := info.NumPending + uint64(info.NumAckPending); held > 0 {
|
||||||
|
f.pending += held
|
||||||
|
where = append(where, fmt.Sprintf("%d on %s", held, c.Stream))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
f.where = fmt.Sprint(where)
|
||||||
|
return f, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// gatherAsks is every ask in the build seat's queue that is in flight or dead, with its bound.
|
||||||
|
func (w *watchdogs) gatherAsks(ctx context.Context, inv *inventory.Inventory) ([]askFacts, error) {
|
||||||
|
if w.js == nil {
|
||||||
|
return nil, errors.New("this controller is not on the bus")
|
||||||
|
}
|
||||||
|
entries, err := inv.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the catalogue cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
seat := buildSeatAmong(entries)
|
||||||
|
q, err := link.ReadQueue(ctx, w.js, seat)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
builds, err := inv.Durations(ctx, inventory.DurationBuild, time.Now().Add(-14*24*time.Hour))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the measured builds cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
var took []time.Duration
|
||||||
|
for _, d := range builds {
|
||||||
|
took = append(took, d.Took)
|
||||||
|
}
|
||||||
|
bound := askDefault
|
||||||
|
if len(took) > 0 {
|
||||||
|
bound = max(askAtLeast, 3*p90(took))
|
||||||
|
}
|
||||||
|
var out []askFacts
|
||||||
|
for _, a := range q.Asks {
|
||||||
|
if a.State != link.AskInFlight && a.State != link.AskDead {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
since := a.Started
|
||||||
|
if since.IsZero() {
|
||||||
|
since = a.AskedAt
|
||||||
|
}
|
||||||
|
what := a.Repository
|
||||||
|
if a.Path != "" {
|
||||||
|
what += " " + a.Path
|
||||||
|
}
|
||||||
|
out = append(out, askFacts{id: a.ID, seat: seat, what: what, state: a.State, on: a.On, since: since, bound: bound})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// lostConsumers is, of the consumers the bus said were deleted, each that is still missing and that
|
||||||
|
// the mesh expects: a consumer removed because its module was unassigned is not lost.
|
||||||
|
func (w *watchdogs) lostConsumers(ctx context.Context, heard []link.Advisory) (map[string]bool, error) {
|
||||||
|
out := map[string]bool{}
|
||||||
|
var deleted []link.Advisory
|
||||||
|
for _, a := range heard {
|
||||||
|
if a.Kind == link.AdvisoryConsumerLost {
|
||||||
|
deleted = append(deleted, a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(deleted) == 0 {
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
if w.js == nil {
|
||||||
|
return nil, errors.New("this controller is not on the bus")
|
||||||
|
}
|
||||||
|
_, expected, err := expectedBusObjects(ctx, w.open.inventory)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
want := map[string]bool{}
|
||||||
|
for _, c := range expected {
|
||||||
|
want[c.Stream+"."+c.Name] = true
|
||||||
|
}
|
||||||
|
for _, a := range deleted {
|
||||||
|
_, err := w.js.Context().ConsumerInfo(a.Stream, a.Consumer)
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, nats.ErrConsumerNotFound):
|
||||||
|
out[a.Stream+"."+a.Consumer] = want[a.Stream+"."+a.Consumer]
|
||||||
|
case err != nil:
|
||||||
|
return nil, fmt.Errorf("whether %s exists cannot be read: %w", link.ConsumerInWords(a.Stream, a.Consumer), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// later is the later of two moments.
|
||||||
|
func later(a, b time.Time) time.Time {
|
||||||
|
if a.After(b) {
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
// watchTheMesh opens the condition store and starts the watchdogs, the bus's advisories and the
|
||||||
|
// self-check, for the serving controller; the returned function stops them. Nil when the store could
|
||||||
|
// not be opened, which is said.
|
||||||
|
func watchTheMesh(ctx context.Context, open *stores, server *link.Server, bus link.OverNATS) func() {
|
||||||
|
keeper, err := keeperOn(ctx, bus.Conn)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("the condition store could NOT be opened, so nothing that goes wrong is kept or said, and "+
|
||||||
|
"status says the conditions cannot be read: %v\n", err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
conditionsFrom = keeper
|
||||||
|
logf := func(format string, args ...any) { fmt.Printf(format+"\n", args...) }
|
||||||
|
stopHearing, err := server.HearAdvisories(logf)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("what the bus says about itself cannot be heard (S9 is blind): %v\n", err)
|
||||||
|
stopHearing = func() {}
|
||||||
|
}
|
||||||
|
host := controlHost(ctx, open.inventory)
|
||||||
|
w := &watchdogs{open: open, server: server, js: server.JetStream(), keeper: keeper, started: time.Now(),
|
||||||
|
acting: link.Holding}
|
||||||
|
d := &doctor{open: open, js: server.JetStream(), keeper: keeper, teller: bus, watchdogs: w, host: host}
|
||||||
|
w.doctor = d
|
||||||
|
doctorFrom = d
|
||||||
|
watching, stop := context.WithCancel(ctx)
|
||||||
|
go w.keep(watching)
|
||||||
|
go d.keep(watching)
|
||||||
|
fmt.Printf("watching the mesh: %d signal(s) every %s, %d probe(s) every %s; what is wrong is kept in %s "+
|
||||||
|
"and said as %s events\n", len(watchedRows()), watchEvery, len(runnableProbes()), doctorEvery,
|
||||||
|
broker.ConditionsBucket, conditions.Seat)
|
||||||
|
return func() {
|
||||||
|
stop()
|
||||||
|
stopHearing()
|
||||||
|
flushing, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
keeper.Close(flushing)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// runnableProbes are the probes the registry runs.
|
||||||
|
func runnableProbes() []probe {
|
||||||
|
var out []probe
|
||||||
|
for _, p := range probeRegistry {
|
||||||
|
if p.run != nil {
|
||||||
|
out = append(out, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -5,7 +5,9 @@ go 1.26.0
|
|||||||
require (
|
require (
|
||||||
github.com/jackc/pgx/v5 v5.10.0
|
github.com/jackc/pgx/v5 v5.10.0
|
||||||
github.com/nats-io/nats.go v1.54.0
|
github.com/nats-io/nats.go v1.54.0
|
||||||
|
github.com/novox/mesh-host v0.0.0
|
||||||
golang.org/x/crypto v0.57.0
|
golang.org/x/crypto v0.57.0
|
||||||
|
golang.org/x/net v0.58.0
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
@@ -15,8 +17,15 @@ require (
|
|||||||
github.com/klauspost/compress v1.20.0 // indirect
|
github.com/klauspost/compress v1.20.0 // indirect
|
||||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||||
github.com/nats-io/nuid v1.0.1 // indirect
|
github.com/nats-io/nuid v1.0.1 // indirect
|
||||||
golang.org/x/net v0.58.0 // indirect
|
|
||||||
golang.org/x/sync v0.23.0 // indirect
|
golang.org/x/sync v0.23.0 // indirect
|
||||||
golang.org/x/sys v0.48.0 // indirect
|
golang.org/x/sys v0.48.0 // indirect
|
||||||
golang.org/x/text v0.42.0 // indirect
|
golang.org/x/text v0.42.0 // indirect
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// The node-engine's own validator (mesh-host/validate, novox/hq to-be 45 D1): one validator, the host's.
|
||||||
|
// The host's module path names no forge a build can fetch from, so the module is read from the one
|
||||||
|
// that holds it, at the host's commit — **once, by whoever moves the pin, into vendor/**, which is
|
||||||
|
// committed. Every build (the build agent's `go build`, the Dockerfile) compiles from vendor/ and
|
||||||
|
// fetches nothing; go refuses to build when vendor/ and this file disagree, so a pin moved without
|
||||||
|
// `go mod vendor` fails loudly, at once, everywhere.
|
||||||
|
replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261006081854-6953b5bafdb2
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
git.novox.be/novox/mesh-host v0.0.0-20261006081854-6953b5bafdb2 h1:b4+F4tTfrPkuJTST+rkwIHpEb4mkVJR9158hr6nnc4g=
|
||||||
|
git.novox.be/novox/mesh-host v0.0.0-20261006081854-6953b5bafdb2/go.mod h1:VlilMCRZ5yyNXg7SNigNBLr0Gt32jrGw5KSNq5JAVYs=
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
|||||||
@@ -0,0 +1,143 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The controller's own key-value buckets (novox/hq to-be 45 §1, §6, §7).
|
||||||
|
//
|
||||||
|
// **What the controller must remember across its own restart, it keeps on the bus.** A call's
|
||||||
|
// outcome lived in the memory of the process that served it (novox/hq issue 265), so a controller
|
||||||
|
// replaced while a push ran answered "no such call" for the one thing its caller had been told to
|
||||||
|
// ask about. The bus already outlives the controller and is the shape ADR 0201 gives a module's
|
||||||
|
// current state: one value per key, written by one owner, read by anybody granted it. These are the
|
||||||
|
// controller's, written by it alone — the writers table of to-be 45 §1 — and asserted on every start
|
||||||
|
// like the streams, so a bus raised from nothing has them before the first call is served.
|
||||||
|
|
||||||
|
// CallsBucket keeps every call of the mesh's own verbs and what came of it; HandActsBucket every act
|
||||||
|
// a person did by hand, with why; ConditionsBucket every condition open now (to-be 45 §2), one key
|
||||||
|
// each, and ConditionHistoryBucket every transition of one — raised, changed, silenced, cleared —
|
||||||
|
// for ninety days.
|
||||||
|
//
|
||||||
|
// **The history is a bucket of its own** because its keys expire and an open condition's must not:
|
||||||
|
// a bucket has one age for every key, and a condition open longer than the history is kept would
|
||||||
|
// otherwise vanish from the store while still true.
|
||||||
|
var (
|
||||||
|
CallsBucket = BucketName(ControllerSeat, "calls")
|
||||||
|
HandActsBucket = BucketName(ControllerSeat, "hand-acts")
|
||||||
|
ConditionsBucket = BucketName(ControllerSeat, "conditions")
|
||||||
|
ConditionHistoryBucket = BucketName(ControllerSeat, "condition-history")
|
||||||
|
)
|
||||||
|
|
||||||
|
// The bounds to-be 45 §6 sets for calls: the last thousand, or fourteen days, whichever is fewer.
|
||||||
|
// A call is two keys — its record, and its answer apart so a listing does not read every answer —
|
||||||
|
// so the stream holds twice as many messages as it keeps calls.
|
||||||
|
const (
|
||||||
|
KeptCallsDurably = 1000
|
||||||
|
CallsKeptFor = 14 * 24 * time.Hour
|
||||||
|
// CallAnswerBytes is the most of one answer kept: a whole declaration is far smaller, and an
|
||||||
|
// answer larger is cut and says so.
|
||||||
|
CallAnswerBytes = 64 << 10
|
||||||
|
// HandActsKeptFor is as long as a condition's history (to-be 45 §2): an act by hand is read
|
||||||
|
// back beside what it addressed.
|
||||||
|
HandActsKeptFor = 90 * 24 * time.Hour
|
||||||
|
// ConditionHistoryKeptFor is how long a condition's transitions are kept (to-be 45 §2).
|
||||||
|
ConditionHistoryKeptFor = 90 * 24 * time.Hour
|
||||||
|
)
|
||||||
|
|
||||||
|
// IsControllerBucket says a bucket is the controller's own, not a module's state nothing declares.
|
||||||
|
func IsControllerBucket(bucket string) bool {
|
||||||
|
return bucket == CallsBucket || bucket == HandActsBucket || bucket == ConditionsBucket ||
|
||||||
|
bucket == ConditionHistoryBucket
|
||||||
|
}
|
||||||
|
|
||||||
|
// ControllerBuckets are the controller's own buckets, in the order they are asserted.
|
||||||
|
func ControllerBuckets() []string {
|
||||||
|
return []string{CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ControllerBucketsAsserter is what raising the controller's buckets needs of a connection.
|
||||||
|
type ControllerBucketsAsserter interface {
|
||||||
|
EnsureControllerBuckets() error
|
||||||
|
}
|
||||||
|
|
||||||
|
// EnsureControllerBuckets creates the controller's buckets if absent and brings their options to
|
||||||
|
// match. An update, never a delete: what they hold is the record of what the mesh was asked.
|
||||||
|
func (j *JetStream) EnsureControllerBuckets() error {
|
||||||
|
js, err := jetstream.New(j.conn)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||||
|
Bucket: CallsBucket,
|
||||||
|
Description: "the calls of the mesh's own verbs and what came of each (novox/hq to-be 45 §6, issue " +
|
||||||
|
"265): written by the controller alone, read through `calls`; the last thousand, or fourteen days",
|
||||||
|
History: 1,
|
||||||
|
TTL: CallsKeptFor,
|
||||||
|
MaxValueSize: CallAnswerBytes + 4<<10,
|
||||||
|
MaxBytes: 2 * KeptCallsDurably * (CallAnswerBytes + 4<<10),
|
||||||
|
Storage: jetstream.FileStorage,
|
||||||
|
}); err != nil {
|
||||||
|
return fmt.Errorf("asserting bucket %s: %w", CallsBucket, err)
|
||||||
|
}
|
||||||
|
// **The count, on the stream under the bucket.** A bucket has an age and a size and no count;
|
||||||
|
// the stream it is made of does, and with one value per key the oldest message is the oldest
|
||||||
|
// call. Asserted after the bucket, every time, because asserting the bucket writes the stream's
|
||||||
|
// configuration whole and puts the count back to none.
|
||||||
|
stream, err := js.Stream(ctx, "KV_"+CallsBucket)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("reading the stream under %s: %w", CallsBucket, err)
|
||||||
|
}
|
||||||
|
cfg := stream.CachedInfo().Config
|
||||||
|
if cfg.MaxMsgs != 2*KeptCallsDurably {
|
||||||
|
cfg.MaxMsgs = 2 * KeptCallsDurably
|
||||||
|
cfg.Discard = jetstream.DiscardOld
|
||||||
|
if _, err := js.UpdateStream(ctx, cfg); err != nil {
|
||||||
|
return fmt.Errorf("bounding %s to the last %d calls: %w", CallsBucket, KeptCallsDurably, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||||
|
Bucket: HandActsBucket,
|
||||||
|
Description: "every act a person did by hand, with why (novox/hq to-be 45 §7): written by the " +
|
||||||
|
"controller's repairing verbs and `hand-act record`, read through `hand-acts`",
|
||||||
|
History: 1,
|
||||||
|
TTL: HandActsKeptFor,
|
||||||
|
MaxValueSize: 16 << 10,
|
||||||
|
MaxBytes: 64 << 20,
|
||||||
|
Storage: jetstream.FileStorage,
|
||||||
|
}); err != nil {
|
||||||
|
return fmt.Errorf("asserting bucket %s: %w", HandActsBucket, err)
|
||||||
|
}
|
||||||
|
// **No age on the open conditions.** A condition is removed when observation clears it and at no
|
||||||
|
// other moment: one that expired would be a fault the store forgot while it was still true.
|
||||||
|
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||||
|
Bucket: ConditionsBucket,
|
||||||
|
Description: "every condition open now, one key each (novox/hq to-be 45 §2): written by the " +
|
||||||
|
"controller alone, raised and cleared by observation, read through `conditions`",
|
||||||
|
History: 1,
|
||||||
|
MaxValueSize: 64 << 10,
|
||||||
|
MaxBytes: 64 << 20,
|
||||||
|
Storage: jetstream.FileStorage,
|
||||||
|
}); err != nil {
|
||||||
|
return fmt.Errorf("asserting bucket %s: %w", ConditionsBucket, err)
|
||||||
|
}
|
||||||
|
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||||
|
Bucket: ConditionHistoryBucket,
|
||||||
|
Description: "every transition of a condition — raised, changed, silenced, cleared — kept ninety " +
|
||||||
|
"days (novox/hq to-be 45 §2): written by the controller alone, read through `conditions history`",
|
||||||
|
History: 1,
|
||||||
|
TTL: ConditionHistoryKeptFor,
|
||||||
|
MaxValueSize: 64 << 10,
|
||||||
|
MaxBytes: 256 << 20,
|
||||||
|
Storage: jetstream.FileStorage,
|
||||||
|
}); err != nil {
|
||||||
|
return fmt.Errorf("asserting bucket %s: %w", ConditionHistoryBucket, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **The controller may write every bucket it writes** (novox/hq to-be 45 §1, issue 269). Writing a
|
||||||
|
// key is a publish to the bucket's own subject, which the management interface's grant does not
|
||||||
|
// cover: the cancelled sets' writes timed out for want of this, and the controller's own buckets
|
||||||
|
// would have.
|
||||||
|
func TestTheControllerMayWriteEveryBucketItWrites(t *testing.T) {
|
||||||
|
p, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := []string{"$KV." + CallsBucket + ".>", "$KV." + HandActsBucket + ".>"}
|
||||||
|
for _, seat := range seatsTheControllerAsks {
|
||||||
|
if hasCancelledSet(seat) {
|
||||||
|
want = append(want, "$KV."+CancelledSetName(seat)+".>")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, subject := range want {
|
||||||
|
if !slices.Contains(p.Publish, subject) {
|
||||||
|
t.Errorf("the controller may not publish %s, so it cannot write that bucket", subject)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if slices.Contains(p.Publish, "$KV.>") {
|
||||||
|
t.Error("the controller may write any bucket, a module's state included")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A machine's node tools may say they are there, as that machine and no other** (novox/hq to-be 45
|
||||||
|
// S11), and the controller may hear the bus's advisories and ask who answers — read-only, named.
|
||||||
|
func TestTheWatchedSignalsMayBeSaidAndHeard(t *testing.T) {
|
||||||
|
tools, err := PermissionsFor(Principal{Kind: KindNodeTools, Node: "anchor", Module: RuntimeModule, PasswordHash: "x"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !slices.Contains(tools.Publish, "mesh.control.anchor.tools-alive") {
|
||||||
|
t.Error("the node tools may not say they are there")
|
||||||
|
}
|
||||||
|
for _, s := range tools.Publish {
|
||||||
|
if strings.Contains(s, "tools-alive") && s != "mesh.control.anchor.tools-alive" {
|
||||||
|
t.Errorf("the node tools may say %s", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, s := range BusAdvisories {
|
||||||
|
if !slices.Contains(controller.Subscribe, s) {
|
||||||
|
t.Errorf("the controller may not hear %s", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !slices.Contains(controller.Publish, "$SRV.INFO") || slices.Contains(controller.Subscribe, "$JS.EVENT.>") {
|
||||||
|
t.Error("the controller may not ask who answers, or hears every API call")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -125,6 +125,16 @@ type Principal struct {
|
|||||||
// goes with the retired seat row.
|
// goes with the retired seat row.
|
||||||
var seatsTheControllerAsks = []string{"node-build-agent", "mesh-build-machine"}
|
var seatsTheControllerAsks = []string{"node-build-agent", "mesh-build-machine"}
|
||||||
|
|
||||||
|
// SeatVerb is one verb of one seat, on every machine holding it.
|
||||||
|
type SeatVerb struct{ Seat, Verb string }
|
||||||
|
|
||||||
|
// VerbsTheSelfCheckAsks are the seat verbs the controller's self-check and watchdogs call (novox/hq
|
||||||
|
// to-be 45 §4): D8 reads every machine's ban list. **Named one by one, and the test that holds them
|
||||||
|
// to the probe registry is the reason they cannot drift** — a probe that calls a verb its grant does
|
||||||
|
// not name is refused by the bus on every run (found live on 2026-10-06: D8 timed out on each
|
||||||
|
// machine, refused). Asked of any machine (`.*`), read-only verbs, nothing else of the seat.
|
||||||
|
var VerbsTheSelfCheckAsks = []SeatVerb{{Seat: "node-intrusion-prevention", Verb: "banned"}}
|
||||||
|
|
||||||
// perMachineEvents are a node-scoped seat's events about the holder itself, whose last token is the
|
// perMachineEvents are a node-scoped seat's events about the holder itself, whose last token is the
|
||||||
// holder's machine (novox/hq ADR 0219): `paused.<node>`, the build agent saying whether it takes work.
|
// holder's machine (novox/hq ADR 0219): `paused.<node>`, the build agent saying whether it takes work.
|
||||||
var perMachineEvents = map[string]bool{"paused.*": true}
|
var perMachineEvents = map[string]bool{"paused.*": true}
|
||||||
@@ -237,6 +247,12 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
|||||||
// building, kill it, pause it, resume it. The queue is the controller's to show and to
|
// building, kill it, pause it, resume it. The queue is the controller's to show and to
|
||||||
// change, and what one machine is doing with an ask it took only that machine can say.
|
// change, and what one machine is doing with an ask it took only that machine can say.
|
||||||
pub = append(pub, "mesh.seat."+seat+".tool.>")
|
pub = append(pub, "mesh.seat."+seat+".tool.>")
|
||||||
|
// **And its cancelled set** (novox/hq ADR 0219, issue 269): a cancel writes the ask's id
|
||||||
|
// there before it deletes the ask, and a write is a publish to the bucket's subject, which
|
||||||
|
// `$JS.API.>` does not cover — so every cancel timed out, refused by this list.
|
||||||
|
if hasCancelledSet(seat) {
|
||||||
|
pub = append(pub, "$KV."+CancelledSetName(seat)+".>")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// **And what the mesh says it did** (novox/hq ADR 0134). The control plane states its own
|
// **And what the mesh says it did** (novox/hq ADR 0134). The control plane states its own
|
||||||
// facts under the seat it holds, because a role's events belong to the role and keep their
|
// facts under the seat it holds, because a role's events belong to the role and keep their
|
||||||
@@ -260,6 +276,13 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
|||||||
sub = append(sub, "mesh.seat."+ControllerSeat+".tool.>")
|
sub = append(sub, "mesh.seat."+ControllerSeat+".tool.>")
|
||||||
// And says so (novox/hq ADR 0197): it answers discovery for the seat it serves.
|
// And says so (novox/hq ADR 0197): it answers discovery for the seat it serves.
|
||||||
sub = append(sub, announcing(ControllerSeat)...)
|
sub = append(sub, announcing(ControllerSeat)...)
|
||||||
|
// And the verbs the self-check reads with, of any machine's holder (novox/hq to-be 45 §4).
|
||||||
|
for _, v := range VerbsTheSelfCheckAsks {
|
||||||
|
pub = append(pub, "mesh.seat."+v.Seat+".tool."+v.Verb+".*")
|
||||||
|
}
|
||||||
|
// And asks who answers (novox/hq to-be 45 §4, D3): the self-check finds every seat's holder by
|
||||||
|
// the same discovery the console reads. The question only; the answers come to its own inbox.
|
||||||
|
pub = append(pub, "$SRV.INFO")
|
||||||
|
|
||||||
// The events it reacts to, and its ack subject on the stream they arrive from
|
// The events it reacts to, and its ack subject on the stream they arrive from
|
||||||
// (streams.go). **Each named, not a pattern**: `mesh.mod.*.event.>` would make the
|
// (streams.go). **Each named, not a pattern**: `mesh.mod.*.event.>` would make the
|
||||||
@@ -288,6 +311,18 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
|||||||
// enrolments and can reach nothing else.
|
// enrolments and can reach nothing else.
|
||||||
pub = append(pub, "_INBOX."+enrolmentPrefix+".>")
|
pub = append(pub, "_INBOX."+enrolmentPrefix+".>")
|
||||||
|
|
||||||
|
// **And its own buckets** (novox/hq to-be 45 §1): the calls it served and the acts done by
|
||||||
|
// hand, which it alone writes. A put is a publish to the bucket's subject, which `$JS.API.>`
|
||||||
|
// does not cover; each bucket named, not `$KV.>`, which would let it write any module's state.
|
||||||
|
for _, bucket := range ControllerBuckets() {
|
||||||
|
pub = append(pub, "$KV."+bucket+".>")
|
||||||
|
}
|
||||||
|
// **And what the bus says about itself, read-only** (novox/hq to-be 45 §3, S9): a durable
|
||||||
|
// consumer that gave up on a message, or one that was deleted. The server already publishes
|
||||||
|
// both in the mesh's own account; the controller says each as a condition in the mesh's words.
|
||||||
|
// Named, not `$JS.EVENT.>`: the other advisories are every API call the mesh makes.
|
||||||
|
sub = append(sub, BusAdvisories...)
|
||||||
|
|
||||||
case KindPerson:
|
case KindPerson:
|
||||||
// Tools, and nothing else. Every subject a person may publish is a tool call; a person
|
// Tools, and nothing else. Every subject a person may publish is a tool call; a person
|
||||||
// who could publish an event would be able to claim a module said something.
|
// who could publish an event would be able to claim a module said something.
|
||||||
@@ -499,6 +534,9 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
|||||||
// that varies is the module, so the pattern is the machine's own assignments.
|
// that varies is the module, so the pattern is the machine's own assignments.
|
||||||
sub = append(sub, "mesh.assignment."+p.Node+".*")
|
sub = append(sub, "mesh.assignment."+p.Node+".*")
|
||||||
pub = append(pub, "$JS.API.DIRECT.GET."+AssignmentsStream+".mesh.assignment."+p.Node+".*")
|
pub = append(pub, "$JS.API.DIRECT.GET."+AssignmentsStream+".mesh.assignment."+p.Node+".*")
|
||||||
|
// And that it is there (novox/hq to-be 45 §3, S11): its own heartbeat, under its machine's
|
||||||
|
// name and no other's, on core NATS like the host's.
|
||||||
|
pub = append(pub, "mesh.control."+p.Node+".tools-alive")
|
||||||
// And every tool on the mesh (ADR 0175, decision 5): any node may call any tool on any
|
// And every tool on the mesh (ADR 0175, decision 5): any node may call any tool on any
|
||||||
// node, as the console already could — the runtime is the console's serving mode.
|
// node, as the console already could — the runtime is the console's serving mode.
|
||||||
invoked, err := invokedSubjects([]string{"*"})
|
invoked, err := invokedSubjects([]string{"*"})
|
||||||
|
|||||||
@@ -160,8 +160,9 @@ func RaiseBuckets(a BucketAsserter, buckets []Bucket) (undeclared []string, err
|
|||||||
return nil, fmt.Errorf("listing the bus's state: %w", err)
|
return nil, fmt.Errorf("listing the bus's state: %w", err)
|
||||||
}
|
}
|
||||||
for _, n := range names {
|
for _, n := range names {
|
||||||
// A seat's cancelled set is the mesh's own (novox/hq ADR 0219), not a module's state.
|
// A seat's cancelled set is the mesh's own (novox/hq ADR 0219), not a module's state; so are
|
||||||
if !declared[n] && !IsCancelledSet(n) {
|
// the controller's own buckets (novox/hq to-be 45 §1).
|
||||||
|
if !declared[n] && !IsCancelledSet(n) && !IsControllerBucket(n) {
|
||||||
undeclared = append(undeclared, n)
|
undeclared = append(undeclared, n)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -20,12 +21,15 @@ func TestTheFactsTheGrantPermitsAreTheFactsTheMeshStates(t *testing.T) {
|
|||||||
t.Fatalf("the grant is written for the %q seat and the mesh states its facts under %q",
|
t.Fatalf("the grant is written for the %q seat and the mesh states its facts under %q",
|
||||||
broker.ControllerSeat, link.MeshControllerSeat)
|
broker.ControllerSeat, link.MeshControllerSeat)
|
||||||
}
|
}
|
||||||
for _, event := range []string{link.KeyApplied, link.KeyRefused, link.KeyBuiltBefore} {
|
// And what is wrong, as it changes, and the self-check's heartbeat (novox/hq to-be 45 §2, §4).
|
||||||
|
states := append([]string{link.KeyApplied, link.KeyRefused, link.KeyBuiltBefore}, conditions.Events...)
|
||||||
|
states = append(states, conditions.HeartbeatEvent)
|
||||||
|
for _, event := range states {
|
||||||
if !slices.Contains(broker.ControllerStates, event) {
|
if !slices.Contains(broker.ControllerStates, event) {
|
||||||
t.Errorf("the mesh states %q and its account may not publish it", event)
|
t.Errorf("the mesh states %q and its account may not publish it", event)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(broker.ControllerStates) != 3 {
|
if len(broker.ControllerStates) != len(states) {
|
||||||
t.Errorf("the grant permits %v, which is more than the mesh states", broker.ControllerStates)
|
t.Errorf("the grant permits %v, which is more than the mesh states", broker.ControllerStates)
|
||||||
}
|
}
|
||||||
// **And the seat says it.** A seat carries the protocol of its role (novox/hq ADR 0129), so the
|
// **And the seat says it.** A seat carries the protocol of its role (novox/hq ADR 0129), so the
|
||||||
|
|||||||
@@ -201,7 +201,23 @@ const ControllerName = "controller"
|
|||||||
// something to say.
|
// something to say.
|
||||||
const ControllerSeat = "mesh-controller"
|
const ControllerSeat = "mesh-controller"
|
||||||
|
|
||||||
var ControllerStates = []string{"applied", "refused", "built-before"}
|
var ControllerStates = []string{"applied", "refused", "built-before",
|
||||||
|
// What is wrong, said as it changes (novox/hq to-be 45 §2): a condition raised, changed in
|
||||||
|
// severity, resolver or silence, and cleared. The operator-channel's holder and any other surface
|
||||||
|
// consume them; the controller tells nobody itself.
|
||||||
|
"condition-raised", "condition-changed", "condition-cleared",
|
||||||
|
// And the self-check's heartbeat, at the end of every run (to-be 45 §4, S10): watched from a
|
||||||
|
// machine that is not the control node, so the controller going quiet is itself said.
|
||||||
|
"doctor-heartbeat"}
|
||||||
|
|
||||||
|
// BusAdvisories are what the bus server says about the mesh's own account that the controller
|
||||||
|
// reads (novox/hq to-be 45 §3, S9): a durable consumer that handed a message over as often as it
|
||||||
|
// may and gave up on it, and one that was deleted. Read-only: an advisory is the server's to
|
||||||
|
// publish, and the controller's subscription changes nothing on the bus.
|
||||||
|
var BusAdvisories = []string{
|
||||||
|
"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>",
|
||||||
|
"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>",
|
||||||
|
}
|
||||||
|
|
||||||
// ControllerFollows are the events the controller reacts to: the catalogue saying a module's
|
// ControllerFollows are the events the controller reacts to: the catalogue saying a module's
|
||||||
// current version moved, and a catalogue that has just started saying it may have missed builds.
|
// current version moved, and a catalogue that has just started saying it may have missed builds.
|
||||||
|
|||||||
+2
-2
@@ -24,8 +24,8 @@ accounts {
|
|||||||
jetstream: enabled
|
jetstream: enabled
|
||||||
users = [
|
users = [
|
||||||
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
||||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>"] }
|
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] }
|
||||||
subscribe: { allow: ["$JS.API.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
|
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
|
||||||
allow_responses: { max: 1, ttl: "1m" }
|
allow_responses: { max: 1, ttl: "1m" }
|
||||||
} }
|
} }
|
||||||
{ user: "enrol.one", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: {
|
{ user: "enrol.one", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: {
|
||||||
|
|||||||
@@ -119,3 +119,41 @@ func TestEveryCatalogueStoreSaysHowItIsBackedUp(t *testing.T) {
|
|||||||
t.Fatal("no module in the catalogue provides a store, so this proved nothing")
|
t.Fatal("no module in the catalogue provides a store, so this proved nothing")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestEveryCatalogueIdentityFitsWhatItRequires is ADR 0225's check over the real catalogue: every
|
||||||
|
// module's identity, on the longest machine name, fits the bound of every provision it wants. An
|
||||||
|
// overflow fails here, in the pull request that introduces it, rather than on the provider's machine
|
||||||
|
// the first time a real machine's name meets the module's (issue 263).
|
||||||
|
func TestEveryCatalogueIdentityFitsWhatItRequires(t *testing.T) {
|
||||||
|
root := catalogueRoot(t)
|
||||||
|
found, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||||
|
if err != nil || len(found) == 0 {
|
||||||
|
t.Fatalf("no manifests under %s: %v", root, err)
|
||||||
|
}
|
||||||
|
shelf := Shelf{}
|
||||||
|
for _, p := range found {
|
||||||
|
raw, err := os.ReadFile(p)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: %v", p, err)
|
||||||
|
}
|
||||||
|
m, err := ParseManifest(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: %v", p, err)
|
||||||
|
}
|
||||||
|
shelf[m.Module] = m
|
||||||
|
}
|
||||||
|
for _, p := range IdentityProblems(shelf, DefaultLongestMachine) {
|
||||||
|
t.Error(p)
|
||||||
|
}
|
||||||
|
// The night it was found: the resolver provision bounds nothing, and the object store still 20.
|
||||||
|
if dns, ok := shelf["dnsmasq"]; ok {
|
||||||
|
if b := dns.IdentityBoundOf("wildcard-resolution"); b.Bounded() {
|
||||||
|
t.Errorf("the resolver provision bounds its consumers' identities: %+v", b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if store, ok := shelf["minio"]; ok {
|
||||||
|
if b := store.IdentityBoundOf("s3-bucket"); b.Max != 20 {
|
||||||
|
t.Errorf("the object store's access key is not bounded at 20: %+v", b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -114,7 +114,9 @@ func consumerInto(value, as string) (string, error) {
|
|||||||
// asDNSLabel writes a minted identity as a DNS label.
|
// asDNSLabel writes a minted identity as a DNS label.
|
||||||
//
|
//
|
||||||
// The mesh's identities are already lower-case letters, digits and `_` (ConsumerIdentity), and
|
// The mesh's identities are already lower-case letters, digits and `_` (ConsumerIdentity), and
|
||||||
// already short enough for the tightest backend they reach (CheckIdentity, twenty characters). So
|
// already inside the bound of the provision serving them: a provider that serves one as a DNS label
|
||||||
|
// bounds it at 63 or less (CheckServes, novox/hq ADR 0225), and one that serves it at all without
|
||||||
|
// saying is held to twenty (IdentityBoundOf). So
|
||||||
// this is the separator and nothing else — no lower-casing of what is already lower case, no
|
// this is the separator and nothing else — no lower-casing of what is already lower case, no
|
||||||
// truncation to a limit the identity is already inside, no padding of a name that is already long
|
// truncation to a limit the identity is already inside, no padding of a name that is already long
|
||||||
// enough. Each of those would be the mesh guessing at a rule it has not been given.
|
// enough. Each of those would be the mesh guessing at a rule it has not been given.
|
||||||
@@ -141,11 +143,31 @@ func CheckServes(m Manifest) []string {
|
|||||||
problems = append(problems, fmt.Sprintf(
|
problems = append(problems, fmt.Sprintf(
|
||||||
"%s serves %s, and the value it serves as %q %s", m.Module, provision, key, err))
|
"%s serves %s, and the value it serves as %q %s", m.Module, provision, key, err))
|
||||||
}
|
}
|
||||||
|
// A label longer than DNS keeps is not truncated here (asDNSLabel), so the offer's own
|
||||||
|
// bound has to keep the identity inside one (ADR 0225).
|
||||||
|
if strings.Contains(text, "${consumer:as:dns}") {
|
||||||
|
if b := m.IdentityBoundOf(provision); !b.Bounded() || b.Max > dnsLabelLimit {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s serves %s's consumers their identity as a DNS label in %q, and bounds "+
|
||||||
|
"that identity at %s: a label keeps %d — state an `identity` of at most %d",
|
||||||
|
m.Module, provision, key, boundWords(b), dnsLabelLimit, dnsLabelLimit))
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return problems
|
return problems
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// dnsLabelLimit is the longest DNS label (RFC 1035 §2.3.4).
|
||||||
|
const dnsLabelLimit = 63
|
||||||
|
|
||||||
|
func boundWords(b IdentityBound) string {
|
||||||
|
if !b.Bounded() {
|
||||||
|
return "nothing"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%d", b.Max)
|
||||||
|
}
|
||||||
|
|
||||||
func sortedServes(serves map[string]map[string]any) []string {
|
func sortedServes(serves map[string]map[string]any) []string {
|
||||||
out := make([]string, 0, len(serves))
|
out := make([]string, 0, len(serves))
|
||||||
for k := range serves {
|
for k := range serves {
|
||||||
|
|||||||
@@ -170,6 +170,10 @@ type Rendering struct {
|
|||||||
// rather than resolved, because who consumes a node is a fact about the rest of the mesh and
|
// rather than resolved, because who consumes a node is a fact about the rest of the mesh and
|
||||||
// resolution answers questions about one machine.
|
// resolution answers questions about one machine.
|
||||||
Grants []Grant
|
Grants []Grant
|
||||||
|
// Withheld is every consumer left out of Grants because its identity overflows the provision's
|
||||||
|
// bound (novox/hq ADR 0225). Composed into nothing; carried so the machine's declaration can say
|
||||||
|
// whom it does not serve, and why, beside what it does.
|
||||||
|
Withheld []Overflow
|
||||||
|
|
||||||
// Ports is where this machine puts what each module needs reachable, by module and by the
|
// Ports is where this machine puts what each module needs reachable, by module and by the
|
||||||
// port the software itself uses (novox/hq ADR 0038).
|
// port the software itself uses (novox/hq ADR 0038).
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package catalogue
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"regexp"
|
"regexp"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -56,13 +57,36 @@ func ConsumerIdentity(node, module string) string {
|
|||||||
return IdentityPrefix + clean(node) + "_" + clean(module)
|
return IdentityPrefix + clean(node) + "_" + clean(module)
|
||||||
}
|
}
|
||||||
|
|
||||||
// identityLimit is the shortest identifier limit among the systems these names reach: an S3 access
|
// IdentityBound is the longest consumer identity one provision's backend keeps, and what keeps it
|
||||||
// key's 20 (novox/hq 04-ISSUES/010). PostgreSQL keeps 63 and MinIO 20, so 20 is the one that binds —
|
// (novox/hq ADR 0049, refined by ADR 0225). Max zero means no bound: the provision keeps no name
|
||||||
// the comment used to name PostgreSQL and was wrong. A name over it is refused, with the remedy a
|
// derived from its consumer, or keeps one in something with no limit the mesh need respect.
|
||||||
// short slug (ADR 0049), not silently cut to fit.
|
type IdentityBound struct {
|
||||||
const identityLimit = 20
|
// Max is the longest identity that backend keeps, in characters; zero for none.
|
||||||
|
Max int `json:"max,omitempty"`
|
||||||
|
// In is what keeps it, in words a refusal can quote: "an S3 access key", "a PostgreSQL role".
|
||||||
|
In string `json:"in,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
// CheckIdentity refuses an identity that would not fit the tightest backend a consumer reaches.
|
// Bounded is whether this bound refuses anything.
|
||||||
|
func (b IdentityBound) Bounded() bool { return b.Max > 0 }
|
||||||
|
|
||||||
|
// DefaultIdentityLimit is the bound on a provision whose provider receives its consumers and does
|
||||||
|
// not say how long a name it keeps: an S3 access key's 20 (novox/hq 04-ISSUES/010, 034), the
|
||||||
|
// tightest backend the mesh has met. It was the bound on every provision until ADR 0225; it stays
|
||||||
|
// the bound on any that has not said otherwise, because a provider that is told each consumer's
|
||||||
|
// identity may create a name from it in a backend nobody has measured.
|
||||||
|
const DefaultIdentityLimit = 20
|
||||||
|
|
||||||
|
// DefaultIdentityBound is DefaultIdentityLimit, said as a bound.
|
||||||
|
var DefaultIdentityBound = IdentityBound{Max: DefaultIdentityLimit,
|
||||||
|
In: "a backend that has not said its limit (the tightest known, an S3 access key's)"}
|
||||||
|
|
||||||
|
// identityLimit is the bound CheckIdentity applies, for a caller that does not know which provision
|
||||||
|
// the identity is for.
|
||||||
|
const identityLimit = DefaultIdentityLimit
|
||||||
|
|
||||||
|
// CheckIdentity refuses an identity that would not fit the tightest backend the mesh knows. A caller
|
||||||
|
// that knows the provision uses CheckIdentityWithin and that provision's own bound (ADR 0225).
|
||||||
//
|
//
|
||||||
// **Truncation is not an error in most of these systems** — a name past the limit is cut to fit and
|
// **Truncation is not an error in most of these systems** — a name past the limit is cut to fit and
|
||||||
// the statement succeeds, so two consumers agreeing for the first N bytes would become one login
|
// the statement succeeds, so two consumers agreeing for the first N bytes would become one login
|
||||||
@@ -70,12 +94,127 @@ const identityLimit = 20
|
|||||||
// name and is the only thing that can choose another. The remedy is a first-class one: give the
|
// name and is the only thing that can choose another. The remedy is a first-class one: give the
|
||||||
// module a short `slug` (ADR 0049), or shorten the machine's name.
|
// module a short `slug` (ADR 0049), or shorten the machine's name.
|
||||||
func CheckIdentity(node, module string) error {
|
func CheckIdentity(node, module string) error {
|
||||||
|
return CheckIdentityWithin(node, module, IdentityBound{Max: identityLimit, In: "a backend (an S3 access key)"})
|
||||||
|
}
|
||||||
|
|
||||||
|
// CheckIdentityWithin refuses an identity that would not fit one provision's bound, and accepts any
|
||||||
|
// identity for a provision with none (novox/hq ADR 0225).
|
||||||
|
func CheckIdentityWithin(node, module string, bound IdentityBound) error {
|
||||||
|
if !bound.Bounded() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
got := ConsumerIdentity(node, module)
|
got := ConsumerIdentity(node, module)
|
||||||
if len(got) <= identityLimit {
|
if len(got) <= bound.Max {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"%s on %s is identified as %q, %d characters where a backend (an S3 access key) keeps %d — "+
|
"%s on %s is identified as %q, %d characters where %s keeps %d — "+
|
||||||
"give the module a shorter `slug` or shorten the machine's name",
|
"give the module a shorter `slug` or shorten the machine's name",
|
||||||
module, node, got, len(got), identityLimit)
|
module, node, got, len(got), bound.In, bound.Max)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Overflow is one consumer whose identity does not fit the provision it requires: left out of its
|
||||||
|
// provider's grants and reported, never a reason to refuse the provider's machine (ADR 0225).
|
||||||
|
type Overflow struct {
|
||||||
|
// Provision is what was required, Provider the machine answering it.
|
||||||
|
Provision string `json:"provision"`
|
||||||
|
Provider string `json:"provider"`
|
||||||
|
// Consumer is the machine, Module the module on it that required it.
|
||||||
|
Consumer string `json:"consumer"`
|
||||||
|
Module string `json:"module"`
|
||||||
|
// Identity is the name the mesh derived, and Bound what it overflows.
|
||||||
|
Identity string `json:"identity"`
|
||||||
|
Bound IdentityBound `json:"bound"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o Overflow) String() string {
|
||||||
|
return fmt.Sprintf("%s on %s requires %s from %s and is identified as %q, %d characters where %s "+
|
||||||
|
"keeps %d — left out of %s's grants until the module's `slug` is shorter",
|
||||||
|
o.Module, o.Consumer, o.Provision, o.Provider, o.Identity, len(o.Identity), o.Bound.In,
|
||||||
|
o.Bound.Max, o.Provider)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Overflowing is every requirement of this machine's modules whose identity overflows the bound of
|
||||||
|
// the provision answering it. The same judgement the provider's composition makes before it grants
|
||||||
|
// (grantsFor), made from the consumer's side so `status` can say it about every machine.
|
||||||
|
func (r Resolution) Overflowing() []Overflow {
|
||||||
|
slugs := map[string]string{}
|
||||||
|
for _, m := range r.Modules {
|
||||||
|
slugs[m.Module] = m.Slug
|
||||||
|
}
|
||||||
|
var out []Overflow
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for _, n := range r.Needs {
|
||||||
|
if n.ByRecord || !n.Identity.Bounded() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
source := IdentitySource(slugs[n.For], n.For)
|
||||||
|
if CheckIdentityWithin(r.Node, source, n.Identity) == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
key := n.Name + "\x00" + n.From + "\x00" + n.For
|
||||||
|
if seen[key] {
|
||||||
|
continue // one line per requirement, however many local names it has
|
||||||
|
}
|
||||||
|
seen[key] = true
|
||||||
|
out = append(out, Overflow{Provision: n.Name, Provider: n.From, Consumer: r.Node, Module: n.For,
|
||||||
|
Identity: ConsumerIdentity(r.Node, source), Bound: n.Identity})
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(i, j int) bool {
|
||||||
|
if out[i].Module != out[j].Module {
|
||||||
|
return out[i].Module < out[j].Module
|
||||||
|
}
|
||||||
|
return out[i].Provision < out[j].Provision
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultLongestMachine is the machine name the catalogue check judges identities on when it is not
|
||||||
|
// told one: the longest name of the mesh this catalogue is written for, so a catalogue that passes
|
||||||
|
// passes on every machine that mesh has. `module check --longest-machine-name` says another mesh's;
|
||||||
|
// a mesh that names a longer machine raises this in the same change (ADR 0225).
|
||||||
|
const DefaultLongestMachine = 6
|
||||||
|
|
||||||
|
// IdentityProblems is every module whose identity would overflow a provision it wants, on a machine
|
||||||
|
// whose name is `longestMachine` characters — judged before merge, over the catalogue alone, so the
|
||||||
|
// pull request that introduces an overflow is the one refused (novox/hq ADR 0225, issue 263). A
|
||||||
|
// provision no module in the shelf offers is not judged: its bound is not known here.
|
||||||
|
func IdentityProblems(shelf Shelf, longestMachine int) []string {
|
||||||
|
offeredBy := map[string][]string{}
|
||||||
|
for _, name := range shelfOrder(shelf) {
|
||||||
|
for _, o := range shelf[name].Offers() {
|
||||||
|
offeredBy[o] = append(offeredBy[o], name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
machine := strings.Repeat("n", longestMachine)
|
||||||
|
var problems []string
|
||||||
|
for _, name := range shelfOrder(shelf) {
|
||||||
|
m := shelf[name]
|
||||||
|
source := IdentitySource(m.Slug, m.Module)
|
||||||
|
for _, want := range m.Wants() {
|
||||||
|
// The tightest bound among the modules offering it: whichever one answers on a given
|
||||||
|
// machine, the identity has to fit it.
|
||||||
|
tightest, by := IdentityBound{}, ""
|
||||||
|
for _, provider := range offeredBy[want] {
|
||||||
|
if provider == name {
|
||||||
|
continue // a module answering its own requirement is not its own consumer
|
||||||
|
}
|
||||||
|
b := shelf[provider].IdentityBoundOf(want)
|
||||||
|
if b.Bounded() && (!tightest.Bounded() || b.Max < tightest.Max) {
|
||||||
|
tightest, by = b, provider
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if CheckIdentityWithin(machine, source, tightest) == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
got := ConsumerIdentity(machine, source)
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s wants %s, and %s keeps its consumers' identities in %s of at most %d characters: "+
|
||||||
|
"on a machine with a %d-character name it is identified as %q, %d — give %s a "+
|
||||||
|
"`slug` of at most %d characters",
|
||||||
|
name, want, by, tightest.In, tightest.Max, longestMachine, got, len(got), name,
|
||||||
|
tightest.Max-len(IdentityPrefix)-longestMachine-1))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return problems
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,190 @@
|
|||||||
|
package catalogue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Each test names the decision it defends: novox/hq ADR 0225, which refines ADR 0049 after issue 263.
|
||||||
|
|
||||||
|
// The provision a module requires sets the bound on its identity, not the tightest backend anywhere.
|
||||||
|
func TestABoundIsTheProvisionsOwn(t *testing.T) {
|
||||||
|
store := Manifest{Module: "objects", Provides: []Offer{{Name: "s3-bucket", Scope: ScopeMesh,
|
||||||
|
Identity: &OfferIdentity{Max: 20, In: "an S3 access key"}}},
|
||||||
|
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}}
|
||||||
|
database := Manifest{Module: "db", Provides: []Offer{{Name: "postgres-database", Scope: ScopeMesh,
|
||||||
|
Identity: &OfferIdentity{Max: 63, In: "a PostgreSQL role"}}},
|
||||||
|
Receives: map[string]string{"postgres-database": "/var/lib/mesh/db/mesh.json"}}
|
||||||
|
if b := store.IdentityBoundOf("s3-bucket"); b.Max != 20 || b.In != "an S3 access key" {
|
||||||
|
t.Errorf("an object store's stated bound was not taken: %+v", b)
|
||||||
|
}
|
||||||
|
if b := database.IdentityBoundOf("postgres-database"); b.Max != 63 {
|
||||||
|
t.Errorf("a database's stated bound was not taken: %+v", b)
|
||||||
|
}
|
||||||
|
// mesh_workstation_keycloak is 25: refused by the object store, accepted by the database.
|
||||||
|
if CheckIdentityWithin("workstation", "keycloak", store.IdentityBoundOf("s3-bucket")) == nil {
|
||||||
|
t.Error("a 25-character identity fit a 20-character access key")
|
||||||
|
}
|
||||||
|
if err := CheckIdentityWithin("workstation", "keycloak", database.IdentityBoundOf("postgres-database")); err != nil {
|
||||||
|
t.Errorf("a database consumer paid the object store's limit: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// What an offer leaves unsaid follows from whether its provider can keep a name at all.
|
||||||
|
func TestAnUnstatedBoundFollowsWhatTheProviderIsTold(t *testing.T) {
|
||||||
|
// Told nothing about its consumers — no receives, nothing served from their identity: no bound.
|
||||||
|
// The resolver provision is exactly this, and its consumers paid an object store's limit (263).
|
||||||
|
resolver := Manifest{Module: "resolver", Provides: []Offer{{Name: "wildcard-resolution", Scope: ScopeMesh}}}
|
||||||
|
if b := resolver.IdentityBoundOf("wildcard-resolution"); b.Bounded() {
|
||||||
|
t.Errorf("a provision that is told nothing of its consumers bounds them: %+v", b)
|
||||||
|
}
|
||||||
|
// Told each consumer and silent about its backend: the old global bound, not none.
|
||||||
|
told := Manifest{Module: "told", Provides: []Offer{{Name: "thing", Scope: ScopeMesh}},
|
||||||
|
Receives: map[string]string{"thing": "/var/lib/mesh/told/mesh.json"}}
|
||||||
|
if b := told.IdentityBoundOf("thing"); b.Max != DefaultIdentityLimit {
|
||||||
|
t.Errorf("a provider told its consumers and silent about its backend is not held to %d: %+v",
|
||||||
|
DefaultIdentityLimit, b)
|
||||||
|
}
|
||||||
|
// Serving a value built from the identity is being told it, too.
|
||||||
|
serving := Manifest{Module: "serving", Provides: []Offer{{Name: "bucket", Scope: ScopeMesh}},
|
||||||
|
Serves: map[string]map[string]any{"bucket": {"name": "b-${consumer:as:dns}"}}}
|
||||||
|
if b := serving.IdentityBoundOf("bucket"); b.Max != DefaultIdentityLimit {
|
||||||
|
t.Errorf("a provider deriving a name from its consumers is not bounded: %+v", b)
|
||||||
|
}
|
||||||
|
// And `false` says it outright, even for a provider that receives.
|
||||||
|
routes := Manifest{Module: "routes", Provides: []Offer{{Name: "route", Scope: ScopeMesh,
|
||||||
|
Identity: &OfferIdentity{None: true}}},
|
||||||
|
Receives: map[string]string{"route": "/var/lib/mesh/routes/mesh.json"}}
|
||||||
|
if b := routes.IdentityBoundOf("route"); b.Bounded() {
|
||||||
|
t.Errorf("`identity: false` still bounds: %+v", b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The field reads as written and writes back the same, and refuses what says nothing.
|
||||||
|
func TestAnOffersIdentityIsParsedStrictly(t *testing.T) {
|
||||||
|
for _, raw := range []string{
|
||||||
|
`{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}`,
|
||||||
|
`{"name":"wildcard-resolution","scope":"mesh","identity":false}`,
|
||||||
|
`{"name":"redis-cache","scope":"mesh","identity":{"in":"a Redis ACL user"}}`,
|
||||||
|
} {
|
||||||
|
var o Offer
|
||||||
|
if err := json.Unmarshal([]byte(raw), &o); err != nil {
|
||||||
|
t.Fatalf("%s: %v", raw, err)
|
||||||
|
}
|
||||||
|
back, err := json.Marshal(o)
|
||||||
|
if err != nil || string(back) != raw {
|
||||||
|
t.Errorf("did not round-trip:\n%s\n%s (%v)", raw, back, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, raw := range []string{
|
||||||
|
`{"name":"x","identity":true}`,
|
||||||
|
`{"name":"x","identity":{"max":20,"in":"y","most":3}}`,
|
||||||
|
} {
|
||||||
|
var o Offer
|
||||||
|
if err := json.Unmarshal([]byte(raw), &o); err == nil {
|
||||||
|
t.Errorf("accepted %s", raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bad := Manifest{Module: "bad", Version: "1", Provides: []Offer{
|
||||||
|
{Name: "unsaid", Scope: ScopeMesh, Identity: &OfferIdentity{Max: 20}},
|
||||||
|
{Name: "tiny", Scope: ScopeMesh, Identity: &OfferIdentity{Max: 4, In: "nothing usable"}},
|
||||||
|
}}
|
||||||
|
raw, err := json.Marshal(bad)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, err = ParseManifest(raw)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "without saying what keeps them") ||
|
||||||
|
!strings.Contains(err.Error(), "the shortest the mesh makes") {
|
||||||
|
t.Fatalf("a bound with no `in`, or too short for any identity, was accepted: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Refused before merge: the catalogue check judges each module's identity, on the longest machine
|
||||||
|
// name, against the bound of every provision it wants — and names the module and the slug to set.
|
||||||
|
func TestTheCatalogueCheckRefusesAnIdentityThatOverflowsWhatItRequires(t *testing.T) {
|
||||||
|
shelf := Shelf{
|
||||||
|
"objects": {Module: "objects", Provides: []Offer{{Name: "s3-bucket", Scope: ScopeMesh,
|
||||||
|
Identity: &OfferIdentity{Max: 20, In: "an S3 access key"}}},
|
||||||
|
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}},
|
||||||
|
"photoalbum": {Module: "photoalbum", Requires: []string{"s3-bucket"}},
|
||||||
|
"files": {Module: "files", Requires: []string{"s3-bucket"}},
|
||||||
|
}
|
||||||
|
problems := IdentityProblems(shelf, 6)
|
||||||
|
if len(problems) != 1 || !strings.Contains(problems[0], "photoalbum wants s3-bucket") ||
|
||||||
|
!strings.Contains(problems[0], `"mesh_nnnnnn_photoalbum", 22`) ||
|
||||||
|
!strings.Contains(problems[0], "`slug` of at most 8 characters") {
|
||||||
|
t.Fatalf("one overflow, named with its remedy, was expected: %q", problems)
|
||||||
|
}
|
||||||
|
// A longer machine name refuses more: the check is about the mesh's machines, not one.
|
||||||
|
if got := IdentityProblems(shelf, 10); len(got) != 2 {
|
||||||
|
t.Fatalf("on a 10-character name both overflow (mesh_nnnnnnnnnn_files is 21): %q", got)
|
||||||
|
}
|
||||||
|
// And a slug is the remedy it names.
|
||||||
|
album := shelf["photoalbum"]
|
||||||
|
album.Slug = "album"
|
||||||
|
shelf["photoalbum"] = album
|
||||||
|
if got := IdentityProblems(shelf, 6); len(got) != 0 {
|
||||||
|
t.Fatalf("a slug that fits is still refused: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tonight's case (issue 263): networkmanager, no slug, requiring the mesh's resolver provision on a
|
||||||
|
// machine with a six-character name. Under ADR 0049's one bound it was refused, and its provider's
|
||||||
|
// whole machine with it; the resolver keeps no name, so it is not refused at all.
|
||||||
|
func TestARequirementOnAKeylessProvisionComposesWithALongName(t *testing.T) {
|
||||||
|
shelf := Shelf{
|
||||||
|
"resolver": {Module: "resolver", Provides: []Offer{{Name: "wildcard-resolution", Scope: ScopeMesh}}},
|
||||||
|
"networkmanager": {Module: "networkmanager", Requires: []string{"wildcard-resolution"}},
|
||||||
|
}
|
||||||
|
if CheckIdentity("laptop", "networkmanager") == nil {
|
||||||
|
t.Fatal("the regression is not reproduced: mesh_laptop_networkmanager fits the old global bound")
|
||||||
|
}
|
||||||
|
if got := IdentityProblems(shelf, 6); len(got) != 0 {
|
||||||
|
t.Fatalf("a requirement on a keyless provision was refused for its length: %q", got)
|
||||||
|
}
|
||||||
|
r := Resolution{Node: "laptop", Modules: []Manifest{shelf["networkmanager"]},
|
||||||
|
Needs: []Needed{{Name: "wildcard-resolution", From: "anchor", For: "networkmanager",
|
||||||
|
Identity: shelf["resolver"].IdentityBoundOf("wildcard-resolution")}}}
|
||||||
|
if got := r.Overflowing(); len(got) != 0 {
|
||||||
|
t.Fatalf("a keyless requirement is reported as overflowing: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The consumer's side of the same judgement the provider's composition makes: what `status` says.
|
||||||
|
func TestAnOverflowingRequirementIsNamedFromTheConsumersSide(t *testing.T) {
|
||||||
|
bound := IdentityBound{Max: 20, In: "an S3 access key"}
|
||||||
|
r := Resolution{Node: "laptop",
|
||||||
|
Modules: []Manifest{{Module: "photoalbum"}, {Module: "files"}, {Module: "gallery", Slug: "gal"}},
|
||||||
|
Needs: []Needed{
|
||||||
|
{Name: "s3-bucket", From: "anchor", For: "photoalbum", Identity: bound},
|
||||||
|
{Name: "s3-bucket", From: "anchor", For: "photoalbum", Local: "second", Identity: bound},
|
||||||
|
{Name: "s3-bucket", From: "anchor", For: "files", Identity: bound},
|
||||||
|
{Name: "s3-bucket", From: "anchor", For: "gallery", Identity: bound},
|
||||||
|
{Name: "licence", From: "records", For: "photoalbum", ByRecord: true, Identity: bound},
|
||||||
|
}}
|
||||||
|
got := r.Overflowing()
|
||||||
|
if len(got) != 1 || got[0].Module != "photoalbum" || got[0].Identity != "mesh_laptop_photoalbum" ||
|
||||||
|
got[0].Provider != "anchor" {
|
||||||
|
t.Fatalf("one overflow, once, was expected: %+v", got)
|
||||||
|
}
|
||||||
|
if said := got[0].String(); !strings.Contains(said, "an S3 access key keeps 20") ||
|
||||||
|
!strings.Contains(said, "slug") {
|
||||||
|
t.Fatalf("the overflow does not say what keeps it or the remedy: %s", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An identity served as a DNS label is never truncated, so the offer's bound must keep it in one.
|
||||||
|
func TestAnIdentityServedAsADNSLabelIsBoundedToOne(t *testing.T) {
|
||||||
|
serves := map[string]map[string]any{"bucket": {"name": "${consumer:as:dns}"}}
|
||||||
|
wide := Manifest{Module: "wide", Serves: serves, Provides: []Offer{{Name: "bucket", Scope: ScopeMesh,
|
||||||
|
Identity: &OfferIdentity{Max: 255, In: "a client id"}}}}
|
||||||
|
if got := CheckServes(wide); len(got) != 1 || !strings.Contains(got[0], "a label keeps 63") {
|
||||||
|
t.Fatalf("a 255-character bound on a DNS label passed: %q", got)
|
||||||
|
}
|
||||||
|
unsaid := Manifest{Module: "unsaid", Serves: serves, Provides: []Offer{{Name: "bucket", Scope: ScopeMesh}}}
|
||||||
|
if got := CheckServes(unsaid); len(got) != 0 {
|
||||||
|
t.Fatalf("the default bound (20) on a DNS label was refused: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -8,6 +8,7 @@ package catalogue
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"regexp"
|
"regexp"
|
||||||
"sort"
|
"sort"
|
||||||
@@ -153,6 +154,84 @@ type Offer struct {
|
|||||||
// a seat's holder gated by the machine's graphical session, and pulling one in for whatever asked
|
// a seat's holder gated by the machine's graphical session, and pulling one in for whatever asked
|
||||||
// is the misassignment research 026 found. Unmet, the requirement is refused naming who could.
|
// is the misassignment research 026 found. Unmet, the requirement is refused naming who could.
|
||||||
Reach string `json:"reach,omitempty"`
|
Reach string `json:"reach,omitempty"`
|
||||||
|
// Identity is the longest consumer identity this provision's backend keeps (novox/hq ADR 0225):
|
||||||
|
// `{"max": 63, "in": "a PostgreSQL role"}`, or `false` for a provision that keeps no name derived
|
||||||
|
// from its consumer. Unsaid, the mesh assumes the tightest backend it knows when the provider is
|
||||||
|
// told its consumers, and no bound when it is not — see IdentityBoundOf.
|
||||||
|
Identity *OfferIdentity `json:"identity,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// OfferIdentity is what an offer says about the names its backend keeps for its consumers.
|
||||||
|
type OfferIdentity struct {
|
||||||
|
// None is set by `"identity": false`: the provision keeps no name derived from its consumer.
|
||||||
|
None bool
|
||||||
|
// Max is the longest identity kept, in characters; zero with In set means no limit worth stating.
|
||||||
|
Max int
|
||||||
|
// In is what keeps it, for a refusal to quote.
|
||||||
|
In string
|
||||||
|
}
|
||||||
|
|
||||||
|
// UnmarshalJSON accepts `false` or `{"max": N, "in": "..."}`.
|
||||||
|
func (i *OfferIdentity) UnmarshalJSON(raw []byte) error {
|
||||||
|
var flag bool
|
||||||
|
if err := json.Unmarshal(raw, &flag); err == nil {
|
||||||
|
if flag {
|
||||||
|
return errors.New("an offer's identity is false (it keeps no name) or {max, in}; true says nothing")
|
||||||
|
}
|
||||||
|
*i = OfferIdentity{None: true}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var full struct {
|
||||||
|
Max int `json:"max,omitempty"`
|
||||||
|
In string `json:"in"`
|
||||||
|
}
|
||||||
|
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||||
|
dec.DisallowUnknownFields()
|
||||||
|
if err := dec.Decode(&full); err != nil {
|
||||||
|
return fmt.Errorf("an offer's identity is false or {max, in}: %w", err)
|
||||||
|
}
|
||||||
|
*i = OfferIdentity{Max: full.Max, In: full.In}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// MarshalJSON writes it back in the form it was written.
|
||||||
|
func (i OfferIdentity) MarshalJSON() ([]byte, error) {
|
||||||
|
if i.None {
|
||||||
|
return []byte("false"), nil
|
||||||
|
}
|
||||||
|
return json.Marshal(struct {
|
||||||
|
Max int `json:"max,omitempty"`
|
||||||
|
In string `json:"in"`
|
||||||
|
}{i.Max, i.In})
|
||||||
|
}
|
||||||
|
|
||||||
|
// IdentityBoundOf is the bound this module's offer of a provision puts on its consumers' identities
|
||||||
|
// (novox/hq ADR 0225).
|
||||||
|
//
|
||||||
|
// **What an offer says, it gets.** Where it says nothing, the bound follows from whether the
|
||||||
|
// provider can keep a name at all: a provider that receives the provision, or serves its consumers
|
||||||
|
// a value built from their identity, is told who each consumer is and may create a name from it in
|
||||||
|
// a backend nobody measured — so it keeps the old global bound, DefaultIdentityBound. One that does
|
||||||
|
// neither is told nothing about its consumers and keeps nothing of them: no bound. The resolver
|
||||||
|
// provision is that case, and its consumers paid an object store's limit until this (issue 263).
|
||||||
|
func (m Manifest) IdentityBoundOf(provision string) IdentityBound {
|
||||||
|
for _, o := range m.Provides {
|
||||||
|
if o.Name != provision || o.Identity == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if o.Identity.None {
|
||||||
|
return IdentityBound{}
|
||||||
|
}
|
||||||
|
return IdentityBound{Max: o.Identity.Max, In: o.Identity.In}
|
||||||
|
}
|
||||||
|
if _, receives := m.Receives[provision]; receives {
|
||||||
|
return DefaultIdentityBound
|
||||||
|
}
|
||||||
|
if served, err := json.Marshal(m.Serves[provision]); err == nil &&
|
||||||
|
bytes.Contains(served, []byte("${consumer:as")) {
|
||||||
|
return DefaultIdentityBound
|
||||||
|
}
|
||||||
|
return IdentityBound{}
|
||||||
}
|
}
|
||||||
|
|
||||||
// MachineReach is whether a provision is usable only on its provider's own machine.
|
// MachineReach is whether a provision is usable only on its provider's own machine.
|
||||||
@@ -206,20 +285,21 @@ func (o *Offer) UnmarshalJSON(raw []byte) error {
|
|||||||
Scope string `json:"scope,omitempty"`
|
Scope string `json:"scope,omitempty"`
|
||||||
Credential *OfferCredential `json:"credential,omitempty"`
|
Credential *OfferCredential `json:"credential,omitempty"`
|
||||||
Reach string `json:"reach,omitempty"`
|
Reach string `json:"reach,omitempty"`
|
||||||
|
Identity *OfferIdentity `json:"identity,omitempty"`
|
||||||
}
|
}
|
||||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||||
dec.DisallowUnknownFields()
|
dec.DisallowUnknownFields()
|
||||||
if err := dec.Decode(&full); err != nil {
|
if err := dec.Decode(&full); err != nil {
|
||||||
return fmt.Errorf("a provided name is either a string or {name, scope, credential, reach}: %w", err)
|
return fmt.Errorf("a provided name is either a string or {name, scope, credential, reach, identity}: %w", err)
|
||||||
}
|
}
|
||||||
o.Name, o.Scope, o.Credential, o.Reach = full.Name, full.Scope, full.Credential, full.Reach
|
o.Name, o.Scope, o.Credential, o.Reach, o.Identity = full.Name, full.Scope, full.Credential, full.Reach, full.Identity
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// MarshalJSON writes back the short form when there is nothing else to say, so a manifest that
|
// MarshalJSON writes back the short form when there is nothing else to say, so a manifest that
|
||||||
// went through the mesh comes out looking like the one that went in.
|
// went through the mesh comes out looking like the one that went in.
|
||||||
func (o Offer) MarshalJSON() ([]byte, error) {
|
func (o Offer) MarshalJSON() ([]byte, error) {
|
||||||
if o.Scope == "" && o.Credential == nil && o.Reach == "" {
|
if o.Scope == "" && o.Credential == nil && o.Reach == "" && o.Identity == nil {
|
||||||
return json.Marshal(o.Name)
|
return json.Marshal(o.Name)
|
||||||
}
|
}
|
||||||
return json.Marshal(struct {
|
return json.Marshal(struct {
|
||||||
@@ -227,7 +307,8 @@ func (o Offer) MarshalJSON() ([]byte, error) {
|
|||||||
Scope string `json:"scope,omitempty"`
|
Scope string `json:"scope,omitempty"`
|
||||||
Credential *OfferCredential `json:"credential,omitempty"`
|
Credential *OfferCredential `json:"credential,omitempty"`
|
||||||
Reach string `json:"reach,omitempty"`
|
Reach string `json:"reach,omitempty"`
|
||||||
}{o.Name, o.Scope, o.Credential, o.Reach})
|
Identity *OfferIdentity `json:"identity,omitempty"`
|
||||||
|
}{o.Name, o.Scope, o.Credential, o.Reach, o.Identity})
|
||||||
}
|
}
|
||||||
|
|
||||||
// Manifest is everything a module says about itself.
|
// Manifest is everything a module says about itself.
|
||||||
@@ -237,9 +318,10 @@ type Manifest struct {
|
|||||||
|
|
||||||
// Slug is a short identifier the mesh uses in place of the module name when it derives a
|
// Slug is a short identifier the mesh uses in place of the module name when it derives a
|
||||||
// consumer's login (novox/hq ADR 0049). Optional: a module with a short name needs none. It
|
// consumer's login (novox/hq ADR 0049). Optional: a module with a short name needs none. It
|
||||||
// exists because `mesh_<node>_<module>` must fit the tightest backend a consumer reaches — an S3
|
// exists because `mesh_<node>_<module>` must fit the bound of every provision the module
|
||||||
// access key is 20 characters — and a long module name would overflow it. A person choosing
|
// requires — an S3 access key's 20 characters is the tightest — and a long module name would
|
||||||
// `kc` for keycloak keeps the identity legible where a hash would not.
|
// overflow it (ADR 0225: the bound is the provision's own, and a keyless one has none). A person
|
||||||
|
// choosing `kc` for keycloak keeps the identity legible where a hash would not.
|
||||||
Slug string `json:"slug,omitempty"`
|
Slug string `json:"slug,omitempty"`
|
||||||
|
|
||||||
// Provides are the names other modules may require. A module always provides its own name;
|
// Provides are the names other modules may require. A module always provides its own name;
|
||||||
@@ -1265,8 +1347,10 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
|||||||
"%q is not a usable module name: lower-case letters, digits, dashes and dots", m.Module))
|
"%q is not a usable module name: lower-case letters, digits, dashes and dots", m.Module))
|
||||||
}
|
}
|
||||||
// A slug is a short identifier the mesh derives a login from (novox/hq ADR 0049). The same
|
// A slug is a short identifier the mesh derives a login from (novox/hq ADR 0049). The same
|
||||||
// charset as a name; its length is checked against a backend's limit at assignment, where the
|
// charset as a name; its length is judged against the bound of each provision it wants on the
|
||||||
// node it joins is known — a slug that is fine on one machine's short name can overflow another's.
|
// longest machine name by the catalogue check (IdentityProblems, ADR 0225), and against the
|
||||||
|
// machine it is on when its provider grants it — a slug that is fine on one machine's short name
|
||||||
|
// can overflow another's.
|
||||||
if m.Slug != "" && !name.MatchString(m.Slug) {
|
if m.Slug != "" && !name.MatchString(m.Slug) {
|
||||||
problems = append(problems, fmt.Sprintf(
|
problems = append(problems, fmt.Sprintf(
|
||||||
"%q is not a usable slug: lower-case letters, digits, dashes and dots", m.Slug))
|
"%q is not a usable slug: lower-case letters, digits, dashes and dots", m.Slug))
|
||||||
@@ -1303,6 +1387,20 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
|||||||
if !name.MatchString(p) {
|
if !name.MatchString(p) {
|
||||||
problems = append(problems, fmt.Sprintf("%q is not a usable name to provide", p))
|
problems = append(problems, fmt.Sprintf("%q is not a usable name to provide", p))
|
||||||
}
|
}
|
||||||
|
// A bound says what keeps the name, so the refusal it causes can say it (ADR 0225); and it
|
||||||
|
// leaves room for the shortest identity the mesh makes, or it would refuse every consumer.
|
||||||
|
if id := offer.Identity; id != nil && !id.None {
|
||||||
|
if strings.TrimSpace(id.In) == "" {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s bounds the identities of %s's consumers without saying what keeps them: `in`",
|
||||||
|
m.Module, p))
|
||||||
|
}
|
||||||
|
if shortest := len(IdentityPrefix) + 3; id.Max < 0 || id.Max > 0 && id.Max < shortest {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s bounds %s's consumers' identities at %d characters, and the shortest the mesh "+
|
||||||
|
"makes is %d", m.Module, p, id.Max, shortest))
|
||||||
|
}
|
||||||
|
}
|
||||||
if offer.Credential != nil {
|
if offer.Credential != nil {
|
||||||
own, declared := m.OwnSecrets[offer.Credential.Own]
|
own, declared := m.OwnSecrets[offer.Credential.Own]
|
||||||
switch {
|
switch {
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ func provided(t *testing.T) map[string]catalogue.Manifest {
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
out := map[string]catalogue.Manifest{}
|
out := map[string]catalogue.Manifest{}
|
||||||
for _, raw := range []map[string]any{
|
for _, raw := range []map[string]any{
|
||||||
overlay.Manifest(),
|
overlay.Manifest(), overlay.DomainManifest(),
|
||||||
} {
|
} {
|
||||||
b, err := json.Marshal(raw)
|
b, err := json.Marshal(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -34,16 +34,20 @@ func provided(t *testing.T) map[string]catalogue.Manifest {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTheShippedPrivateNetworkResolvesOnItsOwn(t *testing.T) {
|
func TestTheShippedNetworkingModulesResolveOnTheirOwn(t *testing.T) {
|
||||||
// **Assigned directly** (novox/hq ADR 0226): the `networking` bundle that required it is
|
got, err := catalogue.Resolve(provided(t), []string{overlay.Domain}, catalogue.Node{Name: "workstation", Site: "house"}, catalogue.World{})
|
||||||
// retired, so the private network's own module is what a machine is given, and it must need
|
|
||||||
// nothing the control plane does not ship beside it.
|
|
||||||
got, err := catalogue.Resolve(provided(t), []string{overlay.Name}, catalogue.Node{Name: "workstation", Site: "house"}, catalogue.World{})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("assigning %s does not work out of the box: %v", overlay.Name, err)
|
t.Fatalf("assigning %s does not work out of the box: %v", overlay.Domain, err)
|
||||||
}
|
}
|
||||||
if len(got.Modules) != 1 || got.Modules[0].Module != overlay.Name {
|
var have []string
|
||||||
t.Fatalf("assigning %s brought %v", overlay.Name, got.Modules)
|
for _, m := range got.Modules {
|
||||||
|
have = append(have, m.Module)
|
||||||
|
}
|
||||||
|
for _, want := range []string{overlay.Domain, overlay.Name} {
|
||||||
|
if !strings.Contains(strings.Join(have, " "), want) {
|
||||||
|
t.Fatalf("%s did not bring in %s: %v", overlay.Domain, want, have)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// **The network writes no names** (novox/hq ADR 0199): /etc/hosts is the hosts seat holder's
|
// **The network writes no names** (novox/hq ADR 0199): /etc/hosts is the hosts seat holder's
|
||||||
@@ -51,52 +55,23 @@ func TestTheShippedPrivateNetworkResolvesOnItsOwn(t *testing.T) {
|
|||||||
// may name that file.
|
// may name that file.
|
||||||
for _, m := range got.Modules {
|
for _, m := range got.Modules {
|
||||||
for name, f := range m.Facts {
|
for name, f := range m.Facts {
|
||||||
if f.Path == "/etc/hosts" {
|
if m.Module == overlay.Name && f.Path == "/etc/hosts" {
|
||||||
t.Fatalf("the network's provider still writes /etc/hosts, as its %q fact", name)
|
t.Fatalf("the network's provider still writes /etc/hosts, as its %q fact", name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTheControlPlaneShipsNoNetworkingBundle(t *testing.T) {
|
func TestAnotherVPNSatisfiesNetworkingWithoutDraggingWireGuardIn(t *testing.T) {
|
||||||
// ADR 0226: one module for the one private network. A bundle shipped beside it again would be
|
// **This inverted, and the inversion is the improvement.** The names used to be a module that
|
||||||
// a second name every machine carries for the same thing.
|
// required the mesh's own addressing, which only WireGuard provided — so choosing another VPN
|
||||||
|
// dragged WireGuard in anyway, and the node-scoped claim existed to at least make that
|
||||||
|
// collision loud. With the names a fact rather than a provision, a person who chose tailscale
|
||||||
|
// gets tailscale, and there is nothing left to collide.
|
||||||
shipped := provided(t)
|
shipped := provided(t)
|
||||||
if len(shipped) != 1 {
|
got, err := catalogue.Resolve(
|
||||||
t.Fatalf("the control plane ships %d modules, want only %s", len(shipped), overlay.Name)
|
withTailscale(shipped),
|
||||||
}
|
[]string{overlay.Domain, "tailscale"},
|
||||||
if _, ok := shipped["networking"]; ok {
|
|
||||||
t.Fatal("the retired networking bundle is shipped again")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestARefusalForWantOfThePrivateNetworkNamesItsModule(t *testing.T) {
|
|
||||||
// The hint in a refusal is a string in the resolver rather than an import of this package. It
|
|
||||||
// named `networking` until ADR 0226; a hint naming a module nobody ships sends a person to
|
|
||||||
// assign something that does not exist.
|
|
||||||
shelf := map[string]catalogue.Manifest{
|
|
||||||
"app": {Module: "app", Version: "1", Requires: []string{"postgres-database"}},
|
|
||||||
"postgres": {Module: "postgres", Version: "1", Provides: catalogue.FromAnywhere("postgres-database")},
|
|
||||||
}
|
|
||||||
_, err := catalogue.Resolve(shelf, []string{"app"}, catalogue.Node{Name: "workstation"},
|
|
||||||
catalogue.World{Offered: map[string][]catalogue.Provider{
|
|
||||||
"postgres-database": {{Node: "anchor", At: "anchor.internal"}}}})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("an app off the private network was pointed at a database on it")
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), "assign "+overlay.Name) {
|
|
||||||
t.Fatalf("the refusal does not name the private network's module %s: %v", overlay.Name, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAnotherVPNIsChosenByAssigningItInstead(t *testing.T) {
|
|
||||||
// What the bundle was for, kept without it: a machine given another VPN answers
|
|
||||||
// private-network from that VPN, and WireGuard is not dragged in by anything.
|
|
||||||
shipped := provided(t)
|
|
||||||
shelf := withTailscale(shipped)
|
|
||||||
shelf["needs-network"] = catalogue.Manifest{Module: "needs-network", Version: "1",
|
|
||||||
Requires: []string{overlay.Requirement}}
|
|
||||||
got, err := catalogue.Resolve(shelf, []string{"needs-network", "tailscale"},
|
|
||||||
catalogue.Node{Name: "workstation", Site: "house"}, catalogue.World{})
|
catalogue.Node{Name: "workstation", Site: "house"}, catalogue.World{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("choosing another VPN was refused: %v", err)
|
t.Fatalf("choosing another VPN was refused: %v", err)
|
||||||
|
|||||||
@@ -1,119 +0,0 @@
|
|||||||
package catalogue
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The public issuer is the proxy's own fact (novox/hq ADR 0226), and the folding of it must not
|
|
||||||
// move the proxy's account.
|
|
||||||
//
|
|
||||||
// route-proxy keeps each ACME authority's account and certificates in a directory named after a
|
|
||||||
// digest of the directory URL and of the root bundle its `trust` container copies in
|
|
||||||
// (examples/route-proxy, forThisAuthority). Until ADR 0226 the URL was rendered from a binding to
|
|
||||||
// `public-acme`'s `acme-ca`, spelled with the port. A URL spelled any other way — even the same
|
|
||||||
// authority without `:443` — or a root bundle from another image is a new authority to the proxy:
|
|
||||||
// a new account, and every routed name ordered again, on two machines at once, against Let's
|
|
||||||
// Encrypt's rate limits. So what the module now states is held to exactly what the binding rendered.
|
|
||||||
|
|
||||||
// renderedBeforeTheFold is route-proxy's acme.env as the binding to public-acme rendered it on every
|
|
||||||
// machine running the proxy, read from the controller's plan on 2026-10-06.
|
|
||||||
const renderedBeforeTheFold = "ACME_DIRECTORY=https://acme-v02.api.letsencrypt.org:443/directory\n" +
|
|
||||||
"ACME_ROOTS=https://acme-v02.api.letsencrypt.org:443\n" +
|
|
||||||
"ACME_ROOTS_PATH=\n"
|
|
||||||
|
|
||||||
// trustImage is the image whose system bundle becomes the public root the account directory is
|
|
||||||
// named after. Moving it renames that directory.
|
|
||||||
const trustImage = "alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b"
|
|
||||||
|
|
||||||
func TestRouteProxyKeepsItsPublicAccountDirectory(t *testing.T) {
|
|
||||||
m := catalogueManifest(t, "route-proxy")
|
|
||||||
for _, r := range m.Requires {
|
|
||||||
if r == "acme-ca" {
|
|
||||||
t.Fatal("route-proxy still asks for acme-ca; the public issuer is its own fact (ADR 0226)")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// As a build would leave it: each artifact a container names resolved to an image. Which image
|
|
||||||
// does not matter to the file checked here.
|
|
||||||
for _, res := range m.Resources {
|
|
||||||
if artifact, ok := res["artifact"].(string); ok {
|
|
||||||
delete(res, "artifact")
|
|
||||||
res["image"] = "registry.invalid/route-proxy/" + artifact +
|
|
||||||
"@sha256:1111111111111111111111111111111111111111111111111111111111111111"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
r := Resolution{
|
|
||||||
Node: "anchor",
|
|
||||||
Modules: []Manifest{m},
|
|
||||||
Needs: []Needed{{
|
|
||||||
Name: "internal-acme-ca", From: "home", At: "home.internal", For: "route-proxy",
|
|
||||||
Serves: map[string]any{
|
|
||||||
"port": float64(9000), "path": "/acme/acme/directory", "roots": "/roots.pem",
|
|
||||||
},
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
// Its own broker credential, sealed as the mesh would; what it is does not matter here.
|
|
||||||
out, err := r.Declaration(Rendering{Needed: map[string]map[string]string{
|
|
||||||
"route-proxy": {"broker": "sealed"}}})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("route-proxy could not be composed for a machine: %v", err)
|
|
||||||
}
|
|
||||||
env := fileNamed(out, "route-proxy.acme-env")
|
|
||||||
if env == nil {
|
|
||||||
t.Fatalf("nothing writes the public issuer's environment: %v", out)
|
|
||||||
}
|
|
||||||
if got, _ := env["content"].(string); got != renderedBeforeTheFold {
|
|
||||||
t.Fatalf("acme.env changed, which moves the proxy's account and reorders every certificate:\n"+
|
|
||||||
"got %q\nwant %q", got, renderedBeforeTheFold)
|
|
||||||
}
|
|
||||||
if fileNamed(out, "route-proxy.bound-acme-ca") != nil {
|
|
||||||
t.Error("a binding to acme-ca is still written")
|
|
||||||
}
|
|
||||||
|
|
||||||
var trust string
|
|
||||||
if m.Build != nil {
|
|
||||||
for _, a := range m.Build.Artifacts {
|
|
||||||
if a.Name == "trust" {
|
|
||||||
trust = a.From
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if trust != trustImage {
|
|
||||||
t.Errorf("the trust image is %q, not %q: its system bundle is the public root the account "+
|
|
||||||
"directory is named after, so moving it reorders every certificate. Move it only with a "+
|
|
||||||
"plan for the account (ADR 0226)", trust, trustImage)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The modules ADR 0226 retired stay retired, and nothing asks for what they provided.
|
|
||||||
func TestTheRetiredNetworkingModulesAreNotInTheCatalogue(t *testing.T) {
|
|
||||||
if _, err := os.Stat("../../../mesh-catalog/modules"); err != nil {
|
|
||||||
t.Skipf("the catalogue is not beside this checkout: %v", err)
|
|
||||||
}
|
|
||||||
for _, gone := range []string{"public-acme", "dhcpcd", "cloudflare-dns"} {
|
|
||||||
if _, err := os.Stat("../../../mesh-catalog/modules/" + gone); err == nil {
|
|
||||||
t.Errorf("%s is in the catalogue again; ADR 0226 retired it", gone)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
entries, err := os.ReadDir("../../../mesh-catalog/modules")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, e := range entries {
|
|
||||||
raw, err := os.ReadFile("../../../mesh-catalog/modules/" + e.Name() + "/module.json")
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
m, err := ParseManifest(raw)
|
|
||||||
if err != nil {
|
|
||||||
continue // judged by the catalogue's own tests
|
|
||||||
}
|
|
||||||
for _, r := range m.Requires {
|
|
||||||
if r == "acme-ca" || r == "public-dns" {
|
|
||||||
t.Errorf("%s requires %q, which nothing in the catalogue provides since ADR 0226",
|
|
||||||
m.Module, r)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -194,6 +194,11 @@ type Needed struct {
|
|||||||
// state, not a consumer missing its key. Set by the plan, which is the only layer that knows a
|
// state, not a consumer missing its key. Set by the plan, which is the only layer that knows a
|
||||||
// licence's manager; empty for every consumer.
|
// licence's manager; empty for every consumer.
|
||||||
Manager bool
|
Manager bool
|
||||||
|
// Identity is the longest consumer identity the answering provision keeps (novox/hq ADR 0225),
|
||||||
|
// from the provider's own offer: what the mesh judges this consumer's identity against, on the
|
||||||
|
// consumer's side for `status` and on the provider's before it grants. No bound for a provision
|
||||||
|
// answered by a record, which keeps no name of anybody's.
|
||||||
|
Identity IdentityBound
|
||||||
}
|
}
|
||||||
|
|
||||||
// Refusal is why a set of assignments cannot become a declaration.
|
// Refusal is why a set of assignments cannot become a declaration.
|
||||||
@@ -403,7 +408,7 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
|
|||||||
needs = append(needs, Needed{
|
needs = append(needs, Needed{
|
||||||
Name: want, From: node.Name, At: at,
|
Name: want, From: node.Name, At: at,
|
||||||
Serves: servedByOne(by, want), For: because[want],
|
Serves: servedByOne(by, want), For: because[want],
|
||||||
SharedOwn: sharedByOne(by, want)})
|
SharedOwn: sharedByOne(by, want), Identity: by.IdentityBoundOf(want)})
|
||||||
} else if served := servedByOne(by, want); len(served) > 0 {
|
} else if served := servedByOne(by, want); len(served) > 0 {
|
||||||
// Answered here with no credential to mint, but the provider serves facts the
|
// Answered here with no credential to mint, but the provider serves facts the
|
||||||
// consumer cannot guess — a port, a model name — and so still needs a binding.
|
// consumer cannot guess — a port, a model name — and so still needs a binding.
|
||||||
@@ -447,11 +452,15 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
shared := ""
|
shared := ""
|
||||||
|
// A provider whose definition is not in hand is held to the tightest bound the
|
||||||
|
// mesh knows, not to none: what it keeps is not known here (ADR 0225).
|
||||||
|
bound := DefaultIdentityBound
|
||||||
if pm, known := catalogue[p.Module]; known {
|
if pm, known := catalogue[p.Module]; known {
|
||||||
shared, _ = pm.SharedCredentialOf(want)
|
shared, _ = pm.SharedCredentialOf(want)
|
||||||
|
bound = pm.IdentityBoundOf(want)
|
||||||
}
|
}
|
||||||
needs = append(needs, Needed{Name: want, From: p.Node, At: p.At,
|
needs = append(needs, Needed{Name: want, From: p.Node, At: p.At,
|
||||||
Serves: p.Serves, For: because[want], SharedOwn: shared})
|
Serves: p.Serves, For: because[want], SharedOwn: shared, Identity: bound})
|
||||||
}
|
}
|
||||||
switch {
|
switch {
|
||||||
case world.Unchecked:
|
case world.Unchecked:
|
||||||
@@ -1019,10 +1028,8 @@ func FromAnywhere(names ...string) []Offer {
|
|||||||
//
|
//
|
||||||
// A string here rather than an import, because the private network is a module the control plane
|
// A string here rather than an import, because the private network is a module the control plane
|
||||||
// ships and this package must not depend on the thing it resolves. The name being wrong would
|
// ships and this package must not depend on the thing it resolves. The name being wrong would
|
||||||
// show up as a refusal naming a module nobody can assign, which a test checks
|
// show up as a refusal naming a module nobody can assign, which a test checks.
|
||||||
// (provided_test.go). The private network's own module since novox/hq ADR 0226 retired the
|
const meshNetwork = "networking"
|
||||||
// `networking` bundle that required it.
|
|
||||||
const meshNetwork = "mesh-wireguard"
|
|
||||||
|
|
||||||
// providersFirst orders a node's modules so that what answers a requirement comes before what
|
// providersFirst orders a node's modules so that what answers a requirement comes before what
|
||||||
// asked for it.
|
// asked for it.
|
||||||
|
|||||||
@@ -83,7 +83,9 @@ var defaultSeats = append([]Seat{
|
|||||||
// `assign` and the rest are a role's interface, not a container's, and stay addressable while
|
// `assign` and the rest are a role's interface, not a container's, and stay addressable while
|
||||||
// the control plane is replaced.
|
// the control plane is replaced.
|
||||||
{Name: ControllerSeatName, Scope: ScopeMesh, Decision: "novox/hq ADR 0079",
|
{Name: ControllerSeatName, Scope: ScopeMesh, Decision: "novox/hq ADR 0079",
|
||||||
Emits: []string{"applied", "refused", "built-before"},
|
// And what is wrong, as it changes, and the self-check's heartbeat (novox/hq to-be 45 §2, §4).
|
||||||
|
Emits: []string{"applied", "refused", "built-before",
|
||||||
|
"condition-raised", "condition-changed", "condition-cleared", "doctor-heartbeat"},
|
||||||
Serves: ControllerVerbs},
|
Serves: ControllerVerbs},
|
||||||
// The store's first verbs (novox/hq ADR 0159): the smallest set that makes the store askable,
|
// The store's first verbs (novox/hq ADR 0159): the smallest set that makes the store askable,
|
||||||
// served by whichever module holds the seat with tools of these names.
|
// served by whichever module holds the seat with tools of these names.
|
||||||
|
|||||||
@@ -24,8 +24,7 @@ var bothResolvers = []Held{
|
|||||||
|
|
||||||
// uplinks is every module in the catalogue holding node-uplink, and so writing the machine's resolver
|
// uplinks is every module in the catalogue holding node-uplink, and so writing the machine's resolver
|
||||||
// file (novox/hq ADR 0223 part 2): the program that would otherwise rewrite it is the one that writes it.
|
// file (novox/hq ADR 0223 part 2): the program that would otherwise rewrite it is the one that writes it.
|
||||||
// dhcpcd's left the catalogue with ADR 0226, held by no machine.
|
var uplinks = []string{"dhcpcd", "networkmanager", "systemd-networkd"}
|
||||||
var uplinks = []string{"networkmanager", "systemd-networkd"}
|
|
||||||
|
|
||||||
// managing is a machine as each uplink module needs it: able to install, run a service and run the
|
// managing is a machine as each uplink module needs it: able to install, run a service and run the
|
||||||
// manager that module is for.
|
// manager that module is for.
|
||||||
|
|||||||
@@ -110,6 +110,8 @@ var ControllerVerbs = []Verb{
|
|||||||
"paths": "with repository: the files the merge would change, comma-separated, from the repository's root",
|
"paths": "with repository: the files the merge would change, comma-separated, from the repository's root",
|
||||||
"modules": "with repository: or the modules it would change, comma-separated",
|
"modules": "with repository: or the modules it would change, comma-separated",
|
||||||
"limit": "how many plans to list (default 10); only when listing",
|
"limit": "how many plans to list (default 10); only when listing",
|
||||||
|
"why": "with stop or close: why it is ended by hand — required, and recorded in the hand-act log (novox/hq to-be 45 §7)",
|
||||||
|
"cause": "with stop or close: the cause in a word, or a condition's kind (optional)",
|
||||||
}, nil)},
|
}, nil)},
|
||||||
{Name: "plan", Description: "What one machine would run, and why: the declaration the mesh would send it — " +
|
{Name: "plan", Description: "What one machine would run, and why: the declaration the mesh would send it — " +
|
||||||
"or, with files, the files it would be given.",
|
"or, with files, the files it would be given.",
|
||||||
@@ -137,12 +139,15 @@ var ControllerVerbs = []Verb{
|
|||||||
{Name: "push", Description: "Send one machine everything it should be. With no machine named it is a push of the " +
|
{Name: "push", Description: "Send one machine everything it should be. With no machine named it is a push of the " +
|
||||||
"WHOLE mesh — every machine that is behind — and the answer says so first; behind says that outright. " +
|
"WHOLE mesh — every machine that is behind — and the answer says so first; behind says that outright. " +
|
||||||
"Answers at once that it is running, with a call id: `calls` with that id says what it sent " +
|
"Answers at once that it is running, with a call id: `calls` with that id says what it sent " +
|
||||||
"(a push can reload the bus, which then refuses any answer still to come).",
|
"(a push can reload the bus, which then refuses any answer still to come). A push by hand is a repair, " +
|
||||||
|
"and says why: recorded in the hand-act log (novox/hq to-be 45 §7).",
|
||||||
Input: schema(map[string]string{
|
Input: schema(map[string]string{
|
||||||
"node": "the machine's name; without it, every machine that is behind",
|
"node": "the machine's name; without it, every machine that is behind",
|
||||||
"behind": "\"true\": every machine that is behind, the whole mesh — the same as naming none, said outright; not with node",
|
"behind": "\"true\": every machine that is behind, the whole mesh — the same as naming none, said outright; not with node",
|
||||||
}, nil, "behind")},
|
"why": "why this is pushed by hand: recorded in the hand-act log",
|
||||||
{Name: "rotate", Description: "Replace a credential. A pair credential, by provision (and a consuming machine, " +
|
"cause": "the cause in a word, or a condition's kind — the word a second push for the same reason uses (optional)",
|
||||||
|
}, []string{"why"}, "behind")},
|
||||||
|
{Name: "rotate", Description: "Replace a credential. A pair credential, by provision (and a consuming machine and module, " +
|
||||||
"else every holder): both ends are re-sent together. Or a module's own secret, by machine, module and " +
|
"else every holder): both ends are re-sent together. Or a module's own secret, by machine, module and " +
|
||||||
"name: made anew and the machine sent, so the module starts again on it — only for a secret its " +
|
"name: made anew and the machine sent, so the module starts again on it — only for a secret its " +
|
||||||
"definition says it reads at start; a value given to the mesh, or one the module applies to a backend, is refused with the reason.",
|
"definition says it reads at start; a value given to the mesh, or one the module applies to a backend, is refused with the reason.",
|
||||||
@@ -150,7 +155,7 @@ var ControllerVerbs = []Verb{
|
|||||||
"provision": "a pair credential: the provision whose credential to replace",
|
"provision": "a pair credential: the provision whose credential to replace",
|
||||||
"consumer": "with provision: only the holder on this machine (optional)",
|
"consumer": "with provision: only the holder on this machine (optional)",
|
||||||
"node": "an own secret: the machine",
|
"node": "an own secret: the machine",
|
||||||
"module": "an own secret: the module",
|
"module": "an own secret: the module; with provision: only this consuming module's credential (optional)",
|
||||||
"secret": "an own secret: its name in the module's definition",
|
"secret": "an own secret: its name in the module's definition",
|
||||||
}, nil)},
|
}, nil)},
|
||||||
{Name: "issue", Description: "Give a module on a machine its account on the bus: minted, and sealed to the " +
|
{Name: "issue", Description: "Give a module on a machine its account on the bus: minted, and sealed to the " +
|
||||||
@@ -206,6 +211,53 @@ var ControllerVerbs = []Verb{
|
|||||||
Input: schema(map[string]string{"node": "one machine; every holder when absent"}, nil)},
|
Input: schema(map[string]string{"node": "one machine; every holder when absent"}, nil)},
|
||||||
{Name: "resume", Description: "The build seat's holder on one machine — or every holder — takes builds again.",
|
{Name: "resume", Description: "The build seat's holder on one machine — or every holder — takes builds again.",
|
||||||
Input: schema(map[string]string{"node": "one machine; every holder when absent"}, nil)},
|
Input: schema(map[string]string{"node": "one machine; every holder when absent"}, nil)},
|
||||||
|
// Acts done by hand, and what the bounds are set from (novox/hq to-be 45 §7, Phase 0).
|
||||||
|
{Name: "hand-act", Description: "Record an act done by hand outside the mesh — a container restarted, a file " +
|
||||||
|
"edited, a service started on a machine — with why and its cause, in the hand-act log beside the pushes and " +
|
||||||
|
"plans ended by hand (novox/hq to-be 45 §7). A cause recorded twice in a fortnight is a healer wanted.",
|
||||||
|
Input: schema(map[string]string{
|
||||||
|
"what": "what was done, in a line",
|
||||||
|
"why": "why it had to be done by hand",
|
||||||
|
"cause": "the cause in a word, or a condition's kind — the word a second act for the same reason uses",
|
||||||
|
"condition": "the key of the condition it addressed, if any (optional)",
|
||||||
|
}, []string{"what", "why", "cause"})},
|
||||||
|
{Name: "hand-acts", Description: "What was done by hand lately — pushes, plans ended, consumers re-made, acts " +
|
||||||
|
"recorded — who, why and the cause of each, and which causes repeat: each repeat is a healer the mesh lacks.",
|
||||||
|
Input: schema(map[string]string{"days": "how many days back (default 14)"}, nil)},
|
||||||
|
{Name: "durations", Description: "How long things take, as the controller measured them: a send to its machine's " +
|
||||||
|
"report (apply), a machine's silence between words (heartbeat-gap), a plan's tier, a build — per machine, " +
|
||||||
|
"repository or module, with median, p90 and max. What the core's bounds are set from (novox/hq to-be 45 Phase 0).",
|
||||||
|
Input: schema(map[string]string{
|
||||||
|
"kind": "one kind: apply, heartbeat-gap, plan-tier or build; every kind when absent",
|
||||||
|
"days": "how many days back (default 14)",
|
||||||
|
}, nil)},
|
||||||
|
// What is wrong, and the self-check (novox/hq to-be 45 §2, §4).
|
||||||
|
{Name: "conditions", Description: "What is wrong with the mesh now: every open condition, urgent first, " +
|
||||||
|
"then oldest — raised by the watchdogs of the signals table, the self-check's probes and the providers' " +
|
||||||
|
"own words, and cleared when observation says it is resolved, never by hand. Given a key, that one " +
|
||||||
|
"whole with its evidence; with history, every transition lately; with silence, stop one's messages " +
|
||||||
|
"for a while — a hand act, which says why (novox/hq to-be 45 §2).",
|
||||||
|
Input: schema(map[string]string{
|
||||||
|
"key": "a condition's key: that one whole; with history, only its transitions",
|
||||||
|
"scope": "only this scope: machine, plan, call, build, merge, provider, seat, bus, core, probe or mesh",
|
||||||
|
"severity": "only urgent, or only warning",
|
||||||
|
"machine": "only those about this machine",
|
||||||
|
"history": "\"true\": every raising, change, silence and clearing lately, oldest first",
|
||||||
|
"days": "with history: how many days back (default 7, at most 90)",
|
||||||
|
"silence": "a condition's key: send no message for it for a while; it stays open and in status",
|
||||||
|
"for": "with silence: how long — 30m, 4h, 2d; at most 7d",
|
||||||
|
"why": "with silence: why — required, and recorded in the hand-act log",
|
||||||
|
"cause": "with silence: the cause in a word (the condition's kind when absent)",
|
||||||
|
}, nil, "history")},
|
||||||
|
{Name: "doctor", Description: "The self-check (novox/hq to-be 45 §4): the last run's verdict at once — " +
|
||||||
|
"each probe of the design's live invariants passed, failed or could not run, and how long ago. With " +
|
||||||
|
"run, a run now; with probes, the registry; with signals, every row of the signals table and the age " +
|
||||||
|
"of its newest signal. Runs every five minutes on its own; each failure is an open condition.",
|
||||||
|
Input: schema(map[string]string{
|
||||||
|
"run": "\"true\": run every probe now and answer the verdict",
|
||||||
|
"probes": "\"true\": the registry — what each probe asserts, and the condition it raises",
|
||||||
|
"signals": "\"true\": the signals table, each row with the age of its newest signal",
|
||||||
|
}, nil, "run", "probes", "signals")},
|
||||||
{Name: "build", Description: "Have the build machine build a repository. Answers at once with the build's id: " +
|
{Name: "build", Description: "Have the build machine build a repository. Answers at once with the build's id: " +
|
||||||
"`builds` with that id follows it line by line, and the module is registered when the outcome comes.",
|
"`builds` with that id follows it line by line, and the module is registered when the outcome comes.",
|
||||||
Input: schema(map[string]string{
|
Input: schema(map[string]string{
|
||||||
|
|||||||
@@ -0,0 +1,159 @@
|
|||||||
|
package conditions
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The condition store on the bus (to-be 45 §2, ADR 0201): the controller's two buckets, asserted at
|
||||||
|
// its start like its calls and its hand-act log.
|
||||||
|
|
||||||
|
// OnTheBus opens the store and its history on a connection.
|
||||||
|
func OnTheBus(ctx context.Context, conn *nats.Conn) (Backend, History, error) {
|
||||||
|
api, err := jetstream.New(conn)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
open, err := api.KeyValue(ctx, broker.ConditionsBucket)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("the condition store %s is not on the bus — the controller asserts it at "+
|
||||||
|
"its start, so one older than this has not: %w", broker.ConditionsBucket, err)
|
||||||
|
}
|
||||||
|
history, err := api.KeyValue(ctx, broker.ConditionHistoryBucket)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("the condition history %s is not on the bus — the controller asserts it "+
|
||||||
|
"at its start, so one older than this has not: %w", broker.ConditionHistoryBucket, err)
|
||||||
|
}
|
||||||
|
return busStore{open}, &busHistory{api: api, kv: history}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type busStore struct{ kv jetstream.KeyValue }
|
||||||
|
|
||||||
|
func (b busStore) Get(ctx context.Context, key string) (Entry, bool, error) {
|
||||||
|
e, err := b.kv.Get(ctx, key)
|
||||||
|
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||||
|
return Entry{}, false, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return Entry{}, false, err
|
||||||
|
}
|
||||||
|
return Entry{Value: e.Value(), Revision: e.Revision()}, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b busStore) Create(ctx context.Context, key string, value []byte) error {
|
||||||
|
_, err := b.kv.Create(ctx, key, value)
|
||||||
|
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||||
|
return ErrMoved
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b busStore) Update(ctx context.Context, key string, value []byte, revision uint64) error {
|
||||||
|
_, err := b.kv.Update(ctx, key, value, revision)
|
||||||
|
return moved(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b busStore) Delete(ctx context.Context, key string, revision uint64) error {
|
||||||
|
return moved(b.kv.Delete(ctx, key, jetstream.LastRevision(revision)))
|
||||||
|
}
|
||||||
|
|
||||||
|
// moved reads the server's refusal of a compare-and-set as what it is.
|
||||||
|
func moved(err error) error {
|
||||||
|
var apiErr *jetstream.APIError
|
||||||
|
if errors.As(err, &apiErr) && apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence {
|
||||||
|
return ErrMoved
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// All is every key, read through a watch that hands over each current value and then says it has.
|
||||||
|
func (b busStore) All(ctx context.Context) (map[string]Entry, error) {
|
||||||
|
w, err := b.kv.WatchAll(ctx, jetstream.IgnoreDeletes())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer func() { _ = w.Stop() }()
|
||||||
|
out := map[string]Entry{}
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, fmt.Errorf("reading the condition store: %w", ctx.Err())
|
||||||
|
case e := <-w.Updates():
|
||||||
|
if e == nil {
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
out[e.Key()] = Entry{Value: e.Value(), Revision: e.Revision()}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// busHistory keeps each transition under a key of its time and a sequence, and reads them back
|
||||||
|
// from a moment through the stream under the bucket — by time, so a read of the last ten minutes
|
||||||
|
// does not read ninety days.
|
||||||
|
type busHistory struct {
|
||||||
|
api jetstream.JetStream
|
||||||
|
kv jetstream.KeyValue
|
||||||
|
seq atomic.Uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *busHistory) Append(ctx context.Context, e Event) error {
|
||||||
|
body, err := json.Marshal(e)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
key := strconv.FormatInt(e.At.UnixNano(), 10) + "-" + strconv.FormatUint(h.seq.Add(1), 10)
|
||||||
|
_, err = h.kv.Put(ctx, key, body)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// historyQuiet is how long a read of the history waits for one more transition before it takes the
|
||||||
|
// stream as read to its end; it answers at once while it holds something.
|
||||||
|
const historyQuiet = 2 * time.Second
|
||||||
|
|
||||||
|
func (h *busHistory) Since(ctx context.Context, since time.Time) ([]Event, error) {
|
||||||
|
start := since
|
||||||
|
consumer, err := h.api.OrderedConsumer(ctx, "KV_"+broker.ConditionHistoryBucket, jetstream.OrderedConsumerConfig{
|
||||||
|
DeliverPolicy: jetstream.DeliverByStartTimePolicy, OptStartTime: &start,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("reading the condition history: %w", err)
|
||||||
|
}
|
||||||
|
info, err := consumer.Info(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("reading the condition history: %w", err)
|
||||||
|
}
|
||||||
|
var out []Event
|
||||||
|
pending := info.NumPending
|
||||||
|
for pending > 0 {
|
||||||
|
msg, err := consumer.Next(jetstream.FetchMaxWait(historyQuiet))
|
||||||
|
if err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
// Nothing more within the quiet wait: read to its end.
|
||||||
|
break
|
||||||
|
}
|
||||||
|
meta, err := msg.Metadata()
|
||||||
|
if err != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
pending = meta.NumPending
|
||||||
|
var e Event
|
||||||
|
if len(msg.Data()) > 0 && json.Unmarshal(msg.Data(), &e) == nil && e.Key != "" {
|
||||||
|
out = append(out, e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.SliceStable(out, func(i, j int) bool { return out[i].At.Before(out[j].At) })
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,117 @@
|
|||||||
|
package conditions
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The condition store against a real server: compare-and-set, an unreadable store refused, and the
|
||||||
|
// history read back by time are claims about what the bus does.
|
||||||
|
|
||||||
|
func busStoreForTest(t *testing.T) (*broker.JetStream, Backend, History) {
|
||||||
|
t.Helper()
|
||||||
|
url := os.Getenv("MESH_TEST_NATS")
|
||||||
|
if url == "" {
|
||||||
|
t.Skip("MESH_TEST_NATS unset")
|
||||||
|
}
|
||||||
|
js, err := broker.Dial(url)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(js.Close)
|
||||||
|
api, err := jetstream.New(js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = api.DeleteKeyValue(t.Context(), broker.ConditionsBucket)
|
||||||
|
_ = api.DeleteKeyValue(t.Context(), broker.ConditionHistoryBucket)
|
||||||
|
if err := js.EnsureControllerBuckets(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
store, history, err := OnTheBus(t.Context(), js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return js, store, history
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A condition outlives the controller that raised it**, and two writers on the bus cannot lose each
|
||||||
|
// other's word: a stale revision is refused as moved.
|
||||||
|
func TestNatsTheStoreKeepsConditionsByCompareAndSet(t *testing.T) {
|
||||||
|
_, store, history := busStoreForTest(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
told := &Told{}
|
||||||
|
k := NewKeeper(ctx, Options{Store: store, History: history, Teller: told})
|
||||||
|
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
k.Close(context.Background())
|
||||||
|
|
||||||
|
again := NewKeeper(ctx, Options{Store: store, History: history})
|
||||||
|
defer again.Close(context.Background())
|
||||||
|
open, err := again.Open(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(open) != 1 || open[0].Observations != 2 {
|
||||||
|
t.Fatalf("a new keeper read %+v", open)
|
||||||
|
}
|
||||||
|
e, _, err := store.Get(ctx, "machine.ace.silent")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := store.Update(ctx, "machine.ace.silent", []byte(`{}`), e.Revision-1); err != ErrMoved {
|
||||||
|
t.Fatalf("a write at a stale revision answered %v", err)
|
||||||
|
}
|
||||||
|
if err := store.Create(ctx, "machine.ace.silent", []byte(`{}`)); err != ErrMoved {
|
||||||
|
t.Fatalf("creating an open condition answered %v", err)
|
||||||
|
}
|
||||||
|
if err := store.Delete(ctx, "machine.ace.silent", e.Revision-1); err != ErrMoved {
|
||||||
|
t.Fatalf("a delete at a stale revision answered %v", err)
|
||||||
|
}
|
||||||
|
if cleared, err := again.Clear(ctx, "machine.ace.silent", "heard"); err != nil || !cleared {
|
||||||
|
t.Fatalf("cleared %v: %v", cleared, err)
|
||||||
|
}
|
||||||
|
// Raised again at once: the store takes a key whose last word was a delete.
|
||||||
|
if _, err := again.Observe(ctx, silent("ace")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, _, _ := again.Get(ctx, "machine.ace.silent")
|
||||||
|
if got.Count != 2 {
|
||||||
|
t.Fatalf("raised again after its clearing as %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The history is read back from a moment, oldest first**, through the stream under its bucket.
|
||||||
|
func TestNatsTheHistoryIsReadByTime(t *testing.T) {
|
||||||
|
_, store, history := busStoreForTest(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
k := NewKeeper(ctx, Options{Store: store, History: history})
|
||||||
|
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := k.Clear(ctx, "machine.ace.silent", "heard"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
k.Close(context.Background())
|
||||||
|
all, err := history.Since(ctx, time.Now().Add(-time.Hour))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(all) != 2 || all[0].Change != ChangeRaised || all[1].Change != ChangeCleared {
|
||||||
|
t.Fatalf("history %+v", all)
|
||||||
|
}
|
||||||
|
none, err := history.Since(ctx, time.Now().Add(time.Hour))
|
||||||
|
if err != nil || len(none) != 0 {
|
||||||
|
t.Fatalf("history from the future: %+v %v", none, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,232 @@
|
|||||||
|
// Package conditions is the condition store (novox/hq to-be 45 §2, ADR 0227 rules 5 and 6).
|
||||||
|
//
|
||||||
|
// **A condition is a durable fact about something the mesh owns that is wrong.** Until this, every
|
||||||
|
// one of the forty-eight core failures of research 031 was noticed because a person or an agent
|
||||||
|
// looked: the mesh's own answers carried the fact for whoever asked, and told nobody. A condition is
|
||||||
|
// raised when an observation says something is wrong past its bound, kept with since-when, evidence
|
||||||
|
// and who can resolve it, said on the bus as it changes, and cleared when an observation says it is
|
||||||
|
// resolved — never by hand.
|
||||||
|
//
|
||||||
|
// The controller is the store's only writer (to-be 45 §1). Two of its processes may write at once —
|
||||||
|
// the serving controller's watchdogs, and a command a person runs to silence one — so every write
|
||||||
|
// is a compare-and-set on the key's revision, and a write that lost the race reads again and redoes
|
||||||
|
// itself rather than overwriting what the other said.
|
||||||
|
package conditions
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Severity is how soon the operator is needed: two levels, no more (to-be 45 §2).
|
||||||
|
type Severity string
|
||||||
|
|
||||||
|
const (
|
||||||
|
// Urgent needs the operator now.
|
||||||
|
Urgent Severity = "urgent"
|
||||||
|
// Warning needs the operator when they can.
|
||||||
|
Warning Severity = "warning"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The scopes a condition's key starts with: what kind of thing is wrong.
|
||||||
|
const (
|
||||||
|
ScopeMachine = "machine"
|
||||||
|
ScopePlan = "plan"
|
||||||
|
ScopeCall = "call"
|
||||||
|
ScopeBuild = "build"
|
||||||
|
ScopeMerge = "merge"
|
||||||
|
ScopeProvider = "provider"
|
||||||
|
ScopeSeat = "seat"
|
||||||
|
ScopeBus = "bus"
|
||||||
|
ScopeCore = "core"
|
||||||
|
ScopeProbe = "probe"
|
||||||
|
ScopeMesh = "mesh"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Scopes is every scope, in the order a person reads them.
|
||||||
|
var Scopes = []string{ScopeMachine, ScopePlan, ScopeCall, ScopeBuild, ScopeMerge, ScopeProvider,
|
||||||
|
ScopeSeat, ScopeBus, ScopeCore, ScopeProbe, ScopeMesh}
|
||||||
|
|
||||||
|
// Who resolves a condition.
|
||||||
|
const (
|
||||||
|
// ResolverSelf clears on observation: the signal returns, the probe passes.
|
||||||
|
ResolverSelf = "self"
|
||||||
|
// ResolverOperator needs a person: a healer's budget spent, or a repair that could only destroy.
|
||||||
|
ResolverOperator = "operator"
|
||||||
|
// ResolverAgent is work handed to an agent (research 017; not raised by anything yet).
|
||||||
|
ResolverAgent = "agent"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ResolverHealer is the resolver of a condition a registered healer works on (Phase 3).
|
||||||
|
func ResolverHealer(name string) string { return "healer:" + name }
|
||||||
|
|
||||||
|
// Subject is what the condition is about: its scope, its id within the scope, and the machine it
|
||||||
|
// concerns when there is one.
|
||||||
|
type Subject struct {
|
||||||
|
Scope string `json:"scope"`
|
||||||
|
ID string `json:"id"`
|
||||||
|
Machine string `json:"machine,omitempty"`
|
||||||
|
// Also are the other machines it concerns: a consumer's, for a provider failing it.
|
||||||
|
Also []string `json:"also,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Evidence is one observation, as it was said.
|
||||||
|
type Evidence struct {
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
Said string `json:"said"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Attempt is one healer's try at a condition (to-be 45 §7; written from Phase 3).
|
||||||
|
type Attempt struct {
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
What string `json:"what"`
|
||||||
|
Outcome string `json:"outcome"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Silence is a person saying they know: no messages until it ends (to-be 45 §2). Recorded as a hand
|
||||||
|
// act; the condition stays open, and `status` still says it.
|
||||||
|
type Silence struct {
|
||||||
|
Until time.Time `json:"until"`
|
||||||
|
By string `json:"by"`
|
||||||
|
Why string `json:"why"`
|
||||||
|
Since time.Time `json:"since"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// KeptEvidence is how many observations a condition keeps, newest first.
|
||||||
|
const KeptEvidence = 10
|
||||||
|
|
||||||
|
// MaxSilence is the longest a condition may be silenced at once: past it, a person says so again.
|
||||||
|
const MaxSilence = 7 * 24 * time.Hour
|
||||||
|
|
||||||
|
// ReopenWithin is how soon after it cleared a condition raised again is the same one again, with its
|
||||||
|
// count increased, rather than news (to-be 45 §2).
|
||||||
|
const ReopenWithin = 10 * time.Minute
|
||||||
|
|
||||||
|
// Condition is one open condition, as the store keeps it and its events carry it.
|
||||||
|
type Condition struct {
|
||||||
|
Key string `json:"key"`
|
||||||
|
// Kind is the condition kind: from the signals table, the probe registry or an event kind.
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Subject Subject `json:"subject"`
|
||||||
|
Severity Severity `json:"severity"`
|
||||||
|
// Summary is one line in the mesh's words.
|
||||||
|
Summary string `json:"summary"`
|
||||||
|
// Evidence is the newest observations, at most KeptEvidence, newest first.
|
||||||
|
Evidence []Evidence `json:"evidence"`
|
||||||
|
// Source is the signals-table row, probe or event that raised it: `S1`, `D3`, `provisioner.failing`.
|
||||||
|
Source string `json:"source"`
|
||||||
|
// Raised is when it was first observed this time; LastObserved the newest observation.
|
||||||
|
Raised time.Time `json:"raised"`
|
||||||
|
LastObserved time.Time `json:"last-observed"`
|
||||||
|
// Observations is how many times it was observed since raised.
|
||||||
|
Observations int `json:"observations"`
|
||||||
|
// Count is how many times it has been raised, a reopening within ReopenWithin counted.
|
||||||
|
Count int `json:"count"`
|
||||||
|
Tried []Attempt `json:"tried,omitempty"`
|
||||||
|
Resolver string `json:"resolver"`
|
||||||
|
// Silenced is null when no silence is in force: said, not left out, so a reader need not guess.
|
||||||
|
Silenced *Silence `json:"silenced"`
|
||||||
|
// Epoch is the controller lease epoch that last wrote it. Zero until the lease exists (to-be 45
|
||||||
|
// Phase 2): no controller holds an epoch yet, and a number invented here would be one nobody
|
||||||
|
// could compare.
|
||||||
|
Epoch uint64 `json:"epoch"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// SilencedAt says whether a person's silence is in force at a moment.
|
||||||
|
func (c Condition) SilencedAt(now time.Time) bool {
|
||||||
|
return c.Silenced != nil && now.Before(c.Silenced.Until)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Show is the verb that shows more about a condition, as a message carries it.
|
||||||
|
func (c Condition) Show() string { return "mesh-controller.conditions key=" + c.Key }
|
||||||
|
|
||||||
|
// Observation is one watchdog, probe or event saying something is wrong now.
|
||||||
|
type Observation struct {
|
||||||
|
Scope string
|
||||||
|
// ID is the thing within the scope; several tokens joined by dots where the thing is named by
|
||||||
|
// several (a provider's module, its machine and the consumer).
|
||||||
|
ID string
|
||||||
|
// Token is the last part of the key, short for the kind: `silent` for a machine, `failing` for a
|
||||||
|
// provider. Kind's own word when empty.
|
||||||
|
Token string
|
||||||
|
Kind string
|
||||||
|
Machine string
|
||||||
|
// Also are the other machines it concerns.
|
||||||
|
Also []string
|
||||||
|
Severity Severity
|
||||||
|
Summary string
|
||||||
|
// Said is this observation's evidence, in the mesh's words; Summary when empty.
|
||||||
|
Said string
|
||||||
|
Source string
|
||||||
|
Resolver string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Key is where the observation's condition is kept: `<scope>.<id>.<kind>`, so the same fault said
|
||||||
|
// again is the same condition.
|
||||||
|
func (o Observation) Key() string {
|
||||||
|
token := o.Token
|
||||||
|
if token == "" {
|
||||||
|
token = o.Kind
|
||||||
|
}
|
||||||
|
return Key(o.Scope, o.ID, token)
|
||||||
|
}
|
||||||
|
|
||||||
|
// unsafeKey is anything a key may not hold: the bus takes letters, digits and `-_/=` in a key's
|
||||||
|
// tokens, and a `*` or `>` would make one a wildcard.
|
||||||
|
var unsafeKey = regexp.MustCompile(`[^A-Za-z0-9_=/-]`)
|
||||||
|
|
||||||
|
// Key composes a condition's key from its parts, each token made safe for the bus: a character the
|
||||||
|
// bus would refuse becomes `_`, so a key is never refused for the name of the thing it is about.
|
||||||
|
func Key(scope, id, token string) string {
|
||||||
|
var parts []string
|
||||||
|
for _, p := range append(append([]string{scope}, strings.Split(id, ".")...), token) {
|
||||||
|
p = unsafeKey.ReplaceAllString(strings.TrimSpace(p), "_")
|
||||||
|
if p == "" {
|
||||||
|
p = "_"
|
||||||
|
}
|
||||||
|
parts = append(parts, p)
|
||||||
|
}
|
||||||
|
return strings.Join(parts, ".")
|
||||||
|
}
|
||||||
|
|
||||||
|
// check refuses an observation that could not be said: a condition with no kind, no scope the mesh
|
||||||
|
// knows, or no severity is one nobody could route.
|
||||||
|
func (o Observation) check() error {
|
||||||
|
known := false
|
||||||
|
for _, s := range Scopes {
|
||||||
|
if s == o.Scope {
|
||||||
|
known = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case !known:
|
||||||
|
return fmt.Errorf("a condition's scope is one of %s, not %q", strings.Join(Scopes, ", "), o.Scope)
|
||||||
|
case strings.TrimSpace(o.ID) == "":
|
||||||
|
return fmt.Errorf("a %s condition names what it is about", o.Scope)
|
||||||
|
case strings.TrimSpace(o.Kind) == "":
|
||||||
|
return fmt.Errorf("the condition %s has no kind", o.Key())
|
||||||
|
case o.Severity != Urgent && o.Severity != Warning:
|
||||||
|
return fmt.Errorf("the condition %s is urgent or a warning, not %q", o.Key(), o.Severity)
|
||||||
|
case strings.TrimSpace(o.Summary) == "":
|
||||||
|
return fmt.Errorf("the condition %s says nothing", o.Key())
|
||||||
|
case strings.TrimSpace(o.Source) == "":
|
||||||
|
return fmt.Errorf("the condition %s does not say what raised it", o.Key())
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Order sorts conditions as `status` says them: urgent before warning, then oldest first.
|
||||||
|
func Order(list []Condition) {
|
||||||
|
sort.SliceStable(list, func(i, j int) bool {
|
||||||
|
if list[i].Severity != list[j].Severity {
|
||||||
|
return list[i].Severity == Urgent
|
||||||
|
}
|
||||||
|
if !list[i].Raised.Equal(list[j].Raised) {
|
||||||
|
return list[i].Raised.Before(list[j].Raised)
|
||||||
|
}
|
||||||
|
return list[i].Key < list[j].Key
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
package conditions
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
|
// The events a condition's life emits (to-be 45 §2), as the mesh-controller seat's own: published on
|
||||||
|
// `mesh.seat.mesh-controller.event.<name>`, on the events stream, so a consumer that was away catches
|
||||||
|
// up. **This is a contract**: the operator-channel's holder is written against these names and the
|
||||||
|
// shape of Event, and the controller learns nothing about telling.
|
||||||
|
const (
|
||||||
|
// EventRaised: a condition was raised — new, or the same fault again within ReopenWithin of its
|
||||||
|
// clearing (Change says which). A reopened condition is not news: its key is the one the
|
||||||
|
// first message was about.
|
||||||
|
EventRaised = "condition-raised"
|
||||||
|
// EventChanged: its severity, its resolver or its silence changed. Not every observation: a
|
||||||
|
// condition observed again is written, and says nothing.
|
||||||
|
EventChanged = "condition-changed"
|
||||||
|
// EventCleared: an observation says it is resolved. The condition is removed from the store and
|
||||||
|
// the transition kept in its history.
|
||||||
|
EventCleared = "condition-cleared"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Events is every event a condition's life emits.
|
||||||
|
var Events = []string{EventRaised, EventChanged, EventCleared}
|
||||||
|
|
||||||
|
// HeartbeatEvent is the self-check's heartbeat (to-be 45 §4, S10), said under the same seat at the end
|
||||||
|
// of every run: `{run, at, interval-seconds, counts: {passed, failed, failed-to-run, deferred}, probes,
|
||||||
|
// controller, why}`. mesh-watcher, on a machine that is not the control node, listens for it.
|
||||||
|
const HeartbeatEvent = "doctor-heartbeat"
|
||||||
|
|
||||||
|
// What changed, as an event's Change and a history entry's says it.
|
||||||
|
const (
|
||||||
|
ChangeRaised = "raised"
|
||||||
|
ChangeReopened = "reopened"
|
||||||
|
ChangeSeverity = "severity"
|
||||||
|
ChangeResolver = "resolver"
|
||||||
|
ChangeSilenced = "silenced"
|
||||||
|
ChangeUnsilenced = "silence-ended"
|
||||||
|
ChangeCleared = "cleared"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Event is the body of every condition event, and the shape a history entry keeps: **the condition
|
||||||
|
// itself, at the top level** — key, kind, subject, severity, summary, source, raised, last-observed,
|
||||||
|
// observations, resolver, silenced (null when not), epoch, and the evidence — with what happened to
|
||||||
|
// it beside. One object a consumer reads the same way whichever of the three it is.
|
||||||
|
type Event struct {
|
||||||
|
// Condition is the condition after the transition — as it was last held, for a clearing.
|
||||||
|
Condition
|
||||||
|
// Event is the event's own name, so a body read without its subject still says what it is.
|
||||||
|
Event string `json:"event"`
|
||||||
|
// At is when the transition happened.
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
// Change is what happened: raised, reopened, severity, resolver, silenced, silence-ended, cleared.
|
||||||
|
Change string `json:"change"`
|
||||||
|
// Was is the value before, for a severity or resolver change.
|
||||||
|
Was string `json:"was,omitempty"`
|
||||||
|
// Why says why it cleared, or why it was silenced.
|
||||||
|
Why string `json:"why,omitempty"`
|
||||||
|
// Cleared is when it cleared, on a clearing.
|
||||||
|
Cleared *time.Time `json:"cleared,omitempty"`
|
||||||
|
// Show is the verb that shows more.
|
||||||
|
Show string `json:"show"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// eventFor is the event a change is said under.
|
||||||
|
func eventFor(change string) string {
|
||||||
|
switch change {
|
||||||
|
case ChangeRaised, ChangeReopened:
|
||||||
|
return EventRaised
|
||||||
|
case ChangeCleared:
|
||||||
|
return EventCleared
|
||||||
|
}
|
||||||
|
return EventChanged
|
||||||
|
}
|
||||||
@@ -0,0 +1,147 @@
|
|||||||
|
package conditions
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"sort"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// InMemory is a store and a history held in this process: for tests, and for nothing else — a
|
||||||
|
// condition kept here is forgotten by a restart, which is the fault the store exists to remove.
|
||||||
|
type InMemory struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
values map[string]Entry
|
||||||
|
revision uint64
|
||||||
|
events []Event
|
||||||
|
// Fail, when set, is what every read and write answers: a store that is away.
|
||||||
|
Fail error
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewInMemory is an empty store.
|
||||||
|
func NewInMemory() *InMemory { return &InMemory{values: map[string]Entry{}} }
|
||||||
|
|
||||||
|
func (m *InMemory) Get(_ context.Context, key string) (Entry, bool, error) {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
if m.Fail != nil {
|
||||||
|
return Entry{}, false, m.Fail
|
||||||
|
}
|
||||||
|
e, ok := m.values[key]
|
||||||
|
return e, ok, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *InMemory) Create(_ context.Context, key string, value []byte) error {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
if m.Fail != nil {
|
||||||
|
return m.Fail
|
||||||
|
}
|
||||||
|
if _, ok := m.values[key]; ok {
|
||||||
|
return ErrMoved
|
||||||
|
}
|
||||||
|
m.revision++
|
||||||
|
m.values[key] = Entry{Value: value, Revision: m.revision}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *InMemory) Update(_ context.Context, key string, value []byte, revision uint64) error {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
if m.Fail != nil {
|
||||||
|
return m.Fail
|
||||||
|
}
|
||||||
|
if e, ok := m.values[key]; !ok || e.Revision != revision {
|
||||||
|
return ErrMoved
|
||||||
|
}
|
||||||
|
m.revision++
|
||||||
|
m.values[key] = Entry{Value: value, Revision: m.revision}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *InMemory) Delete(_ context.Context, key string, revision uint64) error {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
if m.Fail != nil {
|
||||||
|
return m.Fail
|
||||||
|
}
|
||||||
|
if e, ok := m.values[key]; !ok || e.Revision != revision {
|
||||||
|
return ErrMoved
|
||||||
|
}
|
||||||
|
delete(m.values, key)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *InMemory) All(context.Context) (map[string]Entry, error) {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
if m.Fail != nil {
|
||||||
|
return nil, m.Fail
|
||||||
|
}
|
||||||
|
out := make(map[string]Entry, len(m.values))
|
||||||
|
for k, v := range m.values {
|
||||||
|
out[k] = v
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *InMemory) Append(_ context.Context, e Event) error {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
if m.Fail != nil {
|
||||||
|
return m.Fail
|
||||||
|
}
|
||||||
|
m.events = append(m.events, e)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *InMemory) Since(_ context.Context, since time.Time) ([]Event, error) {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
if m.Fail != nil {
|
||||||
|
return nil, m.Fail
|
||||||
|
}
|
||||||
|
var out []Event
|
||||||
|
for _, e := range m.events {
|
||||||
|
if !e.At.Before(since) {
|
||||||
|
out = append(out, e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.SliceStable(out, func(i, j int) bool { return out[i].At.Before(out[j].At) })
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Told is a teller that remembers what it was told, for tests.
|
||||||
|
type Told struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
Events []Event
|
||||||
|
Names []string
|
||||||
|
Fail error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *Told) PublishSeatEvent(_ context.Context, seat, event string, body []byte) error {
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
if t.Fail != nil {
|
||||||
|
return t.Fail
|
||||||
|
}
|
||||||
|
if seat != Seat {
|
||||||
|
return errors.New("told under the wrong seat: " + seat)
|
||||||
|
}
|
||||||
|
var e Event
|
||||||
|
if err := json.Unmarshal(body, &e); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
t.Events = append(t.Events, e)
|
||||||
|
t.Names = append(t.Names, event)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Said is a copy of what was told so far.
|
||||||
|
func (t *Told) Said() []Event {
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
return append([]Event(nil), t.Events...)
|
||||||
|
}
|
||||||
@@ -0,0 +1,502 @@
|
|||||||
|
package conditions
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Backend is where the open conditions are kept: one value per key, written by compare-and-set.
|
||||||
|
type Backend interface {
|
||||||
|
// Get is one key's value and revision; false when it holds none.
|
||||||
|
Get(ctx context.Context, key string) (Entry, bool, error)
|
||||||
|
// Create writes a key that holds nothing, and fails with ErrMoved when it holds something.
|
||||||
|
Create(ctx context.Context, key string, value []byte) error
|
||||||
|
// Update writes a key at the revision it was read at, and fails with ErrMoved when it moved.
|
||||||
|
Update(ctx context.Context, key string, value []byte, revision uint64) error
|
||||||
|
// Delete removes a key at the revision it was read at, and fails with ErrMoved when it moved.
|
||||||
|
Delete(ctx context.Context, key string, revision uint64) error
|
||||||
|
// All is every key's value. An error is an error: never an empty store (ADR 0227 rule 4).
|
||||||
|
All(ctx context.Context) (map[string]Entry, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Entry is one key's value, at a revision.
|
||||||
|
type Entry struct {
|
||||||
|
Value []byte
|
||||||
|
Revision uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
// ErrMoved is a compare-and-set that lost: somebody wrote the key since it was read.
|
||||||
|
var ErrMoved = errors.New("the condition was written by somebody else since it was read")
|
||||||
|
|
||||||
|
// History keeps every transition (to-be 45 §2): appended, read back from a moment.
|
||||||
|
type History interface {
|
||||||
|
Append(ctx context.Context, e Event) error
|
||||||
|
// Since is every transition from a moment, oldest first.
|
||||||
|
Since(ctx context.Context, since time.Time) ([]Event, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Teller says a transition on the bus, as the mesh-controller seat's event. The link's bus is one.
|
||||||
|
type Teller interface {
|
||||||
|
PublishSeatEvent(ctx context.Context, seat, event string, body []byte) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Seat is the role the events are said under (novox/hq ADR 0134): the control plane's.
|
||||||
|
const Seat = "mesh-controller"
|
||||||
|
|
||||||
|
// Keeper raises, observes, silences and clears conditions, and says each transition.
|
||||||
|
type Keeper struct {
|
||||||
|
store Backend
|
||||||
|
history History
|
||||||
|
teller Teller
|
||||||
|
now func() time.Time
|
||||||
|
say func(format string, args ...any)
|
||||||
|
changed func()
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
// cleared is when each recently cleared condition cleared and how often it had been raised, so
|
||||||
|
// one raised again within ReopenWithin is the same one again.
|
||||||
|
cleared map[string]clearing
|
||||||
|
|
||||||
|
// out is the transitions still to be said and kept, in order: said by one goroutine, so a
|
||||||
|
// condition's events arrive in the order they happened, and offered again while the bus is away.
|
||||||
|
out chan Event
|
||||||
|
drained chan struct{}
|
||||||
|
closing sync.Once
|
||||||
|
// Unsaid counts the transitions given up on, for the self-check to say.
|
||||||
|
unsaid int
|
||||||
|
}
|
||||||
|
|
||||||
|
type clearing struct {
|
||||||
|
at time.Time
|
||||||
|
count int
|
||||||
|
silenced *Silence
|
||||||
|
}
|
||||||
|
|
||||||
|
// Options are what a Keeper is made with.
|
||||||
|
type Options struct {
|
||||||
|
Store Backend
|
||||||
|
History History
|
||||||
|
// Teller says the transitions; nil says nothing (a test, or a command run with no bus to say on).
|
||||||
|
Teller Teller
|
||||||
|
Now func() time.Time
|
||||||
|
// Say is where a transition that could not be said or kept is said instead.
|
||||||
|
Say func(format string, args ...any)
|
||||||
|
// Changed is told of every transition, at once — for `status`, which leads with what is open.
|
||||||
|
Changed func()
|
||||||
|
}
|
||||||
|
|
||||||
|
// TellFor is how long one transition is offered to the bus before it is said lost.
|
||||||
|
var TellFor = 10 * time.Minute
|
||||||
|
|
||||||
|
// NewKeeper is a keeper over a store. It reads what cleared lately from the history, so a condition
|
||||||
|
// that cleared just before this controller started and is raised again now is a reopening.
|
||||||
|
func NewKeeper(ctx context.Context, o Options) *Keeper {
|
||||||
|
k := &Keeper{store: o.Store, history: o.History, teller: o.Teller, now: o.Now, say: o.Say, changed: o.Changed,
|
||||||
|
cleared: map[string]clearing{}, out: make(chan Event, 1024), drained: make(chan struct{})}
|
||||||
|
if k.now == nil {
|
||||||
|
k.now = time.Now
|
||||||
|
}
|
||||||
|
if k.say == nil {
|
||||||
|
k.say = func(string, ...any) {}
|
||||||
|
}
|
||||||
|
if k.history != nil {
|
||||||
|
if recent, err := k.history.Since(ctx, k.now().Add(-ReopenWithin)); err == nil {
|
||||||
|
for _, e := range recent {
|
||||||
|
if e.Change == ChangeCleared {
|
||||||
|
k.cleared[e.Key] = clearing{at: e.At, count: e.Condition.Count, silenced: e.Condition.Silenced}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
k.say("what cleared lately could not be read from the condition history, so a condition "+
|
||||||
|
"raised again now is said as new rather than reopened: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
go k.telling()
|
||||||
|
return k
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close says what is still to be said, waiting at most until ctx ends.
|
||||||
|
func (k *Keeper) Close(ctx context.Context) {
|
||||||
|
k.closing.Do(func() { close(k.out) })
|
||||||
|
select {
|
||||||
|
case <-k.drained:
|
||||||
|
case <-ctx.Done():
|
||||||
|
k.say("%d condition transition(s) were not yet said when this process ended", len(k.out))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unsaid is how many transitions were given up on since this keeper started.
|
||||||
|
func (k *Keeper) Unsaid() int {
|
||||||
|
k.mu.Lock()
|
||||||
|
defer k.mu.Unlock()
|
||||||
|
return k.unsaid
|
||||||
|
}
|
||||||
|
|
||||||
|
// tries bounds one compare-and-set: two writers rarely race more than once.
|
||||||
|
const tries = 8
|
||||||
|
|
||||||
|
// Observe records one observation: raises the condition if it is not open, and otherwise adds the
|
||||||
|
// evidence. Says a raising, a reopening, and a change of severity or resolver; an observation that
|
||||||
|
// changes neither is written and said nowhere.
|
||||||
|
func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error) {
|
||||||
|
if err := o.check(); err != nil {
|
||||||
|
return Condition{}, err
|
||||||
|
}
|
||||||
|
key := o.Key()
|
||||||
|
for i := 0; i < tries; i++ {
|
||||||
|
now := k.now().UTC()
|
||||||
|
said := o.Said
|
||||||
|
if said == "" {
|
||||||
|
said = o.Summary
|
||||||
|
}
|
||||||
|
entry, found, err := k.store.Get(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
return Condition{}, fmt.Errorf("reading the condition %s: %w", key, err)
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
c := Condition{Key: key, Kind: o.Kind, Subject: Subject{Scope: o.Scope, ID: o.ID, Machine: o.Machine, Also: o.Also},
|
||||||
|
Severity: o.Severity, Summary: o.Summary, Evidence: []Evidence{{At: now, Said: said}},
|
||||||
|
Source: o.Source, Raised: now, LastObserved: now, Observations: 1, Count: 1,
|
||||||
|
Resolver: orSelf(o.Resolver)}
|
||||||
|
change := ChangeRaised
|
||||||
|
k.mu.Lock()
|
||||||
|
if before, ok := k.cleared[key]; ok && now.Sub(before.at) <= ReopenWithin {
|
||||||
|
c.Count, change = before.count+1, ChangeReopened
|
||||||
|
// A silence a person gave the condition before it cleared still holds: they said
|
||||||
|
// they knew, and the same fault again ten minutes later is what they knew about.
|
||||||
|
if before.silenced != nil && now.Before(before.silenced.Until) {
|
||||||
|
c.Silenced = before.silenced
|
||||||
|
}
|
||||||
|
}
|
||||||
|
k.mu.Unlock()
|
||||||
|
body, err := json.Marshal(c)
|
||||||
|
if err != nil {
|
||||||
|
return Condition{}, err
|
||||||
|
}
|
||||||
|
if err := k.store.Create(ctx, key, body); errors.Is(err, ErrMoved) {
|
||||||
|
continue
|
||||||
|
} else if err != nil {
|
||||||
|
return Condition{}, fmt.Errorf("raising the condition %s: %w", key, err)
|
||||||
|
}
|
||||||
|
k.mu.Lock()
|
||||||
|
delete(k.cleared, key)
|
||||||
|
k.mu.Unlock()
|
||||||
|
k.tell(Event{Condition: c, At: now, Change: change})
|
||||||
|
return c, nil
|
||||||
|
}
|
||||||
|
var c Condition
|
||||||
|
if err := json.Unmarshal(entry.Value, &c); err != nil {
|
||||||
|
return Condition{}, fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err)
|
||||||
|
}
|
||||||
|
var changes []Event
|
||||||
|
if o.Severity != c.Severity {
|
||||||
|
changes = append(changes, Event{Change: ChangeSeverity, Was: string(c.Severity)})
|
||||||
|
c.Severity = o.Severity
|
||||||
|
}
|
||||||
|
if r := orSelf(o.Resolver); o.Resolver != "" && r != c.Resolver {
|
||||||
|
changes = append(changes, Event{Change: ChangeResolver, Was: c.Resolver})
|
||||||
|
c.Resolver = r
|
||||||
|
}
|
||||||
|
c.Summary, c.Source, c.LastObserved = o.Summary, o.Source, now
|
||||||
|
if o.Machine != "" {
|
||||||
|
c.Subject.Machine = o.Machine
|
||||||
|
}
|
||||||
|
if len(o.Also) > 0 {
|
||||||
|
c.Subject.Also = o.Also
|
||||||
|
}
|
||||||
|
c.Observations++
|
||||||
|
c.Evidence = append([]Evidence{{At: now, Said: said}}, c.Evidence...)
|
||||||
|
if len(c.Evidence) > KeptEvidence {
|
||||||
|
c.Evidence = c.Evidence[:KeptEvidence]
|
||||||
|
}
|
||||||
|
body, err := json.Marshal(c)
|
||||||
|
if err != nil {
|
||||||
|
return Condition{}, err
|
||||||
|
}
|
||||||
|
if err := k.store.Update(ctx, key, body, entry.Revision); errors.Is(err, ErrMoved) {
|
||||||
|
continue
|
||||||
|
} else if err != nil {
|
||||||
|
return Condition{}, fmt.Errorf("observing the condition %s: %w", key, err)
|
||||||
|
}
|
||||||
|
for _, e := range changes {
|
||||||
|
e.At, e.Condition = now, c
|
||||||
|
k.tell(e)
|
||||||
|
}
|
||||||
|
return c, nil
|
||||||
|
}
|
||||||
|
return Condition{}, fmt.Errorf("the condition %s kept moving under this write; %d tries", key, tries)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clear removes a condition an observation says is resolved, and says so. False when none was open.
|
||||||
|
func (k *Keeper) Clear(ctx context.Context, key, why string) (bool, error) {
|
||||||
|
for i := 0; i < tries; i++ {
|
||||||
|
entry, found, err := k.store.Get(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("reading the condition %s: %w", key, err)
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
var c Condition
|
||||||
|
if err := json.Unmarshal(entry.Value, &c); err != nil {
|
||||||
|
// Unreadable is not resolved: kept, and said, rather than removed unread.
|
||||||
|
return false, fmt.Errorf("the condition %s on the bus cannot be read, so it is not cleared: %w", key, err)
|
||||||
|
}
|
||||||
|
if err := k.store.Delete(ctx, key, entry.Revision); errors.Is(err, ErrMoved) {
|
||||||
|
continue
|
||||||
|
} else if err != nil {
|
||||||
|
return false, fmt.Errorf("clearing the condition %s: %w", key, err)
|
||||||
|
}
|
||||||
|
now := k.now().UTC()
|
||||||
|
k.mu.Lock()
|
||||||
|
k.cleared[key] = clearing{at: now, count: c.Count, silenced: c.Silenced}
|
||||||
|
k.mu.Unlock()
|
||||||
|
k.tell(Event{Condition: c, At: now, Change: ChangeCleared, Why: why, Cleared: &now})
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
return false, fmt.Errorf("the condition %s kept moving under this clearing; %d tries", key, tries)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reconcile is one source's whole observation: every condition it observes is observed, and every
|
||||||
|
// condition it raised before and no longer observes is cleared — the observation says it is
|
||||||
|
// resolved. A source that could not observe must not call this: an empty observation clears all it
|
||||||
|
// raised, which is exactly the fault of saying "none" for "I could not tell" (ADR 0227 rule 4).
|
||||||
|
func (k *Keeper) Reconcile(ctx context.Context, source string, observed []Observation) error {
|
||||||
|
all, err := k.Open(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
seen := map[string]bool{}
|
||||||
|
var problems []string
|
||||||
|
for _, o := range observed {
|
||||||
|
o.Source = source
|
||||||
|
seen[o.Key()] = true
|
||||||
|
if _, err := k.Observe(ctx, o); err != nil {
|
||||||
|
problems = append(problems, err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, c := range all {
|
||||||
|
if c.Source != source || seen[c.Key] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := k.Clear(ctx, c.Key, source+" no longer observes it"); err != nil {
|
||||||
|
problems = append(problems, err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(problems) > 0 {
|
||||||
|
return errors.New(strings.Join(problems, "; "))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Silence stops a condition's messages for a while, with a reason, by somebody (to-be 45 §2). The
|
||||||
|
// condition stays open and `status` still says it; recording the act in the hand-act log is the
|
||||||
|
// caller's, which knows who acted.
|
||||||
|
func (k *Keeper) Silence(ctx context.Context, key string, d time.Duration, by, why string) (Condition, error) {
|
||||||
|
if strings.TrimSpace(why) == "" {
|
||||||
|
return Condition{}, errors.New("a silence says why: --why <text>")
|
||||||
|
}
|
||||||
|
if d <= 0 || d > MaxSilence {
|
||||||
|
return Condition{}, fmt.Errorf("a condition is silenced for a while, at most %s — not %s", MaxSilence, d)
|
||||||
|
}
|
||||||
|
for i := 0; i < tries; i++ {
|
||||||
|
entry, found, err := k.store.Get(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
return Condition{}, fmt.Errorf("reading the condition %s: %w", key, err)
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
return Condition{}, fmt.Errorf("no condition %s is open — `conditions` lists them", key)
|
||||||
|
}
|
||||||
|
var c Condition
|
||||||
|
if err := json.Unmarshal(entry.Value, &c); err != nil {
|
||||||
|
return Condition{}, fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err)
|
||||||
|
}
|
||||||
|
now := k.now().UTC()
|
||||||
|
c.Silenced = &Silence{Until: now.Add(d), By: by, Why: strings.TrimSpace(why), Since: now}
|
||||||
|
body, err := json.Marshal(c)
|
||||||
|
if err != nil {
|
||||||
|
return Condition{}, err
|
||||||
|
}
|
||||||
|
if err := k.store.Update(ctx, key, body, entry.Revision); errors.Is(err, ErrMoved) {
|
||||||
|
continue
|
||||||
|
} else if err != nil {
|
||||||
|
return Condition{}, fmt.Errorf("silencing the condition %s: %w", key, err)
|
||||||
|
}
|
||||||
|
k.tell(Event{Condition: c, At: now, Change: ChangeSilenced, Why: c.Silenced.Why})
|
||||||
|
return c, nil
|
||||||
|
}
|
||||||
|
return Condition{}, fmt.Errorf("the condition %s kept moving under this silence; %d tries", key, tries)
|
||||||
|
}
|
||||||
|
|
||||||
|
// EndSilences ends every silence that has run out, and says each: the condition is still open, and
|
||||||
|
// its messages start again.
|
||||||
|
func (k *Keeper) EndSilences(ctx context.Context) error {
|
||||||
|
all, err := k.Open(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
now := k.now().UTC()
|
||||||
|
for _, c := range all {
|
||||||
|
if c.Silenced == nil || now.Before(c.Silenced.Until) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for i := 0; i < tries; i++ {
|
||||||
|
entry, found, err := k.store.Get(ctx, c.Key)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
var held Condition
|
||||||
|
if err := json.Unmarshal(entry.Value, &held); err != nil {
|
||||||
|
return fmt.Errorf("the condition %s on the bus cannot be read: %w", c.Key, err)
|
||||||
|
}
|
||||||
|
if held.Silenced == nil || now.Before(held.Silenced.Until) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
was := held.Silenced.Why
|
||||||
|
held.Silenced = nil
|
||||||
|
body, err := json.Marshal(held)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := k.store.Update(ctx, c.Key, body, entry.Revision); errors.Is(err, ErrMoved) {
|
||||||
|
continue
|
||||||
|
} else if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
k.tell(Event{Condition: held, At: now, Change: ChangeUnsilenced, Why: was})
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Open is every open condition, urgent first and then oldest first.
|
||||||
|
func (k *Keeper) Open(ctx context.Context) ([]Condition, error) {
|
||||||
|
return Read(ctx, k.store)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get is one open condition.
|
||||||
|
func (k *Keeper) Get(ctx context.Context, key string) (Condition, bool, error) {
|
||||||
|
return ReadOne(ctx, k.store, key)
|
||||||
|
}
|
||||||
|
|
||||||
|
// HistorySince is every transition from a moment, oldest first.
|
||||||
|
func (k *Keeper) HistorySince(ctx context.Context, since time.Time) ([]Event, error) {
|
||||||
|
if k.history == nil {
|
||||||
|
return nil, errors.New("this keeper has no history to read")
|
||||||
|
}
|
||||||
|
return k.history.Since(ctx, since)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read is every open condition in a store, in the order status says them. A value that cannot be
|
||||||
|
// read is an error naming its key, never a condition left out (ADR 0227 rule 4).
|
||||||
|
func Read(ctx context.Context, store Backend) ([]Condition, error) {
|
||||||
|
all, err := store.All(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the open conditions cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
out := make([]Condition, 0, len(all))
|
||||||
|
for key, e := range all {
|
||||||
|
var c Condition
|
||||||
|
if err := json.Unmarshal(e.Value, &c); err != nil {
|
||||||
|
return nil, fmt.Errorf("the condition %s cannot be read: %w", key, err)
|
||||||
|
}
|
||||||
|
out = append(out, c)
|
||||||
|
}
|
||||||
|
Order(out)
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadOne is one open condition from a store.
|
||||||
|
func ReadOne(ctx context.Context, store Backend, key string) (Condition, bool, error) {
|
||||||
|
e, found, err := store.Get(ctx, key)
|
||||||
|
if err != nil || !found {
|
||||||
|
return Condition{}, found, err
|
||||||
|
}
|
||||||
|
var c Condition
|
||||||
|
if err := json.Unmarshal(e.Value, &c); err != nil {
|
||||||
|
return Condition{}, false, fmt.Errorf("the condition %s cannot be read: %w", key, err)
|
||||||
|
}
|
||||||
|
return c, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// tell queues a transition to be kept and said. Never blocks the caller for long: a queue that is
|
||||||
|
// full is a bus away for a long time, and the transition is said lost rather than holding a watchdog.
|
||||||
|
func (k *Keeper) tell(e Event) {
|
||||||
|
e.Event = eventFor(e.Change)
|
||||||
|
e.Show = e.Condition.Show()
|
||||||
|
if k.changed != nil {
|
||||||
|
k.changed()
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
// A keeper closed while a write was in flight: said, not a panic.
|
||||||
|
if recover() != nil {
|
||||||
|
k.lost(e, errors.New("the keeper was closed"))
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case k.out <- e:
|
||||||
|
default:
|
||||||
|
k.lost(e, errors.New("too many transitions are waiting to be said"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (k *Keeper) lost(e Event, err error) {
|
||||||
|
k.mu.Lock()
|
||||||
|
k.unsaid++
|
||||||
|
k.mu.Unlock()
|
||||||
|
k.say("the condition %s was %s and that could NOT be said or kept: %v", e.Key, e.Change, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// telling keeps and says every transition in order, offering each again while the bus is away.
|
||||||
|
func (k *Keeper) telling() {
|
||||||
|
defer close(k.drained)
|
||||||
|
for e := range k.out {
|
||||||
|
body, err := json.Marshal(e)
|
||||||
|
if err != nil {
|
||||||
|
k.lost(e, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
deadline := time.Now().Add(TellFor)
|
||||||
|
wait := 200 * time.Millisecond
|
||||||
|
kept, said := k.history == nil, k.teller == nil
|
||||||
|
for {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
if !kept {
|
||||||
|
kept = k.history.Append(ctx, e) == nil
|
||||||
|
}
|
||||||
|
if !said {
|
||||||
|
said = k.teller.PublishSeatEvent(ctx, Seat, e.Event, body) == nil
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
if kept && said {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
what := "said"
|
||||||
|
if !kept {
|
||||||
|
what = "kept in the history"
|
||||||
|
}
|
||||||
|
k.lost(e, fmt.Errorf("not %s within %s", what, TellFor))
|
||||||
|
break
|
||||||
|
}
|
||||||
|
time.Sleep(wait)
|
||||||
|
wait = min(2*wait, 10*time.Second)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func orSelf(resolver string) string {
|
||||||
|
if resolver == "" {
|
||||||
|
return ResolverSelf
|
||||||
|
}
|
||||||
|
return resolver
|
||||||
|
}
|
||||||
@@ -0,0 +1,379 @@
|
|||||||
|
package conditions
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// clock is a time a test moves by hand.
|
||||||
|
type clock struct{ at time.Time }
|
||||||
|
|
||||||
|
func (c *clock) now() time.Time { return c.at }
|
||||||
|
func (c *clock) pass(d time.Duration) { c.at = c.at.Add(d) }
|
||||||
|
func newClock() *clock { return &clock{at: time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)} }
|
||||||
|
func keeper(t *testing.T) (*Keeper, *InMemory, *Told, *clock) {
|
||||||
|
t.Helper()
|
||||||
|
store, told, c := NewInMemory(), &Told{}, newClock()
|
||||||
|
k := NewKeeper(t.Context(), Options{Store: store, History: store, Teller: told, Now: c.now,
|
||||||
|
Say: func(f string, a ...any) { t.Logf(f, a...) }})
|
||||||
|
t.Cleanup(func() { k.Close(context.Background()) })
|
||||||
|
return k, store, told, c
|
||||||
|
}
|
||||||
|
|
||||||
|
// settled waits until the teller has been told n events.
|
||||||
|
func settled(t *testing.T, told *Told, n int) []Event {
|
||||||
|
t.Helper()
|
||||||
|
deadline := time.Now().Add(5 * time.Second)
|
||||||
|
for {
|
||||||
|
said := told.Said()
|
||||||
|
if len(said) >= n {
|
||||||
|
return said
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatalf("told %d event(s), want %d: %+v", len(said), n, said)
|
||||||
|
}
|
||||||
|
time.Sleep(5 * time.Millisecond)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func silent(node string) Observation {
|
||||||
|
return Observation{Scope: ScopeMachine, ID: node, Kind: "silent", Machine: node, Severity: Warning,
|
||||||
|
Summary: node + " has not been heard from", Source: "S1"}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A condition is raised once, observed many times, and said on the bus only when it changes**
|
||||||
|
// (to-be 45 §2): an observation that changes nothing is written and said nowhere, or the operator's
|
||||||
|
// channel would hear the same fault every thirty seconds.
|
||||||
|
func TestAConditionIsSaidWhenItChangesNotWhenItIsSeenAgain(t *testing.T) {
|
||||||
|
k, _, told, c := keeper(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
c.pass(time.Minute)
|
||||||
|
}
|
||||||
|
urgent := silent("ace")
|
||||||
|
urgent.Severity = Urgent
|
||||||
|
got, err := k.Observe(ctx, urgent)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.Key != "machine.ace.silent" || got.Observations != 4 || got.Count != 1 || got.Severity != Urgent {
|
||||||
|
t.Fatalf("held %+v", got)
|
||||||
|
}
|
||||||
|
if len(got.Evidence) != 4 || !got.Evidence[0].At.Equal(c.at) {
|
||||||
|
t.Fatalf("evidence is not newest first: %+v", got.Evidence)
|
||||||
|
}
|
||||||
|
said := settled(t, told, 2)
|
||||||
|
if said[0].Event != EventRaised || said[0].Change != ChangeRaised || said[1].Event != EventChanged ||
|
||||||
|
said[1].Change != ChangeSeverity || said[1].Was != string(Warning) {
|
||||||
|
t.Fatalf("said %+v", said)
|
||||||
|
}
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
if n := len(told.Said()); n != 2 {
|
||||||
|
t.Fatalf("said %d events for one raising and one change", n)
|
||||||
|
}
|
||||||
|
for i, name := range told.Names {
|
||||||
|
if name != told.Events[i].Event {
|
||||||
|
t.Errorf("event %d published as %s and says it is %s", i, name, told.Events[i].Event)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Evidence is bounded**: a condition open for a week keeps its newest ten observations, not all.
|
||||||
|
func TestEvidenceKeepsTheNewestTen(t *testing.T) {
|
||||||
|
k, _, _, c := keeper(t)
|
||||||
|
var got Condition
|
||||||
|
for i := 0; i < 25; i++ {
|
||||||
|
var err error
|
||||||
|
if got, err = k.Observe(t.Context(), silent("ace")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
c.pass(time.Minute)
|
||||||
|
}
|
||||||
|
if len(got.Evidence) != KeptEvidence || got.Observations != 25 {
|
||||||
|
t.Fatalf("kept %d evidence of %d observations", len(got.Evidence), got.Observations)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Cleared and raised again within ten minutes is the same condition again** (to-be 45 §2): its
|
||||||
|
// count goes up and it is said as reopened, not as news; a person's silence of it still holds.
|
||||||
|
func TestRaisedAgainSoonAfterClearingReopens(t *testing.T) {
|
||||||
|
k, store, told, c := keeper(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := k.Silence(ctx, "machine.ace.silent", time.Hour, "jochen", "the laptop is on the train"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if cleared, err := k.Clear(ctx, "machine.ace.silent", "heard again"); err != nil || !cleared {
|
||||||
|
t.Fatalf("cleared %v: %v", cleared, err)
|
||||||
|
}
|
||||||
|
c.pass(5 * time.Minute)
|
||||||
|
again, err := k.Observe(ctx, silent("ace"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if again.Count != 2 || again.Silenced == nil {
|
||||||
|
t.Fatalf("reopened as %+v", again)
|
||||||
|
}
|
||||||
|
said := settled(t, told, 4)
|
||||||
|
if said[3].Event != EventRaised || said[3].Change != ChangeReopened {
|
||||||
|
t.Fatalf("the reopening was said as %+v", said[3])
|
||||||
|
}
|
||||||
|
// And from a new keeper — the controller restarted between — reading what cleared from history.
|
||||||
|
if _, err := k.Clear(ctx, "machine.ace.silent", "heard again"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
settled(t, told, 5)
|
||||||
|
k.Close(context.Background())
|
||||||
|
next := NewKeeper(ctx, Options{Store: store, History: store, Now: c.now})
|
||||||
|
defer next.Close(context.Background())
|
||||||
|
c.pass(time.Minute)
|
||||||
|
third, err := next.Observe(ctx, silent("ace"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if third.Count != 3 {
|
||||||
|
t.Fatalf("a controller restarted between cleared and raised said it as new: %+v", third)
|
||||||
|
}
|
||||||
|
// Past the window it is news.
|
||||||
|
if _, err := next.Clear(ctx, "machine.ace.silent", "heard again"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
c.pass(ReopenWithin + time.Minute)
|
||||||
|
fourth, err := next.Observe(ctx, silent("ace"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if fourth.Count != 1 || fourth.Silenced != nil {
|
||||||
|
t.Fatalf("raised past the window as %+v", fourth)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A source's whole observation clears what it no longer observes, and only its own.** A watchdog
|
||||||
|
// that stops seeing a fault says it is resolved; it does not clear what another raised.
|
||||||
|
func TestReconcileClearsOnlyTheSourcesOwn(t *testing.T) {
|
||||||
|
k, _, told, _ := keeper(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
other := Observation{Scope: ScopeProbe, ID: "D3", Kind: "probe-failed", Token: "failed", Severity: Warning,
|
||||||
|
Summary: "D3 did not answer", Source: "doctor"}
|
||||||
|
if _, err := k.Observe(ctx, other); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := k.Reconcile(ctx, "S1", []Observation{silent("ace"), silent("g14")}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := k.Reconcile(ctx, "S1", []Observation{silent("g14")}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
open, err := k.Open(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var keys []string
|
||||||
|
for _, c := range open {
|
||||||
|
keys = append(keys, c.Key)
|
||||||
|
}
|
||||||
|
if strings.Join(keys, ",") != "machine.g14.silent,probe.D3.failed" {
|
||||||
|
t.Fatalf("open after the second observation: %v", keys)
|
||||||
|
}
|
||||||
|
said := settled(t, told, 4)
|
||||||
|
last := said[3]
|
||||||
|
if last.Event != EventCleared || last.Key != "machine.ace.silent" || last.Why == "" {
|
||||||
|
t.Fatalf("the clearing was said as %+v", last)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A store that cannot be read is never an empty one** (ADR 0227 rule 4): reconciling against it
|
||||||
|
// clears nothing and says why.
|
||||||
|
func TestAnUnreadableStoreClearsNothing(t *testing.T) {
|
||||||
|
k, store, _, _ := keeper(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
store.Fail = errors.New("the bus is away")
|
||||||
|
if err := k.Reconcile(ctx, "S1", nil); err == nil {
|
||||||
|
t.Fatal("reconciled against a store it could not read")
|
||||||
|
}
|
||||||
|
if _, err := k.Open(ctx); err == nil {
|
||||||
|
t.Fatal("an unreadable store answered as read")
|
||||||
|
}
|
||||||
|
store.Fail = nil
|
||||||
|
store.values["machine.g14.silent"] = Entry{Value: []byte("{not a condition"), Revision: 99}
|
||||||
|
if _, err := k.Open(ctx); err == nil || !strings.Contains(err.Error(), "machine.g14.silent") {
|
||||||
|
t.Fatalf("an unreadable condition was left out rather than said: %v", err)
|
||||||
|
}
|
||||||
|
if cleared, err := k.Clear(ctx, "machine.g14.silent", "x"); err == nil || cleared {
|
||||||
|
t.Fatal("an unreadable condition was cleared unread")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A silence is bounded, says why, and ends on its own** (to-be 45 §2): the condition stays open
|
||||||
|
// through it, and its messages start again when it ends.
|
||||||
|
func TestASilenceIsBoundedAndEnds(t *testing.T) {
|
||||||
|
k, _, told, c := keeper(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := k.Silence(ctx, "machine.ace.silent", 8*24*time.Hour, "jochen", "away"); err == nil {
|
||||||
|
t.Fatal("silenced for longer than a week")
|
||||||
|
}
|
||||||
|
if _, err := k.Silence(ctx, "machine.ace.silent", time.Hour, "jochen", " "); err == nil {
|
||||||
|
t.Fatal("silenced without a reason")
|
||||||
|
}
|
||||||
|
if _, err := k.Silence(ctx, "machine.nothing.silent", time.Hour, "jochen", "x"); err == nil {
|
||||||
|
t.Fatal("silenced a condition that is not open")
|
||||||
|
}
|
||||||
|
held, err := k.Silence(ctx, "machine.ace.silent", time.Hour, "jochen", "on the train")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !held.SilencedAt(c.at) || held.Silenced.By != "jochen" {
|
||||||
|
t.Fatalf("silenced as %+v", held.Silenced)
|
||||||
|
}
|
||||||
|
c.pass(30 * time.Minute)
|
||||||
|
if err := k.EndSilences(ctx); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
c.pass(31 * time.Minute)
|
||||||
|
if err := k.EndSilences(ctx); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, _, _ := k.Get(ctx, "machine.ace.silent")
|
||||||
|
if got.Silenced != nil {
|
||||||
|
t.Fatalf("a silence past its end still held: %+v", got.Silenced)
|
||||||
|
}
|
||||||
|
said := settled(t, told, 3)
|
||||||
|
if said[1].Change != ChangeSilenced || said[2].Change != ChangeUnsilenced || said[2].Event != EventChanged {
|
||||||
|
t.Fatalf("said %+v", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Two writers never lose each other's word.** The serving controller observes while a person's
|
||||||
|
// command silences: the write that lost the compare-and-set reads again and redoes itself.
|
||||||
|
func TestAWriteThatLostTheRaceRedoesItself(t *testing.T) {
|
||||||
|
k, store, _, _ := keeper(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if _, err := k.Observe(ctx, silent("ace")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
racing := &racingStore{InMemory: store, before: func() {
|
||||||
|
// Another process silences between this keeper's read and its write.
|
||||||
|
other := NewKeeper(ctx, Options{Store: store})
|
||||||
|
defer other.Close(context.Background())
|
||||||
|
if _, err := other.Silence(ctx, "machine.ace.silent", time.Hour, "jochen", "known"); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
k.store = racing
|
||||||
|
got, err := k.Observe(ctx, silent("ace"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.Silenced == nil || got.Observations != 2 {
|
||||||
|
t.Fatalf("the observation overwrote the silence: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// racingStore lets another writer in once, between a read and the write after it.
|
||||||
|
type racingStore struct {
|
||||||
|
*InMemory
|
||||||
|
before func()
|
||||||
|
done bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *racingStore) Update(ctx context.Context, key string, value []byte, revision uint64) error {
|
||||||
|
if !r.done {
|
||||||
|
r.done = true
|
||||||
|
r.before()
|
||||||
|
}
|
||||||
|
return r.InMemory.Update(ctx, key, value, revision)
|
||||||
|
}
|
||||||
|
|
||||||
|
// **An observation that could not be routed is refused**, naming what it lacks.
|
||||||
|
func TestAnObservationSaysWhatItIs(t *testing.T) {
|
||||||
|
k, _, _, _ := keeper(t)
|
||||||
|
for _, o := range []Observation{
|
||||||
|
{Scope: "elsewhere", ID: "x", Kind: "k", Severity: Warning, Summary: "s", Source: "S1"},
|
||||||
|
{Scope: ScopeMachine, ID: "x", Kind: "k", Severity: "loud", Summary: "s", Source: "S1"},
|
||||||
|
{Scope: ScopeMachine, ID: "x", Kind: "k", Severity: Warning, Source: "S1"},
|
||||||
|
{Scope: ScopeMachine, ID: "x", Kind: "k", Severity: Warning, Summary: "s"},
|
||||||
|
} {
|
||||||
|
if _, err := k.Observe(t.Context(), o); err == nil {
|
||||||
|
t.Errorf("observed %+v", o)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A key holds nothing the bus would refuse or read as a wildcard**, whatever the thing is called.
|
||||||
|
func TestAKeyIsSafeForTheBus(t *testing.T) {
|
||||||
|
if got := Key(ScopeProvider, "keycloak.novox.my app*", "failing"); got != "provider.keycloak.novox.my_app_.failing" {
|
||||||
|
t.Fatalf("key %q", got)
|
||||||
|
}
|
||||||
|
if got := Key(ScopeBus, "EVENTS.>", "consumer-lost"); got != "bus.EVENTS._.consumer-lost" {
|
||||||
|
t.Fatalf("key %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The event's shape is a contract** (to-be 45 §2): the operator-channel's holder is written against
|
||||||
|
// these field names. A rename here is a channel that reads nothing, so they are held still.
|
||||||
|
func TestTheEventShapeIsTheContract(t *testing.T) {
|
||||||
|
k, _, told, _ := keeper(t)
|
||||||
|
if _, err := k.Observe(t.Context(), silent("ace")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
said := settled(t, told, 1)
|
||||||
|
body, err := json.Marshal(said[0])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var shape map[string]any
|
||||||
|
if err := json.Unmarshal(body, &shape); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// The condition at the top level, kebab-case, beside what happened to it.
|
||||||
|
for _, field := range []string{"event", "at", "change", "show", "key", "kind", "subject", "severity",
|
||||||
|
"summary", "evidence", "source", "raised", "last-observed", "observations", "count", "resolver",
|
||||||
|
"silenced", "epoch"} {
|
||||||
|
if _, ok := shape[field]; !ok {
|
||||||
|
t.Errorf("the event carries no %q: %s", field, body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if shape["silenced"] != nil {
|
||||||
|
t.Errorf("an unsilenced condition says silenced %v, not null", shape["silenced"])
|
||||||
|
}
|
||||||
|
subject, _ := shape["subject"].(map[string]any)
|
||||||
|
if subject["scope"] != "machine" || subject["id"] != "ace" || subject["machine"] != "ace" {
|
||||||
|
t.Errorf("subject %v", shape["subject"])
|
||||||
|
}
|
||||||
|
if said[0].Show != "mesh-controller.conditions key=machine.ace.silent" {
|
||||||
|
t.Errorf("show is %q", said[0].Show)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A transition the bus will not take is offered again**, and said lost only after TellFor.
|
||||||
|
func TestATransitionIsOfferedAgainWhileTheBusIsAway(t *testing.T) {
|
||||||
|
store, told, c := NewInMemory(), &Told{Fail: errors.New("no responders")}, newClock()
|
||||||
|
k := NewKeeper(t.Context(), Options{Store: store, History: store, Teller: told, Now: c.now})
|
||||||
|
defer k.Close(context.Background())
|
||||||
|
if _, err := k.Observe(t.Context(), silent("ace")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
time.Sleep(300 * time.Millisecond)
|
||||||
|
told.mu.Lock()
|
||||||
|
told.Fail = nil
|
||||||
|
told.mu.Unlock()
|
||||||
|
said := settled(t, told, 1)
|
||||||
|
if said[0].Key != "machine.ace.silent" || k.Unsaid() != 0 {
|
||||||
|
t.Fatalf("said %+v, unsaid %d", said, k.Unsaid())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -284,92 +284,6 @@ func (i *Inventory) Provide(ctx context.Context, m catalogue.Manifest) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// RetireUnshipped removes every module the control plane recorded as its own and no longer ships.
|
|
||||||
//
|
|
||||||
// **`module forget` refuses a provided module**, rightly: the next start would put it back. So a
|
|
||||||
// module the control plane stops shipping — `networking`, retired by novox/hq ADR 0226 — could be
|
|
||||||
// removed by nothing at all, and would sit in the catalogue for ever, assignable and pointing at a
|
|
||||||
// manifest no release carries. This is the other half of Provide: what this release ships is
|
|
||||||
// recorded, and what it does not is taken away.
|
|
||||||
//
|
|
||||||
// **Never from under a machine.** A retired module still assigned somewhere is kept, and named in
|
|
||||||
// `kept` with the machines it is on, so the caller can say "unassign it, and it goes at the next
|
|
||||||
// start". Taking it while assigned would drop whatever it pulled in — for `networking`, the
|
|
||||||
// private network itself — at the next push, with nobody having asked for that. Its settings,
|
|
||||||
// secrets and ports are kept the same way: a provided module that holds any is kept and named,
|
|
||||||
// because discarding them is a person's decision (novox/hq 04-ISSUES/017).
|
|
||||||
func (i *Inventory) RetireUnshipped(ctx context.Context, shipped []string) (retired []string, kept map[string]string, err error) {
|
|
||||||
keep := map[string]bool{}
|
|
||||||
for _, s := range shipped {
|
|
||||||
keep[s] = true
|
|
||||||
}
|
|
||||||
rows, err := i.store.Pool().Query(ctx,
|
|
||||||
`select name from module where source = 'the control plane' order by name`)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
var gone []string
|
|
||||||
for rows.Next() {
|
|
||||||
var name string
|
|
||||||
if err := rows.Scan(&name); err != nil {
|
|
||||||
rows.Close()
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
if !keep[name] {
|
|
||||||
gone = append(gone, name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
rows.Close()
|
|
||||||
if err := rows.Err(); err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
kept = map[string]string{}
|
|
||||||
for _, name := range gone {
|
|
||||||
on, err := i.assignedOn(ctx, name)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
if len(on) > 0 {
|
|
||||||
kept[name] = "still assigned on " + strings.Join(on, ", ") + "; unassign it and it goes at the next start"
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
held, err := i.HeldFor(ctx, name)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
if held.Any() {
|
|
||||||
kept[name] = "the mesh still holds things for it:\n" + strings.Join(held.Lines(), "\n")
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if err := i.discard(ctx, name); err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
retired = append(retired, name)
|
|
||||||
}
|
|
||||||
return retired, kept, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// assignedOn is the machines a module is assigned to, sorted.
|
|
||||||
func (i *Inventory) assignedOn(ctx context.Context, name string) ([]string, error) {
|
|
||||||
rows, err := i.store.Pool().Query(ctx,
|
|
||||||
`select n.name from assignment a join node n on n.id = a.node where a.module = $1
|
|
||||||
order by n.name`, name)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer rows.Close()
|
|
||||||
var on []string
|
|
||||||
for rows.Next() {
|
|
||||||
var node string
|
|
||||||
if err := rows.Scan(&node); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
on = append(on, node)
|
|
||||||
}
|
|
||||||
return on, rows.Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Provided reports whether a module came with the control plane rather than from a repository.
|
// Provided reports whether a module came with the control plane rather than from a repository.
|
||||||
func (i *Inventory) Provided(ctx context.Context, name string) (bool, error) {
|
func (i *Inventory) Provided(ctx context.Context, name string) (bool, error) {
|
||||||
var source *string
|
var source *string
|
||||||
|
|||||||
@@ -0,0 +1,146 @@
|
|||||||
|
package inventory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The durations the core's bounds are set from (novox/hq to-be 45 Phase 0): see migration 0066.
|
||||||
|
|
||||||
|
// The kinds of duration recorded.
|
||||||
|
const (
|
||||||
|
DurationApply = "apply"
|
||||||
|
DurationHeartbeatGap = "heartbeat-gap"
|
||||||
|
DurationPlanTier = "plan-tier"
|
||||||
|
DurationBuild = "build"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DurationKinds are every kind, in the order `durations` shows them.
|
||||||
|
var DurationKinds = []string{DurationApply, DurationHeartbeatGap, DurationPlanTier, DurationBuild}
|
||||||
|
|
||||||
|
// DurationsKeptFor is how long a duration is kept: long enough to set a bound from, and to correct it
|
||||||
|
// in Phase 1's first live week.
|
||||||
|
const DurationsKeptFor = 30 * 24 * time.Hour
|
||||||
|
|
||||||
|
// Duration is one measurement.
|
||||||
|
type Duration struct {
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Subject string `json:"subject"`
|
||||||
|
Node string `json:"node,omitempty"`
|
||||||
|
Ref string `json:"ref"`
|
||||||
|
Started time.Time `json:"started"`
|
||||||
|
Took time.Duration `json:"took"`
|
||||||
|
Detail string `json:"detail,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordDuration keeps one measurement, once: the same thing measured again is not a second row.
|
||||||
|
func (i *Inventory) RecordDuration(ctx context.Context, d Duration) error {
|
||||||
|
if d.Took < 0 {
|
||||||
|
return nil // a clock that went back measures nothing
|
||||||
|
}
|
||||||
|
_, err := i.store.Pool().Exec(ctx,
|
||||||
|
`insert into duration (kind, subject, node, ref, started, took_ms, detail)
|
||||||
|
values ($1, $2, $3, $4, $5, $6, $7) on conflict do nothing`,
|
||||||
|
d.Kind, d.Subject, d.Node, d.Ref, d.Started, d.Took.Milliseconds(), d.Detail)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordApplyDuration measures a machine's report of the declaration it was last sent: from the send
|
||||||
|
// to the first report of it. A report of anything else, or of a send already measured, measures
|
||||||
|
// nothing — a machine reconciling reports the same declaration every few minutes.
|
||||||
|
func (i *Inventory) RecordApplyDuration(ctx context.Context, node, declared, outcome string) error {
|
||||||
|
if declared == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
_, err := i.store.Pool().Exec(ctx,
|
||||||
|
`insert into duration (kind, subject, node, ref, started, took_ms, detail)
|
||||||
|
select $1, n.name, n.name, n.sent || '@' || to_char(n.sent_at at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US'),
|
||||||
|
n.sent_at, (extract(epoch from (now() - n.sent_at)) * 1000)::bigint, $4
|
||||||
|
from node n
|
||||||
|
where n.name = $2 and n.sent = $3 and n.sent_at is not null
|
||||||
|
on conflict do nothing`,
|
||||||
|
DurationApply, node, declared, outcome)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordHeartbeatGap measures the silence before a machine's word: from the last word heard before
|
||||||
|
// it, which the caller read before recording this one.
|
||||||
|
func (i *Inventory) RecordHeartbeatGap(ctx context.Context, node string, before time.Time) error {
|
||||||
|
if before.IsZero() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
return i.RecordDuration(ctx, Duration{Kind: DurationHeartbeatGap, Subject: node, Node: node,
|
||||||
|
Ref: before.UTC().Format(time.RFC3339Nano), Started: before, Took: now.Sub(before)})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Durations is every measurement of a kind since a moment, oldest first; every kind when kind is empty.
|
||||||
|
func (i *Inventory) Durations(ctx context.Context, kind string, since time.Time) ([]Duration, error) {
|
||||||
|
rows, err := i.store.Pool().Query(ctx,
|
||||||
|
`select kind, subject, node, ref, started, took_ms, detail from duration
|
||||||
|
where ($1 = '' or kind = $1) and recorded >= $2 order by recorded`, kind, since)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var out []Duration
|
||||||
|
for rows.Next() {
|
||||||
|
var d Duration
|
||||||
|
var ms int64
|
||||||
|
if err := rows.Scan(&d.Kind, &d.Subject, &d.Node, &d.Ref, &d.Started, &ms, &d.Detail); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
d.Took = time.Duration(ms) * time.Millisecond
|
||||||
|
out = append(out, d)
|
||||||
|
}
|
||||||
|
return out, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// ForgetOldDurations removes what is older than DurationsKeptFor, and says how many.
|
||||||
|
func (i *Inventory) ForgetOldDurations(ctx context.Context) (int64, error) {
|
||||||
|
tag, err := i.store.Pool().Exec(ctx, `delete from duration where recorded < $1`,
|
||||||
|
time.Now().Add(-DurationsKeptFor))
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return tag.RowsAffected(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// planTierLeft is the measurement a plan's save makes when it leaves a tier: the tier moved on, or
|
||||||
|
// the plan ended. Read in the save's own transaction, so two saves cannot both measure one tier.
|
||||||
|
func planTierLeft(ctx context.Context, tx pgx.Tx, p Plan, now time.Time) (entered time.Time, err error) {
|
||||||
|
var oldTier int
|
||||||
|
var oldState string
|
||||||
|
var since time.Time
|
||||||
|
err = tx.QueryRow(ctx,
|
||||||
|
`select tier, state, coalesce(tier_entered, created) from release_plan where id = $1 for update`,
|
||||||
|
p.ID).Scan(&oldTier, &oldState, &since)
|
||||||
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
return now, nil // a new plan enters its first tier now
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return time.Time{}, err
|
||||||
|
}
|
||||||
|
wasOpen := oldState == PlanBuilding || oldState == PlanRolling
|
||||||
|
if !wasOpen || (oldTier == p.Tier && p.Open()) {
|
||||||
|
return since, nil // still in the tier, or already ended
|
||||||
|
}
|
||||||
|
var modules []string
|
||||||
|
if oldTier >= 0 && oldTier < len(p.Tiers) {
|
||||||
|
modules = p.Tiers[oldTier]
|
||||||
|
}
|
||||||
|
detail := fmt.Sprintf("plan %s, tier %d of %d (%v), left %s", p.ID, oldTier, len(p.Tiers), modules, p.State)
|
||||||
|
if p.Open() {
|
||||||
|
detail = fmt.Sprintf("plan %s, tier %d of %d (%v), moved on to tier %d", p.ID, oldTier, len(p.Tiers), modules, p.Tier)
|
||||||
|
}
|
||||||
|
_, err = tx.Exec(ctx,
|
||||||
|
`insert into duration (kind, subject, node, ref, started, took_ms, detail)
|
||||||
|
values ($1, $2, '', $3, $4, $5, $6) on conflict do nothing`,
|
||||||
|
DurationPlanTier, p.Repository, fmt.Sprintf("%s/tier-%d", p.ID, oldTier), since,
|
||||||
|
now.Sub(since).Milliseconds(), detail)
|
||||||
|
return now, err
|
||||||
|
}
|
||||||
@@ -0,0 +1,97 @@
|
|||||||
|
package inventory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **The durations the bounds are set from are recorded once each** (novox/hq to-be 45 Phase 0): a
|
||||||
|
// send measured at its first report and not again at every reconcile that repeats it; a send of
|
||||||
|
// something else measures nothing; a new send is a new measurement.
|
||||||
|
func TestAnApplyIsMeasuredOncePerSend(t *testing.T) {
|
||||||
|
inv := ForTest(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
node, err := inv.AddNode(ctx, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.RecordSent(ctx, node.ID, "d1", nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for range 3 { // the report, then two reconciles saying the same
|
||||||
|
if err := inv.RecordApplyDuration(ctx, "anchor", "d1", OutcomeApplied); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := inv.RecordApplyDuration(ctx, "anchor", "d0", OutcomeApplied); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ds, err := inv.Durations(ctx, DurationApply, time.Now().Add(-time.Hour))
|
||||||
|
if err != nil || len(ds) != 1 || ds[0].Subject != "anchor" || ds[0].Detail != OutcomeApplied || ds[0].Took < 0 {
|
||||||
|
t.Fatalf("%v %+v", err, ds)
|
||||||
|
}
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
if err := inv.RecordSent(ctx, node.ID, "d2", nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.RecordApplyDuration(ctx, "anchor", "d2", OutcomeFailed); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if ds, _ = inv.Durations(ctx, "", time.Now().Add(-time.Hour)); len(ds) != 2 {
|
||||||
|
t.Fatalf("a second send was not measured: %+v", ds)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A machine's silence is the time since its last word, once per word.
|
||||||
|
func TestASilenceIsMeasuredFromTheLastWord(t *testing.T) {
|
||||||
|
inv := ForTest(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
before := time.Now().Add(-90 * time.Second)
|
||||||
|
for range 2 {
|
||||||
|
if err := inv.RecordHeartbeatGap(ctx, "anchor", before); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := inv.RecordHeartbeatGap(ctx, "anchor", time.Time{}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ds, err := inv.Durations(ctx, DurationHeartbeatGap, time.Now().Add(-time.Hour))
|
||||||
|
if err != nil || len(ds) != 1 || ds[0].Took < 90*time.Second || ds[0].Took > 2*time.Minute {
|
||||||
|
t.Fatalf("%v %+v", err, ds)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A plan's tier is measured when the plan leaves it — moving on, or ending — and only then.
|
||||||
|
func TestAPlansTierIsMeasuredWhenItIsLeft(t *testing.T) {
|
||||||
|
inv := ForTest(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
p := Plan{ID: "plan-1", Repository: "novox/mesh-tools", Commit: "abc", Created: time.Now().UTC(),
|
||||||
|
State: PlanBuilding, Tiers: [][]string{{"mesh-tools"}, {"builder"}},
|
||||||
|
Modules: map[string]*PlanModule{"mesh-tools": {}, "builder": {}}}
|
||||||
|
if err := inv.SavePlan(ctx, p); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
p.State = PlanRolling // the same tier, saved again
|
||||||
|
if err := inv.SavePlan(ctx, p); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if ds, _ := inv.Durations(ctx, DurationPlanTier, time.Now().Add(-time.Hour)); len(ds) != 0 {
|
||||||
|
t.Fatalf("a tier not left was measured: %+v", ds)
|
||||||
|
}
|
||||||
|
p.Tier = 1
|
||||||
|
if err := inv.SavePlan(ctx, p); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
p.State = PlanDone
|
||||||
|
if err := inv.SavePlan(ctx, p); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.SavePlan(ctx, p); err != nil { // saved again once ended: nothing more to measure
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ds, err := inv.Durations(ctx, DurationPlanTier, time.Now().Add(-time.Hour))
|
||||||
|
if err != nil || len(ds) != 2 || ds[0].Subject != "novox/mesh-tools" || ds[0].Ref != "plan-1/tier-0" ||
|
||||||
|
ds[1].Ref != "plan-1/tier-1" {
|
||||||
|
t.Fatalf("%v %+v", err, ds)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
-- The durations the core's bounds are set from (novox/hq to-be 45 Phase 0).
|
||||||
|
--
|
||||||
|
-- Every watchdog of the core's signals table has a bound — how long a machine may take to report
|
||||||
|
-- after a send, how long it may be silent, how long a plan's tier or a build may take — and a bound
|
||||||
|
-- guessed is a condition that cries wolf or one that never fires. So the controller records each
|
||||||
|
-- duration as it is observed, and Phase 1 sets the bounds from what was recorded:
|
||||||
|
--
|
||||||
|
-- apply a declaration sent → the machine's first report of that declaration, per machine
|
||||||
|
-- heartbeat-gap one word from a machine → the next, per machine
|
||||||
|
-- plan-tier a plan entering a tier → leaving it, per repository
|
||||||
|
-- build a build asked → its outcome heard, per module
|
||||||
|
--
|
||||||
|
-- One row per thing measured: `ref` names it (the send, the earlier word, the plan's tier, the
|
||||||
|
-- build), so a report repeated by a reconcile is not a second measurement. Kept a month; read
|
||||||
|
-- through `durations`.
|
||||||
|
create table duration (
|
||||||
|
kind text not null,
|
||||||
|
subject text not null,
|
||||||
|
node text not null default '',
|
||||||
|
ref text not null,
|
||||||
|
started timestamptz not null,
|
||||||
|
took_ms bigint not null,
|
||||||
|
detail text not null default '',
|
||||||
|
recorded timestamptz not null default now(),
|
||||||
|
primary key (kind, subject, ref)
|
||||||
|
);
|
||||||
|
|
||||||
|
create index duration_by_kind on duration (kind, recorded);
|
||||||
|
|
||||||
|
-- When a plan entered the tier it is at, so leaving it measures the tier.
|
||||||
|
alter table release_plan add column tier_entered timestamptz;
|
||||||
@@ -28,6 +28,10 @@ type Plan struct {
|
|||||||
Tiers [][]string `json:"tiers"`
|
Tiers [][]string `json:"tiers"`
|
||||||
Modules map[string]*PlanModule `json:"modules"`
|
Modules map[string]*PlanModule `json:"modules"`
|
||||||
Note string `json:"note,omitempty"`
|
Note string `json:"note,omitempty"`
|
||||||
|
// TierEntered is when the plan entered the tier it is at (novox/hq to-be 45 Phase 0), read and
|
||||||
|
// never written from here: a save measures the tier it leaves and stamps the next. What the
|
||||||
|
// watchdog of a plan's progress (S3) reads.
|
||||||
|
TierEntered time.Time `json:"tier_entered,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// PlanModule is one module's state within a plan.
|
// PlanModule is one module's state within a plan.
|
||||||
@@ -80,13 +84,29 @@ func (i *Inventory) SavePlan(ctx context.Context, p Plan) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
_, err = i.store.Pool().Exec(ctx,
|
// **And how long the tier it left took** (novox/hq to-be 45 Phase 0): measured here, where the
|
||||||
`insert into release_plan (id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch)
|
// plan moves, in the same transaction as the move, so no save can move a tier unmeasured or
|
||||||
values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9, $10)
|
// measure one twice.
|
||||||
|
tx, err := i.store.Pool().Begin(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer func() { _ = tx.Rollback(ctx) }()
|
||||||
|
entered, err := planTierLeft(ctx, tx, p, time.Now())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = tx.Exec(ctx,
|
||||||
|
`insert into release_plan (id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch, tier_entered)
|
||||||
|
values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9, $10, $11)
|
||||||
on conflict (id) do update set updated = now(), state = excluded.state, tier = excluded.tier,
|
on conflict (id) do update set updated = now(), state = excluded.state, tier = excluded.tier,
|
||||||
tiers = excluded.tiers, modules = excluded.modules, note = excluded.note, branch = excluded.branch`,
|
tiers = excluded.tiers, modules = excluded.modules, note = excluded.note, branch = excluded.branch,
|
||||||
p.ID, p.Repository, p.Commit, p.Created, p.State, p.Tier, tiers, modules, p.Note, p.Branch)
|
tier_entered = excluded.tier_entered`,
|
||||||
return err
|
p.ID, p.Repository, p.Commit, p.Created, p.State, p.Tier, tiers, modules, p.Note, p.Branch, entered)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return tx.Commit(ctx)
|
||||||
}
|
}
|
||||||
|
|
||||||
// OpenPlans is every plan still being worked, oldest first.
|
// OpenPlans is every plan still being worked, oldest first.
|
||||||
@@ -113,7 +133,8 @@ func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) {
|
|||||||
|
|
||||||
func (i *Inventory) plans(ctx context.Context, tail string) ([]Plan, error) {
|
func (i *Inventory) plans(ctx context.Context, tail string) ([]Plan, error) {
|
||||||
rows, err := i.store.Pool().Query(ctx,
|
rows, err := i.store.Pool().Query(ctx,
|
||||||
`select id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch
|
`select id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch,
|
||||||
|
coalesce(tier_entered, created)
|
||||||
from release_plan `+tail)
|
from release_plan `+tail)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -124,7 +145,7 @@ func (i *Inventory) plans(ctx context.Context, tail string) ([]Plan, error) {
|
|||||||
var p Plan
|
var p Plan
|
||||||
var tiers, modules []byte
|
var tiers, modules []byte
|
||||||
if err := rows.Scan(&p.ID, &p.Repository, &p.Commit, &p.Created, &p.Updated, &p.State,
|
if err := rows.Scan(&p.ID, &p.Repository, &p.Commit, &p.Created, &p.Updated, &p.State,
|
||||||
&p.Tier, &tiers, &modules, &p.Note, &p.Branch); err != nil {
|
&p.Tier, &tiers, &modules, &p.Note, &p.Branch, &p.TierEntered); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := json.Unmarshal(tiers, &p.Tiers); err != nil {
|
if err := json.Unmarshal(tiers, &p.Tiers); err != nil {
|
||||||
|
|||||||
@@ -1,91 +0,0 @@
|
|||||||
package inventory
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/jackc/pgx/v5"
|
|
||||||
)
|
|
||||||
|
|
||||||
// What a release stops shipping is retired at the next start, and never from under a machine
|
|
||||||
// (novox/hq ADR 0226). `module forget` refuses a provided module, so without this a module the
|
|
||||||
// control plane no longer carries could be removed by nothing.
|
|
||||||
|
|
||||||
func TestAModuleTheControlPlaneNoLongerShipsIsRetired(t *testing.T) {
|
|
||||||
inv := fresh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
for _, m := range []string{"mesh-wireguard", "networking"} {
|
|
||||||
if err := inv.Provide(ctx, manifest(m, nil, nil)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
retired, kept, err := inv.RetireUnshipped(ctx, []string{"mesh-wireguard"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if strings.Join(retired, ",") != "networking" || len(kept) != 0 {
|
|
||||||
t.Fatalf("retired %v, kept %v", retired, kept)
|
|
||||||
}
|
|
||||||
if _, err := inv.Provided(ctx, "networking"); !errors.Is(err, pgx.ErrNoRows) {
|
|
||||||
t.Fatalf("networking is still in the catalogue: %v", err)
|
|
||||||
}
|
|
||||||
if provided, err := inv.Provided(ctx, "mesh-wireguard"); err != nil || !provided {
|
|
||||||
t.Fatalf("what this release ships went too: %v %v", provided, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestARetiredModuleStillAssignedIsKeptAndSaidSo(t *testing.T) {
|
|
||||||
inv := fresh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := inv.Provide(ctx, manifest("networking", nil, nil)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := inv.Assign(ctx, "anchor", "networking"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
retired, kept, err := inv.RetireUnshipped(ctx, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(retired) != 0 {
|
|
||||||
// Taking it now would drop whatever it pulled in at the next push — for the bundle, the
|
|
||||||
// private network.
|
|
||||||
t.Fatalf("a module assigned on a machine was retired: %v", retired)
|
|
||||||
}
|
|
||||||
if !strings.Contains(kept["networking"], "anchor") {
|
|
||||||
t.Fatalf("keeping it does not say where it is still assigned: %v", kept)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Unassigned, the next start takes it.
|
|
||||||
if err := inv.Unassign(ctx, "anchor", "networking"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
retired, _, err = inv.RetireUnshipped(ctx, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if strings.Join(retired, ",") != "networking" {
|
|
||||||
t.Fatalf("unassigned, it was still not retired: %v", retired)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAModuleFromARepositoryIsNeverRetiredByThis(t *testing.T) {
|
|
||||||
// Only what the control plane recorded as its own. A module somebody registered is theirs to
|
|
||||||
// forget.
|
|
||||||
inv := fresh(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
if err := inv.RegisterModule(ctx, manifest("public-acme", nil, nil), Source{}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
retired, kept, err := inv.RetireUnshipped(ctx, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(retired) != 0 || len(kept) != 0 {
|
|
||||||
t.Fatalf("a registered module was considered: retired %v, kept %v", retired, kept)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -105,14 +105,27 @@ func (i *Inventory) widenProtocol(ctx context.Context, s catalogue.Seat) error {
|
|||||||
changed := false
|
changed := false
|
||||||
row.Accepts, changed = union(row.Accepts, s.Accepts, changed)
|
row.Accepts, changed = union(row.Accepts, s.Accepts, changed)
|
||||||
row.Emits, changed = union(row.Emits, s.Emits, changed)
|
row.Emits, changed = union(row.Emits, s.Emits, changed)
|
||||||
have := map[string]bool{}
|
have := map[string]int{}
|
||||||
for _, v := range row.Serves {
|
for n, v := range row.Serves {
|
||||||
have[v.Name] = true
|
have[v.Name] = n
|
||||||
}
|
}
|
||||||
for _, v := range s.Serves {
|
for _, v := range s.Serves {
|
||||||
if !have[v.Name] {
|
at, kept := have[v.Name]
|
||||||
|
if !kept {
|
||||||
row.Serves = append(row.Serves, v)
|
row.Serves = append(row.Serves, v)
|
||||||
changed = true
|
changed = true
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// **The controller's own verbs are described by the binary that runs them** (novox/hq
|
||||||
|
// issue 244, to-be 45 §7). Its seat's verbs are not an operator's to reshape: each is a
|
||||||
|
// command line this binary composes from the arguments its own table declares, and the
|
||||||
|
// console judges a call against the row. A row kept from an older build described `push`
|
||||||
|
// without the `behind` and `why` the binary takes, so the console refused an argument the verb
|
||||||
|
// needs. So a verb this binary defines takes this binary's definition; a verb only the row has
|
||||||
|
// — a newer build's, during a roll-out (ADR 0185) — is left as it is.
|
||||||
|
if s.Name == catalogue.ControllerSeatName && !sameVerb(row.Serves[at], v) {
|
||||||
|
row.Serves[at] = v
|
||||||
|
changed = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !changed {
|
if !changed {
|
||||||
@@ -282,3 +295,18 @@ func (i *Inventory) Holdings(ctx context.Context) ([]catalogue.Held, error) {
|
|||||||
}
|
}
|
||||||
return out, rows.Err()
|
return out, rows.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// sameVerb is whether two definitions of a verb say the same, read as the row stores them.
|
||||||
|
func sameVerb(a, b catalogue.Verb) bool {
|
||||||
|
ja, errA := json.Marshal(a)
|
||||||
|
jb, errB := json.Marshal(b)
|
||||||
|
if errA != nil || errB != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
var ra, rb any
|
||||||
|
_ = json.Unmarshal(ja, &ra)
|
||||||
|
_ = json.Unmarshal(jb, &rb)
|
||||||
|
ca, _ := json.Marshal(ra)
|
||||||
|
cb, _ := json.Marshal(rb)
|
||||||
|
return string(ca) == string(cb)
|
||||||
|
}
|
||||||
|
|||||||
@@ -113,3 +113,46 @@ func TestRenameSeatKeepsTheFormerNameAsAnAlias(t *testing.T) {
|
|||||||
t.Fatal("renaming a seat to its own name was accepted")
|
t.Fatal("renaming a seat to its own name was accepted")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **The controller's verbs in the row are the binary's** (novox/hq issue 244, to-be 45 §7): a row
|
||||||
|
// seeded by an older build describes `push` without the arguments this one takes, and the console
|
||||||
|
// judges a call against the row — so re-seeding brings the controller's verbs to this binary's
|
||||||
|
// definition, and keeps a verb only the row has, which a newer build added (ADR 0185).
|
||||||
|
func TestTheControllersVerbsInTheRowAreTheBinarys(t *testing.T) {
|
||||||
|
inv := ForTest(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
old := `[{"name":"push","description":"an older push","input":{"type":"object","properties":{"node":{"type":"string"}}}},
|
||||||
|
{"name":"newer","description":"a verb of a newer build"}]`
|
||||||
|
if _, err := inv.store.Pool().Exec(ctx, `update seat set serves = $1 where name = $2`,
|
||||||
|
[]byte(old), catalogue.ControllerSeatName); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
seats, err := inv.Seats(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
verbs := map[string]catalogue.Verb{}
|
||||||
|
for _, s := range seats {
|
||||||
|
if s.Name == catalogue.ControllerSeatName {
|
||||||
|
for _, v := range s.Serves {
|
||||||
|
verbs[v.Name] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
props, _ := verbs["push"].Input["properties"].(map[string]any)
|
||||||
|
if _, takesWhy := props["why"]; !takesWhy || verbs["push"].Description == "an older push" {
|
||||||
|
t.Fatalf("push in the row is still the older build's: %+v", verbs["push"])
|
||||||
|
}
|
||||||
|
if _, kept := verbs["newer"]; !kept {
|
||||||
|
t.Fatal("a verb only the row has was dropped")
|
||||||
|
}
|
||||||
|
if _, added := verbs["durations"]; !added {
|
||||||
|
t.Fatal("a verb this binary adds was not added")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,76 +0,0 @@
|
|||||||
package inventory
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ProviderStanding is a consumer a provider says it keeps failing (novox/hq ADR 0224).
|
|
||||||
type ProviderStanding struct {
|
|
||||||
// Module is the provider's module, and ProviderNode the machine it runs on.
|
|
||||||
Module string `json:"module"`
|
|
||||||
ProviderNode string `json:"provider-node"`
|
|
||||||
// Provision is the interface it provides, e.g. `oidc-client`.
|
|
||||||
Provision string `json:"provision"`
|
|
||||||
// Consumer is the identity the mesh derived for the consumer, ConsumerNode its machine.
|
|
||||||
Consumer string `json:"consumer"`
|
|
||||||
ConsumerNode string `json:"consumer-node"`
|
|
||||||
// Class is what kind of failure: credentials-rejected, unreachable, secret-unreadable, refused.
|
|
||||||
Class string `json:"class"`
|
|
||||||
Error string `json:"error"`
|
|
||||||
// Since is when the unbroken run of failures began; Attempts how many it has been.
|
|
||||||
Since time.Time `json:"since"`
|
|
||||||
Attempts int `json:"attempts"`
|
|
||||||
// SaidAt is when the controller last heard it. A provider says it again every quarter of an hour
|
|
||||||
// while it lasts, so an old one is a provider that stopped saying anything.
|
|
||||||
SaidAt time.Time `json:"said-at"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// SayAgainWithin is how long a failing standing stays current without being said again: twice the
|
|
||||||
// quarter of an hour a provider repeats it at. Older, and status says the provider has gone quiet.
|
|
||||||
const SayAgainWithin = 30 * time.Minute
|
|
||||||
|
|
||||||
// Quiet says the provider has not repeated this standing for longer than it would while it lasts.
|
|
||||||
func (s ProviderStanding) Quiet(now time.Time) bool { return now.Sub(s.SaidAt) > SayAgainWithin }
|
|
||||||
|
|
||||||
// KeepStanding records a provider's newest word: failing keeps it, recovered removes it, and says
|
|
||||||
// whether a recovery removed anything.
|
|
||||||
func (i *Inventory) KeepStanding(ctx context.Context, failing bool, s ProviderStanding) (bool, error) {
|
|
||||||
if !failing {
|
|
||||||
tag, err := i.store.Pool().Exec(ctx,
|
|
||||||
`delete from provider_standing where module = $1 and provider_node = $2 and consumer = $3`,
|
|
||||||
s.Module, s.ProviderNode, s.Consumer)
|
|
||||||
return err == nil && tag.RowsAffected() > 0, err
|
|
||||||
}
|
|
||||||
_, err := i.store.Pool().Exec(ctx, `
|
|
||||||
insert into provider_standing
|
|
||||||
(module, provider_node, consumer, consumer_node, provision, class, error, since, attempts, said_at)
|
|
||||||
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, now())
|
|
||||||
on conflict (module, provider_node, consumer) do update set
|
|
||||||
consumer_node = excluded.consumer_node, provision = excluded.provision,
|
|
||||||
class = excluded.class, error = excluded.error, since = excluded.since,
|
|
||||||
attempts = excluded.attempts, said_at = excluded.said_at`,
|
|
||||||
s.Module, s.ProviderNode, s.Consumer, s.ConsumerNode, s.Provision, s.Class, s.Error, s.Since, s.Attempts)
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// FailingProviders is every consumer a provider last said it keeps failing, oldest run first.
|
|
||||||
func (i *Inventory) FailingProviders(ctx context.Context) ([]ProviderStanding, error) {
|
|
||||||
rows, err := i.store.Pool().Query(ctx, `
|
|
||||||
select module, provider_node, provision, consumer, consumer_node, class, error, since, attempts, said_at
|
|
||||||
from provider_standing order by since, module, consumer`)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer rows.Close()
|
|
||||||
var out []ProviderStanding
|
|
||||||
for rows.Next() {
|
|
||||||
var s ProviderStanding
|
|
||||||
if err := rows.Scan(&s.Module, &s.ProviderNode, &s.Provision, &s.Consumer, &s.ConsumerNode,
|
|
||||||
&s.Class, &s.Error, &s.Since, &s.Attempts, &s.SaidAt); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
out = append(out, s)
|
|
||||||
}
|
|
||||||
return out, rows.Err()
|
|
||||||
}
|
|
||||||
@@ -1,130 +0,0 @@
|
|||||||
package inventory
|
|
||||||
|
|
||||||
import (
|
|
||||||
"slices"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A provider's standing (novox/hq ADR 0224), from the grant that lets it say so to the row status
|
|
||||||
// reads.
|
|
||||||
|
|
||||||
// The broker spells the events itself because it cannot import the catalogue; the two agree.
|
|
||||||
func TestTheBrokerAndTheCatalogueNameTheSameStandingEvents(t *testing.T) {
|
|
||||||
if broker.ProvisionerFailing != catalogue.ProvisionerFailing ||
|
|
||||||
broker.ProvisionerRecovered != catalogue.ProvisionerRecovered {
|
|
||||||
t.Fatal("the broker and the catalogue disagree about what a provider's standing is called")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **Every provider may say it, whatever its manifest lists**: a provider whose manifest forgot the
|
|
||||||
// events would have its announcement refused by the bus, and fail its consumers as silently as on
|
|
||||||
// 2026-10-05 (issue 179). A module that receives no contributions provides nothing and is given
|
|
||||||
// nothing.
|
|
||||||
func TestEveryProviderIsGrantedItsStandingAndNothingElseIs(t *testing.T) {
|
|
||||||
provider := catalogue.Manifest{Module: "keycloak", Version: "1",
|
|
||||||
Emits: []string{"client.created"}, Receives: map[string]string{"oidc-client": "/x/mesh.json"}}
|
|
||||||
consumer := catalogue.Manifest{Module: "grafana", Version: "1", Emits: []string{"dashboard.saved"}}
|
|
||||||
|
|
||||||
d := declaredFor(provider, nil)
|
|
||||||
for _, e := range []string{"client.created", catalogue.ProvisionerFailing, catalogue.ProvisionerRecovered} {
|
|
||||||
if !slices.Contains(d.Emits, e) {
|
|
||||||
t.Fatalf("a provider is not granted %s: %v", e, d.Emits)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindModule, Node: "anchor",
|
|
||||||
Module: "keycloak", Emits: d.Emits})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !slices.Contains(perms.Publish, "mesh.mod.keycloak.event.provisioner.failing") {
|
|
||||||
t.Fatalf("the bus would refuse a provider's standing: %v", perms.Publish)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got := declaredFor(consumer, nil).Emits; slices.Contains(got, catalogue.ProvisionerFailing) {
|
|
||||||
t.Fatalf("a module that provides nothing was granted a provider's standing: %v", got)
|
|
||||||
}
|
|
||||||
// Declared by hand as well: said once.
|
|
||||||
provider.Emits = append(provider.Emits, catalogue.ProvisionerFailing)
|
|
||||||
n := 0
|
|
||||||
for _, e := range provider.EmitsAll() {
|
|
||||||
if e == catalogue.ProvisionerFailing {
|
|
||||||
n++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if n != 1 {
|
|
||||||
t.Fatalf("%v", provider.EmitsAll())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// And the controller may hear it from every provider, and only those two events.
|
|
||||||
func TestTheControllerHearsEveryProvidersStanding(t *testing.T) {
|
|
||||||
perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, want := range []string{"mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered"} {
|
|
||||||
if !slices.Contains(perms.Subscribe, want) {
|
|
||||||
t.Fatalf("the controller may not hear %s: %v", want, perms.Subscribe)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if slices.Contains(perms.Subscribe, "mesh.mod.*.event.>") {
|
|
||||||
t.Fatal("the controller hears every event in the mesh")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAFailingStandingIsKeptUntilItRecovers(t *testing.T) {
|
|
||||||
inv := ForTest(t)
|
|
||||||
ctx := t.Context()
|
|
||||||
since := time.Date(2026, 10, 5, 0, 49, 0, 0, time.UTC)
|
|
||||||
s := ProviderStanding{Module: "keycloak", ProviderNode: "anchor", Provision: "oidc-client",
|
|
||||||
Consumer: "mesh_home_grafana", ConsumerNode: "home-server", Class: "credentials-rejected",
|
|
||||||
Error: "401 invalid_grant", Since: since, Attempts: 60}
|
|
||||||
if _, err := inv.KeepStanding(ctx, true, s); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
// Said again: one row, the newest word.
|
|
||||||
s.Attempts = 31000
|
|
||||||
if _, err := inv.KeepStanding(ctx, true, s); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
got, err := inv.FailingProviders(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(got) != 1 || got[0].Attempts != 31000 || !got[0].Since.Equal(since) || got[0].ConsumerNode != "home-server" ||
|
|
||||||
got[0].Class != "credentials-rejected" || got[0].SaidAt.IsZero() {
|
|
||||||
t.Fatalf("%+v", got)
|
|
||||||
}
|
|
||||||
if got[0].Quiet(time.Now()) {
|
|
||||||
t.Fatal("a standing just said reads as quiet")
|
|
||||||
}
|
|
||||||
if !got[0].Quiet(time.Now().Add(SayAgainWithin + time.Minute)) {
|
|
||||||
t.Fatal("a standing not said again for longer than a provider repeats it does not read as quiet")
|
|
||||||
}
|
|
||||||
|
|
||||||
// The same consumer from another machine's provider is its own row.
|
|
||||||
other := s
|
|
||||||
other.ProviderNode = "laptop"
|
|
||||||
if _, err := inv.KeepStanding(ctx, true, other); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
cleared, err := inv.KeepStanding(ctx, false, s)
|
|
||||||
if err != nil || !cleared {
|
|
||||||
t.Fatalf("recovered cleared nothing: %v %v", cleared, err)
|
|
||||||
}
|
|
||||||
cleared, err = inv.KeepStanding(ctx, false, s)
|
|
||||||
if err != nil || cleared {
|
|
||||||
t.Fatalf("a recovery for nothing kept said it cleared something: %v %v", cleared, err)
|
|
||||||
}
|
|
||||||
got, err = inv.FailingProviders(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(got) != 1 || got[0].ProviderNode != "laptop" {
|
|
||||||
t.Fatalf("%+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,283 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What the bus says about itself, in the mesh's words (novox/hq to-be 45 §3, S9).
|
||||||
|
//
|
||||||
|
// **The server already says it; nothing listened.** A durable consumer that hands a message over as
|
||||||
|
// often as it may gives up on it and says so on `$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES`; one
|
||||||
|
// deleted says so on `…CONSUMER.DELETED`. The controller's own connection is told when it falls behind
|
||||||
|
// (a slow consumer: the client library dropped messages — issue 184, the controller deaf for 24
|
||||||
|
// minutes) and when the bus refuses it a subject (a permissions violation — issues 183, 217, 265,
|
||||||
|
// days each as a line in a client library's output). Each is recorded here, named in the mesh's words —
|
||||||
|
// which consumer of whose, which subject — for the watchdog to say as a condition, as the refused
|
||||||
|
// reply of issue 265 is said today.
|
||||||
|
//
|
||||||
|
// What the bus says about **other** principals' connections — a module's slow consumer, a module
|
||||||
|
// refused a subject — the server publishes only to a system account, which the mesh's bus does not
|
||||||
|
// have; that half is not heard yet (to-be 45 S9, recorded as deferred).
|
||||||
|
|
||||||
|
// The advisory kinds, as conditions name them.
|
||||||
|
const (
|
||||||
|
AdvisorySlowConsumer = "slow-consumer"
|
||||||
|
AdvisoryMaxDeliveries = "max-deliveries"
|
||||||
|
AdvisoryRefused = "refused"
|
||||||
|
AdvisoryConsumerLost = "consumer-lost"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Advisory is one thing the bus said, kept as its newest word and how often it was said.
|
||||||
|
type Advisory struct {
|
||||||
|
Kind string
|
||||||
|
// ID names what it is about, for the condition's key: `<stream>.<consumer>`, or `controller`.
|
||||||
|
ID string
|
||||||
|
// Stream and Consumer are the consumer it is about, when it is about one.
|
||||||
|
Stream, Consumer string
|
||||||
|
// Said is the newest saying, in the mesh's words.
|
||||||
|
Said string
|
||||||
|
First, Last time.Time
|
||||||
|
Count int
|
||||||
|
}
|
||||||
|
|
||||||
|
// AdvisoryLog keeps what the bus said lately.
|
||||||
|
type AdvisoryLog struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
seen map[string]*Advisory
|
||||||
|
}
|
||||||
|
|
||||||
|
// Advisories is this process's log.
|
||||||
|
var Advisories = &AdvisoryLog{seen: map[string]*Advisory{}}
|
||||||
|
|
||||||
|
// Heard records one advisory.
|
||||||
|
func (l *AdvisoryLog) Heard(a Advisory, at time.Time) {
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
key := a.Kind + "/" + a.ID
|
||||||
|
if had, ok := l.seen[key]; ok {
|
||||||
|
had.Last, had.Said, had.Count = at, a.Said, had.Count+1
|
||||||
|
return
|
||||||
|
}
|
||||||
|
a.First, a.Last, a.Count = at, at, 1
|
||||||
|
l.seen[key] = &a
|
||||||
|
}
|
||||||
|
|
||||||
|
// Since is every advisory said at or after a moment, and forgets the older ones.
|
||||||
|
func (l *AdvisoryLog) Since(since time.Time) []Advisory {
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
var out []Advisory
|
||||||
|
for key, a := range l.seen {
|
||||||
|
if a.Last.Before(since) {
|
||||||
|
delete(l.seen, key)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, *a)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// jsAdvisory is the part of a JetStream advisory the mesh reads.
|
||||||
|
type jsAdvisory struct {
|
||||||
|
Type string `json:"type"`
|
||||||
|
Stream string `json:"stream"`
|
||||||
|
Consumer string `json:"consumer"`
|
||||||
|
StreamSeq uint64 `json:"stream_seq"`
|
||||||
|
Deliveries uint64 `json:"deliveries"`
|
||||||
|
Action string `json:"action"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadAdvisory is one JetStream advisory as the mesh says it; false for one it does not watch.
|
||||||
|
func ReadAdvisory(subject string, body []byte) (Advisory, bool) {
|
||||||
|
var a jsAdvisory
|
||||||
|
if err := json.Unmarshal(body, &a); err != nil || a.Stream == "" || a.Consumer == "" {
|
||||||
|
return Advisory{}, false
|
||||||
|
}
|
||||||
|
who := ConsumerInWords(a.Stream, a.Consumer)
|
||||||
|
id := a.Stream + "." + a.Consumer
|
||||||
|
switch {
|
||||||
|
case strings.HasPrefix(subject, "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES."):
|
||||||
|
return Advisory{Kind: AdvisoryMaxDeliveries, ID: id, Stream: a.Stream, Consumer: a.Consumer,
|
||||||
|
Said: fmt.Sprintf("%s handed message %d over %d times and gave up on it: it will not be delivered "+
|
||||||
|
"again, and what it asked for was not done", who, a.StreamSeq, a.Deliveries)}, true
|
||||||
|
case strings.HasPrefix(subject, "$JS.EVENT.ADVISORY.CONSUMER.DELETED."):
|
||||||
|
if !MeshNamed(a.Stream, a.Consumer) {
|
||||||
|
// A reader's own consumer, gone when it finished — every watch of a bucket and every
|
||||||
|
// read-back of a stream makes one, many a minute: not something the mesh defines.
|
||||||
|
return Advisory{}, false
|
||||||
|
}
|
||||||
|
return Advisory{Kind: AdvisoryConsumerLost, ID: id, Stream: a.Stream, Consumer: a.Consumer,
|
||||||
|
Said: fmt.Sprintf("%s was deleted from the bus", who)}, true
|
||||||
|
}
|
||||||
|
return Advisory{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// MeshNamed says a consumer is one of the durable consumers the mesh defines, by its name's shape:
|
||||||
|
// the controller's own, a machine's declaration consumer, a module's (`<node>_<module>`), a seat's
|
||||||
|
// worker. Every other consumer is a reader's own — an ordered consumer, a bucket's watcher — named at
|
||||||
|
// random by the client library, deleted when the read is done, and not the mesh's to say anything of.
|
||||||
|
func MeshNamed(stream, name string) bool {
|
||||||
|
switch {
|
||||||
|
case strings.HasPrefix(stream, "KV_") || stream == broker.AssignmentsStream:
|
||||||
|
return false // the mesh defines no durable consumer on a bucket's stream, or the memberships'
|
||||||
|
case stream == "NODES":
|
||||||
|
return true
|
||||||
|
case strings.HasPrefix(stream, "SEAT_"):
|
||||||
|
return strings.HasSuffix(name, "_worker")
|
||||||
|
case name == broker.ControllerName:
|
||||||
|
return true
|
||||||
|
case stream == broker.EventsStream:
|
||||||
|
return strings.Contains(name, "_")
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// ConsumerInWords is a durable consumer as the mesh says it: whose, and for what.
|
||||||
|
func ConsumerInWords(stream, name string) string {
|
||||||
|
switch {
|
||||||
|
case name == broker.ControllerName:
|
||||||
|
return "the controller's consumer on " + stream
|
||||||
|
case stream == "NODES":
|
||||||
|
return "how " + name + " hears what it should be (its declaration consumer)"
|
||||||
|
case strings.HasPrefix(stream, "SEAT_") && strings.HasSuffix(name, "_worker"):
|
||||||
|
seat := strings.ToLower(strings.ReplaceAll(strings.TrimPrefix(stream, "SEAT_"), "_", "-"))
|
||||||
|
return "the worker every holder of " + seat + " takes its asks from"
|
||||||
|
case stream == broker.EventsStream:
|
||||||
|
if node, module, ok := strings.Cut(name, "_"); ok {
|
||||||
|
return "how " + module + " on " + node + " hears what it consumes"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "the consumer " + name + " on " + stream
|
||||||
|
}
|
||||||
|
|
||||||
|
// HearAdvisories subscribes what the bus says about the mesh's account, and listens for what it
|
||||||
|
// tells this connection, until the returned function is called. Read-only: nothing is published.
|
||||||
|
func (s *Server) HearAdvisories(logf func(string, ...any)) (func(), error) {
|
||||||
|
if s.js == nil {
|
||||||
|
return nil, errors.New("this control plane is not on the bus, so it cannot hear what the bus says")
|
||||||
|
}
|
||||||
|
conn := s.js.Conn()
|
||||||
|
var subs []*nats.Subscription
|
||||||
|
stop := func() {
|
||||||
|
for _, sub := range subs {
|
||||||
|
_ = sub.Unsubscribe()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, subject := range broker.BusAdvisories {
|
||||||
|
sub, err := conn.Subscribe(subject, func(m *nats.Msg) {
|
||||||
|
if a, ok := ReadAdvisory(m.Subject, m.Data); ok {
|
||||||
|
Advisories.Heard(a, time.Now())
|
||||||
|
logf("the bus says: %s", a.Said)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
stop()
|
||||||
|
return nil, fmt.Errorf("listening to what the bus says (%s): %w", subject, err)
|
||||||
|
}
|
||||||
|
subs = append(subs, sub)
|
||||||
|
}
|
||||||
|
WatchConnection(conn, logf)
|
||||||
|
return stop, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// WatchConnection records what the bus tells this connection about itself: it fell behind, or a
|
||||||
|
// subject was refused it. Chained before whatever handler the connection had, which still runs. A
|
||||||
|
// refused answer to a call is the call log's to say (issue 265), and is not said twice.
|
||||||
|
func WatchConnection(conn *nats.Conn, logf func(string, ...any)) {
|
||||||
|
before := conn.ErrorHandler()
|
||||||
|
conn.SetErrorHandler(func(c *nats.Conn, sub *nats.Subscription, err error) {
|
||||||
|
if a, ok := connectionAdvisory(sub, err); ok {
|
||||||
|
Advisories.Heard(a, time.Now())
|
||||||
|
logf("the bus says: %s", a.Said)
|
||||||
|
}
|
||||||
|
if before != nil {
|
||||||
|
before(c, sub, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// connectionAdvisory is an error the bus handed the controller's connection, as an advisory.
|
||||||
|
func connectionAdvisory(sub *nats.Subscription, err error) (Advisory, bool) {
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, nats.ErrSlowConsumer):
|
||||||
|
subject := "a subscription"
|
||||||
|
if sub != nil {
|
||||||
|
subject = sub.Subject
|
||||||
|
}
|
||||||
|
return Advisory{Kind: AdvisorySlowConsumer, ID: "controller",
|
||||||
|
Said: fmt.Sprintf("the controller fell behind on %s and the client dropped messages it was sent", subject)}, true
|
||||||
|
case errors.Is(err, nats.ErrPermissionViolation):
|
||||||
|
if m := refusedPublish.FindStringSubmatch(err.Error()); m != nil && strings.HasPrefix(m[1], "_INBOX.") &&
|
||||||
|
!strings.HasPrefix(m[1], "_INBOX.enrol.") {
|
||||||
|
return Advisory{}, false // a refused answer to a call, which the call log says against its call
|
||||||
|
}
|
||||||
|
return Advisory{Kind: AdvisoryRefused, ID: "controller",
|
||||||
|
Said: "the bus refused the controller: " + err.Error()}, true
|
||||||
|
}
|
||||||
|
return Advisory{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Refusals of a publish, by subject, for a caller waiting on an answer to it.
|
||||||
|
var refusalWaiters = struct {
|
||||||
|
sync.Mutex
|
||||||
|
hooked map[*nats.Conn]bool
|
||||||
|
by map[string][]chan error
|
||||||
|
}{hooked: map[*nats.Conn]bool{}, by: map[string][]chan error{}}
|
||||||
|
|
||||||
|
// refusalsOf is told when the bus refuses this connection a publish to subject, until stop is called.
|
||||||
|
// The connection's error handler is chained once, before whatever it had, which still runs.
|
||||||
|
func refusalsOf(conn *nats.Conn, subject string) (<-chan error, func()) {
|
||||||
|
ch := make(chan error, 1)
|
||||||
|
refusalWaiters.Lock()
|
||||||
|
defer refusalWaiters.Unlock()
|
||||||
|
if !refusalWaiters.hooked[conn] {
|
||||||
|
refusalWaiters.hooked[conn] = true
|
||||||
|
before := conn.ErrorHandler()
|
||||||
|
conn.SetErrorHandler(func(c *nats.Conn, sub *nats.Subscription, err error) {
|
||||||
|
if m := refusedPublish.FindStringSubmatch(errString(err)); m != nil {
|
||||||
|
refusalWaiters.Lock()
|
||||||
|
for _, w := range refusalWaiters.by[m[1]] {
|
||||||
|
select {
|
||||||
|
case w <- err:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
refusalWaiters.Unlock()
|
||||||
|
}
|
||||||
|
if before != nil {
|
||||||
|
before(c, sub, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
refusalWaiters.by[subject] = append(refusalWaiters.by[subject], ch)
|
||||||
|
return ch, func() {
|
||||||
|
refusalWaiters.Lock()
|
||||||
|
defer refusalWaiters.Unlock()
|
||||||
|
waiting := refusalWaiters.by[subject]
|
||||||
|
for i, w := range waiting {
|
||||||
|
if w == ch {
|
||||||
|
refusalWaiters.by[subject] = append(waiting[:i], waiting[i+1:]...)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(refusalWaiters.by[subject]) == 0 {
|
||||||
|
delete(refusalWaiters.by, subject)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func errString(err error) string {
|
||||||
|
if err == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return err.Error()
|
||||||
|
}
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **A reader's own consumer gone is not a consumer lost**: every bucket watch and stream read-back
|
||||||
|
// makes and deletes one, many a minute, and the bus says so each time (found running the controller
|
||||||
|
// against a real bus: five a tick).
|
||||||
|
func TestOnlyTheMeshsOwnConsumersAreSaidLost(t *testing.T) {
|
||||||
|
deleted := func(stream, consumer string) bool {
|
||||||
|
_, ok := ReadAdvisory("$JS.EVENT.ADVISORY.CONSUMER.DELETED."+stream+"."+consumer,
|
||||||
|
[]byte(`{"type":"io.nats.jetstream.advisory.v1.consumer_action","stream":"`+stream+`","consumer":"`+consumer+`","action":"delete"}`))
|
||||||
|
return ok
|
||||||
|
}
|
||||||
|
for _, c := range [][2]string{{"EVENTS", "controller"}, {"CONTROL", "controller"}, {"NODES", "anchor"},
|
||||||
|
{"EVENTS", "anchor_shop"}, {"SEAT_NODE_BUILD_AGENT", "SEAT_NODE_BUILD_AGENT_worker"}} {
|
||||||
|
if !deleted(c[0], c[1]) {
|
||||||
|
t.Errorf("%s on %s deleted is not said", c[1], c[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, c := range [][2]string{{"KV_mesh-controller_conditions", "381UWW5Y"}, {"EVENTS", "E7vVYoe6"},
|
||||||
|
{"SEAT_NODE_BUILD_AGENT", "Rcnt6jla"}, {"ASSIGNMENTS", "x"}} {
|
||||||
|
if deleted(c[0], c[1]) {
|
||||||
|
t.Errorf("a reader's own consumer %s on %s is said lost", c[1], c[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a, ok := ReadAdvisory("$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.EVENTS.anchor_shop",
|
||||||
|
[]byte(`{"stream":"EVENTS","consumer":"anchor_shop","stream_seq":7,"deliveries":5}`))
|
||||||
|
if !ok || a.Kind != AdvisoryMaxDeliveries || a.ID != "EVENTS.anchor_shop" ||
|
||||||
|
a.Said != "how shop on anchor hears what it consumes handed message 7 over 5 times and gave up on it: "+
|
||||||
|
"it will not be delivered again, and what it asked for was not done" {
|
||||||
|
t.Fatalf("%+v", a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A question the bus refuses is answered as refused at once**, not after its timeout: against a
|
||||||
|
// server whose only user may not publish where it asks.
|
||||||
|
func TestNatsARefusedQuestionIsSaidAtOnce(t *testing.T) {
|
||||||
|
url := os.Getenv("MESH_TEST_NATS_REFUSING")
|
||||||
|
if url == "" {
|
||||||
|
t.Skip("MESH_TEST_NATS_REFUSING unset: a server whose user may not publish mesh.seat.>")
|
||||||
|
}
|
||||||
|
conn, err := nats.Connect(url)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
started := time.Now()
|
||||||
|
_, err = AskSeatTool(t.Context(), conn, "node-intrusion-prevention", "banned", "anchor", map[string]any{}, 10*time.Second)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "the bus refused") {
|
||||||
|
t.Fatalf("answered %v", err)
|
||||||
|
}
|
||||||
|
if took := time.Since(started); took > 3*time.Second {
|
||||||
|
t.Fatalf("a refusal took %s to be known", took)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The refusal reaches whoever waits on that subject, and only them.
|
||||||
|
func TestNatsARefusalReachesItsWaiter(t *testing.T) {
|
||||||
|
url := os.Getenv("MESH_TEST_NATS")
|
||||||
|
if url == "" {
|
||||||
|
t.Skip("MESH_TEST_NATS unset")
|
||||||
|
}
|
||||||
|
conn, err := nats.Connect(url)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
mine, stop := refusalsOf(conn, "mesh.seat.s.tool.v.anchor")
|
||||||
|
defer stop()
|
||||||
|
other, stopOther := refusalsOf(conn, "mesh.seat.s.tool.v.laptop")
|
||||||
|
defer stopOther()
|
||||||
|
conn.ErrorHandler()(conn, nil, fmt.Errorf("%w: Permissions Violation for Publish to %q",
|
||||||
|
nats.ErrPermissionViolation, "mesh.seat.s.tool.v.anchor"))
|
||||||
|
select {
|
||||||
|
case <-mine:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("the waiter was not told")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-other:
|
||||||
|
t.Fatal("another subject's waiter was told")
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -71,6 +71,10 @@ const (
|
|||||||
// next heartbeat, and a stream of them is the mesh's least valuable message competing for
|
// next heartbeat, and a stream of them is the mesh's least valuable message competing for
|
||||||
// retention with its most valuable (design 25 §3).
|
// retention with its most valuable (design 25 §3).
|
||||||
AliveSubjects = "mesh.control.*.alive"
|
AliveSubjects = "mesh.control.*.alive"
|
||||||
|
|
||||||
|
// ToolsAliveSubjects is every machine's node tools saying they are there (novox/hq to-be 45 §3,
|
||||||
|
// S11): core NATS like the host's, for the same reason.
|
||||||
|
ToolsAliveSubjects = "mesh.control.*.tools-alive"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ReportSubject is where one node says what it did. On the CONTROL stream, because it is the
|
// ReportSubject is where one node says what it did. On the CONTROL stream, because it is the
|
||||||
@@ -80,6 +84,9 @@ func ReportSubject(node string) string { return "mesh.control." + node + ".repor
|
|||||||
// AliveSubject is one node's heartbeat.
|
// AliveSubject is one node's heartbeat.
|
||||||
func AliveSubject(node string) string { return "mesh.control." + node + ".alive" }
|
func AliveSubject(node string) string { return "mesh.control." + node + ".alive" }
|
||||||
|
|
||||||
|
// ToolsAliveSubject is one machine's node tools' heartbeat.
|
||||||
|
func ToolsAliveSubject(node string) string { return "mesh.control." + node + ".tools-alive" }
|
||||||
|
|
||||||
// EventSubject is where a module's event lands. Derived from the emitter, never taken from the
|
// EventSubject is where a module's event lands. Derived from the emitter, never taken from the
|
||||||
// caller: a source that could differ from the subject is an envelope that can lie about its
|
// caller: a source that could differ from the subject is an envelope that can lie about its
|
||||||
// origin, and on NATS the account's permissions make the subject the authority (design 29 §2).
|
// origin, and on NATS the account's permissions make the subject the authority (design 29 §2).
|
||||||
|
|||||||
+220
-15
@@ -7,7 +7,9 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"regexp"
|
"regexp"
|
||||||
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -33,8 +35,9 @@ import (
|
|||||||
// either. A variable so a test need not wait.
|
// either. A variable so a test need not wait.
|
||||||
var AnswerWithin = 10 * time.Second
|
var AnswerWithin = 10 * time.Second
|
||||||
|
|
||||||
// KeptCalls is how many calls are kept, newest first; keptAnswer the largest answer kept of a call
|
// KeptCalls is how many calls this process keeps in memory, newest first — to match a refusal to its
|
||||||
// whose caller was sent it. One its caller never had is kept whole.
|
// call, and to answer at once; the bus keeps the last thousand (Durably). keptAnswer is the largest
|
||||||
|
// answer kept of a call whose caller was sent it. One its caller never had is kept whole.
|
||||||
const (
|
const (
|
||||||
KeptCalls = 100
|
KeptCalls = 100
|
||||||
keptAnswer = 64 << 10
|
keptAnswer = 64 << 10
|
||||||
@@ -47,6 +50,10 @@ const (
|
|||||||
// CallFinishedAfter is a call that finished after its caller was told it was still running: its
|
// CallFinishedAfter is a call that finished after its caller was told it was still running: its
|
||||||
// answer is here and nowhere else.
|
// answer is here and nowhere else.
|
||||||
CallFinishedAfter = "finished after its caller was answered"
|
CallFinishedAfter = "finished after its caller was answered"
|
||||||
|
// CallAbandoned is a call whose controller stopped before it finished (novox/hq to-be 45 §6): a
|
||||||
|
// controller starting finds it running under another and says so, rather than leaving it running
|
||||||
|
// for ever in the record.
|
||||||
|
CallAbandoned = "abandoned: the controller running it stopped before it finished"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Call is one call of a role's tool, as `calls` shows it.
|
// Call is one call of a role's tool, as `calls` shows it.
|
||||||
@@ -65,10 +72,27 @@ type Call struct {
|
|||||||
// Refused is the bus refusing the answer this holder sent: the caller got nothing, and this is
|
// Refused is the bus refusing the answer this holder sent: the caller got nothing, and this is
|
||||||
// the only place that says what it would have.
|
// the only place that says what it would have.
|
||||||
Refused string `json:"answer refused by the bus,omitempty"`
|
Refused string `json:"answer refused by the bus,omitempty"`
|
||||||
|
// Caller is the bus principal that asked, read from the inbox its answer went to — every principal
|
||||||
|
// is granted only its own (novox/hq to-be 45 §7: a hand act says who).
|
||||||
|
Caller string `json:"caller,omitempty"`
|
||||||
|
// Holder is the controller process that served it, so one starting can tell its own running
|
||||||
|
// calls from those a stopped one left.
|
||||||
|
Holder string `json:"holder,omitempty"`
|
||||||
|
|
||||||
reply string // the subject the answer went to, which the bus names when it refuses it
|
reply string // the subject the answer went to, which the bus names when it refuses it
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CallKeeper keeps calls where the process serving them does not: the controller's bucket on the
|
||||||
|
// bus (novox/hq to-be 45 §6). A call is kept whole on every change — begun, finished, refused — so a
|
||||||
|
// controller replaced at any moment leaves the last word on each.
|
||||||
|
type CallKeeper interface {
|
||||||
|
Keep(ctx context.Context, c Call) error
|
||||||
|
// Kept is one call with its whole answer.
|
||||||
|
Kept(ctx context.Context, id string) (Call, bool, error)
|
||||||
|
// Recent is the kept calls, newest first, without their answers.
|
||||||
|
Recent(ctx context.Context) ([]Call, error)
|
||||||
|
}
|
||||||
|
|
||||||
// CallLog keeps the latest calls a holder served.
|
// CallLog keeps the latest calls a holder served.
|
||||||
type CallLog struct {
|
type CallLog struct {
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
@@ -78,6 +102,15 @@ type CallLog struct {
|
|||||||
// Follow is the tool that reads this log back, named in a running answer — set by a holder that
|
// Follow is the tool that reads this log back, named in a running answer — set by a holder that
|
||||||
// serves one (the controller's `calls`); without it, the answer points at the holder's journal.
|
// serves one (the controller's `calls`); without it, the answer points at the holder's journal.
|
||||||
Follow string
|
Follow string
|
||||||
|
|
||||||
|
// keeper keeps every call beyond this process, when Durably was given one; writes go through
|
||||||
|
// one goroutine, in order, so a call's last state is the one kept.
|
||||||
|
keeper CallKeeper
|
||||||
|
holder string
|
||||||
|
writes chan Call
|
||||||
|
logger *log.Logger
|
||||||
|
lost int // writes the keeper could not take, said once each
|
||||||
|
keepErr error
|
||||||
}
|
}
|
||||||
|
|
||||||
// Calls is this process's log: one holder process serves its seats on one connection.
|
// Calls is this process's log: one holder process serves its seats on one connection.
|
||||||
@@ -85,16 +118,130 @@ var Calls = NewCallLog()
|
|||||||
|
|
||||||
func NewCallLog() *CallLog { return &CallLog{now: time.Now} }
|
func NewCallLog() *CallLog { return &CallLog{now: time.Now} }
|
||||||
|
|
||||||
|
// keepTries is how many times one call's state is offered to the keeper before it is said lost: the
|
||||||
|
// bus reloading its user list refuses for a moment, and that is exactly when a push runs.
|
||||||
|
const keepTries = 5
|
||||||
|
|
||||||
|
// Durably keeps every call from now on with keeper as well as in memory, under this process's name,
|
||||||
|
// and marks running the calls a controller before this one left running: it stopped, so they cannot
|
||||||
|
// finish (novox/hq to-be 45 §6). Said, naming each.
|
||||||
|
func (l *CallLog) Durably(ctx context.Context, keeper CallKeeper, holder string, logger *log.Logger) error {
|
||||||
|
l.mu.Lock()
|
||||||
|
l.keeper, l.holder, l.logger = keeper, holder, logger
|
||||||
|
if l.writes == nil {
|
||||||
|
l.writes = make(chan Call, 256)
|
||||||
|
go l.keepWrites()
|
||||||
|
}
|
||||||
|
l.mu.Unlock()
|
||||||
|
kept, err := keeper.Recent(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("reading the calls kept on the bus: %w", err)
|
||||||
|
}
|
||||||
|
for _, c := range kept {
|
||||||
|
if c.State != CallRunning || c.Holder == holder {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
whole, found, err := keeper.Kept(ctx, c.ID)
|
||||||
|
if err != nil || !found {
|
||||||
|
whole = c
|
||||||
|
}
|
||||||
|
whole.State = CallAbandoned
|
||||||
|
if err := keeper.Keep(ctx, whole); err != nil {
|
||||||
|
return fmt.Errorf("marking %s abandoned: %w", c.ID, err)
|
||||||
|
}
|
||||||
|
if logger != nil {
|
||||||
|
logger.Printf("%s (%s.%s, asked %s by %s) was running under %s, which stopped: marked abandoned — "+
|
||||||
|
"it may have done part of what it was asked, and nothing will finish it", c.ID, c.Seat, c.Verb,
|
||||||
|
c.Started.Format(time.RFC3339), orSomebody(c.Caller), orSomebody(c.Holder))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func orSomebody(s string) string {
|
||||||
|
if s == "" {
|
||||||
|
return "an unnamed caller"
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// keep queues one call's state for the keeper. Never blocks a call: a queue that is full is a keeper
|
||||||
|
// that is not taking writes, and that is said rather than waited on.
|
||||||
|
func (l *CallLog) keep(c Call) {
|
||||||
|
if l.writes == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case l.writes <- c:
|
||||||
|
default:
|
||||||
|
l.lost++
|
||||||
|
if l.logger != nil {
|
||||||
|
l.logger.Printf("%s (%s.%s) is kept in memory only: the bus is not taking calls' records (%d not kept)",
|
||||||
|
c.ID, c.Seat, c.Verb, l.lost)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *CallLog) keepWrites() {
|
||||||
|
for c := range l.writes {
|
||||||
|
var err error
|
||||||
|
for try := 0; try < keepTries; try++ {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
err = l.keeper.Keep(ctx, c)
|
||||||
|
cancel()
|
||||||
|
if err == nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
time.Sleep(time.Duration(try+1) * time.Second)
|
||||||
|
}
|
||||||
|
if err != nil && l.logger != nil {
|
||||||
|
l.logger.Printf("%s (%s.%s, %s) could not be kept on the bus after %d tries: %v — `calls` "+
|
||||||
|
"answers it from memory until this controller stops", c.ID, c.Seat, c.Verb, c.State, keepTries, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// callerOf is the bus principal an answer goes to: every principal's inbox is `_INBOX.<its user>.`
|
||||||
|
// followed by the client's own random token, and a user may itself hold dots.
|
||||||
|
func callerOf(reply string) string {
|
||||||
|
rest, ok := strings.CutPrefix(reply, "_INBOX.")
|
||||||
|
if !ok {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
tokens := strings.Split(rest, ".")
|
||||||
|
for i, t := range tokens {
|
||||||
|
if i > 0 && isNUID(t) {
|
||||||
|
return strings.Join(tokens[:i], ".")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// isNUID is the client library's random inbox token: twenty-two letters and digits.
|
||||||
|
func isNUID(t string) bool {
|
||||||
|
if len(t) != 22 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, r := range t {
|
||||||
|
if !(r >= '0' && r <= '9' || r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z') {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
func (l *CallLog) begin(seat, verb string, args json.RawMessage, reply string) *Call {
|
func (l *CallLog) begin(seat, verb string, args json.RawMessage, reply string) *Call {
|
||||||
l.mu.Lock()
|
l.mu.Lock()
|
||||||
defer l.mu.Unlock()
|
defer l.mu.Unlock()
|
||||||
l.next++
|
l.next++
|
||||||
c := &Call{ID: "call-" + strconv.FormatInt(l.now().UnixNano(), 10) + "-" + strconv.FormatUint(l.next, 10),
|
c := &Call{ID: "call-" + strconv.FormatInt(l.now().UnixNano(), 10) + "-" + strconv.FormatUint(l.next, 10),
|
||||||
Seat: seat, Verb: verb, Args: args, Started: l.now(), State: CallRunning, reply: reply}
|
Seat: seat, Verb: verb, Args: args, Started: l.now(), State: CallRunning, reply: reply,
|
||||||
|
Caller: callerOf(reply), Holder: l.holder}
|
||||||
l.calls = append(l.calls, c)
|
l.calls = append(l.calls, c)
|
||||||
if len(l.calls) > KeptCalls {
|
if len(l.calls) > KeptCalls {
|
||||||
l.calls = l.calls[len(l.calls)-KeptCalls:]
|
l.calls = l.calls[len(l.calls)-KeptCalls:]
|
||||||
}
|
}
|
||||||
|
l.keep(*c)
|
||||||
return c
|
return c
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -117,29 +264,84 @@ func (l *CallLog) finish(c *Call, answer []byte, failed, answeredAlready bool) {
|
|||||||
} else {
|
} else {
|
||||||
c.State = CallAnswered
|
c.State = CallAnswered
|
||||||
}
|
}
|
||||||
|
l.keep(*c)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Recent is the kept calls, newest first, as copies.
|
// Recent is the kept calls, newest first, as copies: this process's from memory, and, when they are
|
||||||
func (l *CallLog) Recent() []Call {
|
// kept durably, every other the bus holds — a call a controller before this one served included.
|
||||||
|
// Memory wins for a call in both, being the newer word on it. A bus that cannot be read is said in
|
||||||
|
// the error beside what memory holds, never answered as no calls.
|
||||||
|
// Running is every call this process is serving that has not finished, oldest first, without
|
||||||
|
// answers: what the watchdog of a call's bound (novox/hq to-be 45 S7) reads. This process's own,
|
||||||
|
// because a call another controller left running is said abandoned when this one starts.
|
||||||
|
func (l *CallLog) Running() []Call {
|
||||||
l.mu.Lock()
|
l.mu.Lock()
|
||||||
defer l.mu.Unlock()
|
defer l.mu.Unlock()
|
||||||
out := make([]Call, 0, len(l.calls))
|
var out []Call
|
||||||
for i := len(l.calls) - 1; i >= 0; i-- {
|
for _, c := range l.calls {
|
||||||
out = append(out, *l.calls[i])
|
if c.State == CallRunning {
|
||||||
|
running := *c
|
||||||
|
running.Answer = nil
|
||||||
|
out = append(out, running)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get is one kept call.
|
func (l *CallLog) Recent() ([]Call, error) {
|
||||||
func (l *CallLog) Get(id string) (Call, bool) {
|
|
||||||
l.mu.Lock()
|
l.mu.Lock()
|
||||||
defer l.mu.Unlock()
|
out := make([]Call, 0, len(l.calls))
|
||||||
for _, c := range l.calls {
|
seen := map[string]bool{}
|
||||||
if c.ID == id {
|
for i := len(l.calls) - 1; i >= 0; i-- {
|
||||||
return *c, true
|
out = append(out, *l.calls[i])
|
||||||
|
seen[l.calls[i].ID] = true
|
||||||
|
}
|
||||||
|
keeper := l.keeper
|
||||||
|
l.mu.Unlock()
|
||||||
|
if keeper == nil {
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
kept, err := keeper.Recent(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return out, fmt.Errorf("the calls kept on the bus could not be read, so only this controller's own "+
|
||||||
|
"are listed: %w", err)
|
||||||
|
}
|
||||||
|
for _, c := range kept {
|
||||||
|
if !seen[c.ID] {
|
||||||
|
out = append(out, c)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return Call{}, false
|
sort.SliceStable(out, func(i, j int) bool { return out[i].Started.After(out[j].Started) })
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get is one kept call: from memory, or from the bus when this process did not serve it.
|
||||||
|
func (l *CallLog) Get(id string) (Call, bool, error) {
|
||||||
|
l.mu.Lock()
|
||||||
|
for _, c := range l.calls {
|
||||||
|
if c.ID == id {
|
||||||
|
found := *c
|
||||||
|
l.mu.Unlock()
|
||||||
|
return found, true, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
keeper := l.keeper
|
||||||
|
l.mu.Unlock()
|
||||||
|
if keeper == nil {
|
||||||
|
return Call{}, false, nil
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
return keeper.Kept(ctx, id)
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsDurable says whether calls outlive this process.
|
||||||
|
func (l *CallLog) IsDurable() bool {
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
return l.keeper != nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// kept is what a call's arguments are kept as: each argument by name, a value only when it is short
|
// kept is what a call's arguments are kept as: each argument by name, a value only when it is short
|
||||||
@@ -195,6 +397,7 @@ func (l *CallLog) Refusal(err error, logger *log.Logger) bool {
|
|||||||
at := l.now()
|
at := l.now()
|
||||||
hit.Refused = fmt.Sprintf("%s: %s", at.Format(time.RFC3339), err)
|
hit.Refused = fmt.Sprintf("%s: %s", at.Format(time.RFC3339), err)
|
||||||
id, verb, took := hit.ID, hit.Seat+"."+hit.Verb, at.Sub(hit.Started).Round(time.Second)
|
id, verb, took := hit.ID, hit.Seat+"."+hit.Verb, at.Sub(hit.Started).Round(time.Second)
|
||||||
|
l.keep(*hit)
|
||||||
l.mu.Unlock()
|
l.mu.Unlock()
|
||||||
if logger != nil {
|
if logger != nil {
|
||||||
// Said in the mesh's words, beside the library's own line: which call, and where its answer is.
|
// Said in the mesh's words, beside the library's own line: which call, and where its answer is.
|
||||||
@@ -277,6 +480,8 @@ func (l *CallLog) serveCall(seat, verb string, args json.RawMessage, reply strin
|
|||||||
args = json.RawMessage(`{}`)
|
args = json.RawMessage(`{}`)
|
||||||
}
|
}
|
||||||
c := l.begin(seat, verb, kept(args), reply)
|
c := l.begin(seat, verb, kept(args), reply)
|
||||||
|
// Who asked travels with the call, so an act it does by hand says so (novox/hq to-be 45 §7).
|
||||||
|
ctx = context.WithValue(ctx, callerKey{}, c.Caller)
|
||||||
acknowledged := make(chan struct{})
|
acknowledged := make(chan struct{})
|
||||||
var once sync.Once
|
var once sync.Once
|
||||||
ctx = context.WithValue(ctx, answerNowKey{}, func() { once.Do(func() { close(acknowledged) }) })
|
ctx = context.WithValue(ctx, answerNowKey{}, func() { once.Do(func() { close(acknowledged) }) })
|
||||||
|
|||||||
@@ -0,0 +1,115 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Calls kept on the bus (novox/hq to-be 45 §6).
|
||||||
|
//
|
||||||
|
// **Two keys a call**: `<id>` holds the record — verb, arguments as kept, caller, state, times — and
|
||||||
|
// `<id>.answer` the answer, bounded. A listing watches the records alone, so reading the last thousand
|
||||||
|
// calls reads the last thousand small records and not a thousand answers; one call asked by id reads
|
||||||
|
// both. A call's id is one token, so the record's key never holds a dot and `*` matches records only.
|
||||||
|
|
||||||
|
// BusCalls keeps calls in the controller's calls bucket.
|
||||||
|
type BusCalls struct {
|
||||||
|
kv jetstream.KeyValue
|
||||||
|
}
|
||||||
|
|
||||||
|
// CallsOnTheBus opens the calls bucket the controller asserts at its start.
|
||||||
|
func CallsOnTheBus(ctx context.Context, conn *nats.Conn) (*BusCalls, error) {
|
||||||
|
api, err := jetstream.New(conn)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
kv, err := api.KeyValue(ctx, broker.CallsBucket)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the calls bucket %s is not on the bus — the controller asserts it at its "+
|
||||||
|
"start, so one older than this has not: %w", broker.CallsBucket, err)
|
||||||
|
}
|
||||||
|
return &BusCalls{kv: kv}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const answerKey = ".answer"
|
||||||
|
|
||||||
|
// Keep writes a call's record, and its answer when it has one.
|
||||||
|
func (b *BusCalls) Keep(ctx context.Context, c Call) error {
|
||||||
|
answer := c.Answer
|
||||||
|
c.Answer = nil
|
||||||
|
if len(answer) > 0 {
|
||||||
|
if len(answer) > broker.CallAnswerBytes {
|
||||||
|
answer, _ = json.Marshal(map[string]any{"cut": fmt.Sprintf("an answer of %d bytes; the first %d "+
|
||||||
|
"are kept", len(answer), broker.CallAnswerBytes), "start": string(answer[:broker.CallAnswerBytes])})
|
||||||
|
}
|
||||||
|
// The answer before the record, so a record saying a call finished never points at an answer
|
||||||
|
// not yet written.
|
||||||
|
if _, err := b.kv.Put(ctx, c.ID+answerKey, answer); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
record, err := json.Marshal(c)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = b.kv.Put(ctx, c.ID, record)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Kept is one call, with its answer.
|
||||||
|
func (b *BusCalls) Kept(ctx context.Context, id string) (Call, bool, error) {
|
||||||
|
entry, err := b.kv.Get(ctx, id)
|
||||||
|
if errors.Is(err, jetstream.ErrKeyNotFound) || errors.Is(err, jetstream.ErrInvalidKey) {
|
||||||
|
return Call{}, false, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return Call{}, false, err
|
||||||
|
}
|
||||||
|
var c Call
|
||||||
|
if err := json.Unmarshal(entry.Value(), &c); err != nil {
|
||||||
|
return Call{}, false, fmt.Errorf("the record of %s on the bus is not a call: %w", id, err)
|
||||||
|
}
|
||||||
|
answer, err := b.kv.Get(ctx, id+answerKey)
|
||||||
|
switch {
|
||||||
|
case err == nil:
|
||||||
|
c.Answer = json.RawMessage(answer.Value())
|
||||||
|
case !errors.Is(err, jetstream.ErrKeyNotFound):
|
||||||
|
return Call{}, false, err
|
||||||
|
}
|
||||||
|
return c, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recent is every call the bucket holds, newest first, without answers: read once through a watch
|
||||||
|
// of the records, which hands over the current value of each and then says it has.
|
||||||
|
func (b *BusCalls) Recent(ctx context.Context) ([]Call, error) {
|
||||||
|
w, err := b.kv.Watch(ctx, "*", jetstream.IgnoreDeletes())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer func() { _ = w.Stop() }()
|
||||||
|
var out []Call
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, fmt.Errorf("reading the calls bucket: %w", ctx.Err())
|
||||||
|
case entry := <-w.Updates():
|
||||||
|
if entry == nil {
|
||||||
|
// Every current value handed over.
|
||||||
|
sort.SliceStable(out, func(i, j int) bool { return out[i].Started.After(out[j].Started) })
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
var c Call
|
||||||
|
if json.Unmarshal(entry.Value(), &c) == nil && c.ID != "" {
|
||||||
|
out = append(out, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,152 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"log"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The calls bucket against a real server (novox/hq to-be 45 §6): whether a call's outcome is
|
||||||
|
// readable by id from a controller that did not serve it is a claim about what the bus keeps.
|
||||||
|
|
||||||
|
func callsBucket(t *testing.T) *BusCalls {
|
||||||
|
t.Helper()
|
||||||
|
js := aBus(t)
|
||||||
|
api, err := jetstream.New(js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = api.DeleteKeyValue(t.Context(), broker.CallsBucket)
|
||||||
|
if err := js.EnsureControllerBuckets(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
keeper, err := CallsOnTheBus(t.Context(), js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return keeper
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitKept waits until the keeper holds a call in a state, the writes being queued.
|
||||||
|
func waitKept(t *testing.T, keeper CallKeeper, id, state string) Call {
|
||||||
|
t.Helper()
|
||||||
|
deadline := time.Now().Add(5 * time.Second)
|
||||||
|
for {
|
||||||
|
c, found, err := keeper.Kept(context.Background(), id)
|
||||||
|
if err == nil && found && c.State == state {
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatalf("%s never kept as %q: %+v %v %v", id, state, c, found, err)
|
||||||
|
}
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A controller restart keeps every call's outcome** (to-be 45 Phase 0): a call one controller
|
||||||
|
// answered is read whole by id from the next, and a call it left running is said abandoned rather
|
||||||
|
// than running for ever.
|
||||||
|
func TestNatsACallsOutcomeOutlivesItsController(t *testing.T) {
|
||||||
|
keeper := callsBucket(t)
|
||||||
|
first := NewCallLog()
|
||||||
|
if err := first.Durably(t.Context(), keeper, "controller@one", nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
a := newAnswers(t)
|
||||||
|
first.serveCall("mesh-controller", "push", json.RawMessage(`{"node":"anchor","values":"secret"}`),
|
||||||
|
"_INBOX.node-tools.g14.Pe3sGzAtv8jUBYQ6sKSvKz.1",
|
||||||
|
func(context.Context, json.RawMessage) (any, error) { return "anchor told", nil }, a.respond, nil)
|
||||||
|
recent, _ := first.Recent()
|
||||||
|
finished := waitKept(t, keeper, recent[0].ID, CallAnswered)
|
||||||
|
// A second call, still running when its controller stops.
|
||||||
|
left := first.begin("mesh-controller", "plans", json.RawMessage(`{}`), "")
|
||||||
|
waitKept(t, keeper, left.ID, CallRunning)
|
||||||
|
|
||||||
|
var said bytes.Buffer
|
||||||
|
second := NewCallLog()
|
||||||
|
if err := second.Durably(t.Context(), keeper, "controller@two", log.New(&said, "", 0)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
c, found, err := second.Get(finished.ID)
|
||||||
|
if err != nil || !found {
|
||||||
|
t.Fatalf("the next controller cannot read %s: %v %v", finished.ID, found, err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(c.Answer), "anchor told") || c.Caller != "node-tools.g14" || c.Holder != "controller@one" {
|
||||||
|
t.Fatalf("kept %+v", c)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(c.Args), "secret") {
|
||||||
|
t.Fatalf("a setting was kept: %s", c.Args)
|
||||||
|
}
|
||||||
|
abandoned, _, _ := second.Get(left.ID)
|
||||||
|
if abandoned.State != CallAbandoned || !strings.Contains(said.String(), left.ID) {
|
||||||
|
t.Fatalf("a call left running reads %q, and was said: %q", abandoned.State, said.String())
|
||||||
|
}
|
||||||
|
listed, err := second.Recent()
|
||||||
|
if err != nil || len(listed) != 2 {
|
||||||
|
t.Fatalf("the next controller lists %d calls: %v", len(listed), err)
|
||||||
|
}
|
||||||
|
// Its own running calls are not its predecessor's: a controller starting again under the same
|
||||||
|
// name leaves them alone.
|
||||||
|
mine := second.begin("mesh-controller", "status", nil, "")
|
||||||
|
waitKept(t, keeper, mine.ID, CallRunning)
|
||||||
|
if err := second.Durably(t.Context(), keeper, "controller@two", nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if c, _, _ := keeper.Kept(t.Context(), mine.ID); c.State != CallRunning {
|
||||||
|
t.Fatalf("a controller marked its own running call %q", c.State)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The bucket holds the last thousand calls or fourteen days: a call is two keys, so the stream under
|
||||||
|
// it holds twice as many messages, and asserting it again keeps the count.
|
||||||
|
func TestNatsTheCallsBucketIsBounded(t *testing.T) {
|
||||||
|
callsBucket(t)
|
||||||
|
js := aBus(t)
|
||||||
|
if err := js.EnsureControllerBuckets(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
info, err := js.Context().StreamInfo("KV_" + broker.CallsBucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if info.Config.MaxMsgs != 2*broker.KeptCallsDurably || info.Config.MaxAge != broker.CallsKeptFor {
|
||||||
|
t.Fatalf("kept %d messages for %s", info.Config.MaxMsgs, info.Config.MaxAge)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An answer larger than the bound is cut and says so, and the record is still written.
|
||||||
|
func TestNatsAnAnswerLargerThanTheBoundIsCut(t *testing.T) {
|
||||||
|
keeper := callsBucket(t)
|
||||||
|
big, _ := json.Marshal(map[string]string{"result": strings.Repeat("x", broker.CallAnswerBytes+10)})
|
||||||
|
c := Call{ID: "call-1-1", Seat: "mesh-controller", Verb: "plan", State: CallAnswered, Answer: big}
|
||||||
|
if err := keeper.Keep(t.Context(), c); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, found, err := keeper.Kept(t.Context(), "call-1-1")
|
||||||
|
if err != nil || !found || !strings.Contains(string(got.Answer), "the first") {
|
||||||
|
t.Fatalf("%v %v %.200s", found, err, got.Answer)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Who asked is read from the inbox the answer goes to.
|
||||||
|
func TestTheCallerIsTheInboxsPrincipal(t *testing.T) {
|
||||||
|
for reply, want := range map[string]string{
|
||||||
|
"_INBOX.node-tools.g14.Pe3sGzAtv8jUBYQ6sKSvKz.1": "node-tools.g14",
|
||||||
|
"_INBOX.jochen.Pe3sGzAtv8jUBYQ6sKSvKz": "jochen",
|
||||||
|
"_INBOX.x.1": "",
|
||||||
|
"mesh.control.one": "",
|
||||||
|
"": "",
|
||||||
|
} {
|
||||||
|
if got := callerOf(reply); got != want {
|
||||||
|
t.Errorf("%q: %q, want %q", reply, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -62,7 +62,7 @@ func TestACallThatFinishesInTimeAnswersInFull(t *testing.T) {
|
|||||||
if got := a.only()["result"]; got != "all well" {
|
if got := a.only()["result"]; got != "all well" {
|
||||||
t.Fatalf("answered %v", got)
|
t.Fatalf("answered %v", got)
|
||||||
}
|
}
|
||||||
recent := l.Recent()
|
recent, _ := l.Recent()
|
||||||
if len(recent) != 1 || recent[0].State != CallAnswered || string(recent[0].Args) != "{}" {
|
if len(recent) != 1 || recent[0].State != CallAnswered || string(recent[0].Args) != "{}" {
|
||||||
t.Fatalf("kept %+v", recent)
|
t.Fatalf("kept %+v", recent)
|
||||||
}
|
}
|
||||||
@@ -90,12 +90,12 @@ func TestACallThatOutlastsTheWindowSaysItIsRunningAndKeepsItsAnswer(t *testing.T
|
|||||||
if got["running"] != true || id == "" || !strings.Contains(got["output"].(string), id) {
|
if got["running"] != true || id == "" || !strings.Contains(got["output"].(string), id) {
|
||||||
t.Fatalf("the running answer does not name its call: %v", got)
|
t.Fatalf("the running answer does not name its call: %v", got)
|
||||||
}
|
}
|
||||||
if c, _ := l.Get(id); c.State != CallRunning {
|
if c, _, _ := l.Get(id); c.State != CallRunning {
|
||||||
t.Fatalf("while it runs it is kept as %q", c.State)
|
t.Fatalf("while it runs it is kept as %q", c.State)
|
||||||
}
|
}
|
||||||
close(release)
|
close(release)
|
||||||
<-finished
|
<-finished
|
||||||
c, ok := l.Get(id)
|
c, ok, _ := l.Get(id)
|
||||||
if !ok || c.State != CallFinishedAfter || !strings.Contains(string(c.Answer), "anchor told") {
|
if !ok || c.State != CallFinishedAfter || !strings.Contains(string(c.Answer), "anchor told") {
|
||||||
t.Fatalf("its answer was not kept: %+v", c)
|
t.Fatalf("its answer was not kept: %+v", c)
|
||||||
}
|
}
|
||||||
@@ -125,7 +125,7 @@ func TestAnAcknowledgedCallIsAnsweredBeforeItGoesOn(t *testing.T) {
|
|||||||
if got := a.only()["result"].(map[string]any); got["running"] != true {
|
if got := a.only()["result"].(map[string]any); got["running"] != true {
|
||||||
t.Fatalf("an acknowledged call answered %v", got)
|
t.Fatalf("an acknowledged call answered %v", got)
|
||||||
}
|
}
|
||||||
if c := l.Recent()[0]; c.State != CallFinishedAfter || !strings.Contains(string(c.Answer), "sent") {
|
if c := recentOf(l)[0]; c.State != CallFinishedAfter || !strings.Contains(string(c.Answer), "sent") {
|
||||||
t.Fatalf("kept %+v", c)
|
t.Fatalf("kept %+v", c)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -143,7 +143,7 @@ func TestARefusedAnswerIsKeptAgainstItsCall(t *testing.T) {
|
|||||||
if !l.Refusal(refusal, log.New(&logged, "", 0)) {
|
if !l.Refusal(refusal, log.New(&logged, "", 0)) {
|
||||||
t.Fatal("the refusal of a kept call's answer was not recognised")
|
t.Fatal("the refusal of a kept call's answer was not recognised")
|
||||||
}
|
}
|
||||||
c := l.Recent()[0]
|
c := recentOf(l)[0]
|
||||||
if c.Refused == "" || !strings.Contains(logged.String(), c.ID) {
|
if c.Refused == "" || !strings.Contains(logged.String(), c.ID) {
|
||||||
t.Fatalf("the refusal is not kept or not said: %+v / %q", c, logged.String())
|
t.Fatalf("the refusal is not kept or not said: %+v / %q", c, logged.String())
|
||||||
}
|
}
|
||||||
@@ -167,8 +167,13 @@ func TestTheLogKeepsTheNewest(t *testing.T) {
|
|||||||
for i := 0; i < KeptCalls+5; i++ {
|
for i := 0; i < KeptCalls+5; i++ {
|
||||||
l.begin("s", "v", nil, "")
|
l.begin("s", "v", nil, "")
|
||||||
}
|
}
|
||||||
recent := l.Recent()
|
recent, _ := l.Recent()
|
||||||
if len(recent) != KeptCalls || !strings.HasSuffix(recent[0].ID, "-105") || !strings.HasSuffix(recent[KeptCalls-1].ID, "-6") {
|
if len(recent) != KeptCalls || !strings.HasSuffix(recent[0].ID, "-105") || !strings.HasSuffix(recent[KeptCalls-1].ID, "-6") {
|
||||||
t.Fatalf("kept %d, newest %s, oldest %s", len(recent), recent[0].ID, recent[len(recent)-1].ID)
|
t.Fatalf("kept %d, newest %s, oldest %s", len(recent), recent[0].ID, recent[len(recent)-1].ID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func recentOf(l *CallLog) []Call {
|
||||||
|
out, _ := l.Recent()
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|||||||
@@ -380,6 +380,10 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err
|
|||||||
if report.Applied == nil && report.Refused == "" && len(report.Failed) == 0 {
|
if report.Applied == nil && report.Refused == "" && len(report.Failed) == 0 {
|
||||||
if report.Superseded != "" {
|
if report.Superseded != "" {
|
||||||
log.Printf("%s set aside declaration %s for the newer %s", report.Node, report.Declared, report.Superseded)
|
log.Printf("%s set aside declaration %s for the newer %s", report.Node, report.Declared, report.Superseded)
|
||||||
|
} else if err := e.Inventory.RecordHeartbeatGap(ctx, report.Node, node.LastSeen); err != nil {
|
||||||
|
// The silence before this word, which the machine-silent bound will be set from (novox/hq
|
||||||
|
// to-be 45 Phase 0). A measurement lost is said and costs the report nothing.
|
||||||
|
log.Printf("the silence before %s's word could not be recorded: %v", report.Node, err)
|
||||||
}
|
}
|
||||||
return false, e.Inventory.Seen(ctx, node.ID)
|
return false, e.Inventory.Seen(ctx, node.ID)
|
||||||
}
|
}
|
||||||
@@ -435,6 +439,11 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
|
// And how long the machine took, from the send to this first account of it (novox/hq to-be 45
|
||||||
|
// Phase 0): what the sent-not-reported bound will be set from. Said if lost, never a failure.
|
||||||
|
if err := e.Inventory.RecordApplyDuration(ctx, report.Node, report.Declared, doing.Outcome); err != nil {
|
||||||
|
log.Printf("how long %s took to apply could not be recorded: %v", report.Node, err)
|
||||||
|
}
|
||||||
|
|
||||||
// A refusal, a failure, or a bare word that the node is there — none of them is an account of
|
// A refusal, a failure, or a bare word that the node is there — none of them is an account of
|
||||||
// what the machine holds, so each moves last_seen and nothing else. Recording a partial list
|
// what the machine holds, so each moves last_seen and nothing else. Recording a partial list
|
||||||
|
|||||||
@@ -0,0 +1,162 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The hand-act log (novox/hq to-be 45 §7).
|
||||||
|
//
|
||||||
|
// **A repair a person makes by hand is the record of a healer the mesh does not have yet.** Tonight's
|
||||||
|
// pushes of one machine, plans closed because a report never came, a consumer re-made because it fell
|
||||||
|
// a week behind — each was done, and the only trace was a chat. So every verb that repairs by hand
|
||||||
|
// asks why, and writes one entry: who, which verb and arguments, why, when, the condition it
|
||||||
|
// addresses if one is named, and a cause — a word that, recorded twice in a fortnight, says a healer
|
||||||
|
// is wanted (S15, from Phase 3). `hand-act record` is the same entry for an act done outside the mesh.
|
||||||
|
// The controller is the bucket's only writer; its verbs are the way in.
|
||||||
|
|
||||||
|
// HandAct is one entry.
|
||||||
|
type HandAct struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
// By is who: the bus principal a seat call came from, or the account and machine at a shell.
|
||||||
|
By string `json:"by"`
|
||||||
|
// Verb and Args are the act as given: `push`, `plans close`, `broker consumer-reset`, or
|
||||||
|
// `hand-act record` with what was done outside the mesh.
|
||||||
|
Verb string `json:"verb"`
|
||||||
|
Args []string `json:"arguments,omitempty"`
|
||||||
|
Why string `json:"why"`
|
||||||
|
// Cause is the condition kind, or a word the person gives; the verb's own name when neither.
|
||||||
|
Cause string `json:"cause"`
|
||||||
|
// Condition is the condition's key the act addresses, when it names one.
|
||||||
|
Condition string `json:"condition,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// CallerVar carries a seat call's caller to the command the controller runs for it, so an act done
|
||||||
|
// through the console says who asked rather than "the controller".
|
||||||
|
const CallerVar = "MESH_CALLER"
|
||||||
|
|
||||||
|
// Caller is who is acting in this process: the seat call's caller when the controller ran it for
|
||||||
|
// one, otherwise the account and machine at the shell.
|
||||||
|
func Caller() string {
|
||||||
|
if c := strings.TrimSpace(os.Getenv(CallerVar)); c != "" {
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
user := os.Getenv("USER")
|
||||||
|
if user == "" {
|
||||||
|
user = "an unnamed account"
|
||||||
|
}
|
||||||
|
host, _ := os.Hostname()
|
||||||
|
return fmt.Sprintf("%s at a shell on %s", user, host)
|
||||||
|
}
|
||||||
|
|
||||||
|
type callerKey struct{}
|
||||||
|
|
||||||
|
// CallerIn is the caller of the seat call ctx belongs to, empty outside one.
|
||||||
|
func CallerIn(ctx context.Context) string {
|
||||||
|
c, _ := ctx.Value(callerKey{}).(string)
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
var handActSeq atomic.Uint64
|
||||||
|
|
||||||
|
// RecordHandAct writes one entry. Its key is its time and a sequence, so the bucket lists in order.
|
||||||
|
func RecordHandAct(ctx context.Context, conn *nats.Conn, act HandAct) (HandAct, error) {
|
||||||
|
if strings.TrimSpace(act.Why) == "" {
|
||||||
|
return act, errors.New("an act by hand says why: --why <text>")
|
||||||
|
}
|
||||||
|
if act.At.IsZero() {
|
||||||
|
act.At = time.Now().UTC()
|
||||||
|
}
|
||||||
|
if act.ID == "" {
|
||||||
|
act.ID = "act-" + strconv.FormatInt(act.At.UnixNano(), 10) + "-" + strconv.FormatUint(handActSeq.Add(1), 10)
|
||||||
|
}
|
||||||
|
if act.By == "" {
|
||||||
|
act.By = Caller()
|
||||||
|
}
|
||||||
|
if act.Cause == "" {
|
||||||
|
act.Cause = act.Verb
|
||||||
|
}
|
||||||
|
kv, err := handActs(ctx, conn)
|
||||||
|
if err != nil {
|
||||||
|
return act, err
|
||||||
|
}
|
||||||
|
body, err := json.Marshal(act)
|
||||||
|
if err != nil {
|
||||||
|
return act, err
|
||||||
|
}
|
||||||
|
_, err = kv.Put(ctx, act.ID, body)
|
||||||
|
return act, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// HandActs is every entry since a moment, oldest first.
|
||||||
|
func HandActs(ctx context.Context, conn *nats.Conn, since time.Time) ([]HandAct, error) {
|
||||||
|
kv, err := handActs(ctx, conn)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
w, err := kv.WatchAll(ctx, jetstream.IgnoreDeletes())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer func() { _ = w.Stop() }()
|
||||||
|
var out []HandAct
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, fmt.Errorf("reading the hand-act log: %w", ctx.Err())
|
||||||
|
case entry := <-w.Updates():
|
||||||
|
if entry == nil {
|
||||||
|
sort.SliceStable(out, func(i, j int) bool { return out[i].At.Before(out[j].At) })
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
var a HandAct
|
||||||
|
if json.Unmarshal(entry.Value(), &a) == nil && !a.At.Before(since) {
|
||||||
|
out = append(out, a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// RepeatedCauses are the causes recorded more than once within the fortnight before now, with how
|
||||||
|
// often: each is a repair done by hand again, which is what S15 will raise as a healer wanted.
|
||||||
|
func RepeatedCauses(acts []HandAct, now time.Time) map[string]int {
|
||||||
|
counts := map[string]int{}
|
||||||
|
for _, a := range acts {
|
||||||
|
if now.Sub(a.At) <= 14*24*time.Hour {
|
||||||
|
counts[a.Cause]++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for c, n := range counts {
|
||||||
|
if n < 2 {
|
||||||
|
delete(counts, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return counts
|
||||||
|
}
|
||||||
|
|
||||||
|
func handActs(ctx context.Context, conn *nats.Conn) (jetstream.KeyValue, error) {
|
||||||
|
api, err := jetstream.New(conn)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
kv, err := api.KeyValue(ctx, broker.HandActsBucket)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the hand-act log %s is not on the bus — the controller asserts it at its "+
|
||||||
|
"start, so one older than this has not: %w", broker.HandActsBucket, err)
|
||||||
|
}
|
||||||
|
return kv, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The hand-act log against a real server (novox/hq to-be 45 §7): an act is written with who, why and
|
||||||
|
// its cause, read back in order, and a cause recorded twice within a fortnight is found.
|
||||||
|
func TestNatsAnActByHandIsKeptWithWhyAndARepeatIsFound(t *testing.T) {
|
||||||
|
js := aBus(t)
|
||||||
|
api, err := jetstream.New(js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = api.DeleteKeyValue(t.Context(), broker.HandActsBucket)
|
||||||
|
if err := js.EnsureControllerBuckets(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Setenv(CallerVar, "node-tools.g14, through the mesh-controller seat")
|
||||||
|
if _, err := RecordHandAct(t.Context(), js.Conn(), HandAct{Verb: "push", Args: []string{"anchor"}}); err == nil {
|
||||||
|
t.Fatal("an act without why was written")
|
||||||
|
}
|
||||||
|
first, err := RecordHandAct(t.Context(), js.Conn(), HandAct{Verb: "push", Args: []string{"anchor"},
|
||||||
|
Why: "it never reported the send", Cause: "sent-not-reported"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if first.By != "node-tools.g14, through the mesh-controller seat" || first.ID == "" {
|
||||||
|
t.Fatalf("written as %+v", first)
|
||||||
|
}
|
||||||
|
if _, err := RecordHandAct(t.Context(), js.Conn(), HandAct{Verb: "plans close", Args: []string{"plan-1"},
|
||||||
|
Why: "waiting on the same report"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := RecordHandAct(t.Context(), js.Conn(), HandAct{Verb: "push", Args: []string{"ace"},
|
||||||
|
Why: "again", Cause: "sent-not-reported", At: time.Now().UTC().Add(time.Second)}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
acts, err := HandActs(t.Context(), js.Conn(), time.Now().Add(-time.Hour))
|
||||||
|
if err != nil || len(acts) != 3 || acts[0].ID != first.ID || acts[1].Cause != "plans close" {
|
||||||
|
t.Fatalf("%v %+v", err, acts)
|
||||||
|
}
|
||||||
|
repeated := RepeatedCauses(acts, time.Now())
|
||||||
|
if len(repeated) != 1 || repeated["sent-not-reported"] != 2 {
|
||||||
|
t.Fatalf("repeated %v", repeated)
|
||||||
|
}
|
||||||
|
if old, _ := HandActs(t.Context(), js.Conn(), time.Now().Add(time.Hour)); len(old) != 0 {
|
||||||
|
t.Fatalf("acts before the moment asked were listed: %+v", old)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(acts[2].ID, "act-") {
|
||||||
|
t.Fatalf("%q", acts[2].ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -138,6 +138,20 @@ type Signed struct {
|
|||||||
// is current.
|
// is current.
|
||||||
type Alive struct {
|
type Alive struct {
|
||||||
Node string `json:"node"`
|
Node string `json:"node"`
|
||||||
|
// IntervalSeconds is how often the node says it is there (novox/hq to-be 45 §3, S1): the
|
||||||
|
// watchdog's bound is three of them. Zero from a host older than that, which is read as the
|
||||||
|
// interval hosts have always used.
|
||||||
|
IntervalSeconds int `json:"interval_seconds,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ToolsAlive is a machine's node tools saying they are there (novox/hq to-be 45 §3, S11): the
|
||||||
|
// runtime every module's tools and every held seat's verbs are served by. Its own word, apart from the
|
||||||
|
// host's, because a host heard and a runtime gone is a machine nobody can ask anything.
|
||||||
|
type ToolsAlive struct {
|
||||||
|
Node string `json:"node"`
|
||||||
|
IntervalSeconds int `json:"interval_seconds,omitempty"`
|
||||||
|
// Version is the runtime's build, as it says it.
|
||||||
|
Version string `json:"version,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Report is what a node states after applying. It states; the owning context writes.
|
// Report is what a node states after applying. It states; the owning context writes.
|
||||||
|
|||||||
+24
-1
@@ -413,7 +413,30 @@ func AskSeatTool(ctx context.Context, conn *nats.Conn, seat, verb, node string,
|
|||||||
}
|
}
|
||||||
asking, cancel := context.WithTimeout(ctx, timeout)
|
asking, cancel := context.WithTimeout(ctx, timeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
reply, err := conn.RequestWithContext(asking, NodeSeatToolSubject(seat, verb, node), body)
|
subject := NodeSeatToolSubject(seat, verb, node)
|
||||||
|
// **A refused question is known at once** (novox/hq to-be 45, D8 live on 2026-10-06): the bus
|
||||||
|
// says it to the connection and the request would otherwise wait out its whole timeout, reading
|
||||||
|
// as a machine that did not answer.
|
||||||
|
refused, stop := refusalsOf(conn, subject)
|
||||||
|
defer stop()
|
||||||
|
type replied struct {
|
||||||
|
msg *nats.Msg
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
done := make(chan replied, 1)
|
||||||
|
go func() {
|
||||||
|
msg, err := conn.RequestWithContext(asking, subject, body)
|
||||||
|
done <- replied{msg, err}
|
||||||
|
}()
|
||||||
|
var reply *nats.Msg
|
||||||
|
select {
|
||||||
|
case r := <-done:
|
||||||
|
reply, err = r.msg, r.err
|
||||||
|
case why := <-refused:
|
||||||
|
cancel()
|
||||||
|
return Answer{}, fmt.Errorf("the bus refused the controller asking %s.%s of %s — its grants do not "+
|
||||||
|
"name %s: %v", seat, verb, node, subject, why)
|
||||||
|
}
|
||||||
switch {
|
switch {
|
||||||
case errors.Is(err, nats.ErrNoResponders):
|
case errors.Is(err, nats.ErrNoResponders):
|
||||||
return Answer{}, fmt.Errorf("nothing on %s answers %s.%s: its holder is not running, or is "+
|
return Answer{}, fmt.Errorf("nothing on %s answers %s.%s: its holder is not running, or is "+
|
||||||
|
|||||||
@@ -25,11 +25,13 @@ import (
|
|||||||
// on the wire: the wire is the transport's business, and a kind that travelled would be a third
|
// on the wire: the wire is the transport's business, and a kind that travelled would be a third
|
||||||
// name for the same thing.
|
// name for the same thing.
|
||||||
const (
|
const (
|
||||||
KindEnrolment = "enrolment"
|
KindEnrolment = "enrolment"
|
||||||
KindReport = "report"
|
KindReport = "report"
|
||||||
KindHeartbeat = "heartbeat"
|
KindHeartbeat = "heartbeat"
|
||||||
KindBuilt = "built"
|
// KindToolsHeartbeat is a machine's node tools saying they are there (novox/hq to-be 45 S11).
|
||||||
KindModuleMoved = "module-moved"
|
KindToolsHeartbeat = "tools-heartbeat"
|
||||||
|
KindBuilt = "built"
|
||||||
|
KindModuleMoved = "module-moved"
|
||||||
// KindSourceMoved is the forge announcing a merge: a source moved, and what it produces is
|
// KindSourceMoved is the forge announcing a merge: a source moved, and what it produces is
|
||||||
// built without anybody telling the mesh (novox/hq 04-ISSUES/131).
|
// built without anybody telling the mesh (novox/hq 04-ISSUES/131).
|
||||||
KindSourceMoved = "source-moved"
|
KindSourceMoved = "source-moved"
|
||||||
|
|||||||
@@ -89,6 +89,10 @@ func (n *natsInbound) Receive(ctx context.Context, act func(context.Context, Con
|
|||||||
return nil // stopped while standing by
|
return nil // stopped while standing by
|
||||||
}
|
}
|
||||||
defer func() { _ = said.Unsubscribe() }()
|
defer func() { _ = said.Unsubscribe() }()
|
||||||
|
// This controller is the one acting now: its watchdogs and self-check may say what they see
|
||||||
|
// (novox/hq to-be 45 §3). One standing by hears nothing, and would call every machine silent.
|
||||||
|
holding.Store(true)
|
||||||
|
defer holding.Store(false)
|
||||||
|
|
||||||
// Heartbeats, on core NATS and off any stream (design 25 §3). Their own subscription because
|
// Heartbeats, on core NATS and off any stream (design 25 §3). Their own subscription because
|
||||||
// they are their own guarantee: a lost one is the next one.
|
// they are their own guarantee: a lost one is the next one.
|
||||||
@@ -98,6 +102,12 @@ func (n *natsInbound) Receive(ctx context.Context, act func(context.Context, Con
|
|||||||
return fmt.Errorf("subscribing to heartbeats: %w", err)
|
return fmt.Errorf("subscribing to heartbeats: %w", err)
|
||||||
}
|
}
|
||||||
defer func() { _ = alive.Unsubscribe() }()
|
defer func() { _ = alive.Unsubscribe() }()
|
||||||
|
// And each machine's node tools, on the same channel: as cheap, and lost the same way.
|
||||||
|
toolsAlive, err := conn.ChanSubscribe(ToolsAliveSubjects, beats)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("subscribing to the node tools' heartbeats: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = toolsAlive.Unsubscribe() }()
|
||||||
|
|
||||||
// The events the controller follows, when something is listening for them.
|
// The events the controller follows, when something is listening for them.
|
||||||
var events chan *nats.Msg
|
var events chan *nats.Msg
|
||||||
@@ -159,6 +169,8 @@ func (n *natsInbound) deliver(ctx context.Context, act func(context.Context, Con
|
|||||||
}
|
}
|
||||||
m := &natsControl{kind: kind, msg: msg, on: n}
|
m := &natsControl{kind: kind, msg: msg, on: n}
|
||||||
if streamed {
|
if streamed {
|
||||||
|
// The event loop took one: what S4 watches (novox/hq to-be 45 §3).
|
||||||
|
Loop.Took(time.Now())
|
||||||
// A message with no metadata is not from a stream, whatever it was delivered on, and the
|
// A message with no metadata is not from a stream, whatever it was delivered on, and the
|
||||||
// window has nothing to hold it by. Said by leaving the sequence at zero.
|
// window has nothing to hold it by. Said by leaving the sequence at zero.
|
||||||
if meta, err := msg.Metadata(); err == nil {
|
if meta, err := msg.Metadata(); err == nil {
|
||||||
@@ -225,6 +237,8 @@ func kindOfSubject(subject string) (string, bool) {
|
|||||||
return KindReport, true
|
return KindReport, true
|
||||||
case "alive":
|
case "alive":
|
||||||
return KindHeartbeat, true
|
return KindHeartbeat, true
|
||||||
|
case "tools-alive":
|
||||||
|
return KindToolsHeartbeat, true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
switch subject {
|
switch subject {
|
||||||
|
|||||||
@@ -170,6 +170,8 @@ func (s *Server) act(ctx context.Context, m Control) {
|
|||||||
s.reported(ctx, m)
|
s.reported(ctx, m)
|
||||||
case KindHeartbeat:
|
case KindHeartbeat:
|
||||||
s.heartbeat(m)
|
s.heartbeat(m)
|
||||||
|
case KindToolsHeartbeat:
|
||||||
|
s.toolsHeartbeat(m)
|
||||||
case KindBuilt:
|
case KindBuilt:
|
||||||
s.wasBuilt(ctx, m)
|
s.wasBuilt(ctx, m)
|
||||||
case KindModuleMoved:
|
case KindModuleMoved:
|
||||||
@@ -268,6 +270,8 @@ func (s *Server) heartbeat(m Control) {
|
|||||||
_ = m.Drop()
|
_ = m.Drop()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
// The interval it says, for the watchdog's bound (novox/hq to-be 45 S1).
|
||||||
|
HostBeats.Heard(alive.Node, time.Now(), time.Duration(alive.IntervalSeconds)*time.Second, "")
|
||||||
if s.listener != nil {
|
if s.listener != nil {
|
||||||
if _, err := s.listener.Heard(context.Background(), Report{Node: alive.Node}); err != nil {
|
if _, err := s.listener.Heard(context.Background(), Report{Node: alive.Node}); err != nil {
|
||||||
s.log.Printf("could not record that %s is here: %v", alive.Node, err)
|
s.log.Printf("could not record that %s is here: %v", alive.Node, err)
|
||||||
@@ -276,6 +280,22 @@ func (s *Server) heartbeat(m Control) {
|
|||||||
_ = m.Took()
|
_ = m.Took()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// toolsHeartbeat records that a machine's node tools were heard from (novox/hq to-be 45 S11), and
|
||||||
|
// nothing else: in this process's memory, for the watchdog — the next one is a minute away.
|
||||||
|
func (s *Server) toolsHeartbeat(m Control) {
|
||||||
|
var alive ToolsAlive
|
||||||
|
if err := json.Unmarshal(m.Body(), &alive); err != nil || alive.Node == "" {
|
||||||
|
_ = m.Drop()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// The machine is the one in the subject the bus let the runtime publish on, never the body's.
|
||||||
|
if node, ok := nodeOfToolsAlive(m.Subject()); ok {
|
||||||
|
alive.Node = node
|
||||||
|
}
|
||||||
|
ToolsBeats.Heard(alive.Node, time.Now(), time.Duration(alive.IntervalSeconds)*time.Second, alive.Version)
|
||||||
|
_ = m.Took()
|
||||||
|
}
|
||||||
|
|
||||||
// reported records what a node says it did.
|
// reported records what a node says it did.
|
||||||
//
|
//
|
||||||
// A node states; nothing here writes anything the node claimed about itself beyond that it was
|
// A node states; nothing here writes anything the node claimed about itself beyond that it was
|
||||||
|
|||||||
@@ -0,0 +1,227 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What the serving controller hears that its watchdogs read (novox/hq to-be 45 §3).
|
||||||
|
//
|
||||||
|
// **In memory, on purpose.** A heartbeat is the least valuable message the mesh sends — the next one
|
||||||
|
// is a minute away — and what a watchdog needs of it is the newest moment and the interval it said.
|
||||||
|
// A machine's last word is kept in the store as well (`last_seen`), which is what S1 reads; these are
|
||||||
|
// what the store does not keep: the interval, the node tools' word, and when the event loop last took
|
||||||
|
// a message.
|
||||||
|
|
||||||
|
// Beat is one emitter's newest heartbeat.
|
||||||
|
type Beat struct {
|
||||||
|
At time.Time
|
||||||
|
// Every is the interval it said; zero when it said none.
|
||||||
|
Every time.Duration
|
||||||
|
Version string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Beats is the newest heartbeat of each machine, for one kind of emitter.
|
||||||
|
type Beats struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
started time.Time
|
||||||
|
heard map[string]Beat
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewBeats is an empty record, started now.
|
||||||
|
func NewBeats() *Beats { return &Beats{started: time.Now(), heard: map[string]Beat{}} }
|
||||||
|
|
||||||
|
// HostBeats are the node-engines' heartbeats (S1); ToolsBeats the node tools' (S11).
|
||||||
|
var (
|
||||||
|
HostBeats = NewBeats()
|
||||||
|
ToolsBeats = NewBeats()
|
||||||
|
)
|
||||||
|
|
||||||
|
// Heard records one heartbeat.
|
||||||
|
func (b *Beats) Heard(node string, at time.Time, every time.Duration, version string) {
|
||||||
|
b.mu.Lock()
|
||||||
|
defer b.mu.Unlock()
|
||||||
|
b.heard[node] = Beat{At: at, Every: every, Version: version}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Of is one machine's newest heartbeat; false when none was heard since this process started.
|
||||||
|
func (b *Beats) Of(node string) (Beat, bool) {
|
||||||
|
b.mu.Lock()
|
||||||
|
defer b.mu.Unlock()
|
||||||
|
beat, ok := b.heard[node]
|
||||||
|
return beat, ok
|
||||||
|
}
|
||||||
|
|
||||||
|
// Started is when this record began: a machine not heard since is silent since then at the most.
|
||||||
|
func (b *Beats) Started() time.Time {
|
||||||
|
b.mu.Lock()
|
||||||
|
defer b.mu.Unlock()
|
||||||
|
return b.started
|
||||||
|
}
|
||||||
|
|
||||||
|
// nodeOfToolsAlive is the machine a node tools' heartbeat names in its subject.
|
||||||
|
func nodeOfToolsAlive(subject string) (string, bool) {
|
||||||
|
rest, ok := strings.CutPrefix(subject, "mesh.control.")
|
||||||
|
if !ok {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
node, kind, ok := strings.Cut(rest, ".")
|
||||||
|
if !ok || kind != "tools-alive" || node == "" {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
return node, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// LoopActivity is when the controller's event loop last took a message from a stream (S4).
|
||||||
|
type LoopActivity struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
took time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// Loop is this process's event loop.
|
||||||
|
var Loop = &LoopActivity{}
|
||||||
|
|
||||||
|
// Took records that the loop was handed a message.
|
||||||
|
func (l *LoopActivity) Took(at time.Time) {
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
l.took = at
|
||||||
|
}
|
||||||
|
|
||||||
|
// Last is when the loop last took one; zero when it has not since this process started.
|
||||||
|
func (l *LoopActivity) Last() time.Time {
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
return l.took
|
||||||
|
}
|
||||||
|
|
||||||
|
// PowerState is what a machine last said about its power (novox/hq ADR 0211): `sleeping` or
|
||||||
|
// `shutting-down` until it says `booted` or `woke`.
|
||||||
|
type PowerState struct {
|
||||||
|
State string
|
||||||
|
At time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// PowerModule is the module whose events say a machine's power (ADR 0211), and PowerEvents its
|
||||||
|
// events that say whether the machine is about to be away or is back.
|
||||||
|
const PowerModule = "power"
|
||||||
|
|
||||||
|
var PowerEvents = map[string]bool{"sleeping": true, "shutting-down": true, "booted": true, "woke": true}
|
||||||
|
|
||||||
|
// PowerStates is each machine's newest word about its power since a moment, read back from the
|
||||||
|
// events stream on a consumer of its own that acknowledges nothing (as AnnouncedMerges reads). The
|
||||||
|
// machine is the one the runtime stamped on the event (`x-node`); an event without one names none.
|
||||||
|
func (s *Server) PowerStates(ctx context.Context, since time.Time) (map[string]PowerState, error) {
|
||||||
|
if s.js == nil {
|
||||||
|
return nil, errors.New("this control plane is not on the bus, so it cannot read what machines said of their power")
|
||||||
|
}
|
||||||
|
sub, err := s.js.Context().SubscribeSync(EventSubject(PowerModule, ">"), nats.OrderedConsumer(), nats.StartTime(since))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("reading what machines said of their power: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = sub.Unsubscribe() }()
|
||||||
|
out := map[string]PowerState{}
|
||||||
|
for {
|
||||||
|
wait, cancel := context.WithTimeout(ctx, readQuiet)
|
||||||
|
msg, err := sub.NextMsgWithContext(wait)
|
||||||
|
cancel()
|
||||||
|
if err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
meta, err := msg.Metadata()
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
state := strings.TrimPrefix(msg.Subject, "mesh.mod."+PowerModule+".event.")
|
||||||
|
node := msg.Header.Get("x-node")
|
||||||
|
if PowerEvents[state] && node != "" {
|
||||||
|
at := meta.Timestamp
|
||||||
|
var body struct {
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
}
|
||||||
|
if json.Unmarshal(msg.Data, &body) == nil && !body.At.IsZero() {
|
||||||
|
at = body.At
|
||||||
|
}
|
||||||
|
if before, ok := out[node]; !ok || !at.Before(before.At) {
|
||||||
|
out[node] = PowerState{State: state, At: at}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if meta.NumPending == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Away says whether a power state is a machine that said it would be away.
|
||||||
|
func (p PowerState) Away() bool { return p.State == "sleeping" || p.State == "shutting-down" }
|
||||||
|
|
||||||
|
// StaleRefusal is the node-engine's words for a declaration it refused because it is older than the
|
||||||
|
// one it holds (mesh-host `refuseOlder`, novox/hq issue 107): the receiver's refusal S13 counts.
|
||||||
|
const StaleRefusal = "is older than what the mesh last said to this node"
|
||||||
|
|
||||||
|
// IsStaleRefusal says a report's refusal is the node-engine refusing a declaration older than it holds.
|
||||||
|
func IsStaleRefusal(refused string) bool { return strings.Contains(refused, StaleRefusal) }
|
||||||
|
|
||||||
|
// RefusalCount keeps when each machine refused a stale declaration, for S13 (novox/hq to-be 45 §3):
|
||||||
|
// more than five from one writer in five minutes is a writer sending what it has moved past.
|
||||||
|
type RefusalCount struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
per map[string][]time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// StaleRefusals is this process's count.
|
||||||
|
var StaleRefusals = &RefusalCount{per: map[string][]time.Time{}}
|
||||||
|
|
||||||
|
// Refused records one refusal by a machine.
|
||||||
|
func (r *RefusalCount) Refused(node string, at time.Time) {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
r.per[node] = append(r.per[node], at)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Within is how many refusals each machine made since a moment; older ones are forgotten.
|
||||||
|
func (r *RefusalCount) Within(since time.Time) map[string]int {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
out := map[string]int{}
|
||||||
|
for node, times := range r.per {
|
||||||
|
var kept []time.Time
|
||||||
|
for _, t := range times {
|
||||||
|
if !t.Before(since) {
|
||||||
|
kept = append(kept, t)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(kept) == 0 {
|
||||||
|
delete(r.per, node)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
r.per[node] = kept
|
||||||
|
out[node] = len(kept)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// holding is whether this process holds the controller's consumer of what nodes say: the controller
|
||||||
|
// acting, not one standing by for another (issue 213). Until the lease (to-be 45 §6) it is how a
|
||||||
|
// watchdog knows it is the one that hears.
|
||||||
|
var holding atomic.Bool
|
||||||
|
|
||||||
|
// Holding says this process is the controller acting now.
|
||||||
|
func Holding() bool { return holding.Load() }
|
||||||
|
|
||||||
|
// JetStream is the connection the server consumes on.
|
||||||
|
func (s *Server) JetStream() *broker.JetStream { return s.js }
|
||||||
@@ -61,12 +61,15 @@ const TheNetwork = "the-private-network"
|
|||||||
// network. A requirement nothing provides is refused at resolution, so leaving the names here
|
// network. A requirement nothing provides is refused at resolution, so leaving the names here
|
||||||
// would only have documented a mechanism that does not exist.
|
// would only have documented a mechanism that does not exist.
|
||||||
|
|
||||||
// **There is no bundle any more** (novox/hq ADR 0226). A `networking` module requiring this one and
|
// Domain is the module for people who want a network and do not want to choose one.
|
||||||
// nothing else used to be what a machine was assigned, so that "get the network working" was one
|
//
|
||||||
// word and a second VPN could be chosen by assigning it instead. The mesh has one private network,
|
// It has no files of its own — it is requirements and nothing else. Assigning it finds one
|
||||||
// every machine is on it, and the bundle was a second name for this module that every machine
|
// answer to each and takes them silently, so getting a mesh onto a private network is one word.
|
||||||
// carried. A machine is assigned Name directly; another VPN is still chosen by assigning it in its
|
// The day the catalogue holds a second VPN there are two answers, the resolver refuses and names
|
||||||
// place, which is all the bundle ever did.
|
// both, and choosing is assigning the one you want. **That is the whole mechanism**: picking an
|
||||||
|
// implementation is assigning a module, and there is no flavor field, no configuration language,
|
||||||
|
// and nothing to learn.
|
||||||
|
const Domain = "networking"
|
||||||
|
|
||||||
// Generator answers what one node's network configuration is.
|
// Generator answers what one node's network configuration is.
|
||||||
type Generator struct {
|
type Generator struct {
|
||||||
@@ -144,6 +147,19 @@ func Manifest() map[string]any {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DomainManifest is the module that means "get the network working".
|
||||||
|
func DomainManifest() map[string]any {
|
||||||
|
return map[string]any{
|
||||||
|
"module": Domain,
|
||||||
|
"version": "1",
|
||||||
|
// **Only the network now.** It used to require name-resolution as well, answered by a
|
||||||
|
// module that wrote a hosts file and ran nothing. Names are not a provision — they are a
|
||||||
|
// fact the mesh computes, and whatever puts a machine on the private network writes them,
|
||||||
|
// because a mesh name IS an address on that network.
|
||||||
|
"requires": []string{Requirement},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Empty is a network nobody is on.
|
// Empty is a network nobody is on.
|
||||||
//
|
//
|
||||||
// A mesh where no machine was given the module. Legitimate rather than broken — every node still
|
// A mesh where no machine was given the module. Legitimate rather than broken — every node still
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ import (
|
|||||||
// has no module.json for a test reading the catalogue to find. Parsed with the real parser, so a
|
// has no module.json for a test reading the catalogue to find. Parsed with the real parser, so a
|
||||||
// set that forgot this seat refuses the control plane's own module here rather than on a machine.
|
// set that forgot this seat refuses the control plane's own module here rather than on a machine.
|
||||||
func TestThePrivateNetworksClaimIsASeatTheMeshDefines(t *testing.T) {
|
func TestThePrivateNetworksClaimIsASeatTheMeshDefines(t *testing.T) {
|
||||||
for _, composed := range []map[string]any{Manifest()} {
|
for _, composed := range []map[string]any{Manifest(), DomainManifest()} {
|
||||||
raw, err := json.Marshal(composed)
|
raw, err := json.Marshal(composed)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
+9
@@ -0,0 +1,9 @@
|
|||||||
|
language: go
|
||||||
|
|
||||||
|
go:
|
||||||
|
- 1.x
|
||||||
|
- tip
|
||||||
|
|
||||||
|
matrix:
|
||||||
|
allow_failures:
|
||||||
|
- go: tip
|
||||||
+22
@@ -0,0 +1,22 @@
|
|||||||
|
Copyright (c) 2019 Jack Christensen
|
||||||
|
|
||||||
|
MIT License
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining
|
||||||
|
a copy of this software and associated documentation files (the
|
||||||
|
"Software"), to deal in the Software without restriction, including
|
||||||
|
without limitation the rights to use, copy, modify, merge, publish,
|
||||||
|
distribute, sublicense, and/or sell copies of the Software, and to
|
||||||
|
permit persons to whom the Software is furnished to do so, subject to
|
||||||
|
the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be
|
||||||
|
included in all copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||||
|
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||||
|
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||||
|
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
||||||
|
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
||||||
|
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||||
|
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||||
+8
@@ -0,0 +1,8 @@
|
|||||||
|
[](https://godoc.org/github.com/jackc/pgpassfile)
|
||||||
|
[](https://travis-ci.org/jackc/pgpassfile)
|
||||||
|
|
||||||
|
# pgpassfile
|
||||||
|
|
||||||
|
Package pgpassfile is a parser PostgreSQL .pgpass files.
|
||||||
|
|
||||||
|
Extracted and rewritten from original implementation in https://github.com/jackc/pgx.
|
||||||
+110
@@ -0,0 +1,110 @@
|
|||||||
|
// Package pgpassfile is a parser PostgreSQL .pgpass files.
|
||||||
|
package pgpassfile
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Entry represents a line in a PG passfile.
|
||||||
|
type Entry struct {
|
||||||
|
Hostname string
|
||||||
|
Port string
|
||||||
|
Database string
|
||||||
|
Username string
|
||||||
|
Password string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Passfile is the in memory data structure representing a PG passfile.
|
||||||
|
type Passfile struct {
|
||||||
|
Entries []*Entry
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadPassfile reads the file at path and parses it into a Passfile.
|
||||||
|
func ReadPassfile(path string) (*Passfile, error) {
|
||||||
|
f, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
|
||||||
|
return ParsePassfile(f)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParsePassfile reads r and parses it into a Passfile.
|
||||||
|
func ParsePassfile(r io.Reader) (*Passfile, error) {
|
||||||
|
passfile := &Passfile{}
|
||||||
|
|
||||||
|
scanner := bufio.NewScanner(r)
|
||||||
|
for scanner.Scan() {
|
||||||
|
entry := parseLine(scanner.Text())
|
||||||
|
if entry != nil {
|
||||||
|
passfile.Entries = append(passfile.Entries, entry)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return passfile, scanner.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Match (not colons or escaped colon or escaped backslash)+. Essentially gives a split on unescaped
|
||||||
|
// colon.
|
||||||
|
var colonSplitterRegexp = regexp.MustCompile("(([^:]|(\\:)))+")
|
||||||
|
|
||||||
|
// var colonSplitterRegexp = regexp.MustCompile("((?:[^:]|(?:\\:)|(?:\\\\))+)")
|
||||||
|
|
||||||
|
// parseLine parses a line into an *Entry. It returns nil on comment lines or any other unparsable
|
||||||
|
// line.
|
||||||
|
func parseLine(line string) *Entry {
|
||||||
|
const (
|
||||||
|
tmpBackslash = "\r"
|
||||||
|
tmpColon = "\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
|
||||||
|
if strings.HasPrefix(line, "#") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
line = strings.Replace(line, `\\`, tmpBackslash, -1)
|
||||||
|
line = strings.Replace(line, `\:`, tmpColon, -1)
|
||||||
|
|
||||||
|
parts := strings.Split(line, ":")
|
||||||
|
if len(parts) != 5 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unescape escaped colons and backslashes
|
||||||
|
for i := range parts {
|
||||||
|
parts[i] = strings.Replace(parts[i], tmpBackslash, `\`, -1)
|
||||||
|
parts[i] = strings.Replace(parts[i], tmpColon, `:`, -1)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &Entry{
|
||||||
|
Hostname: parts[0],
|
||||||
|
Port: parts[1],
|
||||||
|
Database: parts[2],
|
||||||
|
Username: parts[3],
|
||||||
|
Password: parts[4],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// FindPassword finds the password for the provided hostname, port, database, and username. For a
|
||||||
|
// Unix domain socket hostname must be set to "localhost". An empty string will be returned if no
|
||||||
|
// match is found.
|
||||||
|
//
|
||||||
|
// See https://www.postgresql.org/docs/current/libpq-pgpass.html for more password file information.
|
||||||
|
func (pf *Passfile) FindPassword(hostname, port, database, username string) (password string) {
|
||||||
|
for _, e := range pf.Entries {
|
||||||
|
if (e.Hostname == "*" || e.Hostname == hostname) &&
|
||||||
|
(e.Port == "*" || e.Port == port) &&
|
||||||
|
(e.Database == "*" || e.Database == database) &&
|
||||||
|
(e.Username == "*" || e.Username == username) {
|
||||||
|
return e.Password
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
+22
@@ -0,0 +1,22 @@
|
|||||||
|
Copyright (c) 2020 Jack Christensen
|
||||||
|
|
||||||
|
MIT License
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining
|
||||||
|
a copy of this software and associated documentation files (the
|
||||||
|
"Software"), to deal in the Software without restriction, including
|
||||||
|
without limitation the rights to use, copy, modify, merge, publish,
|
||||||
|
distribute, sublicense, and/or sell copies of the Software, and to
|
||||||
|
permit persons to whom the Software is furnished to do so, subject to
|
||||||
|
the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be
|
||||||
|
included in all copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||||
|
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||||
|
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||||
|
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
||||||
|
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
||||||
|
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||||
|
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user