Compare commits
23
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d059311c0f | ||
|
|
1a28cb3357 | ||
|
|
758537dd4e | ||
|
|
add807f034 | ||
|
|
8ca4b04321 | ||
|
|
8170fc58a3 | ||
|
|
efcdd5dd7d | ||
|
|
5d7d8ee2d6 | ||
|
|
5b7e6ff453 | ||
|
|
cc7fb99f29 | ||
|
|
1e04670052 | ||
|
|
81e5458cbf | ||
|
|
fb74e24c9e | ||
|
|
ca09a07fdf | ||
|
|
9b028b4212 | ||
|
|
d7fab82a89 | ||
|
|
7f65e62743 | ||
|
|
2b01f8786e | ||
|
|
909062e729 | ||
|
|
826dcb91b1 | ||
|
|
193168e086 | ||
|
|
59fdffb979 | ||
|
|
5d47e0bfd6 |
@@ -27,6 +27,8 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
@@ -175,7 +177,9 @@ func dialFor(credential Credential) (*broker.JetStream, string, error) {
|
||||
if !credential.onTheNewBus() {
|
||||
return nil, "", fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL)
|
||||
}
|
||||
js, err := broker.DialPinned(credential.natsURL(), credential.Fingerprint)
|
||||
// Named for what it is, not the controller whose code dials it (novox/hq issue 327).
|
||||
host, _ := os.Hostname()
|
||||
js, err := broker.DialPinned(credential.natsURL(), credential.Fingerprint, nats.Name("build agent on "+host))
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
@@ -187,7 +188,8 @@ func (a *actor) serveUnderTheLease(ctx context.Context, inv *inventory.Inventory
|
||||
a.mu.Lock()
|
||||
a.serving = true
|
||||
a.mu.Unlock()
|
||||
js, err := broker.Dial(address)
|
||||
// Its own connection, held as long as the lease, and named so (novox/hq issue 327).
|
||||
js, err := broker.Dial(address, nats.Name(broker.ConnectionName+" lease"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the mesh is on the bus at %s and this control plane cannot reach it to take the "+
|
||||
"lease: %w", broker.BareAddress(address), err)
|
||||
|
||||
@@ -879,6 +879,10 @@ func heldBy(ctx context.Context) map[string]string {
|
||||
// the mesh runs on today this needs the controller's own connection, so it is handed one; on the bus
|
||||
// being built it dials, because a build request is a one-shot and holds nothing else.
|
||||
func askOverOn(seat string) (link.Builders, error) {
|
||||
// The serving controller asks on its own connection (novox/hq issue 327).
|
||||
if serving := servingBus.Load(); serving != nil {
|
||||
return link.BuildsOn(serving, seat), nil
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -937,11 +941,7 @@ func buildSeatAmong(entries []inventory.Entry) string {
|
||||
// with a consumer of its own that is gone when this returns, so nothing accumulates in the server
|
||||
// for the reading, and filtered by subject, so one build's lines are all that travel.
|
||||
func buildLog(ctx context.Context, id string) error {
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot reach the bus to read a build's log: %w", err)
|
||||
}
|
||||
|
||||
@@ -53,26 +53,9 @@ func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Ra
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// And the work queues of seats that name their caller or their kind, with each holder's worker
|
||||
// (novox/hq ADR 0259 §3): an ask queues until the router takes it, a channel's work until that kind
|
||||
// takes it.
|
||||
trafficStreams, trafficWorkers, err := seatTrafficObjects(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Every one tried, and every failure named: one module's consumer the bus refuses is no reason
|
||||
// the modules after it in the list hear nothing (novox/hq issue 208, where this runs on each send).
|
||||
var failed []error
|
||||
for _, s := range trafficStreams {
|
||||
if err := r.EnsureStream(s); err != nil {
|
||||
failed = append(failed, fmt.Errorf("the work queue %s: %w", s.Name, err))
|
||||
}
|
||||
}
|
||||
for _, c := range trafficWorkers {
|
||||
if err := r.EnsureConsumer(c); err != nil {
|
||||
failed = append(failed, fmt.Errorf("the worker %s on %s: %w", c.Name, c.Stream, err))
|
||||
}
|
||||
}
|
||||
for _, c := range consumers {
|
||||
if err := r.EnsureConsumer(c.Consumer); err != nil {
|
||||
failed = append(failed, fmt.Errorf("how %s on %s hears what it consumes: %w", c.Module, c.Node, err))
|
||||
@@ -147,37 +130,6 @@ func moduleConsumers(ctx context.Context, inv *inventory.Inventory) ([]broker.Mo
|
||||
return broker.ConsumersOf(users), nil
|
||||
}
|
||||
|
||||
// seatTrafficObjects is the work queues and workers of seats that name their caller or their kind, from
|
||||
// the records the user list is composed from.
|
||||
func seatTrafficObjects(ctx context.Context, inv *inventory.Inventory) ([]broker.Stream, []broker.Consumer, error) {
|
||||
records, err := inv.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
streams, workers := broker.SeatTrafficObjects(users)
|
||||
// And the queue of every such seat the catalogue declares, held or not: work queues from registration,
|
||||
// so what is submitted before a holder is assigned waits for it (the correctness review of 2026-10-08).
|
||||
declared, err := inv.DeclaredTrafficSeats(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
have := map[string]bool{}
|
||||
for _, s := range streams {
|
||||
have[s.Name] = true
|
||||
}
|
||||
for _, s := range broker.TrafficQueues(declared) {
|
||||
if !have[s.Name] {
|
||||
streams = append(streams, s)
|
||||
have[s.Name] = true
|
||||
}
|
||||
}
|
||||
return streams, workers, nil
|
||||
}
|
||||
|
||||
// moduleConsumerCount is how many modules hear what they consume, for the raise's one line.
|
||||
func moduleConsumerCount(ctx context.Context, inv *inventory.Inventory) (int, error) {
|
||||
consumers, err := moduleConsumers(ctx, inv)
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
@@ -107,19 +108,20 @@ func conditionsCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
switch sub {
|
||||
case "list":
|
||||
return listConditions(ctx, args)
|
||||
return listConditions(ctx, args, os.Stdout)
|
||||
case "show":
|
||||
return showCondition(ctx, args)
|
||||
return showCondition(ctx, args, os.Stdout)
|
||||
case "silence":
|
||||
return silenceCondition(ctx, args)
|
||||
case "history":
|
||||
return conditionHistory(ctx, args)
|
||||
return conditionHistory(ctx, args, os.Stdout)
|
||||
}
|
||||
return errors.New(conditionsUsage)
|
||||
}
|
||||
|
||||
func listConditions(ctx context.Context, args []string) error {
|
||||
func listConditions(ctx context.Context, args []string, w io.Writer) error {
|
||||
set := flag.NewFlagSet("conditions", flag.ContinueOnError)
|
||||
usageTo(set, w)
|
||||
scope := set.String("scope", "", "only this scope: "+strings.Join(conditions.Scopes, ", "))
|
||||
severity := set.String("severity", "", "only urgent, or only warning")
|
||||
machine := set.String("machine", "", "only those about this machine")
|
||||
@@ -144,20 +146,20 @@ func listConditions(ctx context.Context, args []string) error {
|
||||
}
|
||||
}
|
||||
if *asJSON {
|
||||
return printJSON(map[string]any{"conditions": inBrief(out), "open": len(open), "counted": counted(out),
|
||||
return printJSONTo(w, map[string]any{"conditions": inBrief(out), "open": len(open), "counted": counted(out),
|
||||
"note": "urgent first, then oldest first; a condition clears when observation says so, never by hand; " +
|
||||
"each with its newest evidence — `conditions key=<key>` gives one whole"})
|
||||
}
|
||||
if len(out) == 0 {
|
||||
if len(open) == 0 {
|
||||
fmt.Println("no open conditions")
|
||||
fmt.Fprintln(w, "no open conditions")
|
||||
} else {
|
||||
fmt.Printf("none of the %d open condition(s) is about that\n", len(open))
|
||||
fmt.Fprintf(w, "none of the %d open condition(s) is about that\n", len(open))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for _, line := range conditionLines(out, time.Now()) {
|
||||
fmt.Println(line)
|
||||
fmt.Fprintln(w, line)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -233,8 +235,9 @@ func conditionLines(list []conditions.Condition, now time.Time) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
func showCondition(ctx context.Context, args []string) error {
|
||||
func showCondition(ctx context.Context, args []string, w io.Writer) error {
|
||||
set := flag.NewFlagSet("conditions show", flag.ContinueOnError)
|
||||
usageTo(set, w)
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
rest, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
@@ -266,34 +269,34 @@ func showCondition(ctx context.Context, args []string) error {
|
||||
"history --key %s` what became of it", key, key)
|
||||
}
|
||||
if *asJSON {
|
||||
return printJSON(c)
|
||||
return printJSONTo(w, c)
|
||||
}
|
||||
now := time.Now()
|
||||
fmt.Printf("%s %s\n %s\n\n", strings.ToUpper(string(c.Severity)), c.Key, c.Summary)
|
||||
fmt.Printf(" kind %s\n about %s %s", c.Kind, c.Subject.Scope, c.Subject.ID)
|
||||
fmt.Fprintf(w, "%s %s\n %s\n\n", strings.ToUpper(string(c.Severity)), c.Key, c.Summary)
|
||||
fmt.Fprintf(w, " kind %s\n about %s %s", c.Kind, c.Subject.Scope, c.Subject.ID)
|
||||
if c.Subject.Machine != "" {
|
||||
fmt.Printf(", on %s", c.Subject.Machine)
|
||||
fmt.Fprintf(w, ", on %s", c.Subject.Machine)
|
||||
}
|
||||
fmt.Printf("\n raised by %s\n since %s (%s ago), observed %d time(s), last %s ago\n",
|
||||
fmt.Fprintf(w, "\n raised by %s\n since %s (%s ago), observed %d time(s), last %s ago\n",
|
||||
c.Source, c.Raised.Local().Format("2006-01-02 15:04:05"), roughly(now.Sub(c.Raised)), c.Observations,
|
||||
now.Sub(c.LastObserved).Round(time.Second))
|
||||
if c.Count > 1 {
|
||||
fmt.Printf(" raised %d times, each within ten minutes of clearing\n", c.Count)
|
||||
fmt.Fprintf(w, " raised %d times, each within ten minutes of clearing\n", c.Count)
|
||||
}
|
||||
fmt.Printf(" resolved by %s\n", resolverWords(c.Resolver))
|
||||
fmt.Fprintf(w, " resolved by %s\n", resolverWords(c.Resolver))
|
||||
if c.Silenced != nil {
|
||||
fmt.Printf(" silenced until %s by %s: %s\n", c.Silenced.Until.Local().Format("2006-01-02 15:04"),
|
||||
fmt.Fprintf(w, " silenced until %s by %s: %s\n", c.Silenced.Until.Local().Format("2006-01-02 15:04"),
|
||||
c.Silenced.By, c.Silenced.Why)
|
||||
}
|
||||
if len(c.Tried) > 0 {
|
||||
fmt.Println("\n tried:")
|
||||
fmt.Fprintln(w, "\n tried:")
|
||||
for _, t := range c.Tried {
|
||||
fmt.Printf(" %s %s — %s: %s\n", t.At.Local().Format("2006-01-02 15:04"), orHealer(t.By), t.What, t.Outcome)
|
||||
fmt.Fprintf(w, " %s %s — %s: %s\n", t.At.Local().Format("2006-01-02 15:04"), orHealer(t.By), t.What, t.Outcome)
|
||||
}
|
||||
}
|
||||
fmt.Println("\n evidence, newest first:")
|
||||
fmt.Fprintln(w, "\n evidence, newest first:")
|
||||
for _, e := range c.Evidence {
|
||||
fmt.Printf(" %s %s\n", e.At.Local().Format("2006-01-02 15:04:05"), e.Said)
|
||||
fmt.Fprintf(w, " %s %s\n", e.At.Local().Format("2006-01-02 15:04:05"), e.Said)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -378,8 +381,9 @@ func parseFor(s string) (time.Duration, error) {
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func conditionHistory(ctx context.Context, args []string) error {
|
||||
func conditionHistory(ctx context.Context, args []string, w io.Writer) error {
|
||||
set := flag.NewFlagSet("conditions history", flag.ContinueOnError)
|
||||
usageTo(set, w)
|
||||
days := set.Int("days", 7, "how many days back, at most 90")
|
||||
key := set.String("key", "", "only this condition")
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
@@ -420,10 +424,10 @@ func conditionHistory(ctx context.Context, args []string) error {
|
||||
if out == nil {
|
||||
out = []conditions.Event{}
|
||||
}
|
||||
return printJSON(map[string]any{"history": out, "days": *days})
|
||||
return printJSONTo(w, map[string]any{"history": out, "days": *days})
|
||||
}
|
||||
if len(out) == 0 {
|
||||
fmt.Printf("nothing was raised, changed or cleared in the last %d day(s)\n", *days)
|
||||
fmt.Fprintf(w, "nothing was raised, changed or cleared in the last %d day(s)\n", *days)
|
||||
return nil
|
||||
}
|
||||
for _, e := range out {
|
||||
@@ -440,7 +444,7 @@ func conditionHistory(ctx context.Context, args []string) error {
|
||||
line += " — " + e.Why
|
||||
}
|
||||
}
|
||||
fmt.Println(line)
|
||||
fmt.Fprintln(w, line)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// What a consumer gave up on, answered by the serving controller (novox/hq issue 330).
|
||||
//
|
||||
// **In this process, on its own connection**: DEAD_LETTERS is read and changed on the bus, and the
|
||||
// serving controller is already on it. A verb run as a fresh process would open a connection of its own
|
||||
// for each call (novox/hq issue 327).
|
||||
|
||||
// busHandles are the serving controller's connection and JetStream handle.
|
||||
type busHandles struct {
|
||||
conn *nats.Conn
|
||||
js nats.JetStreamContext
|
||||
}
|
||||
|
||||
// defaultDeadLetters is how many the list says when not asked for more.
|
||||
const defaultDeadLetters = 50
|
||||
|
||||
// causeDeadLetter is the cause a delivery or drop gives when the caller gives none.
|
||||
const causeDeadLetter = "dead-letter"
|
||||
|
||||
// deadLettersAnswer is what `dead-letters` answers: the list, one whole, or what came of delivering one
|
||||
// again or dropping it.
|
||||
func deadLettersAnswer(ctx context.Context, a *verbArguments) (any, error) {
|
||||
serving := servingBus.Load()
|
||||
if serving == nil {
|
||||
return nil, errors.New("this controller is not serving, so it does not read DEAD_LETTERS: ask again, " +
|
||||
"and the serving controller answers")
|
||||
}
|
||||
on := &busHandles{conn: serving.Conn(), js: serving.Context()}
|
||||
// The shape first, and every argument it reads; one given beside it is refused before anything is
|
||||
// done, as every verb refuses what it would pass over (novox/hq issue 244).
|
||||
var deliver, drop, why, cause, idText, consumer, limit string
|
||||
switch {
|
||||
case a.given["deliver"] != "" || a.given["drop"] != "":
|
||||
deliver, drop, why, cause = a.str("deliver"), a.str("drop"), a.str("why"), a.str("cause")
|
||||
case a.given["id"] != "":
|
||||
idText = a.str("id")
|
||||
default:
|
||||
consumer, limit = a.str("consumer"), a.str("limit")
|
||||
}
|
||||
if unused := a.unused(); len(unused) > 0 {
|
||||
return nil, fmt.Errorf("dead-letters did not use %s together with %s, and an argument a verb would pass "+
|
||||
"over is refused: nothing was done", quoteAll(unused), quoteAll(a.usedGiven()))
|
||||
}
|
||||
switch {
|
||||
case deliver != "" && drop != "":
|
||||
return nil, errors.New("dead-letters delivers one again or drops one, not both. Nothing was done")
|
||||
case deliver != "" || drop != "":
|
||||
act, text := "deliver", deliver
|
||||
if drop != "" {
|
||||
act, text = "drop", drop
|
||||
}
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return nil, fmt.Errorf("dead-letters %s is a hand act, and says why: why is required and recorded in "+
|
||||
"the hand-act log (novox/hq to-be 45 §7). Nothing was done", act)
|
||||
}
|
||||
id, err := deadLetterID(text)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return actOnDeadLetter(ctx, on, act, id, why, cause)
|
||||
case idText != "":
|
||||
id, err := deadLetterID(idText)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return link.DeadLetterNamed(on.js, id)
|
||||
}
|
||||
most := defaultDeadLetters
|
||||
if limit != "" {
|
||||
n, err := strconv.Atoi(limit)
|
||||
if err != nil || n <= 0 {
|
||||
return nil, fmt.Errorf("limit is a number of dead letters, not %q", limit)
|
||||
}
|
||||
most = n
|
||||
}
|
||||
held, total, err := link.DeadLetters(on.js, consumer, most)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
answer := map[string]any{"dead_letters": held, "held": total,
|
||||
"note": "newest first; with id, one whole; deliver or drop one with why"}
|
||||
if total == 0 {
|
||||
answer["note"] = "no consumer gave up on a message that is still kept"
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// deadLetterID is a dead letter's id as a caller wrote it.
|
||||
func deadLetterID(text string) (uint64, error) {
|
||||
id, err := strconv.ParseUint(strings.TrimSpace(text), 10, 64)
|
||||
if err != nil || id == 0 {
|
||||
return 0, fmt.Errorf("a dead letter's id is its number in %s, as dead-letters lists it, not %q",
|
||||
broker.DeadLettersStream, text)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// actOnDeadLetter delivers one again or drops it, recorded in the hand-act log before it is done. A log
|
||||
// that cannot be written is said, and the act still happens: the log is never the reason a person's act
|
||||
// is refused (handacts.go).
|
||||
func actOnDeadLetter(ctx context.Context, on *busHandles, act string, id uint64, why, cause string) (any, error) {
|
||||
d, err := link.DeadLetterNamed(on.js, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if act == "deliver" {
|
||||
// Refused before it is recorded: an act that cannot be done is not an act.
|
||||
if _, err := link.AgainTo(on.js, d); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if cause == "" {
|
||||
cause = causeDeadLetter
|
||||
}
|
||||
by := link.CallerIn(ctx)
|
||||
if by == "" {
|
||||
by = "a seat call whose caller the bus did not name"
|
||||
}
|
||||
answer := map[string]any{"dead_letter": d.ID, "consumer": d.Who, "subject": d.Subject}
|
||||
recorded, logErr := link.RecordHandAct(ctx, on.conn, link.HandAct{Verb: "dead-letters " + act,
|
||||
Args: []string{strconv.FormatUint(id, 10), d.Stream + "." + d.Consumer}, Why: why, Cause: cause,
|
||||
Condition: conditions.Key(conditions.ScopeBus, d.Stream+"."+d.Consumer, link.AdvisoryMaxDeliveries),
|
||||
By: by + ", through the " + catalogue.ControllerSeatName + " seat"})
|
||||
if logErr != nil {
|
||||
answer["unrecorded"] = "the hand-act log could not be written, and the act was done all the same: " + logErr.Error()
|
||||
} else {
|
||||
answer["recorded"] = recorded.ID
|
||||
}
|
||||
switch act {
|
||||
case "deliver":
|
||||
_, to, err := link.DeliverAgain(on.js, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
answer["delivered_on"] = to
|
||||
answer["done"] = fmt.Sprintf("dead letter %d was delivered again to %s, and nobody else; it is no longer kept",
|
||||
id, consumerWho(d.Stream, d.Consumer))
|
||||
case "drop":
|
||||
if _, err := link.DropDeadLetter(on.js, id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
answer["done"] = fmt.Sprintf("dead letter %d, which %s gave up on, was dropped for good", id,
|
||||
consumerWho(d.Stream, d.Consumer))
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// The serving controller's bus, with the mesh's streams, for the verb to read and act on.
|
||||
func servingDeadLetters(t *testing.T) *broker.JetStream {
|
||||
t.Helper()
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := servingBus.Load()
|
||||
servingBus.Store(js)
|
||||
t.Cleanup(func() { servingBus.Store(before) })
|
||||
return js
|
||||
}
|
||||
|
||||
func askDeadLetters(t *testing.T, args map[string]any) (any, error) {
|
||||
t.Helper()
|
||||
a, err := readArguments("dead-letters", args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return deadLettersAnswer(context.Background(), a)
|
||||
}
|
||||
|
||||
func TestDeadLettersListsDropsAndRecordsWhy(t *testing.T) {
|
||||
js := servingDeadLetters(t)
|
||||
if _, err := js.Context().Publish("mesh.mod.gitea.event.pull.merged", []byte(`{"n":1}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, err := link.KeepDeadLetter(js.Context(), []byte(`{"stream":"EVENTS","consumer":"media_sonarr","stream_seq":1,"deliveries":5}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
answer, err := askDeadLetters(t, map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
listed := answer.(map[string]any)
|
||||
if listed["held"] != 1 || len(listed["dead_letters"].([]link.DeadLetter)) != 1 {
|
||||
t.Fatalf("listed %v", listed)
|
||||
}
|
||||
|
||||
// Refused before anything is done: an act without why, an argument the shape passes over, both acts.
|
||||
for _, args := range []map[string]any{
|
||||
{"drop": "1"},
|
||||
{"drop": "1", "why": "x", "limit": "3"},
|
||||
{"drop": "1", "deliver": "1", "why": "x"},
|
||||
{"why": "x"},
|
||||
{"id": "nought"},
|
||||
} {
|
||||
if _, err := askDeadLetters(t, args); err == nil {
|
||||
t.Errorf("%v was done", args)
|
||||
}
|
||||
}
|
||||
if _, total, _ := link.DeadLetters(js.Context(), "", 0); total != 1 {
|
||||
t.Fatalf("a refused call changed what is kept: %d left", total)
|
||||
}
|
||||
|
||||
done, err := askDeadLetters(t, map[string]any{"drop": "1", "why": "the media server took the download in by hand"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if said := done.(map[string]any); said["recorded"] == nil || !strings.Contains(said["done"].(string), "dropped") {
|
||||
t.Fatalf("answered %v", said)
|
||||
}
|
||||
acts, err := link.HandActs(context.Background(), js.Conn(), time.Now().Add(-time.Minute))
|
||||
if err != nil || len(acts) != 1 || acts[0].Verb != "dead-letters drop" || acts[0].Cause != causeDeadLetter ||
|
||||
!strings.Contains(acts[0].Condition, "media_sonarr") {
|
||||
t.Fatalf("recorded %+v (%v)", acts, err)
|
||||
}
|
||||
if !personsDecision(acts[0]) {
|
||||
t.Error("dropping a dead letter is counted as a repair, so S15 would want a healer for it")
|
||||
}
|
||||
if _, err := askDeadLetters(t, map[string]any{"id": "1"}); err == nil {
|
||||
t.Errorf("dead letter %d is still answered after it was dropped", d.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// Open while DEAD_LETTERS holds a message for the consumer, in words the operator reads in one pass:
|
||||
// what is held, and where to act.
|
||||
func TestAConsumersDeadLettersAreSaidUntilActedOn(t *testing.T) {
|
||||
f := &signalFacts{now: time.Now(), deadLetters: map[string]int{"EVENTS.media_sonarr": 4, "EVENTS.controller": 1}}
|
||||
found := watchDeadLetters(f)
|
||||
if len(found) != 2 {
|
||||
t.Fatalf("said %d conditions", len(found))
|
||||
}
|
||||
for _, o := range found {
|
||||
if o.Kind != "max-deliveries" || o.Severity != conditions.Warning || o.Needs == "" {
|
||||
t.Errorf("%+v", o)
|
||||
}
|
||||
if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Needs: o.Needs,
|
||||
Explanation: o.Explanation, Resolved: o.Resolved}, "media"); !ok {
|
||||
t.Errorf("%q is not plain: %s", o.Headline, why)
|
||||
}
|
||||
if !strings.Contains(o.Needs, "mesh MCP server") {
|
||||
t.Errorf("does not say where to act: %q", o.Needs)
|
||||
}
|
||||
}
|
||||
sonarr := found[1]
|
||||
if sonarr.ID != "EVENTS.media_sonarr" || sonarr.Machine != "media" ||
|
||||
sonarr.Headline != "Sonarr on media could not handle 4 messages" ||
|
||||
!strings.Contains(sonarr.Summary, "DEAD_LETTERS") {
|
||||
t.Errorf("%+v", sonarr)
|
||||
}
|
||||
if found[0].Headline != "The controller could not handle a message" {
|
||||
t.Errorf("%q", found[0].Headline)
|
||||
}
|
||||
// None held, none said: it clears when they are delivered again or dropped.
|
||||
if left := watchDeadLetters(&signalFacts{now: time.Now()}); len(left) != 0 {
|
||||
t.Fatalf("%v", left)
|
||||
}
|
||||
}
|
||||
@@ -116,11 +116,11 @@ var probeRegistry = []probe{
|
||||
{ID: probeDeliveriesID, Asserts: "no delivery is held past its state's bound unsaid: mesh-delivery's " +
|
||||
"`stalled`, each with the transition its table lets healer H2 take", From: "ADR 0239",
|
||||
Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries},
|
||||
// Root where the trusted parties run (novox/hq ADR 0259 §8): while an agent can become root there without a
|
||||
// person, an answer proven there proves nothing.
|
||||
{ID: "D-root", Asserts: "no agent can become root without a person on a machine where the router or a channel " +
|
||||
"proving its sender runs: not by its own account, and not through a tool that runs its command as an account " +
|
||||
"that can", From: "ADR 0259 §8", Kind: kindAgentRoot, Phase: 2, run: probeAgentRoot},
|
||||
// A client of the bus reconnecting in a loop (novox/hq issue 327), from the server's record of closed
|
||||
// connections, which the bus's own module reads.
|
||||
{ID: probeReconnectsID, Asserts: "no user of the bus had its connection dropped more than twelve times in the " +
|
||||
"last hour: the bus module's nats_closed_connections", From: "issue 327", Kind: kindBusReconnects,
|
||||
Phase: 1, run: probeReconnects},
|
||||
{ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals",
|
||||
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs},
|
||||
// The core's health definitions (novox/hq to-be 45 §8, ADR 0236): what a core component's new build is
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
@@ -14,7 +15,6 @@ import (
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
@@ -82,6 +82,11 @@ var handActVerbs = []handActVerb{
|
||||
{Verb: "retire approve", Decision: "nothing is retired past its bound without a person (ADR 0230)"},
|
||||
{Verb: "retire reject", Decision: "keeping a consumer active is a person's word (ADR 0230)"},
|
||||
{Verb: "cleanup delete", Decision: "nothing retired is deleted without a person (ADR 0230)"},
|
||||
// What becomes of a message a consumer gave up on (novox/hq issue 330): kept until a person says.
|
||||
{Verb: "dead-letters deliver", Decision: "a message a consumer gave up on is delivered again only on a " +
|
||||
"person's word (issue 330)"},
|
||||
{Verb: "dead-letters drop", Decision: "a message a consumer gave up on is let go only on a person's word " +
|
||||
"(issue 330)"},
|
||||
// The sweep run on a person's word rather than after a build: the same decision the records make, at
|
||||
// a moment the person chose (ADR 0251) — never a repair.
|
||||
{Verb: "collect", Decision: "letting the store go of what the records keep for no reason, now rather " +
|
||||
@@ -149,14 +154,10 @@ func onTheBus(f func(*nats.Conn) error) error {
|
||||
if handActConn != nil {
|
||||
return f(handActConn)
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot reach the bus: %w", err)
|
||||
}
|
||||
defer js.Close()
|
||||
return f(js.Conn())
|
||||
}
|
||||
@@ -247,7 +248,13 @@ func handActCommand(ctx context.Context, args []string) error {
|
||||
if len(args) > 0 && args[0] == "list" {
|
||||
args = args[1:]
|
||||
}
|
||||
return listHandActs(ctx, args, os.Stdout)
|
||||
}
|
||||
|
||||
// listHandActs is `hand-acts`: what was done by hand lately, and the causes done more than once.
|
||||
func listHandActs(ctx context.Context, args []string, w io.Writer) error {
|
||||
set := flag.NewFlagSet("hand-acts", flag.ContinueOnError)
|
||||
usageTo(set, w)
|
||||
days := set.Int("days", 14, "how many days back")
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
@@ -265,27 +272,27 @@ func handActCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
fmt.Fprintln(w, string(body))
|
||||
return nil
|
||||
}
|
||||
if len(acts) == 0 {
|
||||
fmt.Printf("nothing was done by hand in the last %d day(s)\n", *days)
|
||||
fmt.Fprintf(w, "nothing was done by hand in the last %d day(s)\n", *days)
|
||||
return nil
|
||||
}
|
||||
for i := len(acts) - 1; i >= 0; i-- {
|
||||
a := acts[i]
|
||||
fmt.Printf("%s %s %s %s\n by %s — %s (cause: %s", a.At.Local().Format("2006-01-02 15:04"), a.ID,
|
||||
fmt.Fprintf(w, "%s %s %s %s\n by %s — %s (cause: %s", a.At.Local().Format("2006-01-02 15:04"), a.ID,
|
||||
a.Verb, strings.Join(a.Args, " "), a.By, a.Why, a.Cause)
|
||||
if a.Condition != "" {
|
||||
fmt.Printf(", condition %s", a.Condition)
|
||||
fmt.Fprintf(w, ", condition %s", a.Condition)
|
||||
}
|
||||
fmt.Println(")")
|
||||
fmt.Fprintln(w, ")")
|
||||
if pushedRecorded(a) {
|
||||
carried := strings.Join(a.Carried, "; ")
|
||||
if carried == "" {
|
||||
carried = recordedBefore[a.ID]
|
||||
}
|
||||
fmt.Printf(" a push of recorded builds, no repair: %s\n", carried)
|
||||
fmt.Fprintf(w, " a push of recorded builds, no repair: %s\n", carried)
|
||||
}
|
||||
}
|
||||
if len(repeated) > 0 {
|
||||
@@ -294,7 +301,7 @@ func handActCommand(ctx context.Context, args []string) error {
|
||||
causes = append(causes, fmt.Sprintf("%s ×%d", c, n))
|
||||
}
|
||||
sort.Strings(causes)
|
||||
fmt.Printf("\ndone by hand more than once in a fortnight — a healer is wanted (to-be 45 S15): %s\n",
|
||||
fmt.Fprintf(w, "\ndone by hand more than once in a fortnight — a healer is wanted (to-be 45 S15): %s\n",
|
||||
strings.Join(causes, ", "))
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/identity"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/licences"
|
||||
@@ -53,6 +54,9 @@ func run() error {
|
||||
return fmt.Errorf("no command given")
|
||||
}
|
||||
|
||||
// Every connection this process dials says what it is (novox/hq issue 327).
|
||||
broker.ConnectionName = connectionName(args[0], os.Getenv(verbVar))
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
// Whatever this process holds of the controller's lease is given back as it ends (novox/hq to-be
|
||||
@@ -416,3 +420,18 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
||||
statusFrom.nudge()
|
||||
return nil
|
||||
}
|
||||
|
||||
// verbVar carries the seat verb a command runs for, from the serving controller to the process it starts.
|
||||
const verbVar = "MESH_VERB"
|
||||
|
||||
// connectionName is what this process's connections say they are in the bus's list (novox/hq issue 327):
|
||||
// the serving controller, a verb's own process and which verb, or a command run at a shell and which.
|
||||
func connectionName(command, verb string) string {
|
||||
switch {
|
||||
case command == "serve":
|
||||
return "mesh-controller serving"
|
||||
case verb != "":
|
||||
return "mesh-controller verb " + verb
|
||||
}
|
||||
return "mesh-controller command " + command
|
||||
}
|
||||
|
||||
@@ -7,7 +7,9 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -684,11 +686,14 @@ func orNotReported(s string) string {
|
||||
}
|
||||
|
||||
// printJSON prints a value as indented JSON, the shape every `--json` answers in.
|
||||
func printJSON(v any) error {
|
||||
func printJSON(v any) error { return printJSONTo(os.Stdout, v) }
|
||||
|
||||
// printJSONTo is printJSON to a writer of the caller's.
|
||||
func printJSONTo(w io.Writer, v any) error {
|
||||
body, err := json.MarshalIndent(v, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
fmt.Fprintln(w, string(body))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -543,10 +543,18 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
||||
Explanation: "The bus reports a listener too slow to keep up, so messages to it are late.",
|
||||
Resolved: "The listener keeps up again"}
|
||||
}),
|
||||
kindBusReconnects: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A client keeps losing the bus",
|
||||
Explanation: "One of the mesh's clients lost its connection to the bus again and again in the last hour. " +
|
||||
"While it reconnects, what it says and what it is asked waits.",
|
||||
Resolved: "Resolved: the client stays connected"}
|
||||
}),
|
||||
"max-deliveries": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A message could not be handled",
|
||||
Explanation: "The bus gave up on a message after trying to hand it over too many times.",
|
||||
Resolved: "Resolved: messages are handled again"}
|
||||
return words{Headline: "A listener gave up on messages",
|
||||
Needs: "deliver them again or drop them, from the mesh MCP server.",
|
||||
Explanation: "A listener on the bus could not handle messages after several tries, so what they asked " +
|
||||
"for was not done. The mesh keeps them until you deliver them again or drop them.",
|
||||
Resolved: "Resolved: the messages given up on were delivered again or dropped"}
|
||||
}),
|
||||
"refused": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The bus refuses some messages",
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
@@ -98,11 +97,9 @@ func buildSeatPause(ctx context.Context, inv *inventory.Inventory, plans []inven
|
||||
if len(holders) == 0 {
|
||||
return pauseView{}
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return pauseView{}
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
// On the serving controller's own connection when this is it: a watchdog tick while a walk waits for a
|
||||
// build dialled one every 30 seconds (novox/hq issue 327).
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not reach the bus to read whether the build seat is paused: %v\n", err)
|
||||
return pauseView{}
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **The planner over edges the store derives**, not edges written by hand: modules registered, builds
|
||||
// recorded with what they stood on and which repositories they read, the relation answered by
|
||||
// inventory.Dependencies (dependenciesOf over the records), and the merge planned by reachOfMerge — the
|
||||
// path a real merge takes, short of the bus.
|
||||
//
|
||||
// **The repository rows are CURRENT BEHAVIOUR, documented — not the rule the operator states**
|
||||
// (novox/hq issue 338, and the decision pending on it): a build that read a repository gives its module a
|
||||
// packages edge to every module built from that repository, and mergeCandidates moves it on any merge to
|
||||
// that repository, whatever the files. So a change to C alone, or to a README, moves the module that
|
||||
// packages C's repository. ADR 0238 §3 records exactly that today ("a repository a recipe names"); the
|
||||
// expectations marked 338 change with that decision.
|
||||
func TestASharedRepositoryIsPlannedFromTheRecordsAsItIsToday(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
ctx := t.Context()
|
||||
asked := time.Now().Add(-time.Hour)
|
||||
register := func(m catalogue.Manifest, repository, path string, against []string, read []inventory.ReadRepository) {
|
||||
t.Helper()
|
||||
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: repository, Seat: "git", Path: path,
|
||||
Ref: "main", BuiltFrom: "old", Asked: asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + m.Module, Repository: repository, Ref: "main",
|
||||
Module: m.Module, Commit: "old", On: "builder", Path: path, Against: against, Read: read, Asked: asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
controllerRead := []inventory.ReadRepository{{Repository: "novox/mesh-controller", Ref: "main"}}
|
||||
agent := catalogue.Manifest{Module: "build-agent", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}}
|
||||
|
||||
// The shape of issue 338.
|
||||
register(catalogue.Manifest{Module: "mesh-controller", Version: "1"}, "novox/mesh-controller", "", nil, nil)
|
||||
register(agent, "novox/mesh-catalog", "modules/build-agent", nil, controllerRead)
|
||||
register(catalogue.Manifest{Module: "route-proxy", Version: "1"}, "novox/mesh-catalog", "modules/route-proxy", nil, controllerRead)
|
||||
register(catalogue.Manifest{Module: "gitea", Version: "1"}, "novox/mesh-catalog", "modules/gitea", nil, nil)
|
||||
// A, B and C in one repository; D built against A's artifact, E declaring B, P packaging the repository.
|
||||
for _, n := range []string{"a", "b", "c"} {
|
||||
register(catalogue.Manifest{Module: n, Version: "1"}, "novox/one", "modules/"+n, nil, nil)
|
||||
}
|
||||
register(catalogue.Manifest{Module: "d", Version: "1"}, "novox/two", "d",
|
||||
[]string{catalogue.ArtifactStoreScheme + "a/runtime@sha256:" + strings.Repeat("0", 64)}, nil)
|
||||
register(catalogue.Manifest{Module: "e", Version: "1", Build: &catalogue.Build{
|
||||
On: []catalogue.BuildsOn{{Arg: "BASE", Module: "b", Artifact: "runtime"}}}}, "novox/two", "e", nil, nil)
|
||||
register(catalogue.Manifest{Module: "p", Version: "1"}, "novox/two", "p", nil,
|
||||
[]inventory.ReadRepository{{Repository: "novox/one", Ref: "main"}})
|
||||
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
read, err := inv.ReadRepositories(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
edges, err := inv.Dependencies(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The edges the hand-written rows of TestASharedRepositoryMovesWhatPackagesItAsItDoesToday use are
|
||||
// the ones derived here.
|
||||
var shared []inventory.Edge
|
||||
in338 := map[string]bool{"mesh-controller": true, "build-agent": true, "route-proxy": true, "gitea": true}
|
||||
for _, e := range edges {
|
||||
if in338[e.From] && in338[e.To] {
|
||||
shared = append(shared, e)
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(shared, sharedRepositoryEdges) {
|
||||
t.Errorf("derived %v\nthe hand-written rows use %v", shared, sharedRepositoryEdges)
|
||||
}
|
||||
// Each kind derived from its record: built against (stands-on), build.on (declared), read (packages).
|
||||
for _, want := range []inventory.Edge{
|
||||
dep("d", inventory.EdgeStandsOn, "a"),
|
||||
dep("e", inventory.EdgeDeclared, "b"),
|
||||
dep("p", inventory.EdgePackages, "a"),
|
||||
dep("p", inventory.EdgePackages, "b"),
|
||||
dep("p", inventory.EdgePackages, "c"),
|
||||
dep("d", inventory.EdgeBuiltBy, "build-agent"),
|
||||
} {
|
||||
found := false
|
||||
for _, e := range edges {
|
||||
found = found || e == want
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("no %s %s %s derived: %v", want.From, want.Kind, want.To, edges)
|
||||
}
|
||||
}
|
||||
|
||||
for _, c := range []struct {
|
||||
what, repo string
|
||||
paths []string
|
||||
want string
|
||||
issue338 bool
|
||||
}{
|
||||
{"A and B changed, C untouched: D after A, E after B; P packages their repository", "one",
|
||||
[]string{"modules/a/x.go", "modules/b/x.go"}, "a,b,p | d,e", false},
|
||||
{"C alone: C, and P, which packages C's repository", "one",
|
||||
[]string{"modules/c/x.go"}, "c,p", true},
|
||||
{"a README of the repository P packages: P moves, nothing built from it does", "one",
|
||||
[]string{"README.md"}, "p", true},
|
||||
{"the dependent's repository: D alone", "two", []string{"d/main.go"}, "d", false},
|
||||
{"a README of the controller's repository: all three, three tiers", "mesh-controller",
|
||||
[]string{"README.md"}, "mesh-controller | build-agent | route-proxy", true},
|
||||
{"the route proxy's directory in the catalogue: it alone", "mesh-catalog",
|
||||
[]string{"modules/route-proxy/module.json"}, "route-proxy", false},
|
||||
{"the build agent's directory: it alone, nothing it builds", "mesh-catalog",
|
||||
[]string{"modules/build-agent/module.json"}, "build-agent", false},
|
||||
} {
|
||||
_, got := planMerge(t, c.repo, c.paths, entries, read, edges)
|
||||
if got != c.want {
|
||||
tag := ""
|
||||
if c.issue338 {
|
||||
tag = " (current behaviour, issue 338)"
|
||||
}
|
||||
t.Errorf("%s: planned %q, wanted %q%s", c.what, got, c.want, tag)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,447 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The delivery planner's rules, as recorded (novox/hq ADR 0162 §1, ADR 0238 §3), held by one table and
|
||||
// one property over reachOfMerge — the planner's one answer to "what does this merge move, and in which
|
||||
// order". A row that fails here on main is a planner that breaks a recorded rule: the row stays, the
|
||||
// expectation is not bent to the code.
|
||||
//
|
||||
// The kinds of edge, as the code reads them (release_plan.go):
|
||||
//
|
||||
// kind widens the plan orders the tiers
|
||||
// stands-on yes yes, after its base is built
|
||||
// declared yes yes, after its base is built
|
||||
// packages yes no, the same tier (a code dependency)
|
||||
// built-by no yes, after the build machine — except for what the build machine stands
|
||||
// on, and for the controller whose worker it binds
|
||||
// worker-of no yes, the build seat's holder after the controller (hq issue 206)
|
||||
|
||||
const (
|
||||
repoOne = "http://forge.internal:20000/novox/one.git"
|
||||
repoTwo = "http://forge.internal:20000/novox/two.git"
|
||||
)
|
||||
|
||||
// dep is one edge of the catalogue's relation: from depends on to, in the way kind says.
|
||||
func dep(from, kind, to string) inventory.Edge {
|
||||
return inventory.Edge{From: from, To: to, Kind: kind}
|
||||
}
|
||||
|
||||
// tiered is a plan's tiers as one line: a tier's modules by comma, tiers by " | ". Empty for no plan.
|
||||
func tiered(tiers [][]string) string {
|
||||
var out []string
|
||||
for _, t := range tiers {
|
||||
out = append(out, strings.Join(t, ","))
|
||||
}
|
||||
return strings.Join(out, " | ")
|
||||
}
|
||||
|
||||
// planMerge is what reachOfMerge plans for a merge of these files into a repository's main: its tiers as
|
||||
// one line, and the files no build reads. It also holds the plan to its own shape: every module it builds
|
||||
// is in exactly one tier.
|
||||
func planMerge(t *testing.T, repo string, paths []string, entries []inventory.Entry,
|
||||
read map[string][]inventory.ReadRepository, edges []inventory.Edge) (mergeReach, string) {
|
||||
t.Helper()
|
||||
m := link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Commit: "head", Paths: paths}
|
||||
r := reachOfMerge(m, entries, read, edges)
|
||||
seen := map[string]int{}
|
||||
for _, tier := range r.Plan.Tiers {
|
||||
for _, name := range tier {
|
||||
seen[name]++
|
||||
}
|
||||
}
|
||||
for name := range r.Plan.Modules {
|
||||
if seen[name] != 1 {
|
||||
t.Errorf("%s/%v: %s is in %d tiers of %v", repo, paths, name, seen[name], r.Plan.Tiers)
|
||||
}
|
||||
}
|
||||
if len(seen) != len(r.Plan.Modules) {
|
||||
t.Errorf("%s/%v: the tiers %v hold modules the plan does not (%d)", repo, paths, r.Plan.Tiers, len(r.Plan.Modules))
|
||||
}
|
||||
return r, tiered(r.Plan.Tiers)
|
||||
}
|
||||
|
||||
// **A merge moves the modules whose directory it changed, and everything built on them; nothing else.**
|
||||
// Each row is a catalogue (modules in directories of one or two repositories), its dependencies with
|
||||
// their kinds, the files one commit changed, and the exact plan: the moved set and its tier order.
|
||||
func TestAPlanIsWhatTheChangeTouchedAndWhatIsBuiltOnIt(t *testing.T) {
|
||||
in := func(repo, dir string, names ...string) []inventory.Entry {
|
||||
var out []inventory.Entry
|
||||
for _, n := range names {
|
||||
d := dir + "/" + n
|
||||
if dir == "" {
|
||||
d = n
|
||||
}
|
||||
out = append(out, fromRepo(n, repo, d))
|
||||
}
|
||||
return out
|
||||
}
|
||||
// Modules a to f, x and z in novox/one under modules/; g in novox/two at g/.
|
||||
entries := append(in(repoOne, "modules", "a", "b", "c", "d", "e", "f", "x", "z"), in(repoTwo, "", "g")...)
|
||||
const (
|
||||
standsOn = inventory.EdgeStandsOn
|
||||
declared = inventory.EdgeDeclared
|
||||
packages = inventory.EdgePackages
|
||||
builtBy = inventory.EdgeBuiltBy
|
||||
workerOf = inventory.EdgeWorkerOf
|
||||
)
|
||||
// The two real cycles the kinds resolve themselves (ADR 0162 §1, hq issue 206).
|
||||
runtime := append(in(repoOne, "modules", "runtime", "builder"), fromRepo("controller", repoTwo, ""))
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
entries []inventory.Entry
|
||||
edges []inventory.Edge
|
||||
repo string
|
||||
paths []string
|
||||
want string // the tiers, " | " between them
|
||||
unread string
|
||||
cycle bool
|
||||
}{
|
||||
// The operator's case: two modules changed, a third beside them in the same repository untouched,
|
||||
// each changed one with a dependent.
|
||||
{what: "A and B changed, C untouched beside them, D on A and E on B",
|
||||
edges: []inventory.Edge{dep("d", standsOn, "a"), dep("e", declared, "b")},
|
||||
repo: "one", paths: []string{"modules/a/main.go", "modules/b/module.json"}, want: "a,b | d,e"},
|
||||
{what: "only A's directory: A and what stands on it",
|
||||
edges: []inventory.Edge{dep("d", standsOn, "a"), dep("e", declared, "b")},
|
||||
repo: "one", paths: []string{"modules/a/main.go"}, want: "a | d"},
|
||||
{what: "only C's directory: C alone, nothing is built on it",
|
||||
edges: []inventory.Edge{dep("d", standsOn, "a"), dep("e", declared, "b")},
|
||||
repo: "one", paths: []string{"modules/c/Dockerfile"}, want: "c"},
|
||||
{what: "only the dependent changed: its base does not move",
|
||||
edges: []inventory.Edge{dep("d", standsOn, "a")},
|
||||
repo: "one", paths: []string{"modules/d/main.go"}, want: "d"},
|
||||
{what: "transitive: F on D on A, A changed",
|
||||
edges: []inventory.Edge{dep("f", standsOn, "d"), dep("d", standsOn, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x"}, want: "a | d | f"},
|
||||
{what: "transitive across kinds: F declared on D, D packages A",
|
||||
edges: []inventory.Edge{dep("f", declared, "d"), dep("d", packages, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x"}, want: "a,d | f"},
|
||||
|
||||
// Each kind alone: X depends on A, A changed (widening), then both changed (ordering).
|
||||
{what: "stands-on (built against A's artifact) widens", edges: []inventory.Edge{dep("x", standsOn, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x"}, want: "a | x"},
|
||||
{what: "stands-on orders", edges: []inventory.Edge{dep("x", standsOn, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a | x"},
|
||||
{what: "declared (build.on) widens", edges: []inventory.Edge{dep("x", declared, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x"}, want: "a | x"},
|
||||
{what: "declared orders", edges: []inventory.Edge{dep("x", declared, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a | x"},
|
||||
{what: "packages widens, into the same tier", edges: []inventory.Edge{dep("x", packages, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x"}, want: "a,x"},
|
||||
{what: "packages does not order", edges: []inventory.Edge{dep("x", packages, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a,x"},
|
||||
{what: "built-by never widens", edges: []inventory.Edge{dep("x", builtBy, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x"}, want: "a"},
|
||||
{what: "built-by orders", edges: []inventory.Edge{dep("x", builtBy, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a | x"},
|
||||
{what: "worker-of never widens", edges: []inventory.Edge{dep("x", workerOf, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x"}, want: "a"},
|
||||
{what: "worker-of orders", edges: []inventory.Edge{dep("x", workerOf, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a | x"},
|
||||
{what: "a change to the base alone does not move what it builds", edges: []inventory.Edge{dep("x", builtBy, "a"),
|
||||
dep("d", builtBy, "a"), dep("e", standsOn, "a")},
|
||||
repo: "one", paths: []string{"modules/a/module.json"}, want: "a | e"},
|
||||
|
||||
// The cycles the kinds resolve: the build machine stands on the runtime image the runtime image is
|
||||
// built by; the build seat's holder follows the controller that is built by it.
|
||||
{what: "the build machine's base comes first, built by the build machine that runs", entries: runtime,
|
||||
edges: []inventory.Edge{dep("runtime", builtBy, "builder"), dep("builder", standsOn, "runtime")},
|
||||
repo: "one", paths: []string{"modules/runtime/Dockerfile", "modules/builder/main.go"}, want: "runtime | builder"},
|
||||
{what: "the runtime image alone takes the build machine on it along", entries: runtime,
|
||||
edges: []inventory.Edge{dep("runtime", builtBy, "builder"), dep("builder", standsOn, "runtime")},
|
||||
repo: "one", paths: []string{"modules/runtime/Dockerfile"}, want: "runtime | builder"},
|
||||
{what: "the build machine alone moves alone", entries: runtime,
|
||||
edges: []inventory.Edge{dep("runtime", builtBy, "builder"), dep("builder", standsOn, "runtime")},
|
||||
repo: "one", paths: []string{"modules/builder/main.go"}, want: "builder"},
|
||||
{what: "the build seat's holder follows the controller it binds the worker of", entries: runtime,
|
||||
edges: []inventory.Edge{dep("controller", builtBy, "builder"), dep("builder", workerOf, "controller")},
|
||||
repo: "two", paths: []string{"cmd/main.go"}, want: "controller"},
|
||||
|
||||
// Two repositories.
|
||||
{what: "a dependent in another repository follows its base",
|
||||
edges: []inventory.Edge{dep("g", standsOn, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x"}, want: "a | g"},
|
||||
{what: "a directory of the same name in another repository is not this one's",
|
||||
edges: []inventory.Edge{dep("g", standsOn, "a")},
|
||||
repo: "two", paths: []string{"modules/a/x"}, want: "", unread: "modules/a/x"},
|
||||
{what: "the dependent's own repository moves the dependent alone",
|
||||
edges: []inventory.Edge{dep("g", standsOn, "a")},
|
||||
repo: "two", paths: []string{"g/main.go"}, want: "g"},
|
||||
|
||||
// A diamond.
|
||||
{what: "a diamond, one side changed", edges: []inventory.Edge{dep("d", standsOn, "a"), dep("d", standsOn, "b")},
|
||||
repo: "one", paths: []string{"modules/a/x"}, want: "a | d"},
|
||||
{what: "a diamond, both sides changed", edges: []inventory.Edge{dep("d", standsOn, "a"), dep("d", standsOn, "b")},
|
||||
repo: "one", paths: []string{"modules/a/x", "modules/b/x"}, want: "a,b | d"},
|
||||
{what: "a diamond on one base", edges: []inventory.Edge{dep("d", standsOn, "a"), dep("d", declared, "b"),
|
||||
dep("a", standsOn, "z"), dep("b", standsOn, "z")},
|
||||
repo: "one", paths: []string{"modules/z/x"}, want: "z | a,b | d"},
|
||||
{what: "a diamond of mixed kinds orders on the ordering side only",
|
||||
edges: []inventory.Edge{dep("d", standsOn, "a"), dep("d", packages, "b")},
|
||||
repo: "one", paths: []string{"modules/b/x"}, want: "b,d"},
|
||||
|
||||
// A cycle the catalogue should never produce: what remains is one last tier, and said.
|
||||
{what: "a cycle is one last tier, not lost", edges: []inventory.Edge{dep("a", standsOn, "b"), dep("b", standsOn, "a"),
|
||||
dep("f", standsOn, "c")},
|
||||
repo: "one", paths: []string{"modules/a/x", "modules/c/x"}, want: "c | f | a,b", cycle: true},
|
||||
|
||||
// Files no build reads.
|
||||
{what: "a README at the root of a repository whose modules all live below it",
|
||||
edges: []inventory.Edge{dep("d", standsOn, "a")},
|
||||
repo: "one", paths: []string{"README.md"}, want: "", unread: "README.md"},
|
||||
{what: "a directory no module lives in", edges: []inventory.Edge{dep("d", standsOn, "a")},
|
||||
repo: "one", paths: []string{"modules/lib/x.go", "modules/README.md"}, want: "",
|
||||
unread: "modules/lib/x.go,modules/README.md"},
|
||||
{what: "a module's directory beside a root file", edges: []inventory.Edge{dep("d", standsOn, "a")},
|
||||
repo: "one", paths: []string{"merge-check.sh", "modules/a/x"}, want: "a | d", unread: "merge-check.sh"},
|
||||
{what: "a directory whose name begins with a module's", edges: []inventory.Edge{dep("d", standsOn, "a")},
|
||||
repo: "one", paths: []string{"modules/ab/x"}, want: "", unread: "modules/ab/x"},
|
||||
} {
|
||||
e := entries
|
||||
if c.entries != nil {
|
||||
e = c.entries
|
||||
}
|
||||
r, got := planMerge(t, c.repo, c.paths, e, nil, c.edges)
|
||||
if got != c.want {
|
||||
t.Errorf("%s: planned %q, wanted %q", c.what, got, c.want)
|
||||
}
|
||||
if u := strings.Join(r.Unread, ","); u != c.unread {
|
||||
t.Errorf("%s: unread %q, wanted %q", c.what, u, c.unread)
|
||||
}
|
||||
// A packages edge in the last tier is no cycle; hasCycle said one on main at 8170fc5.
|
||||
if hasCycle(r.Plan.Tiers, c.edges) != c.cycle {
|
||||
t.Errorf("%s: a cycle said %v, wanted %v (%v)", c.what, !c.cycle, c.cycle, r.Plan.Tiers)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **CURRENT BEHAVIOUR, documented — not the rule the operator states.** novox/hq issue 338 (a module
|
||||
// built from a shared repository moves on every merge to it) and the decision pending on it would change
|
||||
// every row here. Today:
|
||||
//
|
||||
// - mesh-controller is built from its repository's root, so every file of that repository touches it;
|
||||
// - route-proxy and build-agent package the whole of that repository (a build context), so the build
|
||||
// record's `read` makes them move on any merge to it, whatever the files, and dependenciesOf gives
|
||||
// each a packages edge to every module built from it;
|
||||
// - built-by (route-proxy on build-agent) and worker-of (build-agent on the controller) make it three
|
||||
// tiers.
|
||||
//
|
||||
// These follow ADR 0238 §3 as written ("the whole repository for a module built from its root, and a
|
||||
// repository a recipe names"), so they are not failures; when the decision on issue 338 lands, these
|
||||
// expectations change with it. The edges are the ones dependenciesOf derives from this catalogue — held
|
||||
// to that by TestASharedRepositoryIsPlannedFromTheRecordsAsItIsToday, which derives them from the store.
|
||||
func TestASharedRepositoryMovesWhatPackagesItAsItDoesToday(t *testing.T) {
|
||||
const catalogueRepo = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||
const controllerRepo = "http://forge.internal:20000/novox/mesh-controller.git"
|
||||
entries := []inventory.Entry{
|
||||
fromRepo("mesh-controller", controllerRepo, ""),
|
||||
fromRepo("build-agent", catalogueRepo, "modules/build-agent"),
|
||||
fromRepo("route-proxy", catalogueRepo, "modules/route-proxy"),
|
||||
fromRepo("gitea", catalogueRepo, "modules/gitea"),
|
||||
}
|
||||
read := map[string][]inventory.ReadRepository{
|
||||
"build-agent": {{Repository: "novox/mesh-controller", Ref: "main"}},
|
||||
"route-proxy": {{Repository: "novox/mesh-controller", Ref: "main"}},
|
||||
}
|
||||
edges := sharedRepositoryEdges
|
||||
for _, c := range []struct {
|
||||
what, repo string
|
||||
paths []string
|
||||
want string
|
||||
}{
|
||||
// The live three-tier plan of 2026-10-08 (issue 338), in the worker-of order (issue 206) that
|
||||
// TestAMergeIsPlannedInTiersAlongTheThreeKindsOfDependency's controller case holds too.
|
||||
{"a README of the controller's repository moves all three, in three tiers", "mesh-controller",
|
||||
[]string{"README.md"}, "mesh-controller | build-agent | route-proxy"},
|
||||
{"the controller's own code: the same", "mesh-controller",
|
||||
[]string{"cmd/mesh-controller/main.go"}, "mesh-controller | build-agent | route-proxy"},
|
||||
{"the route proxy's program alone: the same, the controller with it", "mesh-controller",
|
||||
[]string{"examples/route-proxy/main.go"}, "mesh-controller | build-agent | route-proxy"},
|
||||
// In the catalogue, where they live, the rule is path-precise.
|
||||
{"the route proxy's directory in the catalogue: it alone", "mesh-catalog",
|
||||
[]string{"modules/route-proxy/module.json"}, "route-proxy"},
|
||||
{"the build agent's directory: it alone, nothing it builds", "mesh-catalog",
|
||||
[]string{"modules/build-agent/module.json"}, "build-agent"},
|
||||
{"another module of the catalogue: neither", "mesh-catalog",
|
||||
[]string{"modules/gitea/index.ts"}, "gitea"},
|
||||
} {
|
||||
r, got := planMerge(t, c.repo, c.paths, entries, read, edges)
|
||||
if got != c.want {
|
||||
t.Errorf("%s: planned %q, wanted %q (as today; issue 338)", c.what, got, c.want)
|
||||
}
|
||||
if c.repo == "mesh-controller" && strings.Join(r.Unread, ",") != "" {
|
||||
t.Errorf("%s: a root-built module reads every file, and %v were said unread", c.what, r.Unread)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sharedRepositoryEdges is what dependenciesOf derives for the catalogue of the test above, sorted as it
|
||||
// sorts them.
|
||||
var sharedRepositoryEdges = []inventory.Edge{
|
||||
dep("build-agent", inventory.EdgePackages, "mesh-controller"),
|
||||
dep("build-agent", inventory.EdgeWorkerOf, "mesh-controller"),
|
||||
dep("gitea", inventory.EdgeBuiltBy, "build-agent"),
|
||||
dep("mesh-controller", inventory.EdgeBuiltBy, "build-agent"),
|
||||
dep("route-proxy", inventory.EdgeBuiltBy, "build-agent"),
|
||||
dep("route-proxy", inventory.EdgePackages, "mesh-controller"),
|
||||
}
|
||||
|
||||
// **The planner's invariant, over random catalogues.** For any catalogue whose dependencies form no cycle
|
||||
// and any set of changed files in one repository:
|
||||
//
|
||||
// - the plan is exactly the modules of that repository whose directory holds a changed file (every file,
|
||||
// for a module built from the root), and everything reachable from them along stands-on, declared and
|
||||
// packages — never along built-by or worker-of;
|
||||
// - every stands-on, declared, built-by and worker-of edge with both ends in the plan has the module
|
||||
// depended on in an earlier tier;
|
||||
// - no cycle is said.
|
||||
//
|
||||
// Seeded, so a failure is replayed by its seed and case.
|
||||
func TestAPlanIsTheTouchedModulesAndWhatIsReachableAlongTheWideningEdges(t *testing.T) {
|
||||
kinds := []string{inventory.EdgeStandsOn, inventory.EdgeDeclared, inventory.EdgePackages,
|
||||
inventory.EdgeBuiltBy, inventory.EdgeWorkerOf}
|
||||
widens := map[string]bool{inventory.EdgeStandsOn: true, inventory.EdgeDeclared: true, inventory.EdgePackages: true}
|
||||
orders := map[string]bool{inventory.EdgeStandsOn: true, inventory.EdgeDeclared: true,
|
||||
inventory.EdgeBuiltBy: true, inventory.EdgeWorkerOf: true}
|
||||
// Directory names drawn from one pool, so two repositories hold directories of the same name, and one
|
||||
// is a prefix of another.
|
||||
dirs := []string{"a", "ab", "b", "c", "lib/x", "lib/y", "modules/a", "modules/a/sub"}
|
||||
files := []string{"README.md", "merge-check.sh", "lib/z.go", "docs/x.md", "modules/README.md"}
|
||||
|
||||
falseCycles, firstFalseCycle := 0, ""
|
||||
for _, seed := range []uint64{1, 2, 3, 0x338, 0x162} {
|
||||
rng := rand.New(rand.NewPCG(seed, seed^0x9e3779b97f4a7c15))
|
||||
for n := 0; n < 100; n++ {
|
||||
repos := 1 + rng.IntN(3)
|
||||
repoName := func(i int) string { return fmt.Sprintf("r%d", i) }
|
||||
count := 1 + rng.IntN(12)
|
||||
var entries []inventory.Entry
|
||||
repoOf, dirOf := map[string]int{}, map[string]string{}
|
||||
for i := 0; i < count; i++ {
|
||||
name := fmt.Sprintf("m%02d", i)
|
||||
repo := rng.IntN(repos)
|
||||
dir := dirs[rng.IntN(len(dirs))]
|
||||
if rng.IntN(12) == 0 {
|
||||
dir = "" // built from the repository's root
|
||||
}
|
||||
repoOf[name], dirOf[name] = repo, dir
|
||||
entries = append(entries, fromRepo(name, "http://forge.internal:20000/novox/"+repoName(repo)+".git", dir))
|
||||
}
|
||||
// A graph with no cycle: a module depends only on modules made before it.
|
||||
var edges []inventory.Edge
|
||||
for i := 1; i < count; i++ {
|
||||
for j := 0; j < i; j++ {
|
||||
if rng.IntN(4) == 0 {
|
||||
edges = append(edges, dep(fmt.Sprintf("m%02d", i), kinds[rng.IntN(len(kinds))], fmt.Sprintf("m%02d", j)))
|
||||
}
|
||||
}
|
||||
}
|
||||
merged := rng.IntN(repos)
|
||||
var paths []string
|
||||
for k := 1 + rng.IntN(4); k > 0; k-- {
|
||||
if rng.IntN(3) == 0 {
|
||||
paths = append(paths, files[rng.IntN(len(files))])
|
||||
} else {
|
||||
paths = append(paths, dirs[rng.IntN(len(dirs))]+"/f.go")
|
||||
}
|
||||
}
|
||||
|
||||
// What the rules say.
|
||||
want := map[string]bool{}
|
||||
for _, e := range entries {
|
||||
name := e.Manifest.Module
|
||||
if repoOf[name] != merged {
|
||||
continue
|
||||
}
|
||||
for _, p := range paths {
|
||||
if d := dirOf[name]; d == "" || p == d || strings.HasPrefix(p, d+"/") {
|
||||
want[name] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
for grew := true; grew; {
|
||||
grew = false
|
||||
for _, e := range edges {
|
||||
if widens[e.Kind] && want[e.To] && !want[e.From] {
|
||||
want[e.From], grew = true, true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
r, _ := planMerge(t, repoName(merged), paths, entries, nil, edges)
|
||||
got := map[string]bool{}
|
||||
tierOf := map[string]int{}
|
||||
for i, tier := range r.Plan.Tiers {
|
||||
for _, name := range tier {
|
||||
got[name], tierOf[name] = true, i
|
||||
}
|
||||
}
|
||||
replay := func() string {
|
||||
return fmt.Sprintf("seed %#x case %d: repository %s, files %v\n modules %v\n edges %v\n tiers %v",
|
||||
seed, n, repoName(merged), paths, describe(entries), edges, r.Plan.Tiers)
|
||||
}
|
||||
if !sameSet(got, want) {
|
||||
t.Fatalf("planned %v, wanted %v\n%s", keys(got), keys(want), replay())
|
||||
}
|
||||
for _, e := range edges {
|
||||
if orders[e.Kind] && got[e.From] && got[e.To] && tierOf[e.To] >= tierOf[e.From] {
|
||||
t.Fatalf("%s %s %s, and %s is in tier %d, not before %s's %d\n%s", e.From, e.Kind, e.To,
|
||||
e.To, tierOf[e.To], e.From, tierOf[e.From], replay())
|
||||
}
|
||||
}
|
||||
if hasCycle(r.Plan.Tiers, edges) {
|
||||
falseCycles++
|
||||
if firstFalseCycle == "" {
|
||||
firstFalseCycle = replay()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Said once, after the other invariants have run over every case, so it hides none of them (hasCycle
|
||||
// counted a packages edge on main at 8170fc5: 19 of these 500 cases).
|
||||
if falseCycles > 0 {
|
||||
t.Errorf("a cycle said of a graph with none in %d of 500 cases; "+
|
||||
"the first:\n%s", falseCycles, firstFalseCycle)
|
||||
}
|
||||
}
|
||||
|
||||
func sameSet(a, b map[string]bool) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for k := range a {
|
||||
if !b[k] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func keys(m map[string]bool) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func describe(entries []inventory.Entry) []string {
|
||||
var out []string
|
||||
for _, e := range entries {
|
||||
out = append(out, fmt.Sprintf("%s@%s:%q", e.Manifest.Module,
|
||||
strings.TrimSuffix(strings.TrimPrefix(e.Source.Repository, "http://forge.internal:20000/novox/"), ".git"),
|
||||
e.Source.Path))
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1,194 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The self-check's probe of who can become root where the trusted parties run (novox/hq ADR 0259 §8, rule 3).
|
||||
//
|
||||
// The router and every channel proving its sender run as accounts of their own, so that no agent reads what
|
||||
// they hold or speaks as them. **Root on their machine undoes all of it.** So on every machine where a module
|
||||
// of its own account runs, this probe asks two questions, and raises an urgent condition while either is yes:
|
||||
//
|
||||
// 1. can an account an agent runs as become root without a person there — passwordless sudo, or a group
|
||||
// that is root by another name (docker, disk)? The agent's account is the one the coding-agent module on
|
||||
// that machine names (`agent_account`), and the operator's account while it names none;
|
||||
// 2. can an agent run a command it chooses, through the mesh's own tools, as an account that can — the login
|
||||
// shell's `execute` runs as the machine's runtime account, which the mesh's acting tools give passwordless
|
||||
// sudo?
|
||||
//
|
||||
// The measurement is the sudo module's on that machine (`sudo_escalation`); a machine it does not run on, or
|
||||
// that does not answer, is a probe that could not run — said, never taken for "no".
|
||||
|
||||
// kindAgentRoot is the condition an agent able to become root where a trusted party runs raises.
|
||||
const kindAgentRoot = "agent-root"
|
||||
|
||||
// The tools the probe asks, of the modules on each machine.
|
||||
const (
|
||||
agentModule = "claude-code"
|
||||
agentStatusTool = "claude_code_status"
|
||||
sudoModule = "sudo"
|
||||
sudoEscalation = "sudo_escalation"
|
||||
loginShellSeat = "node-login-shell"
|
||||
)
|
||||
|
||||
// escalation is the sudo module's answer for one account.
|
||||
type escalation struct {
|
||||
Account string `json:"account"`
|
||||
Root bool `json:"root_without_a_person"`
|
||||
Why string `json:"why"`
|
||||
}
|
||||
|
||||
// agentRootFacts is what one machine says, as the probe reads it.
|
||||
type agentRootFacts struct {
|
||||
Machine string
|
||||
Trusted []string // the modules of their own account on it
|
||||
Agent string // the account agents run as there; "" when none run there
|
||||
AgentNamed bool // the coding-agent module named it, rather than it being taken for the operator's
|
||||
Runtime string // the account the machine's runtime runs as
|
||||
LoginShell bool // the login shell seat is held there, running commands as the runtime's account
|
||||
Answers map[string]escalation
|
||||
}
|
||||
|
||||
// agentRootObservations judges one machine's facts: an urgent condition while an agent can become root there
|
||||
// without a person, one way or the other, naming which.
|
||||
func agentRootObservations(f agentRootFacts) []conditions.Observation {
|
||||
var ways []string
|
||||
if f.Agent != "" {
|
||||
if e := f.Answers[f.Agent]; e.Root {
|
||||
named := "the operator's account, which agents run as while the coding-agent module names no other"
|
||||
if f.AgentNamed {
|
||||
named = "the account agents run as"
|
||||
}
|
||||
ways = append(ways, fmt.Sprintf("%s (%s) can become root without a person: %s", f.Agent, named, e.Why))
|
||||
}
|
||||
}
|
||||
if f.LoginShell && f.Runtime != "" {
|
||||
if e := f.Answers[f.Runtime]; e.Root {
|
||||
ways = append(ways, fmt.Sprintf("the login shell runs any command an agent gives it as %s, which can "+
|
||||
"become root without a person: %s", f.Runtime, e.Why))
|
||||
}
|
||||
}
|
||||
if len(ways) == 0 {
|
||||
return nil
|
||||
}
|
||||
sort.Strings(f.Trusted)
|
||||
return []conditions.Observation{{Scope: conditions.ScopeMachine, ID: f.Machine, Token: kindAgentRoot,
|
||||
Machine: f.Machine, Kind: kindAgentRoot, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("an agent can become root on %s without a person, where %s run as accounts of their "+
|
||||
"own; until it cannot, the router approves nothing proven there (novox/hq ADR 0259 §8): %s",
|
||||
f.Machine, strings.Join(f.Trusted, ", "), strings.Join(ways, "; ")),
|
||||
Headline: "Root without you on " + f.Machine,
|
||||
Needs: "choose how programs working for you on " + f.Machine + " stop becoming root without asking you.",
|
||||
Explanation: "The modules that prove your answers from your phone run on " + f.Machine + ", and a program " +
|
||||
"working for you there can become root without asking you, so it could answer in your name. Until " +
|
||||
"that changes, answers from your phone can only acknowledge.",
|
||||
Resolved: "Nothing on " + f.Machine + " becomes root without you any more"}}
|
||||
}
|
||||
|
||||
// probeAgentRoot is the probe: every machine a module of its own account runs on, judged.
|
||||
func probeAgentRoot(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
inv := d.open.inventory
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
facts := map[string]*agentRootFacts{}
|
||||
agentOn, sudoOn := map[string]bool{}, map[string]bool{}
|
||||
for _, e := range entries {
|
||||
for _, node := range e.On {
|
||||
switch {
|
||||
case e.Manifest.RunsAs != "":
|
||||
f := facts[node]
|
||||
if f == nil {
|
||||
f = &agentRootFacts{Machine: node, Answers: map[string]escalation{}}
|
||||
facts[node] = f
|
||||
}
|
||||
f.Trusted = append(f.Trusted, e.Manifest.Module)
|
||||
case e.Manifest.Module == agentModule:
|
||||
agentOn[node] = true
|
||||
case e.Manifest.Module == sudoModule:
|
||||
sudoOn[node] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Manifest.ClaimsSeat(loginShellSeat) {
|
||||
for _, node := range e.On {
|
||||
if f := facts[node]; f != nil {
|
||||
f.LoginShell = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
machines := make([]string, 0, len(facts))
|
||||
for m := range facts {
|
||||
machines = append(machines, m)
|
||||
}
|
||||
sort.Strings(machines)
|
||||
var out []conditions.Observation
|
||||
var unread []string
|
||||
for _, m := range machines {
|
||||
f := facts[m]
|
||||
record, err := inv.NodeByName(ctx, m)
|
||||
if err != nil {
|
||||
unread = append(unread, m+": "+err.Error())
|
||||
continue
|
||||
}
|
||||
f.Runtime = record.Account
|
||||
if agentOn[m] {
|
||||
f.Agent = record.Account
|
||||
if a, err := link.AskModuleToolOn(ctx, d.js.Conn(), agentModule, agentStatusTool, m, map[string]any{}, 10*time.Second); err == nil && a.Error == "" {
|
||||
var status struct {
|
||||
AgentAccount string `json:"agent_account"`
|
||||
}
|
||||
if json.Unmarshal(a.Result, &status) == nil && status.AgentAccount != "" {
|
||||
f.Agent, f.AgentNamed = status.AgentAccount, true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !sudoOn[m] {
|
||||
unread = append(unread, m+": the sudo module is not assigned there, so who can become root is not measured")
|
||||
continue
|
||||
}
|
||||
var accounts []string
|
||||
for _, a := range []string{f.Agent, f.Runtime} {
|
||||
if a != "" && !slices.Contains(accounts, a) {
|
||||
accounts = append(accounts, a)
|
||||
}
|
||||
}
|
||||
if len(accounts) == 0 {
|
||||
continue
|
||||
}
|
||||
a, err := link.AskModuleToolOn(ctx, d.js.Conn(), sudoModule, sudoEscalation, m, map[string]any{"accounts": accounts}, 15*time.Second)
|
||||
if err == nil && a.Error != "" {
|
||||
err = fmt.Errorf("%s", a.Error)
|
||||
}
|
||||
if err != nil {
|
||||
unread = append(unread, m+": "+err.Error())
|
||||
continue
|
||||
}
|
||||
var answers []escalation
|
||||
if err := json.Unmarshal(a.Result, &answers); err != nil {
|
||||
unread = append(unread, m+": the sudo module's answer could not be read: "+err.Error())
|
||||
continue
|
||||
}
|
||||
for _, e := range answers {
|
||||
f.Answers[e.Account] = e
|
||||
}
|
||||
out = append(out, agentRootObservations(*f)...)
|
||||
}
|
||||
if len(unread) > 0 {
|
||||
return out, fmt.Errorf("who can become root could not be measured: %s", strings.Join(unread, "; "))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0259 §8, rule 3: the probe fails today — agents run as the operator's account, which has
|
||||
// passwordless sudo, and the login shell runs their commands as it — and passes only once neither holds.
|
||||
func TestAnAgentAbleToBecomeRootWhereTheRouterRunsIsSaid(t *testing.T) {
|
||||
root := escalation{Root: true, Why: "(ALL : ALL) NOPASSWD: ALL"}
|
||||
none := escalation{Why: "no rule lets it without a password"}
|
||||
base := func() agentRootFacts {
|
||||
return agentRootFacts{Machine: "anchor", Trusted: []string{"telegram", "messenger"}, Runtime: "ops",
|
||||
Answers: map[string]escalation{}}
|
||||
}
|
||||
|
||||
today := base()
|
||||
today.Agent, today.LoginShell = "ops", true
|
||||
today.Answers["ops"] = root
|
||||
got := agentRootObservations(today)
|
||||
if len(got) != 1 || got[0].Severity != conditions.Urgent || got[0].Kind != kindAgentRoot ||
|
||||
!strings.Contains(got[0].Summary, "the operator's account") || !strings.Contains(got[0].Summary, "the login shell") {
|
||||
t.Fatalf("today: %+v", got)
|
||||
}
|
||||
|
||||
agentsMoved := base()
|
||||
agentsMoved.Agent, agentsMoved.AgentNamed, agentsMoved.LoginShell = "agents", true, true
|
||||
agentsMoved.Answers["agents"], agentsMoved.Answers["ops"] = none, root
|
||||
if got := agentRootObservations(agentsMoved); len(got) != 1 || strings.Contains(got[0].Summary, "agents (") ||
|
||||
!strings.Contains(got[0].Summary, "the login shell") {
|
||||
t.Errorf("agents of their own account, the login shell still the runtime's: %+v", got)
|
||||
}
|
||||
|
||||
closed := base()
|
||||
closed.Agent, closed.AgentNamed = "agents", true
|
||||
closed.Answers["agents"], closed.Answers["ops"] = none, root
|
||||
if got := agentRootObservations(closed); len(got) != 0 {
|
||||
t.Errorf("agents of their own account and no login shell there: %+v", got)
|
||||
}
|
||||
|
||||
noAgents := base()
|
||||
noAgents.Answers["ops"] = root
|
||||
if got := agentRootObservations(noAgents); len(got) != 0 {
|
||||
t.Errorf("no agent runs there and no login shell is held: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Its words are plain, as every condition's are (novox/hq ADR 0253).
|
||||
func TestTheRootConditionIsSaidInPlainWords(t *testing.T) {
|
||||
f := agentRootFacts{Machine: "anchor", Trusted: []string{"telegram"}, Runtime: "ops", Agent: "ops",
|
||||
Answers: map[string]escalation{"ops": {Root: true, Why: "x"}}}
|
||||
o := agentRootObservations(f)[0]
|
||||
if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Explanation: o.Explanation,
|
||||
Needs: o.Needs, Resolved: o.Resolved}, "anchor"); !ok {
|
||||
t.Errorf("not plain: %s", why)
|
||||
}
|
||||
}
|
||||
@@ -875,6 +875,16 @@ func streamDiffers(want broker.Stream, have nats.StreamConfig) string {
|
||||
if perSubject != 0 && have.MaxMsgsPerSubject != perSubject {
|
||||
differs = append(differs, fmt.Sprintf("keeps %d per subject, defined %d", have.MaxMsgsPerSubject, perSubject))
|
||||
}
|
||||
if want.MaxBytes > 0 && have.MaxBytes != want.MaxBytes {
|
||||
differs = append(differs, fmt.Sprintf("holds up to %d bytes, defined %d", have.MaxBytes, want.MaxBytes))
|
||||
}
|
||||
if want.DuplicatesSeconds > 0 && have.Duplicates != time.Duration(want.DuplicatesSeconds)*time.Second {
|
||||
differs = append(differs, fmt.Sprintf("keeps one of a message id for %s, defined %s", have.Duplicates,
|
||||
time.Duration(want.DuplicatesSeconds)*time.Second))
|
||||
}
|
||||
if want.DiscardNew && have.Discard != nats.DiscardNew {
|
||||
differs = append(differs, "drops what it holds when full, defined to refuse what comes next")
|
||||
}
|
||||
return strings.Join(differs, "; ")
|
||||
}
|
||||
|
||||
|
||||
@@ -219,6 +219,10 @@ func serve(ctx context.Context) (err error) {
|
||||
}
|
||||
// The hand-act log is counted for `status` on this connection rather than a new one a minute.
|
||||
handActConn = bus.Conn
|
||||
// And everything else this controller does on the bus for a moment (novox/hq issue 327).
|
||||
servingBus.Store(server.JetStream())
|
||||
// No longer serving: nothing is lent, and dead-letters says it is not read here (novox/hq issue 330).
|
||||
defer servingBus.Store(nil)
|
||||
// And says when it replaced a value given by hand (novox/hq ADR 0228).
|
||||
givenEvents = bus
|
||||
// And a pull request's merge check, asked when the forge announces its head and said when judged
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -37,22 +38,21 @@ const (
|
||||
killAnswer = 75 * time.Second
|
||||
)
|
||||
|
||||
// dialTheBus opens the controller's own connection, for a command that reads or changes the queue.
|
||||
// dialTheBus is the controller's connection, for a command that reads or changes the queue: the serving
|
||||
// controller's own, lent, when this process is it (novox/hq issue 327).
|
||||
func dialTheBus() (*broker.JetStream, error) {
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot reach the bus: %w", err)
|
||||
}
|
||||
return js, nil
|
||||
return aBus()
|
||||
}
|
||||
|
||||
// queueCommand prints every ask in the build seat's work queue.
|
||||
func queueCommand(ctx context.Context, args []string) error {
|
||||
return listQueue(ctx, args, os.Stdout)
|
||||
}
|
||||
|
||||
// listQueue is `queue`: the build queue, as a person reads it or as JSON.
|
||||
func listQueue(ctx context.Context, args []string, w io.Writer) error {
|
||||
set := flag.NewFlagSet("queue", flag.ContinueOnError)
|
||||
usageTo(set, w)
|
||||
asJSON := set.Bool("json", false, "the queue as JSON")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
@@ -72,10 +72,10 @@ func queueCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
fmt.Fprintln(w, string(body))
|
||||
return nil
|
||||
}
|
||||
fmt.Print(queueText(q, time.Now()))
|
||||
fmt.Fprint(w, queueText(q, time.Now()))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// D15: no client of the bus reconnects in a loop (novox/hq issue 327).
|
||||
//
|
||||
// The server's connection total was the one number that would show a client reconnecting, and every verb
|
||||
// the controller served opened and closed a connection of its own, hundreds an hour, so a loop was
|
||||
// invisible in it. The bus's own module reads the server's record of closed connections
|
||||
// (`nats_closed_connections`); this asks it every run, and says each user whose connections were dropped —
|
||||
// closed by anything but the client itself: a read or write error, a stale connection, a slow consumer, a
|
||||
// refused login — more often than reconnectBound in the last hour.
|
||||
|
||||
const (
|
||||
probeReconnectsID = "D15"
|
||||
kindBusReconnects = "bus-reconnects"
|
||||
// reconnectBound is how many dropped connections in an hour one user may have before it is said:
|
||||
// a client that loses its connection every five minutes. Provisional.
|
||||
reconnectBound = 12
|
||||
// busModule is the module that is the bus, and closedTool its tool that reads closed connections.
|
||||
busModule = "nats"
|
||||
closedTool = "nats_closed_connections"
|
||||
closedAsk = 20 * time.Second
|
||||
)
|
||||
|
||||
// closedConnections is what the bus's module answers.
|
||||
type closedConnections struct {
|
||||
Hours float64 `json:"hours"`
|
||||
Reaches bool `json:"reaches"`
|
||||
Users []struct {
|
||||
User string `json:"user"`
|
||||
Closed int `json:"closed"`
|
||||
Dropped int `json:"dropped"`
|
||||
DroppedPerHour float64 `json:"dropped_per_hour"`
|
||||
Names []struct {
|
||||
Name string `json:"name"`
|
||||
Closed int `json:"closed"`
|
||||
Dropped int `json:"dropped"`
|
||||
Reasons map[string]int `json:"reasons"`
|
||||
} `json:"names"`
|
||||
} `json:"users"`
|
||||
}
|
||||
|
||||
// probeReconnects is D15.
|
||||
func probeReconnects(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
if d.js == nil {
|
||||
return nil, fmt.Errorf("no bus to ask the bus's module over")
|
||||
}
|
||||
on, err := d.open.inventory.Running(ctx, busModule)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(on) == 0 {
|
||||
return nil, nil // no bus module assigned: a mesh whose bus is not the mesh's module
|
||||
}
|
||||
return reconnectsOn(ctx, d.js.Conn(), on[0])
|
||||
}
|
||||
|
||||
// reconnectsOn asks the bus module on its machine and says who reconnects in a loop.
|
||||
func reconnectsOn(ctx context.Context, conn *nats.Conn, node string) ([]conditions.Observation, error) {
|
||||
read, err := askClosed(ctx, conn, node)
|
||||
if isNothingServes(err) {
|
||||
// A bus module older than its tool has nothing it can say, which is not a failure of the probe.
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return reconnecting(read), nil
|
||||
}
|
||||
|
||||
// askClosed asks the bus's module, on its machine, who closed connections in the last hour.
|
||||
func askClosed(ctx context.Context, conn *nats.Conn, node string) (closedConnections, error) {
|
||||
var read closedConnections
|
||||
answer, err := link.AskModuleToolOn(ctx, conn, busModule, closedTool, node, map[string]any{"hours": 1}, closedAsk)
|
||||
if err != nil {
|
||||
return read, err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return read, fmt.Errorf("%s on %s answered %s with an error: %s", busModule, node, closedTool, answer.Error)
|
||||
}
|
||||
if err := unmarshalAnswer(answer, &read); err != nil {
|
||||
return read, fmt.Errorf("%s on %s answered %s with something unreadable: %w", busModule, node, closedTool, err)
|
||||
}
|
||||
return read, nil
|
||||
}
|
||||
|
||||
// reconnecting is one observation per user whose connections were dropped more than reconnectBound times
|
||||
// an hour.
|
||||
func reconnecting(read closedConnections) []conditions.Observation {
|
||||
hours := read.Hours
|
||||
if hours <= 0 {
|
||||
hours = 1
|
||||
}
|
||||
var out []conditions.Observation
|
||||
for _, u := range read.Users {
|
||||
if u.User == "" || strings.HasPrefix(u.User, "(") {
|
||||
// Refused before it logged in: no client of the mesh's, so nobody's reconnect loop. A login
|
||||
// refused again and again is a question of its own, not this probe's.
|
||||
continue
|
||||
}
|
||||
perHour := float64(u.Dropped) / hours
|
||||
if perHour <= reconnectBound {
|
||||
continue
|
||||
}
|
||||
reasons := map[string]int{}
|
||||
var names []string
|
||||
for _, n := range u.Names {
|
||||
if n.Dropped == 0 {
|
||||
continue
|
||||
}
|
||||
names = append(names, fmt.Sprintf("%q ×%d", n.Name, n.Dropped))
|
||||
for r, c := range n.Reasons {
|
||||
if r != "Client Closed" {
|
||||
reasons[r] += c
|
||||
}
|
||||
}
|
||||
}
|
||||
var why []string
|
||||
for r, c := range reasons {
|
||||
why = append(why, fmt.Sprintf("%s ×%d", r, c))
|
||||
}
|
||||
sort.Strings(why)
|
||||
partial := ""
|
||||
if !read.Reaches {
|
||||
partial = " (at least: the server's record of closed connections does not reach back the whole hour)"
|
||||
}
|
||||
who, machine := busUserWords(u.User)
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: u.User, Kind: kindBusReconnects,
|
||||
Machine: machine, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the bus dropped %s's connection %d times in the last hour%s, more than %d: a "+
|
||||
"client reconnecting in a loop", u.User, u.Dropped, partial, reconnectBound),
|
||||
Said: fmt.Sprintf("%d of %d closed connections dropped in %.0f h; by name %s; why %s", u.Dropped,
|
||||
u.Closed, hours, strings.Join(names, ", "), strings.Join(why, ", ")),
|
||||
Headline: clip(conditions.Capital(who)+" keeps losing the bus", 60),
|
||||
Explanation: conditions.Capital(fmt.Sprintf("%s lost its connection to the bus %d times in the last hour "+
|
||||
"and connected again each time. While it reconnects, what it says and what it is asked waits.", who,
|
||||
u.Dropped)),
|
||||
Resolved: "Resolved: " + who + " stays connected"})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// busUserWords is a bus user as the operator says it, and the machine it is on: `node.<machine>` the
|
||||
// node-engine, `<machine>.node-tools` the tool runner, `controller` the controller, `<machine>.<module>` a
|
||||
// module.
|
||||
func busUserWords(user string) (string, string) {
|
||||
switch {
|
||||
case user == "controller":
|
||||
return "the controller", ""
|
||||
case strings.HasPrefix(user, "node."):
|
||||
m := strings.TrimPrefix(user, "node.")
|
||||
return "the node-engine on " + m, m
|
||||
}
|
||||
if m, module, ok := strings.Cut(user, "."); ok {
|
||||
if module == "node-tools" {
|
||||
return "the tool runner on " + m, m
|
||||
}
|
||||
return module + " on " + m, m
|
||||
}
|
||||
return "a client of the bus", ""
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// The serving controller's own connection serves what it does for a moment: no connection is opened,
|
||||
// and closing what it was lent leaves the serving one open (novox/hq issue 327).
|
||||
func TestTheServingControllerLendsItsOwnConnection(t *testing.T) {
|
||||
bus := testbus.Start(t)
|
||||
serving, err := broker.Dial(bus.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer serving.Close()
|
||||
if err := serving.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
was := servingBus.Load()
|
||||
servingBus.Store(serving)
|
||||
defer servingBus.Store(was)
|
||||
t.Setenv(broker.NATSVar, bus.ClientURL())
|
||||
|
||||
before, _ := bus.Varz(nil)
|
||||
lent, err := aBus()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lent.Close()
|
||||
if !serving.Conn().IsConnected() {
|
||||
t.Fatal("closing a lent connection closed the serving controller's")
|
||||
}
|
||||
if err := onTheBus(func(*nats.Conn) error { return nil }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handlers, _, err := seatToolHandlers()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
answer, err := handlers["hand-acts"](context.Background(), json.RawMessage(`{}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a := answer.(verbAnswer); !a.OK || a.Answer == nil {
|
||||
t.Fatalf("hand-acts answered %+v", a)
|
||||
}
|
||||
_, _ = handlers["queue"](context.Background(), json.RawMessage(`{}`))
|
||||
after, _ := bus.Varz(nil)
|
||||
if opened := after.TotalConnections - before.TotalConnections; opened != 0 {
|
||||
t.Fatalf("the serving controller opened %d connection(s) of its own", opened)
|
||||
}
|
||||
}
|
||||
|
||||
// A verb that still runs as a process of its own says which in the bus's list of connections.
|
||||
func TestAVerbsOwnProcessNamesItsConnection(t *testing.T) {
|
||||
bus := testbus.Start(t)
|
||||
setup, err := broker.Dial(bus.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := setup.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
setup.Close()
|
||||
asAProcess(t, bus.ClientURL())
|
||||
handlers, _, err := seatToolHandlers()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A silence acts, so it is still its own process; it dials, and finds no such condition.
|
||||
_, _ = handlers["conditions"](context.Background(),
|
||||
json.RawMessage(`{"silence":"bus.nothing.here","for":"1h","why":"a test"}`))
|
||||
names := closedNames(t, bus)
|
||||
found := false
|
||||
for _, n := range names {
|
||||
found = found || n == "mesh-controller verb conditions"
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("the verb's connection was named %q", names)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEachProcessNamesItsConnectionsForWhatItIs(t *testing.T) {
|
||||
for _, c := range []struct{ command, verb, want string }{
|
||||
{"serve", "", "mesh-controller serving"},
|
||||
{"conditions", "conditions", "mesh-controller verb conditions"},
|
||||
{"delivery", "delivery-check", "mesh-controller verb delivery-check"},
|
||||
{"push", "", "mesh-controller command push"},
|
||||
} {
|
||||
if got := connectionName(c.command, c.verb); got != c.want {
|
||||
t.Errorf("%s/%s: %q", c.command, c.verb, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// D15: a user whose connections are dropped more than twelve times an hour is said, in plain words; one
|
||||
// whose client closes its own short connections is not.
|
||||
func TestAClientReconnectingInALoopIsSaid(t *testing.T) {
|
||||
var read closedConnections
|
||||
if err := json.Unmarshal([]byte(`{"hours":1,"reaches":true,"users":[
|
||||
{"user":"ace.node-tools","closed":40,"dropped":40,"dropped_per_hour":40,"names":[
|
||||
{"name":"ace.node-tools","closed":40,"dropped":40,"reasons":{"Stale Connection":30,"Read Error":10}}]},
|
||||
{"user":"controller","closed":300,"dropped":0,"names":[
|
||||
{"name":"mesh-controller verb delivery-check","closed":300,"dropped":0,"reasons":{"Client Closed":300}}]},
|
||||
{"user":"(no user: refused before it logged in)","closed":90,"dropped":90,"names":[{"name":"","closed":90,
|
||||
"dropped":90,"reasons":{"Authentication Failure":90}}]},
|
||||
{"user":"node.anchor","closed":5,"dropped":5,"names":[{"name":"mesh-host/anchor","closed":5,"dropped":5,
|
||||
"reasons":{"Read Error":5}}]}]}`), &read); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found := reconnecting(read)
|
||||
if len(found) != 1 {
|
||||
t.Fatalf("%+v", found)
|
||||
}
|
||||
o := found[0]
|
||||
if o.ID != "ace.node-tools" || o.Machine != "ace" || o.Kind != kindBusReconnects ||
|
||||
o.Headline != "The tool runner on ace keeps losing the bus" || !strings.Contains(o.Said, "Stale Connection ×30") {
|
||||
t.Fatalf("%+v", o)
|
||||
}
|
||||
if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Explanation: o.Explanation,
|
||||
Resolved: o.Resolved}, "ace"); !ok {
|
||||
t.Fatalf("not plain: %s", why)
|
||||
}
|
||||
}
|
||||
|
||||
// The bus's module is asked on its machine, over the bus.
|
||||
func TestTheBusModuleIsAskedWhoClosedConnections(t *testing.T) {
|
||||
bus := testbus.Start(t)
|
||||
conn, err := nats.Connect(bus.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
sub, err := conn.Subscribe(link.ModuleToolOn("nats", "nats_closed_connections", "anchor"), func(m *nats.Msg) {
|
||||
_ = m.Respond([]byte(`{"result":{"hours":1,"reaches":true,"users":[{"user":"controller","closed":2,"dropped":0}]}}`))
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
read, err := askClosed(context.Background(), conn, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(read.Users) != 1 || read.Users[0].Closed != 2 {
|
||||
t.Fatalf("%+v", read)
|
||||
}
|
||||
}
|
||||
|
||||
// A bus module older than the tool answers nothing to the question: the probe passes over it quietly.
|
||||
func TestAnOlderBusModuleIsPassedOverQuietly(t *testing.T) {
|
||||
bus := testbus.Start(t)
|
||||
conn, err := nats.Connect(bus.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
found, err := reconnectsOn(context.Background(), conn, "anchor")
|
||||
if err != nil || len(found) != 0 {
|
||||
t.Fatalf("a bus module without the tool: %v %v", found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A verb answered in the serving controller says its flag errors in its answer, not in the controller's log.
|
||||
func TestAFlagErrorIsSaidInTheAnswer(t *testing.T) {
|
||||
bus := testbus.Start(t)
|
||||
serving, err := broker.Dial(bus.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer serving.Close()
|
||||
was := servingBus.Load()
|
||||
servingBus.Store(serving)
|
||||
defer servingBus.Store(was)
|
||||
answer, read := readHere(context.Background(), []string{"hand-acts", "--bogus"})
|
||||
if !read || answer.OK || !strings.Contains(answer.Output, "flag provided but not defined") {
|
||||
t.Fatalf("answered %+v", answer)
|
||||
}
|
||||
}
|
||||
@@ -157,7 +157,9 @@ func reachableFrom(moved []string, edges []inventory.Edge) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// hasCycle says whether the tiers' last tier holds modules that still depend on each other.
|
||||
// hasCycle says whether the tiers' last tier holds modules that still depend on each other. A packages
|
||||
// edge orders nothing (tiersOf), so a module and what packages its source share a tier by rule: that is
|
||||
// no cycle, and saying one was is a false report in every such plan's log.
|
||||
func hasCycle(tiers [][]string, edges []inventory.Edge) bool {
|
||||
if len(tiers) == 0 {
|
||||
return false
|
||||
@@ -167,6 +169,9 @@ func hasCycle(tiers [][]string, edges []inventory.Edge) bool {
|
||||
last[m] = true
|
||||
}
|
||||
for _, e := range edges {
|
||||
if e.Kind == inventory.EdgePackages {
|
||||
continue
|
||||
}
|
||||
if last[e.From] && last[e.To] {
|
||||
return true
|
||||
}
|
||||
@@ -701,7 +706,6 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
}
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
step := nextRollout(*state, running, policy.Together, reports, now, planWaitBound)
|
||||
// **Sent with others, judged with them** (issue 281): the gate of the send that carried it is its
|
||||
// verdict on its first machine. A failure there stopped the plan already.
|
||||
if state.GatedBy != "" {
|
||||
@@ -715,6 +719,9 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
}
|
||||
passedWith(m, state, state.GatedBy, lead.Gate)
|
||||
}
|
||||
// Read after its pass is taken over from the send that carried it, so a passed gate is never judged
|
||||
// again from the first machine's later reports (novox/hq issue 335).
|
||||
step := nextRollout(*state, running, policy.Together, reports, now, planWaitBound)
|
||||
switch {
|
||||
case step.failed != "":
|
||||
// The first machine refused or failed what it was sent, or never said: the gate failed, and
|
||||
@@ -974,6 +981,19 @@ func failFirstSend(ctx context.Context, open *stores, p *inventory.Plan, module
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
}()
|
||||
g := state.Gate
|
||||
if g != nil && g.Verdict == inventory.GatePassed {
|
||||
// **A guard: a build that passed its gate is never put back for what came after** (novox/hq issue 335).
|
||||
// Not reached while nextRollout answers a passed gate with the rest to send, and advanceOnce reads it
|
||||
// after a carried module takes over its lead's pass; it is here so that a path added later cannot
|
||||
// overturn a verdict. If it is reached, the plan stops and says why, and the build is not marked failed.
|
||||
// The module is left a stopped rollout (its Why said, sent first and not to the rest), which
|
||||
// `plans retry` takes: it sends the first machine again, and the passed gate then sends the rest.
|
||||
state.Why = "passed its gate; its walk then stopped: " + why
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = fmt.Sprintf("%s passed its gate on %s in tier %d (%s) and is kept; its walk stopped after: %s",
|
||||
module, strings.Join(g.Machines, ", "), p.Tier, g.Why, why)
|
||||
return
|
||||
}
|
||||
if g != nil && slices.Contains(g.Returned, module) {
|
||||
// Put back at once when it broke: its rollback was made then, and is not made again.
|
||||
state.Why = "put back when it broke; its send failed: " + g.Why
|
||||
@@ -1061,6 +1081,15 @@ func nextRollout(s inventory.PlanModule, running []string, together bool, report
|
||||
rest = append(rest, n)
|
||||
}
|
||||
}
|
||||
// **A passed gate is the first machine's verdict, and its later reports are not** (novox/hq issue 335).
|
||||
// Once the build passed there, what that machine reports next is about whatever it was sent after —
|
||||
// another walk's send, a push — and says nothing of this build. On 2026-10-08 a build passed on the
|
||||
// laptop, its send to the rest waited on another walk, that walk sent the laptop a new declaration it did
|
||||
// not report for half an hour, and the plan read the silence as the first machine never applying the
|
||||
// passed build: it marked it failed at its gate and put it back. The rest are sent, as the pass said.
|
||||
if s.Gate != nil && s.Gate.Verdict == inventory.GatePassed {
|
||||
return rolloutStep{send: rest}
|
||||
}
|
||||
var waiting, failed []string
|
||||
for _, n := range s.First {
|
||||
r, said := byNode[n]
|
||||
|
||||
@@ -27,6 +27,8 @@ func TestAMergeIsPlannedInTiersAlongTheThreeKindsOfDependency(t *testing.T) {
|
||||
{From: "mesh-controller", To: "builder", Kind: inventory.EdgeBuiltBy},
|
||||
{From: "mesh-tools", To: "builder", Kind: inventory.EdgeBuiltBy},
|
||||
{From: "builder", To: "mesh-tools", Kind: inventory.EdgeStandsOn},
|
||||
// the build seat's holder follows the controller that defines its worker (hq issue 206)
|
||||
{From: "builder", To: "mesh-controller", Kind: inventory.EdgeWorkerOf},
|
||||
{From: "unrelated", To: "alpine", Kind: inventory.EdgeStandsOn},
|
||||
}
|
||||
// The runtime image moved: everything on it, and what is built by what is on it.
|
||||
@@ -62,12 +64,15 @@ func TestAMergeIsPlannedInTiersAlongTheThreeKindsOfDependency(t *testing.T) {
|
||||
if len(small) != 3 {
|
||||
t.Fatalf("a controller merge rebuilds the controller and what packages it: %v", small)
|
||||
}
|
||||
// The builder packages the controller's source (same tier by that edge) and the controller is
|
||||
// built by the builder (next tier by that one): the builder first, then the controller and the
|
||||
// proxy together — a code dependency in one tier, a runtime dependency across tiers.
|
||||
// The builder and the proxy package the controller's source, which orders nothing. The builder holds
|
||||
// the build seat, whose worker the controller defines, so it follows the controller (worker-of,
|
||||
// novox/hq issue 206), and the controller's built-by edge to it yields: the controller is built by the
|
||||
// build machine that is running. The proxy is built by the new builder: the controller, the builder,
|
||||
// the proxy — the live plan of every controller merge. (This read "the builder, then the controller
|
||||
// and the proxy together" before issue 206, and the fixture had no worker-of edge.)
|
||||
smallTiers := tiersOf(small, edges)
|
||||
if len(smallTiers) != 2 || smallTiers[0][0] != "builder" || len(smallTiers[1]) != 2 {
|
||||
t.Fatalf("the builder, then the controller and the proxy together: %v", smallTiers)
|
||||
if got := tiered(smallTiers); got != "mesh-controller | builder | route-proxy" {
|
||||
t.Fatalf("the controller, then the builder, then the proxy: %v", smallTiers)
|
||||
}
|
||||
// The builder alone moved: the builder, and nothing it builds.
|
||||
if only := reachableFrom([]string{"builder"}, edges); len(only) != 1 {
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// novox/hq issue 327, replayed with only what the controller had before its fix, so it can be laid over
|
||||
// the older commit. On 2026-10-08 the bus's connection total rose by 5.2 a minute while the same 16
|
||||
// connections stayed open, and three calls of `conditions` in one second added three: each verb ran as a
|
||||
// process of the controller's own binary, which dialled the bus, logged in and left. The operator's
|
||||
// channel reads `conditions` at least once a minute. A verb that only reads, served by the serving
|
||||
// controller, opens no connection of its own.
|
||||
func TestReplay327(t *testing.T) {
|
||||
bus := testbus.Start(t)
|
||||
serving, err := broker.Dial(bus.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer serving.Close()
|
||||
if err := serving.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
keeper, err := keeperOn(context.Background(), serving.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The serving controller: its keeper and its connection, as serve sets them.
|
||||
keptBefore, connBefore := conditionsFrom, handActConn
|
||||
conditionsFrom, handActConn = keeper, serving.Conn()
|
||||
defer func() { conditionsFrom, handActConn = keptBefore, connBefore }()
|
||||
// A verb that runs as a process of its own runs this controller's binary, on this bus.
|
||||
asAProcess(t, bus.ClientURL())
|
||||
|
||||
handlers, _, err := seatToolHandlers()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
accepted := func() uint64 {
|
||||
v, err := bus.Varz(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return v.TotalConnections
|
||||
}
|
||||
before := accepted()
|
||||
for i := 0; i < 3; i++ {
|
||||
answer, err := handlers["conditions"](context.Background(), json.RawMessage(`{}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a, ok := answer.(verbAnswer); !ok || !a.OK {
|
||||
t.Fatalf("conditions answered %+v", answer)
|
||||
}
|
||||
}
|
||||
if opened := accepted() - before; opened != 0 {
|
||||
t.Fatalf("three conditions calls opened %d connection(s) to the bus; the serving controller is on it already",
|
||||
opened)
|
||||
}
|
||||
}
|
||||
|
||||
// asAProcess makes a verb that runs as a process of its own run this package's binary, on the bus at url:
|
||||
// built into the test's own directory, which goes with the test.
|
||||
func asAProcess(t *testing.T, url string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "mesh-controller")
|
||||
if out, err := exec.Command("go", "build", "-o", path, ".").CombinedOutput(); err != nil {
|
||||
t.Fatalf("the controller could not be built to run a verb as its own process: %v: %s", err, out)
|
||||
}
|
||||
was := ownImage
|
||||
ownImage = func() string { return path }
|
||||
t.Cleanup(func() { ownImage = was })
|
||||
t.Setenv(broker.NATSVar, url)
|
||||
t.Setenv(broker.CertificateVar, "")
|
||||
}
|
||||
|
||||
// closedNames are the names of the connections the bus saw closed.
|
||||
func closedNames(t *testing.T, bus *server.Server) []string {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
var names []string
|
||||
for time.Now().Before(deadline) {
|
||||
connz, err := bus.Connz(&server.ConnzOptions{State: server.ConnClosed})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
names = names[:0]
|
||||
for _, c := range connz.Conns {
|
||||
names = append(names, c.Name)
|
||||
}
|
||||
if len(names) > 0 {
|
||||
return names
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
return names
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// TestReplay335 replays novox/hq issue 335 (2026-10-08): dunst's new build passed its gate on the laptop
|
||||
// (healthy 3 times over 2m5s); its send to the rest was refused while another walk's build waited on the
|
||||
// workstation; that walk then sent the laptop a declaration the laptop did not report on; and thirty minutes
|
||||
// after the first send the plan read the laptop's silence as the passed build never applied, marked it failed
|
||||
// at its gate with the pass's own words, and put it back. Written with only what the controller had before
|
||||
// its fix, so it is laid over the commit before.
|
||||
func TestReplay335(t *testing.T) {
|
||||
t.Run("the first machine's later reports", testAPassedGateIsNotJudgedAgainFromTheFirstMachinesLaterReports)
|
||||
t.Run("a walk stopped after the pass", testAWalkStoppedAfterItsGatePassedKeepsThePass)
|
||||
}
|
||||
|
||||
// novox/hq issue 335: a build that passed its gate on its first machine is not judged again from that
|
||||
// machine's later reports. On 2026-10-08 the send to the rest waited on another walk, that walk sent the
|
||||
// first machine a declaration it did not report for half an hour, and the plan failed the passed build at
|
||||
// the wait's bound and put it back.
|
||||
func testAPassedGateIsNotJudgedAgainFromTheFirstMachinesLaterReports(t *testing.T) {
|
||||
sentAt := time.Date(2026, 10, 8, 16, 44, 45, 0, time.UTC)
|
||||
judged := sentAt.Add(2 * time.Minute)
|
||||
later := sentAt.Add(5 * time.Minute)
|
||||
state := inventory.PlanModule{First: []string{"laptop"}, FirstAt: &sentAt,
|
||||
Gate: &inventory.PlanGate{Machines: []string{"laptop"}, Verdict: inventory.GatePassed,
|
||||
Why: "healthy 3 times over 2m5s", JudgedAt: &judged, Kept: true}}
|
||||
running := []string{"laptop", "workstation"}
|
||||
now := sentAt.Add(30*time.Minute + 9*time.Second)
|
||||
|
||||
for what, reports := range map[string][]inventory.Reported{
|
||||
"no report about what it was sent since": {{Node: "laptop", At: &later, Outcome: inventory.OutcomeApplied, Current: false}},
|
||||
"another send failed there since": {{Node: "laptop", At: &later, Outcome: inventory.OutcomeFailed, Current: true}},
|
||||
"no report at all": nil,
|
||||
} {
|
||||
step := nextRollout(state, running, false, reports, now, 30*time.Minute)
|
||||
if step.failed != "" || step.waiting != "" || !reflect.DeepEqual(step.send, []string{"workstation"}) {
|
||||
t.Errorf("%s: %+v, want the rest sent as the pass said", what, step)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq issue 335: whatever stops a walk after its gate passed, the passed build is not marked failed at
|
||||
// its gate, nor put back.
|
||||
func testAWalkStoppedAfterItsGatePassedKeepsThePass(t *testing.T) {
|
||||
sentAt := time.Date(2026, 10, 8, 16, 44, 45, 0, time.UTC)
|
||||
g := &inventory.PlanGate{Machines: []string{"laptop"}, Verdict: inventory.GatePassed, Why: "healthy 3 times over 2m5s"}
|
||||
state := &inventory.PlanModule{Build: "build-1", First: []string{"laptop"}, FirstAt: &sentAt, Gate: g}
|
||||
p := &inventory.Plan{ID: "plan-1", Modules: map[string]*inventory.PlanModule{"dunst": state}}
|
||||
// No stores: a walk that keeps the pass touches none, and one that reaches for them is putting it back.
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
t.Fatalf("the passed build was taken to be failed and put back: %v", r)
|
||||
}
|
||||
}()
|
||||
failFirstSend(t.Context(), nil, p, "dunst", state, []string{"laptop"}, "laptop did not report it applied within 30m0s",
|
||||
[]string{"workstation"})
|
||||
if g.Verdict != inventory.GatePassed || g.Rollback != "" {
|
||||
t.Fatalf("the passed gate became %q, rollback %q", g.Verdict, g.Rollback)
|
||||
}
|
||||
if strings.Contains(p.Note, "failed its gate") || strings.Contains(p.Note, "put back") ||
|
||||
!strings.Contains(p.Note, "did not report it applied") {
|
||||
t.Fatalf("the note reads %q", p.Note)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq issue 335 review: **a module carried by its lead's send takes over the lead's pass before its own
|
||||
// step is read.** The state is the one a step leaves when the lead's gate passed and the step ended before the
|
||||
// carried module's turn (an error read before it, kept with the plan): the lead passed, the carried module has
|
||||
// no gate of its own yet. Since then another send reached the first machine and it has not reported on it, and
|
||||
// the first send is older than the wait for a first machine's report. Read before the pass is taken over, the
|
||||
// carried module's step said the first machine never applied it, and the passed build was put back.
|
||||
func TestACarriedModuleTakesItsLeadsPassBeforeItsStepIsRead(t *testing.T) {
|
||||
tm := aTierMesh(t, "m01", "m02")
|
||||
ctx := t.Context()
|
||||
inv := tm.open.inventory
|
||||
|
||||
advancePlans(ctx, tm.open)
|
||||
if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) {
|
||||
t.Fatalf("sent %v: the first machine once, for the tier", tm.sent)
|
||||
}
|
||||
p := tm.plan(t)
|
||||
lead, carried := p.Modules["m01"], p.Modules["m02"]
|
||||
if lead.Gate == nil || carried.GatedBy != "m01" {
|
||||
t.Fatalf("m02 is not carried by m01's send: lead %+v, carried %+v", lead.Gate, carried)
|
||||
}
|
||||
// The lead passed; the carried module's turn did not come. The first send is past the wait's bound.
|
||||
sent := time.Now().UTC().Add(-planWaitBound - time.Minute)
|
||||
judged := sent.Add(2 * time.Minute)
|
||||
lead.FirstAt, carried.FirstAt, lead.Gate.Since = &sent, &sent, &sent
|
||||
lead.Gate.Verdict, lead.Gate.Why, lead.Gate.JudgedAt, lead.Gate.Kept = inventory.GatePassed,
|
||||
"healthy 3 times over 2m5s", &judged, true
|
||||
carried.Gate = nil
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Another walk's send reached anchor, which has not reported on it.
|
||||
if err := inv.RecordSent(ctx, nodeID(t, tm.open, "anchor"), "d-anchor-elsewhere",
|
||||
map[string]string{"m01": "c2", "m02": "c2"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
advancePlans(ctx, tm.open)
|
||||
p = tm.plan(t)
|
||||
if p.State == inventory.PlanFailed {
|
||||
t.Fatalf("the plan failed after its gate passed: %s", p.Note)
|
||||
}
|
||||
if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"laptop"}}) {
|
||||
t.Fatalf("sent %v: the rest once, as the pass said", tm.sent)
|
||||
}
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range []string{"m01", "m02"} {
|
||||
if current[m].Commit != "c2" {
|
||||
t.Errorf("%s was put back to %s after its gate passed", m, current[m].Commit)
|
||||
}
|
||||
if failed, _ := inv.GateFailed(ctx, "build-"+m+"-2"); failed {
|
||||
t.Errorf("%s's build was marked failed at its gate after the gate passed", m)
|
||||
}
|
||||
}
|
||||
if g := p.Modules["m02"].Gate; g == nil || g.Verdict != inventory.GatePassed {
|
||||
t.Errorf("m02 did not take over m01's pass: %+v", g)
|
||||
}
|
||||
}
|
||||
@@ -131,7 +131,10 @@ func readinessOf(ctx context.Context, inv *inventory.Inventory) (broker.Readines
|
||||
// **Dialled the way the mesh dials it** — credential and pin — because a bare connect to a
|
||||
// bus that requires TLS and a user fails at the handshake, and the check then reported a
|
||||
// standing server as absent (seen live, 2026-09-28).
|
||||
if js, err := broker.Dial(address, nats.Timeout(5*time.Second)); err == nil {
|
||||
// The serving controller's own connection answers it without a second (novox/hq issue 327).
|
||||
if serving := servingBus.Load(); serving != nil && serving.Conn().IsConnected() {
|
||||
state.ServerStanding = true
|
||||
} else if js, err := broker.Dial(address, nats.Timeout(5*time.Second)); err == nil {
|
||||
state.ServerStanding = true
|
||||
js.Close()
|
||||
}
|
||||
|
||||
@@ -135,12 +135,6 @@ func handOver(ctx context.Context, seatName, to string, adding bool) error {
|
||||
if !ok || nodeName == "" || module == "" {
|
||||
return fmt.Errorf("the new holder is named <node>/<module>, not %q", to)
|
||||
}
|
||||
// A kinded bench is held once per kind, by the claims themselves (novox/hq ADR 0234 §2, ADR 0259): the
|
||||
// record of who holds a seat has no kind, so a handover would name one holder for every kind.
|
||||
if catalogue.KindedBenches[seatName] {
|
||||
return fmt.Errorf("%s is a kinded bench: each kind is held by the module claiming it, and is not handed "+
|
||||
"over by `seat` — assign the module that claims the kind, or unassign the one that does", seatName)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
@@ -64,13 +62,3 @@ func TestAClaimOutsideTheSetIsShownNotHidden(t *testing.T) {
|
||||
t.Fatalf("a claim outside the set was not shown: %+v", outside)
|
||||
}
|
||||
}
|
||||
|
||||
// A kinded bench is not handed over by `seat`: each kind is held by its claim (novox/hq ADR 0259).
|
||||
func TestAKindedBenchIsNotHandedOver(t *testing.T) {
|
||||
for _, bench := range []string{"channel", "intake"} {
|
||||
err := handOver(context.Background(), bench, "anchor/telegram", false)
|
||||
if err == nil || !strings.Contains(err.Error(), "is a kinded bench") {
|
||||
t.Errorf("%s: %v", bench, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"slices"
|
||||
@@ -28,6 +29,9 @@ import (
|
||||
// answer. It also means a refusal is the same refusal in the same words, because it is the same
|
||||
// output.
|
||||
|
||||
// verbKey carries the verb a call is for, to the process it runs.
|
||||
type verbKey struct{}
|
||||
|
||||
// verbAnswer is what a verb answers: what the command printed, whether it succeeded, and — where the
|
||||
// command speaks JSON — the same as data.
|
||||
type verbAnswer struct {
|
||||
@@ -250,6 +254,8 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
return argv, nil
|
||||
case "tools":
|
||||
return nil, errors.New("tools is answered from the records, not by a command")
|
||||
case "dead-letters":
|
||||
return nil, errors.New("dead-letters is answered by the serving controller, on its own connection, not by a command")
|
||||
case "status":
|
||||
return []string{"status", "--json"}, nil
|
||||
case "nodes":
|
||||
@@ -919,6 +925,12 @@ func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
||||
caller = "a seat call whose caller the bus did not name"
|
||||
}
|
||||
cmd.Env = append(cmd.Env, link.CallerVar+"="+caller+", through the "+catalogue.ControllerSeatName+" seat")
|
||||
// And which verb, so the connection it dials says so in the bus's list (novox/hq issue 327).
|
||||
verb, _ := ctx.Value(verbKey{}).(string)
|
||||
if verb == "" {
|
||||
verb = argv[0]
|
||||
}
|
||||
cmd.Env = append(cmd.Env, verbVar+"="+verb)
|
||||
// Two buffers, one answer. What the command *says* is both streams, in the order a person at
|
||||
// a shell would read them; what it *answers as data* is standard output alone — `status --json`
|
||||
// prints its warnings beside the document, and a JSON parsed from the two together parsed
|
||||
@@ -927,18 +939,7 @@ func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
runErr := cmd.Run()
|
||||
answer := verbAnswer{Output: stdout.String() + stderr.String(), OK: runErr == nil}
|
||||
if jsonVerbs[argv[0]] && runErr == nil {
|
||||
var parsed any
|
||||
if json.Unmarshal(bytes.TrimSpace(stdout.Bytes()), &parsed) == nil {
|
||||
answer.Answer = parsed
|
||||
if overviewVerbs[argv[0]] {
|
||||
// Once, as data: the same document again as text doubled an answer that already
|
||||
// outgrew one message of the bus (novox/hq issue 314).
|
||||
answer.Output = stderr.String() + "its answer, as data, is `answer`\n"
|
||||
}
|
||||
}
|
||||
}
|
||||
answer := answerOf(argv, stdout.Bytes(), stderr.String(), runErr == nil)
|
||||
var exit *exec.ExitError
|
||||
if runErr != nil && !errors.As(runErr, &exit) {
|
||||
// Not the command refusing — the command not running at all, which is this process's fault.
|
||||
@@ -953,6 +954,66 @@ func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// answerOf is what a command said, as a verb answers it: both streams as text, and standard output as data
|
||||
// where the command speaks JSON.
|
||||
func answerOf(argv []string, stdout []byte, stderr string, ok bool) verbAnswer {
|
||||
answer := verbAnswer{Output: string(stdout) + stderr, OK: ok}
|
||||
if jsonVerbs[argv[0]] && ok {
|
||||
var parsed any
|
||||
if json.Unmarshal(bytes.TrimSpace(stdout), &parsed) == nil {
|
||||
answer.Answer = parsed
|
||||
if overviewVerbs[argv[0]] {
|
||||
// Once, as data: the same document again as text doubled an answer that already
|
||||
// outgrew one message of the bus (novox/hq issue 314).
|
||||
answer.Output = stderr + "its answer, as data, is `answer`\n"
|
||||
}
|
||||
}
|
||||
}
|
||||
return answer
|
||||
}
|
||||
|
||||
// readHere answers a verb that only reads the bus in the serving controller itself, on its own connection
|
||||
// and keeper (novox/hq issue 327): the same command, writing to the answer rather than to a process's
|
||||
// output, so the answer is the one the command prints. False for any other command line, which runs as a
|
||||
// command of its own. Every `conditions` call — the operator's channel reads it at least once a minute —
|
||||
// was a process that dialled the bus, logged in and left.
|
||||
func readHere(ctx context.Context, argv []string) (verbAnswer, bool) {
|
||||
var read func(context.Context, []string, io.Writer) error
|
||||
args := argv[1:]
|
||||
// Each where this process holds what it reads: the serving keeper, the hand-act log's connection, the
|
||||
// serving connection.
|
||||
switch argv[0] {
|
||||
case "conditions":
|
||||
sub := "list"
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
sub, args = args[0], args[1:]
|
||||
}
|
||||
if conditionsFrom != nil {
|
||||
read = map[string]func(context.Context, []string, io.Writer) error{
|
||||
"list": listConditions, "show": showCondition, "history": conditionHistory}[sub]
|
||||
}
|
||||
case "hand-acts":
|
||||
if handActConn != nil || servingBus.Load() != nil {
|
||||
read = listHandActs
|
||||
}
|
||||
case "queue":
|
||||
if servingBus.Load() != nil {
|
||||
read = listQueue
|
||||
}
|
||||
}
|
||||
if read == nil {
|
||||
return verbAnswer{}, false
|
||||
}
|
||||
var out bytes.Buffer
|
||||
stderr := ""
|
||||
err := read(ctx, args, &out)
|
||||
if err != nil {
|
||||
// As the command says it when it fails (main).
|
||||
stderr = "mesh-controller: " + err.Error() + "\n"
|
||||
}
|
||||
return answerOf(argv, out.Bytes(), stderr, err == nil), true
|
||||
}
|
||||
|
||||
// seatToolHandlers are the handlers for every verb the mesh-controller seat declares, from the
|
||||
// store's row, so a verb the row does not carry is not served. A verb it carries that this binary
|
||||
// cannot run is named at start and answers the reason when called — never a refusal to serve, which
|
||||
@@ -982,6 +1043,9 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
||||
if verb == "calls" {
|
||||
return callsAnswer(link.Calls, a.given["call"])
|
||||
}
|
||||
if verb == "dead-letters" {
|
||||
return deadLettersAnswer(ctx, a)
|
||||
}
|
||||
if verb == "doctor" {
|
||||
// From the serving controller, which runs the self-check and hears the signals
|
||||
// (novox/hq to-be 45 §4): the last verdict at once, or a run now.
|
||||
@@ -1033,6 +1097,7 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx = context.WithValue(ctx, verbKey{}, verb)
|
||||
if verb == "status" && statusFrom != nil {
|
||||
// At once, from the summary the serving controller keeps (novox/hq to-be 45 Phase 0).
|
||||
return statusFrom.answer(ctx)
|
||||
@@ -1041,6 +1106,9 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
||||
// Whatever it did, `status` is composed again once it has.
|
||||
defer statusFrom.nudge()
|
||||
}
|
||||
if answer, read := readHere(ctx, argv); read {
|
||||
return answer, nil
|
||||
}
|
||||
if answersFirst(argv) {
|
||||
// Before anything is sent: a push sends the bus's own machine first, and a broker
|
||||
// reloading its user list forgets the answer it was about to permit (novox/hq issue 265).
|
||||
@@ -1064,7 +1132,10 @@ func actsOnAPlan(args map[string]any) bool {
|
||||
|
||||
// inProcess are the verbs answered by this process rather than by a command it runs: `tools` from
|
||||
// the records, `calls` from what this process served.
|
||||
var inProcess = map[string]bool{"tools": true, "calls": true, "doctor": true}
|
||||
var inProcess = map[string]bool{"tools": true, "calls": true, "doctor": true,
|
||||
// What a consumer gave up on, read and changed on the serving controller's own connection (novox/hq
|
||||
// issue 330).
|
||||
"dead-letters": true}
|
||||
|
||||
// answersFirst is a command line whose caller is answered before it runs: a push, by its verb or
|
||||
// through `command`. A push sends the machine holding the bus first when its user list changed, the
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// The serving controller's own connection, lent to whatever it does for a moment (novox/hq issue 327).
|
||||
//
|
||||
// Every place that needed the bus for a moment dialled it: a verb's own process, and in the serving
|
||||
// controller a watchdog tick reading whether the build seat paused, a walk's step reading readiness, a
|
||||
// queue read. Each paid a connection, a TLS handshake and a login on the control node, and hundreds an
|
||||
// hour hid in the server's connection total the one thing it would show: a client reconnecting in a loop.
|
||||
// The serving controller is on the bus already; what it does is done on that connection.
|
||||
|
||||
// servingBus is the serving controller's connection, set when it starts serving; nil in every other
|
||||
// process, which dials its own.
|
||||
var servingBus atomic.Pointer[broker.JetStream]
|
||||
|
||||
// aBus is a connection for something done for a moment: the serving controller's own, lent — so its
|
||||
// Close closes nothing — when this process is it, and otherwise one dialled for it, named for this
|
||||
// process (broker.ConnectionName), which its Close closes.
|
||||
func aBus() (*broker.JetStream, error) {
|
||||
if serving := servingBus.Load(); serving != nil {
|
||||
return broker.Borrow(serving), nil
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot reach the bus: %w", err)
|
||||
}
|
||||
return js, nil
|
||||
}
|
||||
|
||||
// usageTo sends a command's flag errors and usage to where its answer goes when that is not this process's
|
||||
// output: a verb answered in the serving controller says them in its answer, not in the controller's log.
|
||||
func usageTo(set *flag.FlagSet, w io.Writer) {
|
||||
if w != os.Stdout {
|
||||
set.SetOutput(w)
|
||||
}
|
||||
}
|
||||
@@ -154,8 +154,9 @@ var signalsTable = []signalRow{
|
||||
}},
|
||||
{Row: "S9", Signal: "bus advisories: maximum deliveries, consumer deleted; the controller's own slow " +
|
||||
"consumer and refused subjects", Emitter: "bus server's advisory subjects; the controller's connection",
|
||||
Trigger: "any", Bound: "any occurrence; clears after an hour without another, and a deleted consumer " +
|
||||
"once it exists again or the mesh no longer expects it",
|
||||
Trigger: "any", Bound: "any occurrence; clears after an hour without another, a deleted consumer " +
|
||||
"once it exists again or the mesh no longer expects it, and a message given up on once DEAD_LETTERS " +
|
||||
"no longer holds it (novox/hq issue 330)",
|
||||
Kind: "slow-consumer, max-deliveries, refused, consumer-lost", Severity: conditions.Warning, Phase: 1,
|
||||
needs: func(f *signalFacts) error { return f.advisoriesErr }, watch: watchAdvisories,
|
||||
newest: func(f *signalFacts) time.Time {
|
||||
@@ -485,12 +486,94 @@ func watchAdvisories(f *signalFacts) []conditions.Observation {
|
||||
if a.ID == "controller" {
|
||||
machine = f.host
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: a.ID, Kind: a.Kind,
|
||||
Machine: machine, Severity: severity, Summary: a.Said + times, Said: a.Said})
|
||||
o := conditions.Observation{Scope: conditions.ScopeBus, ID: a.ID, Kind: a.Kind, Token: a.Token,
|
||||
Machine: machine, Severity: severity, Summary: a.Said + times, Said: a.Said}
|
||||
if a.Kind == link.AdvisoryMaxDeliveries && a.Token == link.AdvisoryNotKept {
|
||||
o.Machine = consumerMachine(a.Stream, a.Consumer)
|
||||
o.Headline = clip(conditions.Capital(fmt.Sprintf("%s gave up on a message, not kept",
|
||||
consumerWho(a.Stream, a.Consumer))), 60)
|
||||
o.Explanation = "A listener on the bus could not handle a message, and the mesh could not keep it " +
|
||||
"for you yet. It tries again every minute."
|
||||
o.Resolved = "Resolved: the message is kept"
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
return append(out, watchDeadLetters(f)...)
|
||||
}
|
||||
|
||||
// watchDeadLetters says each consumer that DEAD_LETTERS holds a message for (novox/hq issue 330): open
|
||||
// while it holds any, so it clears when they are delivered again or dropped, never because the server
|
||||
// stopped saying it.
|
||||
func watchDeadLetters(f *signalFacts) []conditions.Observation {
|
||||
keys := make([]string, 0, len(f.deadLetters))
|
||||
for k := range f.deadLetters {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var out []conditions.Observation
|
||||
for _, key := range keys {
|
||||
n := f.deadLetters[key]
|
||||
stream, consumer, _ := strings.Cut(key, ".")
|
||||
messages, them := "a message", "it"
|
||||
if n > 1 {
|
||||
messages, them = fmt.Sprintf("%d messages", n), "them"
|
||||
}
|
||||
who := consumerWho(stream, consumer)
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: key, Kind: link.AdvisoryMaxDeliveries,
|
||||
Machine: consumerMachine(stream, consumer), Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s gave up on %s; %s kept in %s until delivered again or dropped, with why, "+
|
||||
"through the controller's dead-letters verb", link.ConsumerInWords(stream, consumer), messages,
|
||||
map[bool]string{true: "they are", false: "it is"}[n > 1], broker.DeadLettersStream),
|
||||
Said: fmt.Sprintf("%d held for %s", n, key),
|
||||
Headline: clip(conditions.Capital(fmt.Sprintf("%s could not handle %s", who, messages)), 60),
|
||||
Needs: fmt.Sprintf("deliver %s again or drop %s, from the mesh MCP server.", them, them),
|
||||
Explanation: conditions.Capital(fmt.Sprintf("%s was handed %s several times and gave up, so what %s "+
|
||||
"asked for was not done. The mesh keeps %s until you deliver %s again or drop %s.", who, messages,
|
||||
them, them, them, them)),
|
||||
Resolved: "Resolved: the messages it gave up on were delivered again or dropped"})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// consumerWho is a durable consumer's holder as the operator says it: a module on its machine, the
|
||||
// controller, or a seat's holders.
|
||||
func consumerWho(stream, consumer string) string {
|
||||
switch {
|
||||
case consumer == broker.ControllerName:
|
||||
return "the controller"
|
||||
case strings.HasPrefix(stream, "SEAT_") && strings.HasSuffix(consumer, "_worker"):
|
||||
seat := strings.ToLower(strings.ReplaceAll(strings.TrimSuffix(strings.TrimPrefix(consumer, "SEAT_"), "_worker"), "_", "-"))
|
||||
return "the holder of " + seat
|
||||
case stream == broker.EventsStream:
|
||||
if node, module, ok := strings.Cut(consumer, "_"); ok {
|
||||
return module + " on " + node
|
||||
}
|
||||
}
|
||||
return "a listener on the bus"
|
||||
}
|
||||
|
||||
// consumerMachine is the machine a module's consumer is on; empty for the others.
|
||||
func consumerMachine(stream, consumer string) string {
|
||||
if stream == broker.EventsStream {
|
||||
if node, _, ok := strings.Cut(consumer, "_"); ok {
|
||||
return node
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// clip is words at most n characters long, cut at a word.
|
||||
func clip(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
cut := s[:n]
|
||||
if i := strings.LastIndex(cut, " "); i > 0 {
|
||||
cut = cut[:i]
|
||||
}
|
||||
return cut
|
||||
}
|
||||
|
||||
func watchSelfCheck(f *signalFacts) []conditions.Observation {
|
||||
every := f.selfCheck.every
|
||||
if every <= 0 {
|
||||
|
||||
@@ -38,7 +38,7 @@ func unknownFieldObservations(known map[string]catalogue.Manifest) []conditions.
|
||||
Summary: catalogue.UnknownFieldReason(m),
|
||||
Said: m.UnknownField(),
|
||||
Headline: name + " is left out until the controller is updated",
|
||||
Explanation: name + " uses a field this controller does not know, so it is left out of every machine it is on, and nothing of it changes there until the controller is updated.",
|
||||
Explanation: name + " uses a field this controller does not know. Until the controller is updated, nothing of it changes on its machines, and what it adds to other modules and the ports opened for it stop. Its data is still backed up as this controller reads it, which may not be what its newer version asks.",
|
||||
Needs: "update the controller, or register " + name + " again at a version this controller knows.",
|
||||
Resolved: "the controller reads " + name + " again",
|
||||
})
|
||||
|
||||
@@ -75,6 +75,9 @@ type signalFacts struct {
|
||||
|
||||
advisories []link.Advisory
|
||||
lostConsumers map[string]bool
|
||||
// deadLetters are how many messages DEAD_LETTERS holds per consumer, by `<stream>.<consumer>`
|
||||
// (novox/hq issue 330): each consumer's max-deliveries condition is open while it holds any.
|
||||
deadLetters map[string]int
|
||||
advisoriesErr error
|
||||
|
||||
selfCheck selfCheckFacts
|
||||
@@ -332,6 +335,9 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
|
||||
}
|
||||
f.advisories = link.Advisories.Since(now.Add(-advisoryQuiet))
|
||||
f.lostConsumers, f.advisoriesErr = w.lostConsumers(ctx, f.advisories)
|
||||
if f.advisoriesErr == nil && w.js != nil {
|
||||
f.deadLetters, f.advisoriesErr = link.HeldDeadLetters(w.js.Context())
|
||||
}
|
||||
f.handActs, f.handActsErr = w.gatherHandActs(ctx, now)
|
||||
f.facts.taken, _, f.facts.began, f.facts.err = exportedFacts.last()
|
||||
return f
|
||||
|
||||
@@ -38,7 +38,8 @@ type Consumer struct {
|
||||
Push bool
|
||||
// AckWaitSeconds before an unacknowledged delivery is redelivered.
|
||||
AckWaitSeconds int
|
||||
// MaxDeliver before the message is dead-lettered; zero for the mesh's default.
|
||||
// MaxDeliver is how often a message is handed over before the consumer gives it up; zero for no
|
||||
// bound. What a consumer gives up is kept in DEAD_LETTERS by the controller (novox/hq issue 330).
|
||||
MaxDeliver int
|
||||
// MaxAckPending is how many deliveries the server lets stand unacknowledged at once; zero for
|
||||
// the server's default, which is many. **One, for a consumer handled one at a time**
|
||||
@@ -125,9 +126,7 @@ func ConsumerFor(p Principal) (Consumer, bool) {
|
||||
// A module that reacts to anything — a module's events or a role's (novox/hq ADR 0121). Watching
|
||||
// a role was missing here, so the one module that does it got no consumer at all: it started,
|
||||
// connected, and its graph stayed empty with nothing anywhere reporting why.
|
||||
// And one that hears its own answers on a seat it uses (novox/hq ADR 0259 §3): an asker's warrants.
|
||||
hearsItsOwn := len(SeatTrafficOf(p.Module, nil, p.Uses, nil).Subscribe) > 0
|
||||
if p.Kind != KindModule || (len(p.Consumes) == 0 && len(p.Watches) == 0 && !hearsItsOwn) {
|
||||
if p.Kind != KindModule || (len(p.Consumes) == 0 && len(p.Watches) == 0) {
|
||||
return Consumer{}, false
|
||||
}
|
||||
perms, err := PermissionsFor(p)
|
||||
@@ -147,13 +146,16 @@ func ConsumerFor(p Principal) (Consumer, bool) {
|
||||
return Consumer{}, false
|
||||
}
|
||||
sort.Strings(filters)
|
||||
// And the events given up on and delivered again to this consumer alone (novox/hq issue 330).
|
||||
filters = append(filters, AgainFilter(consumerDurable(p)))
|
||||
return Consumer{
|
||||
Name: consumerDurable(p),
|
||||
Stream: consumerStream(p),
|
||||
Filters: filters,
|
||||
AckWaitSeconds: 30,
|
||||
MaxDeliver: 5,
|
||||
Why: "what " + p.Module + " declared it consumes; after max-deliver it dead-letters",
|
||||
Why: "what " + p.Module + " declared it consumes; after max-deliver it gives an event up, and the " +
|
||||
"controller keeps it in DEAD_LETTERS until a person delivers it again or drops it",
|
||||
}, true
|
||||
}
|
||||
|
||||
@@ -236,7 +238,7 @@ func HolderConsumerFor(node, module string, seat DeclaredSeat) (Consumer, bool)
|
||||
//
|
||||
// **No max-deliver, and a long ack wait.** A declaration is settled only after the node has applied
|
||||
// it and reported, which is minutes on a machine pulling images; and a declaration the mesh cannot
|
||||
// get a node to accept is not one to dead-letter, because the stream keeps only the newest per node
|
||||
// get a node to accept is not one to give up on, because the stream keeps only the newest per node
|
||||
// anyway — so there is exactly one message per node to redeliver, for as long as that node is away.
|
||||
func NodeConsumer(node string) Consumer {
|
||||
return Consumer{
|
||||
|
||||
@@ -61,8 +61,9 @@ func TestAModuleGetsOneConsumerCarryingEveryFilter(t *testing.T) {
|
||||
if !ok {
|
||||
t.Fatal("a module that consumes got no consumer")
|
||||
}
|
||||
if len(c.Filters) != 2 {
|
||||
t.Fatalf("expected both subjects as filters, got %v", c.Filters)
|
||||
// Both, and its own share of what is delivered again (novox/hq issue 330).
|
||||
if len(c.Filters) != 3 || c.Filters[2] != "mesh.again.one_audit.>" {
|
||||
t.Fatalf("expected both subjects and its own again filter, got %v", c.Filters)
|
||||
}
|
||||
perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "audit",
|
||||
Consumes: []string{"shop.order.placed"}, PasswordHash: "x"})
|
||||
|
||||
@@ -28,6 +28,8 @@ import (
|
||||
type JetStream struct {
|
||||
conn *nats.Conn
|
||||
js nats.JetStreamContext
|
||||
// borrowed is a connection lent by its owner (Borrow): closing it is the owner's.
|
||||
borrowed bool
|
||||
// Note is how this says something it decided not to fail over. Nil is silent, which is only
|
||||
// right for a caller that has no way to report; the controller sets it.
|
||||
Note func(string, ...any)
|
||||
@@ -40,9 +42,17 @@ func (j *JetStream) note(format string, args ...any) {
|
||||
}
|
||||
}
|
||||
|
||||
// ConnectionName is what a connection this process dials says it is, in the server's list of
|
||||
// connections: the controller's role and what it is doing (novox/hq issue 327). Every connection was
|
||||
// named `mesh-controller` — the serving controller's, each verb's own process, the build agents' — so the
|
||||
// server's list could not say which was which. The process sets it once, at its start; an option a
|
||||
// caller passes to Dial names one connection otherwise.
|
||||
var ConnectionName = "mesh-controller"
|
||||
|
||||
// Dial connects and returns the controller's JetStream handle.
|
||||
func Dial(url string, opts ...nats.Option) (*JetStream, error) {
|
||||
opts = append(opts, nats.Name("mesh-controller"), nats.Timeout(10*time.Second))
|
||||
// The name first, so a name the caller gives is the one that stands.
|
||||
opts = append([]nats.Option{nats.Name(ConnectionName), nats.Timeout(10 * time.Second)}, opts...)
|
||||
// **Pinned, not named.** The bus presents the mesh's own certificate, which names nothing a
|
||||
// public verifier would accept (design 25 §4: a host pins the server's exact certificate and
|
||||
// checks nothing else, and so does this). Without this, the first connection failed with
|
||||
@@ -130,11 +140,20 @@ func (j *JetStream) Conn() *nats.Conn { return j.conn }
|
||||
func (j *JetStream) Context() nats.JetStreamContext { return j.js }
|
||||
|
||||
func (j *JetStream) Close() {
|
||||
if j.conn != nil {
|
||||
if j.conn != nil && !j.borrowed {
|
||||
j.conn.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// Borrow is the same connection for a caller that will close what it was handed when it is done: its
|
||||
// Close closes nothing, and the connection stays its owner's (novox/hq issue 327). How the serving
|
||||
// controller lends its own connection to work that would otherwise dial one of its own.
|
||||
func Borrow(j *JetStream) *JetStream {
|
||||
lent := *j
|
||||
lent.borrowed = true
|
||||
return &lent
|
||||
}
|
||||
|
||||
// EnsureStream creates the stream if it is absent and brings it to match if it is present.
|
||||
//
|
||||
// **Idempotent, because the controller asserts on every start** rather than creating once at
|
||||
@@ -154,6 +173,15 @@ func (j *JetStream) EnsureStream(s Stream) error {
|
||||
Description: s.Why,
|
||||
}
|
||||
want.AllowDirect = s.Direct
|
||||
if s.MaxBytes > 0 {
|
||||
want.MaxBytes = s.MaxBytes
|
||||
}
|
||||
if s.DiscardNew {
|
||||
want.Discard = nats.DiscardNew
|
||||
}
|
||||
if s.DuplicatesSeconds > 0 {
|
||||
want.Duplicates = time.Duration(s.DuplicatesSeconds) * time.Second
|
||||
}
|
||||
if s.Retention == RetentionLastPerSubject {
|
||||
// Last-per-subject is a limits stream with one message kept per subject, not a
|
||||
// retention policy of its own — the state shape, spelled the way the server spells it.
|
||||
|
||||
@@ -3,6 +3,8 @@ package broker
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
@@ -66,3 +68,32 @@ func TestAgainstARealServer(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// A connection says what it is in the server's list (novox/hq issue 327): the process's name, unless the
|
||||
// caller names this one; and a lent connection's Close leaves its owner's open.
|
||||
func TestAConnectionIsNamedAndALentOneIsNotClosed(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
was := ConnectionName
|
||||
ConnectionName = "mesh-controller verb conditions"
|
||||
defer func() { ConnectionName = was }()
|
||||
named, err := Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer named.Close()
|
||||
if got := named.Conn().Opts.Name; got != "mesh-controller verb conditions" {
|
||||
t.Errorf("named %q", got)
|
||||
}
|
||||
lease, err := Dial(url, nats.Name("mesh-controller serving lease"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer lease.Close()
|
||||
if got := lease.Conn().Opts.Name; got != "mesh-controller serving lease" {
|
||||
t.Errorf("a name the caller gave became %q", got)
|
||||
}
|
||||
Borrow(named).Close()
|
||||
if !named.Conn().IsConnected() {
|
||||
t.Error("closing a lent connection closed its owner's")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,12 +50,6 @@ type Membership struct {
|
||||
// and refuses, with the reason, what is not on it — the bus enforces only the union over every
|
||||
// module on the machine.
|
||||
State []StateIssued `json:"state,omitempty"`
|
||||
// SeatTraffic is what this module's code may submit, say, hear, take, ask, answer and read on seats
|
||||
// that name their caller or their kind (novox/hq ADR 0259 §3). The runtime carrying the module
|
||||
// publishes, takes and answers for it only what is listed here: the bus enforces only the union
|
||||
// over every module on the machine, so one module's code reaching another's name or kind through
|
||||
// the runtime is the runtime's to refuse.
|
||||
SeatTraffic *SeatTraffic `json:"seat-traffic,omitempty"`
|
||||
}
|
||||
|
||||
// Served is one address a tool is answered on.
|
||||
@@ -84,8 +78,6 @@ type Placements struct {
|
||||
// Interchangeable is each module whose definition says its instances are the same anywhere,
|
||||
// so the module's plain subject is issued to all of them in one queue.
|
||||
Interchangeable map[string]bool
|
||||
// Kinds is every kind held of a kinded bench, by whom and with what capabilities (ADR 0259 §5).
|
||||
Kinds []KindHeld
|
||||
}
|
||||
|
||||
// AnswersForTheModule says whether an instance of a module on one machine is issued the module's
|
||||
@@ -117,20 +109,6 @@ func MembershipFor(node string, d Declared, where Placements) Membership {
|
||||
}
|
||||
}
|
||||
m.State = stateIssuedFor(d, node)
|
||||
t := SeatTrafficOf(d.Module, d.Holds, d.Uses, d.Watches)
|
||||
for _, s := range append(append([]Seat{}, d.Uses...), d.Watches...) {
|
||||
if !s.Kinded {
|
||||
continue
|
||||
}
|
||||
for _, k := range where.Kinds {
|
||||
if k.Seat == s.Name && !kindListed(t.Kinds, k) {
|
||||
t.Kinds = append(t.Kinds, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(t.Publish)+len(t.Subscribe)+len(t.Answers)+len(t.Workers)+len(t.Records)+len(t.Kinds) > 0 {
|
||||
m.SeatTraffic = &t
|
||||
}
|
||||
if len(d.Invokes) > 0 {
|
||||
m.Reaches = map[string][]string{}
|
||||
for _, t := range d.Invokes {
|
||||
@@ -167,48 +145,5 @@ func PlacementsOf(r Records, interchangeable map[string]bool) Placements {
|
||||
for _, nodes := range p.Nodes {
|
||||
sort.Strings(nodes)
|
||||
}
|
||||
for node, declared := range r.Assigned {
|
||||
for _, d := range declared {
|
||||
for _, s := range d.Holds {
|
||||
if s.Kinded && s.Kind != "" {
|
||||
p.Kinds = append(p.Kinds, KindHeld{Seat: s.Name, Kind: s.Kind, Module: d.Module, Node: node,
|
||||
Capabilities: placedCapabilities(s.Capabilities, d.RunsAs)})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(p.Kinds, func(i, j int) bool {
|
||||
a, b := p.Kinds[i], p.Kinds[j]
|
||||
if a.Seat != b.Seat {
|
||||
return a.Seat < b.Seat
|
||||
}
|
||||
if a.Kind != b.Kind {
|
||||
return a.Kind < b.Kind
|
||||
}
|
||||
return a.Node < b.Node
|
||||
})
|
||||
return p
|
||||
}
|
||||
|
||||
// placedCapabilities is what a kind's claim promises, as far as its placement lets the router believe it
|
||||
// (novox/hq ADR 0259 §8): `verified-sender` only from a holder that runs as an account of its own, on a bus
|
||||
// account of its own — never one the machine's runtime carries as the operator's account.
|
||||
func placedCapabilities(declared []string, runsAs string) []string {
|
||||
var out []string
|
||||
for _, c := range declared {
|
||||
if c == "verified-sender" && runsAs == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func kindListed(list []KindHeld, k KindHeld) bool {
|
||||
for _, x := range list {
|
||||
if x.Seat == k.Seat && x.Kind == k.Kind && x.Module == k.Module && x.Node == k.Node {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
+6
-53
@@ -63,23 +63,6 @@ type Seat struct {
|
||||
Emits []string
|
||||
Serves []string
|
||||
Versions []string // protocol versions served beside the current one; empty for v1 only
|
||||
|
||||
// Kinded says the seat is a kinded bench (novox/hq ADR 0234 §2, ADR 0259 §3): each holder claims one
|
||||
// kind, and its verbs' subjects carry it. Kind is the kind this principal's claim names, for a seat it
|
||||
// holds.
|
||||
Kinded bool
|
||||
Kind string
|
||||
// ByCaller are the accepts and emits whose last token names the calling module (ADR 0259 §3).
|
||||
ByCaller []string
|
||||
// Proofs are the seat's proof verbs: core request and reply, never on a stream (ADR 0259 §3).
|
||||
Proofs []string
|
||||
// Records are the holder's buckets, by their full name, each user reads under its own name.
|
||||
Records []string
|
||||
// Capabilities are what this principal's claim of a kinded bench promises (ADR 0234 §2).
|
||||
Capabilities []string
|
||||
// DeclaredBy is the module that declares the seat. On a kinded bench it alone submits work to a kind
|
||||
// and answers its proofs (novox/hq ADR 0259 §8): the router, not any user or watcher of the bench.
|
||||
DeclaredBy string
|
||||
}
|
||||
|
||||
// A Principal is one user of the bus. Its permissions are derived from what it declares and
|
||||
@@ -429,6 +412,11 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// both in the mesh's own account; the controller says each as a condition in the mesh's words.
|
||||
// Named, not `$JS.EVENT.>`: the other advisories are every API call the mesh makes.
|
||||
sub = append(sub, BusAdvisories...)
|
||||
// **And what a consumer gave up on, kept and delivered again** (novox/hq issue 330): the notice
|
||||
// acknowledged once the message is copied, the copy kept, and an event delivered again to the one
|
||||
// consumer that gave it up. An ask to a seat is not delivered again, so no seat's queue is granted.
|
||||
pub = append(pub, "$JS.ACK."+DeadLetterNoticesStream+"."+ControllerName+".>", deadLetterPrefix+">",
|
||||
againPrefix+">")
|
||||
|
||||
case KindPerson:
|
||||
// Tools, and nothing else. Every subject a person may publish is a tool call; a person
|
||||
@@ -542,9 +530,6 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// 2b. Events of a role it watches, under the seat's own namespace. Subscribe only: watching a
|
||||
// role is hearing what it announced, not taking part in it.
|
||||
for _, w := range p.Watches {
|
||||
if w.Kinded {
|
||||
continue // composed by SeatTrafficOf below
|
||||
}
|
||||
for _, e := range w.Emits {
|
||||
sub = append(sub, seatSubject(w, "event", e))
|
||||
}
|
||||
@@ -563,13 +548,6 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
|
||||
// 3. Seats it holds: full participation.
|
||||
for _, s := range p.Holds {
|
||||
if s.isNewTraffic() {
|
||||
// Composed by SeatTrafficOf below, worker and all; only its tools are served here.
|
||||
for _, t := range s.Serves {
|
||||
sub = append(sub, seatToolSubject(s, t, p.Node))
|
||||
}
|
||||
continue
|
||||
}
|
||||
// Taking work from the role's queue: the worker consumer every holder shares (asked
|
||||
// about, pulled from, acknowledged), on the seat's own stream (novox/hq ADR 0190). A
|
||||
// holder pulls — asks the consumer for its next message, answered on its own inbox —
|
||||
@@ -612,7 +590,7 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// seat's inbound subject and watch other modules' traffic, nor publish its outbound
|
||||
// events and lie about outcomes (design 29 §2).
|
||||
for _, s := range p.Uses {
|
||||
for _, a := range plainVerbs(s, s.Accepts) {
|
||||
for _, a := range s.Accepts {
|
||||
pub = append(pub, seatSubject(s, "accept", a))
|
||||
}
|
||||
for _, t := range s.Serves {
|
||||
@@ -625,12 +603,6 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
pub = append(pub, stateGrants(stateAccess{Module: p.Module, Node: p.Node, Keeps: p.State,
|
||||
PerMachine: p.PerMachine, Reads: p.Reads, KeyedReads: p.KeyedReads})...)
|
||||
|
||||
// 6. Its traffic on seats that name their caller or their kind, ask proofs or keep records
|
||||
// (novox/hq ADR 0259 §3).
|
||||
tp, ts := SeatTrafficOf(p.Module, p.Holds, p.Uses, p.Watches).grants()
|
||||
pub = append(pub, tp...)
|
||||
sub = append(sub, ts...)
|
||||
|
||||
case KindNodeTools:
|
||||
// **One process serves what every module on the machine would have served for itself**
|
||||
// (novox/hq ADR 0175). Each carried module's whole tool namespace — the same grant that
|
||||
@@ -708,25 +680,6 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
pub = append(pub, stateGrants(stateAccess{Module: d.Module, Node: p.Node, Keeps: stateNames(d.State),
|
||||
PerMachine: perMachineNames(d.State), Reads: d.Reads, KeyedReads: d.KeyedReads})...)
|
||||
}
|
||||
// **Never the traffic of a trusted holder** (novox/hq ADR 0259 §8): the machine's runtime runs as the
|
||||
// operator's account, which every agent runs as, so a module saying warrants or speaking for a kind
|
||||
// that proves its sender is never composed into it — refused here, naming it, whatever registration
|
||||
// let through.
|
||||
for _, d := range p.Carries {
|
||||
if why := trustedTraffic(d); why != "" {
|
||||
return Permissions{}, fmt.Errorf("%s on %s is carried by the machine's runtime, and %s: it runs "+
|
||||
"as an account of its own, never the runtime's (novox/hq ADR 0259)", d.Module, p.Node, why)
|
||||
}
|
||||
}
|
||||
// **And the seat traffic of the modules it carries** (novox/hq ADR 0259 §3): a bundle reaches the
|
||||
// bus only through its runtime, so the runtime is granted the union. That one module's code does
|
||||
// not publish under another's name or kind through it is the runtime's to keep, from the seat
|
||||
// traffic each module's membership lists.
|
||||
for _, d := range p.Carries {
|
||||
tp, ts := SeatTrafficOf(d.Module, d.Holds, d.Uses, d.Watches).grants()
|
||||
pub = append(pub, tp...)
|
||||
sub = append(sub, ts...)
|
||||
}
|
||||
sub = unique(sub)
|
||||
pub = unique(pub)
|
||||
}
|
||||
|
||||
@@ -1,305 +0,0 @@
|
||||
package broker
|
||||
|
||||
import "sort"
|
||||
|
||||
// What a module may say and take on the seats it holds, uses and watches, beyond tools (novox/hq ADR
|
||||
// 0259 §3, to-be 46 §10).
|
||||
//
|
||||
// Three rules are added to the ones a seat always had, each a subject whose last token names who may
|
||||
// publish it, granted to that publisher alone — the way a node seat's event about a machine carries the
|
||||
// machine (ADR 0219):
|
||||
//
|
||||
// - **A verb named by its caller** (`by-caller`). A user of the seat submits that accept, and hears that
|
||||
// event, under its own module's name and no other: `accept.ask.<module>`, `event.decided.<module>`. The
|
||||
// holder takes every caller's accept and says the event to any caller. So an ask's asker is a fact the
|
||||
// server enforces, and a warrant reaches only the asker it is for.
|
||||
// - **A kinded bench** (ADR 0234 §2). A holder claims one kind and takes its own kind's accepts, says its
|
||||
// own kind's events and asks its own kind's proofs, and nothing of another kind; a user submits to any
|
||||
// kind. Each kind has its own worker on the seat's queue, so a holder that is away keeps its work and
|
||||
// holds up no other kind.
|
||||
// - **A proof** (`proofs`): core request and reply on `mesh.seat.<seat>.proof.<verb>.<kind>`. No stream's
|
||||
// subjects cover it, so what travels there — a code typed by the operator — is never persisted. A kinded
|
||||
// holder asks with its own kind; the modules that watch the seat answer.
|
||||
//
|
||||
// And one read: **a holder's records**, a bucket the seat names, read by each user under its own name only
|
||||
// (`$KV.<bucket>.<module>.>`), so an asker reads the state of its own asks and no other asker's.
|
||||
|
||||
// Worker is one durable consumer a holder pulls a seat's work from.
|
||||
type Worker struct {
|
||||
Stream string
|
||||
Consumer string
|
||||
Filter string
|
||||
}
|
||||
|
||||
// SeatTraffic is one module's seat traffic beyond tools. Publish and Subscribe are subject patterns, in the
|
||||
// server's wildcards; the runtime that carries the module checks a bundle's request against them, since
|
||||
// the runtime's own principal holds the union of every module it carries.
|
||||
type SeatTraffic struct {
|
||||
// Publish is what it submits (accepts of seats it uses), says (events of seats it holds) and asks
|
||||
// (proofs of seats it holds a kind of).
|
||||
Publish []string `json:"publish,omitempty"`
|
||||
// Subscribe is what it hears: events of seats it uses that are named by caller, events of seats it
|
||||
// watches, and accepts of seats it holds.
|
||||
Subscribe []string `json:"subscribe,omitempty"`
|
||||
// Answers is the proof subjects it answers, as a watcher of a kinded seat.
|
||||
Answers []string `json:"answers,omitempty"`
|
||||
// Workers are the work queues it takes from, as a holder.
|
||||
Workers []Worker `json:"workers,omitempty"`
|
||||
// Records is the direct-get subjects of the records it reads under its own name.
|
||||
Records []string `json:"records,omitempty"`
|
||||
// Kinds are the holders of every kinded bench it uses or watches, with what each promises: the one
|
||||
// account of which channel is which, and what it can carry, that the router judges an answer by. The
|
||||
// controller's, from the claims, never a channel's word (novox/hq ADR 0259 §5).
|
||||
Kinds []KindHeld `json:"kinds,omitempty"`
|
||||
}
|
||||
|
||||
// KindHeld is one kind of a kinded bench and who holds it.
|
||||
type KindHeld struct {
|
||||
Seat string `json:"seat"`
|
||||
Kind string `json:"kind"`
|
||||
Module string `json:"module"`
|
||||
Node string `json:"node"`
|
||||
Capabilities []string `json:"capabilities,omitempty"`
|
||||
}
|
||||
|
||||
// KindedBenches are the seats that may be kinded (ADR 0234 §2): making another is a decision, recorded.
|
||||
var KindedBenches = map[string]bool{"channel": true, "intake": true}
|
||||
|
||||
// WorkerName is the worker a seat's holders pull from: one for the seat, or one per kind on a kinded bench.
|
||||
func WorkerName(seat, kind string) string {
|
||||
if kind == "" {
|
||||
return "SEAT_" + upperSnake(seat) + "_worker"
|
||||
}
|
||||
return "SEAT_" + upperSnake(seat) + "_" + upperSnake(kind) + "_worker"
|
||||
}
|
||||
|
||||
func namesVerb(list []string, s string) bool {
|
||||
for _, x := range list {
|
||||
if x == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// SeatTrafficOf derives one module's seat traffic from the seats it holds, uses and watches. Only seats
|
||||
// carrying one of the rules above are read: every other seat is composed as it always was.
|
||||
func SeatTrafficOf(module string, holds, uses, watches []Seat) SeatTraffic {
|
||||
var t SeatTraffic
|
||||
for _, s := range holds {
|
||||
if !s.isNewTraffic() {
|
||||
continue
|
||||
}
|
||||
kind := ""
|
||||
if s.Kinded {
|
||||
kind = s.Kind
|
||||
if kind == "" || !safeSubject.MatchString(kind) {
|
||||
// A kinded claim without a usable kind is refused at registration; here it is granted
|
||||
// nothing, which is the same answer at the last place it could be asked.
|
||||
continue
|
||||
}
|
||||
}
|
||||
if len(s.Accepts) > 0 {
|
||||
stream := seatStreamName(s.Name)
|
||||
filter := "mesh.seat." + s.Name + ".accept.>"
|
||||
if kind != "" {
|
||||
filter = "mesh.seat." + s.Name + ".accept.*." + kind
|
||||
}
|
||||
t.Workers = append(t.Workers, Worker{Stream: stream, Consumer: WorkerName(s.Name, kind), Filter: filter})
|
||||
}
|
||||
for _, a := range s.Accepts {
|
||||
switch {
|
||||
case kind != "":
|
||||
t.Subscribe = append(t.Subscribe, seatSubject(s, "accept", a+"."+kind))
|
||||
case namesVerb(s.ByCaller, a):
|
||||
t.Subscribe = append(t.Subscribe, seatSubject(s, "accept", a+".*"))
|
||||
}
|
||||
}
|
||||
for _, e := range s.Emits {
|
||||
switch {
|
||||
case kind != "":
|
||||
t.Publish = append(t.Publish, seatSubject(s, "event", e+"."+kind))
|
||||
case namesVerb(s.ByCaller, e):
|
||||
t.Publish = append(t.Publish, seatSubject(s, "event", e+".*"))
|
||||
}
|
||||
}
|
||||
if kind != "" {
|
||||
for _, v := range s.Proofs {
|
||||
t.Publish = append(t.Publish, seatSubject(s, "proof", v+"."+kind))
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, s := range uses {
|
||||
if !s.isNewTraffic() {
|
||||
continue
|
||||
}
|
||||
for _, a := range s.Accepts {
|
||||
switch {
|
||||
case namesVerb(s.ByCaller, a):
|
||||
t.Publish = append(t.Publish, seatSubject(s, "accept", a+"."+module))
|
||||
case s.Kinded && module == s.DeclaredBy:
|
||||
// Work for a kind is put on its queue by the bench's own router, and by no other user.
|
||||
t.Publish = append(t.Publish, seatSubject(s, "accept", a+".*"))
|
||||
}
|
||||
}
|
||||
for _, e := range s.Emits {
|
||||
if namesVerb(s.ByCaller, e) {
|
||||
t.Subscribe = append(t.Subscribe, seatSubject(s, "event", e+"."+module))
|
||||
}
|
||||
}
|
||||
for _, b := range s.Records {
|
||||
if !safeSubject.MatchString(b) {
|
||||
continue
|
||||
}
|
||||
t.Records = append(t.Records, "$JS.API.DIRECT.GET.KV_"+b+".$KV."+b+"."+module+".>")
|
||||
}
|
||||
}
|
||||
for _, w := range watches {
|
||||
if w.Kinded {
|
||||
for _, e := range w.Emits {
|
||||
t.Subscribe = append(t.Subscribe, seatSubject(w, "event", e+".*"))
|
||||
}
|
||||
if module == w.DeclaredBy {
|
||||
// A code is answered by the bench's own router, and by no other watcher.
|
||||
for _, v := range w.Proofs {
|
||||
t.Answers = append(t.Answers, seatSubject(w, "proof", v+".*"))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Publish = unique(t.Publish)
|
||||
t.Subscribe = unique(t.Subscribe)
|
||||
t.Answers = unique(t.Answers)
|
||||
t.Records = unique(t.Records)
|
||||
sort.Slice(t.Workers, func(i, j int) bool { return t.Workers[i].Consumer < t.Workers[j].Consumer })
|
||||
return t
|
||||
}
|
||||
|
||||
// grants is the bus permissions seat traffic needs: the subjects themselves, and the JetStream API a
|
||||
// worker is pulled and acknowledged through and a record is read through.
|
||||
func (t SeatTraffic) grants() (pub, sub []string) {
|
||||
pub = append(pub, t.Publish...)
|
||||
sub = append(sub, t.Subscribe...)
|
||||
sub = append(sub, t.Answers...)
|
||||
for _, w := range t.Workers {
|
||||
pub = append(pub,
|
||||
"$JS.API.CONSUMER.INFO."+w.Stream+"."+w.Consumer,
|
||||
"$JS.API.CONSUMER.MSG.NEXT."+w.Stream+"."+w.Consumer,
|
||||
"$JS.ACK."+w.Stream+"."+w.Consumer+".>")
|
||||
}
|
||||
pub = append(pub, t.Records...)
|
||||
return pub, sub
|
||||
}
|
||||
|
||||
// isNewTraffic says whether a seat carries any of the rules above, so a seat that carries none is
|
||||
// composed exactly as before them.
|
||||
func (s Seat) isNewTraffic() bool {
|
||||
return s.Kinded || len(s.ByCaller) > 0 || len(s.Proofs) > 0 || len(s.Records) > 0
|
||||
}
|
||||
|
||||
// plainVerbs is a seat's accepts or emits with those the rules above compose taken out: a verb named by
|
||||
// its caller and every verb of a kinded bench are composed by SeatTrafficOf and nowhere else.
|
||||
func plainVerbs(s Seat, verbs []string) []string {
|
||||
if s.Kinded {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, v := range verbs {
|
||||
if !namesVerb(s.ByCaller, v) {
|
||||
out = append(out, v)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// SeatTrafficObjects is the work queues and workers the seat traffic of every composed user implies
|
||||
// (novox/hq ADR 0259 §3): a queue for each seat a holder takes work from, and each holder's worker on it —
|
||||
// one per kind on a kinded bench, filtered to that kind, so the kinds never take each other's work. Only
|
||||
// seats carrying the rules above; the mesh's own seats' queues are RaiseSeats'.
|
||||
func SeatTrafficObjects(users []Principal) ([]Stream, []Consumer) {
|
||||
streams := map[string]Stream{}
|
||||
consumers := map[string]Consumer{}
|
||||
add := func(module string, holds []Seat) {
|
||||
for _, w := range SeatTrafficOf(module, holds, nil, nil).Workers {
|
||||
seat := ""
|
||||
for _, s := range holds {
|
||||
if seatStreamName(s.Name) == w.Stream {
|
||||
seat = s.Name
|
||||
}
|
||||
}
|
||||
streams[w.Stream] = Stream{
|
||||
Name: w.Stream,
|
||||
Subjects: []string{"mesh.seat." + seat + ".accept.>"},
|
||||
Retention: RetentionWorkQueue,
|
||||
MaxAge: 7 * 24 * 60 * 60,
|
||||
Why: "work submitted to the " + seat + " seat; its holders take it, each kind its own, and it queues while nobody does",
|
||||
}
|
||||
consumers[w.Consumer] = Consumer{
|
||||
Name: w.Consumer,
|
||||
Stream: w.Stream,
|
||||
Filters: []string{w.Filter},
|
||||
AckWaitSeconds: 60,
|
||||
// No bound on redelivery: a channel away for a day keeps its work, offered again later and
|
||||
// later by its holder's runtime (novox/hq ADR 0259; the correctness review of 2026-10-08).
|
||||
MaxDeliver: 0,
|
||||
Why: module + " holds " + seat + "; it pulls one ask at a time and acknowledges once it has " +
|
||||
"recorded it, so a crash redelivers rather than loses",
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, p := range users {
|
||||
switch p.Kind {
|
||||
case KindModule:
|
||||
add(p.Module, p.Holds)
|
||||
case KindNodeTools:
|
||||
for _, d := range p.Carries {
|
||||
add(d.Module, d.Holds)
|
||||
}
|
||||
}
|
||||
}
|
||||
var ss []Stream
|
||||
for _, s := range streams {
|
||||
ss = append(ss, s)
|
||||
}
|
||||
sort.Slice(ss, func(i, j int) bool { return ss[i].Name < ss[j].Name })
|
||||
var cs []Consumer
|
||||
for _, c := range consumers {
|
||||
cs = append(cs, c)
|
||||
}
|
||||
sort.Slice(cs, func(i, j int) bool { return cs[i].Name < cs[j].Name })
|
||||
return ss, cs
|
||||
}
|
||||
|
||||
// trustedTraffic is why a module's seat traffic is the trusted holder's (novox/hq ADR 0259 §8), or "": it
|
||||
// says a seat's event to one caller each (a warrant), or holds a kind of a kinded bench that proves its sender.
|
||||
func trustedTraffic(d Declared) string {
|
||||
for _, s := range d.Holds {
|
||||
for _, e := range s.Emits {
|
||||
if namesVerb(s.ByCaller, e) {
|
||||
return "it says " + s.Name + "'s " + e + " to one caller each"
|
||||
}
|
||||
}
|
||||
if s.Kinded && namesVerb(s.Capabilities, "verified-sender") {
|
||||
return "it holds " + s.Name + " of kind " + s.Kind + ", which proves its sender"
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// TrafficQueues is the work queue of every seat naming its caller or its kind that accepts work, held or not
|
||||
// (novox/hq ADR 0259 §3): what is submitted before a holder is assigned waits for it.
|
||||
func TrafficQueues(seats []Seat) []Stream {
|
||||
var out []Stream
|
||||
seen := map[string]bool{}
|
||||
for _, s := range seats {
|
||||
if len(s.Accepts) == 0 || !s.isNewTraffic() || seen[s.Name] {
|
||||
continue
|
||||
}
|
||||
seen[s.Name] = true
|
||||
out = append(out, Stream{Name: seatStreamName(s.Name), Subjects: []string{"mesh.seat." + s.Name + ".accept.>"},
|
||||
Retention: RetentionWorkQueue, MaxAge: 7 * 24 * 60 * 60,
|
||||
Why: "work submitted to the " + s.Name + " seat; its holders take it, each kind its own, and it queues while nobody does"})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
|
||||
return out
|
||||
}
|
||||
@@ -1,340 +0,0 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The seats of novox/hq ADR 0259 §3, as the messenger declares them.
|
||||
func operatorChannel() Seat {
|
||||
return Seat{Name: "operator-channel", Scope: "mesh", Accepts: []string{"ask", "cancel"},
|
||||
Emits: []string{"decided"}, Serves: []string{"open", "history", "notify"},
|
||||
ByCaller: []string{"ask", "cancel", "decided"}, Records: []string{"messenger_asks"}}
|
||||
}
|
||||
|
||||
func channelSeat(kind string) Seat {
|
||||
return Seat{Name: "channel", Scope: "mesh", Accepts: []string{"show", "edit", "send"}, Kinded: true, Kind: kind,
|
||||
DeclaredBy: "messenger"}
|
||||
}
|
||||
|
||||
func intakeSeat(kind string) Seat {
|
||||
return Seat{Name: "intake", Scope: "mesh", Emits: []string{"choice", "link"}, Proofs: []string{"code"},
|
||||
Kinded: true, Kind: kind, DeclaredBy: "messenger"}
|
||||
}
|
||||
|
||||
func allowed(patterns []string, subject string) bool {
|
||||
for _, p := range patterns {
|
||||
if subjectMatches(p, subject) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func perms(t *testing.T, p Principal) Permissions {
|
||||
t.Helper()
|
||||
got, err := PermissionsFor(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
func TestAnAskerAsksAndHearsUnderItsOwnNameOnly(t *testing.T) {
|
||||
asker := Principal{Kind: KindModule, Node: "anchor", Module: "mesh-delivery", Uses: []Seat{operatorChannel()}}
|
||||
got := perms(t, asker)
|
||||
for _, s := range []string{
|
||||
"mesh.seat.operator-channel.accept.ask.mesh-delivery",
|
||||
"mesh.seat.operator-channel.accept.cancel.mesh-delivery",
|
||||
"$JS.API.DIRECT.GET.KV_messenger_asks.$KV.messenger_asks.mesh-delivery.a1",
|
||||
} {
|
||||
if !allowed(got.Publish, s) {
|
||||
t.Errorf("an asker may not publish %s", s)
|
||||
}
|
||||
}
|
||||
for _, s := range []string{
|
||||
"mesh.seat.operator-channel.accept.ask.mesh-controller",
|
||||
"mesh.seat.operator-channel.accept.ask.*",
|
||||
"mesh.seat.operator-channel.event.decided.mesh-delivery",
|
||||
"$JS.API.DIRECT.GET.KV_messenger_asks.$KV.messenger_asks.mesh-controller.a1",
|
||||
"$KV.messenger_asks.mesh-delivery.a1",
|
||||
} {
|
||||
if allowed(got.Publish, s) {
|
||||
t.Errorf("an asker may publish %s, which is not its own to submit", s)
|
||||
}
|
||||
}
|
||||
if !allowed(got.Subscribe, "mesh.seat.operator-channel.event.decided.mesh-delivery") {
|
||||
t.Error("an asker does not hear its own warrants")
|
||||
}
|
||||
if allowed(got.Subscribe, "mesh.seat.operator-channel.event.decided.mesh-controller") {
|
||||
t.Error("an asker hears another asker's warrants")
|
||||
}
|
||||
// And its own consumer carries its warrants, so a restart catches up.
|
||||
c, ok := ConsumerFor(asker)
|
||||
if !ok || !allowed(c.Filters, "mesh.seat.operator-channel.event.decided.mesh-delivery") {
|
||||
t.Errorf("the asker's consumer does not carry its warrants: %v", c.Filters)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyTheHolderPublishesAWarrant(t *testing.T) {
|
||||
users, err := Users(Records{
|
||||
Nodes: []string{"anchor"},
|
||||
Assigned: map[string][]Declared{"anchor": {
|
||||
{Module: "messenger", Holds: []Seat{operatorChannel()}, Uses: []Seat{channelSeat("")},
|
||||
Watches: []Seat{{Name: "intake", Emits: []string{"choice", "link"}, Kinded: true, Proofs: []string{"code"}, DeclaredBy: "messenger"}}},
|
||||
{Module: "mesh-delivery", Uses: []Seat{operatorChannel()}},
|
||||
{Module: "telegram", Holds: []Seat{channelSeat("telegram"), intakeSeat("telegram")}},
|
||||
}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, u := range users {
|
||||
got := perms(t, u)
|
||||
says := allowed(got.Publish, "mesh.seat.operator-channel.event.decided.mesh-delivery")
|
||||
if says != (u.Module == "messenger") {
|
||||
t.Errorf("%s %s publish a warrant", u.Username(), map[bool]string{true: "may", false: "may not"}[says])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheHolderTakesEveryCallersAskThroughItsWorker(t *testing.T) {
|
||||
got := perms(t, Principal{Kind: KindModule, Node: "anchor", Module: "messenger", Holds: []Seat{operatorChannel()}})
|
||||
if !allowed(got.Subscribe, "mesh.seat.operator-channel.accept.ask.mesh-delivery") {
|
||||
t.Error("the router does not take an ask")
|
||||
}
|
||||
for _, s := range []string{
|
||||
"$JS.API.CONSUMER.MSG.NEXT.SEAT_OPERATOR_CHANNEL.SEAT_OPERATOR_CHANNEL_worker",
|
||||
"$JS.ACK.SEAT_OPERATOR_CHANNEL.SEAT_OPERATOR_CHANNEL_worker.x",
|
||||
"mesh.seat.operator-channel.event.decided.mesh-controller",
|
||||
} {
|
||||
if !allowed(got.Publish, s) {
|
||||
t.Errorf("the router may not publish %s", s)
|
||||
}
|
||||
}
|
||||
if allowed(got.Publish, "mesh.seat.operator-channel.accept.ask.messenger") {
|
||||
t.Error("the holder may ask its own seat without using it")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAKindedHolderReachesItsOwnKindAndNoOther(t *testing.T) {
|
||||
got := perms(t, Principal{Kind: KindModule, Node: "anchor", Module: "telegram",
|
||||
Holds: []Seat{channelSeat("telegram"), intakeSeat("telegram")}})
|
||||
for _, s := range []string{
|
||||
"mesh.seat.intake.event.choice.telegram", "mesh.seat.intake.event.link.telegram",
|
||||
"mesh.seat.intake.proof.code.telegram",
|
||||
"$JS.API.CONSUMER.MSG.NEXT.SEAT_CHANNEL.SEAT_CHANNEL_TELEGRAM_worker",
|
||||
} {
|
||||
if !allowed(got.Publish, s) {
|
||||
t.Errorf("telegram may not publish %s", s)
|
||||
}
|
||||
}
|
||||
for _, s := range []string{
|
||||
"mesh.seat.intake.event.choice.desktop", "mesh.seat.intake.proof.code.desktop",
|
||||
"mesh.seat.channel.accept.show.telegram",
|
||||
"$JS.API.CONSUMER.MSG.NEXT.SEAT_CHANNEL.SEAT_CHANNEL_DESKTOP_worker",
|
||||
"mesh.seat.operator-channel.event.decided.mesh-delivery",
|
||||
} {
|
||||
if allowed(got.Publish, s) {
|
||||
t.Errorf("telegram may publish %s", s)
|
||||
}
|
||||
}
|
||||
if !allowed(got.Subscribe, "mesh.seat.channel.accept.show.telegram") ||
|
||||
allowed(got.Subscribe, "mesh.seat.channel.accept.show.desktop") {
|
||||
t.Error("telegram does not take exactly its own kind's work")
|
||||
}
|
||||
if allowed(got.Subscribe, "mesh.seat.intake.proof.code.telegram") {
|
||||
t.Error("a channel answers its own proofs")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheWatcherAnswersProofsAndHearsEveryKind(t *testing.T) {
|
||||
got := perms(t, Principal{Kind: KindModule, Node: "anchor", Module: "messenger",
|
||||
Uses: []Seat{channelSeat("")},
|
||||
Watches: []Seat{{Name: "intake", Emits: []string{"choice"}, Kinded: true, Proofs: []string{"code"}, DeclaredBy: "messenger"}}})
|
||||
for _, s := range []string{"mesh.seat.intake.event.choice.telegram", "mesh.seat.intake.proof.code.desktop"} {
|
||||
if !allowed(got.Subscribe, s) {
|
||||
t.Errorf("the router does not hear %s", s)
|
||||
}
|
||||
}
|
||||
if !allowed(got.Publish, "mesh.seat.channel.accept.show.telegram") {
|
||||
t.Error("the router cannot send a channel its work")
|
||||
}
|
||||
if allowed(got.Publish, "mesh.seat.intake.event.choice.telegram") || allowed(got.Publish, "mesh.seat.intake.proof.code.telegram") {
|
||||
t.Error("the router may say a channel's answer or proof")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoStreamKeepsAProof(t *testing.T) {
|
||||
users, _ := Users(Records{Nodes: []string{"anchor"}, Assigned: map[string][]Declared{"anchor": {
|
||||
{Module: "telegram", Holds: []Seat{channelSeat("telegram"), intakeSeat("telegram")}},
|
||||
{Module: "messenger", Holds: []Seat{operatorChannel()}},
|
||||
}}})
|
||||
streams, _ := SeatTrafficObjects(users)
|
||||
streams = append(streams, MeshStreams()...)
|
||||
for _, s := range streams {
|
||||
for _, subject := range s.Subjects {
|
||||
if subjectMatches(subject, "mesh.seat.intake.proof.code.telegram") {
|
||||
t.Errorf("%s keeps a proof (%s)", s.Name, subject)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEachKindHasAWorkerOfItsOwn(t *testing.T) {
|
||||
users, _ := Users(Records{Nodes: []string{"anchor"}, Assigned: map[string][]Declared{"anchor": {
|
||||
{Module: "telegram", Holds: []Seat{channelSeat("telegram")}},
|
||||
{Module: "desk-channel", Holds: []Seat{channelSeat("desktop")}},
|
||||
{Module: "messenger", Holds: []Seat{operatorChannel()}},
|
||||
}}})
|
||||
streams, workers := SeatTrafficObjects(users)
|
||||
names := map[string]string{}
|
||||
for _, w := range workers {
|
||||
names[w.Name] = strings.Join(w.Filters, ",")
|
||||
}
|
||||
want := map[string]string{
|
||||
"SEAT_CHANNEL_TELEGRAM_worker": "mesh.seat.channel.accept.*.telegram",
|
||||
"SEAT_CHANNEL_DESKTOP_worker": "mesh.seat.channel.accept.*.desktop",
|
||||
"SEAT_OPERATOR_CHANNEL_worker": "mesh.seat.operator-channel.accept.>",
|
||||
}
|
||||
for n, f := range want {
|
||||
if names[n] != f {
|
||||
t.Errorf("worker %s filters %q, want %q", n, names[n], f)
|
||||
}
|
||||
}
|
||||
if len(streams) != 2 {
|
||||
t.Errorf("want the queues of channel and operator-channel, got %v", streams)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheRuntimeIsGrantedTheUnionAndTheMembershipEachModulesShare(t *testing.T) {
|
||||
telegram := Declared{Module: "telegram", Holds: []Seat{channelSeat("telegram"), intakeSeat("telegram")}}
|
||||
desk := Declared{Module: "desk-channel", Holds: []Seat{channelSeat("desktop"), intakeSeat("desktop")}}
|
||||
got := perms(t, Principal{Kind: KindNodeTools, Node: "anchor", Module: RuntimeModule, Carries: []Declared{telegram, desk}})
|
||||
for _, s := range []string{"mesh.seat.intake.event.choice.telegram", "mesh.seat.intake.event.choice.desktop"} {
|
||||
if !allowed(got.Publish, s) {
|
||||
t.Errorf("the runtime may not publish %s for a module it carries", s)
|
||||
}
|
||||
}
|
||||
m := MembershipFor("anchor", telegram, Placements{})
|
||||
if m.SeatTraffic == nil || !allowed(m.SeatTraffic.Publish, "mesh.seat.intake.event.choice.telegram") ||
|
||||
allowed(m.SeatTraffic.Publish, "mesh.seat.intake.event.choice.desktop") {
|
||||
t.Errorf("telegram's membership does not list exactly its own kind: %+v", m.SeatTraffic)
|
||||
}
|
||||
if plain := MembershipFor("anchor", Declared{Module: "plain"}, Placements{}); plain.SeatTraffic != nil {
|
||||
t.Error("a module with no such seat is given seat traffic")
|
||||
}
|
||||
}
|
||||
|
||||
func TestASeatWithoutTheNewRulesIsComposedAsBefore(t *testing.T) {
|
||||
old := Seat{Name: "node-build-agent", Scope: "node", Accepts: []string{"build"}, Emits: []string{"built"}}
|
||||
got := perms(t, Principal{Kind: KindModule, Node: "anchor", Module: "builder", Holds: []Seat{old}})
|
||||
for _, s := range []string{"mesh.seat.node-build-agent.event.built",
|
||||
"$JS.API.CONSUMER.MSG.NEXT.SEAT_NODE_BUILD_AGENT.SEAT_NODE_BUILD_AGENT_worker"} {
|
||||
if !allowed(got.Publish, s) {
|
||||
t.Errorf("an old seat's holder lost %s", s)
|
||||
}
|
||||
}
|
||||
if !allowed(got.Subscribe, "mesh.seat.node-build-agent.accept.build") {
|
||||
t.Error("an old seat's holder lost its accept")
|
||||
}
|
||||
}
|
||||
|
||||
// The router learns which channel is which, and what each promises, from the controller's membership:
|
||||
// the claims, never a channel's word (ADR 0259 §5).
|
||||
func TestTheRoutersMembershipNamesEveryKindAndItsCapabilities(t *testing.T) {
|
||||
tg := channelSeat("telegram")
|
||||
tg.Capabilities = []string{"choice", "verified-sender"}
|
||||
desk := channelSeat("desktop")
|
||||
desk.Capabilities = []string{"choice"}
|
||||
router := Declared{Module: "messenger", Holds: []Seat{operatorChannel()}, Uses: []Seat{channelSeat("")}}
|
||||
records := Records{Nodes: []string{"anchor", "laptop"}, Assigned: map[string][]Declared{
|
||||
"anchor": {router, {Module: "telegram", Holds: []Seat{tg}, RunsAs: "telegram"}},
|
||||
"laptop": {{Module: "desk-channel", Holds: []Seat{desk}}},
|
||||
}}
|
||||
where := PlacementsOf(records, nil)
|
||||
m := MembershipFor("anchor", router, where)
|
||||
if m.SeatTraffic == nil || len(m.SeatTraffic.Kinds) != 2 {
|
||||
t.Fatalf("the router is not told the kinds: %+v", m.SeatTraffic)
|
||||
}
|
||||
byKind := map[string]KindHeld{}
|
||||
for _, k := range m.SeatTraffic.Kinds {
|
||||
byKind[k.Kind] = k
|
||||
}
|
||||
if k := byKind["telegram"]; k.Module != "telegram" || k.Node != "anchor" || !namesVerb(k.Capabilities, "verified-sender") {
|
||||
t.Errorf("telegram is %+v", k)
|
||||
}
|
||||
if k := byKind["desktop"]; k.Module != "desk-channel" || namesVerb(k.Capabilities, "verified-sender") {
|
||||
t.Errorf("the desk is %+v", k)
|
||||
}
|
||||
if other := MembershipFor("anchor", Declared{Module: "mesh-delivery", Uses: []Seat{operatorChannel()}}, where); other.SeatTraffic != nil && len(other.SeatTraffic.Kinds) > 0 {
|
||||
t.Error("an asker is told the channels")
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0259 §8: only the bench's own router answers its proofs and puts work on a kind's queue.
|
||||
func TestOnlyTheBenchsRouterAnswersProofsAndSubmitsWork(t *testing.T) {
|
||||
other := perms(t, Principal{Kind: KindModule, Node: "anchor", Module: "eavesdropper",
|
||||
Uses: []Seat{channelSeat("")},
|
||||
Watches: []Seat{{Name: "intake", Emits: []string{"choice"}, Kinded: true, Proofs: []string{"code"}, DeclaredBy: "messenger"}}})
|
||||
if allowed(other.Subscribe, "mesh.seat.intake.proof.code.telegram") {
|
||||
t.Error("a watcher that is not the router answers codes")
|
||||
}
|
||||
if allowed(other.Publish, "mesh.seat.channel.accept.show.telegram") {
|
||||
t.Error("a user that is not the router puts work on a kind's queue")
|
||||
}
|
||||
if !allowed(other.Subscribe, "mesh.seat.intake.event.choice.telegram") {
|
||||
t.Error("a watcher no longer hears the bench's events")
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0259 §8: the machine's runtime runs as the operator's account; it never carries a module
|
||||
// that says warrants or speaks for a kind proving its sender, and such a module has its own account.
|
||||
func TestTheMachinesRuntimeNeverCarriesATrustedHolder(t *testing.T) {
|
||||
tg := channelSeat("telegram")
|
||||
tg.Capabilities = []string{"choice", "verified-sender"}
|
||||
for name, d := range map[string]Declared{
|
||||
"the router": {Module: "messenger", Holds: []Seat{operatorChannel()}},
|
||||
"a verified channel": {Module: "telegram", Holds: []Seat{tg}},
|
||||
} {
|
||||
if _, err := PermissionsFor(Principal{Kind: KindNodeTools, Node: "anchor", Module: RuntimeModule,
|
||||
Carries: []Declared{d}}); err == nil || !strings.Contains(err.Error(), "an account of its own") {
|
||||
t.Errorf("%s was composed into the machine's runtime: %v", name, err)
|
||||
}
|
||||
}
|
||||
desk := channelSeat("desktop")
|
||||
desk.Capabilities = []string{"choice"}
|
||||
if _, err := PermissionsFor(Principal{Kind: KindNodeTools, Node: "anchor", Module: RuntimeModule,
|
||||
Carries: []Declared{{Module: "desk-channel", Holds: []Seat{desk}}}}); err != nil {
|
||||
t.Errorf("a channel proving nothing was refused: %v", err)
|
||||
}
|
||||
users, err := Users(Records{Nodes: []string{"anchor"}, Assigned: map[string][]Declared{"anchor": {
|
||||
{Module: RuntimeModule}, {Module: "telegram", Holds: []Seat{tg}, RunsAs: "telegram"},
|
||||
{Module: "messenger", Holds: []Seat{operatorChannel()}, RunsAs: "messenger"},
|
||||
}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, u := range users {
|
||||
if u.Kind == KindNodeTools {
|
||||
for _, d := range u.Carries {
|
||||
if d.RunsAs != "" {
|
||||
t.Errorf("the machine's runtime carries %s", d.Module)
|
||||
}
|
||||
}
|
||||
if _, err := PermissionsFor(u); err != nil {
|
||||
t.Errorf("the runtime could not be composed: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0259 §8: verified-sender reaches the router only from a holder of its own account.
|
||||
func TestVerifiedSenderIsBelievedOnlyFromAHolderOfItsOwnAccount(t *testing.T) {
|
||||
if got := placedCapabilities([]string{"choice", "verified-sender"}, ""); namesVerb(got, "verified-sender") {
|
||||
t.Errorf("a carried holder keeps verified-sender: %v", got)
|
||||
}
|
||||
if got := placedCapabilities([]string{"choice", "verified-sender"}, "telegram"); !namesVerb(got, "verified-sender") {
|
||||
t.Errorf("a holder of its own account lost verified-sender: %v", got)
|
||||
}
|
||||
}
|
||||
+134
-6
@@ -3,11 +3,13 @@ package broker
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The mesh's own streams.
|
||||
//
|
||||
// **These four and no more** (novox/hq ADR 0116 task 1.4, as revised by ADR 0118). An earlier
|
||||
// **These and no more** (novox/hq ADR 0116 task 1.4, as revised by ADR 0118; the two that keep what a
|
||||
// consumer gave up on added for issue 330). An earlier
|
||||
// reading had the controller create *every* stream at genesis, from a fixed set. That is only the
|
||||
// mesh's own half: a seat's streams are created when the module declaring it is registered, and a
|
||||
// module's durable consumers when it is assigned — neither of which has happened at genesis. What
|
||||
@@ -50,11 +52,80 @@ type Stream struct {
|
||||
// Direct lets a client read a subject's last message without a consumer, which is how a
|
||||
// runtime reads its own membership with no JetStream API beyond one request (ADR 0160).
|
||||
Direct bool
|
||||
// MaxBytes bounds the stream's size, zero for unbounded. With DiscardNew a full stream refuses
|
||||
// what comes next rather than dropping what it holds: the publisher is told, and says so.
|
||||
MaxBytes int64
|
||||
DiscardNew bool
|
||||
// DuplicatesSeconds is the window in which a message id published twice is kept once; zero for the
|
||||
// server's default (two minutes).
|
||||
DuplicatesSeconds int
|
||||
}
|
||||
|
||||
// AssignmentsStream holds every assignment's membership, the newest per subject.
|
||||
const AssignmentsStream = "ASSIGNMENTS"
|
||||
|
||||
// What a durable consumer gave up on is kept (novox/hq issue 330, design 25 §3).
|
||||
//
|
||||
// **The server says it and keeps it; the controller copies it.** A consumer that handed a message over
|
||||
// as often as it may stops offering it and publishes `$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES` with
|
||||
// the stream and the message's sequence. DeadLetterNoticesStream captures those advisories as the
|
||||
// server publishes them, so one said while no controller listens is still there when one starts. The
|
||||
// controller's consumer on it fetches the given-up message by its sequence, while the source stream
|
||||
// still holds it, and keeps a copy in DeadLettersStream under DeadLetterSubject, with the consumer,
|
||||
// the subject, how often it was handed over and when it was given up. It stays there until a person
|
||||
// delivers it again or drops it, with why; a condition is open for as long as it does.
|
||||
const (
|
||||
DeadLetterNoticesStream = "DEAD_LETTER_NOTICES"
|
||||
DeadLettersStream = "DEAD_LETTERS"
|
||||
// MaxDeliveriesAdvisories is the subject the server says a given-up message on.
|
||||
MaxDeliveriesAdvisories = "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>"
|
||||
deadLetterPrefix = "mesh.events.dead."
|
||||
againPrefix = "mesh.again."
|
||||
// DeadLettersBytes bounds the kept copies. Full, the stream refuses the next copy, which is said
|
||||
// as the consumer's condition; it never drops one it holds.
|
||||
DeadLettersBytes = 256 << 20
|
||||
)
|
||||
|
||||
// DeadLetterSubject is where a message one consumer gave up on is kept: `mesh.events.dead.<stream>.<consumer>`.
|
||||
func DeadLetterSubject(stream, consumer string) string {
|
||||
return deadLetterPrefix + stream + "." + consumer
|
||||
}
|
||||
|
||||
// DeadLetterOf is the stream and consumer a kept message's subject names; false for any other subject.
|
||||
func DeadLetterOf(subject string) (stream, consumer string, ok bool) {
|
||||
rest, found := strings.CutPrefix(subject, deadLetterPrefix)
|
||||
if !found {
|
||||
return "", "", false
|
||||
}
|
||||
stream, consumer, ok = strings.Cut(rest, ".")
|
||||
return stream, consumer, ok && stream != "" && consumer != "" && !strings.Contains(consumer, ".")
|
||||
}
|
||||
|
||||
// AgainSubject is where an event given up on is delivered again to the one consumer that gave it up,
|
||||
// and to nobody else: `mesh.again.<consumer>.` and the original subject without its `mesh.`. Every
|
||||
// consumer on EVENTS filters its own (AgainFilter), and a module's runtime reads the event's key from
|
||||
// the tokens around `.event.`, so the handler sees the same key it saw the first time.
|
||||
func AgainSubject(consumer, original string) string {
|
||||
return againPrefix + consumer + "." + strings.TrimPrefix(original, "mesh.")
|
||||
}
|
||||
|
||||
// AgainFilter is the one consumer's share of the subjects events are delivered again on.
|
||||
func AgainFilter(consumer string) string { return againPrefix + consumer + ".>" }
|
||||
|
||||
// OriginalOfAgain is the subject an event delivered again was first published on; false for a subject
|
||||
// that is not one delivered again.
|
||||
func OriginalOfAgain(subject string) (string, bool) {
|
||||
rest, found := strings.CutPrefix(subject, againPrefix)
|
||||
if !found {
|
||||
return "", false
|
||||
}
|
||||
_, original, ok := strings.Cut(rest, ".")
|
||||
if !ok || original == "" {
|
||||
return "", false
|
||||
}
|
||||
return "mesh." + original, true
|
||||
}
|
||||
|
||||
// MeshStreams is the foundation set, in the order a person reads it.
|
||||
//
|
||||
// **CONTROL names its subjects rather than taking `mesh.control.>`**, because heartbeats live
|
||||
@@ -97,7 +168,9 @@ func MeshStreams() []Stream {
|
||||
// A seat's own events ride here too: they are 1:many like any event, and the
|
||||
// `event` token keeps them clear of both the seat's work queue (`accept`) and its
|
||||
// tools (`tool`), which must not be persisted.
|
||||
Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"},
|
||||
// And an event given up on, delivered again to the one consumer that gave it up
|
||||
// (novox/hq issue 330): under `mesh.again.<consumer>.`, which only that consumer filters.
|
||||
Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>", againPrefix + ">"},
|
||||
Retention: RetentionLimits,
|
||||
MaxAge: 7 * 24 * 60 * 60,
|
||||
MaxMsgsPerSubject: 10000,
|
||||
@@ -105,6 +178,26 @@ func MeshStreams() []Stream {
|
||||
"excluded by the event token; per-subject caps keep a noisy emitter from " +
|
||||
"evicting a quiet one without splitting the stream",
|
||||
},
|
||||
{
|
||||
Name: DeadLetterNoticesStream,
|
||||
Subjects: []string{MaxDeliveriesAdvisories},
|
||||
Retention: RetentionWorkQueue,
|
||||
MaxAge: 7 * 24 * 60 * 60,
|
||||
Why: "the server's word that a consumer gave up on a message, kept until the controller has " +
|
||||
"copied the message into DEAD_LETTERS (novox/hq issue 330); a week, the longest the source " +
|
||||
"streams keep what they are about",
|
||||
},
|
||||
{
|
||||
Name: DeadLettersStream,
|
||||
Subjects: []string{deadLetterPrefix + ">"},
|
||||
Retention: RetentionLimits,
|
||||
MaxBytes: DeadLettersBytes,
|
||||
DiscardNew: true,
|
||||
DuplicatesSeconds: 24 * 60 * 60,
|
||||
Why: "every message a consumer gave up on, with its consumer, subject, deliveries and when, kept " +
|
||||
"until a person delivers it again or drops it with why (novox/hq issue 330); no age, and full " +
|
||||
"it refuses the next copy rather than drop one it holds",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -298,7 +391,7 @@ const EventsStream = "EVENTS"
|
||||
//
|
||||
// **Unlimited redelivery on CONTROL, deliberately.** The store window's bound is the controller's,
|
||||
// not the server's (window.go): a message is held with a nak-and-delay until the controller either
|
||||
// takes it or gives up and says so. A max-deliver here would dead-letter a push that was being
|
||||
// takes it or gives up and says so. A max-deliver here would give up on a push that was being
|
||||
// held through a store restart — the exact message the stream exists to protect — some minutes
|
||||
// before the controller had finished deciding about it.
|
||||
func MeshConsumers() []Consumer {
|
||||
@@ -314,7 +407,7 @@ func MeshConsumers() []Consumer {
|
||||
{
|
||||
Name: ControllerName,
|
||||
Stream: "EVENTS",
|
||||
Filters: ControllerFollows,
|
||||
Filters: append(append([]string(nil), ControllerFollows...), AgainFilter(ControllerName)),
|
||||
Push: true,
|
||||
AckWaitSeconds: 30,
|
||||
MaxDeliver: 5,
|
||||
@@ -331,12 +424,47 @@ func MeshConsumers() []Consumer {
|
||||
Resettable: "what it drops is caught up: merges by the catch-up pass (issue 266), build outcomes " +
|
||||
"from the build records (issue 214), a provider's failing word said again (ADR 0224)",
|
||||
Why: "the events the mesh's own controller reacts to, one at a time; after " +
|
||||
"max-deliver it dead-letters, because an announcement it cannot act on will not " +
|
||||
"become actionable",
|
||||
"max-deliver it gives the event up, and the controller keeps it in DEAD_LETTERS until " +
|
||||
"a person delivers it again or drops it",
|
||||
},
|
||||
// What the server said a consumer gave up on (novox/hq issue 330): copied into DEAD_LETTERS and
|
||||
// acknowledged. No max-deliver: a notice the controller could not copy is offered again, and said.
|
||||
{
|
||||
Name: ControllerName,
|
||||
Stream: DeadLetterNoticesStream,
|
||||
Push: true,
|
||||
AckWaitSeconds: 30,
|
||||
Why: "the controller copies each message a consumer gave up on into DEAD_LETTERS; no max-deliver, " +
|
||||
"because a notice it gave up on would lose the message it is about",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// NoticesConsumer is the controller's consumer on DEAD_LETTER_NOTICES (novox/hq issue 330).
|
||||
func NoticesConsumer() Consumer {
|
||||
for _, c := range MeshConsumers() {
|
||||
if c.Stream == DeadLetterNoticesStream {
|
||||
return c
|
||||
}
|
||||
}
|
||||
panic("the mesh's consumers carry none on " + DeadLetterNoticesStream)
|
||||
}
|
||||
|
||||
// AssertServingConsumers are the controller's own consumers its serving cannot go without: all but the
|
||||
// one on DEAD_LETTER_NOTICES, which the keeper of dead letters asserts and retries by itself, so a fault
|
||||
// there never stops the controller serving (novox/hq issue 330).
|
||||
func AssertServingConsumers(e Ensurer) error {
|
||||
for _, c := range MeshConsumers() {
|
||||
if c.Stream == DeadLetterNoticesStream {
|
||||
continue
|
||||
}
|
||||
if err := e.EnsureConsumer(c); err != nil {
|
||||
return fmt.Errorf("asserting consumer %s on %s: %w", c.Name, c.Stream, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Ensurer is the part of a JetStream connection consumer assertion needs, narrow for the reason
|
||||
// Asserter is.
|
||||
type Ensurer interface {
|
||||
|
||||
@@ -127,6 +127,10 @@ func TestEachStreamCarriesTheRetentionItsShapeNeeds(t *testing.T) {
|
||||
"NODES": RetentionLastPerSubject,
|
||||
"EVENTS": RetentionLimits,
|
||||
"ASSIGNMENTS": RetentionLastPerSubject,
|
||||
// What a consumer gave up on (novox/hq issue 330): the server's notice taken once, the message
|
||||
// kept until somebody acts.
|
||||
"DEAD_LETTER_NOTICES": RetentionWorkQueue,
|
||||
"DEAD_LETTERS": RetentionLimits,
|
||||
}
|
||||
got := map[string]Retention{}
|
||||
for _, s := range MeshStreams() {
|
||||
|
||||
+1
-1
@@ -24,7 +24,7 @@ accounts {
|
||||
jetstream: enabled
|
||||
users = [
|
||||
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] }
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] }
|
||||
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
|
||||
@@ -50,9 +50,6 @@ type Declared struct {
|
||||
// Checks are the module's own tools its health asks, each `<module>.<tool>` (novox/hq ADR 0240, to-be
|
||||
// 48 §3): the machine's node-engine asks them of its own node tools, and is granted that and no more.
|
||||
Checks []string
|
||||
// RunsAs is the account the module runs as in a runtime of its own (novox/hq ADR 0259 §8): it is never
|
||||
// carried by the machine's runtime, and reaches the bus on its own account.
|
||||
RunsAs string
|
||||
}
|
||||
|
||||
// Records is what composing a user list needs to know about the mesh, and nothing more.
|
||||
@@ -123,13 +120,10 @@ func Users(r Records) ([]Principal, error) {
|
||||
})
|
||||
}
|
||||
if runtimeHere {
|
||||
var carried []Declared
|
||||
for _, d := range r.Assigned[node] {
|
||||
if d.RunsAs == "" {
|
||||
carried = append(carried, d)
|
||||
}
|
||||
}
|
||||
out = append(out, Principal{Kind: KindNodeTools, Node: node, Module: RuntimeModule, Carries: carried})
|
||||
out = append(out, Principal{
|
||||
Kind: KindNodeTools, Node: node, Module: RuntimeModule,
|
||||
Carries: append([]Declared(nil), r.Assigned[node]...),
|
||||
})
|
||||
}
|
||||
}
|
||||
for _, node := range sortedCopy(r.Enrolling) {
|
||||
|
||||
@@ -133,6 +133,11 @@ var WritersTable = []WriterRow{
|
||||
Others: "—"},
|
||||
{State: "the facts snapshot", Writer: "controller", KeptIn: "the artifact store, facts/latest",
|
||||
Others: "the build seat reads"},
|
||||
// What a consumer gave up on (novox/hq issue 330): copied by the controller from the server's notice,
|
||||
// and delivered again or dropped only through its verb, with why.
|
||||
{State: "a message a consumer gave up on", Writer: "controller", KeptIn: "the bus, the stream " + DeadLettersStream,
|
||||
Others: "read, delivered again or dropped through the controller's dead-letters verb",
|
||||
Subjects: []string{deadLetterPrefix + ">", againPrefix + ">"}, Writes: isController},
|
||||
}
|
||||
|
||||
// CheckWriters refuses a grant that lets a principal publish on a subject the writers table gives
|
||||
|
||||
@@ -30,6 +30,7 @@ var designRows = []string{
|
||||
"a provider's standing",
|
||||
"the operator-channel's open messages",
|
||||
"the facts snapshot",
|
||||
"a message a consumer gave up on",
|
||||
}
|
||||
|
||||
func TestTheWritersTableIsTheDesigns(t *testing.T) {
|
||||
|
||||
@@ -369,11 +369,16 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
// Before placing, because a placement is a setting too.
|
||||
left := r.LeftOut(with.Settings, with.Adopted)
|
||||
kept := make([]Manifest, 0, len(r.Modules))
|
||||
var stillBackedUp []Manifest
|
||||
for _, m := range r.Modules {
|
||||
if why, isLeft := left[m.Module]; isLeft {
|
||||
if leftOut != nil {
|
||||
leftOut[m.Module] = why
|
||||
}
|
||||
// **Its data is still copied** (novox/hq ADR 0262): a module left out runs nothing new, and
|
||||
// the data it already holds on the machine is the reason to keep copying it. Only its data,
|
||||
// as the backup holder's lines are derived from it, and the directories they name.
|
||||
stillBackedUp = append(stillBackedUp, backupView(m))
|
||||
continue
|
||||
}
|
||||
kept = append(kept, m)
|
||||
@@ -979,7 +984,8 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
// seat that places them (novox/hq ADR 0203, ADR 0204). Gathered from every module on
|
||||
// the node, as the jails are, and **last of every placeholder pass**: shell code is a
|
||||
// shell's own syntax, full of `${…}` no pass above should ever be shown.
|
||||
if err := contributionsInto(copied, m, r.Modules, thisMachine, with, r.Capabilities, unplaced); err != nil {
|
||||
contributing := append(append([]Manifest(nil), r.Modules...), stillBackedUp...)
|
||||
if err := contributionsInto(copied, m, contributing, thisMachine, with, r.Capabilities, unplaced); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
copied["id"] = m.Module + "." + fmt.Sprint(resource["id"])
|
||||
@@ -1066,23 +1072,9 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
}
|
||||
owner[fmt.Sprint(process["id"])] = RuntimeModule
|
||||
out = append(out, process)
|
||||
// And a runtime of its own for each module of its own account, after it (novox/hq ADR 0259 §8).
|
||||
owns, err := r.ownRuntimes(with)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, p := range owns {
|
||||
id := fmt.Sprint(p["id"])
|
||||
owner[id] = strings.TrimSuffix(id, "."+OwnRuntimeID())
|
||||
out = append(out, p)
|
||||
}
|
||||
// What each module's bundles are given is read as the account the runtime runs as — not what a
|
||||
// module of its own account is given, which its own account reads.
|
||||
// What each module's bundles are given is read as the account the runtime runs as.
|
||||
words := map[string]map[string]string{}
|
||||
for _, m := range r.Modules {
|
||||
if m.RunsAs != "" {
|
||||
continue
|
||||
}
|
||||
w, err := bundleWords(m, with)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -1,161 +0,0 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The router of novox/hq ADR 0259: it declares the operator's seat and the two kinded benches.
|
||||
func router() Manifest {
|
||||
return Manifest{Module: "messenger", Tools: []string{"open", "history", "notify"},
|
||||
State: []StateDeclaration{{Name: "asks"}}, RunsAs: "messenger", SecretsOwner: "messenger",
|
||||
DefinesSeats: []SeatDeclaration{
|
||||
{Name: "operator-channel", Scope: ScopeMesh, Accepts: []string{"ask", "cancel"}, Emits: []string{"decided"},
|
||||
ByCaller: []string{"ask", "cancel", "decided"}, Records: []string{"asks"},
|
||||
Serves: []Verb{{Name: "open"}, {Name: "history"}, {Name: "notify"}}},
|
||||
{Name: "channel", Scope: ScopeMesh, Kinded: true, Accepts: []string{"show", "edit", "send"}},
|
||||
{Name: "intake", Scope: ScopeMesh, Kinded: true, Emits: []string{"choice", "link"}, Proofs: []string{"code"}},
|
||||
},
|
||||
Claims: []Claim{{Name: "operator-channel", Scope: ScopeMesh}},
|
||||
Uses: []string{"channel"}}
|
||||
}
|
||||
|
||||
func aChannel(module, kind string) Manifest {
|
||||
return Manifest{Module: module, Claims: []Claim{
|
||||
{Name: "channel", Scope: ScopeMesh, Kind: kind}, {Name: "intake", Scope: ScopeMesh, Kind: kind}}}
|
||||
}
|
||||
|
||||
func TestTwoChannelsOfDifferentKindsHoldTheBenches(t *testing.T) {
|
||||
shelf := Shelf{"messenger": router(), "telegram": aChannel("telegram", "telegram"),
|
||||
"desk-channel": aChannel("desk-channel", "desktop")}
|
||||
if got := problemsFor(t, shelf); got != "" {
|
||||
t.Fatalf("two kinds were refused: %s", got)
|
||||
}
|
||||
for _, m := range shelf {
|
||||
if got := declaredSeatProblems(m); len(got) > 0 {
|
||||
t.Fatalf("%s: %v", m.Module, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestASecondClaimOfOneKindIsRefused(t *testing.T) {
|
||||
got := problemsFor(t, Shelf{"messenger": router(), "telegram": aChannel("telegram", "telegram"),
|
||||
"telegram-two": aChannel("telegram-two", "telegram")})
|
||||
if !strings.Contains(got, `of kind "telegram", which telegram already claims`) {
|
||||
t.Fatalf("a second holder of one kind stood: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAKindedBenchNeedsAKindAndNoOtherSeatTakesOne(t *testing.T) {
|
||||
got := problemsFor(t, Shelf{"messenger": router(), "nameless": aChannel("nameless", "")})
|
||||
if !strings.Contains(got, "claims the kinded bench channel and names no kind") {
|
||||
t.Fatalf("a claim without a kind stood: %s", got)
|
||||
}
|
||||
odd := Manifest{Module: "odd", Claims: []Claim{{Name: "operator-channel", Scope: ScopeMesh, Kind: "telegram"}}}
|
||||
got = problemsFor(t, Shelf{"messenger": router(), "odd": odd})
|
||||
if !strings.Contains(got, "only a kinded bench takes a kind") {
|
||||
t.Fatalf("a kind on a seat that is not kinded stood: %s", got)
|
||||
}
|
||||
dotted := problemsFor(t, Shelf{"messenger": router(), "dotted": aChannel("dotted", "a.b")})
|
||||
if !strings.Contains(dotted, "not a usable name") {
|
||||
t.Fatalf("a kind that would widen a subject stood: %s", dotted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyChannelAndIntakeAreKinded(t *testing.T) {
|
||||
m := Manifest{Module: "x", DefinesSeats: []SeatDeclaration{{Name: "pager", Kinded: true, Accepts: []string{"page"}}}}
|
||||
if got := strings.Join(declaredSeatProblems(m), "; "); !strings.Contains(got, "only channel and intake are kinded") {
|
||||
t.Fatalf("another kinded bench was declared: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheNewRulesAreHeldToWhatTheSeatSays(t *testing.T) {
|
||||
m := Manifest{Module: "x", DefinesSeats: []SeatDeclaration{{Name: "thing", Accepts: []string{"do"},
|
||||
ByCaller: []string{"undo"}, Proofs: []string{"code"}, Records: []string{"nothing"}}}}
|
||||
got := strings.Join(declaredSeatProblems(m), "; ")
|
||||
for _, want := range []string{"names thing.undo by its caller, which the seat neither accepts nor emits",
|
||||
"declares proofs on thing, which is not kinded", `read its records "nothing", which it keeps no state of`} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("not refused: %q in %s", want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Two kinds on one machine are two holders, and one kind on two machines is a second claimant.
|
||||
func TestEachKindIsItsOwnHolderWhenResolved(t *testing.T) {
|
||||
modules := []Manifest{aChannel("telegram", "telegram"), aChannel("desk-channel", "desktop")}
|
||||
held, problems := checkClaims(modules, Node{Name: "anchor"}, nil, nil)
|
||||
if len(problems) > 0 || len(held) != 4 {
|
||||
t.Fatalf("two kinds on one machine: held %v, problems %v", held, problems)
|
||||
}
|
||||
_, problems = checkClaims([]Manifest{aChannel("telegram", "telegram")}, Node{Name: "home"}, held, nil)
|
||||
if len(problems) == 0 {
|
||||
t.Fatal("one kind held on two machines was not refused")
|
||||
}
|
||||
}
|
||||
|
||||
// A channel's capabilities come from the fixed vocabulary, and only a kinded claim carries any.
|
||||
func TestCapabilitiesAreTheVocabularysAndOnlyOnAKindedClaim(t *testing.T) {
|
||||
good := aChannel("telegram", "telegram")
|
||||
good.Claims[0].Capabilities = []string{"deliver", "choice", "verified-sender", "max-length:4096"}
|
||||
good.RunsAs = "telegram"
|
||||
if got := problemsFor(t, Shelf{"messenger": router(), "telegram": good}); got != "" {
|
||||
t.Fatalf("the vocabulary was refused: %s", got)
|
||||
}
|
||||
bad := aChannel("telegram", "telegram")
|
||||
bad.Claims[0].Capabilities = []string{"trusted", "max-length:lots"}
|
||||
got := problemsFor(t, Shelf{"messenger": router(), "telegram": bad})
|
||||
for _, w := range []string{`"trusted"`, `"max-length:lots"`} {
|
||||
if !strings.Contains(got, w+", which channel-capabilities/1 does not have") {
|
||||
t.Errorf("%s was not refused: %s", w, got)
|
||||
}
|
||||
}
|
||||
odd := Manifest{Module: "odd", Claims: []Claim{{Name: "operator-channel", Scope: ScopeMesh, Capabilities: []string{"deliver"}}}}
|
||||
if got := problemsFor(t, Shelf{"messenger": router(), "odd": odd}); !strings.Contains(got, "only a kinded bench's claim carries them") {
|
||||
t.Errorf("capabilities on a seat that is not kinded stood: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0259 §8: a module saying warrants, or speaking for a kind that proves its sender, runs as an
|
||||
// account of its own — never carried by the machine's runtime, which runs as the operator's account.
|
||||
func TestATrustedHolderMustRunAsAnAccountOfItsOwn(t *testing.T) {
|
||||
r := router()
|
||||
r.RunsAs = ""
|
||||
if got := problemsFor(t, Shelf{"messenger": r}); !strings.Contains(got, "messenger must run as an account of its own") {
|
||||
t.Errorf("a router on the machine's runtime stood: %s", got)
|
||||
}
|
||||
tg := aChannel("telegram", "telegram")
|
||||
tg.Claims[0].Capabilities = []string{"choice", "verified-sender"}
|
||||
if got := problemsFor(t, Shelf{"messenger": router(), "telegram": tg}); !strings.Contains(got, "telegram must run as an account of its own") {
|
||||
t.Errorf("a verified channel on the machine's runtime stood: %s", got)
|
||||
}
|
||||
desk := aChannel("desk-channel", "desktop")
|
||||
desk.Claims[0].Capabilities = []string{"choice"}
|
||||
if got := problemsFor(t, Shelf{"messenger": router(), "desk-channel": desk}); got != "" {
|
||||
t.Errorf("a channel proving nothing was held to it: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunsAsIsAnAccountOfTheModulesOwn(t *testing.T) {
|
||||
ok := Manifest{Module: "telegram", RunsAs: "telegram", SecretsOwner: "telegram",
|
||||
OwnSecrets: OwnSecrets{"broker": {Path: "/var/lib/telegram/broker"}},
|
||||
Resources: []map[string]any{{"id": "account", "type": "user", "name": "telegram"}}}
|
||||
if got := RunsAsProblems(ok); len(got) != 0 {
|
||||
t.Fatalf("a sound runs-as was refused: %v", got)
|
||||
}
|
||||
for want, change := range map[string]func(*Manifest){
|
||||
"never root": func(m *Manifest) { m.RunsAs, m.SecretsOwner = "root", "root" },
|
||||
"not an account name": func(m *Manifest) { m.RunsAs = "${machine:account}" },
|
||||
"which it does not make": func(m *Manifest) { m.Resources = nil },
|
||||
"declares no own secret": func(m *Manifest) { m.OwnSecrets = nil },
|
||||
"they are the account's own": func(m *Manifest) { m.SecretsOwner = "" },
|
||||
} {
|
||||
m := ok
|
||||
m.Resources = append([]map[string]any(nil), ok.Resources...)
|
||||
m.OwnSecrets = OwnSecrets{"broker": {Path: "/x"}}
|
||||
change(&m)
|
||||
if got := strings.Join(RunsAsProblems(m), "; "); !strings.Contains(got, want) {
|
||||
t.Errorf("want %q, got %q", want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -64,13 +64,6 @@ type Claim struct {
|
||||
// text/template over one piece — its fields and `module` — in the tool's own grammar (novox/hq ADR
|
||||
// 0255). The data is the mesh's, the format the holder's, as a module's facts template is.
|
||||
Renders map[string]string `json:"renders,omitempty"`
|
||||
// Kind is the kind this module holds a kinded bench as (novox/hq ADR 0234 §2, ADR 0259): `telegram`,
|
||||
// `desktop`. Refused on any other seat, and a second claim of one kind is refused.
|
||||
Kind string `json:"kind,omitempty"`
|
||||
// Capabilities are what a channel of this kind promises, from the fixed vocabulary
|
||||
// channel-capabilities/1 (novox/hq ADR 0234 §2): the router judges an answer by these, read from the
|
||||
// controller's record of this claim and never from the channel.
|
||||
Capabilities []string `json:"capabilities,omitempty"`
|
||||
}
|
||||
|
||||
// ServesFor is what this claim offers a seat's protocol: the verbs it names, else the module's
|
||||
@@ -478,6 +471,11 @@ type Manifest struct {
|
||||
// machine's declaration by name until the controller is updated (LeftOut).
|
||||
unknown string
|
||||
|
||||
// bestEffort marks the view of a left-out module that only its backup lines are made from
|
||||
// (backupView, novox/hq ADR 0262): a line of it that cannot be placed is said in the plan's list of
|
||||
// what could not be placed, never an error that would cost the whole machine its declaration.
|
||||
bestEffort bool
|
||||
|
||||
// Data is every kind of data this module keeps — its own, by directory, and what it keeps for
|
||||
// its consumers, by provision — each with a class the mesh protects and watches it by (novox/hq
|
||||
// ADR 0233). One list: the backup holder's lines, the bindings that do not move, what an
|
||||
@@ -642,13 +640,6 @@ type Manifest struct {
|
||||
// cannot use.
|
||||
SecretsOwner string `json:"secrets-owner,omitempty"`
|
||||
|
||||
// RunsAs is the account this module's tools bundle runs as, in a runtime of its own on a bus account of
|
||||
// its own (novox/hq ADR 0259 §8): never the machine's runtime, which runs as the operator's account and
|
||||
// carries every module on the machine. The account is one the module makes (a `user` resource of that
|
||||
// name), owns its secrets (`secrets-owner`), and is neither root nor the operator's. Required of a module
|
||||
// that says a warrant, or speaks for a channel kind that proves its sender.
|
||||
RunsAs string `json:"runs-as,omitempty"`
|
||||
|
||||
// Keeps is where this module wants every operator-sealed secret in the mesh written — the
|
||||
// vault's field, and so far nobody else's (novox/hq ADR 0085, amended).
|
||||
//
|
||||
@@ -1297,16 +1288,21 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error {
|
||||
if asUnknownField(err) == nil {
|
||||
return err
|
||||
}
|
||||
// Read without it where the key is at the top; where it is inside a block, the block's own
|
||||
// decoder refuses it again, and the manifest keeps its name and version alone. Either way the
|
||||
// module is left out of every declaration by name (LeftOut), so nothing runs on a part-read
|
||||
// manifest.
|
||||
// Read without it, at whatever depth it is (prunedFields): the module is left out of every
|
||||
// declaration by name (LeftOut), and still provides what it provides, and still has its data
|
||||
// copied, so nothing that requires it is refused and nothing it holds goes uncopied.
|
||||
unknown = err.Error()
|
||||
fields = manifestFields{}
|
||||
if json.Unmarshal(rest, &fields) != nil {
|
||||
var pruned []string
|
||||
var perr error
|
||||
if fields, pruned, perr = prunedFields(keys); perr != nil {
|
||||
// Not read past: the manifest keeps its name and version alone. It is left out and raised
|
||||
// all the same, and one manifest never fails the whole catalogue.
|
||||
fields = manifestFields{}
|
||||
_ = json.Unmarshal(keys["module"], &fields.Module)
|
||||
_ = json.Unmarshal(keys["version"], &fields.Version)
|
||||
unknown += " (read no further: " + perr.Error() + ")"
|
||||
} else {
|
||||
unknown += " (read without " + strings.Join(pruned, ", ") + ")"
|
||||
}
|
||||
}
|
||||
*m = Manifest(fields)
|
||||
@@ -1624,7 +1620,6 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
||||
// prefix. Whether a seat anybody names exists, and whether a holder answers for it, are
|
||||
// facts about the catalogue and are checked at registration (CatalogueProblems).
|
||||
problems = append(problems, declaredSeatProblems(m)...)
|
||||
problems = append(problems, RunsAsProblems(m)...)
|
||||
if m.Computed != "" && len(m.Resources) > 0 {
|
||||
// One or the other. A module that both ships files and has them computed would leave
|
||||
// nobody able to say where a given file came from.
|
||||
|
||||
@@ -115,16 +115,6 @@ type Held struct {
|
||||
Node string
|
||||
Module string
|
||||
Site string
|
||||
// Kind is the kind a kinded bench is held as (novox/hq ADR 0234 §2): each kind is its own holder.
|
||||
Kind string
|
||||
}
|
||||
|
||||
// heldKey is what one holder holds: the seat, and its kind on a kinded bench.
|
||||
func heldKey(claim, kind string) string {
|
||||
if kind == "" {
|
||||
return canonicalSeat(claim)
|
||||
}
|
||||
return canonicalSeat(claim) + "/" + kind
|
||||
}
|
||||
|
||||
// Resolution is what a node should run, and why.
|
||||
@@ -874,7 +864,7 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
|
||||
// **One seat under either of its names** (novox/hq ADR 0122): a manifest registered before
|
||||
// a rename claims the former name, and one written after it the current — two claimants of
|
||||
// one seat, compared by the seat they resolve to and not by how each spelled it.
|
||||
seat := heldKey(c.Name, c.Kind)
|
||||
seat := canonicalSeat(c.Name)
|
||||
if other, taken := byScope[scope][seat]; taken {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s and %s both claim %q, and only one thing may hold it per %s",
|
||||
@@ -883,14 +873,14 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
|
||||
}
|
||||
byScope[scope][seat] = m.Module
|
||||
held = append(held, Held{Claim: c.Name, Scope: scope, Node: node.Name,
|
||||
Module: m.Module, Site: node.Site, Kind: c.Kind})
|
||||
Module: m.Module, Site: node.Site})
|
||||
}
|
||||
}
|
||||
|
||||
// And against the rest of the mesh, for the scopes that reach past this machine.
|
||||
for _, h := range held {
|
||||
for _, e := range elsewhere {
|
||||
if e.Node == node.Name || heldKey(e.Claim, e.Kind) != heldKey(h.Claim, h.Kind) || e.Scope != h.Scope {
|
||||
if e.Node == node.Name || canonicalSeat(e.Claim) != canonicalSeat(h.Claim) || e.Scope != h.Scope {
|
||||
continue
|
||||
}
|
||||
switch h.Scope {
|
||||
|
||||
@@ -165,10 +165,6 @@ func (r Resolution) runtimeProcess(with Rendering) (map[string]any, error) {
|
||||
if with.Adopted && m.Filtering != nil {
|
||||
continue
|
||||
}
|
||||
if m.RunsAs != "" {
|
||||
// Served by a runtime of its own, on its own account (ownRuntimes): never the machine's.
|
||||
continue
|
||||
}
|
||||
words, err := bundleWords(m, with)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -227,84 +223,6 @@ func (r Resolution) runtimeProcess(with Rendering) (map[string]any, error) {
|
||||
return process, nil
|
||||
}
|
||||
|
||||
// OwnRuntimeID names the process a module of its own account is served by (novox/hq ADR 0259 §8).
|
||||
func OwnRuntimeID() string { return "own-runtime" }
|
||||
|
||||
// ownRuntimes are the processes the modules of their own account are served by (novox/hq ADR 0259 §8): each
|
||||
// the machine's runtime program — the same build, run from the same source — serving that one module alone,
|
||||
// as the module's own account, on the module's own bus credential. Never the machine's runtime, which runs
|
||||
// as the operator's account and carries every module on the machine.
|
||||
func (r Resolution) ownRuntimes(with Rendering) ([]map[string]any, error) {
|
||||
var own []Manifest
|
||||
for _, m := range r.Modules {
|
||||
if m.RunsAs != "" && !(with.Adopted && m.Filtering != nil) {
|
||||
own = append(own, m)
|
||||
}
|
||||
}
|
||||
if len(own) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
var runtime *Manifest
|
||||
for i := range r.Modules {
|
||||
if r.Modules[i].Module == RuntimeModule {
|
||||
runtime = &r.Modules[i]
|
||||
}
|
||||
}
|
||||
if runtime == nil || len(runtime.Bundles) != 1 || runtime.Bundles[0].Binary == "" {
|
||||
return nil, fmt.Errorf("%s runs as its own account in a runtime of its own, and %s is not here to run it "+
|
||||
"from: assign %s to %s first (novox/hq ADR 0259)", own[0].Module, RuntimeModule, RuntimeModule, r.Node)
|
||||
}
|
||||
program := runtime.Bundles[0]
|
||||
var out []map[string]any
|
||||
for _, m := range own {
|
||||
credential, declared := m.OwnSecrets["broker"]
|
||||
if !declared {
|
||||
return nil, fmt.Errorf("%s runs as its own account and declares no own secret broker", m.Module)
|
||||
}
|
||||
var served, restartOn []string
|
||||
for _, b := range m.Bundles {
|
||||
for _, load := range b.Loads {
|
||||
if launcher, has := b.Launchers[load]; has {
|
||||
load = launcher
|
||||
}
|
||||
served = append(served, m.Module+"="+BundlePath(m.Module, b.Name)+"/"+load)
|
||||
}
|
||||
if len(b.Loads) > 0 {
|
||||
restartOn = append(restartOn, m.Module+"."+BundleID(b.Name))
|
||||
}
|
||||
}
|
||||
if len(served) == 0 {
|
||||
return nil, fmt.Errorf("%s runs as its own account and its build produced no bundle to serve", m.Module)
|
||||
}
|
||||
sort.Strings(served)
|
||||
restartOn = append(restartOn, m.Module+"."+NeedID("broker"))
|
||||
sort.Strings(restartOn)
|
||||
env := map[string]string{RuntimeToolModules: strings.Join(served, ","), RuntimeBrokerFile: credential.Path}
|
||||
words, err := bundleWords(m, with)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(words) > 0 {
|
||||
body, err := json.Marshal(map[string]map[string]string{m.Module: words})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
env[RuntimeToolEnv] = string(body)
|
||||
}
|
||||
process := map[string]any{
|
||||
"id": m.Module + "." + OwnRuntimeID(), "type": "process", "name": m.Module + "-runtime",
|
||||
"source": program.Source, "digest": program.Digest,
|
||||
"run": []any{"./" + program.Binary}, "env": env, "restart-on": toAny(restartOn),
|
||||
"user": m.RunsAs,
|
||||
}
|
||||
if err := artifactsInto(process, RuntimeModule, with); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, process)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func toAny(in []string) []any {
|
||||
out := make([]any, 0, len(in))
|
||||
for _, s := range in {
|
||||
|
||||
@@ -457,49 +457,3 @@ func TestAGoToolsBundleIsServedByItsBinary(t *testing.T) {
|
||||
t.Error("a Go bundle loading a file it does not contain was admitted")
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0259 §8: a module of its own account is served by a runtime of its own — the machine's
|
||||
// runtime program, as that account, on that module's own credential — and never by the machine's runtime,
|
||||
// which runs as the operator's account and is given none of its words.
|
||||
func TestAModuleOfItsOwnAccountIsServedByARuntimeOfItsOwn(t *testing.T) {
|
||||
with := Rendering{ArtifactStore: "anchor.internal:5101",
|
||||
Needed: map[string]map[string]string{RuntimeModule: {"broker": "sealed-credential"}, "telegram": {"broker": "own"}}}
|
||||
goRuntime := Manifest{Module: RuntimeModule, Version: "1",
|
||||
OwnSecrets: OwnSecrets{"broker": {Path: "/var/lib/mesh/" + RuntimeModule + "/broker"}},
|
||||
Build: &Build{Artifacts: []Artifact{{Name: "runtime", Kind: ArtifactBundle, Language: "go",
|
||||
System: "arch", From: "cmd/node-tools"}}}}
|
||||
goRuntime, err := goRuntime.Resolve([]Built{{Name: "runtime", Kind: ArtifactBundle,
|
||||
Reference: ArtifactStoreScheme + RuntimeModule + "/runtime/blobs/" + bundleDigest, Digest: bundleDigest}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
telegram := aToolsModule(t, "telegram", "tools/index.js")
|
||||
telegram.RunsAs, telegram.SecretsOwner = "telegram", "telegram"
|
||||
telegram.OwnSecrets = OwnSecrets{"broker": {Path: "/var/lib/telegram/broker"}}
|
||||
out, err := Resolution{Node: "anchor", Account: "ops",
|
||||
Modules: []Manifest{aToolsModule(t, "nftables", "tools/index.js"), telegram, goRuntime}}.Declaration(with)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
machine := fileNamed(out, RuntimeModule+"."+RuntimeProcessID())
|
||||
if served := machine["env"].(map[string]string)[RuntimeToolModules]; strings.Contains(served, "telegram") || !strings.Contains(served, "nftables") {
|
||||
t.Errorf("the machine's runtime serves %q", served)
|
||||
}
|
||||
own := fileNamed(out, "telegram."+OwnRuntimeID())
|
||||
if own == nil {
|
||||
t.Fatalf("telegram has no runtime of its own: %v", ids(out))
|
||||
}
|
||||
env := own["env"].(map[string]string)
|
||||
if own["user"] != "telegram" || fmt.Sprint(own["run"]) != "[./node-tools]" ||
|
||||
env[RuntimeBrokerFile] != "/var/lib/telegram/broker" ||
|
||||
env[RuntimeToolModules] != "telegram="+BundleRoot+"/telegram/tools/tools/index.js" {
|
||||
t.Errorf("its own runtime: user %v run %v env %v", own["user"], own["run"], env)
|
||||
}
|
||||
if _, told := env[RuntimeOperatorAccount]; told {
|
||||
t.Error("a runtime of a module's own account is told the operator's account")
|
||||
}
|
||||
// Without the machine's runtime to run it from, it is refused in words.
|
||||
if _, err := (Resolution{Node: "anchor", Modules: []Manifest{telegram}}).ownRuntimes(with); err == nil {
|
||||
t.Error("a module of its own account composed without a runtime program")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -212,9 +212,20 @@ func seatContributions(modules []Manifest, holder Manifest, placeholder string,
|
||||
return shapedContributions(modules, holder, facts["name"], s, r, where, caps)
|
||||
}
|
||||
var failed error
|
||||
var unplacedLines []string
|
||||
var b strings.Builder
|
||||
for _, m := range inModuleOrder(modules) {
|
||||
named := false
|
||||
// A left-out module's lines are best effort (novox/hq ADR 0262): what cannot be placed is said,
|
||||
// and the machine is declared without it. A placement setting that does not read is not
|
||||
// replaced by the definition's own path, which may not be where the data is.
|
||||
if m.bestEffort && r.Dirs {
|
||||
if _, err := Places(m, with.Settings[m.Module]); err != nil {
|
||||
unplacedLines = append(unplacedLines, fmt.Sprintf("%s's %s for %s, kept while it is left out, "+
|
||||
"is not placed: its placement setting does not read (%v)", m.Module, kind, s.Name, err))
|
||||
continue
|
||||
}
|
||||
}
|
||||
for _, c := range m.allContributions() {
|
||||
if c.Kind != kind || !capable(c, caps) {
|
||||
continue
|
||||
@@ -222,15 +233,12 @@ func seatContributions(modules []Manifest, holder Manifest, placeholder string,
|
||||
if cs, known := SeatNamed(c.Seat); !known || cs.Name != s.Name {
|
||||
continue
|
||||
}
|
||||
if !named {
|
||||
fmt.Fprintf(&b, "%s %s\n", r.Comment, m.Module)
|
||||
named = true
|
||||
}
|
||||
content := c.Content
|
||||
var lineFailed error
|
||||
if r.Dirs {
|
||||
filled, err := dirFill(content, dirsFor(m, with), m.Module)
|
||||
if err != nil && failed == nil {
|
||||
failed = err
|
||||
if err != nil && lineFailed == nil {
|
||||
lineFailed = err
|
||||
}
|
||||
// An operator's path the module was given, as an item of data on it (novox/hq ADR 0233).
|
||||
if accessRef.MatchString(filled) {
|
||||
@@ -238,15 +246,15 @@ func seatContributions(modules []Manifest, holder Manifest, placeholder string,
|
||||
if err == nil {
|
||||
filled, err = accessFill(filled, byID, m.Module)
|
||||
}
|
||||
if err != nil && failed == nil {
|
||||
failed = err
|
||||
if err != nil && lineFailed == nil {
|
||||
lineFailed = err
|
||||
}
|
||||
}
|
||||
for _, key := range machineUsed(filled) {
|
||||
value, has := facts[key]
|
||||
if !has {
|
||||
if failed == nil {
|
||||
failed = fmt.Errorf("%s's %s for %s says ${machine:%s}, and this machine says %s",
|
||||
if lineFailed == nil {
|
||||
lineFailed = fmt.Errorf("%s's %s for %s says ${machine:%s}, and this machine says %s",
|
||||
m.Module, kind, s.Name, key, orNothing(namesOfFacts(facts)))
|
||||
}
|
||||
continue
|
||||
@@ -255,11 +263,25 @@ func seatContributions(modules []Manifest, holder Manifest, placeholder string,
|
||||
}
|
||||
content = filled
|
||||
}
|
||||
if lineFailed != nil {
|
||||
if m.bestEffort {
|
||||
unplacedLines = append(unplacedLines, fmt.Sprintf("%s's %s for %s, kept while it is left "+
|
||||
"out, is not placed: %v", m.Module, kind, s.Name, lineFailed))
|
||||
continue
|
||||
}
|
||||
if failed == nil {
|
||||
failed = lineFailed
|
||||
}
|
||||
}
|
||||
if !named {
|
||||
fmt.Fprintf(&b, "%s %s\n", r.Comment, m.Module)
|
||||
named = true
|
||||
}
|
||||
b.WriteString(content)
|
||||
if !strings.HasSuffix(content, "\n") {
|
||||
b.WriteString("\n")
|
||||
}
|
||||
}
|
||||
}
|
||||
return b.String(), nil, failed
|
||||
return b.String(), unplacedLines, failed
|
||||
}
|
||||
|
||||
@@ -304,6 +304,11 @@ var defaultSeats = append([]Seat{
|
||||
// Where it is held, its holder writes the resolver file and the uplink's holder steps back from it
|
||||
// (node_resolver.go). It knows nothing of any VPN: its verbs route domains to servers over a link.
|
||||
{Name: ResolverSeat, Scope: ScopeNode, Decision: "novox/hq ADR 0247", Serves: resolverVerbs()},
|
||||
// A machine's shares and the shares it mounts (novox/hq ADR 0263): the holder of node-nfs-server
|
||||
// exports a machine's folders to the private network and provides each as `nfs-share`; the holder of
|
||||
// node-mounts writes a mount and an automount unit per share on a machine that asks (shares.go).
|
||||
{Name: NFSServerSeat, Scope: ScopeNode, Decision: "novox/hq ADR 0263", Serves: nfsServerVerbs()},
|
||||
{Name: MountsSeat, Scope: ScopeNode, Decision: "novox/hq ADR 0263", Serves: mountsVerbs()},
|
||||
},
|
||||
// The graphical session's roles (novox/hq ADR 0208), last because they are a workstation's.
|
||||
graphicalSessionSeats()...)
|
||||
|
||||
@@ -2,7 +2,6 @@ package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
@@ -52,35 +51,6 @@ type SeatDeclaration struct {
|
||||
// owns its own, which is why a seat is also the answer for a module that needs retention
|
||||
// its events cannot have.
|
||||
RetainSeconds int `json:"retain-seconds,omitempty"`
|
||||
|
||||
// Kinded makes the seat a kinded bench (novox/hq ADR 0234 §2, ADR 0259 §3): its holders are different
|
||||
// modules, each claiming one kind, and each verb's subject carries the kind. Only the benches in
|
||||
// KindedBenches may be kinded; making another is a decision, recorded.
|
||||
Kinded bool `json:"kinded,omitempty"`
|
||||
// ByCaller are accepts and emits whose subject's last token names the calling module (ADR 0259 §3): a
|
||||
// user submits such an accept, and hears such an event, under its own name and no other.
|
||||
ByCaller []string `json:"by-caller,omitempty"`
|
||||
// Proofs are verbs carried as core request and reply, never on a stream: what travels on them (a code
|
||||
// the operator typed) is never kept (ADR 0259 §3). On a kinded bench a holder asks with its own kind and
|
||||
// the modules watching the seat answer.
|
||||
Proofs []string `json:"proofs,omitempty"`
|
||||
// Records are state buckets of the declaring module that each user reads under its own name — the
|
||||
// keys `<user>.…` and no other (ADR 0259 §3).
|
||||
Records []string `json:"records,omitempty"`
|
||||
}
|
||||
|
||||
// KindedBenches are the seats that may be kinded (novox/hq ADR 0234 §2): `channel` sends to the operator,
|
||||
// `intake` takes what the operator answers. Another is a decision, recorded, as ADR 0223 asks of a bench.
|
||||
var KindedBenches = map[string]bool{"channel": true, "intake": true}
|
||||
|
||||
// NamedByCaller says whether one of the seat's verbs is named by its caller.
|
||||
func (s SeatDeclaration) NamedByCaller(verb string) bool {
|
||||
for _, v := range s.ByCaller {
|
||||
if v == verb {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// At is this declaration's scope, with the default applied. Mesh by default, because a seat
|
||||
@@ -162,7 +132,6 @@ func declaredSeatProblems(m Manifest) []string {
|
||||
"%s declares %s.%s, which is not a usable verb", m.Module, s.Name, v))
|
||||
}
|
||||
}
|
||||
problems = append(problems, trafficProblems(m, s)...)
|
||||
}
|
||||
|
||||
for _, u := range m.Uses {
|
||||
@@ -173,56 +142,6 @@ func declaredSeatProblems(m Manifest) []string {
|
||||
return problems
|
||||
}
|
||||
|
||||
// trafficProblems is what one declaration of the rules of ADR 0259 §3 can be judged on alone.
|
||||
func trafficProblems(m Manifest, s SeatDeclaration) []string {
|
||||
var problems []string
|
||||
if s.Kinded && !KindedBenches[s.Name] {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s declares %s as a kinded bench; only channel and intake are kinded, and another is a "+
|
||||
"decision, recorded (novox/hq ADR 0234)", m.Module, s.Name))
|
||||
}
|
||||
if s.Kinded && len(s.ByCaller) > 0 {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s declares %s kinded and names verbs by their caller; a kinded bench's subjects carry the kind",
|
||||
m.Module, s.Name))
|
||||
}
|
||||
for _, v := range s.ByCaller {
|
||||
inAccepts, inEmits := false, false
|
||||
for _, a := range s.Accepts {
|
||||
inAccepts = inAccepts || a == v
|
||||
}
|
||||
for _, e := range s.Emits {
|
||||
inEmits = inEmits || e == v
|
||||
}
|
||||
if !inAccepts && !inEmits {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s names %s.%s by its caller, which the seat neither accepts nor emits", m.Module, s.Name, v))
|
||||
}
|
||||
}
|
||||
for _, v := range s.Proofs {
|
||||
if !name.MatchString(v) || strings.Contains(v, ".") {
|
||||
problems = append(problems, fmt.Sprintf("%s declares the proof %s.%s, which is not a usable verb",
|
||||
m.Module, s.Name, v))
|
||||
}
|
||||
}
|
||||
if len(s.Proofs) > 0 && !s.Kinded {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s declares proofs on %s, which is not kinded; a proof is asked by a holder of a kind",
|
||||
m.Module, s.Name))
|
||||
}
|
||||
for _, r := range s.Records {
|
||||
kept := false
|
||||
for _, st := range m.State {
|
||||
kept = kept || st.Name == r
|
||||
}
|
||||
if !kept {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s says %s's users read its records %q, which it keeps no state of", m.Module, s.Name, r))
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// A Shelf is every manifest the mesh has registered, by module name.
|
||||
type Shelf map[string]Manifest
|
||||
|
||||
@@ -263,19 +182,8 @@ func CatalogueProblems(shelf Shelf) []string {
|
||||
return ok
|
||||
}
|
||||
|
||||
// Who claims each kind of a kinded bench, so a second claim of one kind is refused (ADR 0234 §2).
|
||||
kindsTaken := map[string]string{}
|
||||
|
||||
for _, module := range shelfOrder(shelf) {
|
||||
m := shelf[module]
|
||||
for _, c := range m.Claims {
|
||||
if c.Kind != "" {
|
||||
if _, isModuleSeat := declared[c.Name]; !isModuleSeat {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s claims %s of kind %q, and only a kinded bench takes a kind", module, c.Name, c.Kind))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A `uses` naming nothing is where ADR 0110's guarantee lands under a derived set: the
|
||||
// same refusal, at the same moment, from a set nobody maintains by hand.
|
||||
@@ -306,7 +214,6 @@ func CatalogueProblems(shelf Shelf) []string {
|
||||
}
|
||||
continue
|
||||
}
|
||||
problems = append(problems, kindProblems(module, c, s, kindsTaken)...)
|
||||
if c.At() != s.At() {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s claims %s at scope %q, and %s declares it at %s",
|
||||
@@ -321,16 +228,6 @@ func CatalogueProblems(shelf Shelf) []string {
|
||||
}
|
||||
}
|
||||
}
|
||||
// **A trusted holder runs as its own account** (novox/hq ADR 0259 §8): a module saying warrants, or
|
||||
// speaking for a kind that proves its sender, is never carried by a machine's runtime.
|
||||
for _, module := range shelfOrder(shelf) {
|
||||
m := shelf[module]
|
||||
if why := TrustedHolding(m, declared); why != "" && m.RunsAs == "" {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s must run as an account of its own (runs-as): %s, and the machine's runtime runs as the "+
|
||||
"operator's account, which every agent runs as (novox/hq ADR 0259)", module, why))
|
||||
}
|
||||
}
|
||||
// A read of a module's state that module does not keep (novox/hq ADR 0201) — said only where the
|
||||
// owner is on the shelf, as a consumer may be installed before its emitter.
|
||||
var manifests []Manifest
|
||||
@@ -342,63 +239,6 @@ func CatalogueProblems(shelf Shelf) []string {
|
||||
return problems
|
||||
}
|
||||
|
||||
// ChannelCapabilities is the fixed vocabulary `channel-capabilities/1` (novox/hq ADR 0234 §2): a word
|
||||
// outside it is refused. `max-length:<N>` takes a number.
|
||||
var ChannelCapabilities = map[string]bool{
|
||||
"deliver": true, "reaches-away": true, "loud": true, "silent": true, "edit": true,
|
||||
"reaches-when-mesh-down": true, "private": true,
|
||||
"choice": true, "reply": true, "threads": true, "operator-first": true,
|
||||
"verified-sender": true, "exact-render": true, "code-factor": true, "key-factor": true,
|
||||
}
|
||||
|
||||
var maxLength = regexp.MustCompile(`^max-length:[1-9][0-9]{0,6}$`)
|
||||
|
||||
// capabilityProblems are the words of a claim outside the vocabulary, and capabilities on a claim of a
|
||||
// seat that is not kinded.
|
||||
func capabilityProblems(module string, c Claim, kinded bool) []string {
|
||||
if len(c.Capabilities) == 0 {
|
||||
return nil
|
||||
}
|
||||
if !kinded {
|
||||
return []string{fmt.Sprintf("%s claims %s with capabilities, and only a kinded bench's claim carries them",
|
||||
module, c.Name)}
|
||||
}
|
||||
var problems []string
|
||||
for _, w := range c.Capabilities {
|
||||
if !ChannelCapabilities[w] && !maxLength.MatchString(w) {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s claims %s with the capability %q, which channel-capabilities/1 does not have", module, c.Name, w))
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// kindProblems is a claim judged against a declared seat's kind: a kinded bench takes one claim per kind,
|
||||
// a usable name; any other seat takes none.
|
||||
func kindProblems(module string, c Claim, s SeatDeclaration, taken map[string]string) []string {
|
||||
if problems := capabilityProblems(module, c, s.Kinded); len(problems) > 0 {
|
||||
return problems
|
||||
}
|
||||
switch {
|
||||
case !s.Kinded && c.Kind != "":
|
||||
return []string{fmt.Sprintf("%s claims %s of kind %q, and only a kinded bench takes a kind",
|
||||
module, c.Name, c.Kind)}
|
||||
case !s.Kinded:
|
||||
return nil
|
||||
case c.Kind == "":
|
||||
return []string{fmt.Sprintf("%s claims the kinded bench %s and names no kind", module, c.Name)}
|
||||
case !name.MatchString(c.Kind) || strings.Contains(c.Kind, "."):
|
||||
return []string{fmt.Sprintf("%s claims %s of kind %q, which is not a usable name", module, c.Name, c.Kind)}
|
||||
}
|
||||
key := c.Name + "/" + c.Kind
|
||||
if first, ok := taken[key]; ok && first != module {
|
||||
return []string{fmt.Sprintf("%s claims %s of kind %q, which %s already claims; a kind has one holder",
|
||||
module, c.Name, c.Kind, first)}
|
||||
}
|
||||
taken[key] = module
|
||||
return nil
|
||||
}
|
||||
|
||||
// unserved is what a seat's protocol promises and the claimant does not answer. Only the tools
|
||||
// are checked: `accepts` and `emits` are wired by the runtime from the declaration, while a tool
|
||||
// is code the module either has or has not written — under the claim's serves, or among its own.
|
||||
@@ -426,65 +266,3 @@ func shelfOrder(shelf Shelf) []string {
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
var accountName = regexp.MustCompile(`^[a-z_][a-z0-9_-]{0,30}$`)
|
||||
|
||||
// RunsAsProblems is what one manifest's `runs-as` is held to (novox/hq ADR 0259 §8): an account of the
|
||||
// module's own making — a `user` resource of that name — that owns its secrets, with a bus account of its own,
|
||||
// and that is neither root nor the operator's.
|
||||
func RunsAsProblems(m Manifest) []string {
|
||||
if m.RunsAs == "" {
|
||||
return nil
|
||||
}
|
||||
var problems []string
|
||||
say := func(format string, a ...any) { problems = append(problems, fmt.Sprintf(format, a...)) }
|
||||
switch {
|
||||
case !accountName.MatchString(m.RunsAs):
|
||||
say("%s runs as %q, which is not an account name of the module's own", m.Module, m.RunsAs)
|
||||
return problems
|
||||
case m.RunsAs == "root":
|
||||
say("%s runs as root; a module of its own account runs as an account it makes, never root", m.Module)
|
||||
}
|
||||
made := false
|
||||
for _, r := range m.Resources {
|
||||
if fmt.Sprint(r["type"]) == "user" && fmt.Sprint(r["name"]) == m.RunsAs {
|
||||
made = true
|
||||
}
|
||||
}
|
||||
if !made {
|
||||
say("%s runs as %s, which it does not make: a user resource named %s", m.Module, m.RunsAs, m.RunsAs)
|
||||
}
|
||||
if _, has := m.OwnSecrets["broker"]; !has {
|
||||
say("%s runs as its own account and declares no own secret broker: its runtime reaches the bus on an "+
|
||||
"account of its own", m.Module)
|
||||
}
|
||||
if m.SecretsOwner != m.RunsAs {
|
||||
say("%s runs as %s, and its secrets belong to %q: they are the account's own", m.Module, m.RunsAs, m.SecretsOwner)
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// TrustedHolding is why a module must run as its own account (novox/hq ADR 0259 §8), or "": it holds a seat
|
||||
// whose events it says to one caller each (a warrant), or speaks for a kind of a kinded bench that proves
|
||||
// its sender. Neither may be carried by the machine's runtime, which runs as the operator's account.
|
||||
func TrustedHolding(m Manifest, declared map[string]SeatDeclaration) string {
|
||||
for _, c := range m.Claims {
|
||||
s, ok := declared[c.Name]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, e := range s.Emits {
|
||||
if s.NamedByCaller(e) {
|
||||
return fmt.Sprintf("it holds %s, whose %s it says to one caller each", c.Name, e)
|
||||
}
|
||||
}
|
||||
if s.Kinded {
|
||||
for _, capability := range c.Capabilities {
|
||||
if capability == "verified-sender" {
|
||||
return fmt.Sprintf("it holds %s of kind %s, which proves its sender", c.Name, c.Kind)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
@@ -46,7 +46,7 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
|
||||
delivered[s.Delivers] = s.Name
|
||||
}
|
||||
}
|
||||
// Thirty-eight with node-resolver (novox/hq ADR 0247); thirty-seven with mesh-delivery (novox/hq ADR 0239); thirty-six since node-resolver-config retired
|
||||
// Forty with node-nfs-server and node-mounts (novox/hq ADR 0263); thirty-eight with node-resolver (novox/hq ADR 0247); thirty-seven with mesh-delivery (novox/hq ADR 0239); thirty-six since node-resolver-config retired
|
||||
// into node-uplink (novox/hq ADR 0223); thirty-seven
|
||||
// since the retired node-dns-resolver went (novox/hq ADR 0220); thirty-eight with
|
||||
// node-backup (novox/hq ADR 0214); thirty-seven with node-message-bus (novox/hq ADR 0215);
|
||||
@@ -57,8 +57,8 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
|
||||
// node-container-runtime (ADR 0207); nineteen with node-environment and node-login-shell (ADR 0203,
|
||||
// ADR 0204); seventeen with node-build-agent (ADR 0190). One fewer once the retired
|
||||
// mesh-build-machine row goes, when no registered manifest claims it.
|
||||
if len(Seats()) != 38 {
|
||||
t.Errorf("the mesh defines %d seats rather than 38; the set is closed, so a change here is "+
|
||||
if len(Seats()) != 40 {
|
||||
t.Errorf("the mesh defines %d seats rather than 40; the set is closed, so a change here is "+
|
||||
"a decision (novox/hq ADR 0110): %s", len(Seats()), seatNames())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,20 +57,27 @@ var operatorsOwn = map[string]bool{
|
||||
"identity": true, "login": true, "user": true, "username": true, "account": true, "owner": true,
|
||||
"uid": true, "gid": true, "puid": true, "pgid": true,
|
||||
"password": true, "pass": true, "passwd": true, "passphrase": true, "secret": true, "token": true,
|
||||
"key": true, "apikey": true, "bearer": true, "cert": true, "credential": true,
|
||||
"key": true, "apikey": true, "bearer": true, "cert": true, "certificate": true, "credential": true,
|
||||
"nameserver": true, "gateway": true, "subnet": true, "sender": true, "recipient": true, "contact": true,
|
||||
"trusted": true, "whitelist": true, "peer": true, "bind": true, "listen": true, "upstream": true,
|
||||
"proxy": true, "admin": true, "mac": true,
|
||||
}
|
||||
|
||||
// operatorsCompounds are names of two words that are the operator's though neither word alone says so
|
||||
// at the end of a key: a client's identifier, and a name the world knows a site or server by.
|
||||
var operatorsCompounds = map[string]bool{
|
||||
"client-id": true, "site-name": true, "server-name": true, "public-name": true, "smtp-relay": true,
|
||||
"host-name": true, "user-name": true, "domain-name": true, "dns-server": true,
|
||||
// Whom a rule lets in or keeps out: a list of addresses or networks.
|
||||
"allow-from": true, "deny-from": true, "allow-list": true,
|
||||
}
|
||||
|
||||
// aboutAnAmount are first words that make a key about how many or whether, never about whom:
|
||||
// `max-tokens` is a number, `show-hostname` a switch.
|
||||
// `max-tokens` is a number, `show-hostname` a switch. Not `allow` or `use`: `allow-from` and
|
||||
// `use-host` name whom.
|
||||
var aboutAnAmount = map[string]bool{
|
||||
"max": true, "min": true, "num": true, "count": true, "show": true, "hide": true, "enable": true,
|
||||
"disable": true, "use": true, "allow": true,
|
||||
"disable": true,
|
||||
}
|
||||
|
||||
// operatorsWord is what in a key's name says its value is the operator's, or "". A key is about its
|
||||
@@ -82,13 +89,23 @@ func operatorsWord(key string) string {
|
||||
return ""
|
||||
}
|
||||
for i, w := range words {
|
||||
if !operatorsOwn[w] && strings.HasSuffix(w, "s") && operatorsOwn[strings.TrimSuffix(w, "s")] {
|
||||
switch {
|
||||
case operatorsOwn[w]:
|
||||
case strings.HasSuffix(w, "ies") && operatorsOwn[strings.TrimSuffix(w, "ies")+"y"]:
|
||||
words[i] = strings.TrimSuffix(w, "ies") + "y"
|
||||
case strings.HasSuffix(w, "s") && operatorsOwn[strings.TrimSuffix(w, "s")]:
|
||||
words[i] = strings.TrimSuffix(w, "s")
|
||||
}
|
||||
}
|
||||
if n := len(words); n > 1 {
|
||||
if pair := words[n-2] + "-" + words[n-1]; operatorsCompounds[pair] {
|
||||
return pair
|
||||
// Where a secret or an identity is kept is the operator's too: `password-file`, `token-path`.
|
||||
if (words[n-1] == "file" || words[n-1] == "path") && operatorsOwn[words[n-2]] {
|
||||
return words[n-2] + "-" + words[n-1]
|
||||
}
|
||||
for _, last := range []string{words[n-1], strings.TrimSuffix(words[n-1], "s")} {
|
||||
if pair := words[n-2] + "-" + last; operatorsCompounds[pair] {
|
||||
return pair
|
||||
}
|
||||
}
|
||||
}
|
||||
if last := words[len(words)-1]; operatorsOwn[last] {
|
||||
|
||||
@@ -223,7 +223,10 @@ func TestANodeCalledDefaultIsANodesLayer(t *testing.T) {
|
||||
func TestAKeyIsTheOperatorsByWhatItIsAbout(t *testing.T) {
|
||||
for _, key := range []string{"max-tokens", "show-hostname", "ghost-opacity", "users-per-page", "mailbox-size",
|
||||
"font-size", "width", "keyboard-delay", "ipv6-preferred", "client-width", "user-agent", "url-timeout",
|
||||
"site-title", "cert-renewal-days", "name", "font-name"} {
|
||||
"site-title", "cert-renewal-days", "name", "font-name", "allow-resize", "use-gpu", "disable-sender-check",
|
||||
"max-recipients", "gateway-timeout", "sender-delay", "upstream-resolvers", "mirror-countries",
|
||||
"pool-region", "proxy-timeout", "listen-backlog", "peer-keepalive", "admin-theme", "log-file",
|
||||
"cache-path"} {
|
||||
if w := operatorsWord(key); w != "" {
|
||||
t.Errorf("%s read as the operator's (%s)", key, w)
|
||||
}
|
||||
@@ -235,7 +238,17 @@ func TestAKeyIsTheOperatorsByWhatItIsAbout(t *testing.T) {
|
||||
"allowed-hosts": "host", "admin-emails": "email", "tokens": "token", "hostname": "hostname",
|
||||
"apikey": "apikey", "servername": "servername", "tls-cert": "cert", "site": "site", "timezone": "timezone",
|
||||
"bearer": "bearer", "bind-ipv4": "ipv4", "listen-ipv6": "ipv6", "site-name": "site-name",
|
||||
"server-name": "server-name", "public-name": "public-name", "smtp-relay": "smtp-relay"} {
|
||||
"server-name": "server-name", "public-name": "public-name", "smtp-relay": "smtp-relay",
|
||||
"host-name": "host-name", "user-name": "user-name", "domain-name": "domain-name",
|
||||
"nameserver": "nameserver", "upstream-nameservers": "nameserver", "dns-server": "dns-server",
|
||||
"dns-servers": "dns-server", "default-gateway": "gateway", "lan-subnet": "subnet", "sender": "sender",
|
||||
"notify-recipients": "recipient", "contact": "contact", "tls-certificate": "certificate",
|
||||
"allow-from": "allow-from", "deny-from": "deny-from", "allow-hosts": "host", "use-host": "host",
|
||||
"trusted": "trusted", "allow-list": "allow-list", "ip-whitelist": "whitelist", "peer": "peer",
|
||||
"wireguard-peers": "peer", "bind": "bind", "listen": "listen", "upstream": "upstream", "http-proxy": "proxy",
|
||||
"trusted-proxies": "proxy", "admin": "admin", "notify-admin": "admin", "wake-mac": "mac",
|
||||
"password-file": "password-file", "key-file": "key-file", "token-path": "token-path",
|
||||
"secret-file": "secret-file", "cert-path": "cert-path"} {
|
||||
if w := operatorsWord(key); w != word {
|
||||
t.Errorf("%s: read %q, want %q", key, w, word)
|
||||
}
|
||||
@@ -332,3 +345,106 @@ func TestAStoredManifestWithAnUnknownKeyIsLeftOutAndRegistrationRefusesIt(t *tes
|
||||
t.Fatal("a malformed stored manifest was read")
|
||||
}
|
||||
}
|
||||
|
||||
// A module left out for a key this controller does not know, inside an entry, is read past that key
|
||||
// alone: it still provides what it provides, its other entries are whole, and a key of the same name
|
||||
// that another entry knows is kept (novox/hq ADR 0262).
|
||||
func TestALeftOutModuleStillProvidesWhatItProvides(t *testing.T) {
|
||||
var m Manifest
|
||||
raw := `{"module": "later", "version": "2",
|
||||
"provides": [{"name": "db", "scope": "mesh"}, {"name": "cache", "a-field-from-later": 1}],
|
||||
"state": [{"name": "s", "history": 3}],
|
||||
"data": {"own": [{"id": "d", "path": "${dir:d}", "class": "valuable", "backup": {"dump": "x", "into": "d", "class": "later"}}]},
|
||||
"resources": [{"id": "d", "type": "directory", "mode": "0700"}]}`
|
||||
if err := json.Unmarshal([]byte(raw), &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(m.Provides) != 2 || m.Provides[0].Name != "db" || m.Provides[1].Name != "cache" {
|
||||
t.Fatalf("provides: %+v", m.Provides)
|
||||
}
|
||||
if len(m.State) != 1 || m.State[0].History != 3 {
|
||||
t.Fatalf("state: %+v", m.State)
|
||||
}
|
||||
if m.Data == nil || len(m.Data.Own) != 1 || m.Data.Own[0].Class != "valuable" {
|
||||
t.Fatalf("data: the item's own class was taken for the backup's unknown one: %+v", m.Data)
|
||||
}
|
||||
for _, want := range []string{"provides[1].a-field-from-later", "data.own[0].backup.class"} {
|
||||
if !strings.Contains(m.UnknownField(), want) {
|
||||
t.Errorf("unknown %q does not say it read without %s", m.UnknownField(), want)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(UnknownFieldReason(m), "left out") {
|
||||
t.Fatal(UnknownFieldReason(m))
|
||||
}
|
||||
}
|
||||
|
||||
// A left-out module's data is still copied: the backup holder on its machine keeps its lines while every
|
||||
// other thing of it is left out.
|
||||
func TestALeftOutModulesDataIsStillBackedUp(t *testing.T) {
|
||||
var later Manifest
|
||||
if err := json.Unmarshal([]byte(`{"module": "later", "version": "2", "a-field-from-later": 1,
|
||||
"resources": [{"id": "d", "type": "directory", "mode": "0700"}, {"id": "rc", "type": "file", "path": "/etc/later.conf", "content": "x\n"}],
|
||||
"data": {"own": [{"id": "d", "path": "${dir:d}", "class": "valuable"}]}}`), &later); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
holder := Manifest{Module: "backups", Version: "1",
|
||||
Claims: []Claim{{Name: BackupSeat, Scope: ScopeNode}},
|
||||
Resources: []map[string]any{{"id": "list", "type": "file", "path": "/etc/backups.list", "mode": "0644",
|
||||
"content": "${contribution:" + BackupSeat + ":backup}"}}}
|
||||
r := anAdoptedAnchor()
|
||||
r.Modules = append(r.Modules, holder, later)
|
||||
composed, err := r.Compose(anchorRendering(false))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, left := composed.LeftOut["later"]; !left {
|
||||
t.Fatalf("not left out: %v", composed.LeftOut)
|
||||
}
|
||||
got := byID(composed.Resources)
|
||||
if _, declared := got["later.rc"]; declared {
|
||||
t.Fatal("the left-out module's file is still declared")
|
||||
}
|
||||
list, _ := got["backups.list"]["content"].(string)
|
||||
if !strings.Contains(list, "# later") || !strings.Contains(list, "path /var/lib/later/d") {
|
||||
t.Fatalf("the left-out module's data is no longer backed up:\n%s", list)
|
||||
}
|
||||
}
|
||||
|
||||
// A left-out module's backup lines are best effort: a line that cannot be placed — an access nobody
|
||||
// placed, a placement setting that does not read — is said among what could not be placed, and the
|
||||
// machine is declared (novox/hq ADR 0262).
|
||||
func TestALeftOutModulesUnplaceableBackupLineCostsOnlyThatLine(t *testing.T) {
|
||||
holder := Manifest{Module: "backups", Version: "1",
|
||||
Claims: []Claim{{Name: BackupSeat, Scope: ScopeNode}},
|
||||
Resources: []map[string]any{{"id": "list", "type": "file", "path": "/etc/backups.list", "mode": "0644",
|
||||
"content": "${contribution:" + BackupSeat + ":backup}"}}}
|
||||
media := Manifest{Module: "media", Version: "1",
|
||||
Accesses: []Access{{ID: "library", Mode: "read"}},
|
||||
Data: &Data{Own: []DataItem{{ID: "library", Path: "${access:library}", Class: "valuable"}}}}
|
||||
placedBadly := Manifest{Module: "notes", Version: "1",
|
||||
Resources: []map[string]any{{"id": "d", "type": "directory", "path": "/srv/notes", "mode": "0700"}},
|
||||
Data: &Data{Own: []DataItem{{ID: "d", Path: "${dir:d}", Class: "valuable"}}}}
|
||||
r := anAdoptedAnchor()
|
||||
r.Modules = append(r.Modules, holder, media, placedBadly)
|
||||
with := anchorRendering(false)
|
||||
with.Settings["notes"] = []Layer{{From: "anchor", Values: map[string]any{PlacesSetting: "not a map"}}}
|
||||
composed, err := r.Compose(with)
|
||||
if err != nil {
|
||||
t.Fatalf("an unplaceable line of a left-out module failed the machine: %v", err)
|
||||
}
|
||||
for _, m := range []string{"media", "notes"} {
|
||||
if _, left := composed.LeftOut[m]; !left {
|
||||
t.Errorf("%s is not left out: %v", m, composed.LeftOut)
|
||||
}
|
||||
}
|
||||
unplaced := strings.Join(composed.Unplaced, "\n")
|
||||
for _, want := range []string{"media's backup for node-backup", "notes's backup for node-backup", "placement setting does not read"} {
|
||||
if !strings.Contains(unplaced, want) {
|
||||
t.Errorf("not said among what could not be placed: %q in\n%s", want, unplaced)
|
||||
}
|
||||
}
|
||||
list, _ := byID(composed.Resources)["backups.list"]["content"].(string)
|
||||
if strings.Contains(list, "/srv/notes") || strings.Contains(list, "${") {
|
||||
t.Fatalf("a line was placed from the definition's default or unfilled:\n%s", list)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
package catalogue
|
||||
|
||||
// A machine's shares, and the shares a machine mounts (novox/hq ADR 0263).
|
||||
//
|
||||
// **Two roles, one per side of the wire.** A machine that shares a directory with the mesh does it
|
||||
// through the holder of `node-nfs-server`: it writes the machine's export file, runs the NFS service,
|
||||
// opens its port to the private network only, and provides each share as the provision `nfs-share`. A
|
||||
// machine that wants the files gets them through the holder of `node-mounts`: it writes a mount unit and
|
||||
// an automount unit per share, so nothing mounts at boot and nothing can fail a boot, and it says a
|
||||
// device that comes and goes is absent rather than failed.
|
||||
//
|
||||
// **One holder per machine on each side.** Two modules writing one machine's export file, or two writing
|
||||
// mount units for one mount point, is the conflict a seat exists to refuse. Both seats deliver nothing:
|
||||
// `nfs-share` is provided at the mesh's scope, by the module holding `node-nfs-server` on the machine that
|
||||
// shares, and a seat at a node's scope cannot be the answer for a provision at the mesh's.
|
||||
//
|
||||
// **The data is the operator's** (ADR 0051). Neither holder creates, chowns or removes anything under a
|
||||
// shared path; the export maps every client to the path's owner, so no client acts as another account on
|
||||
// the server. Their verbs read, and the ones that act take over what a person wrote by hand only on a
|
||||
// person's word: a dataset's export property, an fstab line.
|
||||
|
||||
// NFSServerSeat is the role of the machine that shares directories over NFS (novox/hq ADR 0263).
|
||||
const NFSServerSeat = "node-nfs-server"
|
||||
|
||||
// MountsSeat is the role that mounts a machine's shares and occasional sources (novox/hq ADR 0263).
|
||||
const MountsSeat = "node-mounts"
|
||||
|
||||
// nfsServerVerbs is the contract every holder of node-nfs-server serves (novox/hq ADR 0263).
|
||||
func nfsServerVerbs() []Verb {
|
||||
return []Verb{
|
||||
{Name: "exports", Description: "Every share this machine exports: its name, its path, read-write or " +
|
||||
"read-only, the owner every client is mapped to (uid and gid), each node it is exported to with that " +
|
||||
"node's private address and access (only the nodes the server grants it to and that ask for it), " +
|
||||
"what is granted and not asked for or asked for and not granted, and whether the kernel holds it " +
|
||||
"now. Also the exports found that are not the mesh's: a dataset's sharenfs property, a line in " +
|
||||
"/etc/exports.",
|
||||
Input: schema(map[string]string{}, nil),
|
||||
Replaces: []string{"exportfs -v", "cat /etc/exports", "zfs get sharenfs"}},
|
||||
{Name: "clients", Description: "Which machines have mounted which share now, as the NFS server " +
|
||||
"knows its clients.",
|
||||
Input: schema(map[string]string{}, nil),
|
||||
Replaces: []string{"ss -tn sport = :2049", "cat /proc/fs/nfsd/clients/*/info"}},
|
||||
{Name: "test", Description: "Whether this machine exports one share for the mesh now, and whether its " +
|
||||
"NFS service is up; with an address, also whether the share is exported to that address: a node's " +
|
||||
"own private address, when the server grants it the share and the node asks for it. What a machine " +
|
||||
"mounting the share asks before it mounts.",
|
||||
Input: schema(map[string]string{
|
||||
"share": "the share, by its name",
|
||||
"address": "a node's private address, to ask whether the share is exported to it (optional)",
|
||||
}, []string{"share"}),
|
||||
Replaces: []string{"showmount -e"}},
|
||||
{Name: "reload", Description: "Have the kernel read this machine's export files again now (exportfs " +
|
||||
"-ra) and answer the shares as it then holds them. The module's own process writes its export " +
|
||||
"file; this is for after a change made outside it. Changes no shared path.",
|
||||
Input: schema(map[string]string{}, nil),
|
||||
Replaces: []string{"exportfs -ra"}},
|
||||
{Name: "adopt", Description: "Take over an export a person made by hand: clear a dataset's sharenfs " +
|
||||
"property for a path the mesh's own export now serves — only when that export is live. Without " +
|
||||
"confirm, says what it would do and changes nothing.",
|
||||
Input: schema(map[string]string{
|
||||
"path": "the shared path whose hand-made export is taken over",
|
||||
"confirm": "\"true\": change it (needs why); anything else is a dry run",
|
||||
"why": "why, for the record",
|
||||
}, []string{"path"}, "confirm"),
|
||||
Replaces: []string{"zfs set sharenfs=off"}},
|
||||
}
|
||||
}
|
||||
|
||||
// mountsVerbs is the contract every holder of node-mounts serves (novox/hq ADR 0263).
|
||||
func mountsVerbs() []Verb {
|
||||
return []Verb{
|
||||
{Name: "list", Description: "Every mount point on this machine: its fstab line, the mesh's mount and " +
|
||||
"automount units for it, its state (armed, mounted, absent, unreachable, failed) and since when, " +
|
||||
"and which settings asked for it. A password in a mount's options is never shown.",
|
||||
Input: schema(map[string]string{}, nil),
|
||||
Replaces: []string{"cat /etc/fstab", "findmnt", "systemctl list-units --type=mount"}},
|
||||
{Name: "test", Description: "Whether one share's or occasional source's server answers from this " +
|
||||
"machine now, without touching its mount point.",
|
||||
Input: schema(map[string]string{"name": "the share or source, by its name"}, []string{"name"}),
|
||||
Replaces: []string{"showmount -e", "ping"}},
|
||||
{Name: "mount", Description: "Mount one share or occasional source now, rather than at its first " +
|
||||
"access.",
|
||||
Input: schema(map[string]string{"name": "the share or source, by its name"}, []string{"name"}),
|
||||
Replaces: []string{"mount"}},
|
||||
{Name: "unmount", Description: "Release one share or occasional source now; its automount stays " +
|
||||
"armed, so the next access mounts it again.",
|
||||
Input: schema(map[string]string{"name": "the share or source, by its name"}, []string{"name"}),
|
||||
Replaces: []string{"umount"}},
|
||||
{Name: "adopt", Description: "Take over a mount a person wrote by hand: comment out the /etc/fstab " +
|
||||
"line for a mount point the mesh's own units now serve, keeping a copy of the file — only when " +
|
||||
"the mesh's automount for it is armed. Without confirm, says what it would do and changes nothing.",
|
||||
Input: schema(map[string]string{
|
||||
"mountpoint": "the mount point whose fstab line is taken over",
|
||||
"confirm": "\"true\": change it (needs why); anything else is a dry run",
|
||||
"why": "why, for the record",
|
||||
}, []string{"mountpoint"}, "confirm"),
|
||||
Replaces: []string{"sed -i /etc/fstab"}},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Defends novox/hq ADR 0263: a machine's shares and the shares a machine mounts are two node seats, each
|
||||
// with its verbs required of every holder, each delivering nothing — `nfs-share` is provided at the mesh's
|
||||
// scope by the holder of node-nfs-server, and a node seat cannot answer for a provision at the mesh's.
|
||||
|
||||
func TestTheSharesAndMountsAreNodeSeatsWithTheirVerbs(t *testing.T) {
|
||||
for seat, want := range map[string]string{
|
||||
NFSServerSeat: "exports clients test reload adopt",
|
||||
MountsSeat: "list test mount unmount adopt",
|
||||
} {
|
||||
s, ok := SeatNamed(seat)
|
||||
if !ok {
|
||||
t.Fatalf("%s is not in the mesh's set", seat)
|
||||
}
|
||||
if s.Scope != ScopeNode || s.Decision != "novox/hq ADR 0263" || s.Delivers != "" || s.Replicated {
|
||||
t.Errorf("%s is %+v; a node seat under ADR 0263 that delivers nothing", seat, s)
|
||||
}
|
||||
var got []string
|
||||
for _, v := range s.Serves {
|
||||
got = append(got, v.Name)
|
||||
if v.Optional {
|
||||
t.Errorf("%s.%s is optional; its first holder serves it", seat, v.Name)
|
||||
}
|
||||
if v.Description == "" || v.Input["type"] != "object" || len(v.Replaces) == 0 {
|
||||
t.Errorf("%s.%s has no description, no object schema or says it replaces nothing", seat, v.Name)
|
||||
}
|
||||
}
|
||||
if strings.Join(got, " ") != want {
|
||||
t.Errorf("%s serves %v, not %s", seat, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Both acts that take over what a person wrote by hand are dry runs unless confirmed, and name the path.
|
||||
func TestTheAdoptVerbsAreDryRunsUnlessConfirmed(t *testing.T) {
|
||||
for seat, key := range map[string]string{NFSServerSeat: "path", MountsSeat: "mountpoint"} {
|
||||
s, _ := SeatNamed(seat)
|
||||
for _, v := range s.Serves {
|
||||
if v.Name != "adopt" {
|
||||
continue
|
||||
}
|
||||
props, _ := v.Input["properties"].(map[string]any)
|
||||
confirm, _ := props["confirm"].(map[string]any)
|
||||
if confirm == nil || confirm["enum"] == nil {
|
||||
t.Errorf("%s.adopt has no confirm switch: %v", seat, v.Input)
|
||||
}
|
||||
if req, _ := v.Input["required"].([]string); len(req) != 1 || req[0] != key {
|
||||
t.Errorf("%s.adopt requires %v, not %q alone", seat, v.Input["required"], key)
|
||||
}
|
||||
if !strings.Contains(v.Description, "Without confirm, says what it would do and changes nothing") {
|
||||
t.Errorf("%s.adopt does not say a call without confirm changes nothing: %s", seat, v.Description)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A holder claiming the seat at a node with every verb holds it; one naming a verb the seat does not
|
||||
// promise, or leaving one out, is refused — as the catalogue's nfs-server and mounts modules claim them.
|
||||
func TestAHolderOfTheShareSeatsServesEveryVerbAndNothingElse(t *testing.T) {
|
||||
cases := []struct {
|
||||
seat, module string
|
||||
verbs []string
|
||||
}{
|
||||
{NFSServerSeat, "nfs-server", []string{"exports", "clients", "test", "reload", "adopt"}},
|
||||
{MountsSeat, "mounts", []string{"list", "test", "mount", "unmount", "adopt"}},
|
||||
}
|
||||
for _, c := range cases {
|
||||
seat, _ := SeatNamed(c.seat)
|
||||
holder := Manifest{Module: c.module, Version: "1",
|
||||
Claims: []Claim{{Name: c.seat, Scope: ScopeNode, Serves: c.verbs}}}
|
||||
if c.seat == NFSServerSeat {
|
||||
holder.Provides = []Offer{{Name: "nfs-share", Scope: ScopeMesh}}
|
||||
}
|
||||
if err := CanHold(holder, seat); err != nil {
|
||||
t.Errorf("%s serving every verb is refused: %v", c.module, err)
|
||||
}
|
||||
typo := holder
|
||||
typo.Claims = []Claim{{Name: c.seat, Scope: ScopeNode, Serves: append(append([]string{}, c.verbs...), "export")}}
|
||||
if err := CanHold(typo, seat); err == nil || !strings.Contains(err.Error(), "does not promise") {
|
||||
t.Errorf("%s naming a verb the seat does not promise was accepted: %v", c.module, err)
|
||||
}
|
||||
short := holder
|
||||
short.Claims = []Claim{{Name: c.seat, Scope: ScopeNode, Serves: c.verbs[:len(c.verbs)-1]}}
|
||||
if err := CanHold(short, seat); err == nil || !strings.Contains(err.Error(), "adopt") {
|
||||
t.Errorf("%s leaving adopt out was accepted: %v", c.module, err)
|
||||
}
|
||||
mesh := holder
|
||||
mesh.Claims = []Claim{{Name: c.seat, Scope: ScopeMesh, Serves: c.verbs}}
|
||||
if err := CanHold(mesh, seat); err == nil {
|
||||
t.Errorf("%s claiming a node seat at the mesh's scope was accepted", c.module)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// What each verb replaces is what an agent would type over ssh to read or change a share by hand.
|
||||
func TestTheShareVerbsSayWhatTheyReplace(t *testing.T) {
|
||||
want := map[string]string{
|
||||
NFSServerSeat + ".exports": "exportfs -v",
|
||||
NFSServerSeat + ".adopt": "zfs set sharenfs=off",
|
||||
MountsSeat + ".list": "cat /etc/fstab",
|
||||
MountsSeat + ".adopt": "sed -i /etc/fstab",
|
||||
}
|
||||
for _, s := range DefaultSeats() {
|
||||
for _, v := range s.Serves {
|
||||
cmd, ok := want[s.Name+"."+v.Name]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
delete(want, s.Name+"."+v.Name)
|
||||
found := false
|
||||
for _, r := range v.Replaces {
|
||||
found = found || r == cmd
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("%s.%s does not say it replaces %q: %v", s.Name, v.Name, cmd, v.Replaces)
|
||||
}
|
||||
}
|
||||
}
|
||||
for verb := range want {
|
||||
t.Errorf("%s is not a verb of the compiled seats", verb)
|
||||
}
|
||||
}
|
||||
|
||||
// The confirm switch is the seat's string "true", as every verb's switch is, and its description says so:
|
||||
// a holder handed the boolean reading of "true to change it" would treat the string as a dry run.
|
||||
func TestTheConfirmSwitchIsTheStringTrue(t *testing.T) {
|
||||
for _, seat := range []string{NFSServerSeat, MountsSeat} {
|
||||
s, _ := SeatNamed(seat)
|
||||
for _, v := range s.Serves {
|
||||
props, _ := v.Input["properties"].(map[string]any)
|
||||
confirm, _ := props["confirm"].(map[string]any)
|
||||
if confirm == nil {
|
||||
continue
|
||||
}
|
||||
if confirm["type"] != "string" {
|
||||
t.Errorf("%s.%s confirm is %v, not the string switch", seat, v.Name, confirm["type"])
|
||||
}
|
||||
if d, _ := confirm["description"].(string); !strings.Contains(d, `"true"`) {
|
||||
t.Errorf("%s.%s confirm does not name the string \"true\": %q", seat, v.Name, d)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// reload has the kernel read the export files; the module's process writes its file. The description
|
||||
// must not promise a write it does not do.
|
||||
func TestReloadSaysWhatItDoes(t *testing.T) {
|
||||
s, _ := SeatNamed(NFSServerSeat)
|
||||
for _, v := range s.Serves {
|
||||
if v.Name == "reload" && (strings.Contains(v.Description, "Write this machine's export file") ||
|
||||
!strings.Contains(v.Description, "exportfs")) {
|
||||
t.Errorf("reload's description promises a write or does not name exportfs: %s", v.Description)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A share is exported to each node the server grants it to and that asks for it, at that node's own
|
||||
// address (novox/hq ADR 0263 rule 5) — never to the private network's whole range. The verbs that
|
||||
// describe the export say so, and do not promise the range.
|
||||
func TestTheExportVerbsDescribePerNodeAddresses(t *testing.T) {
|
||||
s, _ := SeatNamed(NFSServerSeat)
|
||||
for _, v := range s.Serves {
|
||||
if v.Name != "exports" && v.Name != "test" {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(v.Description, "private network's range") {
|
||||
t.Errorf("%s still describes the export as the private network's range: %s", v.Name, v.Description)
|
||||
}
|
||||
if !strings.Contains(v.Description, "address") {
|
||||
t.Errorf("%s does not say the export is to each node's address: %s", v.Name, v.Description)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,10 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
@@ -54,5 +57,150 @@ func UnknownFieldReason(m Manifest) string {
|
||||
return ""
|
||||
}
|
||||
return m.Module + " uses a field this controller does not know (" + m.unknown + "); it is left out " +
|
||||
"until the controller is updated (novox/hq ADR 0262)"
|
||||
"until the controller is updated: nothing of it is changed on its machines and its contributions to " +
|
||||
"other modules and its open ports stop. Its data is still backed up as this controller reads it, " +
|
||||
"which may not be what its newer manifest asks, and it still provides what it provides " +
|
||||
"(novox/hq ADR 0262)"
|
||||
}
|
||||
|
||||
// backupView is what of a left-out module still reaches its machine: its data, so the backup holder
|
||||
// keeps copying it, and the directories and accesses its data items name. No contribution, shell code
|
||||
// or environment of its own: those are what leaving it out stops.
|
||||
func backupView(m Manifest) Manifest {
|
||||
view := Manifest{Module: m.Module, Data: m.Data, Accesses: m.Accesses, bestEffort: true}
|
||||
for _, r := range m.Resources {
|
||||
if fmt.Sprint(r["type"]) == "directory" {
|
||||
view.Resources = append(view.Resources, r)
|
||||
}
|
||||
}
|
||||
return view
|
||||
}
|
||||
|
||||
// prunedFields is a stored manifest's fields with every key this controller does not know taken out,
|
||||
// and the keys taken out (novox/hq ADR 0262). A second pass, after the strict one refused: each
|
||||
// top-level field is decoded alone, and where an entry inside it has an unknown key, the one
|
||||
// occurrence whose removal moves the decoder past it is removed — never a key of the same name that
|
||||
// the entry around it knows. So a left-out module still provides what it provides, and its data and
|
||||
// directories are still read, which its backup lines are made from.
|
||||
func prunedFields(keys map[string]json.RawMessage) (manifestFields, []string, error) {
|
||||
var removed []string
|
||||
tree := map[string]any{}
|
||||
for k, raw := range keys {
|
||||
var v any
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
dec.UseNumber()
|
||||
if err := dec.Decode(&v); err != nil {
|
||||
return manifestFields{}, nil, err
|
||||
}
|
||||
tree[k] = v
|
||||
}
|
||||
for _, k := range sortedAnyKeys(tree) {
|
||||
for tries := 0; ; tries++ {
|
||||
err := decodesAlone(k, tree[k])
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
unknown := asUnknownField(err)
|
||||
if unknown == nil || tries > 64 {
|
||||
return manifestFields{}, nil, err
|
||||
}
|
||||
if unknown.Field == k {
|
||||
delete(tree, k)
|
||||
removed = append(removed, k)
|
||||
break
|
||||
}
|
||||
path, ok := removalThatHelps(k, tree[k], unknown.Field, err.Error())
|
||||
if !ok {
|
||||
// The same key unknown in two entries alike: no one removal changes the words.
|
||||
// Every occurrence goes, and the field is judged again.
|
||||
if removeEvery(tree[k], unknown.Field) == 0 {
|
||||
return manifestFields{}, nil, err
|
||||
}
|
||||
path = "…." + unknown.Field
|
||||
}
|
||||
removed = append(removed, k+path)
|
||||
}
|
||||
}
|
||||
raw, err := json.Marshal(tree)
|
||||
if err != nil {
|
||||
return manifestFields{}, nil, err
|
||||
}
|
||||
var fields manifestFields
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&fields); err != nil {
|
||||
return manifestFields{}, nil, err
|
||||
}
|
||||
return fields, removed, nil
|
||||
}
|
||||
|
||||
// decodesAlone is whether one top-level field decodes strictly on its own.
|
||||
func decodesAlone(k string, v any) error {
|
||||
raw, err := json.Marshal(map[string]any{k: v})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var fields manifestFields
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
dec.DisallowUnknownFields()
|
||||
return dec.Decode(&fields)
|
||||
}
|
||||
|
||||
// removalThatHelps removes, from v, the one occurrence of key whose removal changes what the strict
|
||||
// decoder says of field k, and says where it was. Every other occurrence is left as it was.
|
||||
func removalThatHelps(k string, v any, key, said string) (string, bool) {
|
||||
var found bool
|
||||
var where string
|
||||
var walk func(node any, at string) bool
|
||||
walk = func(node any, at string) bool {
|
||||
switch n := node.(type) {
|
||||
case map[string]any:
|
||||
if value, has := n[key]; has {
|
||||
delete(n, key)
|
||||
// Accepted only when the decoder is past it: nothing left, or an unknown key said
|
||||
// elsewhere. A different kind of error means the removal broke the entry; the same
|
||||
// words mean this was not the occurrence it refused.
|
||||
err := decodesAlone(k, v)
|
||||
if err == nil || (asUnknownField(err) != nil && err.Error() != said) {
|
||||
found, where = true, at+"."+key
|
||||
return true
|
||||
}
|
||||
n[key] = value
|
||||
}
|
||||
for _, sub := range sortedAnyKeys(n) {
|
||||
if walk(n[sub], at+"."+sub) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
case []any:
|
||||
for i, item := range n {
|
||||
if walk(item, fmt.Sprintf("%s[%d]", at, i)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
walk(v, "")
|
||||
return where, found
|
||||
}
|
||||
|
||||
// removeEvery removes key from every object in v, and says how many it removed.
|
||||
func removeEvery(v any, key string) int {
|
||||
n := 0
|
||||
switch node := v.(type) {
|
||||
case map[string]any:
|
||||
if _, has := node[key]; has {
|
||||
delete(node, key)
|
||||
n++
|
||||
}
|
||||
for _, sub := range node {
|
||||
n += removeEvery(sub, key)
|
||||
}
|
||||
case []any:
|
||||
for _, item := range node {
|
||||
n += removeEvery(item, key)
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
@@ -424,6 +424,21 @@ var ControllerVerbs = []Verb{
|
||||
"why": "with consumer or older-than: why — required, and recorded in the hand-act log",
|
||||
"cause": "with consumer or older-than: the cause in a word (cleanup-waiting when absent)",
|
||||
}, nil, "confirm")},
|
||||
// What a consumer gave up on (novox/hq issue 330): kept in DEAD_LETTERS until a person acts on it.
|
||||
{Name: "dead-letters", Description: "Every message a consumer on the bus gave up on after handing it over " +
|
||||
"as often as it may, kept in DEAD_LETTERS: whose consumer, the subject, how often it was handed over and " +
|
||||
"when it was given up, newest first. With id: that one whole, with what it said. With deliver: hand it " +
|
||||
"again to the consumer that gave it up, and nobody else; with drop: let it go for good. Delivering and " +
|
||||
"dropping are hand acts, which say why (novox/hq issue 330).",
|
||||
Input: schema(map[string]string{
|
||||
"id": "a dead letter's id, as the list gives it: that one whole, with its message",
|
||||
"consumer": "a consumer's name, or <stream>.<consumer>: only what that one gave up on",
|
||||
"limit": "how many to list, newest first (default 50); only when listing",
|
||||
"deliver": "a dead letter's id: deliver it again to the consumer that gave it up (needs why)",
|
||||
"drop": "a dead letter's id: drop it for good (needs why)",
|
||||
"why": "with deliver or drop: why — required, and recorded in the hand-act log",
|
||||
"cause": "with deliver or drop: the cause in a word (dead-letter when absent)",
|
||||
}, nil)},
|
||||
// The data every machine declares (novox/hq ADR 0233).
|
||||
{Name: "data", Description: "Every item of data every machine declares, as the self-check last measured it: " +
|
||||
"its class (irreplaceable, rebuildable, cache), where it is, its size, its newest write, its newest good " +
|
||||
|
||||
@@ -34,12 +34,9 @@ func (i *Inventory) BusRecords(ctx context.Context) (broker.Records, error) {
|
||||
// that seat promises. **Across the whole catalogue, not one manifest**: a seat is declared by
|
||||
// one module and held by another, which is the whole reason a seat exists (ADR 0118).
|
||||
seats := map[string]catalogue.SeatDeclaration{}
|
||||
// And who declared each, so a seat's records are named as the declaring module's buckets (ADR 0259).
|
||||
declarers := map[string]string{}
|
||||
for _, m := range declared {
|
||||
for _, s := range m.DefinesSeats {
|
||||
seats[s.Name] = s
|
||||
declarers[s.Name] = m.Module
|
||||
}
|
||||
}
|
||||
// And the mesh's own, which carry protocol too (novox/hq ADR 0121). Added after the modules'
|
||||
@@ -81,7 +78,7 @@ func (i *Inventory) BusRecords(ctx context.Context) (broker.Records, error) {
|
||||
"%s is assigned to %s and is not in the catalogue, so what it may say cannot "+
|
||||
"be derived", module, n.Name)
|
||||
}
|
||||
d := declaredFor(m, seats, declarers)
|
||||
d := declaredFor(m, seats)
|
||||
// And the state offered to it as a seat's holder by the modules beside it (novox/hq ADR 0255).
|
||||
// For this machine's key alone (novox/hq ADR 0260): a bar shows its own machine's draw.
|
||||
for _, sr := range catalogue.ReadsGranted(m, onMachine(declared, modules), n.Name) {
|
||||
@@ -124,7 +121,7 @@ func onMachine(declared map[string]catalogue.Manifest, modules []string) []catal
|
||||
|
||||
// declaredFor is one module's manifest as the composer needs it: what it says about itself, and the
|
||||
// protocol of every seat it holds or uses.
|
||||
func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaration, declarers map[string]string) broker.Declared {
|
||||
func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaration) broker.Declared {
|
||||
// A consumed name is a module's event unless it names a seat, and only somebody holding the seat
|
||||
// set can tell (novox/hq ADR 0121). Split here, because the composer cannot look at a name and
|
||||
// know — and a role's event read as a module's is a subscription to a namespace nobody owns.
|
||||
@@ -135,16 +132,6 @@ func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaratio
|
||||
if named {
|
||||
// A seat's event when the seat says it; else the event of the module of that name — a seat and
|
||||
// the module holding it may share a name (mesh-delivery, novox/hq ADR 0239).
|
||||
if s, isASeat := seats[emitter]; isASeat && s.Kinded {
|
||||
// A kinded bench's event is named `<event>` or `<event>.*` and heard from every kind; its
|
||||
// proofs are answered by whoever watches it (ADR 0259 §3).
|
||||
ev := strings.TrimSuffix(event, ".*")
|
||||
if catalogue.SeatSays(s.Emits, ev) {
|
||||
watches = append(watches, broker.Seat{Name: s.Name, Scope: s.Scope, Emits: []string{ev},
|
||||
Kinded: true, Proofs: s.Proofs, DeclaredBy: declarers[s.Name]})
|
||||
continue
|
||||
}
|
||||
}
|
||||
if s, isASeat := seats[emitter]; isASeat && catalogue.SeatSays(s.Emits, event) {
|
||||
watches = append(watches, broker.Seat{Name: s.Name, Emits: []string{event}})
|
||||
continue
|
||||
@@ -168,8 +155,6 @@ func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaratio
|
||||
Reads: m.Reads,
|
||||
// And the tools its health asks (novox/hq ADR 0240): the machine's node-engine is granted them.
|
||||
Checks: catalogue.HealthChecks(m),
|
||||
// And whether it runs as an account of its own (novox/hq ADR 0259 §8).
|
||||
RunsAs: m.RunsAs,
|
||||
}
|
||||
// Whether it can be given an account at all: delivered as its own secret named broker, so one
|
||||
// that declares none has nowhere to read it (novox/hq issue 195).
|
||||
@@ -183,15 +168,12 @@ func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaratio
|
||||
// Every seat with a protocol, the mesh's own included. One that says only who does a job is
|
||||
// not here and grants nothing, which is most of them.
|
||||
if s, hasAProtocol := seats[c.Name]; hasAProtocol {
|
||||
held := asSeat(s, declarers[s.Name])
|
||||
held.Kind = c.Kind
|
||||
held.Capabilities = c.Capabilities
|
||||
d.Holds = append(d.Holds, held)
|
||||
d.Holds = append(d.Holds, asSeat(s))
|
||||
}
|
||||
}
|
||||
for _, name := range m.Uses {
|
||||
if s, declaredSomewhere := seats[name]; declaredSomewhere {
|
||||
d.Uses = append(d.Uses, asSeat(s, declarers[s.Name]))
|
||||
d.Uses = append(d.Uses, asSeat(s))
|
||||
}
|
||||
}
|
||||
return d
|
||||
@@ -222,17 +204,9 @@ func (i *Inventory) DeclaredBuckets(ctx context.Context) ([]broker.Bucket, error
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func asSeat(s catalogue.SeatDeclaration, declarer string) broker.Seat {
|
||||
seat := broker.Seat{Name: s.Name, Scope: s.Scope, Accepts: s.Accepts, Emits: s.Emits,
|
||||
Serves: catalogue.VerbNames(s.Serves), Kinded: s.Kinded, ByCaller: s.ByCaller, Proofs: s.Proofs,
|
||||
DeclaredBy: declarer}
|
||||
// A seat's records are its declaring module's buckets, named as the bus holds them (ADR 0259 §3).
|
||||
for _, r := range s.Records {
|
||||
if declarer != "" {
|
||||
seat.Records = append(seat.Records, broker.BucketName(declarer, r))
|
||||
}
|
||||
}
|
||||
return seat
|
||||
func asSeat(s catalogue.SeatDeclaration) broker.Seat {
|
||||
return broker.Seat{Name: s.Name, Scope: s.Scope, Accepts: s.Accepts, Emits: s.Emits,
|
||||
Serves: catalogue.VerbNames(s.Serves)}
|
||||
}
|
||||
|
||||
// MeshSeats are the mesh's own seats that carry a protocol, as the bus needs them: what to make a work
|
||||
@@ -303,22 +277,3 @@ func heldHere(claimed []broker.Seat, holdings []catalogue.Held, node, module str
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// DeclaredTrafficSeats is every seat any registered module declares that names its caller or its kind
|
||||
// (novox/hq ADR 0259 §3), as the bus needs it: assigned or not, so its work queue exists from registration.
|
||||
func (i *Inventory) DeclaredTrafficSeats(ctx context.Context) ([]broker.Seat, error) {
|
||||
declared, err := i.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot read the catalogue: %w", err)
|
||||
}
|
||||
var out []broker.Seat
|
||||
for _, m := range declared {
|
||||
for _, s := range m.DefinesSeats {
|
||||
seat := asSeat(s, m.Module)
|
||||
if seat.Kinded || len(seat.ByCaller) > 0 {
|
||||
out = append(out, seat)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -34,6 +34,9 @@ const (
|
||||
AdvisoryMaxDeliveries = "max-deliveries"
|
||||
AdvisoryRefused = "refused"
|
||||
AdvisoryConsumerLost = "consumer-lost"
|
||||
// AdvisoryNotKept is the token of a max-deliveries advisory whose message could not be kept in
|
||||
// DEAD_LETTERS (novox/hq issue 330): said from the log, since the stream does not hold it.
|
||||
AdvisoryNotKept = "not-kept"
|
||||
)
|
||||
|
||||
// Advisory is one thing the bus said, kept as its newest word and how often it was said.
|
||||
@@ -43,6 +46,8 @@ type Advisory struct {
|
||||
ID string
|
||||
// Stream and Consumer are the consumer it is about, when it is about one.
|
||||
Stream, Consumer string
|
||||
// Token is the condition key's last part, when it is not Kind.
|
||||
Token string
|
||||
// Said is the newest saying, in the mesh's words.
|
||||
Said string
|
||||
First, Last time.Time
|
||||
@@ -62,7 +67,7 @@ var Advisories = &AdvisoryLog{seen: map[string]*Advisory{}}
|
||||
func (l *AdvisoryLog) Heard(a Advisory, at time.Time) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
key := a.Kind + "/" + a.ID
|
||||
key := a.Kind + "/" + a.ID + "/" + a.Token
|
||||
if had, ok := l.seen[key]; ok {
|
||||
had.Last, had.Said, had.Count = at, a.Said, had.Count+1
|
||||
return
|
||||
@@ -107,8 +112,9 @@ func ReadAdvisory(subject string, body []byte) (Advisory, bool) {
|
||||
switch {
|
||||
case strings.HasPrefix(subject, "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES."):
|
||||
return Advisory{Kind: AdvisoryMaxDeliveries, ID: id, Stream: a.Stream, Consumer: a.Consumer,
|
||||
Said: fmt.Sprintf("%s handed message %d over %d times and gave up on it: it will not be delivered "+
|
||||
"again, and what it asked for was not done", who, a.StreamSeq, a.Deliveries)}, true
|
||||
Said: fmt.Sprintf("%s handed message %d over %d times and gave up on it: what it asked for was not "+
|
||||
"done, and the controller keeps it in %s until it is delivered again or dropped", who, a.StreamSeq,
|
||||
a.Deliveries, broker.DeadLettersStream)}, true
|
||||
case strings.HasPrefix(subject, "$JS.EVENT.ADVISORY.CONSUMER.DELETED."):
|
||||
if !MeshNamed(a.Stream, a.Consumer) {
|
||||
// A reader's own consumer, gone when it finished — every watch of a bucket and every
|
||||
@@ -175,7 +181,12 @@ func (s *Server) HearAdvisories(logf func(string, ...any)) (func(), error) {
|
||||
for _, subject := range broker.BusAdvisories {
|
||||
sub, err := conn.Subscribe(subject, func(m *nats.Msg) {
|
||||
if a, ok := ReadAdvisory(m.Subject, m.Data); ok {
|
||||
Advisories.Heard(a, time.Now())
|
||||
// A message given up on is said from DEAD_LETTERS, where it is kept, for as long as it is
|
||||
// kept (novox/hq issue 330) — not for an hour after the server said it; one that could not
|
||||
// be kept is recorded by whoever tried.
|
||||
if a.Kind != AdvisoryMaxDeliveries {
|
||||
Advisories.Heard(a, time.Now())
|
||||
}
|
||||
logf("the bus says: %s", a.Said)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -37,7 +37,8 @@ func TestOnlyTheMeshsOwnConsumersAreSaidLost(t *testing.T) {
|
||||
[]byte(`{"stream":"EVENTS","consumer":"anchor_shop","stream_seq":7,"deliveries":5}`))
|
||||
if !ok || a.Kind != AdvisoryMaxDeliveries || a.ID != "EVENTS.anchor_shop" ||
|
||||
a.Said != "how shop on anchor hears what it consumes handed message 7 over 5 times and gave up on it: "+
|
||||
"it will not be delivered again, and what it asked for was not done" {
|
||||
"what it asked for was not done, and the controller keeps it in DEAD_LETTERS until it is delivered "+
|
||||
"again or dropped" {
|
||||
t.Fatalf("%+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,6 +47,12 @@ func BuildsOverNATSOn(address, seat string) (Builders, error) {
|
||||
return &natsBuilds{js: js, owned: true, seat: seat}, nil
|
||||
}
|
||||
|
||||
// BuildsOn is the asking side on a connection the caller holds, which Close leaves open: the serving
|
||||
// controller asks on its own rather than dialling one per build (novox/hq issue 327).
|
||||
func BuildsOn(js *broker.JetStream, seat string) Builders {
|
||||
return &natsBuilds{js: js, seat: seat}
|
||||
}
|
||||
|
||||
// role is the seat asked: what the asker was made for, or the current build role for one made
|
||||
// without saying (a test building the struct by hand).
|
||||
func (b *natsBuilds) role() string {
|
||||
|
||||
@@ -8,13 +8,16 @@ import (
|
||||
// JetStream connection the caller has already raised the streams on. Nothing is declared here —
|
||||
// the streams and the controller's consumers are asserted by Raise, before anything is served.
|
||||
func ConnectNats(js *broker.JetStream, enroller Enroller, listener Listener) *Server {
|
||||
logger := newLog()
|
||||
inbound := Nats(js).(*natsInbound)
|
||||
inbound.log = logger
|
||||
return &Server{
|
||||
inbound: Nats(js),
|
||||
inbound: inbound,
|
||||
bus: OverNATS{JS: js.Context(), Conn: js.Conn()},
|
||||
js: js,
|
||||
enroller: enroller,
|
||||
listener: listener,
|
||||
log: newLog(),
|
||||
log: logger,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"log"
|
||||
"regexp"
|
||||
"runtime/debug"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -575,6 +576,19 @@ func (l *CallLog) serveCallWithin(seat, verb string, args json.RawMessage, reply
|
||||
done := make(chan outcome, 1)
|
||||
go func() {
|
||||
var body []byte
|
||||
// **A handler that panics is an answer, not a dead controller.** A verb answered in the serving
|
||||
// process (novox/hq issue 327: conditions, hand-acts, queue, dead-letters) runs on this goroutine,
|
||||
// where a panic would take every other call and the controller with it.
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
if logger != nil {
|
||||
logger.Printf("%s.%s: call %s panicked: %v\n%s", seat, verb, c.ID, p, debug.Stack())
|
||||
}
|
||||
body, _ = json.Marshal(map[string]any{"error": fmt.Sprintf("%s failed inside the controller and "+
|
||||
"answered nothing: %v. It is in the controller's log", verb, p)})
|
||||
done <- outcome{body, true}
|
||||
}
|
||||
}()
|
||||
result, err := handle(ctx, args)
|
||||
failed := err != nil
|
||||
if errors.Is(err, ErrHandingOver) {
|
||||
|
||||
@@ -11,6 +11,10 @@ import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// answers collects what a call answered, and fails a second answer: the bus permits one.
|
||||
@@ -223,3 +227,29 @@ func TestAJoinTokenIsShownToItsCallerAndNotKept(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A verb whose handler panics answers an error, and the controller serves the next call (review of
|
||||
// novox/hq issue 327: read verbs are answered in the serving process now).
|
||||
func TestAHandlerThatPanicsAnswersAnError(t *testing.T) {
|
||||
conn, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
stop, err := OverNATS{Conn: conn}.ServeSeatTools("panicky", map[string]ToolHandler{
|
||||
"boom": func(context.Context, json.RawMessage) (any, error) { panic("nil map") },
|
||||
"fine": func(context.Context, json.RawMessage) (any, error) { return "ok", nil },
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
answer, err := AskMeshSeatTool(context.Background(), conn, "panicky", "boom", map[string]any{}, 5*time.Second)
|
||||
if err != nil || !strings.Contains(answer.Error, "failed inside the controller") {
|
||||
t.Fatalf("a panic answered %+v (%v)", answer, err)
|
||||
}
|
||||
answer, err = AskMeshSeatTool(context.Background(), conn, "panicky", "fine", map[string]any{}, 5*time.Second)
|
||||
if err != nil || answer.Error != "" {
|
||||
t.Fatalf("the call after a panic answered %+v (%v)", answer, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,372 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// What a consumer gave up on, kept until a person acts on it (novox/hq issue 330, design 25 §3).
|
||||
//
|
||||
// A durable consumer hands a message over as often as it may, gives up on it and says so on
|
||||
// `$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES`. The server's notice is captured in its own stream
|
||||
// (broker.DeadLetterNoticesStream), so one said while no controller listens waits for the next. The
|
||||
// serving controller takes each notice, fetches the message it names by its sequence, while the source
|
||||
// stream still holds it, and keeps a copy in DEAD_LETTERS with what the notice said: the consumer, the
|
||||
// subject, how often it was handed over and when it was given up. Until then a message given up on was
|
||||
// kept only by its source, which drops an event after a week, and nothing said which.
|
||||
//
|
||||
// The copy's headers are the message's own, but for the server's (`Nats-…`), which describe the first
|
||||
// publish and would refuse or de-duplicate the copy; they are kept under DeadHeader names.
|
||||
|
||||
// The headers a kept message carries beside its own.
|
||||
const (
|
||||
DeadHeaderStream = "Mesh-Dead-Stream"
|
||||
DeadHeaderConsumer = "Mesh-Dead-Consumer"
|
||||
DeadHeaderSequence = "Mesh-Dead-Sequence"
|
||||
DeadHeaderSubject = "Mesh-Dead-Subject"
|
||||
DeadHeaderDeliveries = "Mesh-Dead-Deliveries"
|
||||
DeadHeaderGaveUp = "Mesh-Dead-Gave-Up"
|
||||
DeadHeaderPublished = "Mesh-Dead-Published"
|
||||
DeadHeaderLost = "Mesh-Dead-Lost"
|
||||
// DeadHeaderMsgID is the message's own de-duplication id, kept, since the copy carries its own.
|
||||
DeadHeaderMsgID = "Mesh-Dead-Msg-Id"
|
||||
// AgainHeader marks a message delivered again, with the kept copy's id it came from.
|
||||
AgainHeader = "Mesh-Delivered-Again"
|
||||
)
|
||||
|
||||
// DeadLetter is one message a consumer gave up on, as DEAD_LETTERS keeps it.
|
||||
type DeadLetter struct {
|
||||
// ID is its sequence in DEAD_LETTERS: what the verb names it by.
|
||||
ID uint64 `json:"id"`
|
||||
Stream string `json:"stream"`
|
||||
Consumer string `json:"consumer"`
|
||||
// Who is the consumer in the mesh's words: whose, and for what.
|
||||
Who string `json:"who"`
|
||||
Subject string `json:"subject"`
|
||||
Sequence uint64 `json:"sequence"`
|
||||
Deliveries uint64 `json:"deliveries"`
|
||||
GaveUp time.Time `json:"gave_up"`
|
||||
Published time.Time `json:"published,omitzero"`
|
||||
// Lost says why the message itself is not kept: its source no longer held it when the notice was
|
||||
// taken. The record of it is kept all the same, so it is said and dropped, never silently missing.
|
||||
Lost string `json:"lost,omitempty"`
|
||||
Size int `json:"size"`
|
||||
// Body and Headers are the message itself, given only for one dead letter asked by its id; a header
|
||||
// with several values keeps them all.
|
||||
Body string `json:"body,omitempty"`
|
||||
Headers map[string][]string `json:"headers,omitempty"`
|
||||
}
|
||||
|
||||
// maxDeliveries is the part of the server's notice a kept message is made from.
|
||||
type maxDeliveries struct {
|
||||
Stream string `json:"stream"`
|
||||
Consumer string `json:"consumer"`
|
||||
StreamSeq uint64 `json:"stream_seq"`
|
||||
Deliveries uint64 `json:"deliveries"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
}
|
||||
|
||||
// KeepDeadLetter keeps the message one maximum-deliveries notice is about. An error leaves nothing
|
||||
// kept, and the notice is to be taken again: the source still holds the message, or a full
|
||||
// DEAD_LETTERS has room again. A source that no longer holds it is no error: the record is kept with
|
||||
// why, so it is said.
|
||||
func KeepDeadLetter(js nats.JetStreamContext, notice []byte) (DeadLetter, error) {
|
||||
var a maxDeliveries
|
||||
if err := json.Unmarshal(notice, &a); err != nil || a.Stream == "" || a.Consumer == "" || a.StreamSeq == 0 {
|
||||
return DeadLetter{}, fmt.Errorf("not a maximum-deliveries notice the mesh can read: %.200s", notice)
|
||||
}
|
||||
gaveUp := a.Timestamp
|
||||
if gaveUp.IsZero() {
|
||||
gaveUp = time.Now()
|
||||
}
|
||||
kept := &nats.Msg{Subject: broker.DeadLetterSubject(a.Stream, a.Consumer), Header: nats.Header{}}
|
||||
raw, err := js.GetMsg(a.Stream, a.StreamSeq)
|
||||
switch {
|
||||
case err == nil:
|
||||
for k, v := range raw.Header {
|
||||
if strings.HasPrefix(k, "Nats-") {
|
||||
continue
|
||||
}
|
||||
kept.Header[k] = v
|
||||
}
|
||||
if id := raw.Header.Get(nats.MsgIdHdr); id != "" {
|
||||
kept.Header.Set(DeadHeaderMsgID, id)
|
||||
}
|
||||
kept.Header.Set(DeadHeaderSubject, raw.Subject)
|
||||
kept.Header.Set(DeadHeaderPublished, raw.Time.UTC().Format(time.RFC3339Nano))
|
||||
kept.Data = raw.Data
|
||||
case errors.Is(err, nats.ErrMsgNotFound) || errors.Is(err, nats.ErrStreamNotFound):
|
||||
kept.Header.Set(DeadHeaderLost, fmt.Sprintf("%s no longer held message %d when the notice was taken: %v",
|
||||
a.Stream, a.StreamSeq, err))
|
||||
default:
|
||||
return DeadLetter{}, fmt.Errorf("message %d of %s could not be read: %w", a.StreamSeq, a.Stream, err)
|
||||
}
|
||||
kept.Header.Set(DeadHeaderStream, a.Stream)
|
||||
kept.Header.Set(DeadHeaderConsumer, a.Consumer)
|
||||
kept.Header.Set(DeadHeaderSequence, strconv.FormatUint(a.StreamSeq, 10))
|
||||
kept.Header.Set(DeadHeaderDeliveries, strconv.FormatUint(a.Deliveries, 10))
|
||||
kept.Header.Set(DeadHeaderGaveUp, gaveUp.UTC().Format(time.RFC3339Nano))
|
||||
// One copy per message given up, however often its notice is taken: a controller that copied and
|
||||
// stopped before acknowledging, or two controllers each taking it.
|
||||
kept.Header.Set(nats.MsgIdHdr, fmt.Sprintf("%s.%s.%d", a.Stream, a.Consumer, a.StreamSeq))
|
||||
ack, err := js.PublishMsg(kept)
|
||||
if err != nil {
|
||||
return DeadLetter{}, fmt.Errorf("message %d of %s could not be kept in %s: %w", a.StreamSeq, a.Stream,
|
||||
broker.DeadLettersStream, err)
|
||||
}
|
||||
return deadLetterOf(ack.Sequence, kept.Subject, kept.Header, kept.Data, false), nil
|
||||
}
|
||||
|
||||
// deadLetterOf reads a kept message.
|
||||
func deadLetterOf(id uint64, subject string, h nats.Header, data []byte, whole bool) DeadLetter {
|
||||
d := DeadLetter{ID: id, Stream: h.Get(DeadHeaderStream), Consumer: h.Get(DeadHeaderConsumer),
|
||||
Subject: h.Get(DeadHeaderSubject), Lost: h.Get(DeadHeaderLost), Size: len(data)}
|
||||
if d.Stream == "" || d.Consumer == "" {
|
||||
d.Stream, d.Consumer, _ = broker.DeadLetterOf(subject)
|
||||
}
|
||||
d.Who = ConsumerInWords(d.Stream, d.Consumer)
|
||||
d.Sequence, _ = strconv.ParseUint(h.Get(DeadHeaderSequence), 10, 64)
|
||||
d.Deliveries, _ = strconv.ParseUint(h.Get(DeadHeaderDeliveries), 10, 64)
|
||||
d.GaveUp, _ = time.Parse(time.RFC3339Nano, h.Get(DeadHeaderGaveUp))
|
||||
d.Published, _ = time.Parse(time.RFC3339Nano, h.Get(DeadHeaderPublished))
|
||||
if whole {
|
||||
d.Body = string(data)
|
||||
d.Headers = map[string][]string{}
|
||||
for k, v := range h {
|
||||
if !strings.HasPrefix(k, "Mesh-Dead-") && !strings.HasPrefix(k, "Nats-") {
|
||||
d.Headers[k] = append([]string(nil), v...)
|
||||
}
|
||||
}
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// HeldDeadLetters is how many messages DEAD_LETTERS holds for each consumer, by `<stream>.<consumer>`.
|
||||
func HeldDeadLetters(js nats.JetStreamContext) (map[string]int, error) {
|
||||
info, err := js.StreamInfo(broker.DeadLettersStream, &nats.StreamInfoRequest{SubjectsFilter: ">"})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s cannot be read: %w", broker.DeadLettersStream, err)
|
||||
}
|
||||
held := map[string]int{}
|
||||
for subject, n := range info.State.Subjects {
|
||||
if stream, consumer, ok := broker.DeadLetterOf(subject); ok && n > 0 {
|
||||
held[stream+"."+consumer] += int(n)
|
||||
}
|
||||
}
|
||||
return held, nil
|
||||
}
|
||||
|
||||
// DeadLetters lists what DEAD_LETTERS holds, newest first, at most most of them (all when most is not
|
||||
// positive); for one consumer when consumer names one (its name, or `<stream>.<consumer>`). The total is
|
||||
// the stream's own count per consumer, so it is right however few are read.
|
||||
func DeadLetters(js nats.JetStreamContext, consumer string, most int) ([]DeadLetter, int, error) {
|
||||
held, err := HeldDeadLetters(js)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
total := 0
|
||||
for key, n := range held {
|
||||
if consumer == "" || key == consumer || strings.HasSuffix(key, "."+consumer) {
|
||||
total += n
|
||||
}
|
||||
}
|
||||
if total == 0 {
|
||||
return nil, 0, nil
|
||||
}
|
||||
info, err := js.StreamInfo(broker.DeadLettersStream)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("%s cannot be read: %w", broker.DeadLettersStream, err)
|
||||
}
|
||||
var out []DeadLetter
|
||||
for seq := info.State.LastSeq; seq >= info.State.FirstSeq && seq > 0; seq-- {
|
||||
if most > 0 && len(out) >= most || len(out) >= total {
|
||||
break
|
||||
}
|
||||
raw, err := js.GetMsg(broker.DeadLettersStream, seq)
|
||||
if errors.Is(err, nats.ErrMsgNotFound) {
|
||||
continue // delivered again or dropped
|
||||
}
|
||||
if err != nil {
|
||||
return out, total, fmt.Errorf("dead letter %d cannot be read: %w", seq, err)
|
||||
}
|
||||
d := deadLetterOf(seq, raw.Subject, raw.Header, raw.Data, false)
|
||||
if consumer != "" && consumer != d.Consumer && consumer != d.Stream+"."+d.Consumer {
|
||||
continue
|
||||
}
|
||||
out = append(out, d)
|
||||
}
|
||||
return out, total, nil
|
||||
}
|
||||
|
||||
// ErrNoDeadLetter is a dead letter's id DEAD_LETTERS does not hold.
|
||||
var ErrNoDeadLetter = errors.New("no such dead letter")
|
||||
|
||||
// DeadLetterNamed is one kept message whole: what it said, and the headers it said it with.
|
||||
func DeadLetterNamed(js nats.JetStreamContext, id uint64) (DeadLetter, error) {
|
||||
raw, err := js.GetMsg(broker.DeadLettersStream, id)
|
||||
if errors.Is(err, nats.ErrMsgNotFound) {
|
||||
return DeadLetter{}, fmt.Errorf("%w: %s holds no message %d — delivered again or dropped already, "+
|
||||
"or never kept; dead-letters lists what it holds", ErrNoDeadLetter, broker.DeadLettersStream, id)
|
||||
}
|
||||
if err != nil {
|
||||
return DeadLetter{}, fmt.Errorf("dead letter %d cannot be read: %w", id, err)
|
||||
}
|
||||
return deadLetterOf(id, raw.Subject, raw.Header, raw.Data, true), nil
|
||||
}
|
||||
|
||||
// AgainTo is where a kept message is delivered again so that only the consumer that gave it up gets
|
||||
// it: an event under that consumer's own again subject on EVENTS — and only when the consumer exists and
|
||||
// filters that subject, so a message is never let go as delivered while nobody receives it. Any other
|
||||
// stream's message is refused, with why: an ask given up on by a seat's worker is not delivered again yet
|
||||
// (novox/hq issue 330's follow-up), since publishing it again leaves the original stuck in the queue.
|
||||
func AgainTo(js nats.JetStreamContext, d DeadLetter) (string, error) {
|
||||
switch {
|
||||
case d.Lost != "":
|
||||
return "", fmt.Errorf("dead letter %d holds no message to deliver: %s. Drop it", d.ID, d.Lost)
|
||||
case d.Subject == "":
|
||||
return "", fmt.Errorf("dead letter %d does not say the subject it was published on, so it cannot be "+
|
||||
"delivered again. Drop it", d.ID)
|
||||
case d.Stream != broker.EventsStream:
|
||||
return "", fmt.Errorf("dead letter %d is from %s, and only an event is delivered again: publishing it "+
|
||||
"again would reach every consumer of its subject, or leave the original in its queue. Drop it, and "+
|
||||
"have its sender say it again", d.ID, d.Stream)
|
||||
}
|
||||
info, err := js.ConsumerInfo(d.Stream, d.Consumer)
|
||||
if errors.Is(err, nats.ErrConsumerNotFound) {
|
||||
return "", fmt.Errorf("dead letter %d was given up by %s, which is no longer on the bus, so nothing would "+
|
||||
"receive it. Drop it, or deliver it again once the module is assigned there again", d.ID, d.Who)
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("whether %s can receive dead letter %d cannot be read: %w", d.Who, d.ID, err)
|
||||
}
|
||||
want := broker.AgainFilter(d.Consumer)
|
||||
filters := append([]string{info.Config.FilterSubject}, info.Config.FilterSubjects...)
|
||||
if !slices.Contains(filters, want) {
|
||||
return "", fmt.Errorf("%s does not yet take events delivered again (it does not filter %s): the controller "+
|
||||
"sets that at the next send to its machine. Nothing was done, and dead letter %d is still kept",
|
||||
d.Who, want, d.ID)
|
||||
}
|
||||
return broker.AgainSubject(d.Consumer, d.Subject), nil
|
||||
}
|
||||
|
||||
// DeliverAgain hands a kept message to the consumer that gave it up, and nobody else, then removes it
|
||||
// from DEAD_LETTERS. The message carries its own headers and AgainHeader; its de-duplication id is the
|
||||
// kept copy's, so asking twice delivers it once.
|
||||
func DeliverAgain(js nats.JetStreamContext, id uint64) (DeadLetter, string, error) {
|
||||
d, err := DeadLetterNamed(js, id)
|
||||
if err != nil {
|
||||
return d, "", err
|
||||
}
|
||||
to, err := AgainTo(js, d)
|
||||
if err != nil {
|
||||
return d, "", err
|
||||
}
|
||||
again := &nats.Msg{Subject: to, Header: nats.Header{}, Data: []byte(d.Body)}
|
||||
for k, v := range d.Headers {
|
||||
again.Header[k] = append([]string(nil), v...)
|
||||
}
|
||||
again.Header.Set(AgainHeader, strconv.FormatUint(id, 10))
|
||||
again.Header.Set(nats.MsgIdHdr, "again."+broker.DeadLettersStream+"."+strconv.FormatUint(id, 10))
|
||||
if _, err := js.PublishMsg(again); err != nil {
|
||||
return d, to, fmt.Errorf("dead letter %d could not be delivered again on %s: %w; it is still kept", id, to, err)
|
||||
}
|
||||
if err := js.DeleteMsg(broker.DeadLettersStream, id); err != nil && !errors.Is(err, nats.ErrMsgNotFound) {
|
||||
return d, to, fmt.Errorf("dead letter %d was delivered again on %s and could not be removed from %s: %w",
|
||||
id, to, broker.DeadLettersStream, err)
|
||||
}
|
||||
return d, to, nil
|
||||
}
|
||||
|
||||
// DropDeadLetter removes a kept message for good.
|
||||
func DropDeadLetter(js nats.JetStreamContext, id uint64) (DeadLetter, error) {
|
||||
d, err := DeadLetterNamed(js, id)
|
||||
if err != nil {
|
||||
return d, err
|
||||
}
|
||||
if err := js.DeleteMsg(broker.DeadLettersStream, id); err != nil {
|
||||
return d, fmt.Errorf("dead letter %d could not be dropped: %w", id, err)
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// noticeRetry is how long a notice whose message could not be kept waits before it is taken again.
|
||||
var noticeRetry = time.Minute
|
||||
|
||||
// keepingGivenUp keeps taking the notices until ctx ends; the returned function stops it. A subscription
|
||||
// that cannot be made is said — in the log, and as the max-deliveries condition of the notices themselves
|
||||
// — and tried again every noticeRetry, so it never stops the controller serving.
|
||||
func keepingGivenUp(ctx context.Context, bus *broker.JetStream, logger *log.Logger) func() {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
for {
|
||||
sub, err := func() (*nats.Subscription, error) {
|
||||
if err := bus.EnsureConsumer(broker.NoticesConsumer()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return keepGivenUp(bus.Context(), logger)
|
||||
}()
|
||||
if err == nil {
|
||||
<-ctx.Done()
|
||||
_ = sub.Unsubscribe()
|
||||
return
|
||||
}
|
||||
logger.Printf("the notices of messages consumers gave up on cannot be taken, so none is kept until "+
|
||||
"they can: %v", err)
|
||||
Advisories.Heard(Advisory{Kind: AdvisoryMaxDeliveries, ID: broker.DeadLetterNoticesStream + "." +
|
||||
broker.ControllerName, Stream: broker.DeadLetterNoticesStream, Consumer: broker.ControllerName,
|
||||
Token: AdvisoryNotKept, Said: "the controller cannot take the notices of messages consumers gave " +
|
||||
"up on, so none is kept: " + err.Error()}, time.Now())
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(noticeRetry):
|
||||
}
|
||||
}
|
||||
}()
|
||||
return func() {
|
||||
cancel()
|
||||
<-done
|
||||
}
|
||||
}
|
||||
|
||||
// keepGivenUp takes the server's maximum-deliveries notices off their stream and keeps the message
|
||||
// each is about, for as long as the subscription stands. A notice that could not be kept is offered
|
||||
// again after noticeRetry, and said: in the log, and as the consumer's max-deliveries condition.
|
||||
func keepGivenUp(js nats.JetStreamContext, logger *log.Logger) (*nats.Subscription, error) {
|
||||
return js.Subscribe("", func(m *nats.Msg) {
|
||||
d, err := KeepDeadLetter(js, m.Data)
|
||||
if err != nil {
|
||||
logger.Printf("a message a consumer gave up on could NOT be kept: %v", err)
|
||||
var a maxDeliveries
|
||||
if json.Unmarshal(m.Data, &a) == nil && a.Stream != "" && a.Consumer != "" {
|
||||
Advisories.Heard(Advisory{Kind: AdvisoryMaxDeliveries, ID: a.Stream + "." + a.Consumer,
|
||||
Stream: a.Stream, Consumer: a.Consumer, Token: AdvisoryNotKept,
|
||||
Said: fmt.Sprintf("%s gave up on message %d, and it could not be kept: %v",
|
||||
ConsumerInWords(a.Stream, a.Consumer), a.StreamSeq, err)}, time.Now())
|
||||
}
|
||||
_ = m.NakWithDelay(noticeRetry)
|
||||
return
|
||||
}
|
||||
if d.Lost != "" {
|
||||
logger.Printf("%s gave up on message %d of %s, which its stream no longer held; kept as dead letter %d "+
|
||||
"without it", d.Who, d.Sequence, d.Stream, d.ID)
|
||||
} else {
|
||||
logger.Printf("%s gave up on message %d of %s (%s) after %d deliveries; kept as dead letter %d",
|
||||
d.Who, d.Sequence, d.Stream, d.Subject, d.Deliveries, d.ID)
|
||||
}
|
||||
_ = m.Ack()
|
||||
}, nats.Bind(broker.DeadLetterNoticesStream, broker.ControllerName), nats.ManualAck())
|
||||
}
|
||||
@@ -0,0 +1,336 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// A module's consumer as the controller makes it, on a bus whose streams the controller asserted.
|
||||
func aModuleConsumer(t *testing.T, js *broker.JetStream, node, module string, consumes ...string) jetstream.Consumer {
|
||||
t.Helper()
|
||||
c, ok := broker.ConsumerFor(broker.Principal{Kind: broker.KindModule, Node: node, Module: module,
|
||||
Consumes: consumes, PasswordHash: "x"})
|
||||
if !ok {
|
||||
t.Fatal("a module that consumes got no consumer")
|
||||
}
|
||||
if err := js.EnsureConsumer(c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reading, err := api.Consumer(t.Context(), broker.EventsStream, c.Name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return reading
|
||||
}
|
||||
|
||||
// next is the one message a consumer hands over within a second, or nil.
|
||||
func next(t *testing.T, c jetstream.Consumer) jetstream.Msg {
|
||||
t.Helper()
|
||||
batch, err := c.Fetch(1, jetstream.FetchMaxWait(time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for msg := range batch.Messages() {
|
||||
return msg
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// keeping takes the notices as the serving controller does, until the test ends.
|
||||
func keeping(t *testing.T, js *broker.JetStream) {
|
||||
t.Helper()
|
||||
if err := broker.AssertMeshConsumers(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sub, err := keepGivenUp(js.Context(), log.New(io.Discard, "", 0))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
}
|
||||
|
||||
// givenUp hands one event to a consumer as often as it may, failing each time, and waits for it kept.
|
||||
func givenUp(t *testing.T, js *broker.JetStream, c jetstream.Consumer) DeadLetter {
|
||||
t.Helper()
|
||||
for {
|
||||
msg := next(t, c)
|
||||
if msg == nil {
|
||||
break
|
||||
}
|
||||
_ = msg.Nak()
|
||||
}
|
||||
var kept []DeadLetter
|
||||
eventually(t, "the event kept", func() bool {
|
||||
kept, _, _ = DeadLetters(js.Context(), c.CachedInfo().Name, 0)
|
||||
return len(kept) == 1
|
||||
})
|
||||
return kept[0]
|
||||
}
|
||||
|
||||
// **Delivered again to the consumer that gave it up, and nobody else**: another module consuming the
|
||||
// same event handled it the first time and must not handle it twice.
|
||||
func TestADeadLetterIsDeliveredAgainToItsConsumerAlone(t *testing.T) {
|
||||
js := aBus(t)
|
||||
keeping(t, js)
|
||||
failing := aModuleConsumer(t, js, "media", "sonarr", "gitea.pull.merged")
|
||||
handling := aModuleConsumer(t, js, "media", "radarr", "gitea.pull.merged")
|
||||
|
||||
const subject = "mesh.mod.gitea.event.pull.merged"
|
||||
msg := &nats.Msg{Subject: subject, Data: []byte(`{"n":1}`), Header: nats.Header{}}
|
||||
msg.Header.Set("x-node", "forge")
|
||||
msg.Header.Set(nats.MsgIdHdr, "event-1")
|
||||
if _, err := js.Context().PublishMsg(msg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m := next(t, handling); m == nil {
|
||||
t.Fatal("the other module was not handed the event")
|
||||
} else {
|
||||
_ = m.Ack()
|
||||
}
|
||||
d := givenUp(t, js, failing)
|
||||
if d.Consumer != "media_sonarr" || d.Stream != "EVENTS" || d.Subject != subject || d.Deliveries != 5 ||
|
||||
d.GaveUp.IsZero() || d.Published.IsZero() || d.Lost != "" {
|
||||
t.Fatalf("kept as %+v", d)
|
||||
}
|
||||
held, err := HeldDeadLetters(js.Context())
|
||||
if err != nil || held["EVENTS.media_sonarr"] != 1 {
|
||||
t.Fatalf("held %v (%v)", held, err)
|
||||
}
|
||||
|
||||
whole, err := DeadLetterNamed(js.Context(), d.ID)
|
||||
if err != nil || whole.Body != `{"n":1}` || len(whole.Headers["x-node"]) != 1 || whole.Headers["x-node"][0] != "forge" {
|
||||
t.Fatalf("one asked whole is %+v (%v)", whole, err)
|
||||
}
|
||||
|
||||
_, to, err := DeliverAgain(js.Context(), d.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if to != "mesh.again.media_sonarr.mod.gitea.event.pull.merged" {
|
||||
t.Fatalf("delivered again on %s", to)
|
||||
}
|
||||
again := next(t, failing)
|
||||
if again == nil {
|
||||
t.Fatal("the consumer that gave it up was not handed it again")
|
||||
}
|
||||
if string(again.Data()) != `{"n":1}` || again.Headers().Get("x-node") != "forge" ||
|
||||
again.Headers().Get(AgainHeader) == "" {
|
||||
t.Fatalf("handed again as %s %v", again.Data(), again.Headers())
|
||||
}
|
||||
if original, ok := broker.OriginalOfAgain(again.Subject()); !ok || original != subject {
|
||||
t.Fatalf("delivered again on %s, which does not say the event's own subject", again.Subject())
|
||||
}
|
||||
_ = again.Ack()
|
||||
if m := next(t, handling); m != nil {
|
||||
t.Fatalf("the module that handled it was handed it twice: %s", m.Subject())
|
||||
}
|
||||
if _, err := DeadLetterNamed(js.Context(), d.ID); !errors.Is(err, ErrNoDeadLetter) {
|
||||
t.Fatalf("still kept after it was delivered again: %v", err)
|
||||
}
|
||||
held, _ = HeldDeadLetters(js.Context())
|
||||
if held["EVENTS.media_sonarr"] != 0 {
|
||||
t.Fatalf("still held: %v", held)
|
||||
}
|
||||
// Asked twice, delivered once: the second finds nothing kept.
|
||||
if _, _, err := DeliverAgain(js.Context(), d.ID); !errors.Is(err, ErrNoDeadLetter) {
|
||||
t.Fatalf("a second delivery answered %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestADeadLetterDroppedIsGone(t *testing.T) {
|
||||
js := aBus(t)
|
||||
keeping(t, js)
|
||||
failing := aModuleConsumer(t, js, "media", "sonarr", "gitea.pull.merged")
|
||||
if _, err := js.Context().Publish("mesh.mod.gitea.event.pull.merged", []byte(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := givenUp(t, js, failing)
|
||||
if _, err := DropDeadLetter(js.Context(), d.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if left, total, err := DeadLetters(js.Context(), "", 0); err != nil || total != 0 || len(left) != 0 {
|
||||
t.Fatalf("after the drop: %v %d %v", left, total, err)
|
||||
}
|
||||
if m := next(t, failing); m != nil {
|
||||
t.Fatalf("a dropped event was handed over: %s", m.Subject())
|
||||
}
|
||||
}
|
||||
|
||||
// A notice whose message its stream no longer holds is kept as a record that says so — said, and
|
||||
// dropped by a person, never silently missing — and cannot be delivered again.
|
||||
func TestANoticeWhoseMessageIsGoneIsKeptAndSaysSo(t *testing.T) {
|
||||
js := aBus(t)
|
||||
d, err := KeepDeadLetter(js.Context(), []byte(`{"stream":"EVENTS","consumer":"media_sonarr","stream_seq":42,`+
|
||||
`"deliveries":5,"timestamp":"2026-10-06T15:51:00Z"}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(d.Lost, "no longer held message 42") || d.Consumer != "media_sonarr" {
|
||||
t.Fatalf("kept as %+v", d)
|
||||
}
|
||||
if _, _, err := DeliverAgain(js.Context(), d.ID); err == nil || !strings.Contains(err.Error(), "Drop it") {
|
||||
t.Fatalf("a record without its message was delivered again: %v", err)
|
||||
}
|
||||
// The same notice taken twice is kept once.
|
||||
if _, err := KeepDeadLetter(js.Context(), []byte(`{"stream":"EVENTS","consumer":"media_sonarr","stream_seq":42,`+
|
||||
`"deliveries":5}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, total, _ := DeadLetters(js.Context(), "media_sonarr", 0); total != 1 {
|
||||
t.Fatalf("one notice taken twice is kept %d times", total)
|
||||
}
|
||||
}
|
||||
|
||||
// What only the giving-up consumer filters is its own: a subject delivered again reads back as the
|
||||
// event's, and a module's runtime reads the same key from it (the tokens around `.event.`).
|
||||
func TestASubjectDeliveredAgainSaysTheEventsOwn(t *testing.T) {
|
||||
for _, original := range []string{"mesh.mod.gitea.event.pull.merged", "mesh.seat.node-build-agent.event.built"} {
|
||||
again := broker.AgainSubject("ace_sonarr", original)
|
||||
if back, ok := broker.OriginalOfAgain(again); !ok || back != original {
|
||||
t.Errorf("%s reads back as %s", again, back)
|
||||
}
|
||||
key := func(subject string) string {
|
||||
before, event, _ := strings.Cut(subject, ".event.")
|
||||
parts := strings.Split(before, ".")
|
||||
return parts[len(parts)-1] + "." + event
|
||||
}
|
||||
if key(again) != key(original) {
|
||||
t.Errorf("%s reads as key %s, the event as %s", again, key(again), key(original))
|
||||
}
|
||||
}
|
||||
if _, ok := broker.OriginalOfAgain("mesh.mod.gitea.event.pull.merged"); ok {
|
||||
t.Error("an event's own subject reads as delivered again")
|
||||
}
|
||||
}
|
||||
|
||||
// Only an event is delivered again, and only to a consumer that is on the bus and filters its again
|
||||
// subject: a dead letter is never let go as delivered while nobody receives it.
|
||||
func TestADeadLetterIsDeliveredAgainOnlyWhereItIsReceived(t *testing.T) {
|
||||
js := aBus(t)
|
||||
keeping(t, js)
|
||||
failing := aModuleConsumer(t, js, "media", "sonarr", "gitea.pull.merged")
|
||||
if _, err := js.Context().Publish("mesh.mod.gitea.event.pull.merged", []byte(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := givenUp(t, js, failing)
|
||||
|
||||
// A consumer as it was before this fix: its filters do not take what is delivered again.
|
||||
c, _ := broker.ConsumerFor(broker.Principal{Kind: broker.KindModule, Node: "media", Module: "sonarr",
|
||||
Consumes: []string{"gitea.pull.merged"}, PasswordHash: "x"})
|
||||
c.Filters = c.Filters[:len(c.Filters)-1]
|
||||
if err := js.EnsureConsumer(c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := DeliverAgain(js.Context(), d.ID); err == nil || !strings.Contains(err.Error(), "does not yet take") {
|
||||
t.Fatalf("delivered to a consumer that does not filter its again subject: %v", err)
|
||||
}
|
||||
if _, err := DeadLetterNamed(js.Context(), d.ID); err != nil {
|
||||
t.Fatalf("a refused delivery let the dead letter go: %v", err)
|
||||
}
|
||||
|
||||
// The consumer gone: nothing would receive it.
|
||||
if err := js.Context().DeleteConsumer(broker.EventsStream, c.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := DeliverAgain(js.Context(), d.ID); err == nil || !strings.Contains(err.Error(), "no longer on the bus") {
|
||||
t.Fatalf("delivered to a consumer that is gone: %v", err)
|
||||
}
|
||||
if _, err := DeadLetterNamed(js.Context(), d.ID); err != nil {
|
||||
t.Fatalf("a refused delivery let the dead letter go: %v", err)
|
||||
}
|
||||
|
||||
// Not an event: refused, whatever stream it is from.
|
||||
for _, other := range []DeadLetter{
|
||||
{ID: 2, Stream: "SEAT_TELEGRAM_SENDER", Consumer: "SEAT_TELEGRAM_SENDER_worker", Subject: "mesh.seat.telegram-sender.accept.send"},
|
||||
{ID: 3, Stream: "KV_x", Consumer: "y", Subject: "$KV.x.k"},
|
||||
} {
|
||||
if _, err := AgainTo(js.Context(), other); err == nil {
|
||||
t.Errorf("%s was given a subject to be delivered again on", other.Stream)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A total says how many are held, however few the list carries.
|
||||
func TestTheListSaysTheTotalAndStopsAtItsLimit(t *testing.T) {
|
||||
js := aBus(t)
|
||||
for seq := 1; seq <= 5; seq++ {
|
||||
if _, err := KeepDeadLetter(js.Context(), []byte(fmt.Sprintf(`{"stream":"EVENTS","consumer":"media_sonarr",`+
|
||||
`"stream_seq":%d,"deliveries":5}`, seq))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
list, total, err := DeadLetters(js.Context(), "media_sonarr", 2)
|
||||
if err != nil || total != 5 || len(list) != 2 || list[0].Sequence != 5 {
|
||||
t.Fatalf("%d of %d (%v): %+v", len(list), total, err, list)
|
||||
}
|
||||
if _, total, _ := DeadLetters(js.Context(), "another_one", 2); total != 0 {
|
||||
t.Fatalf("another consumer's total is %d", total)
|
||||
}
|
||||
}
|
||||
|
||||
// An event the controller itself gave up on, delivered again by a person, is acted on as the event it
|
||||
// was: it arrives under the controller's again subject, which its consumer filters.
|
||||
func TestTheControllerActsOnAnEventDeliveredAgain(t *testing.T) {
|
||||
js := aBus(t)
|
||||
told := &toldAbout{}
|
||||
s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()}
|
||||
if err := s.Follows(told); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Answers(replaysWith{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
go func() { _ = s.Serve(ctx) }()
|
||||
eventually(t, "the controller's event consumer being made", func() bool {
|
||||
_, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName)
|
||||
return err == nil
|
||||
})
|
||||
moved, _ := json.Marshal(Upgraded{Module: "gitea", Commit: "abcdef0123"})
|
||||
if _, err := js.Context().Publish(broker.AgainSubject(broker.ControllerName, broker.ControllerFollows[0]), moved); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
eventually(t, "the upgrade delivered again reaching the controller", func() bool { return told.count() == 1 })
|
||||
}
|
||||
|
||||
// The notices cannot be taken (their stream is gone): said as a condition, and the controller serves on.
|
||||
func TestNoticesThatCannotBeTakenAreSaidAndServingGoesOn(t *testing.T) {
|
||||
js := aBus(t)
|
||||
if err := js.Context().DeleteStream(broker.DeadLetterNoticesStream); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stop := keepingGivenUp(context.Background(), js, quiet())
|
||||
defer stop()
|
||||
eventually(t, "the failure said", func() bool {
|
||||
for _, a := range Advisories.Since(time.Now().Add(-time.Minute)) {
|
||||
if a.Token == AdvisoryNotKept && a.Stream == broker.DeadLetterNoticesStream {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
})
|
||||
held := &counted{}
|
||||
_, stopServing := servingOn(t, js, held)
|
||||
defer stopServing()
|
||||
if _, err := js.Context().Publish("mesh.control.anchor.report", []byte(`{"node":"anchor"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
eventually(t, "a report heard while the notices cannot be taken", func() bool { return held.count() == 1 })
|
||||
}
|
||||
@@ -41,6 +41,15 @@ type natsInbound struct {
|
||||
// that restarts loses these and starts the window again, which is correct — it is holding
|
||||
// nothing, and the messages are all still on the server.
|
||||
since map[uint64]time.Time
|
||||
// log is where it says what it could not do; the standard logger when nobody gave one.
|
||||
log *log.Logger
|
||||
}
|
||||
|
||||
func (n *natsInbound) logger() *log.Logger {
|
||||
if n.log != nil {
|
||||
return n.log
|
||||
}
|
||||
return log.Default()
|
||||
}
|
||||
|
||||
// Nats is the consume side of the bus being built.
|
||||
@@ -70,7 +79,7 @@ func (n *natsInbound) Close() {}
|
||||
// which message is held, and since when — is read and written without a lock because the AMQP loop
|
||||
// never had two. A second goroutine would make that wrong in a way no test would catch.
|
||||
func (n *natsInbound) Receive(ctx context.Context, act func(context.Context, Control)) error {
|
||||
if err := broker.AssertMeshConsumers(n.js); err != nil {
|
||||
if err := broker.AssertServingConsumers(n.js); err != nil {
|
||||
return err
|
||||
}
|
||||
js, conn := n.js.Context(), n.js.Conn()
|
||||
@@ -94,6 +103,13 @@ func (n *natsInbound) Receive(ctx context.Context, act func(context.Context, Con
|
||||
holding.Store(true)
|
||||
defer holding.Store(false)
|
||||
|
||||
// What a consumer gave up on, kept by the controller acting now (novox/hq issue 330): the server's
|
||||
// notices wait in their stream for it, so one said while no controller listened is not lost.
|
||||
// **Never the reason the controller stops serving**: a notice it cannot take yet waits in its stream,
|
||||
// and the failure is said and tried again every minute.
|
||||
stopKeeping := keepingGivenUp(ctx, n.js, n.logger())
|
||||
defer stopKeeping()
|
||||
|
||||
// Heartbeats, on core NATS and off any stream (design 25 §3). Their own subscription because
|
||||
// they are their own guarantee: a lost one is the next one.
|
||||
beats := make(chan *nats.Msg, Prefetch)
|
||||
@@ -167,6 +183,11 @@ func (n *natsInbound) Receive(ctx context.Context, act func(context.Context, Con
|
||||
func (n *natsInbound) deliver(ctx context.Context, act func(context.Context, Control),
|
||||
msg *nats.Msg, streamed bool) {
|
||||
|
||||
// An event the controller gave up on and a person delivered again (novox/hq issue 330) arrives under
|
||||
// the controller's own again subject, and is the event it was: acted on as first published.
|
||||
if original, ok := broker.OriginalOfAgain(msg.Subject); ok {
|
||||
msg.Subject = original
|
||||
}
|
||||
kind, known := kindOfSubject(msg.Subject)
|
||||
if !known {
|
||||
if streamed {
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// novox/hq issue 330, replayed with only what the link had before its fix, so it can be laid over the
|
||||
// older commit. On 2026-10-06 a media server's event consumer gave up on several messages after five
|
||||
// deliveries each. The controller raised a condition for each run and cleared it within minutes; the
|
||||
// messages were kept only by EVENTS, which drops an event after a week, and nothing said which they
|
||||
// were. Design 25 promised a dead-letter stream that does not exist. A consumer that never acknowledges
|
||||
// an event: once it gives up, the event is kept with its consumer and how often it was handed over.
|
||||
func TestReplay330(t *testing.T) {
|
||||
js := aBus(t)
|
||||
consumer, ok := broker.ConsumerFor(broker.Principal{Kind: broker.KindModule, Node: "media", Module: "sonarr",
|
||||
Consumes: []string{"gitea.pull.merged"}, PasswordHash: "x"})
|
||||
if !ok {
|
||||
t.Fatal("a module that consumes got no consumer")
|
||||
}
|
||||
if err := js.EnsureConsumer(consumer); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, stop := servingOn(t, js, &counted{})
|
||||
defer stop()
|
||||
|
||||
const subject = "mesh.mod.gitea.event.pull.merged"
|
||||
if _, err := js.Context().Publish(subject, []byte(`{"repository":"novox/media"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The module's handler fails every time, as the media server's did.
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reading, err := api.Consumer(t.Context(), broker.EventsStream, consumer.Name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for handed := 0; handed < consumer.MaxDeliver; handed++ {
|
||||
batch, err := reading.Fetch(1, jetstream.FetchMaxWait(5*time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n := 0
|
||||
for msg := range batch.Messages() {
|
||||
n++
|
||||
_ = msg.Nak()
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("handed over %d times, then nothing: %v", handed, batch.Error())
|
||||
}
|
||||
}
|
||||
// And it goes on reading, as a module's runtime does: the server gives the event up when it would
|
||||
// hand it over a sixth time.
|
||||
if batch, err := reading.Fetch(1, jetstream.FetchMaxWait(time.Second)); err == nil {
|
||||
for msg := range batch.Messages() {
|
||||
t.Fatalf("handed over a sixth time: %s", msg.Subject())
|
||||
}
|
||||
}
|
||||
|
||||
kept := "mesh.events.dead." + broker.EventsStream + "." + consumer.Name
|
||||
var held *nats.RawStreamMsg
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for time.Now().Before(deadline) && held == nil {
|
||||
if stream, err := js.Context().StreamNameBySubject(kept); err == nil {
|
||||
held, _ = js.Context().GetLastMsg(stream, kept)
|
||||
}
|
||||
if held == nil {
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
if held == nil {
|
||||
t.Fatalf("%s gave up on the event and nothing on the bus keeps it under %s", consumer.Name, kept)
|
||||
}
|
||||
if string(held.Data) != `{"repository":"novox/media"}` {
|
||||
t.Errorf("kept %q, not the event", held.Data)
|
||||
}
|
||||
for header, want := range map[string]string{"Mesh-Dead-Consumer": consumer.Name, "Mesh-Dead-Stream": "EVENTS",
|
||||
"Mesh-Dead-Subject": subject, "Mesh-Dead-Deliveries": "5"} {
|
||||
if got := held.Header.Get(header); got != want {
|
||||
t.Errorf("the kept event's %s is %q, not %q", header, got, want)
|
||||
}
|
||||
}
|
||||
if held.Header.Get("Mesh-Dead-Gave-Up") == "" {
|
||||
t.Error("the kept event does not say when it was given up")
|
||||
}
|
||||
}
|
||||
@@ -71,6 +71,7 @@
|
||||
"bus",
|
||||
"retire",
|
||||
"cleanup",
|
||||
"dead-letters",
|
||||
"data",
|
||||
"build",
|
||||
"artifacts",
|
||||
|
||||
Reference in New Issue
Block a user