Join: merge A5 (wip/registry-a5)
This commit is contained in:
@@ -0,0 +1,22 @@
|
||||
package main
|
||||
|
||||
import "nats-tools/internal/bussnap"
|
||||
|
||||
// The snapshot itself is bussnap's, which the keeper takes one entry's with too (design 25 § 12.3).
|
||||
type (
|
||||
SnapshotOptions = bussnap.SnapshotOptions
|
||||
apiError = bussnap.APIError
|
||||
streamState = bussnap.StreamState
|
||||
)
|
||||
|
||||
var (
|
||||
Snapshot = bussnap.Snapshot
|
||||
StreamNames = bussnap.StreamNames
|
||||
request = bussnap.Request
|
||||
truncate = bussnap.Truncate
|
||||
)
|
||||
|
||||
const (
|
||||
apiStreamInfo = bussnap.APIStreamInfo
|
||||
apiStreamRestore = bussnap.APIStreamRestore
|
||||
)
|
||||
@@ -31,6 +31,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"nats-tools/internal/busarchive"
|
||||
"nats-tools/internal/bussnap"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
@@ -200,15 +201,4 @@ func logf(format string, args ...any) {
|
||||
fmt.Fprintf(os.Stderr, format+"\n", args...)
|
||||
}
|
||||
|
||||
func human(n int64) string {
|
||||
const unit = 1024
|
||||
if n < unit {
|
||||
return fmt.Sprintf("%d B", n)
|
||||
}
|
||||
div, exp := int64(unit), 0
|
||||
for v := n / unit; v >= unit; v /= unit {
|
||||
div *= unit
|
||||
exp++
|
||||
}
|
||||
return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp])
|
||||
}
|
||||
func human(n int64) string { return bussnap.Human(n) }
|
||||
|
||||
@@ -0,0 +1,301 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
|
||||
|
||||
"nats-tools/internal/keeper"
|
||||
)
|
||||
|
||||
// The seat mesh-broker's reading verbs (design 25 § 12.3), served by this bundle: every entry the keeper made, one
|
||||
// entry, what the bus holds against what was asked, and the orphans; each read from the keeper's own state asked and
|
||||
// the bus's monitoring endpoint, never from another part's state. And nats_history: every event the bus module said of
|
||||
// one entry, in order.
|
||||
|
||||
// heldEntries reads the keeper's own state asked: every module it holds.
|
||||
type heldEntries func() (map[string]keeper.Asked, error)
|
||||
|
||||
// HeldInState reads asked through the runtime that launched this bundle.
|
||||
func HeldInState() (map[string]keeper.Asked, error) {
|
||||
s := stdio.State(keeper.StateAsked)
|
||||
keys, err := s.Keys()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the keeper's own %s cannot be read: %w", keeper.StateAsked, err)
|
||||
}
|
||||
out := map[string]keeper.Asked{}
|
||||
for _, key := range keys {
|
||||
if key == keeper.KeyStatus {
|
||||
continue
|
||||
}
|
||||
e, err := s.Get(key)
|
||||
if err != nil || e == nil {
|
||||
continue
|
||||
}
|
||||
var a keeper.Asked
|
||||
if json.Unmarshal(e.Value, &a) == nil {
|
||||
out[key] = a
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// busStream is one stream as the monitoring endpoint answers it.
|
||||
type busStream struct {
|
||||
Config jetstream.StreamConfig `json:"config"`
|
||||
State struct {
|
||||
Messages uint64 `json:"messages"`
|
||||
Bytes uint64 `json:"bytes"`
|
||||
} `json:"state"`
|
||||
}
|
||||
|
||||
// streamsHeld reads every stream the bus holds from the monitoring endpoint's jsz, by name.
|
||||
func streamsHeld(jsz []byte) (map[string]busStream, error) {
|
||||
var doc struct {
|
||||
Accounts []struct {
|
||||
Streams []struct {
|
||||
Name string `json:"name"`
|
||||
busStream
|
||||
} `json:"stream_detail"`
|
||||
} `json:"account_details"`
|
||||
}
|
||||
if err := json.Unmarshal(jsz, &doc); err != nil {
|
||||
return nil, fmt.Errorf("the bus's streams cannot be read: %w", err)
|
||||
}
|
||||
out := map[string]busStream{}
|
||||
for _, a := range doc.Accounts {
|
||||
for _, s := range a.Streams {
|
||||
out[s.Name] = s.busStream
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// heldOf is one entry as the seat's list, show and orphans answer it (mesh-contracts meshbroker.Held): its bus name, its
|
||||
// template, its options or configuration as the manifest asked them, the settings the bus holds of each of its objects
|
||||
// by stream, and its fill summed over them against their caps.
|
||||
func heldOf(module, local string, m keeper.Made, held map[string]busStream) meshbroker.Held {
|
||||
h := meshbroker.Held{BusName: keeper.BusName(module, local), Module: module, LocalName: local, StorageType: m.Plan.Storage,
|
||||
Template: m.Plan.Template, Options: anyOf(m.Plan.Entry.Options), Configuration: anyOf(m.Plan.Entry.Configuration)}
|
||||
settings := map[string]any{}
|
||||
var fill meshbroker.Fill
|
||||
for _, o := range m.Plan.Objects {
|
||||
s, ok := held[o.Stream()]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
settings[o.Stream()] = keeper.SettingsOf(o.Kind, s.Config)
|
||||
fill.Bytes += int64(s.State.Bytes)
|
||||
fill.Count += int64(s.State.Messages)
|
||||
fill.Cap += s.Config.MaxBytes
|
||||
}
|
||||
if len(settings) > 0 {
|
||||
h.Settings, h.Fill = settings, &fill
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// differencesOf are an entry's objects the bus holds otherwise than asked, or not at all, as the seat's check answers
|
||||
// them: each object's settings asked and held by stream, and whether the difference only grows (apply may act on it).
|
||||
func differencesOf(module, local string, m keeper.Made, held map[string]busStream) (meshbroker.CheckAnswerDifferencesItem, bool) {
|
||||
d := meshbroker.CheckAnswerDifferencesItem{Module: module, LocalName: local, Asked: map[string]any{}, Held: map[string]any{}, Grows: true}
|
||||
differs := false
|
||||
for _, o := range m.Plan.Objects {
|
||||
s, ok := held[o.Stream()]
|
||||
if !ok {
|
||||
d.Asked[o.Stream()], d.Grows, differs = o.Settings, false, true
|
||||
continue
|
||||
}
|
||||
h := keeper.SettingsOf(o.Kind, s.Config)
|
||||
if keeper.Differs(o.Settings, h) == "" {
|
||||
continue
|
||||
}
|
||||
d.Asked[o.Stream()], d.Held[o.Stream()], differs = o.Settings, h, true
|
||||
d.Grows = d.Grows && keeper.Grows(o.Settings, h)
|
||||
}
|
||||
return d, differs
|
||||
}
|
||||
|
||||
// anyOf is a manifest's options or configuration as the contract carries them.
|
||||
func anyOf(m map[string]json.RawMessage) map[string]any {
|
||||
if len(m) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := map[string]any{}
|
||||
for k, v := range m {
|
||||
var x any
|
||||
if json.Unmarshal(v, &x) == nil {
|
||||
out[k] = x
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// brokerTools are the four reading verbs and nats_history.
|
||||
func brokerTools(read heldEntries, jsz func() ([]byte, error), history func(module, local string, limit int) ([]keeper.Event, error)) []stdio.Tool {
|
||||
both := func() (map[string]keeper.Asked, map[string]busStream, error) {
|
||||
entries, err := read()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
doc, err := jsz()
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the bus's monitoring endpoint cannot be read: %w", err)
|
||||
}
|
||||
held, err := streamsHeld(doc)
|
||||
return entries, held, err
|
||||
}
|
||||
each := func(entries map[string]keeper.Asked, held map[string]busStream, keep func(keeper.Made) bool) []meshbroker.Held {
|
||||
out := []meshbroker.Held{}
|
||||
for _, module := range sortedNames(entries) {
|
||||
for _, local := range sortedNames(entries[module].Made) {
|
||||
if m := entries[module].Made[local]; keep(m) {
|
||||
out = append(out, heldOf(module, local, m, held))
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
entryArgs := map[string]any{"module": str("the module, e.g. mesh-knowledge"), "local-name": str("its local name, e.g. records")}
|
||||
return []stdio.Tool{
|
||||
{Name: "mesh-broker.list",
|
||||
Description: "Every module's kept data on the bus that the bus module created (design 25 § 12.3): each entry's bus name, template and options as asked, the settings the bus holds of each of its objects, and its fill against its cap. Narrowed by a module.",
|
||||
Input: map[string]any{"module": str("one module")},
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
entries, held, err := both()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if m := arg(a, "module"); m != "" {
|
||||
entries = map[string]keeper.Asked{m: entries[m]}
|
||||
}
|
||||
return meshbroker.ListAnswer{Entries: each(entries, held, func(keeper.Made) bool { return true })}, nil
|
||||
}},
|
||||
{Name: "mesh-broker.show",
|
||||
Description: "One entry of a module's kept data on the bus: its bus name, template and options as asked, the settings the bus holds of each of its objects, and its fill against its cap. Whether it is refused, waiting, drifted, full or orphaned is what nats_history says of it.",
|
||||
Input: entryArgs,
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
entries, held, err := both()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
module, local := arg(a, "module"), arg(a, "local-name")
|
||||
m, ok := entries[module].Made[local]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("the bus module made no entry %q of %q", local, module)
|
||||
}
|
||||
return heldOf(module, local, m, held), nil
|
||||
}},
|
||||
{Name: "mesh-broker.check",
|
||||
Description: "What the bus holds against what was asked, entry by entry: each object missing, or differing from its manifest's settings, with both. A difference that only grows is applied by the bus module itself; any other is a drift, raised to the operator.",
|
||||
Input: map[string]any{"module": str("one module")},
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
entries, held, err := both()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if m := arg(a, "module"); m != "" {
|
||||
entries = map[string]keeper.Asked{m: entries[m]}
|
||||
}
|
||||
out := meshbroker.CheckAnswer{Differences: []meshbroker.CheckAnswerDifferencesItem{}}
|
||||
for _, module := range sortedNames(entries) {
|
||||
for _, local := range sortedNames(entries[module].Made) {
|
||||
m := entries[module].Made[local]
|
||||
if !m.Created || m.Orphaned {
|
||||
continue
|
||||
}
|
||||
if d, differs := differencesOf(module, local, m, held); differs {
|
||||
out.Differences = append(out.Differences, d)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}},
|
||||
{Name: "mesh-broker.orphans",
|
||||
Description: "Kept data on the bus that no accepted manifest asks for any more: each goes only by the bus module's drop, on the operator's warrant (nats_drop). Never an entry of a refused or set-aside manifest, nor a foundation part's old section.",
|
||||
Run: func(map[string]any) (any, error) {
|
||||
entries, held, err := both()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return meshbroker.OrphansAnswer{Entries: each(entries, held, func(m keeper.Made) bool { return m.Orphaned })}, nil
|
||||
}},
|
||||
{Name: "nats_history",
|
||||
Description: "Every event the bus module said of one entry of a module's kept data, in order (created, refused, drifted, drift-cleared, full, full-cleared, orphaned, recreated, dropped), from its own stream, kept 30 days.",
|
||||
Input: map[string]any{"module": str("the module"), "local": str("its local name"), "limit": num("at most this many, the newest (default 100)")},
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
module, local := arg(a, "module"), arg(a, "local")
|
||||
if module == "" || local == "" {
|
||||
return nil, fmt.Errorf("history names one entry: a module and its local name")
|
||||
}
|
||||
events, err := history(module, local, limitOf(a, 100))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"module": module, "local": local, "events": events}, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
// StreamHistory reads the bus module's events of one entry from its own stream, on the tools' own credential.
|
||||
func StreamHistory(bus Bus) func(module, local string, limit int) ([]keeper.Event, error) {
|
||||
return func(module, local string, limit int) ([]keeper.Event, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
js, done, err := bus(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer done()
|
||||
cons, err := js.OrderedConsumer(ctx, BrokerEventsStream, jetstream.OrderedConsumerConfig{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bus module's stream %s cannot be read: %w", BrokerEventsStream, err)
|
||||
}
|
||||
info, err := cons.Info(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []keeper.Event
|
||||
for left := info.NumPending; left > 0; {
|
||||
batch, err := cons.Fetch(int(min(left, 256)), jetstream.FetchMaxWait(5*time.Second))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
n := uint64(0)
|
||||
for m := range batch.Messages() {
|
||||
n++
|
||||
var e keeper.Event
|
||||
if json.Unmarshal(m.Data(), &e) == nil && e.Module == module && e.Local == local {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
if n == 0 {
|
||||
break
|
||||
}
|
||||
left -= min(n, left)
|
||||
}
|
||||
if len(out) > limit {
|
||||
out = out[len(out)-limit:]
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
|
||||
// BrokerEventsStream is the bus module's own stream on the bus (catalogue.StreamBusName("broker-events")).
|
||||
const BrokerEventsStream = "BROKER_EVENTS"
|
||||
|
||||
func sortedNames[V any](m map[string]V) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
"nats-tools/internal/keeper"
|
||||
)
|
||||
|
||||
// The seat's reading verbs and nats_history (design 25 § 12.3; CAT-N7): list and show answer every entry the keeper
|
||||
// made with its settings asked and held and its fill against its cap; check names what is missing or differs; orphans
|
||||
// what no manifest asks for; history one entry's events in order. Each from the keeper's own state and the bus's
|
||||
// monitoring endpoint.
|
||||
func TestTheSeatsReadingVerbsAnswerFromTheKeeperAndTheBus(t *testing.T) {
|
||||
plan := func(local, storage, template string, options string) keeper.Plan {
|
||||
e := catalogue.BusStateEntry{Name: local, Storage: storage, Template: template}
|
||||
if options != "" {
|
||||
_ = json.Unmarshal([]byte(options), &e.Options)
|
||||
}
|
||||
p, err := keeper.PlanOf("m", e)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
held := map[string]keeper.Asked{"m": {Module: "m", Made: map[string]keeper.Made{
|
||||
"drafts": {Plan: plan("drafts", "key-value", "state", ""), Created: true},
|
||||
"log": {Plan: plan("log", "stream", "log", `{"max-mib":16}`), Created: true, Drift: "m_log: max-bytes asked 16777216, held 33554432"},
|
||||
"old": {Plan: plan("old", "stream", "events", ""), Created: true, Orphaned: true},
|
||||
}}}
|
||||
jsz := `{"account_details":[{"stream_detail":[
|
||||
{"name":"KV_m_drafts","config":{"name":"KV_m_drafts","retention":"limits","discard":"new","max_bytes":67108864,"max_msgs_per_subject":1,"max_msg_size":262144},"state":{"messages":3,"bytes":1048576}},
|
||||
{"name":"m_log","config":{"name":"m_log","retention":"limits","discard":"new","max_bytes":33554432,"max_msg_size":262144},"state":{"messages":9,"bytes":9000}}]}]}`
|
||||
tools := map[string]stdio.Tool{}
|
||||
for _, tl := range brokerTools(func() (map[string]keeper.Asked, error) { return held, nil },
|
||||
func() ([]byte, error) { return []byte(jsz), nil },
|
||||
func(module, local string, limit int) ([]keeper.Event, error) {
|
||||
return []keeper.Event{{Module: module, Local: local, Why: "first"}, {Module: module, Local: local, Why: "second"}}[:limit], nil
|
||||
}) {
|
||||
tools[tl.Name] = tl
|
||||
}
|
||||
for _, row := range []struct {
|
||||
tool string
|
||||
args map[string]any
|
||||
want []string
|
||||
err string
|
||||
}{
|
||||
// Each answers the seat's contract (mesh-contracts meshbroker ListAnswer, Held, CheckAnswer, OrphansAnswer).
|
||||
{"mesh-broker.list", nil, []string{`{"entries":[`, `"bus-name":"m_drafts"`, `"local-name":"drafts"`, `"storage-type":"key-value"`,
|
||||
`"KV_m_drafts":{"max-bytes":67108864`, `"fill":{"bytes":1048576,"cap":67108864,"count":3}`, `"options":{"max-mib":16}`}, ""},
|
||||
{"mesh-broker.show", map[string]any{"module": "m", "local-name": "log"}, []string{`"bus-name":"m_log"`, `"template":"log"`,
|
||||
`"m_log":{"max-bytes":33554432`}, ""},
|
||||
{"mesh-broker.show", map[string]any{"module": "m", "local-name": "none"}, nil, `made no entry "none" of "m"`},
|
||||
{"mesh-broker.check", nil, []string{`{"differences":[{`, `"local-name":"log"`, `"asked":{"m_log":{"max-bytes":16777216`,
|
||||
`"held":{"m_log":{"max-bytes":33554432`}, ""},
|
||||
{"mesh-broker.check", map[string]any{"module": "x"}, []string{`{"differences":[]}`}, ""},
|
||||
{"mesh-broker.orphans", nil, []string{`{"entries":[{`, `"local-name":"old"`}, ""},
|
||||
{"nats_history", map[string]any{"module": "m", "local": "log", "limit": float64(1)}, []string{`"why":"first"`}, ""},
|
||||
{"nats_history", map[string]any{"module": "m"}, nil, "names one entry"},
|
||||
} {
|
||||
t.Run(fmt.Sprintf("%s %v", row.tool, row.args), func(t *testing.T) {
|
||||
args := row.args
|
||||
if args == nil {
|
||||
args = map[string]any{}
|
||||
}
|
||||
out, err := tools[row.tool].Run(args)
|
||||
if row.err != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), row.err) {
|
||||
t.Fatalf("answered %v (%v): refused, naming %q", out, err, row.err)
|
||||
}
|
||||
return
|
||||
}
|
||||
raw, _ := json.Marshal(out)
|
||||
for _, w := range row.want {
|
||||
if !strings.Contains(string(raw), w) {
|
||||
t.Errorf("%s lacks %s", raw, w)
|
||||
}
|
||||
}
|
||||
if row.tool == "mesh-broker.check" && strings.Contains(string(raw), `"local-name":"drafts"`) {
|
||||
t.Errorf("check names drafts, which the bus holds as asked: %s", raw)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,360 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
|
||||
"nats-tools/internal/bussnap"
|
||||
"nats-tools/internal/keeper"
|
||||
)
|
||||
|
||||
// The keeper (`nats-tools keep`): the bus module's process on its own account (own secret broker-keeper, ADR 0436 §1),
|
||||
// started by the node-engine as the manifest's process keeper. It asks the indexer's first view once, then hears
|
||||
// manifest-accepted on its durable consumer, makes what each asks for, reads the fill every minute, and acts on the
|
||||
// persons' requests its tools put in its own state requests (requests.go). It serves nothing (ADR 0436 §3).
|
||||
|
||||
// AcceptedOf reads a manifest-accepted body: mesh-sdk's delivery.ManifestAccepted, with the indexer's revision of the
|
||||
// verdict when the event carries it (a field mesh-sdk is asked to add; 0 until then).
|
||||
func AcceptedOf(body []byte) (keeper.Accepted, error) {
|
||||
var a struct {
|
||||
delivery.ManifestAccepted
|
||||
Revision uint64 `json:"revision"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &a); err != nil {
|
||||
return keeper.Accepted{}, err
|
||||
}
|
||||
if a.Module == "" {
|
||||
return keeper.Accepted{}, fmt.Errorf("a manifest-accepted names no module")
|
||||
}
|
||||
return keeper.Accepted{Module: a.Module, Digest: a.Digest, Revision: a.Revision, BusState: a.BusState}, nil
|
||||
}
|
||||
|
||||
// AcceptedSubject is the indexer's manifest-accepted on the shared events stream.
|
||||
var AcceptedSubject = "mesh.mod." + keeper.FirstViewOwner + ".event." + delivery.ManifestAcceptedEvent
|
||||
|
||||
// The router's records of asks, read under this module's own name (ADR 0259 §3), as the mail sender reads them.
|
||||
const asksBucket = "messenger_asks"
|
||||
|
||||
// processStore is the keeper's own state through its process connection.
|
||||
type processStore struct{ c *process.Conn }
|
||||
|
||||
func (s processStore) Get(state, key string) ([]byte, uint64, bool, error) {
|
||||
e, err := s.c.State(state).Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
return e.Value, e.Revision, true, nil
|
||||
}
|
||||
|
||||
func (s processStore) Put(state, key string, value []byte) (uint64, error) {
|
||||
return s.c.State(state).Put(key, json.RawMessage(value))
|
||||
}
|
||||
|
||||
func (s processStore) Delete(state, key string) error { return s.c.State(state).Delete(key) }
|
||||
|
||||
func (s processStore) Keys(state string) ([]string, error) { return s.c.State(state).Keys() }
|
||||
|
||||
// streamEmitter says an event of the seat mesh-broker, with the SDK's headers, twice under one event id (the lead's ruling,
|
||||
// registry milestone, 13:0x; design 25 § 12.3): on the seat's own subject `mesh.seat.mesh-broker.event.<event>`, which the
|
||||
// shared events stream keeps 7 days and every consumer of `mesh-broker.<event>` hears, like any seat's event; then on the
|
||||
// bus module's own stream (its old `streams` section, 30 days, 256 MiB), which its tool `history` reads. The seat's subject
|
||||
// first: a reader hears it before the history holds it, never the other way round.
|
||||
func streamEmitter(js jetstream.JetStream, a process.Account) func(string, keeper.Event) error {
|
||||
return func(event string, body keeper.Event) error {
|
||||
data, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
id := eventID()
|
||||
at := time.Now().UTC().Format(time.RFC3339Nano)
|
||||
for _, subject := range EventSubjects(a, event) {
|
||||
msg := nats.NewMsg(subject)
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", a.Module)
|
||||
msg.Header.Set("x-node", a.Node)
|
||||
msg.Header.Set("x-time", at)
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Data = data
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
_, err = js.PublishMsg(ctx, msg, jetstream.WithMsgID(id))
|
||||
cancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("saying %s on %s: %w", event, subject, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// EventSubjects are where one event of the seat mesh-broker is said, in order: the seat's subject, then the bus module's
|
||||
// own stream.
|
||||
func EventSubjects(a process.Account, event string) []string {
|
||||
return []string{seatdef.EventSubject(catalogue.BrokerSeat, event), a.StreamSubject(event)}
|
||||
}
|
||||
|
||||
func eventID() string {
|
||||
var b [16]byte
|
||||
_, _ = rand.Read(b[:])
|
||||
return hex.EncodeToString(b[:])
|
||||
}
|
||||
|
||||
// sayer raises and clears the keeper's conditions through the condition keeper, as every module says them.
|
||||
func sayer(js jetstream.JetStream) func(asks.Condition) error {
|
||||
return func(c asks.Condition) error {
|
||||
subject, err := asks.ConditionSubject(keeper.Module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := asks.EncodeCondition(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_, err = js.Publish(ctx, subject, data)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// firstView asks the indexer's indexer_bus_state as the keeper's own account.
|
||||
func firstView(c *process.Conn) func() (keeper.FirstView, error) {
|
||||
return func() (keeper.FirstView, error) {
|
||||
raw, err := c.Call(keeper.FirstViewOwner+"."+keeper.FirstViewVerb, map[string]any{})
|
||||
if err != nil {
|
||||
return keeper.FirstView{}, err
|
||||
}
|
||||
return FirstViewOf(raw)
|
||||
}
|
||||
}
|
||||
|
||||
// FirstViewOf reads indexer_bus_state's answer, as the tool runner hands it back ({result, error} or bare).
|
||||
func FirstViewOf(raw []byte) (keeper.FirstView, error) {
|
||||
var wrapped struct {
|
||||
Result json.RawMessage `json:"result"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
if json.Unmarshal(raw, &wrapped) == nil && (wrapped.Result != nil || wrapped.Error != "") {
|
||||
if wrapped.Error != "" {
|
||||
return keeper.FirstView{}, fmt.Errorf("%s", wrapped.Error)
|
||||
}
|
||||
raw = wrapped.Result
|
||||
}
|
||||
var v keeper.FirstView
|
||||
if err := json.Unmarshal(raw, &v); err != nil {
|
||||
return keeper.FirstView{}, fmt.Errorf("the answer is not a first view: %w", err)
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
func percent(getenv func(string) string, name string, fallback int64) int64 {
|
||||
if n, err := strconv.ParseInt(strings.TrimSpace(getenv(name)), 10, 64); err == nil && n > 0 && n < 100 {
|
||||
return n
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
// runKeeper is the keeper's process: it returns when ctx ends, or when it has no credential.
|
||||
func runKeeper(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
|
||||
logger := log.New(logOut, "nats-keeper: ", log.LstdFlags)
|
||||
account, err := process.ReadBrokerAccount(getenv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
conn, err := process.Connect(account)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer conn.Close()
|
||||
nc, js, err := account.Dial(nats.Name(account.Node + "." + account.Module + ".keeper"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer nc.Close()
|
||||
recorder := handacts.Recorder{Ask: conn.Call}
|
||||
k := &keeper.Keeper{
|
||||
Bus: keeper.JetStream{JS: js, Snap: bussnap.Into(nc, getenv("MESH_NATS_SNAPSHOTS_DIR"))},
|
||||
Store: processStore{conn}, Emit: streamEmitter(js, account), Say: sayer(js), View: firstView(conn),
|
||||
Act: func(what, why, cause string) error {
|
||||
_, err := recorder.Record(handacts.Request{What: what, Why: why, Cause: cause})
|
||||
return err
|
||||
},
|
||||
Warrant: func(id string) (*asks.Ask, *asks.Warrant, error) {
|
||||
e, err := conn.Ask(asksBucket, id)
|
||||
if err != nil || e == nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var rec struct {
|
||||
Ask *asks.Ask `json:"ask"`
|
||||
Warrant *asks.Warrant `json:"warrant"`
|
||||
}
|
||||
if err := json.Unmarshal(e.Value, &rec); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return rec.Ask, rec.Warrant, nil
|
||||
},
|
||||
Logf: logger.Printf,
|
||||
Raise: percent(getenv, "MESH_NATS_FULL_RAISE", 75), Clear: percent(getenv, "MESH_NATS_FULL_CLEAR", 70),
|
||||
}
|
||||
logger.Printf("started on the account %s: asking %s.%s for the first view", account.User, keeper.FirstViewOwner, keeper.FirstViewVerb)
|
||||
return Keep(ctx, k, hearer(js, account, logger.Printf), requestsOf(conn), logger.Printf)
|
||||
}
|
||||
|
||||
// hearer binds the keeper's durable consumer on EVENTS (<node>_nats_keeper, filtered on manifest-accepted) and hands
|
||||
// each message to handle, which acknowledges it once applied.
|
||||
func hearer(js jetstream.JetStream, a process.Account, logf func(string, ...any)) func(context.Context, func(subject string, body []byte) error) error {
|
||||
return func(ctx context.Context, handle func(string, []byte) error) error {
|
||||
cons, err := js.Consumer(ctx, process.EventsStream, a.QueueConsumerName())
|
||||
if err != nil {
|
||||
return fmt.Errorf("the keeper's consumer %s on %s: %w", a.QueueConsumerName(), process.EventsStream, err)
|
||||
}
|
||||
cc, err := process.HearEvents(cons, func(m jetstream.Msg) {
|
||||
if err := handle(m.Subject(), m.Data()); err != nil {
|
||||
logf("%v: delivered again", err)
|
||||
_ = m.NakWithDelay(30 * time.Second)
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
}, logf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() { <-ctx.Done(); cc.Stop() }()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Keep is the keeper's loop, apart from the bus so a test drives it: the first view asked until it answers, on its
|
||||
// backoff; then hear, which applies each manifest-accepted; a fill pass every minute; the waiting entries made again
|
||||
// on the same backoff; and the persons' requests acted on as they come. It returns when ctx ends.
|
||||
func Keep(ctx context.Context, k *keeper.Keeper, hear func(context.Context, func(string, []byte) error) error,
|
||||
requests func(context.Context, func()) error, logf func(string, ...any)) error {
|
||||
var wait time.Duration
|
||||
for {
|
||||
next, err := k.AskFirstView(wait)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
logf("the first view did not answer (%v): nothing is created or orphaned until it does; asked again in %s", err, next)
|
||||
wait = next
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(next):
|
||||
}
|
||||
}
|
||||
logf("the first view answered at revision %d", k.Status().FirstView.Revision)
|
||||
heard := make(chan struct{}, 1)
|
||||
poke := func() {
|
||||
select {
|
||||
case heard <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
for backoff := 10 * time.Second; ; backoff = min(backoff*2, 5*time.Minute) {
|
||||
err := hear(ctx, func(subject string, body []byte) error {
|
||||
if subject != AcceptedSubject && !strings.HasSuffix(subject, ".event."+delivery.ManifestAcceptedEvent) {
|
||||
return nil
|
||||
}
|
||||
a, err := AcceptedOf(body)
|
||||
if err != nil {
|
||||
logf("a manifest-accepted cannot be read (%v): taken, and nothing made of it", err)
|
||||
return nil
|
||||
}
|
||||
k.Heard(a)
|
||||
poke()
|
||||
return nil
|
||||
})
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
logf("manifest-accepted cannot be heard yet (%v): asked again in %s", err, backoff)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
}
|
||||
asked := make(chan struct{}, 1)
|
||||
if requests != nil {
|
||||
if err := requests(ctx, func() {
|
||||
select {
|
||||
case asked <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}); err != nil {
|
||||
logf("the persons' requests cannot be watched (%v): acted on at each fill pass", err)
|
||||
}
|
||||
}
|
||||
tick := time.NewTicker(time.Minute)
|
||||
defer tick.Stop()
|
||||
retry := 10 * time.Second
|
||||
var again <-chan time.Time
|
||||
k.FillPass()
|
||||
for {
|
||||
if len(k.Waiting()) > 0 && again == nil {
|
||||
again = time.After(retry)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-tick.C:
|
||||
k.FillPass()
|
||||
// A recreate or a drop waits for the operator's warrant: read again once a minute, as an asker reads its asks.
|
||||
k.ActOnRequests()
|
||||
case <-asked:
|
||||
k.ActOnRequests()
|
||||
case <-heard:
|
||||
case <-again:
|
||||
again = nil
|
||||
k.CheckAll()
|
||||
if len(k.Waiting()) > 0 {
|
||||
retry = min(retry*2, 5*time.Minute)
|
||||
logf("%d entr(y/ies) wait to be made: %s; made again in %s", len(k.Waiting()), strings.Join(k.Waiting(), "; "), retry)
|
||||
} else {
|
||||
retry = 10 * time.Second
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// keepMain runs the keeper until a signal.
|
||||
func keepMain() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
if err := runKeeper(ctx, os.Getenv, os.Stderr); err != nil && ctx.Err() == nil {
|
||||
fmt.Fprintln(os.Stderr, "nats-keeper:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// requestsOf watches the keeper's own state requests, waking act at each change.
|
||||
func requestsOf(c *process.Conn) func(context.Context, func()) error {
|
||||
return func(_ context.Context, act func()) error {
|
||||
return c.State(keeper.StateRequests).Watch("", func(stdio.StateChange) error {
|
||||
act()
|
||||
return nil
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
"nats-tools/internal/keeper"
|
||||
)
|
||||
|
||||
// The keeper's inputs (design 25 § 12.2; CAT-N2): a manifest-accepted read with the indexer's revision when it carries
|
||||
// one (0 when not yet), its entries whole; the first view of indexer_bus_state read bare or as the tool runner wraps it,
|
||||
// an error answer an error; anything else refused, naming why.
|
||||
func TestTheKeepersInputsAreRead(t *testing.T) {
|
||||
for _, row := range []struct {
|
||||
name, body, want, refused string
|
||||
view bool
|
||||
}{
|
||||
{name: "accepted with its revision", body: `{"v":1,"module":"m","digest":"sha256:a","judge":"j","revision":7,"bus-state":[{"name":"drafts","storage":"key-value","template":"state"}]}`,
|
||||
want: "m sha256:a 7 [drafts/key-value/state]"},
|
||||
{name: "accepted before the event carries a revision", body: `{"v":1,"module":"m","digest":"sha256:b","judge":"j","bus-state":[]}`,
|
||||
want: "m sha256:b 0 []"},
|
||||
{name: "accepted naming no module", body: `{"v":1,"digest":"x"}`, refused: "names no module"},
|
||||
{name: "first view as the tool runner answers it", view: true,
|
||||
body: `{"result":{"revision":12,"modules":[{"module":"m","digest":"d","revision":11,"bus-state":[{"name":"log","storage":"stream","template":"log"}]}]}}`,
|
||||
want: "12: m d 11 [log/stream/log]"},
|
||||
{name: "first view bare", view: true, body: `{"revision":3,"modules":[]}`, want: "3:"},
|
||||
{name: "first view answering an error", view: true, body: `{"error":"not granted"}`, refused: "not granted"},
|
||||
{name: "first view that is none", view: true, body: `[1,2]`, refused: "is not a first view"},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
var got string
|
||||
var err error
|
||||
if row.view {
|
||||
v, e := FirstViewOf([]byte(row.body))
|
||||
err = e
|
||||
got = fmt.Sprintf("%d:", v.Revision)
|
||||
for _, m := range v.Modules {
|
||||
got += " " + line(m.Module, m.Digest, m.Revision, m.BusState)
|
||||
}
|
||||
} else {
|
||||
a, e := AcceptedOf([]byte(row.body))
|
||||
err = e
|
||||
got = line(a.Module, a.Digest, a.Revision, a.BusState)
|
||||
}
|
||||
if row.refused != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), row.refused) {
|
||||
t.Fatalf("read %s (%v): refused, naming %q", got, err, row.refused)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil || got != row.want {
|
||||
t.Errorf("read %q (%v), want %q", got, err, row.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func line(module, digest string, revision uint64, entries []catalogue.BusStateEntry) string {
|
||||
es := []string{}
|
||||
for _, e := range entries {
|
||||
es = append(es, e.Name+"/"+e.Storage+"/"+e.Template)
|
||||
}
|
||||
return fmt.Sprintf("%s %s %d [%s]", module, digest, revision, strings.Join(es, " "))
|
||||
}
|
||||
|
||||
// The keeper's health tool (CAT-N6): healthy naming its pass; unhealthy, naming why, when its pass is old or verdicts
|
||||
// wait on its consumer `<node>_nats_keeper` past the bound.
|
||||
func TestTheKeepersHealthToolReadsItsStatusAndItsConsumer(t *testing.T) {
|
||||
at := time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC)
|
||||
jsz := `{"account_details":[{"stream_detail":[{"name":"EVENTS","consumer_detail":[{"name":"anchor_nats_keeper","num_pending":3},{"name":"anchor_nats","num_pending":9}]}]}]}`
|
||||
status := keeper.Status{Started: at.Add(-time.Hour), Pass: at.Add(-time.Minute), Heard: at.Add(-time.Minute),
|
||||
FirstView: keeper.FirstViewStatus{Owner: keeper.FirstViewOwner, Verb: keeper.FirstViewVerb, Answered: at.Add(-time.Hour)}}
|
||||
none := func() (map[string]keeper.Asked, error) { return nil, nil }
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
now time.Time
|
||||
state string
|
||||
why string
|
||||
}{
|
||||
{"its pass is fresh", at, "healthy", "last pass 2026-11-04T11:59:00Z"},
|
||||
{"its pass and its verdicts are old", at.Add(5 * time.Minute), "unhealthy", "3 accepted verdict(s) wait on its consumer"},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
a := keeperHealth(func() (keeper.Status, error) { return status, nil }, none,
|
||||
func() ([]byte, error) { return []byte(jsz), nil }, func() time.Time { return row.now })
|
||||
if a.State != row.state || !strings.Contains(a.Why, row.why) {
|
||||
t.Errorf("answered %+v: %s, naming %q", a, row.state, row.why)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
|
||||
"nats-tools/internal/keeper"
|
||||
)
|
||||
|
||||
// The keeper's health, as the node-engine asks it of the process keeper (design 25 § 12.2-12.3; CAT-N6): its status
|
||||
// from its own state asked, what it holds, and how many accepted verdicts wait on its consumer, read from the bus's
|
||||
// monitoring endpoint.
|
||||
|
||||
// StatusInState reads the keeper's status through the runtime.
|
||||
func StatusInState() (keeper.Status, error) {
|
||||
e, err := stdio.State(keeper.StateAsked).Get(keeper.KeyStatus)
|
||||
if err != nil || e == nil {
|
||||
return keeper.Status{}, err
|
||||
}
|
||||
var s keeper.Status
|
||||
return s, json.Unmarshal(e.Value, &s)
|
||||
}
|
||||
|
||||
// keeperPending is how many messages wait on the keeper's consumer on EVENTS (`<node>_nats_keeper`), or -1 unread.
|
||||
func keeperPending(jsz []byte) int {
|
||||
var doc struct {
|
||||
Accounts []struct {
|
||||
Streams []struct {
|
||||
Name string `json:"name"`
|
||||
Consumers []struct {
|
||||
Name string `json:"name"`
|
||||
Pending int `json:"num_pending"`
|
||||
} `json:"consumer_detail"`
|
||||
} `json:"stream_detail"`
|
||||
} `json:"account_details"`
|
||||
}
|
||||
if json.Unmarshal(jsz, &doc) != nil {
|
||||
return -1
|
||||
}
|
||||
for _, a := range doc.Accounts {
|
||||
for _, s := range a.Streams {
|
||||
for _, c := range s.Consumers {
|
||||
if s.Name == "EVENTS" && strings.HasSuffix(c.Name, "_"+keeper.Module+"_keeper") {
|
||||
return c.Pending
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// keeperHealth is the tool's answer.
|
||||
func keeperHealth(status func() (keeper.Status, error), read heldEntries, jsz func() ([]byte, error), now func() time.Time) health.Answer {
|
||||
s, err := status()
|
||||
if err != nil {
|
||||
return health.Unhealthy("The keeper's status cannot be read from its own state: " + err.Error())
|
||||
}
|
||||
held, _ := read()
|
||||
pending := -1
|
||||
if doc, err := jsz(); err == nil {
|
||||
pending = keeperPending(doc)
|
||||
}
|
||||
h := keeper.HealthOf(s, held, pending, now())
|
||||
why := h.Why + "; last pass " + h.Pass
|
||||
for _, l := range []struct {
|
||||
name string
|
||||
lines []string
|
||||
}{{"refused", h.Refused}, {"waiting", h.Waiting}, {"past the raise mark", h.Full}} {
|
||||
if len(l.lines) > 0 {
|
||||
why += "; " + l.name + ": " + strings.Join(l.lines, ", ")
|
||||
}
|
||||
}
|
||||
if !h.Healthy {
|
||||
return health.Unhealthy("The keeper is unhealthy: " + why + ".")
|
||||
}
|
||||
return health.Healthy("The keeper is healthy: " + why + ".")
|
||||
}
|
||||
|
||||
func keeperHealthTool(status func() (keeper.Status, error), read heldEntries, jsz func() ([]byte, error)) stdio.Tool {
|
||||
return stdio.Tool{Name: "nats_keeper_health",
|
||||
Description: "The bus module's keeper's health, as the node-engine asks it: unhealthy, naming why, when its fill pass is more than three minutes old, its first view (the indexer's indexer_bus_state) has not answered, or accepted verdicts wait on its consumer unanswered past three minutes; it names beside the entries it could not create, the ones waiting, and those past the raise mark (another module's full entry raises that module's condition and leaves the keeper healthy).",
|
||||
Run: func(map[string]any) (any, error) {
|
||||
return keeperHealth(status, read, jsz, func() time.Time { return time.Now().UTC() }), nil
|
||||
}}
|
||||
}
|
||||
@@ -0,0 +1,372 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"nats-tools/internal/bussnap"
|
||||
"nats-tools/internal/keeper"
|
||||
)
|
||||
|
||||
// kvStore is the keeper's own state on the real bus: one bucket per state, as the grants part creates its old sections.
|
||||
type kvStore struct {
|
||||
js jetstream.JetStream
|
||||
}
|
||||
|
||||
func (s kvStore) kv(state string) (jetstream.KeyValue, error) {
|
||||
return s.js.KeyValue(context.Background(), "nats_"+state)
|
||||
}
|
||||
|
||||
func (s kvStore) Get(state, key string) ([]byte, uint64, bool, error) {
|
||||
kv, err := s.kv(state)
|
||||
if err != nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
e, err := kv.Get(context.Background(), key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, 0, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
return e.Value(), e.Revision(), true, nil
|
||||
}
|
||||
|
||||
func (s kvStore) Put(state, key string, value []byte) (uint64, error) {
|
||||
kv, err := s.kv(state)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return kv.Put(context.Background(), key, value)
|
||||
}
|
||||
|
||||
func (s kvStore) Delete(state, key string) error {
|
||||
kv, err := s.kv(state)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return kv.Delete(context.Background(), key)
|
||||
}
|
||||
|
||||
func (s kvStore) Keys(state string) ([]string, error) {
|
||||
kv, err := s.kv(state)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keys, err := kv.Keys(context.Background())
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return keys, err
|
||||
}
|
||||
|
||||
func eventually(t *testing.T, what string, ok func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for !ok() {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("not within 10 s: %s", what)
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// The bus module runs whole (design 25 § 12.7, "The bus module runs whole" and "The bus module's first view"; CAT-N10):
|
||||
// its keeper against the real bus server, hearing manifest-accepted on its durable consumer, creating on the bus,
|
||||
// saying its events on its own stream BROKER_EVENTS, keeping its state in its own buckets, and acting on a person's
|
||||
// requests. One server for every case, in order, as the mesh runs them.
|
||||
func TestTheBusModuleRunsWhole(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
s, err := server.NewServer(&server.Options{Port: -1, JetStream: true, StoreDir: dir, NoLog: true, NoSigs: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go s.Start()
|
||||
if !s.ReadyForConnections(5 * time.Second) {
|
||||
t.Fatal("the server did not start")
|
||||
}
|
||||
defer s.Shutdown()
|
||||
nc, err := nats.Connect(s.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer nc.Close()
|
||||
js, _ := jetstream.New(nc)
|
||||
ctx := context.Background()
|
||||
account := process.Account{Node: "anchor", Module: keeper.Module, User: "anchor.nats.keeper"}
|
||||
// What the grants part creates for the bus module from its manifest's old sections: EVENTS and the keeper's
|
||||
// consumer on it, its own stream, and its two states.
|
||||
for _, c := range []jetstream.StreamConfig{
|
||||
{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}},
|
||||
{Name: BrokerEventsStream, Subjects: []string{"mesh.stream.nats.event.>"}, MaxAge: 30 * 24 * time.Hour, MaxBytes: 256 << 20},
|
||||
{Name: "CONDITIONS", Subjects: []string{"mesh.seat.operator-channel.accept.condition.>"}},
|
||||
} {
|
||||
if _, err := js.CreateStream(ctx, c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := js.CreateOrUpdateConsumer(ctx, "EVENTS", jetstream.ConsumerConfig{Durable: account.QueueConsumerName(),
|
||||
FilterSubject: AcceptedSubject, AckPolicy: jetstream.AckExplicitPolicy, MaxDeliver: 5}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, st := range []string{keeper.StateAsked, keeper.StateRequests} {
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "nats_" + st, History: 5}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
store := kvStore{js}
|
||||
snapshots := t.TempDir()
|
||||
var mu sync.Mutex
|
||||
view := keeper.FirstView{Revision: 1}
|
||||
var acts []string
|
||||
var ask *asks.Ask
|
||||
var warrant *asks.Warrant
|
||||
newKeeper := func() *keeper.Keeper {
|
||||
return &keeper.Keeper{Bus: keeper.JetStream{JS: js, Snap: bussnap.Into(nc, snapshots)}, Store: store,
|
||||
Emit: streamEmitter(js, account), Say: sayer(js),
|
||||
View: func() (keeper.FirstView, error) { mu.Lock(); defer mu.Unlock(); return view, nil },
|
||||
Act: func(what, why, cause string) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
acts = append(acts, cause+": "+what)
|
||||
return nil
|
||||
},
|
||||
Warrant: func(string) (*asks.Ask, *asks.Warrant, error) { mu.Lock(); defer mu.Unlock(); return ask, warrant, nil },
|
||||
Logf: t.Logf}
|
||||
}
|
||||
run := func(k *keeper.Keeper) func() {
|
||||
kctx, stop := context.WithCancel(ctx)
|
||||
done := make(chan struct{})
|
||||
watch := func(c context.Context, act func()) error {
|
||||
kv, _ := js.KeyValue(ctx, "nats_"+keeper.StateRequests)
|
||||
w, err := kv.WatchAll(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() {
|
||||
for range w.Updates() {
|
||||
act()
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
go func() { _ = Keep(kctx, k, hearer(js, account, t.Logf), watch, t.Logf); close(done) }()
|
||||
return func() { stop(); <-done }
|
||||
}
|
||||
accepted := func(module string, revision uint64, entries ...catalogue.BusStateEntry) {
|
||||
body, _ := json.Marshal(map[string]any{"v": 1, "module": module, "digest": fmt.Sprintf("sha256:%d", revision),
|
||||
"judge": "j", "revision": revision, "bus-state": entries})
|
||||
if _, err := js.Publish(ctx, AcceptedSubject, body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// saidOn reads what one stream holds of the bus module's events, by its subjects' prefix: its own stream (its
|
||||
// history), or EVENTS on the seat mesh-broker's subjects (what every consumer of mesh-broker.<event> hears).
|
||||
saidOn := func(stream, prefix string) []string {
|
||||
cons, err := js.OrderedConsumer(ctx, stream, jetstream.OrderedConsumerConfig{FilterSubjects: []string{prefix + ">"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, _ := cons.Info(ctx)
|
||||
var out []string
|
||||
if info.NumPending == 0 {
|
||||
return nil
|
||||
}
|
||||
batch, _ := cons.Fetch(int(info.NumPending), jetstream.FetchMaxWait(2*time.Second))
|
||||
for m := range batch.Messages() {
|
||||
var e keeper.Event
|
||||
_ = json.Unmarshal(m.Data(), &e)
|
||||
out = append(out, strings.TrimPrefix(m.Subject(), prefix)+" "+e.Module+" "+e.Local)
|
||||
}
|
||||
return out
|
||||
}
|
||||
said := func() []string { return saidOn(BrokerEventsStream, "mesh.stream.nats.event.") }
|
||||
saidHas := func(want string) func() bool {
|
||||
return func() bool {
|
||||
for _, l := range said() {
|
||||
if l == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
held := func(stream string) (*jetstream.StreamInfo, bool) {
|
||||
st, err := js.Stream(ctx, stream)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
info, err := st.Info(ctx)
|
||||
return info, err == nil
|
||||
}
|
||||
state := entry("drafts", "key-value", "state", nil)
|
||||
log := entry("log", "stream", "log", map[string]string{"max-mib": "1"})
|
||||
|
||||
k := newKeeper()
|
||||
stop := run(k)
|
||||
t.Run("a verdict, then created, with its binding", func(t *testing.T) {
|
||||
accepted("m", 2, state, log)
|
||||
eventually(t, "created m drafts and m log", func() bool { return saidHas("created m drafts")() && saidHas("created m log")() })
|
||||
// Said on the seat's subject as well, on the shared events stream (the lead's ruling, 13:0x).
|
||||
if seat := saidOn("EVENTS", "mesh.seat.mesh-broker.event."); !slices.Contains(seat, "created m drafts") ||
|
||||
!slices.Contains(seat, "created m log") {
|
||||
t.Errorf("EVENTS holds %v of mesh-broker's events: created m drafts and m log, as on the bus module's own stream", seat)
|
||||
}
|
||||
if _, ok := held("KV_m_drafts"); !ok {
|
||||
t.Error("the bus holds no KV_m_drafts")
|
||||
}
|
||||
if info, ok := held("m_log"); !ok || info.Config.Discard != jetstream.DiscardNew || info.Config.Subjects[0] != "mesh.bus.m.log.>" {
|
||||
t.Errorf("m_log is %+v: a log refuses when full, on the module's own subjects", info)
|
||||
}
|
||||
})
|
||||
t.Run("a restart catches up a verdict given while it was down", func(t *testing.T) {
|
||||
stop()
|
||||
accepted("n", 3, entry("jobs", "stream", "work-queue", nil))
|
||||
k = newKeeper()
|
||||
stop = run(k)
|
||||
eventually(t, "created n jobs", saidHas("created n jobs"))
|
||||
})
|
||||
t.Run("a replay older than the first view is not applied; a set-aside module keeps its key", func(t *testing.T) {
|
||||
stop()
|
||||
mu.Lock()
|
||||
view = keeper.FirstView{Revision: 9, Modules: []keeper.Accepted{{Module: "p", Revision: 9, BusState: []catalogue.BusStateEntry{state}}}}
|
||||
mu.Unlock()
|
||||
accepted("p", 4)
|
||||
k = newKeeper()
|
||||
stop = run(k)
|
||||
eventually(t, "created p drafts", saidHas("created p drafts"))
|
||||
eventually(t, "the replay heard", func() bool {
|
||||
info, _ := js.Consumer(ctx, "EVENTS", account.QueueConsumerName())
|
||||
i, _ := info.Info(ctx)
|
||||
return i.NumPending == 0 && i.NumAckPending == 0
|
||||
})
|
||||
if saidHas("orphaned p drafts")() {
|
||||
t.Error("the replay at revision 4 orphaned what the first view at 9 asked")
|
||||
}
|
||||
if _, _, found, _ := store.Get(keeper.StateAsked, "m"); !found {
|
||||
t.Error("m, missing from the first view, lost its key")
|
||||
}
|
||||
})
|
||||
t.Run("drift is said, a growth applied and the drift cleared", func(t *testing.T) {
|
||||
accepted("m", 5, state, entry("log", "stream", "log", map[string]string{"max-mib": "1", "max-age-days": "2"}))
|
||||
eventually(t, "drifted m log", saidHas("drifted m log"))
|
||||
accepted("m", 6, state, entry("log", "stream", "log", map[string]string{"max-mib": "4"}))
|
||||
eventually(t, "drift-cleared m log", saidHas("drift-cleared m log"))
|
||||
if info, _ := held("m_log"); info.Config.MaxBytes != 4<<20 {
|
||||
t.Errorf("m_log's cap is %d: the larger cap applied", info.Config.MaxBytes)
|
||||
}
|
||||
})
|
||||
t.Run("a refused create hands no binding", func(t *testing.T) {
|
||||
bare := entry("raw", "stream", "", nil)
|
||||
accepted("m", 7, state, entry("log", "stream", "log", map[string]string{"max-mib": "4"}), bare)
|
||||
eventually(t, "refused m raw", saidHas("refused m raw"))
|
||||
if saidHas("created m raw")() {
|
||||
t.Error("created m raw")
|
||||
}
|
||||
})
|
||||
t.Run("each template at a small cap refuses, full raised at the raise mark and cleared below the clear mark", func(t *testing.T) {
|
||||
for i := 0; ; i++ {
|
||||
_, err := js.Publish(ctx, fmt.Sprintf("mesh.bus.m.log.k%d", i), make([]byte, 200<<10))
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if i > 40 {
|
||||
t.Fatal("a 4 MiB log took more than 8 MiB")
|
||||
}
|
||||
}
|
||||
k.FillPass()
|
||||
eventually(t, "full m log", saidHas("full m log"))
|
||||
st, _ := js.Stream(ctx, "m_log")
|
||||
_ = st.Purge(ctx)
|
||||
k.FillPass()
|
||||
eventually(t, "full-cleared m log", saidHas("full-cleared m log"))
|
||||
})
|
||||
t.Run("an orphan is listed, and goes only by drop on the operator's warrant, after a snapshot", func(t *testing.T) {
|
||||
accepted("m", 8, entry("log", "stream", "log", map[string]string{"max-mib": "4"}))
|
||||
eventually(t, "orphaned m drafts", saidHas("orphaned m drafts"))
|
||||
put := func(r keeper.Request) {
|
||||
raw, _ := json.Marshal(r)
|
||||
if _, err := store.Put(keeper.StateRequests, r.ID, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
get := func(id string) keeper.Request {
|
||||
raw, _, _, _ := store.Get(keeper.StateRequests, id)
|
||||
var r keeper.Request
|
||||
_ = json.Unmarshal(raw, &r)
|
||||
return r
|
||||
}
|
||||
put(keeper.Request{ID: "r1", Verb: keeper.VerbDrop, Module: "m", Local: "drafts", Caller: "person.jo", Why: "unused"})
|
||||
eventually(t, "the drop with no ask refused", func() bool { return strings.Contains(get("r1").Error, "operator's warrant") })
|
||||
r := keeper.Request{ID: "r2", Verb: keeper.VerbDrop, Module: "m", Local: "drafts", Caller: "person.jo", Why: "unused"}
|
||||
a, _ := askFor(r, time.Now())
|
||||
r.Ask = a.ID
|
||||
mu.Lock()
|
||||
ask = &a
|
||||
mu.Unlock()
|
||||
put(r)
|
||||
k.ActOnRequests()
|
||||
if _, ok := held("KV_m_drafts"); !ok || !get("r2").Done.IsZero() {
|
||||
t.Fatal("dropped before the operator chose")
|
||||
}
|
||||
mu.Lock()
|
||||
warrant = &asks.Warrant{Ask: a.ID, Asker: keeper.Module, Outcome: asks.OutcomeChosen, By: &asks.Person{Who: asks.Operator},
|
||||
Option: "go", Label: "Remove", Level: asks.Destroy, At: time.Now(), AskDigest: a.Digest()}
|
||||
mu.Unlock()
|
||||
k.ActOnRequests()
|
||||
eventually(t, "dropped m drafts", saidHas("dropped m drafts"))
|
||||
if _, ok := held("KV_m_drafts"); ok {
|
||||
t.Error("KV_m_drafts is still on the bus")
|
||||
}
|
||||
if p := get("r2").Snapshot; p == "" {
|
||||
t.Error("no snapshot taken before the drop")
|
||||
} else if st, err := os.Stat(p); err != nil || st.Size() == 0 {
|
||||
t.Errorf("the snapshot %s is not there whole: %v", p, err)
|
||||
}
|
||||
mu.Lock()
|
||||
recorded := strings.Join(acts, "\n")
|
||||
mu.Unlock()
|
||||
if !strings.Contains(recorded, "nats_drop: dropped drafts of m") {
|
||||
t.Errorf("recorded %s: the drop as a hand act", recorded)
|
||||
}
|
||||
})
|
||||
t.Run("nats_first_view_again asks at once and records the hand act", func(t *testing.T) {
|
||||
raw, _ := json.Marshal(keeper.Request{ID: "r3", Verb: keeper.VerbFirstViewAgain, Why: "a verdict thought missed", Caller: "person.jo"})
|
||||
_, _ = store.Put(keeper.StateRequests, "r3", raw)
|
||||
eventually(t, "the first view asked again", func() bool {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return strings.Contains(strings.Join(acts, "\n"), "nats_first_view_again")
|
||||
})
|
||||
})
|
||||
stop()
|
||||
}
|
||||
|
||||
func entry(name, storage, template string, options map[string]string) catalogue.BusStateEntry {
|
||||
e := catalogue.BusStateEntry{Name: name, Storage: storage, Template: template}
|
||||
if len(options) > 0 {
|
||||
e.Options = map[string]json.RawMessage{}
|
||||
for k, v := range options {
|
||||
e.Options[k] = json.RawMessage(v)
|
||||
}
|
||||
}
|
||||
return e
|
||||
}
|
||||
@@ -152,8 +152,37 @@ func tools(monitor Monitor, users func(context.Context) ([]byte, error), written
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := stdio.Serve("", append(append(tools(LiveMonitor(), ReadUsers, UsersWritten), removalTools(LiveBus(), SaidByTheRuntime)...), expiryTools(LiveBus(), SaidByTheRuntime, RecordedBySeat)...)); err != nil {
|
||||
if len(os.Args) > 1 && os.Args[1] == "keep" {
|
||||
keepMain()
|
||||
return
|
||||
}
|
||||
if err := stdio.Serve("", allTools()); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// The seat mesh-broker's reading verbs, which this bundle serves (design 25 § 12.3): every entry, one entry, what the
|
||||
// bus holds against what was asked, and what no manifest asks for. Its acting verbs (create, apply, recreate, drop) are
|
||||
// the keeper's own work, on its own account alone (keeper.go), so no other account, the tool runner's included, holds
|
||||
// them.
|
||||
var ReadingVerbs = []string{"list", "show", "check", "orphans"}
|
||||
|
||||
// Events are the seat mesh-broker's events (ADR 0595 §6, ADR 0620 §2): said on the seat's subjects on the shared events stream
|
||||
// and kept 30 days on the bus module's own stream broker-events (BROKER_EVENTS), which `history` reads (keep.go).
|
||||
var Events = []string{"created", "refused", "drifted", "drift-cleared", "full", "full-cleared", "orphaned", "recreated", "dropped"}
|
||||
|
||||
// allTools is every tool and seat verb this bundle serves.
|
||||
func allTools() []stdio.Tool {
|
||||
monitor := LiveMonitor()
|
||||
jsz := func() ([]byte, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
return monitor(ctx, "/jsz?accounts=1&streams=1&config=1")
|
||||
}
|
||||
out := append(tools(monitor, ReadUsers, UsersWritten), removalTools(LiveBus(), SaidByTheRuntime, HeldInState)...)
|
||||
out = append(out, expiryTools(LiveBus(), SaidByTheRuntime, RecordedBySeat)...)
|
||||
out = append(out, brokerTools(HeldInState, jsz, StreamHistory(LiveBus()))...)
|
||||
out = append(out, keeperHealthTool(StatusInState, HeldInState, jsz))
|
||||
return append(out, requestTools(RequestsInState())...)
|
||||
}
|
||||
|
||||
@@ -2,6 +2,8 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
@@ -150,6 +152,100 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) {
|
||||
t.Errorf("the tools' credential is %q: the process nats-tools' own secret broker-nats-tools, under a word of the bundle's own (the runtime strips MESH_BROKER_FILE)", a.Env["NATS_TOOLS_CREDENTIAL"])
|
||||
}
|
||||
}
|
||||
// The bus module provides bus state (design 25 § 12.2-12.3, ADR 0595 §4-§6; CAT-N1): its keeper is a process of
|
||||
// its own on its own account (own secret broker-keeper, ADR 0436 §1), creating only bus state; it hears the indexer's
|
||||
// manifest-accepted, keeps what it heard in its own old state `asked` (ADR 0595 §10) and the persons' requests in
|
||||
// `requests`; its nine events are its own (ADR 0620 §2), on its own stream kept 30 days at 256 MiB; the seat's four
|
||||
// reading verbs are served, its four acting verbs are the keeper's own work and no other account's; its health is
|
||||
// the keeper's own tool, beside the server's tcp check.
|
||||
var nm struct {
|
||||
Claims []struct {
|
||||
Name string `json:"name"`
|
||||
Serves []string `json:"serves"`
|
||||
} `json:"claims"`
|
||||
Consumes []string `json:"consumes"`
|
||||
Invokes []string `json:"invokes"`
|
||||
Uses []string `json:"uses"`
|
||||
OwnSecrets map[string]json.RawMessage `json:"own-secrets"`
|
||||
Streams []struct {
|
||||
Name string `json:"name"`
|
||||
Events []string `json:"events"`
|
||||
MaxAgeDays int `json:"max-age-days"`
|
||||
MaxMiB int `json:"max-mib"`
|
||||
} `json:"streams"`
|
||||
State []struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"state"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &nm); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(nm.Claims) != 1 || nm.Claims[0].Name != "mesh-broker" || !slices.Equal(nm.Claims[0].Serves, ReadingVerbs) {
|
||||
t.Errorf("claims are %+v: mesh-broker, serving its reading verbs %v alone", nm.Claims, ReadingVerbs)
|
||||
}
|
||||
if !slices.Equal(nm.Consumes, []string{"mesh-delivery-indexer.manifest-accepted"}) {
|
||||
t.Errorf("consumes %v: the indexer's manifest-accepted alone (ADR 0595 §4)", nm.Consumes)
|
||||
}
|
||||
for _, want := range []string{"mesh-delivery-indexer.indexer_bus_state", "seat:mesh-hand-acts.hand-act"} {
|
||||
if !slices.Contains(nm.Invokes, want) {
|
||||
t.Errorf("invokes %v lack %s", nm.Invokes, want)
|
||||
}
|
||||
}
|
||||
if !slices.Contains(nm.Uses, "operator-channel") {
|
||||
t.Errorf("uses %v: the operator channel, for the warrant of recreate and drop", nm.Uses)
|
||||
}
|
||||
if len(nm.Streams) != 1 || nm.Streams[0].Name != "broker-events" || !slices.Equal(nm.Streams[0].Events, Events) ||
|
||||
nm.Streams[0].MaxAgeDays != 30 || nm.Streams[0].MaxMiB != 256 {
|
||||
t.Errorf("streams are %+v: broker-events carrying %v, 30 days, 256 MiB", nm.Streams, Events)
|
||||
}
|
||||
for _, e := range Events {
|
||||
if slices.Contains(m.Emits, e) {
|
||||
t.Errorf("emits lists %s, which its stream carries: an event is named in one place (ADR 0363)", e)
|
||||
}
|
||||
}
|
||||
var states []string
|
||||
for _, s := range nm.State {
|
||||
states = append(states, s.Name)
|
||||
}
|
||||
if !slices.Equal(states, []string{"asked", "requests"}) {
|
||||
t.Errorf("state is %v: asked and requests, old sections (ADR 0595 §10)", states)
|
||||
}
|
||||
if _, ok := nm.OwnSecrets["broker-keeper"]; !ok {
|
||||
t.Error("own-secrets lack broker-keeper, the keeper's own account (ADR 0436 §1)")
|
||||
}
|
||||
keeper := byID["keeper"]
|
||||
bus, _ := keeper["bus"].(map[string]any)
|
||||
health, _ := keeper["health"].(map[string]any)
|
||||
if keeper["type"] != "process" || keeper["artifact"] != "tools" || fmt.Sprint(keeper["run"]) != "[./nats-tools keep]" ||
|
||||
fmt.Sprint(bus["state"]) != "[asked requests]" ||
|
||||
health["kind"] != "tool" || health["tool"] != "nats_keeper_health" {
|
||||
t.Errorf("the keeper is %v: a process of the tools bundle, run as keep, keeping asked and requests, judged by nats_keeper_health", keeper)
|
||||
}
|
||||
// Its bus declares keeps, the one key more ADR 0626 §1 allows it, read through the SDK as the grant composition reads it.
|
||||
var whole catalogue.Manifest
|
||||
if err := json.Unmarshal(raw, &whole); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b := whole.ProcessBuses()["keeper"]; !b.Keeps {
|
||||
t.Errorf("the keeper's bus declares no keeps, so it would be composed no bus state to keep (ADR 0626 §1): %+v", b)
|
||||
}
|
||||
if h, _ := server["health"].(map[string]any); h["kind"] != "tcp" || h["endpoint"] != "bus" {
|
||||
t.Errorf("the server's health is %v: tcp on bus, the check the keeper's tool check stands beside", server["health"])
|
||||
}
|
||||
served := map[string]bool{}
|
||||
for _, tl := range allTools() {
|
||||
served[tl.Name] = true
|
||||
}
|
||||
for _, name := range m.Tools {
|
||||
if !served[name] {
|
||||
t.Errorf("the manifest lists the tool %s, which the bundle does not serve", name)
|
||||
}
|
||||
}
|
||||
for _, verb := range ReadingVerbs {
|
||||
if !served["mesh-broker."+verb] {
|
||||
t.Errorf("the bundle does not serve the seat verb mesh-broker.%s", verb)
|
||||
}
|
||||
}
|
||||
args, _ := server["args"].([]any)
|
||||
if len(args) != 2 || args[0] != "sh" || args[1] != "/opt/mesh-entrypoint/entrypoint.sh" {
|
||||
t.Errorf("args are %v: the entrypoint run through sh, so no execute bit is relied on", args)
|
||||
|
||||
@@ -333,9 +333,23 @@ func RemoveStream(ctx context.Context, bus Bus, said Said, a map[string]any) (an
|
||||
}
|
||||
|
||||
// removalTools are the module's three tools that change the bus; every other tool here reads.
|
||||
func removalTools(bus Bus, said Said) []stdio.Tool {
|
||||
// removalTools are the three removal tools. **They refuse an entry of the new form** (design 25 § 12.3; ADR 0595 §4 as
|
||||
// its insight of 2026-11-03 has it): the mesh deletes a module's kept data only through the bus module's drop and
|
||||
// recreate, so a stream, a bucket or a consumer of a stream the keeper made (its state asked, read through newForm) is
|
||||
// refused, naming nats_drop, before the bus is reached; the old sections' entries are removed as before. A newForm
|
||||
// that cannot be read refuses too: what cannot be told apart is not removed.
|
||||
func removalTools(bus Bus, said Said, newForm heldEntries) []stdio.Tool {
|
||||
run := func(f func(context.Context, Bus, Said, map[string]any) (any, error)) func(map[string]any) (any, error) {
|
||||
return func(a map[string]any) (any, error) { return f(context.Background(), bus, said, a) }
|
||||
return func(a map[string]any) (any, error) {
|
||||
stream := arg(a, "stream")
|
||||
if b := arg(a, "bucket"); b != "" {
|
||||
stream = "KV_" + b
|
||||
}
|
||||
if err := refuseNewForm(newForm, stream); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f(context.Background(), bus, said, a)
|
||||
}
|
||||
}
|
||||
return []stdio.Tool{
|
||||
{Name: "nats_consumer_remove",
|
||||
@@ -353,3 +367,24 @@ func removalTools(bus Bus, said Said) []stdio.Tool {
|
||||
Run: run(RemoveKeys)},
|
||||
}
|
||||
}
|
||||
|
||||
// refuseNewForm refuses a stream the keeper made for a module's bus state, naming the tool that removes it.
|
||||
func refuseNewForm(newForm heldEntries, stream string) error {
|
||||
if newForm == nil || stream == "" {
|
||||
return nil
|
||||
}
|
||||
held, err := newForm()
|
||||
if err != nil {
|
||||
return fmt.Errorf("whether %s is a module's kept data of the new form cannot be read (%v), so nothing was removed", stream, err)
|
||||
}
|
||||
for module, a := range held {
|
||||
for local, m := range a.Made {
|
||||
for _, o := range m.Plan.Objects {
|
||||
if o.Stream() == stream {
|
||||
return fmt.Errorf("%s is %s's kept data %q, which the bus module made: it is removed only by nats_drop (an orphan, on the operator's warrant) or rebuilt by nats_recreate, never by this tool; nothing was removed", stream, module, local)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
|
||||
"nats-tools/internal/keeper"
|
||||
)
|
||||
|
||||
// A person's acting tools (design 25 § 12.3; CAT-N8): nats_create and nats_apply (acts), nats_recreate and nats_drop
|
||||
// (dangerous, granted only by name). None holds a grant of the seat's acting verbs: each hands its request to the
|
||||
// keeper through the keeper's own state requests, stamped with the caller the tool runner proved, and reads its end
|
||||
// back. recreate and drop first raise the operator's ask, binding exactly that act; the keeper acts only on its warrant,
|
||||
// after a snapshot.
|
||||
|
||||
// requestStore is the keeper's state requests, as this bundle reaches it.
|
||||
type requestStore struct {
|
||||
Put func(key string, r keeper.Request) error
|
||||
Get func(key string) (*keeper.Request, error)
|
||||
Ask func(a asks.Ask) error
|
||||
Now func() time.Time
|
||||
Wait func(d time.Duration)
|
||||
}
|
||||
|
||||
// RequestsInState is the store through the runtime that launched this bundle.
|
||||
func RequestsInState() requestStore {
|
||||
s := stdio.State(keeper.StateRequests)
|
||||
return requestStore{
|
||||
Put: func(key string, r keeper.Request) error { _, err := s.Put(key, r); return err },
|
||||
Get: func(key string) (*keeper.Request, error) {
|
||||
e, err := s.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, err
|
||||
}
|
||||
var r keeper.Request
|
||||
return &r, json.Unmarshal(e.Value, &r)
|
||||
},
|
||||
Ask: func(a asks.Ask) error { return stdio.SeatPublish(asks.AskSubject(keeper.Module), a, a.ID) },
|
||||
Now: func() time.Time { return time.Now().UTC() },
|
||||
Wait: time.Sleep,
|
||||
}
|
||||
}
|
||||
|
||||
func requestID(now time.Time) string {
|
||||
var b [4]byte
|
||||
_, _ = rand.Read(b[:])
|
||||
return now.Format("20060102t150405") + "-" + hex.EncodeToString(b[:])
|
||||
}
|
||||
|
||||
// askFor is the operator's ask for a recreate or a drop of one entry, its approve option binding exactly that act.
|
||||
func askFor(r keeper.Request, now time.Time) (asks.Ask, error) {
|
||||
binds, err := asks.ActDigest(r.Act())
|
||||
if err != nil {
|
||||
return asks.Ask{}, err
|
||||
}
|
||||
what := map[string]string{keeper.VerbDrop: "remove", keeper.VerbRecreate: "rebuild"}[r.Verb]
|
||||
name := keeper.BusName(r.Module, r.Local)
|
||||
return asks.Ask{ID: "bus-" + r.ID, Who: asks.Operator, Expires: now.Add(24 * time.Hour),
|
||||
OnExpiry: "nothing is removed and the entry stays as it is",
|
||||
About: keeper.ConditionKey(map[string]string{keeper.VerbDrop: "orphaned", keeper.VerbRecreate: "drifted"}[r.Verb], r.Module, r.Local),
|
||||
Headline: fmt.Sprintf("%s's kept data %s: %s it?", r.Module, r.Local, what),
|
||||
Explanation: fmt.Sprintf("%s asks to %s %s on the bus, %s. A snapshot of it is taken first; its data is lost from the bus.",
|
||||
r.Caller, what, name, r.Why),
|
||||
Options: []asks.Option{
|
||||
{ID: "go", Label: strings.ToUpper(what[:1]) + what[1:], Does: fmt.Sprintf("%s %s after a snapshot", what, name), Level: asks.Destroy, Binds: binds},
|
||||
{ID: "keep", Label: "Keep it", Does: "nothing is removed", Level: asks.Acknowledge},
|
||||
}}, nil
|
||||
}
|
||||
|
||||
func requestTools(rs requestStore) []stdio.Tool {
|
||||
entry := map[string]any{"module": str("the module, e.g. mesh-knowledge"), "local": str("its local name, e.g. records"),
|
||||
"why": str("why, in plain words: kept with the act"), "request": str("a request this tool made before: its end is read, nothing new is asked")}
|
||||
tool := func(verb, description string) stdio.Tool {
|
||||
return stdio.Tool{Name: "nats_" + verb, Description: description, Input: entry, Run: func(a map[string]any) (any, error) {
|
||||
if id := arg(a, "request"); id != "" {
|
||||
r, err := rs.Get(id)
|
||||
if err != nil || r == nil {
|
||||
return nil, fmt.Errorf("no request %s is held (%v)", id, err)
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
r := keeper.Request{Verb: verb, Module: arg(a, "module"), Local: arg(a, "local"), Why: strings.TrimSpace(arg(a, "why")),
|
||||
// The caller the tool runner proved, never a name the arguments give.
|
||||
Caller: stdio.CallerOf(a), At: rs.Now()}
|
||||
if r.Module == "" || r.Local == "" || r.Why == "" {
|
||||
return nil, fmt.Errorf("nats_%s names one entry, a module and its local name, and says why", verb)
|
||||
}
|
||||
r.ID = requestID(r.At)
|
||||
if verb == keeper.VerbRecreate || verb == keeper.VerbDrop {
|
||||
ask, err := askFor(r, r.At)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.Ask = ask.ID
|
||||
if err := rs.Put(r.ID, r); err != nil {
|
||||
return nil, fmt.Errorf("the request cannot be handed to the keeper: %w", err)
|
||||
}
|
||||
if err := rs.Ask(ask); err != nil {
|
||||
return nil, fmt.Errorf("the operator's ask cannot be raised, so nothing will be removed: %w", err)
|
||||
}
|
||||
return map[string]any{"request": r.ID, "ask": ask.ID, "state": "the operator is asked; the keeper acts once they approve, after a snapshot",
|
||||
"read": fmt.Sprintf("nats_%s with request %s", verb, r.ID)}, nil
|
||||
}
|
||||
if err := rs.Put(r.ID, r); err != nil {
|
||||
return nil, fmt.Errorf("the request cannot be handed to the keeper: %w", err)
|
||||
}
|
||||
return awaitEnd(rs, r, "nats_"+verb), nil
|
||||
}}
|
||||
}
|
||||
firstView := stdio.Tool{Name: "nats_first_view_again",
|
||||
Description: "Ask the bus module's keeper to ask the indexer's indexer_bus_state, its first view of every accepted manifest's bus state, at once (design 25 § 12.2): by hand, beside its own backoff, recorded as a hand act; answers what the first view answered and its revision.",
|
||||
Input: map[string]any{"why": str("why, in plain words: kept with the act"), "request": str("a request made before: its end is read")},
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
if id := arg(a, "request"); id != "" {
|
||||
r, err := rs.Get(id)
|
||||
if err != nil || r == nil {
|
||||
return nil, fmt.Errorf("no request %s is held (%v)", id, err)
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
r := keeper.Request{Verb: keeper.VerbFirstViewAgain, Why: strings.TrimSpace(arg(a, "why")), Caller: stdio.CallerOf(a), At: rs.Now()}
|
||||
if r.Why == "" {
|
||||
return nil, fmt.Errorf("nats_first_view_again says why")
|
||||
}
|
||||
r.ID = requestID(r.At)
|
||||
if err := rs.Put(r.ID, r); err != nil {
|
||||
return nil, fmt.Errorf("the request cannot be handed to the keeper: %w", err)
|
||||
}
|
||||
return awaitEnd(rs, r, "nats_first_view_again"), nil
|
||||
}}
|
||||
return []stdio.Tool{
|
||||
firstView,
|
||||
tool(keeper.VerbCreate, "Create one entry of a module's kept data on the bus now, as its accepted manifest asks for it (design 25 § 12.3): handed to the bus module's keeper, which alone may create, and recorded as a hand act."),
|
||||
tool(keeper.VerbApply, "Apply what a module's manifest asks of one entry where the bus holds less (a larger cap, a longer age): only a difference that grows; handed to the keeper and recorded as a hand act."),
|
||||
tool(keeper.VerbRecreate, "Rebuild one entry whose settings drifted in a way that does not grow: the operator is asked first, and the keeper acts only on their warrant, after a snapshot of the entry. Its data is lost from the bus (the snapshot keeps it). Dangerous."),
|
||||
tool(keeper.VerbDrop, "Remove one orphan, an entry no accepted manifest asks for: the operator is asked first, and the keeper acts only on their warrant, after a snapshot of the entry. Dangerous."),
|
||||
}
|
||||
}
|
||||
|
||||
// awaitEnd reads a request's end back: the keeper acts on its watch of requests at once, so the end is read for at most
|
||||
// half a minute, and otherwise how to read it later is answered.
|
||||
func awaitEnd(rs requestStore, r keeper.Request, tool string) any {
|
||||
for wait := 100 * time.Millisecond; wait < 20*time.Second; wait *= 2 {
|
||||
rs.Wait(wait)
|
||||
if got, err := rs.Get(r.ID); err == nil && got != nil && !got.Done.IsZero() {
|
||||
return got
|
||||
}
|
||||
}
|
||||
return map[string]any{"request": r.ID, "state": "handed to the keeper, not ended yet", "read": fmt.Sprintf("%s with request %s", tool, r.ID)}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
|
||||
"nats-tools/internal/keeper"
|
||||
)
|
||||
|
||||
// A person's acting tools hand their request to the keeper (design 25 § 12.3; CAT-N8): stamped with the caller the
|
||||
// tool runner proved, whatever name the arguments give; a create's end read back; a drop raises the operator's ask
|
||||
// first, binding exactly that act, and is not acted on by the tool.
|
||||
func TestAPersonsRequestIsHandedToTheKeeperAsTheProvedCaller(t *testing.T) {
|
||||
held := map[string]keeper.Request{}
|
||||
var raised []asks.Ask
|
||||
rs := requestStore{
|
||||
Put: func(key string, r keeper.Request) error {
|
||||
if r.Verb == keeper.VerbCreate {
|
||||
// The keeper's end, as its watch writes it.
|
||||
r.Done, r.Outcome = time.Unix(1, 0), "created"
|
||||
}
|
||||
held[key] = r
|
||||
return nil
|
||||
},
|
||||
Get: func(key string) (*keeper.Request, error) {
|
||||
r, ok := held[key]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
return &r, nil
|
||||
},
|
||||
Ask: func(a asks.Ask) error { raised = append(raised, a); return nil },
|
||||
Now: func() time.Time { return time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC) },
|
||||
Wait: func(time.Duration) {},
|
||||
}
|
||||
tools := map[string]stdio.Tool{}
|
||||
for _, tl := range requestTools(rs) {
|
||||
tools[tl.Name] = tl
|
||||
}
|
||||
out, err := tools["nats_create"].Run(map[string]any{"module": "m", "local": "log", "why": "a test", "caller": "person.mallory"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r := out.(*keeper.Request); r.Caller != stdio.Unproven || r.Outcome != "created" {
|
||||
t.Errorf("answered %+v: acted for the caller the runtime proved (here none, %q), never person.mallory", r, stdio.Unproven)
|
||||
}
|
||||
out, err = tools["nats_drop"].Run(map[string]any{"module": "m", "local": "old", "why": "no manifest asks for it"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
answer := out.(map[string]any)
|
||||
r := held[answer["request"].(string)]
|
||||
if len(raised) != 1 || r.Ask != raised[0].ID || !r.Done.IsZero() {
|
||||
t.Fatalf("raised %+v, held %+v: the ask raised, the request waiting for it", raised, r)
|
||||
}
|
||||
binds, _ := asks.ActDigest(asks.Act{"verb": "drop", "module": "m", "local": "old"})
|
||||
if o, _ := raised[0].Option("go"); o.Level != asks.Destroy || o.Binds != binds || !strings.Contains(raised[0].Explanation, "snapshot") {
|
||||
t.Errorf("the ask is %+v: its option destroys exactly this drop, after a snapshot", raised[0])
|
||||
}
|
||||
if _, err := tools["nats_drop"].Run(map[string]any{"module": "m", "local": "old"}); err == nil || !strings.Contains(err.Error(), "says why") {
|
||||
t.Errorf("a drop without why: %v", err)
|
||||
}
|
||||
if got, _ := tools["nats_drop"].Run(map[string]any{"request": r.ID}); got.(*keeper.Request).ID != r.ID {
|
||||
t.Errorf("reading the request back answered %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,9 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"nats-tools/internal/keeper"
|
||||
)
|
||||
|
||||
const connz = `{"connections":[
|
||||
@@ -220,6 +223,17 @@ func TestTheRemovalToolsRefuseBeforeTheyReachTheBus(t *testing.T) {
|
||||
return nil, nil, nil
|
||||
}
|
||||
said := func(string, any) error { t.Fatal("an audit event was said for a refused call"); return nil }
|
||||
newForm := func() (map[string]keeper.Asked, error) {
|
||||
made := map[string]keeper.Made{}
|
||||
for _, e := range []catalogue.BusStateEntry{{Name: "log", Storage: "stream", Template: "log"}, {Name: "drafts", Storage: "key-value", Template: "state"}} {
|
||||
p, err := keeper.PlanOf("m", e)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
made[e.Name] = keeper.Made{Plan: p, Created: true}
|
||||
}
|
||||
return map[string]keeper.Asked{"m": {Module: "m", Made: made}}, nil
|
||||
}
|
||||
for _, c := range []struct {
|
||||
tool string
|
||||
args map[string]any
|
||||
@@ -238,9 +252,13 @@ func TestTheRemovalToolsRefuseBeforeTheyReachTheBus(t *testing.T) {
|
||||
{"nats_stream_remove", map[string]any{"stream": "ASSIGNMENTS", "why": " "}, "why is required"},
|
||||
{"nats_stream_remove", map[string]any{"stream": "A.>", "why": "x"}, "stream is required"},
|
||||
{"nats_stream_remove", map[string]any{"stream": "", "why": "x"}, "stream is required"},
|
||||
// CAT-N9: an entry of the new form goes only by drop or recreate; an old section's goes as before.
|
||||
{"nats_stream_remove", map[string]any{"stream": "m_log", "why": "x"}, "removed only by nats_drop"},
|
||||
{"nats_bucket_remove", map[string]any{"bucket": "m_drafts", "node": "g14", "why": "x"}, "m's kept data \"drafts\""},
|
||||
{"nats_consumer_remove", map[string]any{"stream": "m_log", "consumer": "c", "why": "x"}, "removed only by nats_drop"},
|
||||
} {
|
||||
var run func(map[string]any) (any, error)
|
||||
for _, tl := range removalTools(bus, said) {
|
||||
for _, tl := range removalTools(bus, said, newForm) {
|
||||
if tl.Name == c.tool {
|
||||
run = tl.Run
|
||||
}
|
||||
|
||||
+81
-21
@@ -1,4 +1,7 @@
|
||||
package main
|
||||
// Package bussnap takes a snapshot of the bus's streams with the server's own snapshot API (novox/hq ADR 0235): the
|
||||
// nightly dump of every stream, and the bus module's keeper's snapshot of one entry before it is rebuilt or removed
|
||||
// (design 25 § 12.3).
|
||||
package bussnap
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
@@ -10,22 +13,26 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"nats-tools/internal/busarchive"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// The JetStream API's subjects this program speaks. Each is granted to the bus's own module and
|
||||
// nothing else is (mesh-controller internal/broker, BusSnapshotGrants); a test holds the two lists
|
||||
// equal against a real server.
|
||||
const (
|
||||
apiStreamNames = "$JS.API.STREAM.NAMES"
|
||||
apiStreamInfo = "$JS.API.STREAM.INFO."
|
||||
apiStreamSnapshot = "$JS.API.STREAM.SNAPSHOT."
|
||||
apiStreamRestore = "$JS.API.STREAM.RESTORE."
|
||||
APIStreamNames = "$JS.API.STREAM.NAMES"
|
||||
APIStreamInfo = "$JS.API.STREAM.INFO."
|
||||
APIStreamSnapshot = "$JS.API.STREAM.SNAPSHOT."
|
||||
APIStreamRestore = "$JS.API.STREAM.RESTORE."
|
||||
)
|
||||
|
||||
// ChunkSize is what the server is asked to send at a time: its own default. The server sends up to
|
||||
@@ -42,20 +49,22 @@ type SnapshotOptions struct {
|
||||
// TempDir is where one stream's archive waits until it is written out; the system's when empty.
|
||||
TempDir string
|
||||
Log func(format string, args ...any)
|
||||
// Streams, when set, are the only streams taken (the keeper's snapshot of one entry); every stream when empty.
|
||||
Streams []string
|
||||
}
|
||||
|
||||
// apiError is the error every JetStream API answer may carry.
|
||||
type apiError struct {
|
||||
// APIError is the error every JetStream API answer may carry.
|
||||
type APIError struct {
|
||||
Code int `json:"code"`
|
||||
ErrCode int `json:"err_code"`
|
||||
Description string `json:"description"`
|
||||
}
|
||||
|
||||
func (e *apiError) Error() string { return fmt.Sprintf("%s (%d)", e.Description, e.Code) }
|
||||
func (e *APIError) Error() string { return fmt.Sprintf("%s (%d)", e.Description, e.Code) }
|
||||
|
||||
// streamState is what this program reads of a stream's state; the whole of it is kept as the server
|
||||
// StreamState is what this program reads of a stream's state; the whole of it is kept as the server
|
||||
// gave it, in the stream's meta.
|
||||
type streamState struct {
|
||||
type StreamState struct {
|
||||
Messages uint64 `json:"messages"`
|
||||
Bytes uint64 `json:"bytes"`
|
||||
FirstSeq uint64 `json:"first_seq"`
|
||||
@@ -71,8 +80,8 @@ type streamConfig struct {
|
||||
Storage string `json:"storage"`
|
||||
}
|
||||
|
||||
// request asks the JetStream API and decodes its answer, refusing an answer that carries an error.
|
||||
func request(ctx context.Context, nc *nats.Conn, subject string, body any, into any) error {
|
||||
// Request asks the JetStream API and decodes its answer, refusing an answer that carries an error.
|
||||
func Request(ctx context.Context, nc *nats.Conn, subject string, body any, into any) error {
|
||||
var payload []byte
|
||||
if body != nil {
|
||||
var err error
|
||||
@@ -90,10 +99,10 @@ func request(ctx context.Context, nc *nats.Conn, subject string, body any, into
|
||||
return fmt.Errorf("%s: %w", subject, err)
|
||||
}
|
||||
var wrapped struct {
|
||||
Error *apiError `json:"error"`
|
||||
Error *APIError `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(msg.Data, &wrapped); err != nil {
|
||||
return fmt.Errorf("%s answered something that is not JSON: %q", subject, truncate(msg.Data))
|
||||
return fmt.Errorf("%s answered something that is not JSON: %q", subject, Truncate(msg.Data))
|
||||
}
|
||||
if wrapped.Error != nil {
|
||||
return fmt.Errorf("%s: %w", subject, wrapped.Error)
|
||||
@@ -104,7 +113,7 @@ func request(ctx context.Context, nc *nats.Conn, subject string, body any, into
|
||||
return nil
|
||||
}
|
||||
|
||||
func truncate(b []byte) string {
|
||||
func Truncate(b []byte) string {
|
||||
if len(b) > 200 {
|
||||
return string(b[:200]) + "…"
|
||||
}
|
||||
@@ -121,7 +130,7 @@ func StreamNames(ctx context.Context, nc *nats.Conn) ([]string, error) {
|
||||
Limit int `json:"limit"`
|
||||
Streams []string `json:"streams"`
|
||||
}
|
||||
if err := request(ctx, nc, apiStreamNames, map[string]int{"offset": len(names)}, &page); err != nil {
|
||||
if err := Request(ctx, nc, APIStreamNames, map[string]int{"offset": len(names)}, &page); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names = append(names, page.Streams...)
|
||||
@@ -148,6 +157,18 @@ func Snapshot(ctx context.Context, nc *nats.Conn, out io.Writer, o SnapshotOptio
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(o.Streams) > 0 {
|
||||
var only []string
|
||||
for _, n := range names {
|
||||
if slices.Contains(o.Streams, n) {
|
||||
only = append(only, n)
|
||||
}
|
||||
}
|
||||
if len(only) != len(o.Streams) {
|
||||
return nil, fmt.Errorf("the bus holds %v of the streams %v asked for: a snapshot missing one would pass for a whole one", only, o.Streams)
|
||||
}
|
||||
names = only
|
||||
}
|
||||
if len(names) == 0 {
|
||||
// The mesh's own streams always exist once the controller has run; a bus with none is not
|
||||
// one to call backed up.
|
||||
@@ -178,7 +199,7 @@ func Snapshot(ctx context.Context, nc *nats.Conn, out io.Writer, o SnapshotOptio
|
||||
var info struct {
|
||||
Config streamConfig `json:"config"`
|
||||
}
|
||||
if err := request(ctx, nc, apiStreamInfo+name, nil, &info); err != nil {
|
||||
if err := Request(ctx, nc, APIStreamInfo+name, nil, &info); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if info.Config.Storage == "memory" {
|
||||
@@ -202,7 +223,7 @@ func Snapshot(ctx context.Context, nc *nats.Conn, out io.Writer, o SnapshotOptio
|
||||
m.StreamBytes += int64(entry.Bytes)
|
||||
m.ArchiveBytes += entry.ArchiveBytes
|
||||
o.Log("%s: %d message(s), %s, last sequence %d, %d consumer(s) → %s in %.2fs",
|
||||
name, entry.Messages, human(int64(entry.Bytes)), entry.LastSeq, entry.Consumers, human(entry.ArchiveBytes), entry.Seconds)
|
||||
name, entry.Messages, Human(int64(entry.Bytes)), entry.LastSeq, entry.Consumers, Human(entry.ArchiveBytes), entry.Seconds)
|
||||
}
|
||||
m.Finished = time.Now().UTC()
|
||||
m.Seconds = m.Finished.Sub(m.Started).Seconds()
|
||||
@@ -315,7 +336,7 @@ func snapshotStream(ctx context.Context, nc *nats.Conn, name string, f *os.File)
|
||||
State json.RawMessage `json:"state"`
|
||||
}
|
||||
req := map[string]any{"deliver_subject": deliver, "chunk_size": ChunkSize}
|
||||
if err := request(ctx, nc, apiStreamSnapshot+name, req, &resp); err != nil {
|
||||
if err := Request(ctx, nc, APIStreamSnapshot+name, req, &resp); err != nil {
|
||||
return busarchive.StreamEntry{}, nil, err
|
||||
}
|
||||
select {
|
||||
@@ -339,7 +360,7 @@ func snapshotStream(ctx context.Context, nc *nats.Conn, name string, f *os.File)
|
||||
}
|
||||
|
||||
var cfg streamConfig
|
||||
var state streamState
|
||||
var state StreamState
|
||||
if err := json.Unmarshal(resp.Config, &cfg); err != nil {
|
||||
return busarchive.StreamEntry{}, nil, fmt.Errorf("its configuration: %w", err)
|
||||
}
|
||||
@@ -371,3 +392,42 @@ func snapshotStream(ctx context.Context, nc *nats.Conn, name string, f *os.File)
|
||||
ArchiveSHA256: hex.EncodeToString(hash.Sum(nil)), Seconds: time.Since(started).Seconds(),
|
||||
}, meta, nil
|
||||
}
|
||||
|
||||
// Human is a size in bytes as a person reads it: 1.5 MiB.
|
||||
func Human(n int64) string {
|
||||
const unit = 1024
|
||||
if n < unit {
|
||||
return fmt.Sprintf("%d B", n)
|
||||
}
|
||||
div, exp := int64(unit), 0
|
||||
for v := n / unit; v >= unit; v /= unit {
|
||||
div *= unit
|
||||
exp++
|
||||
}
|
||||
return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp])
|
||||
}
|
||||
|
||||
// Into is the keeper's snapshot of some streams before it rebuilds or removes them (design 25 § 12.3): one tar under
|
||||
// dir, named for the streams and the time, written whole before its name is given, so a partial one is never taken
|
||||
// for it. It answers the file's path.
|
||||
func Into(nc *nats.Conn, dir string) func(ctx context.Context, streams []string) (string, error) {
|
||||
return func(ctx context.Context, streams []string) (string, error) {
|
||||
if dir == "" {
|
||||
return "", errors.New("no snapshots directory is set (MESH_NATS_SNAPSHOTS_DIR), so no snapshot is taken and nothing is removed")
|
||||
}
|
||||
name := filepath.Join(dir, "entry-"+strings.Join(streams, "+")+"-"+time.Now().UTC().Format("20060102T150405Z")+".tar")
|
||||
f, err := os.CreateTemp(dir, ".entry-*.partial")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
if _, err := Snapshot(ctx, nc, f, SnapshotOptions{Streams: streams, TempDir: dir}); err != nil {
|
||||
f.Close()
|
||||
return "", err
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return name, os.Rename(f.Name(), name)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package keeper
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
)
|
||||
|
||||
// FillPass reads the fill of every entry that refuses when full (ADR 0595 §6), raising `full` at the raise mark and
|
||||
// clearing it only below the clear mark, so an entry at the mark does not flap. It is the keeper's one periodic read.
|
||||
func (k *Keeper) FillPass() {
|
||||
raise, clear := k.marks()
|
||||
keys, err := k.Store.Keys(StateAsked)
|
||||
if err != nil {
|
||||
k.logf("the fill pass cannot read its own %s: %v", StateAsked, err)
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
for _, module := range keys {
|
||||
held, _, found := k.asked(module)
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
changed := false
|
||||
for _, local := range sortedKeys(held.Made) {
|
||||
m := held.Made[local]
|
||||
if !m.Created {
|
||||
continue
|
||||
}
|
||||
var most int64 = -1
|
||||
for _, o := range m.Plan.Objects {
|
||||
if !o.Settings.Refuses() || o.Settings.MaxBytes <= 0 {
|
||||
continue
|
||||
}
|
||||
_, f, there, err := k.Bus.Held(ctx, o)
|
||||
if err != nil || !there {
|
||||
continue
|
||||
}
|
||||
if p := int64(f.Bytes) * 100 / o.Settings.MaxBytes; p > most {
|
||||
most = p
|
||||
}
|
||||
}
|
||||
if most < 0 {
|
||||
continue
|
||||
}
|
||||
was := m
|
||||
m.Fill = most
|
||||
switch {
|
||||
case !m.Full && most >= raise:
|
||||
m.Full = true
|
||||
k.emit(EventFull, m.Plan, held.Revision, fmt.Sprintf("%d%% of its cap, past the raise mark %d%%", most, raise), nil, nil)
|
||||
k.raise(conditionFull, module, local, asks.SeverityWarning, "Kept data is filling up",
|
||||
"a larger cap in the module's manifest, or data removed",
|
||||
fmt.Sprintf("%s of %s holds %d%% of its cap and refuses new data when full; it is cleared below %d%%", local, module, most, clear))
|
||||
case m.Full && most < clear:
|
||||
m.Full = false
|
||||
k.emit(EventFullCleared, m.Plan, held.Revision, fmt.Sprintf("%d%% of its cap, below the clear mark %d%%", most, clear), nil, nil)
|
||||
k.clear(conditionFull, module, local)
|
||||
}
|
||||
if m.Fill != was.Fill || m.Full != was.Full {
|
||||
held.Made[local], changed = m, true
|
||||
}
|
||||
}
|
||||
if changed {
|
||||
k.write(held)
|
||||
}
|
||||
}
|
||||
k.setStatus(func(s *Status) { s.Pass = k.now() })
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package keeper
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Bound is how long the keeper may go without its fill pass, or leave an accepted verdict unanswered, before it is
|
||||
// unhealthy (design 25 § 12.2): three minutes, so the only guard against a store filling never stops without a sound.
|
||||
const Bound = 3 * time.Minute
|
||||
|
||||
// Health is the keeper's health as its tool answers it: healthy or not, why, and what it names beside (the entries it
|
||||
// could not create, the ones waiting, the ones past the raise mark). Another module's entry past the raise mark raises
|
||||
// that module's condition and leaves the keeper healthy.
|
||||
type Health struct {
|
||||
Healthy bool `json:"healthy"`
|
||||
Why string `json:"why"`
|
||||
Pass string `json:"pass"`
|
||||
Refused []string `json:"refused,omitempty"`
|
||||
Waiting []string `json:"waiting,omitempty"`
|
||||
Full []string `json:"full,omitempty"`
|
||||
}
|
||||
|
||||
// HealthOf judges the keeper from its status, what it holds, and pending: how many accepted verdicts wait on its
|
||||
// durable consumer unanswered (-1 when it could not be read). Pure.
|
||||
func HealthOf(s Status, held map[string]Asked, pending int, now time.Time) Health {
|
||||
h := Health{Healthy: true}
|
||||
var why []string
|
||||
switch {
|
||||
case s.Pass.IsZero() && (s.Started.IsZero() || now.Sub(s.Started) > Bound):
|
||||
why = append(why, "the keeper has made no fill pass since its start")
|
||||
h.Pass = "none"
|
||||
case !s.Pass.IsZero():
|
||||
h.Pass = s.Pass.Format(time.RFC3339)
|
||||
if now.Sub(s.Pass) > Bound {
|
||||
why = append(why, fmt.Sprintf("its last fill pass is %s old, more than %s", now.Sub(s.Pass).Round(time.Second), Bound))
|
||||
}
|
||||
}
|
||||
if s.FirstView.Answered.IsZero() {
|
||||
w := fmt.Sprintf("its first view, %s.%s, has not answered", s.FirstView.Owner, s.FirstView.Verb)
|
||||
if s.FirstView.Error != "" {
|
||||
w += ": " + s.FirstView.Error
|
||||
}
|
||||
why = append(why, w)
|
||||
}
|
||||
if pending > 0 && (s.Heard.IsZero() || now.Sub(s.Heard) > Bound) {
|
||||
why = append(why, fmt.Sprintf("%d accepted verdict(s) wait on its consumer unanswered for more than %s", pending, Bound))
|
||||
}
|
||||
for _, module := range sortedKeys(held) {
|
||||
for _, local := range sortedKeys(held[module].Made) {
|
||||
m := held[module].Made[local]
|
||||
name := BusName(module, local)
|
||||
switch {
|
||||
case m.Refused != "":
|
||||
h.Refused = append(h.Refused, name+": "+m.Refused)
|
||||
case m.Waiting != "":
|
||||
h.Waiting = append(h.Waiting, name+": "+m.Waiting)
|
||||
}
|
||||
if m.Full {
|
||||
h.Full = append(h.Full, fmt.Sprintf("%s: %d%%", name, m.Fill))
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(why) > 0 {
|
||||
h.Healthy, h.Why = false, strings.Join(why, "; ")
|
||||
} else {
|
||||
h.Why = "its fill pass runs and its first view is held"
|
||||
}
|
||||
return h
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package keeper
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// JetStream is the bus as the keeper's own account reaches it (Bus): each object a stream, a key-value bucket or an
|
||||
// object store of JetStream, read, created, grown and deleted by its stream.
|
||||
type JetStream struct {
|
||||
JS jetstream.JetStream
|
||||
// Snap takes a snapshot of streams and says where it is kept (bussnap through the keeper's own connection).
|
||||
Snap func(ctx context.Context, streams []string) (string, error)
|
||||
}
|
||||
|
||||
// Held reads the object's stream.
|
||||
func (b JetStream) Held(ctx context.Context, o Object) (Settings, Fill, bool, error) {
|
||||
s, err := b.JS.Stream(ctx, o.Stream())
|
||||
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
return Settings{}, Fill{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return Settings{}, Fill{}, false, retryable(err)
|
||||
}
|
||||
info, err := s.Info(ctx)
|
||||
if err != nil {
|
||||
return Settings{}, Fill{}, false, retryable(err)
|
||||
}
|
||||
return SettingsOf(o.Kind, info.Config), Fill{Bytes: info.State.Bytes, Messages: info.State.Msgs}, true, nil
|
||||
}
|
||||
|
||||
// SettingsOf reads a stream's configuration as the settings an object of kind is asked with.
|
||||
func SettingsOf(kind string, c jetstream.StreamConfig) Settings {
|
||||
s := Settings{MaxBytes: c.MaxBytes, MaxAgeSeconds: int64(c.MaxAge / time.Second), Discard: DiscardOld,
|
||||
WorkQueue: c.Retention == jetstream.WorkQueuePolicy, DuplicatesSeconds: int64(c.Duplicates / time.Second)}
|
||||
if c.Discard == jetstream.DiscardNew {
|
||||
s.Discard = DiscardNew
|
||||
}
|
||||
if c.MaxMsgSize > 0 {
|
||||
s.MaxMsgSize = int64(c.MaxMsgSize)
|
||||
}
|
||||
switch kind {
|
||||
case KindBucket:
|
||||
s.History = c.MaxMsgsPerSubject
|
||||
s.KeyTTL = c.AllowMsgTTL
|
||||
s.DuplicatesSeconds = 0
|
||||
case KindObjects:
|
||||
s.DuplicatesSeconds = 0
|
||||
default:
|
||||
if c.MaxMsgsPerSubject > 0 {
|
||||
s.History = c.MaxMsgsPerSubject
|
||||
}
|
||||
if !s.WorkQueue {
|
||||
s.DuplicatesSeconds = 0
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Create creates the object; a refusal of the account's grant is ErrRetry: the grants part composes the keeper's grant
|
||||
// to create a module's names from the same accepted manifests, and may not have yet.
|
||||
func (b JetStream) Create(ctx context.Context, o Object) error {
|
||||
var err error
|
||||
switch o.Kind {
|
||||
case KindBucket:
|
||||
_, err = b.JS.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: o.Name, History: uint8(o.Settings.History),
|
||||
TTL: time.Duration(o.Settings.MaxAgeSeconds) * time.Second, MaxBytes: o.Settings.MaxBytes,
|
||||
MaxValueSize: int32(o.Settings.MaxMsgSize), Storage: jetstream.FileStorage, Replicas: 1,
|
||||
LimitMarkerTTL: limitMarker(o.Settings.KeyTTL)})
|
||||
case KindObjects:
|
||||
_, err = b.JS.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: o.Name, MaxBytes: o.Settings.MaxBytes,
|
||||
Storage: jetstream.FileStorage, Replicas: 1})
|
||||
default:
|
||||
_, err = b.JS.CreateStream(ctx, streamConfig(o))
|
||||
}
|
||||
return retryable(err)
|
||||
}
|
||||
|
||||
func limitMarker(on bool) time.Duration {
|
||||
if on {
|
||||
return time.Second
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func streamConfig(o Object) jetstream.StreamConfig {
|
||||
c := jetstream.StreamConfig{Name: o.Name, Subjects: o.Subjects, Retention: jetstream.LimitsPolicy,
|
||||
MaxBytes: o.Settings.MaxBytes, MaxAge: time.Duration(o.Settings.MaxAgeSeconds) * time.Second,
|
||||
Discard: jetstream.DiscardOld, Storage: jetstream.FileStorage, Replicas: 1, MaxMsgs: -1, MaxMsgsPerSubject: -1}
|
||||
if o.Settings.Discard == DiscardNew {
|
||||
c.Discard = jetstream.DiscardNew
|
||||
}
|
||||
if o.Settings.WorkQueue {
|
||||
c.Retention = jetstream.WorkQueuePolicy
|
||||
c.Duplicates = time.Duration(o.Settings.DuplicatesSeconds) * time.Second
|
||||
}
|
||||
if o.Settings.MaxMsgSize > 0 {
|
||||
c.MaxMsgSize = int32(o.Settings.MaxMsgSize)
|
||||
}
|
||||
if o.Settings.History > 0 {
|
||||
c.MaxMsgsPerSubject = o.Settings.History
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// Update brings the held stream to the object's settings, keeping every other part of its configuration.
|
||||
func (b JetStream) Update(ctx context.Context, o Object) error {
|
||||
s, err := b.JS.Stream(ctx, o.Stream())
|
||||
if err != nil {
|
||||
return retryable(err)
|
||||
}
|
||||
info, err := s.Info(ctx)
|
||||
if err != nil {
|
||||
return retryable(err)
|
||||
}
|
||||
c := info.Config
|
||||
c.MaxBytes = o.Settings.MaxBytes
|
||||
c.MaxAge = time.Duration(o.Settings.MaxAgeSeconds) * time.Second
|
||||
if o.Settings.History > 0 {
|
||||
c.MaxMsgsPerSubject = o.Settings.History
|
||||
}
|
||||
_, err = b.JS.UpdateStream(ctx, c)
|
||||
return retryable(err)
|
||||
}
|
||||
|
||||
// Delete deletes the object's stream.
|
||||
func (b JetStream) Delete(ctx context.Context, o Object) error {
|
||||
err := b.JS.DeleteStream(ctx, o.Stream())
|
||||
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
return nil
|
||||
}
|
||||
return retryable(err)
|
||||
}
|
||||
|
||||
// Snapshot takes the streams' snapshot.
|
||||
func (b JetStream) Snapshot(ctx context.Context, streams []string) (string, error) {
|
||||
if b.Snap == nil {
|
||||
return "", fmt.Errorf("no snapshot is taken here, so nothing is removed")
|
||||
}
|
||||
return b.Snap(ctx, streams)
|
||||
}
|
||||
|
||||
// retryable marks a refusal of the account's permissions, or a bus that answers nothing, as ErrRetry.
|
||||
func retryable(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
words := strings.ToLower(err.Error())
|
||||
if strings.Contains(words, "permission") || strings.Contains(words, "authorization") || errors.Is(err, context.DeadlineExceeded) ||
|
||||
strings.Contains(words, "no responders") || strings.Contains(words, "timeout") {
|
||||
return fmt.Errorf("%w: %v", ErrRetry, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,617 @@
|
||||
package keeper
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
)
|
||||
|
||||
// Module is the bus module's own name: the publisher of its events and its conditions.
|
||||
const Module = "nats"
|
||||
|
||||
// The keeper's own states (old `state` sections, ADR 0595 §10): what it last heard per module, and the persons'
|
||||
// requests its tools hand it.
|
||||
const (
|
||||
StateAsked = "asked"
|
||||
StateRequests = "requests"
|
||||
// KeyStatus is the keeper's own status in StateAsked, read by its health tool: a key no module can have, since a
|
||||
// module's name holds no underscore.
|
||||
KeyStatus = "_keeper"
|
||||
)
|
||||
|
||||
// The seat mesh-broker's events (ADR 0595 §6): on the seat's subjects, and kept on the bus module's own stream (ADR 0620 §2).
|
||||
const (
|
||||
EventCreated = "created"
|
||||
EventRefused = "refused"
|
||||
EventDrifted = "drifted"
|
||||
EventDriftCleared = "drift-cleared"
|
||||
EventFull = "full"
|
||||
EventFullCleared = "full-cleared"
|
||||
EventOrphaned = "orphaned"
|
||||
EventRecreated = "recreated"
|
||||
EventDropped = "dropped"
|
||||
)
|
||||
|
||||
// Event is the body of every bus module event, in the field names of the seat mesh-broker's definition (its message
|
||||
// `said`, mesh-contracts go/seats/meshbroker Said): the entry (module, local name, storage type, template, options,
|
||||
// configuration), its bus name, the settings asked and held by bus object, its fill, the snapshot a recreate or drop
|
||||
// took, and why (refused, drifted). Beside the definition's fields: binding, the names a module receives (a ledger's
|
||||
// history, state, indexes and snapshot, which one bus name cannot carry), and revision, the indexer's revision of the
|
||||
// verdict it acted on; both asked of the definition.
|
||||
//
|
||||
// Written here until this module takes mesh-contracts' generated type; the swap changes no byte of the body.
|
||||
type Event struct {
|
||||
V int `json:"v"`
|
||||
Module string `json:"module"`
|
||||
Local string `json:"local-name"`
|
||||
Storage string `json:"storage-type,omitempty"`
|
||||
Template string `json:"template,omitempty"`
|
||||
Options map[string]json.RawMessage `json:"options,omitempty"`
|
||||
Configuration map[string]json.RawMessage `json:"configuration,omitempty"`
|
||||
BusName string `json:"bus-name"`
|
||||
Binding membership.BusBinding `json:"binding"`
|
||||
Asked map[string]Settings `json:"asked,omitempty"`
|
||||
Held map[string]Settings `json:"held,omitempty"`
|
||||
Fill *EventFill `json:"fill,omitempty"`
|
||||
Snapshot string `json:"snapshot,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
Revision uint64 `json:"revision"`
|
||||
}
|
||||
|
||||
// EventFill is an entry's fill as an event says it: its messages, its bytes and its cap.
|
||||
type EventFill struct {
|
||||
Count uint64 `json:"count"`
|
||||
Bytes uint64 `json:"bytes"`
|
||||
Cap int64 `json:"cap"`
|
||||
}
|
||||
|
||||
// Accepted is a manifest-accepted the keeper heard, or a module of its first view: the module, its manifest's digest,
|
||||
// the indexer's revision of the verdict (0 when the event does not say it yet), and its bus-state entries whole.
|
||||
type Accepted struct {
|
||||
Module string `json:"module"`
|
||||
Digest string `json:"digest"`
|
||||
Revision uint64 `json:"revision"`
|
||||
BusState []catalogue.BusStateEntry `json:"bus-state"`
|
||||
}
|
||||
|
||||
// FirstView is the indexer's answer of indexer_bus_state: every accepted manifest's bus-state entries, by module.
|
||||
type FirstView struct {
|
||||
Revision uint64 `json:"revision"`
|
||||
Modules []Accepted `json:"modules"`
|
||||
}
|
||||
|
||||
// Asked is what the keeper holds of one module, the value of its key in StateAsked: what its newest accepted manifest
|
||||
// asks for, and every entry the keeper made for it and the bus holds, by local name.
|
||||
type Asked struct {
|
||||
V int `json:"v"`
|
||||
Module string `json:"module"`
|
||||
Digest string `json:"digest,omitempty"`
|
||||
Revision uint64 `json:"revision"`
|
||||
Entries []catalogue.BusStateEntry `json:"entries"`
|
||||
Made map[string]Made `json:"made,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// Made is one entry as the keeper made it: its plan and what stands of it now.
|
||||
type Made struct {
|
||||
Plan Plan `json:"plan"`
|
||||
// Created says every object of it is on the bus and its `created` was said.
|
||||
Created bool `json:"created"`
|
||||
// Refused is why it could not be made, Waiting why it waits to be made again (a refusal the bus may lift, such as a
|
||||
// grant not composed yet).
|
||||
Refused string `json:"refused,omitempty"`
|
||||
Waiting string `json:"waiting,omitempty"`
|
||||
// Drift is how what the bus holds differs from what was asked, in a way that does not grow.
|
||||
Drift string `json:"drift,omitempty"`
|
||||
// Full is whether its fill is past the raise mark, Fill the last fill read, in percent of its cap.
|
||||
Full bool `json:"full,omitempty"`
|
||||
Fill int64 `json:"fill,omitempty"`
|
||||
// Orphaned says no manifest asks for it any more: it goes only by drop.
|
||||
Orphaned bool `json:"orphaned,omitempty"`
|
||||
}
|
||||
|
||||
// Status is the keeper's own state, KeyStatus: what its health tool reads.
|
||||
type Status struct {
|
||||
Started time.Time `json:"started"`
|
||||
// Pass is when the last fill pass ended.
|
||||
Pass time.Time `json:"pass"`
|
||||
// FirstView is the indexer's first view: when asked, when answered and at which revision, or why not.
|
||||
FirstView FirstViewStatus `json:"first-view"`
|
||||
// Heard is when the last manifest-accepted was heard, and Unanswered since when one has waited unhandled.
|
||||
Heard time.Time `json:"heard,omitempty"`
|
||||
Unanswered time.Time `json:"unanswered,omitempty"`
|
||||
}
|
||||
|
||||
// FirstViewStatus is what the keeper holds of its first view.
|
||||
type FirstViewStatus struct {
|
||||
Owner string `json:"owner"`
|
||||
Verb string `json:"verb"`
|
||||
Asked time.Time `json:"asked,omitempty"`
|
||||
Answered time.Time `json:"answered,omitempty"`
|
||||
Revision uint64 `json:"revision"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Next time.Time `json:"next,omitempty"`
|
||||
}
|
||||
|
||||
// The first view's owner and verb (design 25 § 12.2).
|
||||
const (
|
||||
FirstViewOwner = "mesh-delivery-indexer"
|
||||
FirstViewVerb = "indexer_bus_state"
|
||||
)
|
||||
|
||||
// Bus is the bus as the keeper's own account reaches it.
|
||||
type Bus interface {
|
||||
// Held reads the object's stream: its settings, whether it is there, and its fill (bytes and messages).
|
||||
Held(ctx context.Context, o Object) (Settings, Fill, bool, error)
|
||||
Create(ctx context.Context, o Object) error
|
||||
// Update brings a held object to o's settings: only a difference that grows (Grows), or a recreate's rebuild.
|
||||
Update(ctx context.Context, o Object) error
|
||||
Delete(ctx context.Context, o Object) error
|
||||
// Snapshot takes a snapshot of the streams and says where it is kept.
|
||||
Snapshot(ctx context.Context, streams []string) (string, error)
|
||||
}
|
||||
|
||||
// Fill is how much one object holds: its bytes and messages.
|
||||
type Fill struct {
|
||||
Bytes uint64 `json:"bytes"`
|
||||
Messages uint64 `json:"messages"`
|
||||
}
|
||||
|
||||
// ErrRetry marks a refusal the bus may lift without a change of the manifest (a create the keeper's account is not yet
|
||||
// granted): the entry waits and is made again on the backoff, never said refused.
|
||||
var ErrRetry = errors.New("refused for now")
|
||||
|
||||
// Store is the keeper's own state.
|
||||
type Store interface {
|
||||
Get(state, key string) ([]byte, uint64, bool, error)
|
||||
Put(state, key string, value []byte) (uint64, error)
|
||||
Delete(state, key string) error
|
||||
Keys(state string) ([]string, error)
|
||||
}
|
||||
|
||||
// Keeper is the bus module's keeper.
|
||||
type Keeper struct {
|
||||
Bus Bus
|
||||
Store Store
|
||||
// Emit says one of the bus module's events; Say raises or clears a condition through the condition keeper.
|
||||
Emit func(event string, body Event) error
|
||||
Say func(asks.Condition) error
|
||||
// View asks the indexer's indexer_bus_state.
|
||||
View func() (FirstView, error)
|
||||
// Act records a hand act through the seat mesh-hand-acts; Warrant reads the router's record of an ask.
|
||||
Act func(what, why, cause string) error
|
||||
Warrant func(askID string) (*asks.Ask, *asks.Warrant, error)
|
||||
Now func() time.Time
|
||||
Logf func(string, ...any)
|
||||
// Raise and Clear are the fill marks in percent (ADR 0595 §6): settings of the bus module, 75 and 70 by default.
|
||||
Raise, Clear int64
|
||||
|
||||
mu sync.Mutex
|
||||
status Status
|
||||
viewed bool
|
||||
}
|
||||
|
||||
func (k *Keeper) now() time.Time {
|
||||
if k.Now == nil {
|
||||
return time.Now().UTC()
|
||||
}
|
||||
return k.Now().UTC()
|
||||
}
|
||||
|
||||
func (k *Keeper) logf(format string, a ...any) {
|
||||
if k.Logf != nil {
|
||||
k.Logf(format, a...)
|
||||
}
|
||||
}
|
||||
|
||||
func (k *Keeper) marks() (int64, int64) {
|
||||
raise, clear := k.Raise, k.Clear
|
||||
if raise <= 0 {
|
||||
raise = 75
|
||||
}
|
||||
if clear <= 0 || clear >= raise {
|
||||
clear = raise - 5
|
||||
}
|
||||
return raise, clear
|
||||
}
|
||||
|
||||
// Viewed is whether the first view was answered: before it the keeper creates and orphans nothing.
|
||||
func (k *Keeper) Viewed() bool {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
return k.viewed
|
||||
}
|
||||
|
||||
// Status is the keeper's status now.
|
||||
func (k *Keeper) Status() Status {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
return k.status
|
||||
}
|
||||
|
||||
func (k *Keeper) setStatus(f func(*Status)) {
|
||||
k.mu.Lock()
|
||||
if k.status.Started.IsZero() {
|
||||
k.status.Started = k.now()
|
||||
}
|
||||
k.status.FirstView.Owner, k.status.FirstView.Verb = FirstViewOwner, FirstViewVerb
|
||||
f(&k.status)
|
||||
s := k.status
|
||||
k.mu.Unlock()
|
||||
if k.Store != nil {
|
||||
raw, _ := json.Marshal(s)
|
||||
if _, err := k.Store.Put(StateAsked, KeyStatus, raw); err != nil {
|
||||
k.logf("the keeper's status cannot be written to its own %s: %v", StateAsked, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AskFirstView asks the indexer's first view once and applies it (design 25 § 12.2): every module it answers is
|
||||
// applied as an accepted verdict, a module the keeper holds and the view leaves out is kept (a set-aside manifest is
|
||||
// missing from it, never removed), and then check runs over what is held. It answers the error and when to ask again,
|
||||
// on the backoff from 10 s doubling to 5 min; the keeper's health is unhealthy meanwhile, naming the verb.
|
||||
func (k *Keeper) AskFirstView(wait time.Duration) (time.Duration, error) {
|
||||
at := k.now()
|
||||
k.setStatus(func(s *Status) { s.FirstView.Asked = at })
|
||||
view, err := k.View()
|
||||
if err != nil {
|
||||
next := wait * 2
|
||||
if wait <= 0 {
|
||||
next = 10 * time.Second
|
||||
}
|
||||
if next > 5*time.Minute {
|
||||
next = 5 * time.Minute
|
||||
}
|
||||
k.setStatus(func(s *Status) {
|
||||
s.FirstView.Error = fmt.Sprintf("%s.%s did not answer: %v", FirstViewOwner, FirstViewVerb, err)
|
||||
s.FirstView.Next = at.Add(next)
|
||||
})
|
||||
return next, err
|
||||
}
|
||||
k.mu.Lock()
|
||||
k.viewed = true
|
||||
k.mu.Unlock()
|
||||
k.setStatus(func(s *Status) {
|
||||
s.FirstView.Answered, s.FirstView.Revision, s.FirstView.Error, s.FirstView.Next = k.now(), view.Revision, "", time.Time{}
|
||||
})
|
||||
for _, a := range view.Modules {
|
||||
k.Heard(a)
|
||||
}
|
||||
k.CheckAll()
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// Heard applies one accepted verdict (manifest-accepted, or a module of the first view): applied only when newer than
|
||||
// what the keeper holds for that module by the indexer's revision alone, so a replay never takes an entry back (a
|
||||
// verdict that says no revision is applied when its digest differs). Each entry asked is made where the bus lacks it;
|
||||
// each entry made that is no longer asked is an orphan. An empty verdict ("this module asks nothing now") ends the
|
||||
// module's key once nothing it made is held.
|
||||
func (k *Keeper) Heard(a Accepted) {
|
||||
k.setStatus(func(s *Status) { s.Heard = k.now() })
|
||||
if !k.Viewed() {
|
||||
k.logf("manifest-accepted of %s heard before the first view: kept on the consumer until it is answered", a.Module)
|
||||
return
|
||||
}
|
||||
held, _, found := k.asked(a.Module)
|
||||
if found && a.Revision != 0 && a.Revision <= held.Revision {
|
||||
k.logf("manifest-accepted of %s at revision %d is not newer than the %d held: not applied", a.Module, a.Revision, held.Revision)
|
||||
return
|
||||
}
|
||||
if found && a.Revision == 0 && a.Digest != "" && a.Digest == held.Digest {
|
||||
return
|
||||
}
|
||||
if !found {
|
||||
held = Asked{V: 1, Module: a.Module}
|
||||
}
|
||||
if held.Made == nil {
|
||||
held.Made = map[string]Made{}
|
||||
}
|
||||
held.Digest, held.Entries, held.At = a.Digest, a.BusState, k.now()
|
||||
if a.Revision > held.Revision {
|
||||
held.Revision = a.Revision
|
||||
}
|
||||
asked := map[string]bool{}
|
||||
for _, e := range a.BusState {
|
||||
asked[e.Name] = true
|
||||
k.makeEntry(&held, e)
|
||||
}
|
||||
for _, local := range sortedKeys(held.Made) {
|
||||
m := held.Made[local]
|
||||
if asked[local] || m.Orphaned {
|
||||
continue
|
||||
}
|
||||
if !m.Created {
|
||||
// Never made: nothing on the bus to orphan.
|
||||
delete(held.Made, local)
|
||||
k.clear(conditionRefused, a.Module, local)
|
||||
continue
|
||||
}
|
||||
m.Orphaned = true
|
||||
held.Made[local] = m
|
||||
k.emit(EventOrphaned, m.Plan, held.Revision, "no accepted manifest of "+a.Module+" asks for it now", nil, nil)
|
||||
k.raise(conditionOrphaned, a.Module, local, asks.SeverityWarning, "Kept data no manifest asks for",
|
||||
"the operator's drop of it, or a manifest asking for it again",
|
||||
fmt.Sprintf("%s of %s is on the bus and no accepted manifest asks for it: it goes only by the bus module's drop, on your warrant (nats_drop), or a manifest asking for it again keeps it", BusName(a.Module, local), a.Module))
|
||||
}
|
||||
k.write(held)
|
||||
}
|
||||
|
||||
// makeEntry makes one asked entry where the bus lacks it, compares it where it holds it, and says what it did.
|
||||
func (k *Keeper) makeEntry(held *Asked, e catalogue.BusStateEntry) {
|
||||
module := held.Module
|
||||
plan, err := PlanOf(module, e)
|
||||
m := held.Made[e.Name]
|
||||
if err != nil {
|
||||
if m.Refused != err.Error() {
|
||||
k.emit(EventRefused, Plan{Module: module, Local: e.Name, Storage: e.Storage, Template: e.Template}, held.Revision, err.Error(), nil, nil)
|
||||
k.raise(conditionRefused, module, e.Name, asks.SeverityWarning, "The bus module refused to create kept data",
|
||||
"a manifest that asks for it in a form the bus module makes",
|
||||
fmt.Sprintf("%s of %s was not created: %v. The module gets no binding for it and fails at its start, naming it", e.Name, module, err))
|
||||
}
|
||||
m.Refused, m.Waiting = err.Error(), ""
|
||||
held.Made[e.Name] = m
|
||||
return
|
||||
}
|
||||
wasOrphan := m.Orphaned
|
||||
m.Plan, m.Orphaned = plan, false
|
||||
if wasOrphan {
|
||||
// Asked again: kept, and its binding said again with created (design 25 § 12.3).
|
||||
m.Created = false
|
||||
k.clear(conditionOrphaned, module, e.Name)
|
||||
}
|
||||
k.ensure(held, &m)
|
||||
held.Made[e.Name] = m
|
||||
}
|
||||
|
||||
// ensure brings one planned entry onto the bus: each object the bus lacks created, a held one compared. A difference
|
||||
// that only grows is applied; any other is a drift, said and never fixed quietly.
|
||||
func (k *Keeper) ensure(held *Asked, m *Made) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
asked, holds := map[string]Settings{}, map[string]Settings{}
|
||||
var drifts []string
|
||||
for _, o := range m.Plan.Objects {
|
||||
asked[o.Name] = o.Settings
|
||||
s, _, found, err := k.Bus.Held(ctx, o)
|
||||
if err != nil {
|
||||
m.Waiting = fmt.Sprintf("%s cannot be read: %v", o.Stream(), err)
|
||||
return
|
||||
}
|
||||
if !found {
|
||||
if err := k.Bus.Create(ctx, o); err != nil {
|
||||
if errors.Is(err, ErrRetry) {
|
||||
m.Waiting = fmt.Sprintf("%s cannot be created yet: %v", o.Stream(), err)
|
||||
return
|
||||
}
|
||||
m.Refused, m.Waiting = fmt.Sprintf("%s: %v", o.Stream(), err), ""
|
||||
k.emit(EventRefused, m.Plan, held.Revision, m.Refused, asked, nil)
|
||||
k.raise(conditionRefused, held.Module, m.Plan.Local, asks.SeverityWarning, "The bus module refused to create kept data",
|
||||
"a look at why the bus refused it",
|
||||
fmt.Sprintf("%s of %s was not created: %s. The module gets no binding for it and fails at its start, naming it", m.Plan.Local, held.Module, m.Refused))
|
||||
return
|
||||
}
|
||||
holds[o.Name] = o.Settings
|
||||
continue
|
||||
}
|
||||
holds[o.Name] = s
|
||||
if Differs(o.Settings, s) == "" {
|
||||
continue
|
||||
}
|
||||
if Grows(o.Settings, s) {
|
||||
if err := k.Bus.Update(ctx, o); err != nil {
|
||||
m.Waiting = fmt.Sprintf("%s's larger settings cannot be applied yet: %v", o.Stream(), err)
|
||||
return
|
||||
}
|
||||
holds[o.Name] = o.Settings
|
||||
continue
|
||||
}
|
||||
drifts = append(drifts, o.Stream()+": "+Differs(o.Settings, s))
|
||||
}
|
||||
wasRefused := m.Refused != ""
|
||||
m.Waiting, m.Refused = "", ""
|
||||
if wasRefused {
|
||||
k.clear(conditionRefused, held.Module, m.Plan.Local)
|
||||
}
|
||||
if !m.Created {
|
||||
m.Created = true
|
||||
k.emit(EventCreated, m.Plan, held.Revision, "", asked, holds)
|
||||
}
|
||||
drift := strings.Join(drifts, "; ")
|
||||
switch {
|
||||
case drift != "" && drift != m.Drift:
|
||||
m.Drift = drift
|
||||
k.emit(EventDrifted, m.Plan, held.Revision, drift, asked, holds)
|
||||
k.raise(conditionDrifted, held.Module, m.Plan.Local, asks.SeverityWarning, "Kept data differs from what was asked",
|
||||
"a decision: change the manifest back, or the bus module's recreate on your warrant",
|
||||
fmt.Sprintf("%s of %s differs from what its manifest asks, in a way that does not only grow: %s", m.Plan.Local, held.Module, drift))
|
||||
case drift == "" && m.Drift != "":
|
||||
m.Drift = ""
|
||||
k.emit(EventDriftCleared, m.Plan, held.Revision, "", asked, holds)
|
||||
k.clear(conditionDrifted, held.Module, m.Plan.Local)
|
||||
}
|
||||
}
|
||||
|
||||
// CheckAll compares every entry the keeper holds with the bus, making what is missing (design 25 § 12.2's check at
|
||||
// start) and making again what waits; it answers what it found, one line per entry.
|
||||
func (k *Keeper) CheckAll() []CheckLine {
|
||||
var out []CheckLine
|
||||
keys, err := k.Store.Keys(StateAsked)
|
||||
if err != nil {
|
||||
return []CheckLine{{Why: "the keeper's own " + StateAsked + " cannot be read: " + err.Error()}}
|
||||
}
|
||||
for _, module := range keys {
|
||||
if module == KeyStatus {
|
||||
continue
|
||||
}
|
||||
held, _, found := k.asked(module)
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
for _, local := range sortedKeys(held.Made) {
|
||||
m := held.Made[local]
|
||||
if m.Orphaned || m.Plan.Local == "" {
|
||||
out = append(out, lineOf(module, local, m))
|
||||
continue
|
||||
}
|
||||
for _, e := range held.Entries {
|
||||
if e.Name == local {
|
||||
k.makeEntry(&held, e)
|
||||
}
|
||||
}
|
||||
out = append(out, lineOf(module, local, held.Made[local]))
|
||||
}
|
||||
k.write(held)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// CheckLine is one entry as check answers it.
|
||||
type CheckLine struct {
|
||||
Module string `json:"module,omitempty"`
|
||||
Local string `json:"local,omitempty"`
|
||||
Bus string `json:"bus,omitempty"`
|
||||
Created bool `json:"created"`
|
||||
Refused string `json:"refused,omitempty"`
|
||||
Waiting string `json:"waiting,omitempty"`
|
||||
Drift string `json:"drift,omitempty"`
|
||||
Orphaned bool `json:"orphaned,omitempty"`
|
||||
Full bool `json:"full,omitempty"`
|
||||
Fill int64 `json:"fill-percent,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
}
|
||||
|
||||
func lineOf(module, local string, m Made) CheckLine {
|
||||
return CheckLine{Module: module, Local: local, Bus: BusName(module, local), Created: m.Created, Refused: m.Refused,
|
||||
Waiting: m.Waiting, Drift: m.Drift, Orphaned: m.Orphaned, Full: m.Full, Fill: m.Fill}
|
||||
}
|
||||
|
||||
// Waiting is every entry that waits to be made again, `<module> <local>: <why>`: what the backoff retries.
|
||||
func (k *Keeper) Waiting() []string {
|
||||
var out []string
|
||||
keys, _ := k.Store.Keys(StateAsked)
|
||||
for _, module := range keys {
|
||||
held, _, found := k.asked(module)
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
for _, local := range sortedKeys(held.Made) {
|
||||
if w := held.Made[local].Waiting; w != "" {
|
||||
out = append(out, module+" "+local+": "+w)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// The kinds of condition the keeper raises, each naming the module and the entry (design 25 § 12.3).
|
||||
const (
|
||||
conditionFull = "full"
|
||||
conditionDrifted = "drifted"
|
||||
conditionOrphaned = "orphaned"
|
||||
conditionRefused = "refused"
|
||||
)
|
||||
|
||||
// ConditionKey is the key of one of the keeper's conditions: `nats.<kind>.<module>_<local>`.
|
||||
func ConditionKey(kind, module, local string) string {
|
||||
return Module + "." + kind + "." + BusName(module, local)
|
||||
}
|
||||
|
||||
func (k *Keeper) raise(kind, module, local, severity, headline, needs, explanation string) {
|
||||
if k.Say == nil {
|
||||
return
|
||||
}
|
||||
c := asks.Condition{Key: ConditionKey(kind, module, local), Subject: asks.Scope{Scope: "module", ID: module},
|
||||
State: asks.StateRaised, Severity: severity, Headline: headline, Needs: needs,
|
||||
Summary: explanation,
|
||||
Explanation: asks.NeedsYou + needs + ". " + explanation,
|
||||
Resolved: "cleared by the bus module once it no longer holds"}
|
||||
if err := k.Say(c); err != nil {
|
||||
k.logf("the condition %s cannot be raised: %v", c.Key, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (k *Keeper) clear(kind, module, local string) {
|
||||
if k.Say == nil {
|
||||
return
|
||||
}
|
||||
c := asks.Condition{Key: ConditionKey(kind, module, local), Subject: asks.Scope{Scope: "module", ID: module}, State: asks.StateCleared}
|
||||
if err := k.Say(c); err != nil {
|
||||
k.logf("the condition %s cannot be cleared: %v", c.Key, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (k *Keeper) emit(event string, p Plan, revision uint64, why string, asked, held map[string]Settings) {
|
||||
if k.Emit == nil {
|
||||
return
|
||||
}
|
||||
body := Event{V: 1, Module: p.Module, Local: p.Local, Storage: p.Storage, Template: p.Template, Options: p.Entry.Options,
|
||||
Configuration: p.Entry.Configuration, BusName: BusName(p.Module, p.Local), Binding: p.Binding,
|
||||
Asked: asked, Held: held, Why: why, Revision: revision}
|
||||
if event == EventDropped {
|
||||
// A dropped entry's names only: the binding to remove.
|
||||
body.Binding.Types = nil
|
||||
}
|
||||
if err := k.Emit(event, body); err != nil {
|
||||
k.logf("the event %s of %s %s could not be said: %v", event, p.Module, p.Local, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (k *Keeper) asked(module string) (Asked, uint64, bool) {
|
||||
raw, rev, found, err := k.Store.Get(StateAsked, module)
|
||||
if err != nil || !found {
|
||||
return Asked{}, 0, false
|
||||
}
|
||||
var a Asked
|
||||
if json.Unmarshal(raw, &a) != nil {
|
||||
return Asked{}, 0, false
|
||||
}
|
||||
return a, rev, true
|
||||
}
|
||||
|
||||
// write keeps a module's key, or removes it when its manifest asks for nothing and nothing it made is held: a key goes
|
||||
// only so, never because a module is missing from a first view.
|
||||
func (k *Keeper) write(a Asked) {
|
||||
if len(a.Entries) == 0 && len(a.Made) == 0 {
|
||||
if err := k.Store.Delete(StateAsked, a.Module); err != nil {
|
||||
k.logf("the key of %s cannot be removed from its own %s: %v", a.Module, StateAsked, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
raw, _ := json.Marshal(a)
|
||||
if _, err := k.Store.Put(StateAsked, a.Module, raw); err != nil {
|
||||
k.logf("the key of %s cannot be written to its own %s: %v", a.Module, StateAsked, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Held is every module the keeper holds, by name: what list, show and orphans answer from.
|
||||
func (k *Keeper) Held() map[string]Asked {
|
||||
out := map[string]Asked{}
|
||||
keys, _ := k.Store.Keys(StateAsked)
|
||||
for _, module := range keys {
|
||||
if module == KeyStatus {
|
||||
continue
|
||||
}
|
||||
if a, _, found := k.asked(module); found {
|
||||
out[module] = a
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sortedKeys[V any](m map[string]V) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for key := range m {
|
||||
out = append(out, key)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,571 @@
|
||||
package keeper
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// fakeBus holds objects by stream, as a bus would, and refuses what a test tells it to.
|
||||
type fakeBus struct {
|
||||
mu sync.Mutex
|
||||
held map[string]Settings
|
||||
fill map[string]uint64
|
||||
refuse map[string]error
|
||||
created []string
|
||||
updated []string
|
||||
deleted []string
|
||||
snaps [][]string
|
||||
}
|
||||
|
||||
func newFakeBus() *fakeBus {
|
||||
return &fakeBus{held: map[string]Settings{}, fill: map[string]uint64{}, refuse: map[string]error{}}
|
||||
}
|
||||
|
||||
func (b *fakeBus) Held(_ context.Context, o Object) (Settings, Fill, bool, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
s, ok := b.held[o.Stream()]
|
||||
return s, Fill{Bytes: b.fill[o.Stream()]}, ok, nil
|
||||
}
|
||||
|
||||
func (b *fakeBus) Create(_ context.Context, o Object) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if err := b.refuse[o.Stream()]; err != nil {
|
||||
return err
|
||||
}
|
||||
b.held[o.Stream()] = o.Settings
|
||||
b.created = append(b.created, o.Stream())
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *fakeBus) Update(_ context.Context, o Object) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.held[o.Stream()] = o.Settings
|
||||
b.updated = append(b.updated, o.Stream())
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *fakeBus) Delete(_ context.Context, o Object) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
delete(b.held, o.Stream())
|
||||
b.deleted = append(b.deleted, o.Stream())
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *fakeBus) Snapshot(_ context.Context, streams []string) (string, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.snaps = append(b.snaps, streams)
|
||||
if err := b.refuse["snapshot"]; err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "/snapshots/" + strings.Join(streams, "+") + ".tar", nil
|
||||
}
|
||||
|
||||
// memStore is a key-value state in memory.
|
||||
type memStore struct {
|
||||
mu sync.Mutex
|
||||
kv map[string][]byte
|
||||
revs map[string]uint64
|
||||
rev uint64
|
||||
}
|
||||
|
||||
func newMemStore() *memStore { return &memStore{kv: map[string][]byte{}, revs: map[string]uint64{}} }
|
||||
|
||||
func (s *memStore) Get(state, key string) ([]byte, uint64, bool, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
v, ok := s.kv[state+"/"+key]
|
||||
return v, s.revs[state+"/"+key], ok, nil
|
||||
}
|
||||
|
||||
func (s *memStore) Put(state, key string, value []byte) (uint64, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.rev++
|
||||
s.kv[state+"/"+key], s.revs[state+"/"+key] = value, s.rev
|
||||
return s.rev, nil
|
||||
}
|
||||
|
||||
func (s *memStore) Delete(state, key string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
delete(s.kv, state+"/"+key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *memStore) Keys(state string) ([]string, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var out []string
|
||||
for k := range s.kv {
|
||||
if st, key, _ := strings.Cut(k, "/"); st == state {
|
||||
out = append(out, key)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// said is what a keeper said: its events and its conditions, in order.
|
||||
type said struct {
|
||||
mu sync.Mutex
|
||||
events []string
|
||||
bodies []Event
|
||||
conds []string
|
||||
}
|
||||
|
||||
func (s *said) emit(event string, body Event) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.events = append(s.events, event+" "+body.Module+" "+body.Local)
|
||||
s.bodies = append(s.bodies, body)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *said) say(c asks.Condition) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.conds = append(s.conds, c.State+" "+c.Key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *said) reset() {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.events, s.bodies, s.conds = nil, nil, nil
|
||||
}
|
||||
|
||||
func newKeeper(view FirstView) (*Keeper, *fakeBus, *memStore, *said) {
|
||||
bus, store, out := newFakeBus(), newMemStore(), &said{}
|
||||
at := time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC)
|
||||
k := &Keeper{Bus: bus, Store: store, Emit: out.emit, Say: out.say,
|
||||
View: func() (FirstView, error) { return view, nil },
|
||||
Now: func() time.Time { return at }}
|
||||
return k, bus, store, out
|
||||
}
|
||||
|
||||
func entry(name, storage, template string, options map[string]string) catalogue.BusStateEntry {
|
||||
e := catalogue.BusStateEntry{Name: name, Storage: storage, Template: template}
|
||||
if len(options) > 0 {
|
||||
e.Options = map[string]json.RawMessage{}
|
||||
for k, v := range options {
|
||||
e.Options[k] = json.RawMessage(v)
|
||||
}
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
// What one entry is on the bus (design 25 § 12.1, CAT-N3): each template's objects, named `<module>_<local>`, its
|
||||
// binding, and the fixed settings no option changes; and the entries the keeper refuses itself, naming why.
|
||||
func TestEachTemplateIsPlannedUnderTheModulesNames(t *testing.T) {
|
||||
ledger := entry("records", catalogue.StorageLedger, catalogue.TemplateLedger, map[string]string{"max-mib": "16"})
|
||||
ledger.Types = []catalogue.LedgerType{{Name: "decision", Indexes: []catalogue.LedgerIndex{{Name: "by-topic", On: []string{"topic"}}}}}
|
||||
bare := entry("raw", catalogue.StorageStream, "", nil)
|
||||
bare.Configuration = map[string]json.RawMessage{"when-full": json.RawMessage(`"drop-oldest"`), "max-mib": json.RawMessage("8"), "max-age-days": json.RawMessage("3")}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
entry catalogue.BusStateEntry
|
||||
objects string
|
||||
binding string
|
||||
refused string
|
||||
}{
|
||||
{name: "state", entry: entry("drafts", catalogue.StorageKeyValue, catalogue.TemplateState, map[string]string{"history": "5"}),
|
||||
objects: "key-value mesh-knowledge_drafts {64 MiB, age 0, history 5, new, size 262144}",
|
||||
binding: "bucket mesh-knowledge_drafts"},
|
||||
{name: "events, default age and cap", entry: entry("events", catalogue.StorageStream, catalogue.TemplateEvents, nil),
|
||||
objects: "stream mesh-knowledge_events [mesh.bus.mesh-knowledge.events.>] {1024 MiB, age 604800, history 0, old, size 0}",
|
||||
binding: "stream mesh-knowledge_events on mesh.bus.mesh-knowledge.events"},
|
||||
{name: "work queue", entry: entry("jobs", catalogue.StorageStream, catalogue.TemplateWorkQueue, map[string]string{"max-mib": "4", "duplicates": "120"}),
|
||||
objects: "stream mesh-knowledge_jobs [mesh.bus.mesh-knowledge.jobs.>] {4 MiB, age 0, history 0, new, size 0, work-queue, duplicates 120}",
|
||||
binding: "stream mesh-knowledge_jobs on mesh.bus.mesh-knowledge.jobs"},
|
||||
{name: "log", entry: entry("log", catalogue.StorageStream, catalogue.TemplateLog, nil),
|
||||
objects: "stream mesh-knowledge_log [mesh.bus.mesh-knowledge.log.>] {1024 MiB, age 0, history 0, new, size 262144}",
|
||||
binding: "stream mesh-knowledge_log on mesh.bus.mesh-knowledge.log"},
|
||||
{name: "objects", entry: entry("files", catalogue.StorageObjectStore, catalogue.TemplateObjects, map[string]string{"max-mib": "32"}),
|
||||
objects: "object-store mesh-knowledge_files {32 MiB, age 0, history 0, new, size 0}",
|
||||
binding: "objects mesh-knowledge_files"},
|
||||
{name: "ledger", entry: ledger,
|
||||
objects: "stream mesh-knowledge_records [mesh.ledger.mesh-knowledge.records.history.>] {16 MiB, age 0, history 0, new, size 262144}; " +
|
||||
"key-value mesh-knowledge_records_state {64 MiB, age 0, history 1, new, size 262144}; " +
|
||||
"key-value mesh-knowledge_records_index-by-topic {64 MiB, age 0, history 1, new, size 262144}; " +
|
||||
"stream mesh-knowledge_records_snapshot [mesh.ledger.mesh-knowledge.records.snapshot.>] {16 MiB, age 0, history 1, old, size 0}",
|
||||
binding: "history mesh-knowledge_records on mesh.ledger.mesh-knowledge.records.history, state mesh-knowledge_records_state, indexes map[by-topic:mesh-knowledge_records_index-by-topic], snapshot mesh.ledger.mesh-knowledge.records.snapshot"},
|
||||
{name: "bare stream", entry: bare,
|
||||
objects: "stream mesh-knowledge_raw [mesh.bus.mesh-knowledge.raw.>] {8 MiB, age 259200, history 0, old, size 0}",
|
||||
binding: "stream mesh-knowledge_raw on mesh.bus.mesh-knowledge.raw"},
|
||||
{name: "a local name holding the separator", entry: entry("my_drafts", catalogue.StorageKeyValue, catalogue.TemplateState, nil),
|
||||
refused: `the local name "my_drafts" is empty or holds "_"`},
|
||||
{name: "objects with no cap", entry: entry("files", catalogue.StorageObjectStore, catalogue.TemplateObjects, nil),
|
||||
refused: "gives no max-mib"},
|
||||
{name: "a bare stream with no when-full", entry: entry("raw", catalogue.StorageStream, "", nil),
|
||||
refused: "must give when-full"},
|
||||
{name: "a template on the wrong storage type", entry: entry("x", catalogue.StorageKeyValue, catalogue.TemplateLog, nil),
|
||||
refused: "is none the bus module makes"},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
p, err := PlanOf("mesh-knowledge", row.entry)
|
||||
if row.refused != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), row.refused) {
|
||||
t.Fatalf("planned %+v (%v): refused, naming %q", p, err, row.refused)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var objs []string
|
||||
for _, o := range p.Objects {
|
||||
s := o.Settings
|
||||
w := fmt.Sprintf("%s %s", o.Kind, o.Name)
|
||||
if len(o.Subjects) > 0 {
|
||||
w += fmt.Sprintf(" %v", o.Subjects)
|
||||
}
|
||||
w += fmt.Sprintf(" {%d MiB, age %d, history %d, %s, size %d", s.MaxBytes/MiB, s.MaxAgeSeconds, s.History, s.Discard, s.MaxMsgSize)
|
||||
if s.WorkQueue {
|
||||
w += fmt.Sprintf(", work-queue, duplicates %d", s.DuplicatesSeconds)
|
||||
}
|
||||
objs = append(objs, w+"}")
|
||||
}
|
||||
if got := strings.Join(objs, "; "); got != row.objects {
|
||||
t.Errorf("objects\n got %s\nwant %s", got, row.objects)
|
||||
}
|
||||
b := p.Binding
|
||||
var binding string
|
||||
switch {
|
||||
case b.History != "":
|
||||
binding = fmt.Sprintf("history %s on %s, state %s, indexes %v, snapshot %s", b.History, b.Subject, b.State, b.Indexes, b.Snapshot)
|
||||
case b.Stream != "":
|
||||
binding = fmt.Sprintf("stream %s on %s", b.Stream, b.Subject)
|
||||
case b.Bucket != "":
|
||||
binding = "bucket " + b.Bucket
|
||||
case b.ObjectStore != "":
|
||||
binding = "objects " + b.ObjectStore
|
||||
}
|
||||
if binding != row.binding || b.Local != row.entry.Name || b.Storage != row.entry.Storage {
|
||||
t.Errorf("binding %s (%+v), want %s", binding, b, row.binding)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The keeper makes what an accepted verdict asks for, once, and says it (design 25 § 12.2, ADR 0595 §4; CAT-N2, N3):
|
||||
// nothing before its first view; a replay not newer than what it holds is not applied; a refusal of its grant waits and
|
||||
// is made on the backoff, never said refused; a refusal of the entry is said and raised, and cleared by a later make.
|
||||
func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) {
|
||||
drafts := entry("drafts", catalogue.StorageKeyValue, catalogue.TemplateState, nil)
|
||||
log := entry("log", catalogue.StorageStream, catalogue.TemplateLog, nil)
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
run func(t *testing.T, k *Keeper, bus *fakeBus, out *said)
|
||||
}{
|
||||
{"created once, binding said, nothing before the first view", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
||||
k.mu.Lock()
|
||||
k.viewed = false
|
||||
k.mu.Unlock()
|
||||
k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts}})
|
||||
if len(bus.created) != 0 {
|
||||
t.Fatalf("created %v before the first view", bus.created)
|
||||
}
|
||||
if _, err := k.AskFirstView(0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts, log}})
|
||||
k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts, log}})
|
||||
if want := "[KV_m_drafts m_log]"; fmt.Sprint(bus.created) != want {
|
||||
t.Errorf("created %v, want %s", bus.created, want)
|
||||
}
|
||||
if want := "[created m drafts created m log]"; fmt.Sprint(out.events) != want {
|
||||
t.Errorf("said %v, want %s", out.events, want)
|
||||
}
|
||||
if b := out.bodies[1]; b.Binding.Stream != "m_log" || b.Binding.Subject != "mesh.bus.m.log" || b.Revision != 3 || b.Asked["m_log"].MaxBytes == 0 {
|
||||
t.Errorf("created carries %+v: the binding, the revision and the settings asked", b)
|
||||
}
|
||||
// In the field names of the seat mesh-broker's message said (mesh-contracts), beside binding and revision.
|
||||
raw, _ := json.Marshal(out.bodies[1])
|
||||
for _, w := range []string{`"module":"m"`, `"local-name":"log"`, `"storage-type":"stream"`, `"template":"log"`, `"bus-name":"m_log"`, `"binding":{`, `"revision":3`} {
|
||||
if !strings.Contains(string(raw), w) {
|
||||
t.Errorf("the body %s lacks %s", raw, w)
|
||||
}
|
||||
}
|
||||
}},
|
||||
{"a replay not newer is not applied", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
||||
k.Heard(Accepted{Module: "m", Digest: "d2", Revision: 9, BusState: []catalogue.BusStateEntry{drafts}})
|
||||
k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 4, BusState: nil})
|
||||
if a := k.Held()["m"]; a.Revision != 9 || len(a.Entries) != 1 || a.Made["drafts"].Orphaned {
|
||||
t.Errorf("held %+v: the replay at 4 took back what 9 asked", a)
|
||||
}
|
||||
}},
|
||||
{"a grant not composed yet waits, then is made", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
||||
bus.refuse["m_log"] = fmt.Errorf("%w: permissions violation", ErrRetry)
|
||||
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}})
|
||||
if w := k.Waiting(); len(w) != 1 || len(out.events) != 0 {
|
||||
t.Fatalf("waiting %v, said %v: waits, says nothing", w, out.events)
|
||||
}
|
||||
delete(bus.refuse, "m_log")
|
||||
k.CheckAll()
|
||||
if len(k.Waiting()) != 0 || fmt.Sprint(out.events) != "[created m log]" {
|
||||
t.Errorf("waiting %v, said %v: made at the next check", k.Waiting(), out.events)
|
||||
}
|
||||
}},
|
||||
{"a refused create is said and raised, cleared by a later create", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
||||
bus.refuse["KV_m_drafts"] = errors.New("insufficient resources")
|
||||
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
|
||||
if fmt.Sprint(out.events) != "[refused m drafts]" || fmt.Sprint(out.conds) != "[raised nats.refused.m_drafts]" ||
|
||||
!strings.Contains(out.bodies[0].Why, "insufficient resources") {
|
||||
t.Fatalf("said %v %v %+v", out.events, out.conds, out.bodies)
|
||||
}
|
||||
delete(bus.refuse, "KV_m_drafts")
|
||||
out.reset()
|
||||
k.CheckAll()
|
||||
if fmt.Sprint(out.events) != "[created m drafts]" || fmt.Sprint(out.conds) != "[cleared nats.refused.m_drafts]" {
|
||||
t.Errorf("said %v %v: created, the refusal cleared", out.events, out.conds)
|
||||
}
|
||||
}},
|
||||
{"an entry the keeper cannot plan is refused, naming why", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
||||
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{entry("a_b", catalogue.StorageKeyValue, catalogue.TemplateState, nil)}})
|
||||
if len(bus.created) != 0 || fmt.Sprint(out.events) != "[refused m a_b]" {
|
||||
t.Errorf("created %v, said %v", bus.created, out.events)
|
||||
}
|
||||
}},
|
||||
// CAT-N4: check and drift (ADR 0595 §5, §6): a difference that only grows is applied, said by nothing; any other
|
||||
// is said and raised, never fixed quietly, and cleared once the bus holds what is asked again.
|
||||
{"a held entry that grows is applied without a person", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
||||
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}})
|
||||
out.reset()
|
||||
bigger := entry("log", catalogue.StorageStream, catalogue.TemplateLog, map[string]string{"max-mib": "2048"})
|
||||
k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{bigger}})
|
||||
if fmt.Sprint(bus.updated) != "[m_log]" || bus.held["m_log"].MaxBytes != 2048*MiB || len(out.events)+len(out.conds) != 0 {
|
||||
t.Errorf("updated %v to %d, said %v %v: grown quietly", bus.updated, bus.held["m_log"].MaxBytes, out.events, out.conds)
|
||||
}
|
||||
}},
|
||||
{"a held entry that would shrink is a drift, said, raised and cleared", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
||||
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}})
|
||||
out.reset()
|
||||
smaller := entry("log", catalogue.StorageStream, catalogue.TemplateLog, map[string]string{"max-mib": "16"})
|
||||
k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{smaller}})
|
||||
if len(bus.updated) != 0 || fmt.Sprint(out.events) != "[drifted m log]" || fmt.Sprint(out.conds) != "[raised nats.drifted.m_log]" ||
|
||||
!strings.Contains(out.bodies[0].Why, "max-bytes asked 16777216, held 1073741824") {
|
||||
t.Fatalf("updated %v, said %v %v %+v: the drift said, nothing changed", bus.updated, out.events, out.conds, out.bodies)
|
||||
}
|
||||
if l := k.CheckAll(); len(l) != 1 || l[0].Drift == "" || len(out.events) != 1 {
|
||||
t.Errorf("check answers %+v and said %v: the drift once", l, out.events)
|
||||
}
|
||||
k.Heard(Accepted{Module: "m", Revision: 3, BusState: []catalogue.BusStateEntry{log}})
|
||||
if fmt.Sprint(out.events) != "[drifted m log drift-cleared m log]" || out.conds[len(out.conds)-1] != "cleared nats.drifted.m_log" {
|
||||
t.Errorf("said %v %v: cleared once asked as held", out.events, out.conds)
|
||||
}
|
||||
}},
|
||||
{"a bus that holds an entry the keeper never made is taken as made", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
||||
p, _ := PlanOf("m", drafts)
|
||||
bus.held["KV_m_drafts"] = p.Objects[0].Settings
|
||||
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
|
||||
if len(bus.created) != 0 || fmt.Sprint(out.events) != "[created m drafts]" {
|
||||
t.Errorf("created %v, said %v: its binding said, nothing made twice", bus.created, out.events)
|
||||
}
|
||||
}},
|
||||
// CAT-N5: orphans (design 25 § 12.2-12.3): an empty manifest-accepted asks nothing now, so what was made is an
|
||||
// orphan, said and raised, kept until a drop; asked again it is kept and its binding said again; a module missing
|
||||
// from the first view (set aside to wait) keeps its key; one never made leaves nothing behind.
|
||||
{"an empty verdict orphans what was made, and asking again keeps it", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
||||
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
|
||||
out.reset()
|
||||
k.Heard(Accepted{Module: "m", Revision: 2})
|
||||
a, held := k.Held()["m"]
|
||||
if !held || !a.Made["drafts"].Orphaned || len(bus.deleted) != 0 || fmt.Sprint(out.events) != "[orphaned m drafts]" ||
|
||||
fmt.Sprint(out.conds) != "[raised nats.orphaned.m_drafts]" {
|
||||
t.Fatalf("held %v %+v, deleted %v, said %v %v: orphaned, kept, raised", held, a, bus.deleted, out.events, out.conds)
|
||||
}
|
||||
k.Heard(Accepted{Module: "m", Revision: 3})
|
||||
k.Heard(Accepted{Module: "m", Revision: 4, BusState: []catalogue.BusStateEntry{drafts}})
|
||||
if fmt.Sprint(out.events) != "[orphaned m drafts created m drafts]" || out.conds[len(out.conds)-1] != "cleared nats.orphaned.m_drafts" {
|
||||
t.Errorf("said %v %v: orphaned once, then created again and cleared", out.events, out.conds)
|
||||
}
|
||||
}},
|
||||
{"a module missing from the first view keeps its key; one never made leaves none", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
||||
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
|
||||
k.View = func() (FirstView, error) { return FirstView{Revision: 5}, nil }
|
||||
if _, err := k.AskFirstView(0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a := k.Held()["m"]; len(a.Entries) != 1 || a.Made["drafts"].Orphaned {
|
||||
t.Errorf("held %+v: kept as asked, never orphaned by a first view", a)
|
||||
}
|
||||
bus.refuse["KV_n_x"] = errors.New("no")
|
||||
k.Heard(Accepted{Module: "n", Revision: 1, BusState: []catalogue.BusStateEntry{entry("x", catalogue.StorageKeyValue, catalogue.TemplateState, nil)}})
|
||||
k.Heard(Accepted{Module: "n", Revision: 2})
|
||||
if _, held := k.Held()["n"]; held {
|
||||
t.Errorf("n's key is still held: nothing of it was made, so its empty verdict ends it")
|
||||
}
|
||||
}},
|
||||
// CAT-N6: fill (ADR 0595 §6): an entry that refuses when full raises `full` at the raise mark and clears only
|
||||
// below the clear mark; one that drops its oldest is never read; the health names the pass.
|
||||
{"full at the raise mark, cleared only below the clear mark", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
||||
events := entry("events", catalogue.StorageStream, catalogue.TemplateEvents, map[string]string{"max-mib": "1"})
|
||||
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts, events}})
|
||||
out.reset()
|
||||
bus.fill["m_events"] = 1 * uint64(MiB)
|
||||
var steps []string
|
||||
for _, mib := range []uint64{40, 48, 46, 45, 44} {
|
||||
bus.fill["KV_m_drafts"] = mib * uint64(MiB)
|
||||
k.FillPass()
|
||||
steps = append(steps, fmt.Sprintf("%d:%d", mib, len(out.events)))
|
||||
}
|
||||
if fmt.Sprint(steps) != "[40:0 48:1 46:1 45:1 44:2]" || fmt.Sprint(out.events) != "[full m drafts full-cleared m drafts]" ||
|
||||
fmt.Sprint(out.conds) != "[raised nats.full.m_drafts cleared nats.full.m_drafts]" {
|
||||
t.Errorf("said %v %v: full at 48 of 64 MiB (75%%), kept at 46 and 45 (71%%, 70%%), cleared at 44 (68%%)", out.events, out.conds)
|
||||
}
|
||||
h := HealthOf(k.Status(), k.Held(), 0, k.now())
|
||||
if !h.Healthy || h.Pass == "none" {
|
||||
t.Errorf("health %+v: healthy, naming its pass", h)
|
||||
}
|
||||
}},
|
||||
{"the health is unhealthy past its bound, naming why", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
||||
at := k.now()
|
||||
k.FillPass()
|
||||
later := at.Add(4 * time.Minute)
|
||||
if h := HealthOf(k.Status(), nil, 0, later); h.Healthy || !strings.Contains(h.Why, "last fill pass is 4m0s old") {
|
||||
t.Errorf("health %+v: unhealthy, its pass too old", h)
|
||||
}
|
||||
if h := HealthOf(k.Status(), nil, 2, later); !strings.Contains(h.Why, "2 accepted verdict(s) wait") {
|
||||
t.Errorf("health %+v: names the verdicts waiting", h)
|
||||
}
|
||||
k.View = func() (FirstView, error) { return FirstView{}, errors.New("no responders") }
|
||||
k.mu.Lock()
|
||||
k.status.FirstView.Answered = time.Time{}
|
||||
k.mu.Unlock()
|
||||
if next, err := k.AskFirstView(10 * time.Second); err == nil || next != 20*time.Second {
|
||||
t.Fatalf("asked again in %s (%v): on the backoff", next, err)
|
||||
}
|
||||
if h := HealthOf(k.Status(), nil, 0, at); h.Healthy || !strings.Contains(h.Why, "mesh-delivery-indexer.indexer_bus_state") {
|
||||
t.Errorf("health %+v: unhealthy, naming the verb", h)
|
||||
}
|
||||
}},
|
||||
// CAT-N8: a person's requests: a drop removes an orphan alone, only on the operator's warrant binding exactly
|
||||
// that drop, after a snapshot, said dropped and recorded; a recreate rebuilds a drifted entry the same way.
|
||||
{"a drop waits for the warrant, refuses another act's, then acts after a snapshot", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
||||
store := k.Store.(*memStore)
|
||||
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts, log}})
|
||||
k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{log}})
|
||||
var acts []string
|
||||
k.Act = func(what, why, cause string) error { acts = append(acts, cause+": "+what); return nil }
|
||||
ask := asks.Ask{ID: "bus-1", Who: asks.Operator, Expires: k.now().Add(time.Hour)}
|
||||
var w *asks.Warrant
|
||||
k.Warrant = func(string) (*asks.Ask, *asks.Warrant, error) { return &ask, w, nil }
|
||||
put := func(r Request) Request {
|
||||
raw, _ := json.Marshal(r)
|
||||
_, _ = store.Put(StateRequests, r.ID, raw)
|
||||
k.ActOnRequests()
|
||||
raw, _, _, _ = store.Get(StateRequests, r.ID)
|
||||
var got Request
|
||||
_ = json.Unmarshal(raw, &got)
|
||||
return got
|
||||
}
|
||||
if got := put(Request{ID: "r0", Verb: VerbDrop, Module: "m", Local: "log", Ask: "bus-1", Caller: "person.jo"}); !strings.Contains(got.Error, "drop removes an orphan alone") {
|
||||
t.Errorf("dropping an asked entry: %+v", got)
|
||||
}
|
||||
if got := put(Request{ID: "r1", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo"}); !got.Done.IsZero() || len(bus.deleted) != 0 {
|
||||
t.Fatalf("acted with no warrant: %+v, deleted %v", got, bus.deleted)
|
||||
}
|
||||
binds := func(act asks.Act) string { d, _ := asks.ActDigest(act); return d }
|
||||
choose := func(act asks.Act) {
|
||||
ask.Options = []asks.Option{{ID: "go", Label: "Remove", Level: asks.Destroy, Binds: binds(act)}}
|
||||
w = &asks.Warrant{Ask: "bus-1", Asker: Module, Outcome: asks.OutcomeChosen, By: &asks.Person{Who: asks.Operator},
|
||||
Option: "go", Label: "Remove", Level: asks.Destroy, At: k.now(), AskDigest: ask.Digest()}
|
||||
}
|
||||
choose(asks.Act{"verb": "drop", "module": "m", "local": "log"})
|
||||
if got := put(Request{ID: "r1", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo"}); !strings.Contains(got.Error, "bound") || len(bus.deleted) != 0 {
|
||||
t.Fatalf("acted on a warrant for another act: %+v, deleted %v", got, bus.deleted)
|
||||
}
|
||||
// ADR 0626 §4: the keeper trusts nothing in the request; each of these is refused, said why, and deletes nothing.
|
||||
refused := func(r Request, why string) {
|
||||
t.Helper()
|
||||
if got := put(r); got.Done.IsZero() || !strings.Contains(got.Error, why) || len(bus.deleted) != 0 {
|
||||
t.Errorf("%s: ended %+v, deleted %v: refused for %q", r.ID, got, bus.deleted, why)
|
||||
}
|
||||
}
|
||||
refused(Request{ID: "r1a", Verb: VerbDrop, Module: "m", Local: "drafts", Caller: "person.jo"}, "names no ask")
|
||||
choose(asks.Act{"verb": "drop", "module": "m", "local": "drafts"})
|
||||
w.Level, w.Label, ask.Options[0].Level = asks.Approve, "Keep", asks.Approve
|
||||
w.AskDigest = ask.Digest()
|
||||
refused(Request{ID: "r1b", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo"}, "nothing is removed")
|
||||
choose(asks.Act{"verb": "drop", "module": "m", "local": "drafts"})
|
||||
w.Asker = "mesh-grants"
|
||||
refused(Request{ID: "r1c", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo"}, "no warrant")
|
||||
refused(Request{ID: "r1d", Verb: VerbCreate, Module: "m", Local: "drafts", Caller: "person.jo"}, "no accepted manifest of m asks for drafts")
|
||||
choose(asks.Act{"verb": "drop", "module": "m", "local": "drafts"})
|
||||
bus.refuse["snapshot"] = errors.New("no space")
|
||||
refused(Request{ID: "r1e", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo"}, "the snapshot before drop failed, so nothing was removed")
|
||||
delete(bus.refuse, "snapshot")
|
||||
bus.snaps = nil
|
||||
out.reset()
|
||||
got := put(Request{ID: "r2", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo", Why: "unused"})
|
||||
if got.Error != "" || fmt.Sprint(bus.snaps) != "[[KV_m_drafts]]" || fmt.Sprint(bus.deleted) != "[KV_m_drafts]" ||
|
||||
fmt.Sprint(out.events) != "[dropped m drafts]" || len(acts) != 1 || !strings.Contains(acts[0], "nats_drop: dropped drafts of m") ||
|
||||
!strings.Contains(acts[0], "person.jo") {
|
||||
t.Errorf("ended %+v, snapshots %v, deleted %v, said %v, recorded %v", got, bus.snaps, bus.deleted, out.events, acts)
|
||||
}
|
||||
if _, made := k.Held()["m"].Made["drafts"]; made {
|
||||
t.Error("the dropped entry is still held")
|
||||
}
|
||||
}},
|
||||
{"a recreate rebuilds a drifted entry after a snapshot", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
||||
store := k.Store.(*memStore)
|
||||
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}})
|
||||
k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{entry("log", catalogue.StorageStream, catalogue.TemplateLog, map[string]string{"max-mib": "16"})}})
|
||||
binds, _ := asks.ActDigest(asks.Act{"verb": "recreate", "module": "m", "local": "log"})
|
||||
ask := asks.Ask{ID: "bus-2", Who: asks.Operator, Expires: k.now().Add(time.Hour), Options: []asks.Option{{ID: "go", Label: "Rebuild", Level: asks.Destroy, Binds: binds}}}
|
||||
w := &asks.Warrant{Ask: "bus-2", Asker: Module, Outcome: asks.OutcomeChosen, By: &asks.Person{Who: asks.Operator},
|
||||
Option: "go", Label: "Rebuild", Level: asks.Destroy, At: k.now(), AskDigest: ask.Digest()}
|
||||
k.Warrant = func(string) (*asks.Ask, *asks.Warrant, error) { return &ask, w, nil }
|
||||
out.reset()
|
||||
raw, _ := json.Marshal(Request{ID: "r3", Verb: VerbRecreate, Module: "m", Local: "log", Ask: "bus-2"})
|
||||
_, _ = store.Put(StateRequests, "r3", raw)
|
||||
k.ActOnRequests()
|
||||
if bus.held["m_log"].MaxBytes != 16*MiB || fmt.Sprint(out.events) != "[recreated m log]" ||
|
||||
fmt.Sprint(out.conds) != "[cleared nats.drifted.m_log]" || k.Held()["m"].Made["log"].Drift != "" {
|
||||
t.Errorf("held %+v, said %v %v: rebuilt as asked, the drift cleared", bus.held["m_log"], out.events, out.conds)
|
||||
}
|
||||
}},
|
||||
// CAT-N2b: nats_first_view_again asks the first view at once, by hand, recorded as a hand act.
|
||||
{"the first view asked again by hand", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
||||
store := k.Store.(*memStore)
|
||||
var acts []string
|
||||
k.Act = func(what, why, cause string) error { acts = append(acts, cause+": "+what); return nil }
|
||||
k.View = func() (FirstView, error) {
|
||||
return FirstView{Revision: 8, Modules: []Accepted{{Module: "m", Revision: 8, BusState: []catalogue.BusStateEntry{drafts}}}}, nil
|
||||
}
|
||||
raw, _ := json.Marshal(Request{ID: "r9", Verb: VerbFirstViewAgain, Why: "a verdict thought missed", Caller: "person.jo"})
|
||||
_, _ = store.Put(StateRequests, "r9", raw)
|
||||
k.ActOnRequests()
|
||||
raw, _, _, _ = store.Get(StateRequests, "r9")
|
||||
var got Request
|
||||
_ = json.Unmarshal(raw, &got)
|
||||
if got.Outcome != "the first view answered at revision 8" || fmt.Sprint(bus.created) != "[KV_m_drafts]" ||
|
||||
len(acts) != 1 || !strings.Contains(acts[0], "nats_first_view_again: asked mesh-delivery-indexer.indexer_bus_state again by hand") {
|
||||
t.Errorf("ended %+v, created %v, recorded %v", got, bus.created, acts)
|
||||
}
|
||||
}},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
k, bus, _, out := newKeeper(FirstView{Revision: 1})
|
||||
if _, err := k.AskFirstView(0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
row.run(t, k, bus, out)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,349 @@
|
||||
// Package keeper is the bus module's keeper (novox/hq design 25 § 12.2-12.3, ADR 0595 §4-§6): it creates the bus state
|
||||
// every accepted manifest asks for in `requires`, compares what the bus holds with what was asked, finds orphans and
|
||||
// reads the fill of what refuses when full, and says each on the bus module's own events. It reads nothing of another
|
||||
// part's state: what to create comes on the indexer's manifest-accepted, and its first view from the indexer's tool
|
||||
// indexer_bus_state. Everything it touches it reaches through the interfaces in keeper.go; this file is pure.
|
||||
package keeper
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
)
|
||||
|
||||
// The kinds of bus object an entry is made of: a stream, a key-value bucket (stream KV_<name>) and an object store
|
||||
// (stream OBJ_<name>).
|
||||
const (
|
||||
KindStream = "stream"
|
||||
KindBucket = "key-value"
|
||||
KindObjects = "object-store"
|
||||
)
|
||||
|
||||
// Separator joins a module's name and a local name into a bus name (design 25 § 12.1: `<module>_<local>`); a local
|
||||
// name holding it is refused, so no two modules' names meet.
|
||||
const Separator = "_"
|
||||
|
||||
// Fixed settings of the templates (design 25 § 12.1's table): never options.
|
||||
const (
|
||||
MiB = int64(1) << 20
|
||||
StateCap = 64 * MiB
|
||||
ValueMost = 256 << 10
|
||||
EntryMost = 256 << 10
|
||||
DefaultCapMiB = 1024
|
||||
EventsDays = 7
|
||||
)
|
||||
|
||||
// Settings are one bus object's settings, as asked and as held: what check compares.
|
||||
type Settings struct {
|
||||
// MaxBytes is its cap in bytes; MaxAgeSeconds its age, 0 for none.
|
||||
MaxBytes int64 `json:"max-bytes"`
|
||||
MaxAgeSeconds int64 `json:"max-age-seconds,omitempty"`
|
||||
// History is how many values a key keeps (a bucket's), or how many messages a subject keeps (a snapshot's); 0 none.
|
||||
History int64 `json:"history,omitempty"`
|
||||
// Discard is what it does when full: "new" (refuses) or "old" (drops the oldest).
|
||||
Discard string `json:"discard"`
|
||||
// WorkQueue says a message is removed once taken.
|
||||
WorkQueue bool `json:"work-queue,omitempty"`
|
||||
// MaxMsgSize is the largest message (a value, a log entry), 0 none.
|
||||
MaxMsgSize int64 `json:"max-msg-size,omitempty"`
|
||||
// DuplicatesSeconds is a work queue's duplicate window.
|
||||
DuplicatesSeconds int64 `json:"duplicates-seconds,omitempty"`
|
||||
// KeyTTL says a bucket's key may be given a lifetime of its own.
|
||||
KeyTTL bool `json:"key-ttl,omitempty"`
|
||||
}
|
||||
|
||||
// The words of Discard.
|
||||
const (
|
||||
DiscardNew = "new"
|
||||
DiscardOld = "old"
|
||||
)
|
||||
|
||||
// Refuses says the object refuses a new message when full: the ones the fill pass reads (ADR 0595 §6).
|
||||
func (s Settings) Refuses() bool { return s.Discard == DiscardNew }
|
||||
|
||||
// Object is one bus object an entry is made of: its kind, its name as the kind names it (a bucket by its bucket name,
|
||||
// a stream by its stream name), its subjects for a stream, and its settings.
|
||||
type Object struct {
|
||||
Kind string `json:"kind"`
|
||||
Name string `json:"name"`
|
||||
Subjects []string `json:"subjects,omitempty"`
|
||||
Settings Settings `json:"settings"`
|
||||
}
|
||||
|
||||
// Stream is the stream the bus keeps the object on: KV_<bucket>, OBJ_<store>, or the stream itself.
|
||||
func (o Object) Stream() string {
|
||||
switch o.Kind {
|
||||
case KindBucket:
|
||||
return "KV_" + o.Name
|
||||
case KindObjects:
|
||||
return "OBJ_" + o.Name
|
||||
}
|
||||
return o.Name
|
||||
}
|
||||
|
||||
// Plan is what one entry is on the bus: its binding (the names the module receives) and the objects it is made of.
|
||||
type Plan struct {
|
||||
Module string `json:"module"`
|
||||
Local string `json:"local"`
|
||||
Storage string `json:"storage"`
|
||||
Template string `json:"template,omitempty"`
|
||||
Binding membership.BusBinding `json:"binding"`
|
||||
Objects []Object `json:"objects"`
|
||||
Entry catalogue.BusStateEntry `json:"entry"`
|
||||
}
|
||||
|
||||
// BusName is `<module>_<local>`, the name the mesh gives an entry on the bus (design 25 § 12.1).
|
||||
func BusName(module, local string) string { return module + Separator + local }
|
||||
|
||||
// StreamSubject is the subject a stream entry's messages are under (`mesh.bus.<module>.<local>`, messages on its `.>`).
|
||||
func StreamSubject(module, local string) string { return "mesh.bus." + module + "." + local }
|
||||
|
||||
// PlanOf is the plan of one entry of module, or why it cannot be made: a local name holding the separator, a storage
|
||||
// type and template that do not go together, an option of the wrong shape. The judge refuses each of these first
|
||||
// (ADR 0595 §8); this is the keeper's own guard, so a manifest judged by an older judge is refused, never half-made.
|
||||
func PlanOf(module string, e catalogue.BusStateEntry) (Plan, error) {
|
||||
if e.Name == "" || strings.Contains(e.Name, Separator) || strings.ContainsAny(e.Name, ".*> ") {
|
||||
return Plan{}, fmt.Errorf("the local name %q is empty or holds %q, a dot, a wildcard or a space: the mesh names it %s on the bus, and two modules' names must never meet", e.Name, Separator, BusName(module, "<local>"))
|
||||
}
|
||||
name := BusName(module, e.Name)
|
||||
p := Plan{Module: module, Local: e.Name, Storage: e.Storage, Template: e.Template, Entry: e,
|
||||
Binding: membership.BusBinding{Local: e.Name, Storage: e.Storage, Template: e.Template}}
|
||||
opts := options{raw: e.Options}
|
||||
conf := options{raw: e.Configuration}
|
||||
streamOf := func(s Settings) Object {
|
||||
subject := StreamSubject(module, e.Name)
|
||||
p.Binding.Stream, p.Binding.Subject = name, subject
|
||||
return Object{Kind: KindStream, Name: name, Subjects: []string{subject + ".>"}, Settings: s}
|
||||
}
|
||||
switch {
|
||||
case e.Storage == catalogue.StorageKeyValue && e.Template == catalogue.TemplateState:
|
||||
s, err := stateSettings(opts)
|
||||
if err != nil {
|
||||
return Plan{}, err
|
||||
}
|
||||
p.Binding.Bucket = name
|
||||
p.Objects = []Object{{Kind: KindBucket, Name: name, Settings: s}}
|
||||
case e.Storage == catalogue.StorageStream && e.Template == catalogue.TemplateEvents:
|
||||
age, err := opts.int(catalogue.OptionAgeDays, EventsDays)
|
||||
if err != nil {
|
||||
return Plan{}, err
|
||||
}
|
||||
capMiB, err := opts.int(catalogue.OptionCapMiB, DefaultCapMiB)
|
||||
if err != nil {
|
||||
return Plan{}, err
|
||||
}
|
||||
p.Objects = []Object{streamOf(Settings{MaxBytes: capMiB * MiB, MaxAgeSeconds: days(age), Discard: DiscardOld})}
|
||||
case e.Storage == catalogue.StorageStream && e.Template == catalogue.TemplateWorkQueue:
|
||||
capMiB, err := opts.int(catalogue.OptionCapMiB, DefaultCapMiB)
|
||||
if err != nil {
|
||||
return Plan{}, err
|
||||
}
|
||||
age, err := opts.int(catalogue.OptionAgeDays, 0)
|
||||
if err != nil {
|
||||
return Plan{}, err
|
||||
}
|
||||
dup, err := opts.int(catalogue.OptionDuplicates, 0)
|
||||
if err != nil {
|
||||
return Plan{}, err
|
||||
}
|
||||
p.Objects = []Object{streamOf(Settings{MaxBytes: capMiB * MiB, MaxAgeSeconds: days(age), Discard: DiscardNew,
|
||||
WorkQueue: true, DuplicatesSeconds: dup})}
|
||||
case e.Storage == catalogue.StorageStream && e.Template == catalogue.TemplateLog:
|
||||
s, err := logSettings(opts)
|
||||
if err != nil {
|
||||
return Plan{}, err
|
||||
}
|
||||
p.Objects = []Object{streamOf(s)}
|
||||
case e.Storage == catalogue.StorageObjectStore && e.Template == catalogue.TemplateObjects:
|
||||
capMiB, err := opts.int(catalogue.OptionCapMiB, 0)
|
||||
if err != nil {
|
||||
return Plan{}, err
|
||||
}
|
||||
if capMiB <= 0 {
|
||||
return Plan{}, fmt.Errorf("objects %q gives no %s, which the template objects requires", e.Name, catalogue.OptionCapMiB)
|
||||
}
|
||||
p.Binding.ObjectStore = name
|
||||
p.Objects = []Object{{Kind: KindObjects, Name: name, Settings: Settings{MaxBytes: capMiB * MiB, Discard: DiscardNew}}}
|
||||
case e.Storage == catalogue.StorageLedger && e.Template == catalogue.TemplateLedger:
|
||||
history, err := logSettings(opts)
|
||||
if err != nil {
|
||||
return Plan{}, err
|
||||
}
|
||||
// Keeping days only in a ledger's own configuration (design 25 § 12.1).
|
||||
age, err := conf.int(catalogue.OptionAgeDays, 0)
|
||||
if err != nil {
|
||||
return Plan{}, err
|
||||
}
|
||||
history.MaxAgeSeconds = days(age)
|
||||
state, err := stateSettings(options{})
|
||||
if err != nil {
|
||||
return Plan{}, err
|
||||
}
|
||||
subject := buscompose.LedgerHistorySubject(module, e.Name)
|
||||
snapshot := buscompose.LedgerSnapshotSubject(module, e.Name)
|
||||
p.Binding.History, p.Binding.Subject = name, subject
|
||||
p.Binding.State = name + "_state"
|
||||
p.Binding.Snapshot = snapshot
|
||||
p.Binding.Types = e.Types
|
||||
p.Objects = []Object{
|
||||
{Kind: KindStream, Name: name, Subjects: []string{subject + ".>"}, Settings: history},
|
||||
{Kind: KindBucket, Name: p.Binding.State, Settings: state},
|
||||
}
|
||||
var indexes []string
|
||||
for _, t := range e.Types {
|
||||
for _, ix := range t.Indexes {
|
||||
indexes = append(indexes, ix.Name)
|
||||
}
|
||||
}
|
||||
sort.Strings(indexes)
|
||||
for _, ix := range indexes {
|
||||
if p.Binding.Indexes == nil {
|
||||
p.Binding.Indexes = map[string]string{}
|
||||
}
|
||||
if _, twice := p.Binding.Indexes[ix]; twice {
|
||||
continue
|
||||
}
|
||||
p.Binding.Indexes[ix] = name + "_index-" + ix
|
||||
p.Objects = append(p.Objects, Object{Kind: KindBucket, Name: p.Binding.Indexes[ix], Settings: state})
|
||||
}
|
||||
// The snapshot: the latest state per instance, never expiring (ADR 0597 §8), one message per subject.
|
||||
p.Objects = append(p.Objects, Object{Kind: KindStream, Name: name + "_snapshot", Subjects: []string{snapshot + ".>"},
|
||||
Settings: Settings{MaxBytes: history.MaxBytes, History: 1, Discard: DiscardOld}})
|
||||
case e.Storage == catalogue.StorageStream && e.Template == "":
|
||||
// A bare stream: its own configuration, which must give when full, a cap and an age (design 25 § 12.1).
|
||||
var full string
|
||||
if raw, ok := conf.raw[catalogue.ConfigWhenFull]; ok {
|
||||
_ = json.Unmarshal(raw, &full)
|
||||
}
|
||||
capMiB, err := conf.int(catalogue.OptionCapMiB, 0)
|
||||
if err != nil {
|
||||
return Plan{}, err
|
||||
}
|
||||
age, err := conf.int(catalogue.OptionAgeDays, -1)
|
||||
if err != nil {
|
||||
return Plan{}, err
|
||||
}
|
||||
discard := map[string]string{catalogue.WhenFullRefuse: DiscardNew, catalogue.WhenFullDropOldest: DiscardOld}[full]
|
||||
if discard == "" || capMiB <= 0 || age < 0 {
|
||||
return Plan{}, fmt.Errorf("the bare stream %q must give %s (refuse or drop-oldest), %s and %s in its configuration", e.Name,
|
||||
catalogue.ConfigWhenFull, catalogue.OptionCapMiB, catalogue.OptionAgeDays)
|
||||
}
|
||||
p.Objects = []Object{streamOf(Settings{MaxBytes: capMiB * MiB, MaxAgeSeconds: days(age), Discard: discard})}
|
||||
default:
|
||||
return Plan{}, fmt.Errorf("the storage type %q with the template %q is none the bus module makes (design 25 § 12.1)", e.Storage, e.Template)
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func stateSettings(o options) (Settings, error) {
|
||||
history, err := o.int(catalogue.OptionPastValues, 1)
|
||||
if err != nil {
|
||||
return Settings{}, err
|
||||
}
|
||||
ttl, err := o.int(catalogue.OptionTimeToLive, 0)
|
||||
if err != nil {
|
||||
return Settings{}, err
|
||||
}
|
||||
keyTTL, err := o.bool(catalogue.OptionLifetimePerKey)
|
||||
if err != nil {
|
||||
return Settings{}, err
|
||||
}
|
||||
if perMachine, err := o.bool(catalogue.OptionPerMachine); err != nil || perMachine {
|
||||
if err != nil {
|
||||
return Settings{}, err
|
||||
}
|
||||
return Settings{}, fmt.Errorf("the option %s (one bucket per machine) is not made by the bus module yet: the module keeps one bucket", catalogue.OptionPerMachine)
|
||||
}
|
||||
return Settings{MaxBytes: StateCap, MaxAgeSeconds: ttl, History: history, Discard: DiscardNew, MaxMsgSize: ValueMost, KeyTTL: keyTTL}, nil
|
||||
}
|
||||
|
||||
func logSettings(o options) (Settings, error) {
|
||||
capMiB, err := o.int(catalogue.OptionCapMiB, DefaultCapMiB)
|
||||
if err != nil {
|
||||
return Settings{}, err
|
||||
}
|
||||
age, err := o.int(catalogue.OptionAgeDays, 0)
|
||||
if err != nil {
|
||||
return Settings{}, err
|
||||
}
|
||||
return Settings{MaxBytes: capMiB * MiB, MaxAgeSeconds: days(age), Discard: DiscardNew, MaxMsgSize: EntryMost}, nil
|
||||
}
|
||||
|
||||
func days(n int64) int64 {
|
||||
if n <= 0 {
|
||||
return 0
|
||||
}
|
||||
return int64((time.Duration(n) * 24 * time.Hour) / time.Second)
|
||||
}
|
||||
|
||||
// options reads a template's options as the manifest wrote them.
|
||||
type options struct{ raw map[string]json.RawMessage }
|
||||
|
||||
func (o options) int(name string, fallback int64) (int64, error) {
|
||||
raw, ok := o.raw[name]
|
||||
if !ok {
|
||||
return fallback, nil
|
||||
}
|
||||
n, err := strconv.ParseInt(strings.TrimSpace(string(raw)), 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("the option %s is %s, not a whole number", name, raw)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (o options) bool(name string) (bool, error) {
|
||||
raw, ok := o.raw[name]
|
||||
if !ok {
|
||||
return false, nil
|
||||
}
|
||||
var b bool
|
||||
if err := json.Unmarshal(raw, &b); err != nil {
|
||||
return false, fmt.Errorf("the option %s is %s, not true or false", name, raw)
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// Grows is whether held reaches asked by growing alone (a larger cap, a longer age, more past values), so the keeper
|
||||
// applies it without a person (ADR 0595 §6); false for any other difference, which is a drift a person decides. Equal
|
||||
// settings grow by nothing and are no difference (Differs).
|
||||
func Grows(asked, held Settings) bool {
|
||||
if asked.Discard != held.Discard || asked.WorkQueue != held.WorkQueue || asked.MaxMsgSize != held.MaxMsgSize ||
|
||||
asked.DuplicatesSeconds != held.DuplicatesSeconds || asked.KeyTTL != held.KeyTTL {
|
||||
return false
|
||||
}
|
||||
return asked.MaxBytes >= held.MaxBytes && longer(asked.MaxAgeSeconds, held.MaxAgeSeconds) && asked.History >= held.History
|
||||
}
|
||||
|
||||
// longer is whether age a keeps at least as long as b, 0 being for ever.
|
||||
func longer(a, b int64) bool {
|
||||
if a == 0 {
|
||||
return true
|
||||
}
|
||||
return b != 0 && a >= b
|
||||
}
|
||||
|
||||
// Differs says how held differs from asked, field by field, or "" when it does not.
|
||||
func Differs(asked, held Settings) string {
|
||||
var out []string
|
||||
add := func(name string, a, h any) {
|
||||
if fmt.Sprint(a) != fmt.Sprint(h) {
|
||||
out = append(out, fmt.Sprintf("%s asked %v, held %v", name, a, h))
|
||||
}
|
||||
}
|
||||
add("max-bytes", asked.MaxBytes, held.MaxBytes)
|
||||
add("max-age-seconds", asked.MaxAgeSeconds, held.MaxAgeSeconds)
|
||||
add("history", asked.History, held.History)
|
||||
add("discard", asked.Discard, held.Discard)
|
||||
add("work-queue", asked.WorkQueue, held.WorkQueue)
|
||||
add("max-msg-size", asked.MaxMsgSize, held.MaxMsgSize)
|
||||
add("duplicates-seconds", asked.DuplicatesSeconds, held.DuplicatesSeconds)
|
||||
add("key-ttl", asked.KeyTTL, held.KeyTTL)
|
||||
return strings.Join(out, "; ")
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
package keeper
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
)
|
||||
|
||||
// A person's request (design 25 § 12.3; CAT-N8): the bus module's tools hold no grant of the seat's acting verbs, so
|
||||
// each hands its request to the keeper through the keeper's own state requests, stamped with the caller the tool runner
|
||||
// proved (never a name the request's body gives), and reads the keeper's answer back from the same key. create and
|
||||
// apply act at once; recreate and drop only once the operator's warrant on the ask the tool raised is held, after a
|
||||
// snapshot of the entry's streams. Every act is recorded as a hand act.
|
||||
|
||||
// The verbs a request names.
|
||||
const (
|
||||
VerbCreate = "create"
|
||||
VerbApply = "apply"
|
||||
VerbRecreate = "recreate"
|
||||
VerbDrop = "drop"
|
||||
// VerbFirstViewAgain asks the indexer's first view at once, by hand (design 25 § 12.2), beside the backoff.
|
||||
VerbFirstViewAgain = "first-view-again"
|
||||
)
|
||||
|
||||
// Request is one person's request, the value of its key in StateRequests.
|
||||
type Request struct {
|
||||
ID string `json:"id"`
|
||||
Verb string `json:"verb"`
|
||||
Module string `json:"module"`
|
||||
Local string `json:"local"`
|
||||
Why string `json:"why"`
|
||||
// Caller is who asked, as the tool runner proved it (stdio.CallerOf), written by the tool.
|
||||
Caller string `json:"caller"`
|
||||
At time.Time `json:"at"`
|
||||
// Ask is the id of the operator's ask the tool raised for a recreate or a drop; the keeper acts only on its warrant.
|
||||
Ask string `json:"ask,omitempty"`
|
||||
// Done is when the keeper ended it, Outcome what it did, Error why not.
|
||||
Done time.Time `json:"done,omitempty"`
|
||||
Outcome string `json:"outcome,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Snapshot string `json:"snapshot,omitempty"`
|
||||
}
|
||||
|
||||
// Act is what an option of the operator's ask binds for a recreate or a drop: the verb and the entry (asks.ActDigest).
|
||||
func (r Request) Act() asks.Act {
|
||||
return asks.Act{"verb": r.Verb, "module": r.Module, "local": r.Local}
|
||||
}
|
||||
|
||||
// ErrNoWarrant is a request that waits for the operator's word: kept, acted on once the warrant is held.
|
||||
var ErrNoWarrant = errors.New("the operator has not answered yet")
|
||||
|
||||
// ActOnRequests acts on every request not ended yet, oldest first, and writes each one's end back to its key.
|
||||
func (k *Keeper) ActOnRequests() {
|
||||
keys, err := k.Store.Keys(StateRequests)
|
||||
if err != nil {
|
||||
k.logf("the persons' requests cannot be read: %v", err)
|
||||
return
|
||||
}
|
||||
for _, key := range keys {
|
||||
raw, _, found, err := k.Store.Get(StateRequests, key)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
var r Request
|
||||
if json.Unmarshal(raw, &r) != nil || !r.Done.IsZero() {
|
||||
continue
|
||||
}
|
||||
outcome, err := k.act(&r)
|
||||
if errors.Is(err, ErrNoWarrant) {
|
||||
continue
|
||||
}
|
||||
r.Done, r.Outcome = k.now(), outcome
|
||||
if err != nil {
|
||||
r.Error = err.Error()
|
||||
}
|
||||
data, _ := json.Marshal(r)
|
||||
if _, err := k.Store.Put(StateRequests, key, data); err != nil {
|
||||
k.logf("the end of the request %s cannot be written: %v", key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (k *Keeper) act(r *Request) (string, error) {
|
||||
if r.Verb == VerbFirstViewAgain {
|
||||
_, err := k.AskFirstView(0)
|
||||
s := k.Status().FirstView
|
||||
k.record(r, fmt.Sprintf("asked %s.%s again by hand: answered %v at revision %d", s.Owner, s.Verb, err == nil, s.Revision))
|
||||
if err != nil {
|
||||
return "asked, not answered", err
|
||||
}
|
||||
return fmt.Sprintf("the first view answered at revision %d", s.Revision), nil
|
||||
}
|
||||
if !k.Viewed() {
|
||||
return "", fmt.Errorf("%w: the keeper holds no first view yet, so it acts on nothing", ErrNoWarrant)
|
||||
}
|
||||
held, _, found := k.asked(r.Module)
|
||||
if !found {
|
||||
return "nothing done", fmt.Errorf("the bus module holds no entry of %s", r.Module)
|
||||
}
|
||||
m, made := held.Made[r.Local]
|
||||
asked := false
|
||||
for _, e := range held.Entries {
|
||||
asked = asked || e.Name == r.Local
|
||||
}
|
||||
switch r.Verb {
|
||||
case VerbCreate, VerbApply:
|
||||
if !asked {
|
||||
return "nothing done", fmt.Errorf("no accepted manifest of %s asks for %s: the bus module creates only what one asks", r.Module, r.Local)
|
||||
}
|
||||
for _, e := range held.Entries {
|
||||
if e.Name == r.Local {
|
||||
k.makeEntry(&held, e)
|
||||
}
|
||||
}
|
||||
k.write(held)
|
||||
m = held.Made[r.Local]
|
||||
k.record(r, fmt.Sprintf("%s of %s's %s: created %v, drift %q, waiting %q", r.Verb, r.Module, r.Local, m.Created, m.Drift, m.Waiting))
|
||||
if m.Refused != "" || m.Waiting != "" {
|
||||
return "not made", fmt.Errorf("%s%s", m.Refused, m.Waiting)
|
||||
}
|
||||
return fmt.Sprintf("%s: the bus holds %s as asked%s", r.Verb, BusName(r.Module, r.Local), map[bool]string{true: ", but for a drift a person decides: " + m.Drift}[m.Drift != ""]), nil
|
||||
case VerbRecreate, VerbDrop:
|
||||
default:
|
||||
return "nothing done", fmt.Errorf("%q is no verb a person asks the bus module for: create, apply, recreate or drop", r.Verb)
|
||||
}
|
||||
if !made || !m.Created {
|
||||
return "nothing done", fmt.Errorf("the bus module made no %s of %s", r.Local, r.Module)
|
||||
}
|
||||
if r.Verb == VerbDrop && !m.Orphaned {
|
||||
return "nothing done", fmt.Errorf("%s of %s is asked for by its manifest: drop removes an orphan alone", r.Local, r.Module)
|
||||
}
|
||||
if r.Verb == VerbRecreate && m.Orphaned {
|
||||
return "nothing done", fmt.Errorf("%s of %s is an orphan: drop it, or ask for it in a manifest again", r.Local, r.Module)
|
||||
}
|
||||
if err := k.warranted(r); err != nil {
|
||||
return "nothing done", err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||||
defer cancel()
|
||||
var streams []string
|
||||
for _, o := range m.Plan.Objects {
|
||||
streams = append(streams, o.Stream())
|
||||
}
|
||||
path, err := k.Bus.Snapshot(ctx, streams)
|
||||
if err != nil {
|
||||
return "nothing done", fmt.Errorf("the snapshot before %s failed, so nothing was removed: %w", r.Verb, err)
|
||||
}
|
||||
r.Snapshot = path
|
||||
for _, o := range m.Plan.Objects {
|
||||
if err := k.Bus.Delete(ctx, o); err != nil {
|
||||
return "partly done", fmt.Errorf("%s could not be removed after its snapshot at %s: %w", o.Stream(), path, err)
|
||||
}
|
||||
}
|
||||
if r.Verb == VerbDrop {
|
||||
delete(held.Made, r.Local)
|
||||
k.write(held)
|
||||
k.emit(EventDropped, m.Plan, held.Revision, r.Why, nil, nil)
|
||||
k.clear(conditionOrphaned, r.Module, r.Local)
|
||||
k.record(r, fmt.Sprintf("dropped %s of %s on the operator's warrant (snapshot %s)", r.Local, r.Module, path))
|
||||
return fmt.Sprintf("dropped %s, its snapshot at %s", BusName(r.Module, r.Local), path), nil
|
||||
}
|
||||
for _, o := range m.Plan.Objects {
|
||||
if err := k.Bus.Create(ctx, o); err != nil {
|
||||
m.Created, m.Refused = false, fmt.Sprintf("%s: %v", o.Stream(), err)
|
||||
held.Made[r.Local] = m
|
||||
k.write(held)
|
||||
return "partly done", fmt.Errorf("%s was removed and could not be made again (its snapshot at %s): %w", o.Stream(), path, err)
|
||||
}
|
||||
}
|
||||
wasDrift := m.Drift != ""
|
||||
m.Drift = ""
|
||||
held.Made[r.Local] = m
|
||||
k.write(held)
|
||||
k.emit(EventRecreated, m.Plan, held.Revision, r.Why, nil, nil)
|
||||
if wasDrift {
|
||||
k.clear(conditionDrifted, r.Module, r.Local)
|
||||
}
|
||||
k.record(r, fmt.Sprintf("recreated %s of %s on the operator's warrant (snapshot %s)", r.Local, r.Module, path))
|
||||
return fmt.Sprintf("recreated %s as asked, its old data's snapshot at %s", BusName(r.Module, r.Local), path), nil
|
||||
}
|
||||
|
||||
// warranted holds a recreate or a drop to the operator's warrant on the ask its tool raised: chosen by the operator,
|
||||
// on that ask, binding exactly this act.
|
||||
func (k *Keeper) warranted(r *Request) error {
|
||||
if r.Ask == "" || k.Warrant == nil {
|
||||
return fmt.Errorf("a %s acts only on the operator's warrant, and this request names no ask", r.Verb)
|
||||
}
|
||||
ask, w, err := k.Warrant(r.Ask)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: the router's record of the ask %s cannot be read now: %v", ErrNoWarrant, r.Ask, err)
|
||||
}
|
||||
if ask == nil || w == nil {
|
||||
return ErrNoWarrant
|
||||
}
|
||||
o, err := w.For(Module, *ask)
|
||||
if err != nil {
|
||||
return fmt.Errorf("no warrant: %w", err)
|
||||
}
|
||||
if o.Level != asks.Destroy {
|
||||
return fmt.Errorf("the operator chose %s, so nothing is removed", o.Label)
|
||||
}
|
||||
return o.Performs(r.Act())
|
||||
}
|
||||
|
||||
func (k *Keeper) record(r *Request, what string) {
|
||||
if k.Act == nil {
|
||||
return
|
||||
}
|
||||
if err := k.Act(what+", asked by "+r.Caller, r.Why, "nats_"+strings.ReplaceAll(r.Verb, "-", "_")); err != nil {
|
||||
k.logf("the hand act %q could not be recorded: %v", what, err)
|
||||
}
|
||||
}
|
||||
+115
-7
@@ -24,14 +24,24 @@
|
||||
"claims": [
|
||||
{
|
||||
"name": "mesh-broker",
|
||||
"scope": "mesh"
|
||||
"scope": "mesh",
|
||||
"serves": [
|
||||
"list",
|
||||
"show",
|
||||
"check",
|
||||
"orphans"
|
||||
]
|
||||
}
|
||||
],
|
||||
"bus-users": "/var/lib/nats-module/conf/accounts.conf",
|
||||
"core-credentials": true,
|
||||
"own-secrets": {
|
||||
"broker": "${dir:mesh-state}/broker",
|
||||
"broker-nats-tools": "${dir:tools-secret}/broker-nats-tools"
|
||||
"broker-nats-tools": "${dir:tools-secret}/broker-nats-tools",
|
||||
"broker-keeper": {
|
||||
"path": "${dir:keeper-state}/broker-keeper",
|
||||
"owner": "nats-keeper"
|
||||
}
|
||||
},
|
||||
"capabilities": [
|
||||
"container-runtime"
|
||||
@@ -43,16 +53,51 @@
|
||||
"bucket-expiry-set"
|
||||
],
|
||||
"invokes": [
|
||||
"seat:mesh-hand-acts.hand-act"
|
||||
"seat:mesh-hand-acts.hand-act",
|
||||
"mesh-delivery-indexer.indexer_bus_state"
|
||||
],
|
||||
"consumes": [
|
||||
"mesh-delivery-indexer.manifest-accepted"
|
||||
],
|
||||
"uses": [
|
||||
"operator-channel"
|
||||
],
|
||||
"streams": [
|
||||
{
|
||||
"name": "broker-events",
|
||||
"events": [
|
||||
"created",
|
||||
"refused",
|
||||
"drifted",
|
||||
"drift-cleared",
|
||||
"full",
|
||||
"full-cleared",
|
||||
"orphaned",
|
||||
"recreated",
|
||||
"dropped"
|
||||
],
|
||||
"max-age-days": 30,
|
||||
"max-mib": 256
|
||||
}
|
||||
],
|
||||
"state": [
|
||||
{
|
||||
"name": "asked",
|
||||
"history": 5
|
||||
},
|
||||
{
|
||||
"name": "requests",
|
||||
"history": 5,
|
||||
"ttl-seconds": 86400
|
||||
}
|
||||
],
|
||||
"consumes": [],
|
||||
"listens": [
|
||||
{
|
||||
"name": "bus",
|
||||
"port": 4222,
|
||||
"protocol": "tcp",
|
||||
"from": "mesh",
|
||||
"why": "the mesh bus — every link the mesh has, over TLS, reached across the overlay"
|
||||
"why": "the mesh bus \u2014 every link the mesh has, over TLS, reached across the overlay"
|
||||
}
|
||||
],
|
||||
"tools": [
|
||||
@@ -69,7 +114,14 @@
|
||||
"nats_consumer_remove",
|
||||
"nats_bucket_remove",
|
||||
"nats_stream_remove",
|
||||
"nats_bucket_expiry"
|
||||
"nats_bucket_expiry",
|
||||
"nats_history",
|
||||
"nats_create",
|
||||
"nats_apply",
|
||||
"nats_recreate",
|
||||
"nats_drop",
|
||||
"nats_keeper_health",
|
||||
"nats_first_view_again"
|
||||
],
|
||||
"guards": [
|
||||
8222
|
||||
@@ -96,6 +148,13 @@
|
||||
]
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"id": "keeper-account",
|
||||
"type": "user",
|
||||
"name": "nats-keeper",
|
||||
"shell": "/usr/bin/nologin",
|
||||
"home": "/var/lib/nats-keeper"
|
||||
},
|
||||
{
|
||||
"id": "jetstream-data",
|
||||
"type": "directory",
|
||||
@@ -117,6 +176,14 @@
|
||||
"mode": "0711",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "keeper-state",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/nats-keeper",
|
||||
"mode": "0700",
|
||||
"owner": "nats-keeper",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "snapshots",
|
||||
"type": "directory",
|
||||
@@ -158,6 +225,43 @@
|
||||
"artifact": "entrypoint",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "keeper",
|
||||
"type": "process",
|
||||
"name": "nats-keeper",
|
||||
"artifact": "tools",
|
||||
"run": [
|
||||
"./nats-tools",
|
||||
"keep"
|
||||
],
|
||||
"user": "nats-keeper",
|
||||
"env": {
|
||||
"MESH_BROKER_FILE": "${dir:keeper-state}/broker-keeper",
|
||||
"MESH_NATS_FULL_RAISE": "75",
|
||||
"MESH_NATS_FULL_CLEAR": "70",
|
||||
"MESH_NATS_SNAPSHOTS_DIR": "${dir:keeper-state}"
|
||||
},
|
||||
"bus": {
|
||||
"state": [
|
||||
"asked",
|
||||
"requests"
|
||||
],
|
||||
"asks": true,
|
||||
"keeps": true,
|
||||
"invokes": [
|
||||
"mesh-delivery-indexer.indexer_bus_state",
|
||||
"seat:mesh-hand-acts.hand-act"
|
||||
]
|
||||
},
|
||||
"health": {
|
||||
"kind": "tool",
|
||||
"tool": "nats_keeper_health",
|
||||
"interval": "60s",
|
||||
"timeout": "10s",
|
||||
"looks": 2,
|
||||
"grace": "180s"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "server",
|
||||
"type": "container",
|
||||
@@ -181,7 +285,11 @@
|
||||
"restart-on": [
|
||||
"program-snapshot",
|
||||
"program-entrypoint"
|
||||
]
|
||||
],
|
||||
"health": {
|
||||
"kind": "tcp",
|
||||
"endpoint": "bus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"accesses": [
|
||||
|
||||
Reference in New Issue
Block a user