230 lines
7.1 KiB
Go
230 lines
7.1 KiB
Go
package bus
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/membership"
|
|
"github.com/nats-io/nats.go"
|
|
"github.com/nats-io/nats.go/jetstream"
|
|
|
|
"github.com/novox/mesh-tools/node-tools/internal/ledger"
|
|
)
|
|
|
|
// A ledger on the bus (novox/hq ADR 0597, design 25 § 12.4; issue 1966): its history a stream with a subject per
|
|
// instance under the binding's subject, its state a key-value bucket, its snapshots a subject per instance under the
|
|
// binding's snapshot subject, the latest only (the stream that holds them keeps one per subject, as the bus module
|
|
// creates it). **Only the module's ledger connection publishes on its history and its snapshots** (ADR 0597 §9): what
|
|
// is read, and the state, go on the runner's own.
|
|
|
|
// LedgerStores are the stores of one ledger binding: reads and the state on c, the history's appends and the snapshots'
|
|
// writes on writer, the module's ledger connection.
|
|
func (c *Conn) LedgerStores(b membership.BusBinding, writer *Conn) (ledger.History, ledger.State, ledger.Snapshots) {
|
|
return &busHistory{read: c, write: writer, stream: b.History, prefix: prefixOf(b.Subject)},
|
|
&busState{c: c, bucket: b.State},
|
|
&busSnapshots{read: c, write: writer, prefix: prefixOf(b.Snapshot)}
|
|
}
|
|
|
|
// prefixOf is a binding's subject as the prefix an instance's key is added to: written with or without its `.>`.
|
|
func prefixOf(subject string) string {
|
|
return strings.TrimSuffix(strings.TrimSuffix(subject, ">"), ".")
|
|
}
|
|
|
|
// publishExpecting publishes body on subject on the connection, written only when the subject's last message is still
|
|
// at expect (0: it has none); ledger.ErrRaced when it is not.
|
|
func publishExpecting(c *Conn, subject string, body []byte, expect uint64) (uint64, error) {
|
|
msg := nats.NewMsg(subject)
|
|
msg.Data = body
|
|
msg.Header.Set(jetstream.ExpectedLastSubjSeqHeader, strconv.FormatUint(expect, 10))
|
|
reply, err := c.nc.RequestMsg(msg, StateTimeout)
|
|
if errors.Is(err, nats.ErrNoResponders) {
|
|
return 0, fmt.Errorf("no stream on the bus keeps %s", subject)
|
|
}
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
var ack struct {
|
|
Seq uint64 `json:"seq"`
|
|
Error *struct {
|
|
ErrCode uint16 `json:"err_code"`
|
|
Description string `json:"description"`
|
|
} `json:"error"`
|
|
}
|
|
if err := json.Unmarshal(reply.Data, &ack); err != nil {
|
|
return 0, fmt.Errorf("the bus answered the write to %s with what is not an acknowledgement: %q", subject, reply.Data)
|
|
}
|
|
if ack.Error != nil {
|
|
if code := jetstream.ErrorCode(ack.Error.ErrCode); code == jetstream.JSErrCodeStreamWrongLastSequence ||
|
|
code == jetstream.JSErrCodeStreamWrongLastSequenceConstant {
|
|
return 0, ledger.ErrRaced
|
|
}
|
|
return 0, fmt.Errorf("the bus refused the write to %s: %s (%d)", subject, ack.Error.Description, ack.Error.ErrCode)
|
|
}
|
|
return ack.Seq, nil
|
|
}
|
|
|
|
// lastOn is the last message on a subject of the stream that keeps it: its bytes and sequence, nil and 0 for none.
|
|
func lastOn(c *Conn, stream, subject string) ([]byte, uint64, error) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
|
defer cancel()
|
|
js, err := c.jetStream()
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
if stream == "" {
|
|
if stream, err = js.StreamNameBySubject(ctx, subject); err != nil {
|
|
return nil, 0, fmt.Errorf("no stream on the bus keeps %s: %w", subject, err)
|
|
}
|
|
}
|
|
s, err := js.Stream(ctx, stream)
|
|
if err != nil {
|
|
return nil, 0, fmt.Errorf("the stream %s: %w", stream, err)
|
|
}
|
|
m, err := s.GetLastMsgForSubject(ctx, subject)
|
|
if errors.Is(err, jetstream.ErrMsgNotFound) {
|
|
return nil, 0, nil
|
|
}
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
return m.Data, m.Sequence, nil
|
|
}
|
|
|
|
type busHistory struct {
|
|
read, write *Conn
|
|
stream, prefix string
|
|
}
|
|
|
|
func (h *busHistory) Last(key string) (uint64, error) {
|
|
_, seq, err := lastOn(h.read, h.stream, h.prefix+"."+key)
|
|
return seq, err
|
|
}
|
|
|
|
func (h *busHistory) Append(key string, op []byte, expect uint64) (uint64, error) {
|
|
if h.write == nil {
|
|
return 0, errors.New("the ledger's history is published only on its module's ledger connection, and none is held")
|
|
}
|
|
return publishExpecting(h.write, h.prefix+"."+key, op, expect)
|
|
}
|
|
|
|
func (h *busHistory) Read(keys string, from uint64, limit int) ([]ledger.Kept, error) {
|
|
var out []ledger.Kept
|
|
for {
|
|
want := StreamReadMost
|
|
if limit > 0 && limit-len(out) < want {
|
|
want = limit - len(out)
|
|
}
|
|
page, err := h.read.readStream(h.stream, h.prefix+"."+keys, from, want)
|
|
if err != nil {
|
|
return out, err
|
|
}
|
|
for _, m := range page.Messages {
|
|
out = append(out, ledger.Kept{Sequence: m.Sequence, Data: m.Body})
|
|
}
|
|
if page.Next == 0 || (limit > 0 && len(out) >= limit) {
|
|
return out, nil
|
|
}
|
|
from = page.Next
|
|
}
|
|
}
|
|
|
|
type busState struct {
|
|
c *Conn
|
|
bucket string
|
|
}
|
|
|
|
func (s *busState) kv(ctx context.Context) (jetstream.KeyValue, error) {
|
|
js, err := s.c.jetStream()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
kv, err := js.KeyValue(ctx, s.bucket)
|
|
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
|
return nil, fmt.Errorf("the ledger's state %s is not on the bus", s.bucket)
|
|
}
|
|
return kv, err
|
|
}
|
|
|
|
func (s *busState) Get(key string) ([]byte, uint64, error) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
|
defer cancel()
|
|
kv, err := s.kv(ctx)
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
e, err := kv.Get(ctx, key)
|
|
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
|
return nil, 0, nil
|
|
}
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
return e.Value(), e.Revision(), nil
|
|
}
|
|
|
|
// isFull is a write a bucket or stream refused because it holds as much as it may.
|
|
func isFull(err error) bool {
|
|
msg := err.Error()
|
|
return strings.Contains(msg, "maximum bytes exceeded") || strings.Contains(msg, "maximum messages exceeded") ||
|
|
strings.Contains(msg, "insufficient resources")
|
|
}
|
|
|
|
func (s *busState) Write(key string, value []byte, revision uint64) (uint64, error) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
|
defer cancel()
|
|
kv, err := s.kv(ctx)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
var rev uint64
|
|
if revision == 0 {
|
|
rev, err = kv.Create(ctx, key, value)
|
|
} else {
|
|
rev, err = kv.Update(ctx, key, value, revision)
|
|
}
|
|
switch {
|
|
case err == nil:
|
|
return rev, nil
|
|
case errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) || isWrongSequence(err):
|
|
return 0, ledger.ErrChanged
|
|
case isFull(err):
|
|
return 0, fmt.Errorf("%w: %v", ledger.ErrFull, err)
|
|
}
|
|
return 0, err
|
|
}
|
|
|
|
func (s *busState) Keys() ([]string, error) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
|
defer cancel()
|
|
kv, err := s.kv(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
keys, err := kv.Keys(ctx)
|
|
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
|
return nil, nil
|
|
}
|
|
return keys, err
|
|
}
|
|
|
|
type busSnapshots struct {
|
|
read, write *Conn
|
|
prefix string
|
|
}
|
|
|
|
func (s *busSnapshots) Get(key string) ([]byte, uint64, error) {
|
|
return lastOn(s.read, "", s.prefix+"."+key)
|
|
}
|
|
|
|
func (s *busSnapshots) Put(key string, value []byte, expect uint64) error {
|
|
if s.write == nil {
|
|
return errors.New("a ledger's snapshot is published only on its module's ledger connection, and none is held")
|
|
}
|
|
_, err := publishExpecting(s.write, s.prefix+"."+key, value, expect)
|
|
return err
|
|
}
|