216 lines
6.3 KiB
Go
216 lines
6.3 KiB
Go
package main
|
|
|
|
// The live bus of the item events (items.go): the module's own broker account, as the recovery module's process holds its own
|
|
// (design 47 G A-150: the process `night` has a bus account of its own, says its items and reads the recovery module's restores).
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/keys"
|
|
"git.novox.be/novox/mesh-sdk/go/process"
|
|
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
|
|
|
"github.com/nats-io/nats.go/jetstream"
|
|
)
|
|
|
|
// The streams and subjects of the seats' events the holder reads and says.
|
|
const (
|
|
backupSeat = "node-backup"
|
|
recoverySeat = "mesh-recovery"
|
|
streamItemStarted = "SEAT_node-backup_item-started"
|
|
streamItemEnded = "SEAT_node-backup_item-ended"
|
|
streamRestoreStarted = "SEAT_mesh-recovery_restore-started"
|
|
streamRestored = "SEAT_mesh-recovery_restored"
|
|
readWait = 5 * time.Second
|
|
)
|
|
|
|
func recoveryEventSubject(event, module, kept string) string {
|
|
return "mesh.seat." + recoverySeat + ".event." + event + "." + module + "." + kept
|
|
}
|
|
|
|
func itemEventSubject(event, node, module, item string) string {
|
|
return "mesh.seat." + backupSeat + ".event." + event + "." + node + "." + module + "." + item
|
|
}
|
|
|
|
// liveBus is ItemBus over the module's own account.
|
|
type liveBus struct {
|
|
conn *process.Conn
|
|
js jetstream.JetStream
|
|
node string
|
|
}
|
|
|
|
// connectItems opens the bus as the module's own account when the process was given its credential (MESH_BROKER_FILE), and
|
|
// answers how to find the holder of the seat mesh-broker; nil, nil when it was not.
|
|
func connectItems(getenv func(string) string) (ItemBus, func() (string, error), func(), error) {
|
|
if strings.TrimSpace(getenv(process.BrokerFileEnv)) == "" {
|
|
return nil, nil, func() {}, nil
|
|
}
|
|
account, err := process.ReadBrokerAccount(getenv)
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
conn, err := process.Connect(account)
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
nc, js, err := account.Dial()
|
|
if err != nil {
|
|
conn.Close()
|
|
return nil, nil, nil, err
|
|
}
|
|
return liveBus{conn: conn, js: js, node: account.Node},
|
|
func() (string, error) { return seatlist.HolderOf(conn.Call, BrokerSeat) },
|
|
func() { nc.Close(); conn.Close() }, nil
|
|
}
|
|
|
|
func (l liveBus) Say(event string, ref ItemRef, body map[string]any) error {
|
|
return l.conn.EmitSeat(backupSeat, event+"."+l.node+"."+keys.EscapeName(ref.Module)+"."+keys.EscapeName(ref.Item), body)
|
|
}
|
|
|
|
// last reads every subject's last message of a stream through an ordered consumer made for this one read (§556).
|
|
func (l liveBus) last(ctx context.Context, stream, filter string, take func(subject string, data []byte)) error {
|
|
cons, err := l.js.OrderedConsumer(ctx, stream, jetstream.OrderedConsumerConfig{
|
|
FilterSubjects: []string{filter}, DeliverPolicy: jetstream.DeliverLastPerSubjectPolicy})
|
|
if err != nil {
|
|
return fmt.Errorf("the read of %s: %w", stream, err)
|
|
}
|
|
info, err := cons.Info(ctx)
|
|
if err != nil {
|
|
return fmt.Errorf("the read of %s: %w", stream, err)
|
|
}
|
|
for left := info.NumPending; left > 0; {
|
|
m, err := cons.Next(jetstream.FetchMaxWait(readWait))
|
|
if err != nil {
|
|
return fmt.Errorf("the read of %s: %w", stream, err)
|
|
}
|
|
take(m.Subject(), m.Data())
|
|
meta, err := m.Metadata()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
left = meta.NumPending
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func atOf(data []byte) time.Time {
|
|
var b struct {
|
|
At time.Time `json:"at"`
|
|
}
|
|
_ = json.Unmarshal(data, &b)
|
|
return b.At
|
|
}
|
|
|
|
func (l liveBus) Unended() ([]ItemRef, error) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*readWait)
|
|
defer cancel()
|
|
started, ended := map[ItemRef]time.Time{}, map[ItemRef]time.Time{}
|
|
for _, c := range []struct {
|
|
stream, event string
|
|
into map[ItemRef]time.Time
|
|
}{{streamItemStarted, EventItemStarted, started}, {streamItemEnded, EventItemEnded, ended}} {
|
|
prefix := itemEventSubject(c.event, l.node, "", "")
|
|
prefix = strings.TrimSuffix(prefix, "..")
|
|
c := c
|
|
err := l.last(ctx, c.stream, itemEventSubject(c.event, l.node, "*", "*"), func(subject string, data []byte) {
|
|
module, item, ok := strings.Cut(strings.TrimPrefix(subject, prefix+"."), ".")
|
|
if !ok {
|
|
return
|
|
}
|
|
m, e1 := keys.UnescapeName(module)
|
|
i, e2 := keys.UnescapeName(item)
|
|
if e1 == nil && e2 == nil {
|
|
c.into[ItemRef{m, i}] = atOf(data)
|
|
}
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
var out []ItemRef
|
|
for ref, at := range started {
|
|
if end, ok := ended[ref]; !ok || end.Before(at) {
|
|
out = append(out, ref)
|
|
}
|
|
}
|
|
sortRefs(out)
|
|
return out, nil
|
|
}
|
|
|
|
func (l liveBus) Restoring(module string) (string, error) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*readWait)
|
|
defer cancel()
|
|
type act struct {
|
|
id string
|
|
at time.Time
|
|
}
|
|
started, restored := map[string]act{}, map[string]act{}
|
|
for _, c := range []struct {
|
|
stream, event string
|
|
into map[string]act
|
|
}{{streamRestoreStarted, "restore-started", started}, {streamRestored, "restored", restored}} {
|
|
prefix := recoveryEventSubject(c.event, keys.EscapeName(module), "")
|
|
c := c
|
|
err := l.last(ctx, c.stream, recoveryEventSubject(c.event, keys.EscapeName(module), "*"), func(subject string, data []byte) {
|
|
var b struct {
|
|
Act string `json:"act"`
|
|
At time.Time `json:"at"`
|
|
}
|
|
if json.Unmarshal(data, &b) == nil {
|
|
c.into[strings.TrimPrefix(subject, prefix)] = act{b.Act, b.At}
|
|
}
|
|
})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
}
|
|
var running []string
|
|
for kept, s := range started {
|
|
if r, ok := restored[kept]; !ok || r.id != s.id {
|
|
running = append(running, s.id)
|
|
}
|
|
}
|
|
if len(running) == 0 {
|
|
return "", nil
|
|
}
|
|
if len(running) > 1 {
|
|
sortStrings(running)
|
|
}
|
|
return running[0], nil
|
|
}
|
|
|
|
func (l liveBus) WaitRestored(ctx context.Context, want string) error {
|
|
cons, err := l.js.OrderedConsumer(ctx, streamRestored, jetstream.OrderedConsumerConfig{
|
|
FilterSubjects: []string{"mesh.seat." + recoverySeat + ".event.restored.*.*"}, DeliverPolicy: jetstream.DeliverLastPerSubjectPolicy})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for {
|
|
m, err := cons.Next(jetstream.FetchMaxWait(time.Minute))
|
|
if err != nil {
|
|
if ctx.Err() != nil {
|
|
return ctx.Err()
|
|
}
|
|
continue
|
|
}
|
|
var b struct {
|
|
Act string `json:"act"`
|
|
}
|
|
if json.Unmarshal(m.Data(), &b) == nil && b.Act == want {
|
|
return nil
|
|
}
|
|
}
|
|
}
|
|
|
|
func sortStrings(s []string) {
|
|
for i := 1; i < len(s); i++ {
|
|
for j := i; j > 0 && s[j] < s[j-1]; j-- {
|
|
s[j], s[j-1] = s[j-1], s[j]
|
|
}
|
|
}
|
|
}
|