Files
mesh-catalog/modules/restic/cmd/restic-backups/itembus.go
T

216 lines
6.3 KiB
Go

package main
// The live bus of the item events (items.go): the module's own broker account, as the recovery module's process holds its own
// (design 47 G A-150: the process `night` has a bus account of its own, says its items and reads the recovery module's restores).
import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"github.com/nats-io/nats.go/jetstream"
)
// The streams and subjects of the seats' events the holder reads and says.
const (
backupSeat = "node-backup"
recoverySeat = "mesh-recovery"
streamItemStarted = "SEAT_node-backup_item-started"
streamItemEnded = "SEAT_node-backup_item-ended"
streamRestoreStarted = "SEAT_mesh-recovery_restore-started"
streamRestored = "SEAT_mesh-recovery_restored"
readWait = 5 * time.Second
)
func recoveryEventSubject(event, module, kept string) string {
return "mesh.seat." + recoverySeat + ".event." + event + "." + module + "." + kept
}
func itemEventSubject(event, node, module, item string) string {
return "mesh.seat." + backupSeat + ".event." + event + "." + node + "." + module + "." + item
}
// liveBus is ItemBus over the module's own account.
type liveBus struct {
conn *process.Conn
js jetstream.JetStream
node string
}
// connectItems opens the bus as the module's own account when the process was given its credential (MESH_BROKER_FILE), and
// answers how to find the holder of the seat mesh-broker; nil, nil when it was not.
func connectItems(getenv func(string) string) (ItemBus, func() (string, error), func(), error) {
if strings.TrimSpace(getenv(process.BrokerFileEnv)) == "" {
return nil, nil, func() {}, nil
}
account, err := process.ReadBrokerAccount(getenv)
if err != nil {
return nil, nil, nil, err
}
conn, err := process.Connect(account)
if err != nil {
return nil, nil, nil, err
}
nc, js, err := account.Dial()
if err != nil {
conn.Close()
return nil, nil, nil, err
}
return liveBus{conn: conn, js: js, node: account.Node},
func() (string, error) { return seatlist.HolderOf(conn.Call, BrokerSeat) },
func() { nc.Close(); conn.Close() }, nil
}
func (l liveBus) Say(event string, ref ItemRef, body map[string]any) error {
return l.conn.EmitSeat(backupSeat, event+"."+l.node+"."+keys.EscapeName(ref.Module)+"."+keys.EscapeName(ref.Item), body)
}
// last reads every subject's last message of a stream through an ordered consumer made for this one read (§556).
func (l liveBus) last(ctx context.Context, stream, filter string, take func(subject string, data []byte)) error {
cons, err := l.js.OrderedConsumer(ctx, stream, jetstream.OrderedConsumerConfig{
FilterSubjects: []string{filter}, DeliverPolicy: jetstream.DeliverLastPerSubjectPolicy})
if err != nil {
return fmt.Errorf("the read of %s: %w", stream, err)
}
info, err := cons.Info(ctx)
if err != nil {
return fmt.Errorf("the read of %s: %w", stream, err)
}
for left := info.NumPending; left > 0; {
m, err := cons.Next(jetstream.FetchMaxWait(readWait))
if err != nil {
return fmt.Errorf("the read of %s: %w", stream, err)
}
take(m.Subject(), m.Data())
meta, err := m.Metadata()
if err != nil {
return err
}
left = meta.NumPending
}
return nil
}
func atOf(data []byte) time.Time {
var b struct {
At time.Time `json:"at"`
}
_ = json.Unmarshal(data, &b)
return b.At
}
func (l liveBus) Unended() ([]ItemRef, error) {
ctx, cancel := context.WithTimeout(context.Background(), 2*readWait)
defer cancel()
started, ended := map[ItemRef]time.Time{}, map[ItemRef]time.Time{}
for _, c := range []struct {
stream, event string
into map[ItemRef]time.Time
}{{streamItemStarted, EventItemStarted, started}, {streamItemEnded, EventItemEnded, ended}} {
prefix := itemEventSubject(c.event, l.node, "", "")
prefix = strings.TrimSuffix(prefix, "..")
c := c
err := l.last(ctx, c.stream, itemEventSubject(c.event, l.node, "*", "*"), func(subject string, data []byte) {
module, item, ok := strings.Cut(strings.TrimPrefix(subject, prefix+"."), ".")
if !ok {
return
}
m, e1 := keys.UnescapeName(module)
i, e2 := keys.UnescapeName(item)
if e1 == nil && e2 == nil {
c.into[ItemRef{m, i}] = atOf(data)
}
})
if err != nil {
return nil, err
}
}
var out []ItemRef
for ref, at := range started {
if end, ok := ended[ref]; !ok || end.Before(at) {
out = append(out, ref)
}
}
sortRefs(out)
return out, nil
}
func (l liveBus) Restoring(module string) (string, error) {
ctx, cancel := context.WithTimeout(context.Background(), 2*readWait)
defer cancel()
type act struct {
id string
at time.Time
}
started, restored := map[string]act{}, map[string]act{}
for _, c := range []struct {
stream, event string
into map[string]act
}{{streamRestoreStarted, "restore-started", started}, {streamRestored, "restored", restored}} {
prefix := recoveryEventSubject(c.event, keys.EscapeName(module), "")
c := c
err := l.last(ctx, c.stream, recoveryEventSubject(c.event, keys.EscapeName(module), "*"), func(subject string, data []byte) {
var b struct {
Act string `json:"act"`
At time.Time `json:"at"`
}
if json.Unmarshal(data, &b) == nil {
c.into[strings.TrimPrefix(subject, prefix)] = act{b.Act, b.At}
}
})
if err != nil {
return "", err
}
}
var running []string
for kept, s := range started {
if r, ok := restored[kept]; !ok || r.id != s.id {
running = append(running, s.id)
}
}
if len(running) == 0 {
return "", nil
}
if len(running) > 1 {
sortStrings(running)
}
return running[0], nil
}
func (l liveBus) WaitRestored(ctx context.Context, want string) error {
cons, err := l.js.OrderedConsumer(ctx, streamRestored, jetstream.OrderedConsumerConfig{
FilterSubjects: []string{"mesh.seat." + recoverySeat + ".event.restored.*.*"}, DeliverPolicy: jetstream.DeliverLastPerSubjectPolicy})
if err != nil {
return err
}
for {
m, err := cons.Next(jetstream.FetchMaxWait(time.Minute))
if err != nil {
if ctx.Err() != nil {
return ctx.Err()
}
continue
}
var b struct {
Act string `json:"act"`
}
if json.Unmarshal(m.Data(), &b) == nil && b.Act == want {
return nil
}
}
}
func sortStrings(s []string) {
for i := 1; i < len(s); i++ {
for j := i; j > 0 && s[j] < s[j-1]; j-- {
s[j], s[j-1] = s[j-1], s[j]
}
}
}