329 lines
14 KiB
Go
329 lines
14 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"regexp"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
|
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
|
|
|
"nats-tools/internal/keeper"
|
|
)
|
|
|
|
// The seat mesh-broker's snapshot verbs and nats_snapshots (design 47 G §681, ADR 0675; A-44, A-45, A-59). The bus module's tools
|
|
// bundle answers them as it answers every seat verb, with the caller the tool runner proved. snapshot-get reads the keeper's
|
|
// snapshot file and sends it in chunks (SeatPublish), each acknowledged by the caller (SeatAnswer's subscription); the verbs
|
|
// that act on the bus or remove a file are handed to the keeper through the state requests, as the persons' tools are.
|
|
|
|
// ChunkSize is at most what one chunk carries: 512 KiB.
|
|
const ChunkSize = 512 << 10
|
|
|
|
// AckWithin is how long the next chunk waits for its acknowledgement.
|
|
const AckWithin = 30 * time.Second
|
|
|
|
// callToken is the shape of the subject token a caller chooses for a send.
|
|
var callToken = regexp.MustCompile(`^[A-Za-z0-9_-]{16,64}$`)
|
|
|
|
// ChunkSubject and AckSubject are the subjects of the nth chunk of a send and of its acknowledgement.
|
|
func ChunkSubject(call string, n int64) string {
|
|
return "mesh.seat." + meshbroker.Seat + ".chunk." + meshbroker.VerbSnapshotGet + "." + call + "." + strconv.FormatInt(n, 10)
|
|
}
|
|
|
|
// AckSubject is the subject the caller acknowledges the nth chunk on.
|
|
func AckSubject(call string, n int64) string {
|
|
return "mesh.seat." + meshbroker.Seat + ".ack." + meshbroker.VerbSnapshotGet + "." + call + "." + strconv.FormatInt(n, 10)
|
|
}
|
|
|
|
// AckPattern is every acknowledgement of snapshot-get.
|
|
const AckPattern = "mesh.seat.mesh-broker.ack.snapshot-get.*.*"
|
|
|
|
// Chunk is the body of one chunk: its number, how many there are, and its bytes.
|
|
type Chunk struct {
|
|
N int64 `json:"n"`
|
|
Of int64 `json:"of"`
|
|
Data []byte `json:"data"`
|
|
}
|
|
|
|
// snapshotWorld is what the snapshot verbs reach beside the requests: the file, the runtime's seat traffic and the bus's
|
|
// users.
|
|
type snapshotWorld struct {
|
|
// Open opens the snapshot file the keeper wrote.
|
|
Open func(path string) (io.ReadCloser, error)
|
|
// Publish says one chunk on its subject (stdio.SeatPublish), Answer subscribes to the acknowledgements (stdio.SeatAnswer).
|
|
Publish func(subject string, body any, id string) error
|
|
Answer func(subject string, handle func(subject string, body json.RawMessage) (any, error)) error
|
|
// Granted says whether the caller may hear the chunks and acknowledge them (the bus's users, nats_user_can).
|
|
Granted func(caller, call string) bool
|
|
Logf func(string, ...any)
|
|
// Within is how long an acknowledgement is waited for.
|
|
Within time.Duration
|
|
// KeepDays is snapshot-keep-days.
|
|
KeepDays int
|
|
|
|
once sync.Once
|
|
onceErr error
|
|
mu sync.Mutex
|
|
waiting map[string]chan struct{}
|
|
}
|
|
|
|
// liveSnapshots is the world as this bundle reaches it.
|
|
func liveSnapshots() *snapshotWorld {
|
|
return &snapshotWorld{
|
|
Open: func(path string) (io.ReadCloser, error) { return os.Open(path) },
|
|
Publish: stdio.SeatPublish, Answer: stdio.SeatAnswer,
|
|
Granted: func(caller, call string) bool {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
|
defer cancel()
|
|
conf, err := ReadUsers(ctx)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
a, err := UserCan(conf, caller, "subscribe", ChunkSubject(call, 1))
|
|
if err != nil || a["allowed"] != true {
|
|
return false
|
|
}
|
|
a, err = UserCan(conf, caller, "publish", AckSubject(call, 1))
|
|
return err == nil && a["allowed"] == true
|
|
},
|
|
Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "nats-tools: "+f+"\n", a...) },
|
|
Within: AckWithin, KeepDays: int(number(os.Getenv, "MESH_NATS_SNAPSHOT_KEEP_DAYS", keeper.DefaultSnapshotKeepDays)),
|
|
}
|
|
}
|
|
|
|
// hearAcks subscribes once to the acknowledgements of every send.
|
|
func (w *snapshotWorld) hearAcks() error {
|
|
w.once.Do(func() {
|
|
w.waiting = map[string]chan struct{}{}
|
|
w.onceErr = w.Answer(AckPattern, func(subject string, _ json.RawMessage) (any, error) {
|
|
p := strings.Split(subject, ".")
|
|
if len(p) == 7 {
|
|
w.mu.Lock()
|
|
ch := w.waiting[p[5]+"."+p[6]]
|
|
w.mu.Unlock()
|
|
if ch != nil {
|
|
select {
|
|
case ch <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
}
|
|
return map[string]any{}, nil
|
|
})
|
|
})
|
|
return w.onceErr
|
|
}
|
|
|
|
// digestOf reads the file whole: its size and the `sha256:` of its bytes.
|
|
func (w *snapshotWorld) digestOf(path string) (int64, string, error) {
|
|
f, err := w.Open(path)
|
|
if err != nil {
|
|
return 0, "", err
|
|
}
|
|
defer f.Close()
|
|
h := sha256.New()
|
|
n, err := io.Copy(h, f)
|
|
if err != nil {
|
|
return 0, "", err
|
|
}
|
|
return n, "sha256:" + hex.EncodeToString(h.Sum(nil)), nil
|
|
}
|
|
|
|
// get answers snapshot-get: the fields, then the chunks sent after the answer when the caller is granted the chunk subjects.
|
|
func (w *snapshotWorld) get(rs requestStore, caller, call, request string) (meshbroker.SnapshotGetAnswer, error) {
|
|
refuse := func(why string) (meshbroker.SnapshotGetAnswer, error) {
|
|
return meshbroker.SnapshotGetAnswer{Error: why}, nil
|
|
}
|
|
if !callToken.MatchString(call) {
|
|
return refuse("call is one token of 16 to 64 letters, digits, - or _: 128 random bits the caller chooses")
|
|
}
|
|
rec, err := rs.Get(request)
|
|
if err != nil {
|
|
return meshbroker.SnapshotGetAnswer{}, fmt.Errorf("the snapshot of the request %s cannot be read: %w", request, err)
|
|
}
|
|
if rec == nil || (rec.Path == "" && rec.Snapshot == "") {
|
|
return refuse("no snapshot of the request " + request + " is held")
|
|
}
|
|
if !rec.Recorded() {
|
|
return refuse("snapshot taken before milestone G")
|
|
}
|
|
// A protected snapshot is never handed out: the keeper restores it in place (O-7).
|
|
if rec.Protected {
|
|
return refuse("snapshot protected")
|
|
}
|
|
size, digest, err := w.digestOf(rec.Path)
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
return refuse("snapshot missing")
|
|
}
|
|
if err != nil {
|
|
return meshbroker.SnapshotGetAnswer{}, fmt.Errorf("the snapshot %s cannot be read: %w", rec.Path, err)
|
|
}
|
|
if digest != rec.Digest || size != rec.Size {
|
|
return refuse("snapshot damaged: request " + request)
|
|
}
|
|
at := rec.At
|
|
out := meshbroker.SnapshotGetAnswer{Request: rec.ID, Module: rec.Module, Local: rec.Local, Stream: keeper.BusName(rec.Module, rec.Local),
|
|
Size: size, Digest: digest, At: &at}
|
|
if size == 0 || !w.Granted(caller, call) {
|
|
return out, nil
|
|
}
|
|
if err := w.hearAcks(); err != nil {
|
|
return meshbroker.SnapshotGetAnswer{}, fmt.Errorf("the acknowledgements cannot be heard, so no chunk is sent: %w", err)
|
|
}
|
|
out.Chunks = (size + ChunkSize - 1) / ChunkSize
|
|
go w.send(rec.Path, call, out.Chunks)
|
|
return out, nil
|
|
}
|
|
|
|
// send sends the file in chunks, each acknowledged before the next; a missed acknowledgement ends the send, said in the journal.
|
|
func (w *snapshotWorld) send(path, call string, total int64) {
|
|
f, err := w.Open(path)
|
|
if err != nil {
|
|
w.Logf("the snapshot %s cannot be opened to send (%v): the send %s ends", path, err, call)
|
|
return
|
|
}
|
|
defer f.Close()
|
|
buf := make([]byte, ChunkSize)
|
|
for n := int64(1); n <= total; n++ {
|
|
read, err := io.ReadFull(f, buf)
|
|
if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) && !errors.Is(err, io.EOF) {
|
|
w.Logf("the snapshot %s cannot be read (%v): the send %s ends at chunk %d", path, err, call, n)
|
|
return
|
|
}
|
|
key := call + "." + strconv.FormatInt(n, 10)
|
|
ack := make(chan struct{}, 1)
|
|
w.mu.Lock()
|
|
w.waiting[key] = ack
|
|
w.mu.Unlock()
|
|
err = w.Publish(ChunkSubject(call, n), Chunk{N: n, Of: total, Data: buf[:read]}, key)
|
|
if err == nil {
|
|
select {
|
|
case <-ack:
|
|
case <-time.After(w.Within):
|
|
err = fmt.Errorf("no acknowledgement within %s", w.Within)
|
|
}
|
|
}
|
|
w.mu.Lock()
|
|
delete(w.waiting, key)
|
|
w.mu.Unlock()
|
|
if err != nil {
|
|
w.Logf("the send %s of the snapshot %s ends at chunk %d of %d: %v", call, path, n, total, err)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// handed hands a request to the keeper and reads its answer: accepted or busy with the act's id, or why not.
|
|
func handed(rs requestStore, r keeper.Request) (accepted, busy, why string) {
|
|
r.ID, r.At = requestID(rs.Now()), rs.Now()
|
|
if err := rs.Put(r.ID, r); err != nil {
|
|
return "", "", fmt.Sprintf("the request cannot be handed to the keeper: %v", err)
|
|
}
|
|
end, ok := awaitEnd(rs, r, "nats_requests").(*keeper.Request)
|
|
if !ok {
|
|
return "", "", fmt.Sprintf("the keeper has not answered request %s yet: nats_requests reads it", r.ID)
|
|
}
|
|
switch {
|
|
case end.Error != "":
|
|
return "", "", end.Error
|
|
case end.Outcome == "busy":
|
|
return "", end.ActID, ""
|
|
case end.ActID != "":
|
|
return end.ActID, "", ""
|
|
}
|
|
return end.ID, "", ""
|
|
}
|
|
|
|
func boolean(description string) map[string]any {
|
|
return map[string]any{"type": "boolean", "description": description}
|
|
}
|
|
|
|
func flag(a map[string]any, k string) bool { b, _ := a[k].(bool); return b }
|
|
|
|
// snapshotTools are the four verbs of the seat mesh-broker that concern the keeper's snapshots, and nats_snapshots.
|
|
func snapshotTools(rs requestStore, w *snapshotWorld) []stdio.Tool {
|
|
why := func(a map[string]any) string { return strings.TrimSpace(arg(a, "why")) }
|
|
need := func(a map[string]any, names ...string) error {
|
|
for _, n := range names {
|
|
if strings.TrimSpace(arg(a, n)) == "" {
|
|
return fmt.Errorf("%s is required", n)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
return []stdio.Tool{
|
|
{Name: "mesh-broker.snapshot-get",
|
|
Description: "Send the stream snapshot the bus module took before a drop or recreate: its fields, then the file in chunks of at most 512 KiB on the chunk subject of the call, each acknowledged by the caller on the ack subject within 30 seconds. A caller granted no chunk subjects is answered the fields alone, with chunks 0. Refuses snapshot damaged, snapshot missing, snapshot protected and snapshot taken before milestone G (design 47 G 681).",
|
|
Input: map[string]any{"request": str("the snapshot's request, as nats_snapshots lists it"), "call": str("128 random bits the caller chooses: the subject token of this send's chunks and acknowledgements")},
|
|
Run: func(a map[string]any) (any, error) {
|
|
if err := need(a, "request", "call"); err != nil {
|
|
return nil, err
|
|
}
|
|
return w.get(rs, stdio.CallerOf(a), arg(a, "call"), arg(a, "request"))
|
|
}},
|
|
{Name: "mesh-broker.snapshot-restore",
|
|
Description: "Restore, in place, the snapshot the bus module took before a drop or recreate of a stream no other account hears: the keeper checks the file against its digest (with verify true it says so and restores nothing), refuses `names live` while a stream of its names exists, and restores it under its own name within 60 minutes. Answers at once accepted with the act's id, busy or error; the outcome comes on snapshot-restored (design 47 G 681).",
|
|
Input: map[string]any{"request": str("the snapshot's request"), "verify": boolean("check the file against its digest and restore nothing"), "why": str("why, which the hand-act log keeps")},
|
|
Run: func(a map[string]any) (any, error) {
|
|
if err := need(a, "request", "why"); err != nil {
|
|
return nil, err
|
|
}
|
|
acc, busy, bad := handed(rs, keeper.Request{Verb: keeper.VerbSnapshotRestore, Of: arg(a, "request"), Verify: flag(a, "verify"), Why: why(a), Caller: stdio.CallerOf(a)})
|
|
return meshbroker.SnapshotRestoreAnswer{Accepted: acc, Busy: busy, Error: bad}, nil
|
|
}},
|
|
{Name: "mesh-broker.snapshot-restore-cancel",
|
|
Description: "Cancel a running snapshot restore by its act: the keeper removes its own partial stream and says snapshot-restored failed. Answers accepted, or an error when that act has ended (design 47 G 681).",
|
|
Input: map[string]any{"act": str("the restore's act id"), "why": str("why, which the hand-act log keeps")},
|
|
Run: func(a map[string]any) (any, error) {
|
|
if err := need(a, "act", "why"); err != nil {
|
|
return nil, err
|
|
}
|
|
acc, busy, bad := handed(rs, keeper.Request{Verb: keeper.VerbSnapshotRestoreCancel, Of: arg(a, "act"), Why: why(a), Caller: stdio.CallerOf(a)})
|
|
return meshbroker.SnapshotRestoreCancelAnswer{Accepted: acc, Busy: busy, Error: bad}, nil
|
|
}},
|
|
{Name: "mesh-broker.snapshot-remove",
|
|
Description: "Remove the keeper's snapshot of one request now, or every snapshot past snapshot-keep-days (expired true), written in the hand-act log first; a snapshot of before milestone G goes only by its request (design 47 G 681).",
|
|
Input: map[string]any{"request": str("the snapshot's request"), "expired": boolean("every snapshot past snapshot-keep-days"), "why": str("why, which the hand-act log keeps")},
|
|
Run: func(a map[string]any) (any, error) {
|
|
if err := need(a, "why"); err != nil {
|
|
return nil, err
|
|
}
|
|
acc, busy, bad := handed(rs, keeper.Request{Verb: keeper.VerbSnapshotRemove, Of: arg(a, "request"), AllExpired: flag(a, "expired"), Why: why(a), Caller: stdio.CallerOf(a)})
|
|
return meshbroker.SnapshotRemoveAnswer{Accepted: acc, Busy: busy, Error: bad}, nil
|
|
}},
|
|
{Name: "nats_snapshots",
|
|
Description: "The keeper's snapshots of kept data taken before a drop or recreate, from its state requests: each one's request, module, local name, stream, size, digest, time, whether it is protected (restored only in place, never sent), whether it is expired (past snapshot-keep-days, removed at the keeper's next request) and whether it is recorded (a snapshot of before milestone G holds no digest and is sent or removed by hand only).",
|
|
Run: func(map[string]any) (any, error) {
|
|
keys, err := rs.Keys()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
sort.Strings(keys)
|
|
out := []map[string]any{}
|
|
for _, key := range keys {
|
|
if strings.HasPrefix(key, keeper.ActedPrefix) {
|
|
continue
|
|
}
|
|
r, err := rs.Get(key)
|
|
if err != nil || r == nil || (r.Path == "" && r.Snapshot == "") {
|
|
continue
|
|
}
|
|
out = append(out, map[string]any{"request": r.ID, "module": r.Module, "local": r.Local, "stream": keeper.BusName(r.Module, r.Local),
|
|
"size": r.Size, "digest": r.Digest, "at": r.At, "protected": r.Protected, "expired": r.Expired(rs.Now(), w.KeepDays), "recorded": r.Recorded()})
|
|
}
|
|
return map[string]any{"snapshots": out}, nil
|
|
}},
|
|
}
|
|
}
|