mesh-catalog/modules/nats/internal/bussnap/snapshot.go

// Package bussnap takes a snapshot of the bus's streams with the server's own snapshot API (novox/hq ADR 0235): the
// nightly dump of every stream, and the bus module's keeper's snapshot of one entry before it is rebuilt or removed
// (design 25 § 12.3).
package bussnap

import (
	"archive/tar"
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"slices"
	"sort"
	"strings"
	"sync"
	"time"

	"nats-tools/internal/busarchive"

	"github.com/nats-io/nats.go"
)

// The JetStream API's subjects this program speaks. Each is granted to the bus's own module and
// nothing else is (mesh-controller internal/broker, BusSnapshotGrants); a test holds the two lists
// equal against a real server.
const (
	APIStreamNames    = "$JS.API.STREAM.NAMES"
	APIStreamInfo     = "$JS.API.STREAM.INFO."
	APIStreamSnapshot = "$JS.API.STREAM.SNAPSHOT."
	APIStreamRestore  = "$JS.API.STREAM.RESTORE."
)

// ChunkSize is what the server is asked to send at a time: its own default. The server sends up to
// an 8 MiB window ahead of the acknowledgements and waits for them past it, so one stream costs the
// server at most that much in flight, whatever its size.
const ChunkSize = 128 * 1024

// SnapshotOptions bounds a snapshot.
type SnapshotOptions struct {
	// StreamTimeout is the longest one stream may take; past it the snapshot fails, naming it.
	StreamTimeout time.Duration
	// Pause is a rest between two streams, so a night's snapshot is never one long burst.
	Pause time.Duration
	// TempDir is where one stream's archive waits until it is written out; the system's when empty.
	TempDir string
	Log     func(format string, args ...any)
	// Streams, when set, are the only streams taken (the keeper's snapshot of one entry); every stream when empty.
	Streams []string
}

// APIError is the error every JetStream API answer may carry.
type APIError struct {
	Code        int    `json:"code"`
	ErrCode     int    `json:"err_code"`
	Description string `json:"description"`
}

func (e *APIError) Error() string { return fmt.Sprintf("%s (%d)", e.Description, e.Code) }

// StreamState is what this program reads of a stream's state; the whole of it is kept as the server
// gave it, in the stream's meta.
type StreamState struct {
	Messages  uint64 `json:"messages"`
	Bytes     uint64 `json:"bytes"`
	FirstSeq  uint64 `json:"first_seq"`
	LastSeq   uint64 `json:"last_seq"`
	NumDelete int    `json:"num_deleted"`
	Consumers int    `json:"consumer_count"`
}

type streamConfig struct {
	Name      string   `json:"name"`
	Subjects  []string `json:"subjects"`
	Retention string   `json:"retention"`
	Storage   string   `json:"storage"`
}

// Request asks the JetStream API and decodes its answer, refusing an answer that carries an error.
func Request(ctx context.Context, nc *nats.Conn, subject string, body any, into any) error {
	var payload []byte
	if body != nil {
		var err error
		if payload, err = json.Marshal(body); err != nil {
			return err
		}
	}
	rctx, cancel := context.WithTimeout(ctx, 30*time.Second)
	defer cancel()
	msg, err := nc.RequestWithContext(rctx, subject, payload)
	if err != nil {
		if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, nats.ErrTimeout) || errors.Is(err, nats.ErrNoResponders) {
			return fmt.Errorf("%s was not answered (%v): the bus refuses a subject this user is not granted by not answering it", subject, err)
		}
		return fmt.Errorf("%s: %w", subject, err)
	}
	var wrapped struct {
		Error *APIError `json:"error"`
	}
	if err := json.Unmarshal(msg.Data, &wrapped); err != nil {
		return fmt.Errorf("%s answered something that is not JSON: %q", subject, Truncate(msg.Data))
	}
	if wrapped.Error != nil {
		return fmt.Errorf("%s: %w", subject, wrapped.Error)
	}
	if into != nil {
		return json.Unmarshal(msg.Data, into)
	}
	return nil
}

func Truncate(b []byte) string {
	if len(b) > 200 {
		return string(b[:200]) + "…"
	}
	return string(b)
}

// StreamNames is every stream in the account, in name order, all pages of them.
func StreamNames(ctx context.Context, nc *nats.Conn) ([]string, error) {
	var names []string
	for {
		var page struct {
			Total   int      `json:"total"`
			Offset  int      `json:"offset"`
			Limit   int      `json:"limit"`
			Streams []string `json:"streams"`
		}
		if err := Request(ctx, nc, APIStreamNames, map[string]int{"offset": len(names)}, &page); err != nil {
			return nil, err
		}
		names = append(names, page.Streams...)
		if len(page.Streams) == 0 || len(names) >= page.Total {
			break
		}
	}
	sort.Strings(names)
	return names, nil
}

// Snapshot writes every stream, one at a time, as a tar: the manifest first, then each stream's
// meta and archive under streams/<name>/. Any stream failing fails the whole: a partial copy of the
// bus written where a whole one is expected is the silent failure this exists to prevent.
func Snapshot(ctx context.Context, nc *nats.Conn, out io.Writer, o SnapshotOptions) (*busarchive.Manifest, error) {
	if o.StreamTimeout <= 0 {
		o.StreamTimeout = 10 * time.Minute
	}
	if o.Log == nil {
		o.Log = func(string, ...any) {}
	}
	m := &busarchive.Manifest{Format: 1, Server: nc.ConnectedServerVersion(), Started: time.Now().UTC()}
	names, err := StreamNames(ctx, nc)
	if err != nil {
		return nil, err
	}
	if len(o.Streams) > 0 {
		var only []string
		for _, n := range names {
			if slices.Contains(o.Streams, n) {
				only = append(only, n)
			}
		}
		if len(only) != len(o.Streams) {
			return nil, fmt.Errorf("the bus holds %v of the streams %v asked for: a snapshot missing one would pass for a whole one", only, o.Streams)
		}
		names = only
	}
	if len(names) == 0 {
		// The mesh's own streams always exist once the controller has run; a bus with none is not
		// one to call backed up.
		return nil, errors.New("the bus holds no streams; a snapshot of nothing would pass for a backup")
	}

	type taken struct {
		entry   busarchive.StreamEntry
		meta    []byte
		archive *os.File
	}
	var files []taken
	defer func() {
		for _, t := range files {
			t.archive.Close()
			os.Remove(t.archive.Name())
		}
	}()

	for i, name := range names {
		if i > 0 && o.Pause > 0 {
			select {
			case <-time.After(o.Pause):
			case <-ctx.Done():
				return nil, ctx.Err()
			}
		}
		var info struct {
			Config streamConfig `json:"config"`
		}
		if err := Request(ctx, nc, APIStreamInfo+name, nil, &info); err != nil {
			return nil, err
		}
		if info.Config.Storage == "memory" {
			m.Skipped = append(m.Skipped, busarchive.Skipped{Name: name, Why: "memory storage: it holds nothing across a restart, and the server cannot snapshot it"})
			o.Log("%s: skipped, memory storage", name)
			continue
		}
		f, err := os.CreateTemp(o.TempDir, "mesh-nats-snapshot-*")
		if err != nil {
			return nil, err
		}
		files = append(files, taken{archive: f})
		sctx, cancel := context.WithTimeout(ctx, o.StreamTimeout)
		entry, meta, err := snapshotStream(sctx, nc, name, f)
		cancel()
		if err != nil {
			return nil, fmt.Errorf("stream %s: %w", name, err)
		}
		files[len(files)-1].entry, files[len(files)-1].meta = entry, meta
		m.Messages += entry.Messages
		m.StreamBytes += int64(entry.Bytes)
		m.ArchiveBytes += entry.ArchiveBytes
		o.Log("%s: %d message(s), %s, last sequence %d, %d consumer(s) → %s in %.2fs",
			name, entry.Messages, Human(int64(entry.Bytes)), entry.LastSeq, entry.Consumers, Human(entry.ArchiveBytes), entry.Seconds)
	}
	m.Finished = time.Now().UTC()
	m.Seconds = m.Finished.Sub(m.Started).Seconds()
	for _, t := range files {
		m.Streams = append(m.Streams, t.entry)
	}

	tw := tar.NewWriter(out)
	manifest, err := json.MarshalIndent(m, "", "  ")
	if err != nil {
		return nil, err
	}
	if err := writeEntry(tw, busarchive.ManifestName, append(manifest, '\n'), m.Finished); err != nil {
		return nil, err
	}
	for _, t := range files {
		if err := writeEntry(tw, t.entry.Meta, t.meta, m.Finished); err != nil {
			return nil, err
		}
		if _, err := t.archive.Seek(0, io.SeekStart); err != nil {
			return nil, err
		}
		hdr := &tar.Header{Name: t.entry.Archive, Mode: 0o600, Size: t.entry.ArchiveBytes, ModTime: m.Finished, Format: tar.FormatPAX}
		if err := tw.WriteHeader(hdr); err != nil {
			return nil, err
		}
		if _, err := io.Copy(tw, t.archive); err != nil {
			return nil, err
		}
	}
	if err := tw.Close(); err != nil {
		return nil, err
	}
	return m, nil
}

func writeEntry(tw *tar.Writer, name string, body []byte, at time.Time) error {
	if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(body)), ModTime: at, Format: tar.FormatPAX}); err != nil {
		return err
	}
	_, err := tw.Write(body)
	return err
}

// snapshotStream takes one stream's snapshot into f: the server's chunks, each acknowledged as it
// arrives so the server's window keeps moving, until the server's last, empty message. It answers
// the stream's entry (the archive's size and checksum included) and its meta.
//
// The protocol is the server's (nats-server jetstream_api.go, streamSnapshot): the request names a
// subject to deliver to; the server answers with the stream's configuration and its state at the
// snapshot, then sends the archive in chunks whose reply subject is the acknowledgement it waits for
// past its window — two seconds without one and it gives up with "408 No Flow Response". The end is
// an empty message whose status header says 204, or the error that ended it.
func snapshotStream(ctx context.Context, nc *nats.Conn, name string, f *os.File) (busarchive.StreamEntry, []byte, error) {
	started := time.Now()
	deliver := nc.NewRespInbox()

	hash := sha256.New()
	var written int64
	var mu sync.Mutex
	var failed error
	done := make(chan struct{})
	var once sync.Once
	finish := func(err error) {
		mu.Lock()
		if failed == nil {
			failed = err
		}
		mu.Unlock()
		once.Do(func() { close(done) })
	}
	sub, err := nc.Subscribe(deliver, func(msg *nats.Msg) {
		if len(msg.Data) == 0 {
			if status := msg.Header.Get("Status"); status != "" && status != "204" {
				finish(fmt.Errorf("the server ended the snapshot: %s %s", status, msg.Header.Get("Description")))
				return
			}
			finish(nil)
			return
		}
		// Acknowledged before it is written: the chunk is already in memory, and a slow disk must not
		// read to the server as a reader that went away.
		if msg.Reply != "" {
			if err := nc.Publish(msg.Reply, nil); err != nil {
				finish(fmt.Errorf("acknowledging a chunk: %w", err))
				return
			}
		}
		if _, err := f.Write(msg.Data); err != nil {
			finish(fmt.Errorf("writing the archive: %w", err))
			return
		}
		hash.Write(msg.Data)
		written += int64(len(msg.Data))
	})
	if err != nil {
		return busarchive.StreamEntry{}, nil, err
	}
	defer sub.Unsubscribe()
	// Every chunk is held until the callback has it; the window bounds how many there can be.
	if err := sub.SetPendingLimits(-1, -1); err != nil {
		return busarchive.StreamEntry{}, nil, err
	}
	if err := nc.FlushWithContext(ctx); err != nil {
		return busarchive.StreamEntry{}, nil, err
	}

	var resp struct {
		Config json.RawMessage `json:"config"`
		State  json.RawMessage `json:"state"`
	}
	req := map[string]any{"deliver_subject": deliver, "chunk_size": ChunkSize}
	if err := Request(ctx, nc, APIStreamSnapshot+name, req, &resp); err != nil {
		return busarchive.StreamEntry{}, nil, err
	}
	select {
	case <-done:
	case <-ctx.Done():
		return busarchive.StreamEntry{}, nil, fmt.Errorf("not finished within its bound: %w", ctx.Err())
	}
	// The subscription is drained before the archive is read back, so no callback is still writing.
	_ = sub.Unsubscribe()
	mu.Lock()
	err = failed
	mu.Unlock()
	if err != nil {
		return busarchive.StreamEntry{}, nil, err
	}
	if written == 0 {
		return busarchive.StreamEntry{}, nil, errors.New("the server sent an empty archive")
	}
	if err := f.Sync(); err != nil {
		return busarchive.StreamEntry{}, nil, err
	}

	var cfg streamConfig
	var state StreamState
	if err := json.Unmarshal(resp.Config, &cfg); err != nil {
		return busarchive.StreamEntry{}, nil, fmt.Errorf("its configuration: %w", err)
	}
	if err := json.Unmarshal(resp.State, &state); err != nil {
		return busarchive.StreamEntry{}, nil, fmt.Errorf("its state: %w", err)
	}
	// The archive reads to its end — s2 frames, then a tar — before it is called a snapshot.
	if _, err := f.Seek(0, io.SeekStart); err != nil {
		return busarchive.StreamEntry{}, nil, err
	}
	if err := busarchive.ReadsThrough(f); err != nil {
		return busarchive.StreamEntry{}, nil, fmt.Errorf("the archive the server sent does not read: %w", err)
	}

	// The meta, in the shape the restore API takes and the nats CLI writes (backup.json): the
	// server's own configuration and state, kept as raw JSON so a field this program does not know
	// is never dropped on the way back.
	meta, err := json.MarshalIndent(map[string]json.RawMessage{"config": resp.Config, "state": resp.State}, "", "  ")
	if err != nil {
		return busarchive.StreamEntry{}, nil, err
	}
	metaSum := sha256.Sum256(meta)
	return busarchive.StreamEntry{
		Name: name, Subjects: cfg.Subjects, Retention: cfg.Retention,
		Messages: state.Messages, Bytes: state.Bytes, FirstSeq: state.FirstSeq, LastSeq: state.LastSeq,
		Deleted: state.NumDelete, Consumers: state.Consumers,
		Meta: "streams/" + name + "/backup.json", MetaSHA256: hex.EncodeToString(metaSum[:]),
		Archive: "streams/" + name + "/stream.tar.s2", ArchiveBytes: written,
		ArchiveSHA256: hex.EncodeToString(hash.Sum(nil)), Seconds: time.Since(started).Seconds(),
	}, meta, nil
}

// Human is a size in bytes as a person reads it: 1.5 MiB.
func Human(n int64) string {
	const unit = 1024
	if n < unit {
		return fmt.Sprintf("%d B", n)
	}
	div, exp := int64(unit), 0
	for v := n / unit; v >= unit; v /= unit {
		div *= unit
		exp++
	}
	return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp])
}

// Into is the keeper's snapshot of some streams before it rebuilds or removes them (design 25 § 12.3): one tar under
// dir, named for the streams and the time, written whole before its name is given, so a partial one is never taken
// for it. It answers the file's path.
func Into(nc *nats.Conn, dir string) func(ctx context.Context, streams []string) (string, error) {
	return func(ctx context.Context, streams []string) (string, error) {
		if dir == "" {
			return "", errors.New("no snapshots directory is set (MESH_NATS_SNAPSHOTS_DIR), so no snapshot is taken and nothing is removed")
		}
		name := filepath.Join(dir, "entry-"+strings.Join(streams, "+")+"-"+time.Now().UTC().Format("20060102T150405Z")+".tar")
		f, err := os.CreateTemp(dir, ".entry-*.partial")
		if err != nil {
			return "", err
		}
		defer os.Remove(f.Name())
		if _, err := Snapshot(ctx, nc, f, SnapshotOptions{Streams: streams, TempDir: dir}); err != nil {
			f.Close()
			return "", err
		}
		if err := f.Close(); err != nil {
			return "", err
		}
		return name, os.Rename(f.Name(), name)
	}
}