mesh-catalog/modules/nats/internal/keeper/kept.go

package keeper

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"slices"
	"strings"
	"time"

	"nats-tools/internal/busarchive"
)

// A module's kept data on the bus, backed up before the module moves and restored beside the live data when a want takes
// it back (milestone G, design 47 "A module's kept data on the bus, backed up and restored with its delivery"). The
// keeper takes it on its own account, as it takes a snapshot before a recreate (ADR 0235 §5): the streams of one kept
// data as the server's own snapshot, the position of each durable consumer the module reads with, and the catch-up tool
// the newer manifest declares, in one directory the backup holder keeps as a restore point. A restore never goes over
// the live data: the live data is first snapshotted into a set-aside directory under a name of its own, and only then
// removed from the bus; only discard-aside removes the set-aside copy, on a person's call.

const (
	// VerbSnapshotKept takes a restore point of one kept data; VerbRestoreKept restores one beside the live data.
	VerbSnapshotKept = "snapshot-kept"
	VerbRestoreKept  = "restore-kept"
	// VerbDiscardAside removes the data a restore set aside, by its id.
	VerbDiscardAside = "discard-aside"
)

// StateKept is the keeper's own state of the restore points it took and the data it set aside (never their bytes, which
// are files in the keeper's directory): `point.<module>.<local>.<form>.<fingerprint>` and `aside.<id>`. Read by the
// bus module's tool nats_kept and by the seat mesh-broker's `aside`.
const StateKept = "kept"

// PointFile and ArchiveFile are the two files of a restore point's directory; AsideFile of a set-aside directory.
const (
	PointFile   = "point.json"
	ArchiveFile = "bus.tar"
	AsideFile   = "aside.json"
)

// Position is where one durable consumer of the module stood on a stream: the stream sequence it had acknowledged up to,
// and its configuration as the server held it, so a restore can make it again at that place.
type Position struct {
	Stream   string          `json:"stream"`
	Consumer string          `json:"consumer"`
	AckFloor uint64          `json:"ack-floor"`
	Config   json.RawMessage `json:"config"`
}

// KeptBus is what the keeper needs of the bus beyond Bus to take and restore a kept data.
type KeptBus interface {
	// Existing answers which of the stream names the bus holds.
	Existing(ctx context.Context, names []string) ([]string, error)
	// Positions are the module's durable consumers on every stream (a consumer is the module's when it is named for it).
	Positions(ctx context.Context, module string) ([]Position, error)
	// SetPositions makes each consumer again from the place it stood.
	SetPositions(ctx context.Context, ps []Position) error
	// RestoreArchive puts every stream of an archive into the bus, which must hold none of them.
	RestoreArchive(ctx context.Context, a *busarchive.Archive) error
	// DeleteStreams removes streams by name.
	DeleteStreams(ctx context.Context, names []string) error
}

// Point is a restore point's description, its directory's point.json.
type Point struct {
	Module      string     `json:"module"`
	Local       string     `json:"local"`
	Form        int        `json:"form"`
	Fingerprint string     `json:"fingerprint"`
	CatchUp     string     `json:"catch-up,omitempty"`
	Streams     []string   `json:"streams"`
	Positions   []Position `json:"positions"`
	At          time.Time  `json:"at"`
	Bytes       int64      `json:"bytes"`
}

// Aside is data a restore set aside: the live kept data as it was when the restore removed it.
type Aside struct {
	ID      string    `json:"id"`
	Module  string    `json:"module"`
	Local   string    `json:"local"`
	Streams []string  `json:"streams"`
	At      time.Time `json:"at"`
	Bytes   int64     `json:"bytes"`
	By      string    `json:"by"`
	Why     string    `json:"why"`
	Dir     string    `json:"dir"`
}

// PointKey is a restore point's key in StateKept.
func PointKey(module, local string, form int, fingerprint string) string {
	return fmt.Sprintf("point.%s.%s.%d.%s", module, local, form, fingerprint)
}

// safeName is one path component made of what a module, a kept data and a fingerprint are made of.
func safeName(s string) bool {
	return s != "" && s != "." && s != ".." && !strings.ContainsAny(s, "/\\\x00 *>.")
}

func (k *Keeper) pointDir(module, local string, form int, fingerprint string) string {
	return filepath.Join(k.KeptDir, "points", module, local, fmt.Sprintf("f%d-%s", form, fingerprint))
}

// streamsOf is the streams one kept data is: its plan's objects where the keeper made it, else the streams the bus holds
// under its bus name (a foundation part's own section, made by the grants part).
func (k *Keeper) streamsOf(ctx context.Context, module, local string) ([]string, error) {
	if held, _, found := k.asked(module); found {
		if m, made := held.Made[local]; made && len(m.Plan.Objects) > 0 {
			var out []string
			for _, o := range m.Plan.Objects {
				out = append(out, o.Stream())
			}
			return out, nil
		}
	}
	name := BusName(module, local)
	got, err := k.Kept.Existing(ctx, []string{"KV_" + name, "OBJ_" + name, name})
	if err != nil {
		return nil, err
	}
	if len(got) == 0 {
		return nil, fmt.Errorf("the bus holds no kept data %s of %s (looked for KV_%s, OBJ_%s and %s)", local, module, name, name, name)
	}
	return got, nil
}

func (k *Keeper) keptReady(r *Request) error {
	if k.Kept == nil || k.KeptDir == "" {
		return errors.New("no directory for restore points is set (MESH_NATS_SNAPSHOTS_DIR), so no kept data is taken or restored")
	}
	if !safeName(r.Module) || !safeName(r.Local) || (!r.AsideOnly && !safeName(r.Fingerprint)) {
		return fmt.Errorf("%q, %q and %q are not a module, a kept data and a fingerprint: each is one plain name", r.Module, r.Local, r.Fingerprint)
	}
	return nil
}

// SnapshotKept takes the restore point of one kept data: the file pair in a directory named for its form and fingerprint,
// written whole before its name is given, and its record in StateKept. It answers the directory.
func (k *Keeper) SnapshotKept(ctx context.Context, r *Request) (string, error) {
	if err := k.keptReady(r); err != nil {
		return "", err
	}
	streams, err := k.streamsOf(ctx, r.Module, r.Local)
	if err != nil {
		return "", err
	}
	// The positions first: a consumer that moves while the snapshot is taken is then a little behind its archive, and a
	// restored consumer that is behind hears a message twice, never misses one.
	positions, err := k.Kept.Positions(ctx, r.Module)
	if err != nil {
		return "", fmt.Errorf("the consumers' positions of %s cannot be read, so no restore point is taken: %w", r.Module, err)
	}
	tar, err := k.Bus.Snapshot(ctx, streams)
	if err != nil {
		return "", fmt.Errorf("the snapshot of %s's %s failed, so no restore point is taken: %w", r.Module, r.Local, err)
	}
	dir := k.pointDir(r.Module, r.Local, r.Form, r.Fingerprint)
	if err := os.MkdirAll(filepath.Dir(dir), 0o700); err != nil {
		return "", err
	}
	partial := dir + ".partial"
	_ = os.RemoveAll(partial)
	if err := os.Mkdir(partial, 0o700); err != nil {
		return "", err
	}
	if err := os.Rename(tar, filepath.Join(partial, ArchiveFile)); err != nil {
		_ = os.RemoveAll(partial)
		return "", fmt.Errorf("the snapshot cannot be kept as a restore point: %w", err)
	}
	st, _ := os.Stat(filepath.Join(partial, ArchiveFile))
	p := Point{Module: r.Module, Local: r.Local, Form: r.Form, Fingerprint: r.Fingerprint, CatchUp: r.CatchUp,
		Streams: streams, Positions: positions, At: k.now()}
	if st != nil {
		p.Bytes = st.Size()
	}
	raw, _ := json.MarshalIndent(p, "", "  ")
	if err := os.WriteFile(filepath.Join(partial, PointFile), append(raw, '\n'), 0o600); err != nil {
		_ = os.RemoveAll(partial)
		return "", err
	}
	// A restore point taken again at the same form and fingerprint replaces the earlier one, whole.
	_ = os.RemoveAll(dir)
	if err := os.Rename(partial, dir); err != nil {
		return "", err
	}
	val, _ := json.Marshal(p)
	if _, err := k.Store.Put(StateKept, PointKey(r.Module, r.Local, r.Form, r.Fingerprint), val); err != nil {
		k.logf("the record of the restore point %s cannot be written to its own %s: %v", dir, StateKept, err)
	}
	r.Dir = dir
	k.record(r, fmt.Sprintf("took the restore point of %s's %s at form %d before %s (%d stream(s), %d consumer position(s), %d bytes)",
		r.Module, r.Local, r.Form, r.Fingerprint, len(streams), len(positions), p.Bytes))
	return fmt.Sprintf("restore point of %s's %s at form %d before %s in %s", r.Module, r.Local, r.Form, r.Fingerprint, dir), nil
}

// readPoint reads and checks a restore point's directory: its point.json names what the request names, its files are
// plain files (never a link), and its archive matches its own manifest.
func readPoint(dir string, r *Request) (*Point, *busarchive.Archive, error) {
	for _, name := range []string{PointFile, ArchiveFile} {
		fi, err := os.Lstat(filepath.Join(dir, name))
		if err != nil {
			return nil, nil, fmt.Errorf("the restore point %s holds no %s: %w", dir, name, err)
		}
		if !fi.Mode().IsRegular() {
			return nil, nil, fmt.Errorf("%s of the restore point %s is not a plain file", name, dir)
		}
	}
	raw, err := os.ReadFile(filepath.Join(dir, PointFile))
	if err != nil {
		return nil, nil, err
	}
	var p Point
	if err := json.Unmarshal(raw, &p); err != nil {
		return nil, nil, fmt.Errorf("%s of %s does not read: %w", PointFile, dir, err)
	}
	if p.Module != r.Module || p.Local != r.Local || p.Form != r.Form || p.Fingerprint != r.Fingerprint {
		return nil, nil, fmt.Errorf("the directory %s holds the restore point of %s's %s at form %d before %s, not of %s's %s at form %d before %s",
			dir, p.Module, p.Local, p.Form, p.Fingerprint, r.Module, r.Local, r.Form, r.Fingerprint)
	}
	f, err := os.Open(filepath.Join(dir, ArchiveFile))
	if err != nil {
		return nil, nil, err
	}
	defer f.Close()
	a, err := busarchive.ReadArchive(f, "")
	if err != nil {
		return nil, nil, fmt.Errorf("the archive of the restore point %s does not read: %w", dir, err)
	}
	if err := a.Verify(); err != nil {
		a.Close()
		return nil, nil, fmt.Errorf("the archive of the restore point %s does not match its own manifest, so nothing is restored: %w", dir, err)
	}
	return &p, a, nil
}

// RestoreKept restores one kept data beside the live data: the live data is snapshotted into a set-aside directory first
// and only then removed from the bus, and the restore point's streams are put in its place; the consumers the module
// reads with on other streams are set back to their positions. It answers what was set aside, where.
func (k *Keeper) RestoreKept(ctx context.Context, r *Request) (string, error) {
	if err := k.keptReady(r); err != nil {
		return "", err
	}
	var p *Point
	var a *busarchive.Archive
	var live []string
	if r.AsideOnly {
		// `aside-only` (design 47 G §405): the live data is set aside and nothing is restored.
		streams, err := k.streamsOf(ctx, r.Module, r.Local)
		if err != nil {
			return "", err
		}
		live = streams
	} else {
		if !filepath.IsAbs(r.From) {
			return "", fmt.Errorf("the restore point's directory is given as an absolute path, not %q", r.From)
		}
		var err error
		p, a, err = readPoint(filepath.Clean(r.From), r)
		if err != nil {
			return "", err
		}
		defer a.Close()
		live, err = k.Kept.Existing(ctx, p.Streams)
		if err != nil {
			return "", err
		}
	}
	id := fmt.Sprintf("%s.%s.%s", r.Module, r.Local, k.now().Format("20060102t150405"))
	var aside *Aside
	if len(live) > 0 {
		// Aside before anything is removed: a snapshot that fails leaves the live data where it is.
		tar, err := k.Bus.Snapshot(ctx, live)
		if err != nil {
			return "", fmt.Errorf("the live %s of %s could not be set aside, so nothing was removed: %w", r.Local, r.Module, err)
		}
		dir := filepath.Join(k.KeptDir, "aside", id)
		if err := os.MkdirAll(dir, 0o700); err != nil {
			return "", err
		}
		if err := os.Rename(tar, filepath.Join(dir, ArchiveFile)); err != nil {
			return "", fmt.Errorf("the live data's snapshot cannot be kept: %w", err)
		}
		st, _ := os.Stat(filepath.Join(dir, ArchiveFile))
		aside = &Aside{ID: id, Module: r.Module, Local: r.Local, Streams: live, At: k.now(), By: r.Caller, Why: r.Why, Dir: dir}
		if st != nil {
			aside.Bytes = st.Size()
		}
		raw, _ := json.MarshalIndent(aside, "", "  ")
		if err := os.WriteFile(filepath.Join(dir, AsideFile), append(raw, '\n'), 0o600); err != nil {
			return "", err
		}
		val, _ := json.Marshal(aside)
		if _, err := k.Store.Put(StateKept, "aside."+id, val); err != nil {
			return "", fmt.Errorf("what is set aside cannot be recorded, so nothing was removed: %w", err)
		}
		if err := k.Kept.DeleteStreams(ctx, live); err != nil {
			return "", fmt.Errorf("the live data was set aside in %s and could not be removed from the bus: %w", dir, err)
		}
	}
	if r.AsideOnly {
		r.Dir = ""
		where := "nothing was live, so nothing was set aside"
		if aside != nil {
			r.Dir = aside.Dir
			where = fmt.Sprintf("the live data is set aside as %s (%d bytes) in %s", aside.ID, aside.Bytes, aside.Dir)
		}
		k.record(r, fmt.Sprintf("set aside %s's %s and restored nothing; %s", r.Module, r.Local, where))
		return fmt.Sprintf("set aside %s's %s; %s", r.Module, r.Local, where), nil
	}
	if err := k.Kept.RestoreArchive(ctx, a); err != nil {
		msg := fmt.Sprintf("the restore point could not be put on the bus: %v", err)
		if aside != nil {
			if back, rerr := k.putBack(ctx, aside); rerr != nil {
				msg += fmt.Sprintf("; the live data, set aside in %s, could not be put back either: %v", aside.Dir, rerr)
			} else {
				msg += fmt.Sprintf("; the live data is back as it was (%s)", back)
			}
		}
		return "", errors.New(msg)
	}
	// The consumers on streams the restore point did not carry (the snapshot of a stream carries its own consumers).
	var apply []Position
	for _, pos := range p.Positions {
		if !slices.Contains(p.Streams, pos.Stream) {
			apply = append(apply, pos)
		}
	}
	if len(apply) > 0 {
		if err := k.Kept.SetPositions(ctx, apply); err != nil {
			return "", fmt.Errorf("the kept data is restored, but the consumers' positions could not be set back: %w", err)
		}
	}
	r.Dir, r.CatchUp = "", p.CatchUp
	where := "nothing was live, so nothing was set aside"
	if aside != nil {
		r.Dir = aside.Dir
		where = fmt.Sprintf("the live data is set aside as %s (%d bytes) in %s", aside.ID, aside.Bytes, aside.Dir)
	}
	k.record(r, fmt.Sprintf("restored %s's %s from the restore point at form %d before %s beside the live data; %s", r.Module, r.Local, r.Form, r.Fingerprint, where))
	return fmt.Sprintf("restored %s's %s from its restore point; %s; %d consumer position(s) set back; catch-up tool %q", r.Module, r.Local, where, len(apply), p.CatchUp), nil
}

// putBack restores set-aside data to the bus, after a restore failed.
func (k *Keeper) putBack(ctx context.Context, a *Aside) (string, error) {
	f, err := os.Open(filepath.Join(a.Dir, ArchiveFile))
	if err != nil {
		return "", err
	}
	defer f.Close()
	arch, err := busarchive.ReadArchive(f, "")
	if err != nil {
		return "", err
	}
	defer arch.Close()
	if err := k.Kept.DeleteStreams(ctx, a.Streams); err != nil {
		return "", err
	}
	if err := k.Kept.RestoreArchive(ctx, arch); err != nil {
		return "", err
	}
	return "from " + a.Dir, nil
}

// DiscardAside removes the data a restore set aside, by its id (Request.From), on a person's call: the request says who
// and why, and is recorded as a hand act before the data goes.
func (k *Keeper) DiscardAside(r *Request) (string, error) {
	if k.KeptDir == "" {
		return "", errors.New("no directory for restore points is set (MESH_NATS_SNAPSHOTS_DIR)")
	}
	id := strings.TrimSpace(r.From)
	if !safeAsideID(id) {
		return "", fmt.Errorf("%q is not the id of data set aside", id)
	}
	if strings.TrimSpace(r.Why) == "" {
		return "", errors.New("discarding set-aside data says why")
	}
	raw, _, found, err := k.Store.Get(StateKept, "aside."+id)
	if err != nil {
		return "", err
	}
	if !found {
		return "nothing done", fmt.Errorf("no data is set aside as %s", id)
	}
	var a Aside
	_ = json.Unmarshal(raw, &a)
	k.record(r, fmt.Sprintf("discarded the data set aside as %s (%s's %s, %d bytes)", id, a.Module, a.Local, a.Bytes))
	if r.Unrecorded != "" {
		return "nothing done", fmt.Errorf("the hand act cannot be recorded, so the set-aside data stays: %s", r.Unrecorded)
	}
	if err := os.RemoveAll(filepath.Join(k.KeptDir, "aside", id)); err != nil {
		return "partly done", err
	}
	if err := k.Store.Delete(StateKept, "aside."+id); err != nil {
		return "partly done", err
	}
	return fmt.Sprintf("discarded the data set aside as %s", id), nil
}

func safeAsideID(id string) bool {
	return id != "" && !strings.ContainsAny(id, "/\\\x00 *>") && !strings.HasPrefix(id, ".")
}