NATS10: the bus module serves no restore point tool and keeps no state kept

Reverts bbd9f638, c2e017a4 and b6884796; cmd/mesh-nats-snapshot/restore.go is as before them. The manifest test
asserts the claim serves only the reading verbs and the keeper's state is asked, requests, spent.

Builds: design 47 G §554, §555

Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ
This commit is contained in:
2026-11-11 15:45:03 +01:00
parent 9f62e0a2ba
commit bea1c446ea
16 changed files with 126 additions and 1395 deletions
-126
View File
@@ -1,126 +0,0 @@
package bussnap
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"strings"
"time"
"github.com/nats-io/nats.go"
"nats-tools/internal/busarchive"
)
// restoreChunk is what is sent at a time; the server answers each before the next goes.
const restoreChunk = 64 * 1024
// RestoreAll puts every stream of the archive into the server nc reaches, one at a time, and holds
// each restored stream's state to the manifest. **Nothing is restored over anything**: a server that
// already holds any of the streams is refused before the first is sent.
func RestoreAll(ctx context.Context, nc *nats.Conn, a *busarchive.Archive, log func(string, ...any)) error {
var present []string
for _, s := range a.Manifest.Streams {
err := Request(ctx, nc, APIStreamInfo+s.Name, nil, nil)
var api *APIError
switch {
case err == nil:
present = append(present, s.Name)
case errors.As(err, &api) && api.Code == 404:
default:
return err
}
}
if len(present) > 0 {
return fmt.Errorf("the server already holds %s; nothing is restored over a live stream — restore into a new "+
"store (--into) and swap it in with the bus stopped", strings.Join(present, ", "))
}
for _, s := range a.Manifest.Streams {
started := time.Now()
if err := restoreStream(ctx, nc, a, s); err != nil {
return fmt.Errorf("stream %s: %w", s.Name, err)
}
log("%s: restored, at least %d message(s) to sequence %d, in %.2fs", s.Name, s.Messages, s.LastSeq, time.Since(started).Seconds())
}
return nil
}
func restoreStream(ctx context.Context, nc *nats.Conn, a *busarchive.Archive, s busarchive.StreamEntry) error {
meta, archive, err := a.Open(s)
if err != nil {
return err
}
defer archive.Close()
var req map[string]json.RawMessage
if err := json.Unmarshal(meta, &req); err != nil || req["config"] == nil {
return errors.New("its meta is not a stream's configuration and state")
}
var resp struct {
DeliverSubject string `json:"deliver_subject"`
}
if err := Request(ctx, nc, APIStreamRestore+s.Name, req, &resp); err != nil {
return err
}
if resp.DeliverSubject == "" {
return errors.New("the server gave nowhere to send the archive")
}
buf := make([]byte, restoreChunk)
for {
n, err := archive.Read(buf)
if n > 0 {
cctx, cancel := context.WithTimeout(ctx, 30*time.Second)
reply, rerr := nc.RequestWithContext(cctx, resp.DeliverSubject, buf[:n])
cancel()
if rerr != nil {
return fmt.Errorf("sending the archive: %w", rerr)
}
if len(reply.Data) > 0 {
return fmt.Errorf("the server refused the archive: %s", Truncate(reply.Data))
}
}
if err == io.EOF {
break
}
if err != nil {
return err
}
}
// The end: an empty message, answered once the server has rebuilt the stream from what it was
// sent — which for a large stream takes a while.
fctx, cancel := context.WithTimeout(ctx, 30*time.Minute)
defer cancel()
final, err := nc.RequestWithContext(fctx, resp.DeliverSubject, nil)
if err != nil {
return fmt.Errorf("finishing the restore: %w", err)
}
var created struct {
Error *APIError `json:"error"`
State StreamState `json:"state"`
}
if err := json.Unmarshal(final.Data, &created); err != nil {
return fmt.Errorf("the server's last answer is not JSON: %q", Truncate(final.Data))
}
if created.Error != nil {
return created.Error
}
return HeldTo(created.State, s)
}
// HeldTo is whether a restored stream holds at least what its snapshot said it held.
//
// **At least, not exactly — found by the restore test, not assumed.** The state the server answers a
// snapshot request with is taken when the snapshot starts; the stream goes on taking messages while
// its blocks are read out, and a block read a moment later carries some of what was appended in that
// moment. So a stream written during its snapshot restores with a later last sequence than the
// manifest says (90075 where it said 90024, with a writer publishing every 2ms through it), and of
// the messages in that tail some may be there and some not. Every message up to the manifest's last
// sequence is there, exactly; that is the snapshot's promise, and what the test holds it to message
// by message. What must never happen is a restored stream that ends before the manifest said it did.
func HeldTo(restored StreamState, s busarchive.StreamEntry) error {
if restored.LastSeq < s.LastSeq || (s.Messages > 0 && restored.Messages == 0) {
return fmt.Errorf("restored with %d message(s) up to sequence %d; the snapshot held %d up to %d",
restored.Messages, restored.LastSeq, s.Messages, s.LastSeq)
}
return nil
}
-3
View File
@@ -219,9 +219,6 @@ type Keeper struct {
Warrant func(askID string) (*oc.Ask, *oc.Warrant, error)
Now func() time.Time
Logf func(string, ...any)
// Kept takes and restores a module's kept data, in KeptDir (kept.go); nil where the process has no such bus.
Kept KeptBus
KeptDir string
// Raise and Clear are the fill marks in percent (ADR 0595 §6): settings of the bus module, 75 and 70 by default.
Raise, Clear int64
-405
View File
@@ -1,405 +0,0 @@
package keeper
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"time"
"nats-tools/internal/busarchive"
)
// A module's kept data on the bus, backed up before the module moves and restored beside the live data when a want takes
// it back (milestone G, design 47 "A module's kept data on the bus, backed up and restored with its delivery"). The
// keeper takes it on its own account, as it takes a snapshot before a recreate (ADR 0235 §5): the streams of one kept
// data as the server's own snapshot, the position of each durable consumer the module reads with, and the catch-up tool
// the newer manifest declares, in one directory the backup holder keeps as a restore point. A restore never goes over
// the live data: the live data is first snapshotted into a set-aside directory under a name of its own, and only then
// removed from the bus; only discard-aside removes the set-aside copy, on a person's call.
const (
// VerbSnapshotKept takes a restore point of one kept data; VerbRestoreKept restores one beside the live data.
VerbSnapshotKept = "snapshot-kept"
VerbRestoreKept = "restore-kept"
// VerbDiscardAside removes the data a restore set aside, by its id.
VerbDiscardAside = "discard-aside"
)
// StateKept is the keeper's own state of the restore points it took and the data it set aside (never their bytes, which
// are files in the keeper's directory): `point.<module>.<local>.<form>.<fingerprint>` and `aside.<id>`. Read by the
// bus module's tool nats_kept and by the seat mesh-broker's `aside`.
const StateKept = "kept"
// PointFile and ArchiveFile are the two files of a restore point's directory; AsideFile of a set-aside directory.
const (
PointFile = "point.json"
ArchiveFile = "bus.tar"
AsideFile = "aside.json"
)
// Position is where one durable consumer of the module stood on a stream: the stream sequence it had acknowledged up to,
// and its configuration as the server held it, so a restore can make it again at that place.
type Position struct {
Stream string `json:"stream"`
Consumer string `json:"consumer"`
AckFloor uint64 `json:"ack-floor"`
Config json.RawMessage `json:"config"`
}
// KeptBus is what the keeper needs of the bus beyond Bus to take and restore a kept data.
type KeptBus interface {
// Existing answers which of the stream names the bus holds.
Existing(ctx context.Context, names []string) ([]string, error)
// Positions are the module's durable consumers on every stream (a consumer is the module's when it is named for it).
Positions(ctx context.Context, module string) ([]Position, error)
// SetPositions makes each consumer again from the place it stood.
SetPositions(ctx context.Context, ps []Position) error
// RestoreArchive puts every stream of an archive into the bus, which must hold none of them.
RestoreArchive(ctx context.Context, a *busarchive.Archive) error
// DeleteStreams removes streams by name.
DeleteStreams(ctx context.Context, names []string) error
}
// Point is a restore point's description, its directory's point.json.
type Point struct {
Module string `json:"module"`
Local string `json:"local"`
Form int `json:"form"`
Fingerprint string `json:"fingerprint"`
CatchUp string `json:"catch-up,omitempty"`
Streams []string `json:"streams"`
Positions []Position `json:"positions"`
At time.Time `json:"at"`
Bytes int64 `json:"bytes"`
}
// Aside is data a restore set aside: the live kept data as it was when the restore removed it.
type Aside struct {
ID string `json:"id"`
Module string `json:"module"`
Local string `json:"local"`
Streams []string `json:"streams"`
At time.Time `json:"at"`
Bytes int64 `json:"bytes"`
By string `json:"by"`
Why string `json:"why"`
Dir string `json:"dir"`
}
// PointKey is a restore point's key in StateKept.
func PointKey(module, local string, form int, fingerprint string) string {
return fmt.Sprintf("point.%s.%s.%d.%s", module, local, form, fingerprint)
}
// safeName is one path component made of what a module, a kept data and a fingerprint are made of.
func safeName(s string) bool {
return s != "" && s != "." && s != ".." && !strings.ContainsAny(s, "/\\\x00 *>.")
}
func (k *Keeper) pointDir(module, local string, form int, fingerprint string) string {
return filepath.Join(k.KeptDir, "points", module, local, fmt.Sprintf("f%d-%s", form, fingerprint))
}
// streamsOf is the streams one kept data is: its plan's objects where the keeper made it, else the streams the bus holds
// under its bus name (a foundation part's own section, made by the grants part).
func (k *Keeper) streamsOf(ctx context.Context, module, local string) ([]string, error) {
if held, _, found := k.asked(module); found {
if m, made := held.Made[local]; made && len(m.Plan.Objects) > 0 {
var out []string
for _, o := range m.Plan.Objects {
out = append(out, o.Stream())
}
return out, nil
}
}
name := BusName(module, local)
got, err := k.Kept.Existing(ctx, []string{"KV_" + name, "OBJ_" + name, name})
if err != nil {
return nil, err
}
if len(got) == 0 {
return nil, fmt.Errorf("the bus holds no kept data %s of %s (looked for KV_%s, OBJ_%s and %s)", local, module, name, name, name)
}
return got, nil
}
func (k *Keeper) keptReady(r *Request) error {
if k.Kept == nil || k.KeptDir == "" {
return errors.New("no directory for restore points is set (MESH_NATS_SNAPSHOTS_DIR), so no kept data is taken or restored")
}
if !safeName(r.Module) || !safeName(r.Local) || (!r.AsideOnly && !safeName(r.Fingerprint)) {
return fmt.Errorf("%q, %q and %q are not a module, a kept data and a fingerprint: each is one plain name", r.Module, r.Local, r.Fingerprint)
}
return nil
}
// SnapshotKept takes the restore point of one kept data: the file pair in a directory named for its form and fingerprint,
// written whole before its name is given, and its record in StateKept. It answers the directory.
func (k *Keeper) SnapshotKept(ctx context.Context, r *Request) (string, error) {
if err := k.keptReady(r); err != nil {
return "", err
}
streams, err := k.streamsOf(ctx, r.Module, r.Local)
if err != nil {
return "", err
}
// The positions first: a consumer that moves while the snapshot is taken is then a little behind its archive, and a
// restored consumer that is behind hears a message twice, never misses one.
positions, err := k.Kept.Positions(ctx, r.Module)
if err != nil {
return "", fmt.Errorf("the consumers' positions of %s cannot be read, so no restore point is taken: %w", r.Module, err)
}
tar, err := k.Bus.Snapshot(ctx, streams)
if err != nil {
return "", fmt.Errorf("the snapshot of %s's %s failed, so no restore point is taken: %w", r.Module, r.Local, err)
}
dir := k.pointDir(r.Module, r.Local, r.Form, r.Fingerprint)
if err := os.MkdirAll(filepath.Dir(dir), 0o700); err != nil {
return "", err
}
partial := dir + ".partial"
_ = os.RemoveAll(partial)
if err := os.Mkdir(partial, 0o700); err != nil {
return "", err
}
if err := os.Rename(tar, filepath.Join(partial, ArchiveFile)); err != nil {
_ = os.RemoveAll(partial)
return "", fmt.Errorf("the snapshot cannot be kept as a restore point: %w", err)
}
st, _ := os.Stat(filepath.Join(partial, ArchiveFile))
p := Point{Module: r.Module, Local: r.Local, Form: r.Form, Fingerprint: r.Fingerprint, CatchUp: r.CatchUp,
Streams: streams, Positions: positions, At: k.now()}
if st != nil {
p.Bytes = st.Size()
}
raw, _ := json.MarshalIndent(p, "", " ")
if err := os.WriteFile(filepath.Join(partial, PointFile), append(raw, '\n'), 0o600); err != nil {
_ = os.RemoveAll(partial)
return "", err
}
// A restore point taken again at the same form and fingerprint replaces the earlier one, whole.
_ = os.RemoveAll(dir)
if err := os.Rename(partial, dir); err != nil {
return "", err
}
val, _ := json.Marshal(p)
if _, err := k.Store.Put(StateKept, PointKey(r.Module, r.Local, r.Form, r.Fingerprint), val); err != nil {
k.logf("the record of the restore point %s cannot be written to its own %s: %v", dir, StateKept, err)
}
r.Dir = dir
k.record(r, fmt.Sprintf("took the restore point of %s's %s at form %d before %s (%d stream(s), %d consumer position(s), %d bytes)",
r.Module, r.Local, r.Form, r.Fingerprint, len(streams), len(positions), p.Bytes))
return fmt.Sprintf("restore point of %s's %s at form %d before %s in %s", r.Module, r.Local, r.Form, r.Fingerprint, dir), nil
}
// readPoint reads and checks a restore point's directory: its point.json names what the request names, its files are
// plain files (never a link), and its archive matches its own manifest.
func readPoint(dir string, r *Request) (*Point, *busarchive.Archive, error) {
for _, name := range []string{PointFile, ArchiveFile} {
fi, err := os.Lstat(filepath.Join(dir, name))
if err != nil {
return nil, nil, fmt.Errorf("the restore point %s holds no %s: %w", dir, name, err)
}
if !fi.Mode().IsRegular() {
return nil, nil, fmt.Errorf("%s of the restore point %s is not a plain file", name, dir)
}
}
raw, err := os.ReadFile(filepath.Join(dir, PointFile))
if err != nil {
return nil, nil, err
}
var p Point
if err := json.Unmarshal(raw, &p); err != nil {
return nil, nil, fmt.Errorf("%s of %s does not read: %w", PointFile, dir, err)
}
if p.Module != r.Module || p.Local != r.Local || p.Form != r.Form || p.Fingerprint != r.Fingerprint {
return nil, nil, fmt.Errorf("the directory %s holds the restore point of %s's %s at form %d before %s, not of %s's %s at form %d before %s",
dir, p.Module, p.Local, p.Form, p.Fingerprint, r.Module, r.Local, r.Form, r.Fingerprint)
}
f, err := os.Open(filepath.Join(dir, ArchiveFile))
if err != nil {
return nil, nil, err
}
defer f.Close()
a, err := busarchive.ReadArchive(f, "")
if err != nil {
return nil, nil, fmt.Errorf("the archive of the restore point %s does not read: %w", dir, err)
}
if err := a.Verify(); err != nil {
a.Close()
return nil, nil, fmt.Errorf("the archive of the restore point %s does not match its own manifest, so nothing is restored: %w", dir, err)
}
return &p, a, nil
}
// RestoreKept restores one kept data beside the live data: the live data is snapshotted into a set-aside directory first
// and only then removed from the bus, and the restore point's streams are put in its place; the consumers the module
// reads with on other streams are set back to their positions. It answers what was set aside, where.
func (k *Keeper) RestoreKept(ctx context.Context, r *Request) (string, error) {
if err := k.keptReady(r); err != nil {
return "", err
}
var p *Point
var a *busarchive.Archive
var live []string
if r.AsideOnly {
// `aside-only` (design 47 G §405): the live data is set aside and nothing is restored.
streams, err := k.streamsOf(ctx, r.Module, r.Local)
if err != nil {
return "", err
}
live = streams
} else {
if !filepath.IsAbs(r.From) {
return "", fmt.Errorf("the restore point's directory is given as an absolute path, not %q", r.From)
}
var err error
p, a, err = readPoint(filepath.Clean(r.From), r)
if err != nil {
return "", err
}
defer a.Close()
live, err = k.Kept.Existing(ctx, p.Streams)
if err != nil {
return "", err
}
}
id := fmt.Sprintf("%s.%s.%s", r.Module, r.Local, k.now().Format("20060102t150405"))
var aside *Aside
if len(live) > 0 {
// Aside before anything is removed: a snapshot that fails leaves the live data where it is.
tar, err := k.Bus.Snapshot(ctx, live)
if err != nil {
return "", fmt.Errorf("the live %s of %s could not be set aside, so nothing was removed: %w", r.Local, r.Module, err)
}
dir := filepath.Join(k.KeptDir, "aside", id)
if err := os.MkdirAll(dir, 0o700); err != nil {
return "", err
}
if err := os.Rename(tar, filepath.Join(dir, ArchiveFile)); err != nil {
return "", fmt.Errorf("the live data's snapshot cannot be kept: %w", err)
}
st, _ := os.Stat(filepath.Join(dir, ArchiveFile))
aside = &Aside{ID: id, Module: r.Module, Local: r.Local, Streams: live, At: k.now(), By: r.Caller, Why: r.Why, Dir: dir}
if st != nil {
aside.Bytes = st.Size()
}
raw, _ := json.MarshalIndent(aside, "", " ")
if err := os.WriteFile(filepath.Join(dir, AsideFile), append(raw, '\n'), 0o600); err != nil {
return "", err
}
val, _ := json.Marshal(aside)
if _, err := k.Store.Put(StateKept, "aside."+id, val); err != nil {
return "", fmt.Errorf("what is set aside cannot be recorded, so nothing was removed: %w", err)
}
if err := k.Kept.DeleteStreams(ctx, live); err != nil {
return "", fmt.Errorf("the live data was set aside in %s and could not be removed from the bus: %w", dir, err)
}
}
if r.AsideOnly {
r.Dir = ""
where := "nothing was live, so nothing was set aside"
if aside != nil {
r.Dir = aside.Dir
where = fmt.Sprintf("the live data is set aside as %s (%d bytes) in %s", aside.ID, aside.Bytes, aside.Dir)
}
k.record(r, fmt.Sprintf("set aside %s's %s and restored nothing; %s", r.Module, r.Local, where))
return fmt.Sprintf("set aside %s's %s; %s", r.Module, r.Local, where), nil
}
if err := k.Kept.RestoreArchive(ctx, a); err != nil {
msg := fmt.Sprintf("the restore point could not be put on the bus: %v", err)
if aside != nil {
if back, rerr := k.putBack(ctx, aside); rerr != nil {
msg += fmt.Sprintf("; the live data, set aside in %s, could not be put back either: %v", aside.Dir, rerr)
} else {
msg += fmt.Sprintf("; the live data is back as it was (%s)", back)
}
}
return "", errors.New(msg)
}
// The consumers on streams the restore point did not carry (the snapshot of a stream carries its own consumers).
var apply []Position
for _, pos := range p.Positions {
if !slices.Contains(p.Streams, pos.Stream) {
apply = append(apply, pos)
}
}
if len(apply) > 0 {
if err := k.Kept.SetPositions(ctx, apply); err != nil {
return "", fmt.Errorf("the kept data is restored, but the consumers' positions could not be set back: %w", err)
}
}
r.Dir, r.CatchUp = "", p.CatchUp
where := "nothing was live, so nothing was set aside"
if aside != nil {
r.Dir = aside.Dir
where = fmt.Sprintf("the live data is set aside as %s (%d bytes) in %s", aside.ID, aside.Bytes, aside.Dir)
}
k.record(r, fmt.Sprintf("restored %s's %s from the restore point at form %d before %s beside the live data; %s", r.Module, r.Local, r.Form, r.Fingerprint, where))
return fmt.Sprintf("restored %s's %s from its restore point; %s; %d consumer position(s) set back; catch-up tool %q", r.Module, r.Local, where, len(apply), p.CatchUp), nil
}
// putBack restores set-aside data to the bus, after a restore failed.
func (k *Keeper) putBack(ctx context.Context, a *Aside) (string, error) {
f, err := os.Open(filepath.Join(a.Dir, ArchiveFile))
if err != nil {
return "", err
}
defer f.Close()
arch, err := busarchive.ReadArchive(f, "")
if err != nil {
return "", err
}
defer arch.Close()
if err := k.Kept.DeleteStreams(ctx, a.Streams); err != nil {
return "", err
}
if err := k.Kept.RestoreArchive(ctx, arch); err != nil {
return "", err
}
return "from " + a.Dir, nil
}
// DiscardAside removes the data a restore set aside, by its id (Request.From), on a person's call: the request says who
// and why, and is recorded as a hand act before the data goes.
func (k *Keeper) DiscardAside(r *Request) (string, error) {
if k.KeptDir == "" {
return "", errors.New("no directory for restore points is set (MESH_NATS_SNAPSHOTS_DIR)")
}
id := strings.TrimSpace(r.From)
if !safeAsideID(id) {
return "", fmt.Errorf("%q is not the id of data set aside", id)
}
if strings.TrimSpace(r.Why) == "" {
return "", errors.New("discarding set-aside data says why")
}
raw, _, found, err := k.Store.Get(StateKept, "aside."+id)
if err != nil {
return "", err
}
if !found {
return "nothing done", fmt.Errorf("no data is set aside as %s", id)
}
var a Aside
_ = json.Unmarshal(raw, &a)
k.record(r, fmt.Sprintf("discarded the data set aside as %s (%s's %s, %d bytes)", id, a.Module, a.Local, a.Bytes))
if r.Unrecorded != "" {
return "nothing done", fmt.Errorf("the hand act cannot be recorded, so the set-aside data stays: %s", r.Unrecorded)
}
if err := os.RemoveAll(filepath.Join(k.KeptDir, "aside", id)); err != nil {
return "partly done", err
}
if err := k.Store.Delete(StateKept, "aside."+id); err != nil {
return "partly done", err
}
return fmt.Sprintf("discarded the data set aside as %s", id), nil
}
func safeAsideID(id string) bool {
return id != "" && !strings.ContainsAny(id, "/\\\x00 *>") && !strings.HasPrefix(id, ".")
}
@@ -1,128 +0,0 @@
package keeper
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"nats-tools/internal/busarchive"
"nats-tools/internal/bussnap"
)
// KeptJetStream is KeptBus on the keeper's own connection.
type KeptJetStream struct {
NC *nats.Conn
JS jetstream.JetStream
Logf func(string, ...any)
}
// Existing answers which of the names are streams the bus holds.
func (b KeptJetStream) Existing(ctx context.Context, names []string) ([]string, error) {
var out []string
for _, n := range names {
_, err := b.JS.Stream(ctx, n)
switch {
case err == nil:
out = append(out, n)
case errors.Is(err, jetstream.ErrStreamNotFound):
default:
return nil, err
}
}
return out, nil
}
// ConsumerOf is whether a durable consumer is the module's: named for it, alone or after the machine that runs it
// (`<node>_<module>`, process.Account.QueueConsumerName).
func ConsumerOf(consumer, module string) bool {
return consumer == module || strings.HasSuffix(consumer, "_"+module)
}
// Positions are the module's durable consumers on every stream, each with the stream sequence it has acknowledged up to.
func (b KeptJetStream) Positions(ctx context.Context, module string) ([]Position, error) {
var out []Position
streams := b.JS.ListStreams(ctx)
var names []string
for s := range streams.Info() {
names = append(names, s.Config.Name)
}
if err := streams.Err(); err != nil {
return nil, err
}
for _, name := range names {
st, err := b.JS.Stream(ctx, name)
if err != nil {
return nil, err
}
list := st.ListConsumers(ctx)
for ci := range list.Info() {
if !ConsumerOf(ci.Name, module) {
continue
}
cfg, err := json.Marshal(ci.Config)
if err != nil {
return nil, err
}
out = append(out, Position{Stream: name, Consumer: ci.Name, AckFloor: ci.AckFloor.Stream, Config: cfg})
}
if err := list.Err(); err != nil {
return nil, err
}
}
return out, nil
}
// SetPositions makes each consumer again, from the sequence after the one it had acknowledged. A consumer's place cannot
// be edited, so it is removed and made with its own configuration and a new start.
func (b KeptJetStream) SetPositions(ctx context.Context, ps []Position) error {
for _, p := range ps {
st, err := b.JS.Stream(ctx, p.Stream)
if err != nil {
return fmt.Errorf("the stream %s of %s's position cannot be read: %w", p.Stream, p.Consumer, err)
}
var cfg jetstream.ConsumerConfig
if err := json.Unmarshal(p.Config, &cfg); err != nil {
return fmt.Errorf("the configuration of %s on %s does not read: %w", p.Consumer, p.Stream, err)
}
info, err := st.Info(ctx)
if err != nil {
return err
}
start := p.AckFloor + 1
if start > info.State.LastSeq+1 {
start = info.State.LastSeq + 1
}
cfg.DeliverPolicy, cfg.OptStartSeq, cfg.OptStartTime = jetstream.DeliverByStartSequencePolicy, start, nil
if err := st.DeleteConsumer(ctx, p.Consumer); err != nil && !errors.Is(err, jetstream.ErrConsumerNotFound) {
return fmt.Errorf("%s on %s cannot be removed to be made again: %w", p.Consumer, p.Stream, err)
}
if _, err := st.CreateConsumer(ctx, cfg); err != nil {
return fmt.Errorf("%s on %s cannot be made again at %d: %w", p.Consumer, p.Stream, start, err)
}
}
return nil
}
// RestoreArchive puts every stream of the archive into the bus, which must hold none of them.
func (b KeptJetStream) RestoreArchive(ctx context.Context, a *busarchive.Archive) error {
log := b.Logf
if log == nil {
log = func(string, ...any) {}
}
return bussnap.RestoreAll(ctx, b.NC, a, log)
}
// DeleteStreams removes streams by name.
func (b KeptJetStream) DeleteStreams(ctx context.Context, names []string) error {
for _, n := range names {
if err := b.JS.DeleteStream(ctx, n); err != nil && !errors.Is(err, jetstream.ErrStreamNotFound) {
return fmt.Errorf("%s: %w", n, err)
}
}
return nil
}
-311
View File
@@ -1,311 +0,0 @@
package keeper
import (
"archive/tar"
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/klauspost/compress/s2"
"nats-tools/internal/busarchive"
)
// tarBus is the fake bus whose snapshot is a real archive of the streams asked for, as the server's is.
type tarBus struct {
*fakeBus
dir string
n int
}
func (b *tarBus) Snapshot(ctx context.Context, streams []string) (string, error) {
if _, err := b.fakeBus.Snapshot(ctx, streams); err != nil {
return "", err
}
b.n++
path := filepath.Join(b.dir, fmt.Sprintf(".entry-%d.tar", b.n))
return path, os.WriteFile(path, archiveOf(streams), 0o600)
}
func sha(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) }
// archiveOf is a snapshot archive of the streams, each holding one block.
func archiveOf(streams []string) []byte {
var out bytes.Buffer
tw := tar.NewWriter(&out)
put := func(name string, b []byte) {
_ = tw.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(b))})
_, _ = tw.Write(b)
}
m := busarchive.Manifest{Format: 1, Finished: time.Date(2026, 11, 9, 3, 0, 0, 0, time.UTC)}
files := map[string][]byte{}
for _, name := range streams {
var raw bytes.Buffer
itw := tar.NewWriter(&raw)
_ = itw.WriteHeader(&tar.Header{Name: "msgs.blk", Mode: 0o600, Size: 3})
_, _ = itw.Write([]byte("abc"))
itw.Close()
var arc bytes.Buffer
w := s2.NewWriter(&arc)
_, _ = w.Write(raw.Bytes())
w.Close()
meta := []byte(`{"config":{"name":"` + name + `"}}`)
e := busarchive.StreamEntry{Name: name, Meta: "streams/" + name + "/backup.json", MetaSHA256: sha(meta),
Archive: "streams/" + name + "/stream.tar.s2", ArchiveBytes: int64(arc.Len()), ArchiveSHA256: sha(arc.Bytes())}
m.Streams = append(m.Streams, e)
files[e.Meta], files[e.Archive] = meta, arc.Bytes()
}
raw, _ := json.Marshal(m)
put(busarchive.ManifestName, raw)
for _, s := range m.Streams {
put(s.Meta, files[s.Meta])
put(s.Archive, files[s.Archive])
}
tw.Close()
return out.Bytes()
}
// keptBus is the fake bus's streams and consumers, as the server holds them.
type keptBus struct {
streams map[string]bool
positions []Position
set []Position
restored [][]string
refuse error
}
func (b *keptBus) Existing(_ context.Context, names []string) ([]string, error) {
var out []string
for _, n := range names {
if b.streams[n] {
out = append(out, n)
}
}
return out, nil
}
func (b *keptBus) Positions(context.Context, string) ([]Position, error) { return b.positions, nil }
func (b *keptBus) SetPositions(_ context.Context, ps []Position) error {
b.set = append(b.set, ps...)
return nil
}
func (b *keptBus) RestoreArchive(_ context.Context, a *busarchive.Archive) error {
if err := b.refuse; err != nil {
b.refuse = nil // refused once, as a server that was busy
return err
}
var names []string
for _, s := range a.Manifest.Streams {
if b.streams[s.Name] {
return fmt.Errorf("the server already holds %s", s.Name)
}
names = append(names, s.Name)
}
for _, n := range names {
b.streams[n] = true
}
b.restored = append(b.restored, names)
return nil
}
func (b *keptBus) DeleteStreams(_ context.Context, names []string) error {
for _, n := range names {
delete(b.streams, n)
}
return nil
}
func keptKeeper(t *testing.T) (*Keeper, *keptBus, *memStore, *[]string) {
t.Helper()
k, fb, store, _ := newKeeper(FirstView{})
dir := t.TempDir()
kb := &keptBus{streams: map[string]bool{"KV_idx_modules": true},
positions: []Position{{Stream: "forge", Consumer: "ace_idx", AckFloor: 41, Config: json.RawMessage(`{"durable_name":"ace_idx"}`)},
{Stream: "KV_idx_modules", Consumer: "ace_idx", AckFloor: 7, Config: json.RawMessage(`{"durable_name":"ace_idx"}`)}}}
acts := &[]string{}
k.Bus, k.Kept, k.KeptDir = &tarBus{fakeBus: fb, dir: dir}, kb, dir
k.Act = func(what, why, cause string) error { *acts = append(*acts, what); return nil }
return k, kb, store, acts
}
func take(t *testing.T, k *Keeper) *Request {
t.Helper()
r := &Request{Verb: VerbSnapshotKept, Module: "idx", Local: "modules", Form: 0, Fingerprint: "abc123", CatchUp: "indexer_take_in", Caller: "ace.restic", Why: "before form 1"}
if _, err := k.SnapshotKept(context.Background(), r); err != nil {
t.Fatal(err)
}
return r
}
// A module's kept data on the bus has a restore point before its module moves, and a restore never goes over the live
// data (design 47 "A module's kept data on the bus, backed up and restored with its delivery").
func TestAKeptDataIsBackedUpAndRestoredBesideTheLiveData(t *testing.T) {
for _, row := range []struct {
name string
run func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string)
}{
{"a restore point holds the archive, the positions and the catch-up tool, keyed by form and fingerprint", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) {
r := take(t, k)
want := filepath.Join(k.KeptDir, "points", "idx", "modules", "f0-abc123")
if r.Dir != want {
t.Fatalf("restore point in %s, want %s", r.Dir, want)
}
raw, err := os.ReadFile(filepath.Join(want, PointFile))
if err != nil {
t.Fatal(err)
}
var p Point
if err := json.Unmarshal(raw, &p); err != nil || p.CatchUp != "indexer_take_in" || len(p.Positions) != 2 || p.Streams[0] != "KV_idx_modules" || p.Bytes == 0 {
t.Errorf("point.json = %s (%v)", raw, err)
}
if _, err := os.Stat(filepath.Join(want, ArchiveFile)); err != nil {
t.Error(err)
}
if _, _, found, _ := store.Get(StateKept, PointKey("idx", "modules", 0, "abc123")); !found {
t.Error("the restore point is not recorded in the state kept")
}
if len(*acts) != 1 || !strings.Contains((*acts)[0], "took the restore point") {
t.Errorf("hand acts %v", *acts)
}
}},
{"aside-only sets the live data aside whole and restores nothing", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) {
r := &Request{Verb: VerbRestoreKept, Module: "idx", Local: "modules", AsideOnly: true, Caller: "ace.restic", Why: "newer build stopped"}
if _, err := k.RestoreKept(context.Background(), r); err != nil {
t.Fatal(err)
}
if kb.streams["KV_idx_modules"] || len(kb.restored) != 0 {
t.Errorf("streams %v restored %v: the live data is removed and nothing restored", kb.streams, kb.restored)
}
aside, err := os.ReadDir(filepath.Join(k.KeptDir, "aside"))
if err != nil || len(aside) != 1 || r.Dir == "" {
t.Fatalf("aside: %v %v dir %q", aside, err, r.Dir)
}
}},
{"a kept data the bus does not hold is refused naming what was looked for, and nothing is written", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) {
r := &Request{Module: "idx", Local: "gone", Fingerprint: "abc123"}
_, err := k.SnapshotKept(context.Background(), r)
if err == nil || !strings.Contains(err.Error(), "KV_idx_gone") {
t.Fatalf("err = %v", err)
}
if _, err := os.Stat(filepath.Join(k.KeptDir, "points")); err == nil {
t.Error("a directory was made for a snapshot that was not taken")
}
}},
{"a name that is not one plain name is refused", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) {
for _, r := range []*Request{{Module: "../x", Local: "modules", Fingerprint: "a"}, {Module: "idx", Local: "mod/ules", Fingerprint: "a"}, {Module: "idx", Local: "modules", Fingerprint: ""}} {
if _, err := k.SnapshotKept(context.Background(), r); err == nil {
t.Errorf("%+v was taken", r)
}
}
}},
{"a restore sets the live data aside whole, removes it, restores the archive and sets back the other consumers", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) {
p := take(t, k)
r := &Request{Verb: VerbRestoreKept, Module: "idx", Local: "modules", Form: 0, Fingerprint: "abc123", From: p.Dir, Caller: "ace.restic", Why: "revert"}
if _, err := k.RestoreKept(context.Background(), r); err != nil {
t.Fatal(err)
}
if !kb.streams["KV_idx_modules"] || len(kb.restored) != 1 {
t.Errorf("streams %v restored %v", kb.streams, kb.restored)
}
if r.CatchUp != "indexer_take_in" {
t.Errorf("the catch-up tool is %q", r.CatchUp)
}
aside, err := os.ReadDir(filepath.Join(k.KeptDir, "aside"))
if err != nil || len(aside) != 1 {
t.Fatalf("aside: %v %v", aside, err)
}
if _, err := os.Stat(filepath.Join(k.KeptDir, "aside", aside[0].Name(), ArchiveFile)); err != nil {
t.Error("the live data's archive is not kept")
}
if _, _, found, _ := store.Get(StateKept, "aside."+aside[0].Name()); !found {
t.Error("what is set aside is not recorded")
}
// Only the consumer on a stream the point did not carry is made again; a stream's own comes with its snapshot.
if len(kb.set) != 1 || kb.set[0].Stream != "forge" || kb.set[0].AckFloor != 41 {
t.Errorf("positions set back: %+v", kb.set)
}
}},
{"a restore point of another fingerprint or a linked file is refused before anything is removed", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) {
p := take(t, k)
bad := *p
bad.Fingerprint = "other"
if _, err := k.RestoreKept(context.Background(), &bad); err == nil || !kb.streams["KV_idx_modules"] {
t.Errorf("err = %v, live %v", err, kb.streams)
}
link := filepath.Join(p.Dir, ArchiveFile)
real := filepath.Join(p.Dir, "real.tar")
if err := os.Rename(link, real); err != nil {
t.Fatal(err)
}
if err := os.Symlink(real, link); err != nil {
t.Fatal(err)
}
r := &Request{Module: "idx", Local: "modules", Fingerprint: "abc123", From: p.Dir}
if _, err := k.RestoreKept(context.Background(), r); err == nil || !strings.Contains(err.Error(), "plain file") || !kb.streams["KV_idx_modules"] {
t.Errorf("err = %v, live %v", err, kb.streams)
}
}},
{"a live data that cannot be set aside stays where it is", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) {
p := take(t, k)
k.Bus.(*tarBus).fakeBus.refuse["snapshot"] = errors.New("the server refused")
r := &Request{Module: "idx", Local: "modules", Fingerprint: "abc123", From: p.Dir}
if _, err := k.RestoreKept(context.Background(), r); err == nil || !kb.streams["KV_idx_modules"] || !strings.Contains(err.Error(), "nothing was removed") {
t.Errorf("err = %v, live %v", err, kb.streams)
}
}},
{"a restore that fails puts the live data back and says so", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) {
p := take(t, k)
kb.refuse = errors.New("the server refused the archive")
r := &Request{Module: "idx", Local: "modules", Fingerprint: "abc123", From: p.Dir}
_, err := k.RestoreKept(context.Background(), r)
if err == nil || !strings.Contains(err.Error(), "the live data is back as it was") {
t.Fatalf("err = %v", err)
}
}},
{"set-aside data goes only on a call that says why and is recorded first", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) {
p := take(t, k)
if _, err := k.RestoreKept(context.Background(), &Request{Module: "idx", Local: "modules", Fingerprint: "abc123", From: p.Dir}); err != nil {
t.Fatal(err)
}
entries, _ := os.ReadDir(filepath.Join(k.KeptDir, "aside"))
id := entries[0].Name()
if _, err := k.DiscardAside(&Request{From: id}); err == nil {
t.Error("discarded with no why")
}
if _, err := k.DiscardAside(&Request{From: "../points", Why: "x"}); err == nil {
t.Error("discarded a path")
}
k.Act = func(string, string, string) error { return errors.New("the log is down") }
r := &Request{From: id, Why: "the revert held", Caller: "ace.operator"}
k.record(r, "x") // sets Unrecorded as a failing log does
if _, err := k.DiscardAside(r); err == nil {
t.Error("discarded with the act unrecorded")
}
if _, err := os.Stat(filepath.Join(k.KeptDir, "aside", id)); err != nil {
t.Error("the data went with its act unrecorded")
}
k.Act = func(what, why, cause string) error { *acts = append(*acts, what); return nil }
if _, err := k.DiscardAside(&Request{From: id, Why: "the revert held", Caller: "ace.operator"}); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(filepath.Join(k.KeptDir, "aside", id)); err == nil {
t.Error("still there")
}
if _, _, found, _ := store.Get(StateKept, "aside."+id); found {
t.Error("its record stays")
}
}},
} {
t.Run(row.name, func(t *testing.T) {
k, kb, store, acts := keptKeeper(t)
row.run(t, k, kb, store, acts)
})
}
}
-34
View File
@@ -46,15 +46,6 @@ type Request struct {
Snapshot string `json:"snapshot,omitempty"`
// Waiting is why it waits, while it waits: the operator's warrant, or the keeper's first view.
Waiting string `json:"waiting,omitempty"`
// Form, Fingerprint and CatchUp name a restore point (kept.go); From is the directory a restore reads, or the id of the
// data a discard removes; Dir is what the keeper answers: the restore point's directory, or the data set aside.
Form int `json:"form,omitempty"`
Fingerprint string `json:"fingerprint,omitempty"`
CatchUp string `json:"catch-up,omitempty"`
// AsideOnly is the argument `aside-only` of restore-kept: the live data is set aside and nothing is restored.
AsideOnly bool `json:"aside-only,omitempty"`
From string `json:"from,omitempty"`
Dir string `json:"dir,omitempty"`
// Unrecorded is why the hand act of what it did could not be recorded, when it could not: said, never only logged.
Unrecorded string `json:"unrecorded,omitempty"`
}
@@ -145,10 +136,6 @@ func (k *Keeper) act(r *Request) (string, error) {
}
return fmt.Sprintf("the first view answered at revision %d", s.Revision), nil
}
switch r.Verb {
case VerbSnapshotKept, VerbRestoreKept, VerbDiscardAside:
return k.actKept(r)
}
if !k.Viewed() {
return "", fmt.Errorf("%w: the keeper holds no first view yet, so it acts on nothing", ErrNoWarrant)
}
@@ -308,24 +295,3 @@ func (k *Keeper) record(r *Request, what string) {
k.logf("the hand act %q could not be recorded: %v", what, err)
}
}
// actKept acts on a request about a kept data's restore point. These need no first view or warrant: a snapshot changes
// nothing, a restore sets the live data aside whole before it removes it, and a discard names who and why (kept.go).
func (k *Keeper) actKept(r *Request) (string, error) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
defer cancel()
var out string
var err error
switch r.Verb {
case VerbSnapshotKept:
out, err = k.SnapshotKept(ctx, r)
case VerbRestoreKept:
out, err = k.RestoreKept(ctx, r)
default:
out, err = k.DiscardAside(r)
}
if err != nil {
return "nothing done", err
}
return out, nil
}