NATS10: the bus module serves no restore point tool and keeps no state kept
Revertsbbd9f638,c2e017a4and b6884796; cmd/mesh-nats-snapshot/restore.go is as before them. The manifest test asserts the claim serves only the reading verbs and the keeper's state is asked, requests, spent. Builds: design 47 G §554, §555 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ
This commit is contained in:
@@ -1,126 +0,0 @@
|
||||
package bussnap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"nats-tools/internal/busarchive"
|
||||
)
|
||||
|
||||
// restoreChunk is what is sent at a time; the server answers each before the next goes.
|
||||
const restoreChunk = 64 * 1024
|
||||
|
||||
// RestoreAll puts every stream of the archive into the server nc reaches, one at a time, and holds
|
||||
// each restored stream's state to the manifest. **Nothing is restored over anything**: a server that
|
||||
// already holds any of the streams is refused before the first is sent.
|
||||
func RestoreAll(ctx context.Context, nc *nats.Conn, a *busarchive.Archive, log func(string, ...any)) error {
|
||||
var present []string
|
||||
for _, s := range a.Manifest.Streams {
|
||||
err := Request(ctx, nc, APIStreamInfo+s.Name, nil, nil)
|
||||
var api *APIError
|
||||
switch {
|
||||
case err == nil:
|
||||
present = append(present, s.Name)
|
||||
case errors.As(err, &api) && api.Code == 404:
|
||||
default:
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(present) > 0 {
|
||||
return fmt.Errorf("the server already holds %s; nothing is restored over a live stream — restore into a new "+
|
||||
"store (--into) and swap it in with the bus stopped", strings.Join(present, ", "))
|
||||
}
|
||||
for _, s := range a.Manifest.Streams {
|
||||
started := time.Now()
|
||||
if err := restoreStream(ctx, nc, a, s); err != nil {
|
||||
return fmt.Errorf("stream %s: %w", s.Name, err)
|
||||
}
|
||||
log("%s: restored, at least %d message(s) to sequence %d, in %.2fs", s.Name, s.Messages, s.LastSeq, time.Since(started).Seconds())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func restoreStream(ctx context.Context, nc *nats.Conn, a *busarchive.Archive, s busarchive.StreamEntry) error {
|
||||
meta, archive, err := a.Open(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer archive.Close()
|
||||
var req map[string]json.RawMessage
|
||||
if err := json.Unmarshal(meta, &req); err != nil || req["config"] == nil {
|
||||
return errors.New("its meta is not a stream's configuration and state")
|
||||
}
|
||||
var resp struct {
|
||||
DeliverSubject string `json:"deliver_subject"`
|
||||
}
|
||||
if err := Request(ctx, nc, APIStreamRestore+s.Name, req, &resp); err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.DeliverSubject == "" {
|
||||
return errors.New("the server gave nowhere to send the archive")
|
||||
}
|
||||
buf := make([]byte, restoreChunk)
|
||||
for {
|
||||
n, err := archive.Read(buf)
|
||||
if n > 0 {
|
||||
cctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
reply, rerr := nc.RequestWithContext(cctx, resp.DeliverSubject, buf[:n])
|
||||
cancel()
|
||||
if rerr != nil {
|
||||
return fmt.Errorf("sending the archive: %w", rerr)
|
||||
}
|
||||
if len(reply.Data) > 0 {
|
||||
return fmt.Errorf("the server refused the archive: %s", Truncate(reply.Data))
|
||||
}
|
||||
}
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// The end: an empty message, answered once the server has rebuilt the stream from what it was
|
||||
// sent — which for a large stream takes a while.
|
||||
fctx, cancel := context.WithTimeout(ctx, 30*time.Minute)
|
||||
defer cancel()
|
||||
final, err := nc.RequestWithContext(fctx, resp.DeliverSubject, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("finishing the restore: %w", err)
|
||||
}
|
||||
var created struct {
|
||||
Error *APIError `json:"error"`
|
||||
State StreamState `json:"state"`
|
||||
}
|
||||
if err := json.Unmarshal(final.Data, &created); err != nil {
|
||||
return fmt.Errorf("the server's last answer is not JSON: %q", Truncate(final.Data))
|
||||
}
|
||||
if created.Error != nil {
|
||||
return created.Error
|
||||
}
|
||||
return HeldTo(created.State, s)
|
||||
}
|
||||
|
||||
// HeldTo is whether a restored stream holds at least what its snapshot said it held.
|
||||
//
|
||||
// **At least, not exactly — found by the restore test, not assumed.** The state the server answers a
|
||||
// snapshot request with is taken when the snapshot starts; the stream goes on taking messages while
|
||||
// its blocks are read out, and a block read a moment later carries some of what was appended in that
|
||||
// moment. So a stream written during its snapshot restores with a later last sequence than the
|
||||
// manifest says (90075 where it said 90024, with a writer publishing every 2ms through it), and of
|
||||
// the messages in that tail some may be there and some not. Every message up to the manifest's last
|
||||
// sequence is there, exactly; that is the snapshot's promise, and what the test holds it to message
|
||||
// by message. What must never happen is a restored stream that ends before the manifest said it did.
|
||||
func HeldTo(restored StreamState, s busarchive.StreamEntry) error {
|
||||
if restored.LastSeq < s.LastSeq || (s.Messages > 0 && restored.Messages == 0) {
|
||||
return fmt.Errorf("restored with %d message(s) up to sequence %d; the snapshot held %d up to %d",
|
||||
restored.Messages, restored.LastSeq, s.Messages, s.LastSeq)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -219,9 +219,6 @@ type Keeper struct {
|
||||
Warrant func(askID string) (*oc.Ask, *oc.Warrant, error)
|
||||
Now func() time.Time
|
||||
Logf func(string, ...any)
|
||||
// Kept takes and restores a module's kept data, in KeptDir (kept.go); nil where the process has no such bus.
|
||||
Kept KeptBus
|
||||
KeptDir string
|
||||
// Raise and Clear are the fill marks in percent (ADR 0595 §6): settings of the bus module, 75 and 70 by default.
|
||||
Raise, Clear int64
|
||||
|
||||
|
||||
@@ -1,405 +0,0 @@
|
||||
package keeper
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"nats-tools/internal/busarchive"
|
||||
)
|
||||
|
||||
// A module's kept data on the bus, backed up before the module moves and restored beside the live data when a want takes
|
||||
// it back (milestone G, design 47 "A module's kept data on the bus, backed up and restored with its delivery"). The
|
||||
// keeper takes it on its own account, as it takes a snapshot before a recreate (ADR 0235 §5): the streams of one kept
|
||||
// data as the server's own snapshot, the position of each durable consumer the module reads with, and the catch-up tool
|
||||
// the newer manifest declares, in one directory the backup holder keeps as a restore point. A restore never goes over
|
||||
// the live data: the live data is first snapshotted into a set-aside directory under a name of its own, and only then
|
||||
// removed from the bus; only discard-aside removes the set-aside copy, on a person's call.
|
||||
|
||||
const (
|
||||
// VerbSnapshotKept takes a restore point of one kept data; VerbRestoreKept restores one beside the live data.
|
||||
VerbSnapshotKept = "snapshot-kept"
|
||||
VerbRestoreKept = "restore-kept"
|
||||
// VerbDiscardAside removes the data a restore set aside, by its id.
|
||||
VerbDiscardAside = "discard-aside"
|
||||
)
|
||||
|
||||
// StateKept is the keeper's own state of the restore points it took and the data it set aside (never their bytes, which
|
||||
// are files in the keeper's directory): `point.<module>.<local>.<form>.<fingerprint>` and `aside.<id>`. Read by the
|
||||
// bus module's tool nats_kept and by the seat mesh-broker's `aside`.
|
||||
const StateKept = "kept"
|
||||
|
||||
// PointFile and ArchiveFile are the two files of a restore point's directory; AsideFile of a set-aside directory.
|
||||
const (
|
||||
PointFile = "point.json"
|
||||
ArchiveFile = "bus.tar"
|
||||
AsideFile = "aside.json"
|
||||
)
|
||||
|
||||
// Position is where one durable consumer of the module stood on a stream: the stream sequence it had acknowledged up to,
|
||||
// and its configuration as the server held it, so a restore can make it again at that place.
|
||||
type Position struct {
|
||||
Stream string `json:"stream"`
|
||||
Consumer string `json:"consumer"`
|
||||
AckFloor uint64 `json:"ack-floor"`
|
||||
Config json.RawMessage `json:"config"`
|
||||
}
|
||||
|
||||
// KeptBus is what the keeper needs of the bus beyond Bus to take and restore a kept data.
|
||||
type KeptBus interface {
|
||||
// Existing answers which of the stream names the bus holds.
|
||||
Existing(ctx context.Context, names []string) ([]string, error)
|
||||
// Positions are the module's durable consumers on every stream (a consumer is the module's when it is named for it).
|
||||
Positions(ctx context.Context, module string) ([]Position, error)
|
||||
// SetPositions makes each consumer again from the place it stood.
|
||||
SetPositions(ctx context.Context, ps []Position) error
|
||||
// RestoreArchive puts every stream of an archive into the bus, which must hold none of them.
|
||||
RestoreArchive(ctx context.Context, a *busarchive.Archive) error
|
||||
// DeleteStreams removes streams by name.
|
||||
DeleteStreams(ctx context.Context, names []string) error
|
||||
}
|
||||
|
||||
// Point is a restore point's description, its directory's point.json.
|
||||
type Point struct {
|
||||
Module string `json:"module"`
|
||||
Local string `json:"local"`
|
||||
Form int `json:"form"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
CatchUp string `json:"catch-up,omitempty"`
|
||||
Streams []string `json:"streams"`
|
||||
Positions []Position `json:"positions"`
|
||||
At time.Time `json:"at"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
}
|
||||
|
||||
// Aside is data a restore set aside: the live kept data as it was when the restore removed it.
|
||||
type Aside struct {
|
||||
ID string `json:"id"`
|
||||
Module string `json:"module"`
|
||||
Local string `json:"local"`
|
||||
Streams []string `json:"streams"`
|
||||
At time.Time `json:"at"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
By string `json:"by"`
|
||||
Why string `json:"why"`
|
||||
Dir string `json:"dir"`
|
||||
}
|
||||
|
||||
// PointKey is a restore point's key in StateKept.
|
||||
func PointKey(module, local string, form int, fingerprint string) string {
|
||||
return fmt.Sprintf("point.%s.%s.%d.%s", module, local, form, fingerprint)
|
||||
}
|
||||
|
||||
// safeName is one path component made of what a module, a kept data and a fingerprint are made of.
|
||||
func safeName(s string) bool {
|
||||
return s != "" && s != "." && s != ".." && !strings.ContainsAny(s, "/\\\x00 *>.")
|
||||
}
|
||||
|
||||
func (k *Keeper) pointDir(module, local string, form int, fingerprint string) string {
|
||||
return filepath.Join(k.KeptDir, "points", module, local, fmt.Sprintf("f%d-%s", form, fingerprint))
|
||||
}
|
||||
|
||||
// streamsOf is the streams one kept data is: its plan's objects where the keeper made it, else the streams the bus holds
|
||||
// under its bus name (a foundation part's own section, made by the grants part).
|
||||
func (k *Keeper) streamsOf(ctx context.Context, module, local string) ([]string, error) {
|
||||
if held, _, found := k.asked(module); found {
|
||||
if m, made := held.Made[local]; made && len(m.Plan.Objects) > 0 {
|
||||
var out []string
|
||||
for _, o := range m.Plan.Objects {
|
||||
out = append(out, o.Stream())
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
name := BusName(module, local)
|
||||
got, err := k.Kept.Existing(ctx, []string{"KV_" + name, "OBJ_" + name, name})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(got) == 0 {
|
||||
return nil, fmt.Errorf("the bus holds no kept data %s of %s (looked for KV_%s, OBJ_%s and %s)", local, module, name, name, name)
|
||||
}
|
||||
return got, nil
|
||||
}
|
||||
|
||||
func (k *Keeper) keptReady(r *Request) error {
|
||||
if k.Kept == nil || k.KeptDir == "" {
|
||||
return errors.New("no directory for restore points is set (MESH_NATS_SNAPSHOTS_DIR), so no kept data is taken or restored")
|
||||
}
|
||||
if !safeName(r.Module) || !safeName(r.Local) || (!r.AsideOnly && !safeName(r.Fingerprint)) {
|
||||
return fmt.Errorf("%q, %q and %q are not a module, a kept data and a fingerprint: each is one plain name", r.Module, r.Local, r.Fingerprint)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SnapshotKept takes the restore point of one kept data: the file pair in a directory named for its form and fingerprint,
|
||||
// written whole before its name is given, and its record in StateKept. It answers the directory.
|
||||
func (k *Keeper) SnapshotKept(ctx context.Context, r *Request) (string, error) {
|
||||
if err := k.keptReady(r); err != nil {
|
||||
return "", err
|
||||
}
|
||||
streams, err := k.streamsOf(ctx, r.Module, r.Local)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// The positions first: a consumer that moves while the snapshot is taken is then a little behind its archive, and a
|
||||
// restored consumer that is behind hears a message twice, never misses one.
|
||||
positions, err := k.Kept.Positions(ctx, r.Module)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the consumers' positions of %s cannot be read, so no restore point is taken: %w", r.Module, err)
|
||||
}
|
||||
tar, err := k.Bus.Snapshot(ctx, streams)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the snapshot of %s's %s failed, so no restore point is taken: %w", r.Module, r.Local, err)
|
||||
}
|
||||
dir := k.pointDir(r.Module, r.Local, r.Form, r.Fingerprint)
|
||||
if err := os.MkdirAll(filepath.Dir(dir), 0o700); err != nil {
|
||||
return "", err
|
||||
}
|
||||
partial := dir + ".partial"
|
||||
_ = os.RemoveAll(partial)
|
||||
if err := os.Mkdir(partial, 0o700); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.Rename(tar, filepath.Join(partial, ArchiveFile)); err != nil {
|
||||
_ = os.RemoveAll(partial)
|
||||
return "", fmt.Errorf("the snapshot cannot be kept as a restore point: %w", err)
|
||||
}
|
||||
st, _ := os.Stat(filepath.Join(partial, ArchiveFile))
|
||||
p := Point{Module: r.Module, Local: r.Local, Form: r.Form, Fingerprint: r.Fingerprint, CatchUp: r.CatchUp,
|
||||
Streams: streams, Positions: positions, At: k.now()}
|
||||
if st != nil {
|
||||
p.Bytes = st.Size()
|
||||
}
|
||||
raw, _ := json.MarshalIndent(p, "", " ")
|
||||
if err := os.WriteFile(filepath.Join(partial, PointFile), append(raw, '\n'), 0o600); err != nil {
|
||||
_ = os.RemoveAll(partial)
|
||||
return "", err
|
||||
}
|
||||
// A restore point taken again at the same form and fingerprint replaces the earlier one, whole.
|
||||
_ = os.RemoveAll(dir)
|
||||
if err := os.Rename(partial, dir); err != nil {
|
||||
return "", err
|
||||
}
|
||||
val, _ := json.Marshal(p)
|
||||
if _, err := k.Store.Put(StateKept, PointKey(r.Module, r.Local, r.Form, r.Fingerprint), val); err != nil {
|
||||
k.logf("the record of the restore point %s cannot be written to its own %s: %v", dir, StateKept, err)
|
||||
}
|
||||
r.Dir = dir
|
||||
k.record(r, fmt.Sprintf("took the restore point of %s's %s at form %d before %s (%d stream(s), %d consumer position(s), %d bytes)",
|
||||
r.Module, r.Local, r.Form, r.Fingerprint, len(streams), len(positions), p.Bytes))
|
||||
return fmt.Sprintf("restore point of %s's %s at form %d before %s in %s", r.Module, r.Local, r.Form, r.Fingerprint, dir), nil
|
||||
}
|
||||
|
||||
// readPoint reads and checks a restore point's directory: its point.json names what the request names, its files are
|
||||
// plain files (never a link), and its archive matches its own manifest.
|
||||
func readPoint(dir string, r *Request) (*Point, *busarchive.Archive, error) {
|
||||
for _, name := range []string{PointFile, ArchiveFile} {
|
||||
fi, err := os.Lstat(filepath.Join(dir, name))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the restore point %s holds no %s: %w", dir, name, err)
|
||||
}
|
||||
if !fi.Mode().IsRegular() {
|
||||
return nil, nil, fmt.Errorf("%s of the restore point %s is not a plain file", name, dir)
|
||||
}
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(dir, PointFile))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var p Point
|
||||
if err := json.Unmarshal(raw, &p); err != nil {
|
||||
return nil, nil, fmt.Errorf("%s of %s does not read: %w", PointFile, dir, err)
|
||||
}
|
||||
if p.Module != r.Module || p.Local != r.Local || p.Form != r.Form || p.Fingerprint != r.Fingerprint {
|
||||
return nil, nil, fmt.Errorf("the directory %s holds the restore point of %s's %s at form %d before %s, not of %s's %s at form %d before %s",
|
||||
dir, p.Module, p.Local, p.Form, p.Fingerprint, r.Module, r.Local, r.Form, r.Fingerprint)
|
||||
}
|
||||
f, err := os.Open(filepath.Join(dir, ArchiveFile))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
a, err := busarchive.ReadArchive(f, "")
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the archive of the restore point %s does not read: %w", dir, err)
|
||||
}
|
||||
if err := a.Verify(); err != nil {
|
||||
a.Close()
|
||||
return nil, nil, fmt.Errorf("the archive of the restore point %s does not match its own manifest, so nothing is restored: %w", dir, err)
|
||||
}
|
||||
return &p, a, nil
|
||||
}
|
||||
|
||||
// RestoreKept restores one kept data beside the live data: the live data is snapshotted into a set-aside directory first
|
||||
// and only then removed from the bus, and the restore point's streams are put in its place; the consumers the module
|
||||
// reads with on other streams are set back to their positions. It answers what was set aside, where.
|
||||
func (k *Keeper) RestoreKept(ctx context.Context, r *Request) (string, error) {
|
||||
if err := k.keptReady(r); err != nil {
|
||||
return "", err
|
||||
}
|
||||
var p *Point
|
||||
var a *busarchive.Archive
|
||||
var live []string
|
||||
if r.AsideOnly {
|
||||
// `aside-only` (design 47 G §405): the live data is set aside and nothing is restored.
|
||||
streams, err := k.streamsOf(ctx, r.Module, r.Local)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
live = streams
|
||||
} else {
|
||||
if !filepath.IsAbs(r.From) {
|
||||
return "", fmt.Errorf("the restore point's directory is given as an absolute path, not %q", r.From)
|
||||
}
|
||||
var err error
|
||||
p, a, err = readPoint(filepath.Clean(r.From), r)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer a.Close()
|
||||
live, err = k.Kept.Existing(ctx, p.Streams)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
id := fmt.Sprintf("%s.%s.%s", r.Module, r.Local, k.now().Format("20060102t150405"))
|
||||
var aside *Aside
|
||||
if len(live) > 0 {
|
||||
// Aside before anything is removed: a snapshot that fails leaves the live data where it is.
|
||||
tar, err := k.Bus.Snapshot(ctx, live)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the live %s of %s could not be set aside, so nothing was removed: %w", r.Local, r.Module, err)
|
||||
}
|
||||
dir := filepath.Join(k.KeptDir, "aside", id)
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.Rename(tar, filepath.Join(dir, ArchiveFile)); err != nil {
|
||||
return "", fmt.Errorf("the live data's snapshot cannot be kept: %w", err)
|
||||
}
|
||||
st, _ := os.Stat(filepath.Join(dir, ArchiveFile))
|
||||
aside = &Aside{ID: id, Module: r.Module, Local: r.Local, Streams: live, At: k.now(), By: r.Caller, Why: r.Why, Dir: dir}
|
||||
if st != nil {
|
||||
aside.Bytes = st.Size()
|
||||
}
|
||||
raw, _ := json.MarshalIndent(aside, "", " ")
|
||||
if err := os.WriteFile(filepath.Join(dir, AsideFile), append(raw, '\n'), 0o600); err != nil {
|
||||
return "", err
|
||||
}
|
||||
val, _ := json.Marshal(aside)
|
||||
if _, err := k.Store.Put(StateKept, "aside."+id, val); err != nil {
|
||||
return "", fmt.Errorf("what is set aside cannot be recorded, so nothing was removed: %w", err)
|
||||
}
|
||||
if err := k.Kept.DeleteStreams(ctx, live); err != nil {
|
||||
return "", fmt.Errorf("the live data was set aside in %s and could not be removed from the bus: %w", dir, err)
|
||||
}
|
||||
}
|
||||
if r.AsideOnly {
|
||||
r.Dir = ""
|
||||
where := "nothing was live, so nothing was set aside"
|
||||
if aside != nil {
|
||||
r.Dir = aside.Dir
|
||||
where = fmt.Sprintf("the live data is set aside as %s (%d bytes) in %s", aside.ID, aside.Bytes, aside.Dir)
|
||||
}
|
||||
k.record(r, fmt.Sprintf("set aside %s's %s and restored nothing; %s", r.Module, r.Local, where))
|
||||
return fmt.Sprintf("set aside %s's %s; %s", r.Module, r.Local, where), nil
|
||||
}
|
||||
if err := k.Kept.RestoreArchive(ctx, a); err != nil {
|
||||
msg := fmt.Sprintf("the restore point could not be put on the bus: %v", err)
|
||||
if aside != nil {
|
||||
if back, rerr := k.putBack(ctx, aside); rerr != nil {
|
||||
msg += fmt.Sprintf("; the live data, set aside in %s, could not be put back either: %v", aside.Dir, rerr)
|
||||
} else {
|
||||
msg += fmt.Sprintf("; the live data is back as it was (%s)", back)
|
||||
}
|
||||
}
|
||||
return "", errors.New(msg)
|
||||
}
|
||||
// The consumers on streams the restore point did not carry (the snapshot of a stream carries its own consumers).
|
||||
var apply []Position
|
||||
for _, pos := range p.Positions {
|
||||
if !slices.Contains(p.Streams, pos.Stream) {
|
||||
apply = append(apply, pos)
|
||||
}
|
||||
}
|
||||
if len(apply) > 0 {
|
||||
if err := k.Kept.SetPositions(ctx, apply); err != nil {
|
||||
return "", fmt.Errorf("the kept data is restored, but the consumers' positions could not be set back: %w", err)
|
||||
}
|
||||
}
|
||||
r.Dir, r.CatchUp = "", p.CatchUp
|
||||
where := "nothing was live, so nothing was set aside"
|
||||
if aside != nil {
|
||||
r.Dir = aside.Dir
|
||||
where = fmt.Sprintf("the live data is set aside as %s (%d bytes) in %s", aside.ID, aside.Bytes, aside.Dir)
|
||||
}
|
||||
k.record(r, fmt.Sprintf("restored %s's %s from the restore point at form %d before %s beside the live data; %s", r.Module, r.Local, r.Form, r.Fingerprint, where))
|
||||
return fmt.Sprintf("restored %s's %s from its restore point; %s; %d consumer position(s) set back; catch-up tool %q", r.Module, r.Local, where, len(apply), p.CatchUp), nil
|
||||
}
|
||||
|
||||
// putBack restores set-aside data to the bus, after a restore failed.
|
||||
func (k *Keeper) putBack(ctx context.Context, a *Aside) (string, error) {
|
||||
f, err := os.Open(filepath.Join(a.Dir, ArchiveFile))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer f.Close()
|
||||
arch, err := busarchive.ReadArchive(f, "")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer arch.Close()
|
||||
if err := k.Kept.DeleteStreams(ctx, a.Streams); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := k.Kept.RestoreArchive(ctx, arch); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "from " + a.Dir, nil
|
||||
}
|
||||
|
||||
// DiscardAside removes the data a restore set aside, by its id (Request.From), on a person's call: the request says who
|
||||
// and why, and is recorded as a hand act before the data goes.
|
||||
func (k *Keeper) DiscardAside(r *Request) (string, error) {
|
||||
if k.KeptDir == "" {
|
||||
return "", errors.New("no directory for restore points is set (MESH_NATS_SNAPSHOTS_DIR)")
|
||||
}
|
||||
id := strings.TrimSpace(r.From)
|
||||
if !safeAsideID(id) {
|
||||
return "", fmt.Errorf("%q is not the id of data set aside", id)
|
||||
}
|
||||
if strings.TrimSpace(r.Why) == "" {
|
||||
return "", errors.New("discarding set-aside data says why")
|
||||
}
|
||||
raw, _, found, err := k.Store.Get(StateKept, "aside."+id)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !found {
|
||||
return "nothing done", fmt.Errorf("no data is set aside as %s", id)
|
||||
}
|
||||
var a Aside
|
||||
_ = json.Unmarshal(raw, &a)
|
||||
k.record(r, fmt.Sprintf("discarded the data set aside as %s (%s's %s, %d bytes)", id, a.Module, a.Local, a.Bytes))
|
||||
if r.Unrecorded != "" {
|
||||
return "nothing done", fmt.Errorf("the hand act cannot be recorded, so the set-aside data stays: %s", r.Unrecorded)
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(k.KeptDir, "aside", id)); err != nil {
|
||||
return "partly done", err
|
||||
}
|
||||
if err := k.Store.Delete(StateKept, "aside."+id); err != nil {
|
||||
return "partly done", err
|
||||
}
|
||||
return fmt.Sprintf("discarded the data set aside as %s", id), nil
|
||||
}
|
||||
|
||||
func safeAsideID(id string) bool {
|
||||
return id != "" && !strings.ContainsAny(id, "/\\\x00 *>") && !strings.HasPrefix(id, ".")
|
||||
}
|
||||
@@ -1,128 +0,0 @@
|
||||
package keeper
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"nats-tools/internal/busarchive"
|
||||
"nats-tools/internal/bussnap"
|
||||
)
|
||||
|
||||
// KeptJetStream is KeptBus on the keeper's own connection.
|
||||
type KeptJetStream struct {
|
||||
NC *nats.Conn
|
||||
JS jetstream.JetStream
|
||||
Logf func(string, ...any)
|
||||
}
|
||||
|
||||
// Existing answers which of the names are streams the bus holds.
|
||||
func (b KeptJetStream) Existing(ctx context.Context, names []string) ([]string, error) {
|
||||
var out []string
|
||||
for _, n := range names {
|
||||
_, err := b.JS.Stream(ctx, n)
|
||||
switch {
|
||||
case err == nil:
|
||||
out = append(out, n)
|
||||
case errors.Is(err, jetstream.ErrStreamNotFound):
|
||||
default:
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ConsumerOf is whether a durable consumer is the module's: named for it, alone or after the machine that runs it
|
||||
// (`<node>_<module>`, process.Account.QueueConsumerName).
|
||||
func ConsumerOf(consumer, module string) bool {
|
||||
return consumer == module || strings.HasSuffix(consumer, "_"+module)
|
||||
}
|
||||
|
||||
// Positions are the module's durable consumers on every stream, each with the stream sequence it has acknowledged up to.
|
||||
func (b KeptJetStream) Positions(ctx context.Context, module string) ([]Position, error) {
|
||||
var out []Position
|
||||
streams := b.JS.ListStreams(ctx)
|
||||
var names []string
|
||||
for s := range streams.Info() {
|
||||
names = append(names, s.Config.Name)
|
||||
}
|
||||
if err := streams.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, name := range names {
|
||||
st, err := b.JS.Stream(ctx, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
list := st.ListConsumers(ctx)
|
||||
for ci := range list.Info() {
|
||||
if !ConsumerOf(ci.Name, module) {
|
||||
continue
|
||||
}
|
||||
cfg, err := json.Marshal(ci.Config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, Position{Stream: name, Consumer: ci.Name, AckFloor: ci.AckFloor.Stream, Config: cfg})
|
||||
}
|
||||
if err := list.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// SetPositions makes each consumer again, from the sequence after the one it had acknowledged. A consumer's place cannot
|
||||
// be edited, so it is removed and made with its own configuration and a new start.
|
||||
func (b KeptJetStream) SetPositions(ctx context.Context, ps []Position) error {
|
||||
for _, p := range ps {
|
||||
st, err := b.JS.Stream(ctx, p.Stream)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the stream %s of %s's position cannot be read: %w", p.Stream, p.Consumer, err)
|
||||
}
|
||||
var cfg jetstream.ConsumerConfig
|
||||
if err := json.Unmarshal(p.Config, &cfg); err != nil {
|
||||
return fmt.Errorf("the configuration of %s on %s does not read: %w", p.Consumer, p.Stream, err)
|
||||
}
|
||||
info, err := st.Info(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
start := p.AckFloor + 1
|
||||
if start > info.State.LastSeq+1 {
|
||||
start = info.State.LastSeq + 1
|
||||
}
|
||||
cfg.DeliverPolicy, cfg.OptStartSeq, cfg.OptStartTime = jetstream.DeliverByStartSequencePolicy, start, nil
|
||||
if err := st.DeleteConsumer(ctx, p.Consumer); err != nil && !errors.Is(err, jetstream.ErrConsumerNotFound) {
|
||||
return fmt.Errorf("%s on %s cannot be removed to be made again: %w", p.Consumer, p.Stream, err)
|
||||
}
|
||||
if _, err := st.CreateConsumer(ctx, cfg); err != nil {
|
||||
return fmt.Errorf("%s on %s cannot be made again at %d: %w", p.Consumer, p.Stream, start, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RestoreArchive puts every stream of the archive into the bus, which must hold none of them.
|
||||
func (b KeptJetStream) RestoreArchive(ctx context.Context, a *busarchive.Archive) error {
|
||||
log := b.Logf
|
||||
if log == nil {
|
||||
log = func(string, ...any) {}
|
||||
}
|
||||
return bussnap.RestoreAll(ctx, b.NC, a, log)
|
||||
}
|
||||
|
||||
// DeleteStreams removes streams by name.
|
||||
func (b KeptJetStream) DeleteStreams(ctx context.Context, names []string) error {
|
||||
for _, n := range names {
|
||||
if err := b.JS.DeleteStream(ctx, n); err != nil && !errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
return fmt.Errorf("%s: %w", n, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,311 +0,0 @@
|
||||
package keeper
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/klauspost/compress/s2"
|
||||
|
||||
"nats-tools/internal/busarchive"
|
||||
)
|
||||
|
||||
// tarBus is the fake bus whose snapshot is a real archive of the streams asked for, as the server's is.
|
||||
type tarBus struct {
|
||||
*fakeBus
|
||||
dir string
|
||||
n int
|
||||
}
|
||||
|
||||
func (b *tarBus) Snapshot(ctx context.Context, streams []string) (string, error) {
|
||||
if _, err := b.fakeBus.Snapshot(ctx, streams); err != nil {
|
||||
return "", err
|
||||
}
|
||||
b.n++
|
||||
path := filepath.Join(b.dir, fmt.Sprintf(".entry-%d.tar", b.n))
|
||||
return path, os.WriteFile(path, archiveOf(streams), 0o600)
|
||||
}
|
||||
|
||||
func sha(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) }
|
||||
|
||||
// archiveOf is a snapshot archive of the streams, each holding one block.
|
||||
func archiveOf(streams []string) []byte {
|
||||
var out bytes.Buffer
|
||||
tw := tar.NewWriter(&out)
|
||||
put := func(name string, b []byte) {
|
||||
_ = tw.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(b))})
|
||||
_, _ = tw.Write(b)
|
||||
}
|
||||
m := busarchive.Manifest{Format: 1, Finished: time.Date(2026, 11, 9, 3, 0, 0, 0, time.UTC)}
|
||||
files := map[string][]byte{}
|
||||
for _, name := range streams {
|
||||
var raw bytes.Buffer
|
||||
itw := tar.NewWriter(&raw)
|
||||
_ = itw.WriteHeader(&tar.Header{Name: "msgs.blk", Mode: 0o600, Size: 3})
|
||||
_, _ = itw.Write([]byte("abc"))
|
||||
itw.Close()
|
||||
var arc bytes.Buffer
|
||||
w := s2.NewWriter(&arc)
|
||||
_, _ = w.Write(raw.Bytes())
|
||||
w.Close()
|
||||
meta := []byte(`{"config":{"name":"` + name + `"}}`)
|
||||
e := busarchive.StreamEntry{Name: name, Meta: "streams/" + name + "/backup.json", MetaSHA256: sha(meta),
|
||||
Archive: "streams/" + name + "/stream.tar.s2", ArchiveBytes: int64(arc.Len()), ArchiveSHA256: sha(arc.Bytes())}
|
||||
m.Streams = append(m.Streams, e)
|
||||
files[e.Meta], files[e.Archive] = meta, arc.Bytes()
|
||||
}
|
||||
raw, _ := json.Marshal(m)
|
||||
put(busarchive.ManifestName, raw)
|
||||
for _, s := range m.Streams {
|
||||
put(s.Meta, files[s.Meta])
|
||||
put(s.Archive, files[s.Archive])
|
||||
}
|
||||
tw.Close()
|
||||
return out.Bytes()
|
||||
}
|
||||
|
||||
// keptBus is the fake bus's streams and consumers, as the server holds them.
|
||||
type keptBus struct {
|
||||
streams map[string]bool
|
||||
positions []Position
|
||||
set []Position
|
||||
restored [][]string
|
||||
refuse error
|
||||
}
|
||||
|
||||
func (b *keptBus) Existing(_ context.Context, names []string) ([]string, error) {
|
||||
var out []string
|
||||
for _, n := range names {
|
||||
if b.streams[n] {
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (b *keptBus) Positions(context.Context, string) ([]Position, error) { return b.positions, nil }
|
||||
func (b *keptBus) SetPositions(_ context.Context, ps []Position) error {
|
||||
b.set = append(b.set, ps...)
|
||||
return nil
|
||||
}
|
||||
func (b *keptBus) RestoreArchive(_ context.Context, a *busarchive.Archive) error {
|
||||
if err := b.refuse; err != nil {
|
||||
b.refuse = nil // refused once, as a server that was busy
|
||||
return err
|
||||
}
|
||||
var names []string
|
||||
for _, s := range a.Manifest.Streams {
|
||||
if b.streams[s.Name] {
|
||||
return fmt.Errorf("the server already holds %s", s.Name)
|
||||
}
|
||||
names = append(names, s.Name)
|
||||
}
|
||||
for _, n := range names {
|
||||
b.streams[n] = true
|
||||
}
|
||||
b.restored = append(b.restored, names)
|
||||
return nil
|
||||
}
|
||||
func (b *keptBus) DeleteStreams(_ context.Context, names []string) error {
|
||||
for _, n := range names {
|
||||
delete(b.streams, n)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func keptKeeper(t *testing.T) (*Keeper, *keptBus, *memStore, *[]string) {
|
||||
t.Helper()
|
||||
k, fb, store, _ := newKeeper(FirstView{})
|
||||
dir := t.TempDir()
|
||||
kb := &keptBus{streams: map[string]bool{"KV_idx_modules": true},
|
||||
positions: []Position{{Stream: "forge", Consumer: "ace_idx", AckFloor: 41, Config: json.RawMessage(`{"durable_name":"ace_idx"}`)},
|
||||
{Stream: "KV_idx_modules", Consumer: "ace_idx", AckFloor: 7, Config: json.RawMessage(`{"durable_name":"ace_idx"}`)}}}
|
||||
acts := &[]string{}
|
||||
k.Bus, k.Kept, k.KeptDir = &tarBus{fakeBus: fb, dir: dir}, kb, dir
|
||||
k.Act = func(what, why, cause string) error { *acts = append(*acts, what); return nil }
|
||||
return k, kb, store, acts
|
||||
}
|
||||
|
||||
func take(t *testing.T, k *Keeper) *Request {
|
||||
t.Helper()
|
||||
r := &Request{Verb: VerbSnapshotKept, Module: "idx", Local: "modules", Form: 0, Fingerprint: "abc123", CatchUp: "indexer_take_in", Caller: "ace.restic", Why: "before form 1"}
|
||||
if _, err := k.SnapshotKept(context.Background(), r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// A module's kept data on the bus has a restore point before its module moves, and a restore never goes over the live
|
||||
// data (design 47 "A module's kept data on the bus, backed up and restored with its delivery").
|
||||
func TestAKeptDataIsBackedUpAndRestoredBesideTheLiveData(t *testing.T) {
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
run func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string)
|
||||
}{
|
||||
{"a restore point holds the archive, the positions and the catch-up tool, keyed by form and fingerprint", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) {
|
||||
r := take(t, k)
|
||||
want := filepath.Join(k.KeptDir, "points", "idx", "modules", "f0-abc123")
|
||||
if r.Dir != want {
|
||||
t.Fatalf("restore point in %s, want %s", r.Dir, want)
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(want, PointFile))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var p Point
|
||||
if err := json.Unmarshal(raw, &p); err != nil || p.CatchUp != "indexer_take_in" || len(p.Positions) != 2 || p.Streams[0] != "KV_idx_modules" || p.Bytes == 0 {
|
||||
t.Errorf("point.json = %s (%v)", raw, err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(want, ArchiveFile)); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if _, _, found, _ := store.Get(StateKept, PointKey("idx", "modules", 0, "abc123")); !found {
|
||||
t.Error("the restore point is not recorded in the state kept")
|
||||
}
|
||||
if len(*acts) != 1 || !strings.Contains((*acts)[0], "took the restore point") {
|
||||
t.Errorf("hand acts %v", *acts)
|
||||
}
|
||||
}},
|
||||
{"aside-only sets the live data aside whole and restores nothing", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) {
|
||||
r := &Request{Verb: VerbRestoreKept, Module: "idx", Local: "modules", AsideOnly: true, Caller: "ace.restic", Why: "newer build stopped"}
|
||||
if _, err := k.RestoreKept(context.Background(), r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if kb.streams["KV_idx_modules"] || len(kb.restored) != 0 {
|
||||
t.Errorf("streams %v restored %v: the live data is removed and nothing restored", kb.streams, kb.restored)
|
||||
}
|
||||
aside, err := os.ReadDir(filepath.Join(k.KeptDir, "aside"))
|
||||
if err != nil || len(aside) != 1 || r.Dir == "" {
|
||||
t.Fatalf("aside: %v %v dir %q", aside, err, r.Dir)
|
||||
}
|
||||
}},
|
||||
{"a kept data the bus does not hold is refused naming what was looked for, and nothing is written", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) {
|
||||
r := &Request{Module: "idx", Local: "gone", Fingerprint: "abc123"}
|
||||
_, err := k.SnapshotKept(context.Background(), r)
|
||||
if err == nil || !strings.Contains(err.Error(), "KV_idx_gone") {
|
||||
t.Fatalf("err = %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(k.KeptDir, "points")); err == nil {
|
||||
t.Error("a directory was made for a snapshot that was not taken")
|
||||
}
|
||||
}},
|
||||
{"a name that is not one plain name is refused", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) {
|
||||
for _, r := range []*Request{{Module: "../x", Local: "modules", Fingerprint: "a"}, {Module: "idx", Local: "mod/ules", Fingerprint: "a"}, {Module: "idx", Local: "modules", Fingerprint: ""}} {
|
||||
if _, err := k.SnapshotKept(context.Background(), r); err == nil {
|
||||
t.Errorf("%+v was taken", r)
|
||||
}
|
||||
}
|
||||
}},
|
||||
{"a restore sets the live data aside whole, removes it, restores the archive and sets back the other consumers", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) {
|
||||
p := take(t, k)
|
||||
r := &Request{Verb: VerbRestoreKept, Module: "idx", Local: "modules", Form: 0, Fingerprint: "abc123", From: p.Dir, Caller: "ace.restic", Why: "revert"}
|
||||
if _, err := k.RestoreKept(context.Background(), r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !kb.streams["KV_idx_modules"] || len(kb.restored) != 1 {
|
||||
t.Errorf("streams %v restored %v", kb.streams, kb.restored)
|
||||
}
|
||||
if r.CatchUp != "indexer_take_in" {
|
||||
t.Errorf("the catch-up tool is %q", r.CatchUp)
|
||||
}
|
||||
aside, err := os.ReadDir(filepath.Join(k.KeptDir, "aside"))
|
||||
if err != nil || len(aside) != 1 {
|
||||
t.Fatalf("aside: %v %v", aside, err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(k.KeptDir, "aside", aside[0].Name(), ArchiveFile)); err != nil {
|
||||
t.Error("the live data's archive is not kept")
|
||||
}
|
||||
if _, _, found, _ := store.Get(StateKept, "aside."+aside[0].Name()); !found {
|
||||
t.Error("what is set aside is not recorded")
|
||||
}
|
||||
// Only the consumer on a stream the point did not carry is made again; a stream's own comes with its snapshot.
|
||||
if len(kb.set) != 1 || kb.set[0].Stream != "forge" || kb.set[0].AckFloor != 41 {
|
||||
t.Errorf("positions set back: %+v", kb.set)
|
||||
}
|
||||
}},
|
||||
{"a restore point of another fingerprint or a linked file is refused before anything is removed", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) {
|
||||
p := take(t, k)
|
||||
bad := *p
|
||||
bad.Fingerprint = "other"
|
||||
if _, err := k.RestoreKept(context.Background(), &bad); err == nil || !kb.streams["KV_idx_modules"] {
|
||||
t.Errorf("err = %v, live %v", err, kb.streams)
|
||||
}
|
||||
link := filepath.Join(p.Dir, ArchiveFile)
|
||||
real := filepath.Join(p.Dir, "real.tar")
|
||||
if err := os.Rename(link, real); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(real, link); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := &Request{Module: "idx", Local: "modules", Fingerprint: "abc123", From: p.Dir}
|
||||
if _, err := k.RestoreKept(context.Background(), r); err == nil || !strings.Contains(err.Error(), "plain file") || !kb.streams["KV_idx_modules"] {
|
||||
t.Errorf("err = %v, live %v", err, kb.streams)
|
||||
}
|
||||
}},
|
||||
{"a live data that cannot be set aside stays where it is", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) {
|
||||
p := take(t, k)
|
||||
k.Bus.(*tarBus).fakeBus.refuse["snapshot"] = errors.New("the server refused")
|
||||
r := &Request{Module: "idx", Local: "modules", Fingerprint: "abc123", From: p.Dir}
|
||||
if _, err := k.RestoreKept(context.Background(), r); err == nil || !kb.streams["KV_idx_modules"] || !strings.Contains(err.Error(), "nothing was removed") {
|
||||
t.Errorf("err = %v, live %v", err, kb.streams)
|
||||
}
|
||||
}},
|
||||
{"a restore that fails puts the live data back and says so", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) {
|
||||
p := take(t, k)
|
||||
kb.refuse = errors.New("the server refused the archive")
|
||||
r := &Request{Module: "idx", Local: "modules", Fingerprint: "abc123", From: p.Dir}
|
||||
_, err := k.RestoreKept(context.Background(), r)
|
||||
if err == nil || !strings.Contains(err.Error(), "the live data is back as it was") {
|
||||
t.Fatalf("err = %v", err)
|
||||
}
|
||||
}},
|
||||
{"set-aside data goes only on a call that says why and is recorded first", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) {
|
||||
p := take(t, k)
|
||||
if _, err := k.RestoreKept(context.Background(), &Request{Module: "idx", Local: "modules", Fingerprint: "abc123", From: p.Dir}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entries, _ := os.ReadDir(filepath.Join(k.KeptDir, "aside"))
|
||||
id := entries[0].Name()
|
||||
if _, err := k.DiscardAside(&Request{From: id}); err == nil {
|
||||
t.Error("discarded with no why")
|
||||
}
|
||||
if _, err := k.DiscardAside(&Request{From: "../points", Why: "x"}); err == nil {
|
||||
t.Error("discarded a path")
|
||||
}
|
||||
k.Act = func(string, string, string) error { return errors.New("the log is down") }
|
||||
r := &Request{From: id, Why: "the revert held", Caller: "ace.operator"}
|
||||
k.record(r, "x") // sets Unrecorded as a failing log does
|
||||
if _, err := k.DiscardAside(r); err == nil {
|
||||
t.Error("discarded with the act unrecorded")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(k.KeptDir, "aside", id)); err != nil {
|
||||
t.Error("the data went with its act unrecorded")
|
||||
}
|
||||
k.Act = func(what, why, cause string) error { *acts = append(*acts, what); return nil }
|
||||
if _, err := k.DiscardAside(&Request{From: id, Why: "the revert held", Caller: "ace.operator"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(k.KeptDir, "aside", id)); err == nil {
|
||||
t.Error("still there")
|
||||
}
|
||||
if _, _, found, _ := store.Get(StateKept, "aside."+id); found {
|
||||
t.Error("its record stays")
|
||||
}
|
||||
}},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
k, kb, store, acts := keptKeeper(t)
|
||||
row.run(t, k, kb, store, acts)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -46,15 +46,6 @@ type Request struct {
|
||||
Snapshot string `json:"snapshot,omitempty"`
|
||||
// Waiting is why it waits, while it waits: the operator's warrant, or the keeper's first view.
|
||||
Waiting string `json:"waiting,omitempty"`
|
||||
// Form, Fingerprint and CatchUp name a restore point (kept.go); From is the directory a restore reads, or the id of the
|
||||
// data a discard removes; Dir is what the keeper answers: the restore point's directory, or the data set aside.
|
||||
Form int `json:"form,omitempty"`
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
CatchUp string `json:"catch-up,omitempty"`
|
||||
// AsideOnly is the argument `aside-only` of restore-kept: the live data is set aside and nothing is restored.
|
||||
AsideOnly bool `json:"aside-only,omitempty"`
|
||||
From string `json:"from,omitempty"`
|
||||
Dir string `json:"dir,omitempty"`
|
||||
// Unrecorded is why the hand act of what it did could not be recorded, when it could not: said, never only logged.
|
||||
Unrecorded string `json:"unrecorded,omitempty"`
|
||||
}
|
||||
@@ -145,10 +136,6 @@ func (k *Keeper) act(r *Request) (string, error) {
|
||||
}
|
||||
return fmt.Sprintf("the first view answered at revision %d", s.Revision), nil
|
||||
}
|
||||
switch r.Verb {
|
||||
case VerbSnapshotKept, VerbRestoreKept, VerbDiscardAside:
|
||||
return k.actKept(r)
|
||||
}
|
||||
if !k.Viewed() {
|
||||
return "", fmt.Errorf("%w: the keeper holds no first view yet, so it acts on nothing", ErrNoWarrant)
|
||||
}
|
||||
@@ -308,24 +295,3 @@ func (k *Keeper) record(r *Request, what string) {
|
||||
k.logf("the hand act %q could not be recorded: %v", what, err)
|
||||
}
|
||||
}
|
||||
|
||||
// actKept acts on a request about a kept data's restore point. These need no first view or warrant: a snapshot changes
|
||||
// nothing, a restore sets the live data aside whole before it removes it, and a discard names who and why (kept.go).
|
||||
func (k *Keeper) actKept(r *Request) (string, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||
defer cancel()
|
||||
var out string
|
||||
var err error
|
||||
switch r.Verb {
|
||||
case VerbSnapshotKept:
|
||||
out, err = k.SnapshotKept(ctx, r)
|
||||
case VerbRestoreKept:
|
||||
out, err = k.RestoreKept(ctx, r)
|
||||
default:
|
||||
out, err = k.DiscardAside(r)
|
||||
}
|
||||
if err != nil {
|
||||
return "nothing done", err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user