package keeper import ( "context" "encoding/json" "errors" "fmt" "os" "path/filepath" "slices" "strings" "time" "nats-tools/internal/busarchive" ) // A module's kept data on the bus, backed up before the module moves and restored beside the live data when a want takes // it back (milestone G, design 47 "A module's kept data on the bus, backed up and restored with its delivery"). The // keeper takes it on its own account, as it takes a snapshot before a recreate (ADR 0235 §5): the streams of one kept // data as the server's own snapshot, the position of each durable consumer the module reads with, and the catch-up tool // the newer manifest declares, in one directory the backup holder keeps as a restore point. A restore never goes over // the live data: the live data is first snapshotted into a set-aside directory under a name of its own, and only then // removed from the bus; only discard-aside removes the set-aside copy, on a person's call. const ( // VerbSnapshotKept takes a restore point of one kept data; VerbRestoreKept restores one beside the live data. VerbSnapshotKept = "snapshot-kept" VerbRestoreKept = "restore-kept" // VerbDiscardAside removes the data a restore set aside, by its id. VerbDiscardAside = "discard-aside" ) // StateKept is the keeper's own state of the restore points it took and the data it set aside (never their bytes, which // are files in the keeper's directory): `point...
.` and `aside.`. Read by the // bus module's tool nats_kept and by the seat mesh-broker's `aside`. const StateKept = "kept" // PointFile and ArchiveFile are the two files of a restore point's directory; AsideFile of a set-aside directory. const ( PointFile = "point.json" ArchiveFile = "bus.tar" AsideFile = "aside.json" ) // Position is where one durable consumer of the module stood on a stream: the stream sequence it had acknowledged up to, // and its configuration as the server held it, so a restore can make it again at that place. type Position struct { Stream string `json:"stream"` Consumer string `json:"consumer"` AckFloor uint64 `json:"ack-floor"` Config json.RawMessage `json:"config"` } // KeptBus is what the keeper needs of the bus beyond Bus to take and restore a kept data. type KeptBus interface { // Existing answers which of the stream names the bus holds. Existing(ctx context.Context, names []string) ([]string, error) // Positions are the module's durable consumers on every stream (a consumer is the module's when it is named for it). Positions(ctx context.Context, module string) ([]Position, error) // SetPositions makes each consumer again from the place it stood. SetPositions(ctx context.Context, ps []Position) error // RestoreArchive puts every stream of an archive into the bus, which must hold none of them. RestoreArchive(ctx context.Context, a *busarchive.Archive) error // DeleteStreams removes streams by name. DeleteStreams(ctx context.Context, names []string) error } // Point is a restore point's description, its directory's point.json. type Point struct { Module string `json:"module"` Local string `json:"local"` Form int `json:"form"` Fingerprint string `json:"fingerprint"` CatchUp string `json:"catch-up,omitempty"` Streams []string `json:"streams"` Positions []Position `json:"positions"` At time.Time `json:"at"` Bytes int64 `json:"bytes"` } // Aside is data a restore set aside: the live kept data as it was when the restore removed it. type Aside struct { ID string `json:"id"` Module string `json:"module"` Local string `json:"local"` Streams []string `json:"streams"` At time.Time `json:"at"` Bytes int64 `json:"bytes"` By string `json:"by"` Why string `json:"why"` Dir string `json:"dir"` } // PointKey is a restore point's key in StateKept. func PointKey(module, local string, form int, fingerprint string) string { return fmt.Sprintf("point.%s.%s.%d.%s", module, local, form, fingerprint) } // safeName is one path component made of what a module, a kept data and a fingerprint are made of. func safeName(s string) bool { return s != "" && s != "." && s != ".." && !strings.ContainsAny(s, "/\\\x00 *>.") } func (k *Keeper) pointDir(module, local string, form int, fingerprint string) string { return filepath.Join(k.KeptDir, "points", module, local, fmt.Sprintf("f%d-%s", form, fingerprint)) } // streamsOf is the streams one kept data is: its plan's objects where the keeper made it, else the streams the bus holds // under its bus name (a foundation part's own section, made by the grants part). func (k *Keeper) streamsOf(ctx context.Context, module, local string) ([]string, error) { if held, _, found := k.asked(module); found { if m, made := held.Made[local]; made && len(m.Plan.Objects) > 0 { var out []string for _, o := range m.Plan.Objects { out = append(out, o.Stream()) } return out, nil } } name := BusName(module, local) got, err := k.Kept.Existing(ctx, []string{"KV_" + name, "OBJ_" + name, name}) if err != nil { return nil, err } if len(got) == 0 { return nil, fmt.Errorf("the bus holds no kept data %s of %s (looked for KV_%s, OBJ_%s and %s)", local, module, name, name, name) } return got, nil } func (k *Keeper) keptReady(r *Request) error { if k.Kept == nil || k.KeptDir == "" { return errors.New("no directory for restore points is set (MESH_NATS_SNAPSHOTS_DIR), so no kept data is taken or restored") } if !safeName(r.Module) || !safeName(r.Local) || (!r.AsideOnly && !safeName(r.Fingerprint)) { return fmt.Errorf("%q, %q and %q are not a module, a kept data and a fingerprint: each is one plain name", r.Module, r.Local, r.Fingerprint) } return nil } // SnapshotKept takes the restore point of one kept data: the file pair in a directory named for its form and fingerprint, // written whole before its name is given, and its record in StateKept. It answers the directory. func (k *Keeper) SnapshotKept(ctx context.Context, r *Request) (string, error) { if err := k.keptReady(r); err != nil { return "", err } streams, err := k.streamsOf(ctx, r.Module, r.Local) if err != nil { return "", err } // The positions first: a consumer that moves while the snapshot is taken is then a little behind its archive, and a // restored consumer that is behind hears a message twice, never misses one. positions, err := k.Kept.Positions(ctx, r.Module) if err != nil { return "", fmt.Errorf("the consumers' positions of %s cannot be read, so no restore point is taken: %w", r.Module, err) } tar, err := k.Bus.Snapshot(ctx, streams) if err != nil { return "", fmt.Errorf("the snapshot of %s's %s failed, so no restore point is taken: %w", r.Module, r.Local, err) } dir := k.pointDir(r.Module, r.Local, r.Form, r.Fingerprint) if err := os.MkdirAll(filepath.Dir(dir), 0o700); err != nil { return "", err } partial := dir + ".partial" _ = os.RemoveAll(partial) if err := os.Mkdir(partial, 0o700); err != nil { return "", err } if err := os.Rename(tar, filepath.Join(partial, ArchiveFile)); err != nil { _ = os.RemoveAll(partial) return "", fmt.Errorf("the snapshot cannot be kept as a restore point: %w", err) } st, _ := os.Stat(filepath.Join(partial, ArchiveFile)) p := Point{Module: r.Module, Local: r.Local, Form: r.Form, Fingerprint: r.Fingerprint, CatchUp: r.CatchUp, Streams: streams, Positions: positions, At: k.now()} if st != nil { p.Bytes = st.Size() } raw, _ := json.MarshalIndent(p, "", " ") if err := os.WriteFile(filepath.Join(partial, PointFile), append(raw, '\n'), 0o600); err != nil { _ = os.RemoveAll(partial) return "", err } // A restore point taken again at the same form and fingerprint replaces the earlier one, whole. _ = os.RemoveAll(dir) if err := os.Rename(partial, dir); err != nil { return "", err } val, _ := json.Marshal(p) if _, err := k.Store.Put(StateKept, PointKey(r.Module, r.Local, r.Form, r.Fingerprint), val); err != nil { k.logf("the record of the restore point %s cannot be written to its own %s: %v", dir, StateKept, err) } r.Dir = dir k.record(r, fmt.Sprintf("took the restore point of %s's %s at form %d before %s (%d stream(s), %d consumer position(s), %d bytes)", r.Module, r.Local, r.Form, r.Fingerprint, len(streams), len(positions), p.Bytes)) return fmt.Sprintf("restore point of %s's %s at form %d before %s in %s", r.Module, r.Local, r.Form, r.Fingerprint, dir), nil } // readPoint reads and checks a restore point's directory: its point.json names what the request names, its files are // plain files (never a link), and its archive matches its own manifest. func readPoint(dir string, r *Request) (*Point, *busarchive.Archive, error) { for _, name := range []string{PointFile, ArchiveFile} { fi, err := os.Lstat(filepath.Join(dir, name)) if err != nil { return nil, nil, fmt.Errorf("the restore point %s holds no %s: %w", dir, name, err) } if !fi.Mode().IsRegular() { return nil, nil, fmt.Errorf("%s of the restore point %s is not a plain file", name, dir) } } raw, err := os.ReadFile(filepath.Join(dir, PointFile)) if err != nil { return nil, nil, err } var p Point if err := json.Unmarshal(raw, &p); err != nil { return nil, nil, fmt.Errorf("%s of %s does not read: %w", PointFile, dir, err) } if p.Module != r.Module || p.Local != r.Local || p.Form != r.Form || p.Fingerprint != r.Fingerprint { return nil, nil, fmt.Errorf("the directory %s holds the restore point of %s's %s at form %d before %s, not of %s's %s at form %d before %s", dir, p.Module, p.Local, p.Form, p.Fingerprint, r.Module, r.Local, r.Form, r.Fingerprint) } f, err := os.Open(filepath.Join(dir, ArchiveFile)) if err != nil { return nil, nil, err } defer f.Close() a, err := busarchive.ReadArchive(f, "") if err != nil { return nil, nil, fmt.Errorf("the archive of the restore point %s does not read: %w", dir, err) } if err := a.Verify(); err != nil { a.Close() return nil, nil, fmt.Errorf("the archive of the restore point %s does not match its own manifest, so nothing is restored: %w", dir, err) } return &p, a, nil } // RestoreKept restores one kept data beside the live data: the live data is snapshotted into a set-aside directory first // and only then removed from the bus, and the restore point's streams are put in its place; the consumers the module // reads with on other streams are set back to their positions. It answers what was set aside, where. func (k *Keeper) RestoreKept(ctx context.Context, r *Request) (string, error) { if err := k.keptReady(r); err != nil { return "", err } var p *Point var a *busarchive.Archive var live []string if r.AsideOnly { // `aside-only` (design 47 G §405): the live data is set aside and nothing is restored. streams, err := k.streamsOf(ctx, r.Module, r.Local) if err != nil { return "", err } live = streams } else { if !filepath.IsAbs(r.From) { return "", fmt.Errorf("the restore point's directory is given as an absolute path, not %q", r.From) } var err error p, a, err = readPoint(filepath.Clean(r.From), r) if err != nil { return "", err } defer a.Close() live, err = k.Kept.Existing(ctx, p.Streams) if err != nil { return "", err } } id := fmt.Sprintf("%s.%s.%s", r.Module, r.Local, k.now().Format("20060102t150405")) var aside *Aside if len(live) > 0 { // Aside before anything is removed: a snapshot that fails leaves the live data where it is. tar, err := k.Bus.Snapshot(ctx, live) if err != nil { return "", fmt.Errorf("the live %s of %s could not be set aside, so nothing was removed: %w", r.Local, r.Module, err) } dir := filepath.Join(k.KeptDir, "aside", id) if err := os.MkdirAll(dir, 0o700); err != nil { return "", err } if err := os.Rename(tar, filepath.Join(dir, ArchiveFile)); err != nil { return "", fmt.Errorf("the live data's snapshot cannot be kept: %w", err) } st, _ := os.Stat(filepath.Join(dir, ArchiveFile)) aside = &Aside{ID: id, Module: r.Module, Local: r.Local, Streams: live, At: k.now(), By: r.Caller, Why: r.Why, Dir: dir} if st != nil { aside.Bytes = st.Size() } raw, _ := json.MarshalIndent(aside, "", " ") if err := os.WriteFile(filepath.Join(dir, AsideFile), append(raw, '\n'), 0o600); err != nil { return "", err } val, _ := json.Marshal(aside) if _, err := k.Store.Put(StateKept, "aside."+id, val); err != nil { return "", fmt.Errorf("what is set aside cannot be recorded, so nothing was removed: %w", err) } if err := k.Kept.DeleteStreams(ctx, live); err != nil { return "", fmt.Errorf("the live data was set aside in %s and could not be removed from the bus: %w", dir, err) } } if r.AsideOnly { r.Dir = "" where := "nothing was live, so nothing was set aside" if aside != nil { r.Dir = aside.Dir where = fmt.Sprintf("the live data is set aside as %s (%d bytes) in %s", aside.ID, aside.Bytes, aside.Dir) } k.record(r, fmt.Sprintf("set aside %s's %s and restored nothing; %s", r.Module, r.Local, where)) return fmt.Sprintf("set aside %s's %s; %s", r.Module, r.Local, where), nil } if err := k.Kept.RestoreArchive(ctx, a); err != nil { msg := fmt.Sprintf("the restore point could not be put on the bus: %v", err) if aside != nil { if back, rerr := k.putBack(ctx, aside); rerr != nil { msg += fmt.Sprintf("; the live data, set aside in %s, could not be put back either: %v", aside.Dir, rerr) } else { msg += fmt.Sprintf("; the live data is back as it was (%s)", back) } } return "", errors.New(msg) } // The consumers on streams the restore point did not carry (the snapshot of a stream carries its own consumers). var apply []Position for _, pos := range p.Positions { if !slices.Contains(p.Streams, pos.Stream) { apply = append(apply, pos) } } if len(apply) > 0 { if err := k.Kept.SetPositions(ctx, apply); err != nil { return "", fmt.Errorf("the kept data is restored, but the consumers' positions could not be set back: %w", err) } } r.Dir, r.CatchUp = "", p.CatchUp where := "nothing was live, so nothing was set aside" if aside != nil { r.Dir = aside.Dir where = fmt.Sprintf("the live data is set aside as %s (%d bytes) in %s", aside.ID, aside.Bytes, aside.Dir) } k.record(r, fmt.Sprintf("restored %s's %s from the restore point at form %d before %s beside the live data; %s", r.Module, r.Local, r.Form, r.Fingerprint, where)) return fmt.Sprintf("restored %s's %s from its restore point; %s; %d consumer position(s) set back; catch-up tool %q", r.Module, r.Local, where, len(apply), p.CatchUp), nil } // putBack restores set-aside data to the bus, after a restore failed. func (k *Keeper) putBack(ctx context.Context, a *Aside) (string, error) { f, err := os.Open(filepath.Join(a.Dir, ArchiveFile)) if err != nil { return "", err } defer f.Close() arch, err := busarchive.ReadArchive(f, "") if err != nil { return "", err } defer arch.Close() if err := k.Kept.DeleteStreams(ctx, a.Streams); err != nil { return "", err } if err := k.Kept.RestoreArchive(ctx, arch); err != nil { return "", err } return "from " + a.Dir, nil } // DiscardAside removes the data a restore set aside, by its id (Request.From), on a person's call: the request says who // and why, and is recorded as a hand act before the data goes. func (k *Keeper) DiscardAside(r *Request) (string, error) { if k.KeptDir == "" { return "", errors.New("no directory for restore points is set (MESH_NATS_SNAPSHOTS_DIR)") } id := strings.TrimSpace(r.From) if !safeAsideID(id) { return "", fmt.Errorf("%q is not the id of data set aside", id) } if strings.TrimSpace(r.Why) == "" { return "", errors.New("discarding set-aside data says why") } raw, _, found, err := k.Store.Get(StateKept, "aside."+id) if err != nil { return "", err } if !found { return "nothing done", fmt.Errorf("no data is set aside as %s", id) } var a Aside _ = json.Unmarshal(raw, &a) k.record(r, fmt.Sprintf("discarded the data set aside as %s (%s's %s, %d bytes)", id, a.Module, a.Local, a.Bytes)) if r.Unrecorded != "" { return "nothing done", fmt.Errorf("the hand act cannot be recorded, so the set-aside data stays: %s", r.Unrecorded) } if err := os.RemoveAll(filepath.Join(k.KeptDir, "aside", id)); err != nil { return "partly done", err } if err := k.Store.Delete(StateKept, "aside."+id); err != nil { return "partly done", err } return fmt.Sprintf("discarded the data set aside as %s", id), nil } func safeAsideID(id string) bool { return id != "" && !strings.ContainsAny(id, "/\\\x00 *>") && !strings.HasPrefix(id, ".") }