Merge branch 'feat/47-G-fix9-recovery' into feat/47-G-join10
This commit is contained in:
@@ -0,0 +1,4 @@
|
||||
/mesh-recovery
|
||||
/cmd/mesh-recovery/mesh-recovery
|
||||
/cmd/mesh-recovery-tools/mesh-recovery-tools
|
||||
/mesh-recovery-tools
|
||||
@@ -0,0 +1,9 @@
|
||||
# mesh-recovery
|
||||
|
||||
Holds the seat `mesh-recovery` (design 47 G §554): the recovery module takes, sets aside and restores a module's kept
|
||||
data on the live bus, one act at a time, each act in the hand-act log, its result said as an event under the act's id.
|
||||
Go: one process (`cmd/mesh-recovery`) that answers the seat's verbs itself (A-107, A-115) and a tools bundle
|
||||
(`cmd/mesh-recovery-tools`) whose `recovery_*` tools call those verbs and hold no grant of their own (§656).
|
||||
|
||||
Its data item `points` holds the restore points and the asides; the digests of the points are on the bus in its state
|
||||
`digests` (§567). Its settings are `points-cap`, `points-filling-mark`, `point-keep-days` and `dump-wait-minutes`.
|
||||
@@ -0,0 +1,129 @@
|
||||
// mesh-recovery-tools: the recovery module's tools, its command-line interface (design 47 G §554, §656, §657). Each hand
|
||||
// tool calls one verb of the seat mesh-recovery with the `asker` the tool runner proved for the call and holds no grant of
|
||||
// its own: the bus proves the tool runner's account, the verb logs both in the hand-act log. stdout is the MCP channel.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
|
||||
|
||||
func main() {
|
||||
if err := stdio.Serve("", Tools(stdio.Ask, stdio.CallerOf)); err != nil {
|
||||
logf("%v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Ask is how a bundle calls a seat's verb: the key `seat:<seat>.<verb>` and the arguments.
|
||||
type Ask func(key string, body any) (json.RawMessage, error)
|
||||
|
||||
func obj(props map[string]any, required ...string) map[string]any {
|
||||
m := map[string]any{"type": "object", "properties": props}
|
||||
if len(required) > 0 {
|
||||
m["required"] = required
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func str(d string) map[string]any { return map[string]any{"type": "string", "description": d} }
|
||||
|
||||
var (
|
||||
module = str("the module")
|
||||
kept = str("the kept data, its local name in the module's manifest")
|
||||
why = str("why, which the hand-act log keeps")
|
||||
)
|
||||
|
||||
// hand is a tool that calls a verb of the seat: the arguments go through as they are, with the asker the tool runner proved.
|
||||
func hand(ask Ask, callerOf func(map[string]any) string, name, verb, description string, props map[string]any, required ...string) stdio.Tool {
|
||||
props["why"] = why
|
||||
return stdio.Tool{Name: name, Description: description + " Class act: written in the hand-act log first with the account the bus proved and who asked; answers accepted, busy or error at once and the result comes as the act's ending event.",
|
||||
Input: obj(props, append(required, "why")...),
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
body := map[string]any{}
|
||||
for k, v := range a {
|
||||
body[k] = v
|
||||
}
|
||||
body["asker"] = callerOf(a)
|
||||
raw, err := ask("seat:mesh-recovery."+verb, body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out any
|
||||
return out, json.Unmarshal(raw, &out)
|
||||
}}
|
||||
}
|
||||
|
||||
// Tools are what the runtime serves, named recovery_ (§554); each acting tool is one verb of the seat (§656).
|
||||
func Tools(ask Ask, callerOf func(map[string]any) string) []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{Name: "recovery_kept_points",
|
||||
Description: "Every restore point and aside the recovery module holds, with their sizes, and the act that runs with since when and what it waits on; narrowed to a module and a kept data. Changes nothing (also the module's declared health check, §657).",
|
||||
Input: obj(map[string]any{"module": module, "kept": kept}),
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
body := map[string]any{}
|
||||
for _, k := range []string{"module", "kept"} {
|
||||
if s, _ := a[k].(string); strings.TrimSpace(s) != "" {
|
||||
body[k] = s
|
||||
}
|
||||
}
|
||||
raw, err := ask("seat:mesh-recovery.kept-points", body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return keptPointsHealth(raw)
|
||||
}},
|
||||
hand(ask, callerOf, "recovery_take", "take", "Take a restore point of one module's kept data now, at the form of the given fingerprint.",
|
||||
map[string]any{"module": module, "kept": kept, "fingerprint": str("the fingerprint whose form the point is taken at")}, "module", "kept", "fingerprint"),
|
||||
hand(ask, callerOf, "recovery_restore", "restore", "Restore one point (snapshot, a point's name from recovery_kept_points) or a keeper's snapshot (request), exactly one of them.",
|
||||
map[string]any{"module": module, "kept": kept, "snapshot": str("the restore point's name"), "request": str("the keeper's snapshot by its request"),
|
||||
"from-backup": map[string]any{"type": "boolean", "description": "ask the backup holder for the points directory when the point is missing, only with snapshot"}}, "module", "kept"),
|
||||
hand(ask, callerOf, "recovery_retry", "retry", "Ask again the last failed act of a module and kept data, with its fields.",
|
||||
map[string]any{"module": module, "kept": kept}, "module", "kept"),
|
||||
hand(ask, callerOf, "recovery_aside", "aside", "Set the live kept data aside without restoring: a move, the data stays in the aside until it is put back or discarded.",
|
||||
map[string]any{"module": module, "kept": kept}, "module", "kept"),
|
||||
hand(ask, callerOf, "recovery_discard_aside", "discard-aside", "Remove one aside's directory and nothing else.",
|
||||
map[string]any{"module": module, "kept": kept, "id": str("the aside's id")}, "module", "kept", "id"),
|
||||
hand(ask, callerOf, "recovery_restore_aside", "restore-aside", "Put an aside back under the live names it was taken from, refused while live objects of those names exist.",
|
||||
map[string]any{"module": module, "kept": kept, "id": str("the aside's id")}, "module", "kept", "id"),
|
||||
hand(ask, callerOf, "recovery_cancel", "cancel", "End an accepted act that has not begun its work (one waiting on the dump) failed, cancelled; refused naming the step of an act that has begun.",
|
||||
map[string]any{"act": str("the act's id")}, "act"),
|
||||
hand(ask, callerOf, "recovery_expire_points", "expire-points", "Remove the restore points past their keep days now, never the newest of a kept data.", map[string]any{}),
|
||||
}
|
||||
}
|
||||
|
||||
// keptPointsHealth is what kept-points answers with the health answer beside it, which the module's declared health
|
||||
// check reads (§554, §657, A-129): unhealthy while an aside only it holds the data of is listed as unrestored.
|
||||
func keptPointsHealth(raw json.RawMessage) (map[string]any, error) {
|
||||
var out map[string]any
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var kp struct {
|
||||
Unrestored []struct {
|
||||
ID string `json:"id"`
|
||||
Names []string `json:"names"`
|
||||
} `json:"unrestored"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &kp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
h := health.Healthy("No aside holds data the live bus lacks.")
|
||||
if len(kp.Unrestored) > 0 {
|
||||
var said []string
|
||||
for _, a := range kp.Unrestored {
|
||||
said = append(said, a.ID+" ("+strings.Join(a.Names, ", ")+")")
|
||||
}
|
||||
h = health.Unhealthy(fmt.Sprintf("%d aside(s) hold data whose live objects were removed and not restored: %s; recovery_restore_aside puts one back.",
|
||||
len(kp.Unrestored), strings.Join(said, "; ")))
|
||||
}
|
||||
out["state"], out["healthy"], out["why"] = h.State, h.Healthy, h.Why
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// The bundle serves exactly the tools the manifest names, each named recovery_, and each acting tool calls its verb of the
|
||||
// seat with the asker the tool runner proved, written into the call, and nothing of its own (design 47 G §554, §656, §682).
|
||||
func TestEachToolCallsItsVerbWithTheProvedAsker(t *testing.T) {
|
||||
var key string
|
||||
var sent map[string]any
|
||||
ask := func(k string, b any) (json.RawMessage, error) {
|
||||
key, sent = k, b.(map[string]any)
|
||||
return json.RawMessage(`{"accepted":"act-1"}`), nil
|
||||
}
|
||||
tools := Tools(ask, func(map[string]any) string { return "claude@shanks" })
|
||||
raw, _ := os.ReadFile("../../module.json")
|
||||
var m struct {
|
||||
Tools []string `json:"tools"`
|
||||
Invokes []string `json:"invokes"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &m)
|
||||
var names []string
|
||||
verbs := map[string]string{"recovery_take": "take", "recovery_restore": "restore", "recovery_retry": "retry", "recovery_aside": "aside",
|
||||
"recovery_discard_aside": "discard-aside", "recovery_restore_aside": "restore-aside", "recovery_cancel": "cancel", "recovery_expire_points": "expire-points"}
|
||||
for _, tool := range tools {
|
||||
names = append(names, tool.Name)
|
||||
if !strings.HasPrefix(tool.Name, "recovery_") {
|
||||
t.Errorf("tool %s is not named recovery_", tool.Name)
|
||||
}
|
||||
if verb, acts := verbs[tool.Name]; acts {
|
||||
if _, err := tool.Run(map[string]any{"module": "app", "why": "w"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if key != "seat:mesh-recovery."+verb || sent["asker"] != "claude@shanks" || sent["module"] != "app" {
|
||||
t.Errorf("%s called %s with %v", tool.Name, key, sent)
|
||||
}
|
||||
if !slices.Contains(m.Invokes, key) {
|
||||
t.Errorf("the manifest does not invoke %s", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
slices.Sort(names)
|
||||
want := slices.Clone(m.Tools)
|
||||
slices.Sort(want)
|
||||
if !slices.Equal(names, want) {
|
||||
t.Errorf("tools %v, the manifest names %v", names, want)
|
||||
}
|
||||
}
|
||||
|
||||
// recovery_kept_points answers what kept-points answers with the health answer beside it: unhealthy while an aside is
|
||||
// unrestored, healthy otherwise, as the declared health check reads it (§657, A-129).
|
||||
func TestKeptPointsIsTheHealthCheck(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
answer string
|
||||
state string
|
||||
}{
|
||||
{`{"points":[],"asides":[],"unrestored":[]}`, health.StateHealthy},
|
||||
{`{"points":[],"asides":[{"id":"x","names":["app_notes"],"size":1}],"unrestored":[{"id":"x","names":["app_notes"],"size":1}]}`, health.StateUnhealthy},
|
||||
} {
|
||||
tools := Tools(func(string, any) (json.RawMessage, error) { return json.RawMessage(c.answer), nil }, func(map[string]any) string { return "" })
|
||||
out, err := tools[0].Run(map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(out)
|
||||
a, err := health.Read(raw)
|
||||
if err != nil || a.State != c.state || !strings.Contains(string(raw), `"unrestored"`) {
|
||||
t.Errorf("%s answered %s: %+v, %v", c.answer, raw, a, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
|
||||
"git.novox.be/novox/mesh-sdk/go/bussnap"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"mesh-recovery/internal/recovery"
|
||||
)
|
||||
|
||||
// liveBus is the bus as the recovery module's own account sees it (design 47 G §554): a stream's snapshot and restore
|
||||
// through mesh-sdk's bussnap (§655, A-125), its deletion, and its durable consumers' positions.
|
||||
type liveBus struct {
|
||||
nc *nats.Conn
|
||||
js jetstream.JetStream
|
||||
scratch string
|
||||
logf func(string, ...any)
|
||||
}
|
||||
|
||||
func (b liveBus) Objects(ctx context.Context, streams []string) ([]recovery.Object, error) {
|
||||
var out []recovery.Object
|
||||
for _, name := range streams {
|
||||
s, err := b.js.Stream(ctx, name)
|
||||
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
out = append(out, recovery.Object{Stream: name})
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stream %s: %w", name, err)
|
||||
}
|
||||
info, err := s.Info(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stream %s: %w", name, err)
|
||||
}
|
||||
o := recovery.Object{Stream: name, Exists: true, Bytes: int64(info.State.Bytes)}
|
||||
names := s.ListConsumers(ctx)
|
||||
for ci := range names.Info() {
|
||||
if ci.Config.Durable == "" {
|
||||
continue
|
||||
}
|
||||
o.Consumers = append(o.Consumers, seat.Consumer{Stream: name, Name: ci.Name, Sequence: int64(ci.AckFloor.Stream)})
|
||||
}
|
||||
if err := names.Err(); err != nil {
|
||||
return nil, fmt.Errorf("the consumers of %s: %w", name, err)
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (b liveBus) Take(ctx context.Context, stream string, w io.Writer) error {
|
||||
_, err := bussnap.Snapshot(ctx, b.nc, w, bussnap.SnapshotOptions{Streams: []string{stream}, TempDir: b.scratch, Log: b.logf})
|
||||
return err
|
||||
}
|
||||
|
||||
func (b liveBus) Delete(ctx context.Context, stream string) error {
|
||||
return bussnap.DeleteStream(ctx, b.nc, stream)
|
||||
}
|
||||
|
||||
func (b liveBus) Restore(ctx context.Context, file string) error {
|
||||
scratch, err := os.MkdirTemp(b.scratch, ".restore-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.RemoveAll(scratch)
|
||||
return bussnap.RestoreFile(ctx, b.nc, file, scratch, b.logf)
|
||||
}
|
||||
|
||||
// SetConsumers sets each durable consumer to its recorded sequence: the server's snapshot carries its consumers, so one
|
||||
// restored with its stream stands as it was taken; one missing is made again from the sequence after its acknowledged one.
|
||||
func (b liveBus) SetConsumers(ctx context.Context, cs []seat.Consumer) error {
|
||||
for _, c := range cs {
|
||||
s, err := b.js.Stream(ctx, c.Stream)
|
||||
if err != nil {
|
||||
return fmt.Errorf("stream %s: %w", c.Stream, err)
|
||||
}
|
||||
if _, err := s.Consumer(ctx, c.Name); err == nil {
|
||||
continue
|
||||
} else if !errors.Is(err, jetstream.ErrConsumerNotFound) {
|
||||
return fmt.Errorf("the consumer %s of %s: %w", c.Name, c.Stream, err)
|
||||
}
|
||||
if _, err := s.CreateConsumer(ctx, jetstream.ConsumerConfig{Durable: c.Name, AckPolicy: jetstream.AckExplicitPolicy,
|
||||
DeliverPolicy: jetstream.DeliverByStartSequencePolicy, OptStartSeq: uint64(c.Sequence) + 1}); err != nil {
|
||||
return fmt.Errorf("the consumer %s of %s at %d: %w", c.Name, c.Stream, c.Sequence, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var _ recovery.Bus = liveBus{}
|
||||
@@ -0,0 +1,163 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"mesh-recovery/internal/recovery"
|
||||
)
|
||||
|
||||
// The seat node-backup's item events (design 47 G §554, §563), each kept last per subject on a stream of its own.
|
||||
const (
|
||||
backupSeat = "node-backup"
|
||||
itemStarted = "item-started"
|
||||
itemEnded = "item-ended"
|
||||
itemStartedSteam = "SEAT_node-backup_item-started"
|
||||
itemEndedStream = "SEAT_node-backup_item-ended"
|
||||
)
|
||||
|
||||
func itemSubject(event, node, module, item string) string {
|
||||
return "mesh.seat." + backupSeat + ".event." + event + "." + node + "." + module + "." + item
|
||||
}
|
||||
|
||||
// dumpView is what the backup holders say of each item, `<node>.<module>.<item>` to the `at` of its last start and of its
|
||||
// last end, read as §556 has it. A dump of an item runs while its last start is newer than its last end.
|
||||
type dumpView struct {
|
||||
mu sync.Mutex
|
||||
started map[string]time.Time
|
||||
ended map[string]time.Time
|
||||
changed chan struct{}
|
||||
// busModule is the module that holds the seat mesh-broker now (A-126), whose items are the bus's.
|
||||
busModule func() (string, error)
|
||||
js jetstream.JetStream
|
||||
}
|
||||
|
||||
func newDumpView(js jetstream.JetStream, busModule func() (string, error)) *dumpView {
|
||||
return &dumpView{started: map[string]time.Time{}, ended: map[string]time.Time{}, changed: make(chan struct{}),
|
||||
busModule: busModule, js: js}
|
||||
}
|
||||
|
||||
// hear takes one item event, an older one by `at` than the one held changing nothing, and wakes every wait.
|
||||
func (d *dumpView) hear(m jetstream.Msg) {
|
||||
d.take(m.Subject(), m.Data())
|
||||
_ = m.Ack()
|
||||
}
|
||||
|
||||
func (d *dumpView) take(subject string, data []byte) {
|
||||
var body struct {
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
rest, ok := strings.CutPrefix(subject, "mesh.seat."+backupSeat+".event.")
|
||||
if !ok || json.Unmarshal(data, &body) != nil {
|
||||
return
|
||||
}
|
||||
event, item, ok := strings.Cut(rest, ".")
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
into := d.started
|
||||
if event == itemEnded {
|
||||
into = d.ended
|
||||
} else if event != itemStarted {
|
||||
return
|
||||
}
|
||||
if old, ok := into[item]; ok && body.At.Before(old) {
|
||||
return
|
||||
}
|
||||
into[item] = body.At
|
||||
close(d.changed)
|
||||
d.changed = make(chan struct{})
|
||||
}
|
||||
|
||||
// runs is the first item of the module whose dump runs, or "".
|
||||
func (d *dumpView) runs(module string) string {
|
||||
token := keys.EscapeName(module)
|
||||
var on []string
|
||||
for item, at := range d.started {
|
||||
parts := strings.Split(item, ".")
|
||||
if len(parts) != 3 || parts[1] != token {
|
||||
continue
|
||||
}
|
||||
if end, ok := d.ended[item]; !ok || end.Before(at) {
|
||||
on = append(on, item)
|
||||
}
|
||||
}
|
||||
sort.Strings(on)
|
||||
if len(on) == 0 {
|
||||
return ""
|
||||
}
|
||||
return on[0]
|
||||
}
|
||||
|
||||
// readNow reads the last start and end of each of the bus module's items once more, through an ordered consumer made for
|
||||
// that read (§556): the restore said its restore-started, acknowledged, before it reads (§563).
|
||||
func (d *dumpView) readNow(ctx context.Context, module string) error {
|
||||
for _, c := range []struct{ stream, event string }{{itemStartedSteam, itemStarted}, {itemEndedStream, itemEnded}} {
|
||||
cons, err := d.js.OrderedConsumer(ctx, c.stream, jetstream.OrderedConsumerConfig{
|
||||
FilterSubjects: []string{itemSubject(c.event, "*", keys.EscapeName(module), "*")},
|
||||
DeliverPolicy: jetstream.DeliverLastPerSubjectPolicy})
|
||||
if err != nil {
|
||||
return fmt.Errorf("the read of %s: %w", c.stream, err)
|
||||
}
|
||||
info, err := cons.Info(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the read of %s: %w", c.stream, err)
|
||||
}
|
||||
for left := info.NumPending; left > 0; {
|
||||
m, err := cons.Next(jetstream.FetchMaxWait(5 * time.Second))
|
||||
if err != nil {
|
||||
return fmt.Errorf("the read of %s: %w", c.stream, err)
|
||||
}
|
||||
d.take(m.Subject(), m.Data())
|
||||
meta, err := m.Metadata()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
left = meta.NumPending
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Wait returns once no dump of the bus module's items runs; it calls waiting once, naming the item, when it has to wait
|
||||
// (§554, §563). It wakes on the item events, never on a timer.
|
||||
func (d *dumpView) Wait(ctx context.Context, waiting func(on string)) error {
|
||||
module, err := d.busModule()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := d.readNow(ctx, module); err != nil {
|
||||
return err
|
||||
}
|
||||
said := false
|
||||
for {
|
||||
d.mu.Lock()
|
||||
on, ch := d.runs(module), d.changed
|
||||
d.mu.Unlock()
|
||||
if on == "" {
|
||||
return nil
|
||||
}
|
||||
if !said {
|
||||
waiting("the item-ended of the dump of " + on)
|
||||
said = true
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-ch:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var _ recovery.Dump = (*dumpView)(nil)
|
||||
@@ -0,0 +1,82 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"mesh-recovery/internal/recovery"
|
||||
)
|
||||
|
||||
// boundState is the module's own state bounds, whose keys live a time of their own (key-ttl): a bound key's expiry is
|
||||
// heard as the bus's own change, so nothing in the process waits on a timer (design 47 G A-77, A-138).
|
||||
type boundState interface {
|
||||
CreateTTL(key string, value any, ttl time.Duration) (uint64, error)
|
||||
Watch(key string, handle func(stdio.StateChange) error) error
|
||||
Delete(key string) error
|
||||
}
|
||||
|
||||
// errBoundEnded ends a bound key's watch once the key is gone.
|
||||
var errBoundEnded = errors.New("the bound key is removed")
|
||||
|
||||
// stateBounds times an act's waits by bound keys of the state bounds (A-77, A-143): the key is watched before it is
|
||||
// written, so its expiry is never missed, and the watch ends with the key's removal.
|
||||
type stateBounds struct{ s boundState }
|
||||
|
||||
func (b stateBounds) Bound(key string, ttl time.Duration, expired func()) (func(), error) {
|
||||
var ended atomic.Bool
|
||||
if err := b.s.Watch(key, func(c stdio.StateChange) error {
|
||||
if c.Op == "put" || c.Current {
|
||||
return nil
|
||||
}
|
||||
if !ended.Swap(true) {
|
||||
expired()
|
||||
}
|
||||
return errBoundEnded
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := b.s.CreateTTL(key, map[string]any{"ttl": ttl.String()}, ttl); err != nil {
|
||||
ended.Store(true)
|
||||
_ = b.s.Delete(key)
|
||||
return nil, err
|
||||
}
|
||||
return func() {
|
||||
if !ended.Swap(true) {
|
||||
_ = b.s.Delete(key)
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
|
||||
var _ recovery.Bounds = stateBounds{}
|
||||
|
||||
// expiryKey is the bound key of §551's daily removal.
|
||||
const expiryKey = "expiry"
|
||||
|
||||
// expireOnBound runs due each time the bound key `expiry`, written with the time to live `every` (expire-every-hours), is
|
||||
// removed by the bus, and writes it again (§551, A-138). A key a run before this start wrote is kept, so a restart does not
|
||||
// put the next removal off; its removal at the start is the start's own run.
|
||||
func expireOnBound(s boundState, every time.Duration, due func(), logf func(string, ...any)) error {
|
||||
arm := func() error {
|
||||
_, err := s.CreateTTL(expiryKey, map[string]any{"every": every.String()}, every)
|
||||
if errors.Is(err, stdio.ErrChanged) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if err := s.Watch(expiryKey, func(c stdio.StateChange) error {
|
||||
if c.Op == "put" || c.Current {
|
||||
return nil
|
||||
}
|
||||
due()
|
||||
if err := arm(); err != nil {
|
||||
logf("the bound of the next removal of points past their keep days is not written: %v", err)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
return arm()
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"mesh-recovery/internal/recovery"
|
||||
)
|
||||
|
||||
// liveKeeper is the bus module's keeper reached by the seat mesh-broker's verbs (design 47 G §681): snapshot-get's chunks
|
||||
// received in mesh-sdk's form (A-139), snapshot-restore and its cancel called, and snapshot-restored heard.
|
||||
type liveKeeper struct {
|
||||
nc *nats.Conn
|
||||
call Call
|
||||
restored *restoredView
|
||||
}
|
||||
|
||||
func (k liveKeeper) verb(verb string, in, out any) error {
|
||||
raw, err := k.call("seat:"+meshbroker.Seat+"."+verb, in)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s's %s: %w", meshbroker.Seat, verb, err)
|
||||
}
|
||||
if err := json.Unmarshal(raw, out); err != nil {
|
||||
return fmt.Errorf("%s's %s answered what does not read: %w", meshbroker.Seat, verb, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get subscribes to the chunks of a call of its own before it asks snapshot-get, so none is missed, then receives them in
|
||||
// order, acknowledging each (§681, A-139).
|
||||
func (k liveKeeper) Get(ctx context.Context, request string, fits func(meshbroker.SnapshotGetAnswer) error, w io.Writer) (meshbroker.SnapshotGetAnswer, error) {
|
||||
var bits [16]byte
|
||||
if _, err := rand.Read(bits[:]); err != nil {
|
||||
return meshbroker.SnapshotGetAnswer{}, err
|
||||
}
|
||||
send := buscalls.SeatChunks{Seat: meshbroker.Seat, Verb: meshbroker.VerbSnapshotGet, Call: hex.EncodeToString(bits[:])}
|
||||
chunks := make(chan *nats.Msg, 4)
|
||||
sub, err := k.nc.ChanSubscribe(send.Chunks(), chunks)
|
||||
if err != nil {
|
||||
return meshbroker.SnapshotGetAnswer{}, fmt.Errorf("the chunks of snapshot-get cannot be heard: %w", err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
var ans meshbroker.SnapshotGetAnswer
|
||||
if err := k.verb(meshbroker.VerbSnapshotGet, meshbroker.SnapshotGetInput{Call: send.Call, Request: request}, &ans); err != nil {
|
||||
return ans, err
|
||||
}
|
||||
if ans.Error != "" {
|
||||
return ans, nil
|
||||
}
|
||||
if err := fits(ans); err != nil {
|
||||
return ans, err
|
||||
}
|
||||
if ans.Size > 0 && ans.Chunks == 0 {
|
||||
return ans, fmt.Errorf("the keeper answered the request %s's %d bytes with no chunk: this account is granted no chunk subjects of snapshot-get", request, ans.Size)
|
||||
}
|
||||
next := func() (string, []byte, error) {
|
||||
bound := time.NewTimer(buscalls.SeatAckWithin)
|
||||
defer bound.Stop()
|
||||
select {
|
||||
case m := <-chunks:
|
||||
return m.Subject, m.Data, nil
|
||||
case <-bound.C:
|
||||
return "", nil, buscalls.ErrSeatChunkMissed
|
||||
case <-ctx.Done():
|
||||
return "", nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
ack := func(subject string) error {
|
||||
_, err := k.nc.Request(subject, []byte("{}"), buscalls.SeatAckWithin)
|
||||
return err
|
||||
}
|
||||
if _, err := send.Receive(ans.Chunks, next, ack, w); err != nil {
|
||||
return ans, fmt.Errorf("the snapshot of the request %s: %w", request, err)
|
||||
}
|
||||
return ans, nil
|
||||
}
|
||||
|
||||
func (k liveKeeper) Restore(_ context.Context, request string, verify bool, why string) (meshbroker.SnapshotRestoreAnswer, error) {
|
||||
var ans meshbroker.SnapshotRestoreAnswer
|
||||
err := k.verb(meshbroker.VerbSnapshotRestore, meshbroker.SnapshotRestoreInput{Request: request, Verify: verify, Why: why}, &ans)
|
||||
return ans, err
|
||||
}
|
||||
|
||||
func (k liveKeeper) CancelRestore(_ context.Context, act, why string) (meshbroker.SnapshotRestoreCancelAnswer, error) {
|
||||
var ans meshbroker.SnapshotRestoreCancelAnswer
|
||||
err := k.verb(meshbroker.VerbSnapshotRestoreCancel, meshbroker.SnapshotRestoreCancelInput{Act: act, Why: why}, &ans)
|
||||
return ans, err
|
||||
}
|
||||
|
||||
func (k liveKeeper) Restored(ctx context.Context, request, act string) (meshbroker.SnapshotRestoredEvent, bool) {
|
||||
return k.restored.wait(ctx, request, act)
|
||||
}
|
||||
|
||||
var _ recovery.Keeper = liveKeeper{}
|
||||
|
||||
// restoredView is the keeper's snapshot-restored as heard, each request and act kept (§681): read as §556 has it, the last
|
||||
// of each subject first, so an outcome said before a wait began wakes it too.
|
||||
type restoredView struct {
|
||||
mu sync.Mutex
|
||||
by map[string]meshbroker.SnapshotRestoredEvent
|
||||
changed chan struct{}
|
||||
}
|
||||
|
||||
func newRestoredView() *restoredView {
|
||||
return &restoredView{by: map[string]meshbroker.SnapshotRestoredEvent{}, changed: make(chan struct{})}
|
||||
}
|
||||
|
||||
func (v *restoredView) hear(m jetstream.Msg) {
|
||||
var ev meshbroker.SnapshotRestoredEvent
|
||||
if err := json.Unmarshal(m.Data(), &ev); err == nil && ev.Request != "" && ev.Act != "" {
|
||||
v.take(ev)
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
|
||||
func (v *restoredView) take(ev meshbroker.SnapshotRestoredEvent) {
|
||||
v.mu.Lock()
|
||||
v.by[ev.Request+"\x00"+ev.Act] = ev
|
||||
close(v.changed)
|
||||
v.changed = make(chan struct{})
|
||||
v.mu.Unlock()
|
||||
}
|
||||
|
||||
func (v *restoredView) wait(ctx context.Context, request, act string) (meshbroker.SnapshotRestoredEvent, bool) {
|
||||
for {
|
||||
v.mu.Lock()
|
||||
ev, ok := v.by[request+"\x00"+act]
|
||||
changed := v.changed
|
||||
v.mu.Unlock()
|
||||
if ok {
|
||||
return ev, true
|
||||
}
|
||||
select {
|
||||
case <-changed:
|
||||
case <-ctx.Done():
|
||||
return meshbroker.SnapshotRestoredEvent{}, false
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
// mesh-recovery: the recovery module's process (design 47 G §554). One process on an account of its own holds the seat
|
||||
// mesh-recovery and answers its verbs itself over the bus with the caller the bus proved (A-107); it takes, sets aside and
|
||||
// restores modules' kept data on the live bus, one act at a time, each act's result said as the seat's event.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"mesh-recovery/internal/recovery"
|
||||
)
|
||||
|
||||
// The settings' files and the points directory, as the manifest's process section sets them (A-112, §633).
|
||||
const (
|
||||
envPoints = "MESH_RECOVERY_POINTS"
|
||||
envCap = "MESH_RECOVERY_POINTS_CAP_FILE"
|
||||
envKeepDays = "MESH_RECOVERY_KEEP_DAYS_FILE"
|
||||
envMark = "MESH_RECOVERY_FILLING_MARK_FILE"
|
||||
envDumpWait = "MESH_RECOVERY_DUMP_WAIT_FILE"
|
||||
envRestore = "MESH_RECOVERY_RESTORE_WAIT_FILE"
|
||||
envExpire = "MESH_RECOVERY_EXPIRE_EVERY_FILE"
|
||||
)
|
||||
|
||||
// The settings' defaults: the design's values (§551, §564).
|
||||
const (
|
||||
defaultCap = 10 << 30
|
||||
defaultKeepDays = 90
|
||||
)
|
||||
|
||||
// defaultExpireEvery is how often §551's removal runs beside its start and each snapshot-kept: once a day (expire-every-hours).
|
||||
const defaultExpireEvery = 24 * time.Hour
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
if err := run(ctx, os.Getenv, os.Stderr); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "mesh-recovery:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// digests is the state digests, the points' digests kept on the bus apart from their bytes (§567).
|
||||
type digests struct{ s process.State }
|
||||
|
||||
func (d digests) Put(key string, v []byte) error {
|
||||
_, err := d.s.Put(key, json.RawMessage(v))
|
||||
return err
|
||||
}
|
||||
|
||||
func (d digests) Get(key string) ([]byte, bool, error) {
|
||||
e, err := d.s.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return e.Value, true, nil
|
||||
}
|
||||
|
||||
func (d digests) Delete(key string) error { return d.s.Delete(key) }
|
||||
|
||||
// run is the process: it returns when ctx ends, after the running act ended.
|
||||
func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
|
||||
logger := log.New(logOut, "mesh-recovery: ", log.LstdFlags)
|
||||
dir := getenv(envPoints)
|
||||
if dir == "" {
|
||||
return errors.New(envPoints + " names no points directory, so nothing can be kept")
|
||||
}
|
||||
cap, err := readNumber(getenv(envCap), defaultCap)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
keepDays, err := readFraction(getenv(envKeepDays), defaultKeepDays)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
mark, err := readNumber(getenv(envMark), recovery.DefaultFillingMark)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dumpWait, err := readMinutes(getenv(envDumpWait), recovery.DefaultDumpWait)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
restoreWait, err := readMinutes(getenv(envRestore), recovery.DefaultRestoreWait)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
expireEvery, err := readFraction(getenv(envExpire), defaultExpireEvery.Hours())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
account, err := process.ReadBrokerAccount(getenv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
conn, err := process.Connect(account)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer conn.Close()
|
||||
nc, js, err := account.Dial(nats.Name(account.Node + "." + account.Module))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer nc.Close()
|
||||
|
||||
// What the engine reads of the bus, each read as §556 has it before it acts on it: the node-engine seat's running
|
||||
// (A-132) and the backup holders' item events of the bus module (§563).
|
||||
running := newRunningView()
|
||||
if _, err := account.HearSeatEvents(ctx, js, nodeengine.StreamRunning, running.hear, logger.Printf, process.OnRead(running.onRead)); err != nil {
|
||||
return err
|
||||
}
|
||||
dump := newDumpView(js, func() (string, error) { return seatlist.HolderOf(conn.Call, meshbroker.Seat) })
|
||||
for _, s := range []string{itemStartedSteam, itemEndedStream} {
|
||||
if _, err := account.HearSeatEvents(ctx, js, s, dump.hear, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// The keeper's snapshot-restored, each request and act, for a restore in place (§681).
|
||||
restored := newRestoredView()
|
||||
if _, err := account.HearSeatEvents(ctx, js, meshbroker.StreamSnapshotRestored, restored.hear, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
// The module's findings go through mesh-sdk's held writer over its own state findings (§561), started before the
|
||||
// engine's start, so what the start writes is held and written.
|
||||
writer, err := views.NewHeldWriter(views.WriterConfig[meshconditions.Finding, meshconditions.FindingCleared, meshconditions.FindingsStated, meshconditions.FindingsAskedEvent]{
|
||||
Asked: meshconditions.SubjectFindingsAsked, Module: account.Module, Node: account.Node, State: conn.State("findings"),
|
||||
Kinds: recovery.FindingKinds, JS: js, Conn: nc, Logf: logger.Printf})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := writer.Start(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
defer writer.Close()
|
||||
e := &recovery.Engine{
|
||||
Dir: dir,
|
||||
Bus: liveBus{nc: nc, js: js, scratch: dir, logf: logger.Printf},
|
||||
Seats: liveSeats{emit: func(seat, event string, body any) error { return conn.EmitSeat(seat, event, body) },
|
||||
recorder: meshhandactsclient.Recorder{Ask: conn.Call}, call: conn.Call, callOn: conn.CallOn,
|
||||
node: account.Node, points: dir, running: running},
|
||||
Digests: digests{conn.State("digests")},
|
||||
Dump: dump,
|
||||
Keeper: liveKeeper{nc: nc, call: conn.Call, restored: restored},
|
||||
Bounds: stateBounds{conn.State("bounds")},
|
||||
Settings: recovery.Settings{Cap: cap, KeepDays: keepDays, FillingMark: int(mark), DumpWait: dumpWait, RestoreWait: restoreWait},
|
||||
Findings: writer,
|
||||
Node: account.Node,
|
||||
Logf: logger.Printf,
|
||||
}
|
||||
for _, x := range e.Start(ctx) {
|
||||
logger.Printf("the aside %s of %s's %s holds data the live bus lacks: recovery_restore_aside puts it back", x.Aside.ID, x.Module, x.Kept)
|
||||
}
|
||||
stopAnswers, err := conn.AnswerSeat(recovery.Seat, e.Handlers())
|
||||
if err != nil {
|
||||
return fmt.Errorf("the seat %s's verbs cannot be answered: %w", recovery.Seat, err)
|
||||
}
|
||||
defer stopAnswers()
|
||||
logger.Printf("started on the account %s, holding the seat %s", account.User, recovery.Seat)
|
||||
expire := func(why string) {
|
||||
if a := e.Accept(recovery.Act{Kind: recovery.KindExpire, Why: why, Caller: account.User}); a.Accepted == "" {
|
||||
logger.Printf("the removal of points past their keep days is not run now (%s%s); it runs at the next snapshot-kept or day", a.Busy, a.Error)
|
||||
}
|
||||
}
|
||||
expire("the removal of points past their keep days at the start (design 47 G section 551)")
|
||||
every := time.Duration(expireEvery * float64(time.Hour))
|
||||
if err := expireOnBound(conn.State("bounds"), every, func() {
|
||||
expire("the removal of points past their keep days on its bound of " + every.String() + " (design 47 G section 551)")
|
||||
}, logger.Printf); err != nil {
|
||||
return fmt.Errorf("the bound of the removal of points past their keep days: %w", err)
|
||||
}
|
||||
<-ctx.Done()
|
||||
// A move or a restart stops the process: it takes no new call, the running act ends through its own failure path, and
|
||||
// every accepted act ends stopped, so its caller asks again (§562, A-141).
|
||||
e.Stop("its stop signal")
|
||||
stopAnswers()
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// The process's own readings of the mesh (design 47 G §554, §563, A-121, A-132): the bus names of a kept data from its
|
||||
// manifest, the fingerprint a module runs at from the node-engine seat's running, and whether a dump of the bus module runs.
|
||||
func TestWhatTheProcessReads(t *testing.T) {
|
||||
t.Run("bus names of a kept data (A-121)", func(t *testing.T) {
|
||||
m, err := catalogue.Decode([]byte(`{"module":"app","version":"1","state":[{"name":"old"}],"logs":[{"name":"journal"}],
|
||||
"streams":[{"name":"APP_EVENTS"}],"objects":[{"name":"files"}],
|
||||
"requires":[{"name":"notes","storage":"key-value","template":"state"}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for kept, want := range map[string]string{"notes": "KV_app_notes OBJ_app_notes-objects", "old": "KV_app_old",
|
||||
"journal": "LOG_app_journal", "APP_EVENTS": "APP_EVENTS", "files": "OBJ_app_files"} {
|
||||
got, err := KeptStreams("app", kept, m)
|
||||
if err != nil || strings.Join(got, " ") != want {
|
||||
t.Errorf("%s: %v, %v; want %s", kept, got, err, want)
|
||||
}
|
||||
}
|
||||
if _, err := KeptStreams("app", "none", m); err == nil {
|
||||
t.Error("a kept data the manifest does not declare was named")
|
||||
}
|
||||
})
|
||||
t.Run("the running fingerprint (A-132)", func(t *testing.T) {
|
||||
v := newRunningView()
|
||||
if _, err := v.Fingerprint("app"); err == nil {
|
||||
t.Error("answered before the read of running ended")
|
||||
}
|
||||
v.onRead(process.ReadDone)
|
||||
at := time.Date(2026, 11, 12, 1, 0, 0, 0, time.UTC)
|
||||
v.by["a\x00app"] = nodeengine.RunningEvent{Node: "a", Module: "app", Fingerprint: "src2:1", At: at}
|
||||
v.by["b\x00app"] = nodeengine.RunningEvent{Node: "b", Module: "app", Stopped: true, At: at}
|
||||
if fp, err := v.Fingerprint("app"); err != nil || fp != "src2:1" {
|
||||
t.Errorf("%q, %v", fp, err)
|
||||
}
|
||||
v.by["b\x00app"] = nodeengine.RunningEvent{Node: "b", Module: "app", Fingerprint: "src2:2", At: at}
|
||||
if _, err := v.Fingerprint("app"); err == nil || !strings.Contains(err.Error(), "src2:2 on b") {
|
||||
t.Errorf("two fingerprints answered %v", err)
|
||||
}
|
||||
})
|
||||
t.Run("the keeper's snapshot-restored of a request and act (§681)", func(t *testing.T) {
|
||||
v := newRestoredView()
|
||||
v.take(meshbroker.SnapshotRestoredEvent{Request: "req-1", Act: "act-0", Outcome: "failed"})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, ok := v.wait(ctx, "req-1", "act-1"); ok {
|
||||
t.Error("another act's outcome woke the wait")
|
||||
}
|
||||
woken := make(chan meshbroker.SnapshotRestoredEvent, 1)
|
||||
go func() {
|
||||
ev, _ := v.wait(context.Background(), "req-1", "act-1")
|
||||
woken <- ev
|
||||
}()
|
||||
v.take(meshbroker.SnapshotRestoredEvent{Request: "req-1", Act: "act-1", Outcome: "done"})
|
||||
if ev := <-woken; ev.Outcome != "done" {
|
||||
t.Errorf("woken by %+v", ev)
|
||||
}
|
||||
if ev, ok := v.wait(ctx, "req-1", "act-1"); !ok || ev.Outcome != "done" {
|
||||
t.Errorf("an outcome said before the wait is not taken: %+v %v", ev, ok)
|
||||
}
|
||||
})
|
||||
t.Run("every bound is a setting, a fraction allowed (§557, A-138)", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
write := func(name, v string) string {
|
||||
p := filepath.Join(dir, name)
|
||||
_ = os.WriteFile(p, []byte(v+"\n"), 0o600)
|
||||
return p
|
||||
}
|
||||
if d, err := readMinutes(write("wait", "0.05"), time.Hour); err != nil || d != 3*time.Second {
|
||||
t.Errorf("0.05 minutes read %s, %v", d, err)
|
||||
}
|
||||
if n, err := readFraction(write("days", "0.00002"), 90); err != nil || n != 0.00002 {
|
||||
t.Errorf("a fraction of a day read %v, %v", n, err)
|
||||
}
|
||||
if n, err := readFraction(filepath.Join(dir, "none"), 24); err != nil || n != 24 {
|
||||
t.Errorf("a missing setting read %v, %v", n, err)
|
||||
}
|
||||
if _, err := readFraction(write("zero", "0"), 24); err == nil {
|
||||
t.Error("a bound of 0 was taken")
|
||||
}
|
||||
})
|
||||
t.Run("the expiry runs on its bound key's expiry (§551, A-77, A-138)", func(t *testing.T) {
|
||||
s := &fakeBounds{}
|
||||
runs := 0
|
||||
if err := expireOnBound(s, 2*time.Second, func() { runs++ }, t.Logf); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = s.watch(stdio.StateChange{Key: expiryKey, Op: "put"})
|
||||
_ = s.watch(stdio.StateChange{Key: expiryKey, Op: "delete"})
|
||||
if runs != 1 || len(s.created) != 2 || s.created[1] != 2*time.Second {
|
||||
t.Errorf("%d runs, keys written %v", runs, s.created)
|
||||
}
|
||||
s.exists = true // a key a run before this start wrote stands
|
||||
if err := expireOnBound(s, time.Hour, func() {}, t.Logf); err != nil {
|
||||
t.Errorf("a standing bound key is refused: %v", err)
|
||||
}
|
||||
})
|
||||
t.Run("an act's wait is a bound key, its expiry heard on its watch (A-143)", func(t *testing.T) {
|
||||
s := &fakeBounds{}
|
||||
expired := 0
|
||||
end, err := stateBounds{s}.Bound("dump-wait.act-1", 3*time.Second, func() { expired++ })
|
||||
if err != nil || len(s.created) != 1 || s.created[0] != 3*time.Second {
|
||||
t.Fatalf("%v, keys written %v", err, s.created)
|
||||
}
|
||||
if s.watch(stdio.StateChange{Key: "dump-wait.act-1", Op: "put"}) != nil || expired != 0 {
|
||||
t.Error("the key's own write ended the wait")
|
||||
}
|
||||
if s.watch(stdio.StateChange{Key: "dump-wait.act-1", Op: "delete"}) == nil || expired != 1 {
|
||||
t.Errorf("the key's expiry: %d calls, its watch not ended", expired)
|
||||
}
|
||||
end()
|
||||
if expired != 1 || len(s.deleted) != 0 {
|
||||
t.Errorf("an expired key's end: %d calls, deleted %v", expired, s.deleted)
|
||||
}
|
||||
s = &fakeBounds{}
|
||||
end, _ = stateBounds{s}.Bound("restore-wait.act-2", time.Minute, func() { expired++ })
|
||||
end()
|
||||
if s.watch(stdio.StateChange{Key: "restore-wait.act-2", Op: "delete"}) == nil || expired != 1 ||
|
||||
strings.Join(s.deleted, " ") != "restore-wait.act-2" {
|
||||
t.Errorf("an ended key's removal: %d calls, deleted %v", expired, s.deleted)
|
||||
}
|
||||
})
|
||||
t.Run("a dump of the bus module runs (§563)", func(t *testing.T) {
|
||||
d := newDumpView(nil, nil)
|
||||
start := time.Date(2026, 11, 12, 2, 0, 0, 0, time.UTC)
|
||||
d.take(itemSubject(itemStarted, "srv", "nats", "data"), []byte(`{"at":"`+start.Format(time.RFC3339)+`"}`))
|
||||
d.take(itemSubject(itemStarted, "srv", "app", "data"), []byte(`{"at":"`+start.Format(time.RFC3339)+`"}`))
|
||||
if on := d.runs("nats"); on != "srv.nats.data" {
|
||||
t.Errorf("runs %q", on)
|
||||
}
|
||||
d.take(itemSubject(itemEnded, "srv", "nats", "data"), []byte(`{"at":"`+start.Add(time.Minute).Format(time.RFC3339)+`","outcome":"done"}`))
|
||||
if on := d.runs("nats"); on != "" {
|
||||
t.Errorf("an ended dump runs: %q", on)
|
||||
}
|
||||
d.take(itemSubject(itemEnded, "srv", "nats", "data"), []byte(`{"at":"`+start.Add(-time.Hour).Format(time.RFC3339)+`","outcome":"done"}`))
|
||||
if on := d.runs("nats"); on != "" {
|
||||
t.Errorf("an older end changed what runs: %q", on)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// fakeBounds is the state bounds: the keys written with their time to live, and the watch's handler.
|
||||
type fakeBounds struct {
|
||||
created []time.Duration
|
||||
exists bool
|
||||
watch func(stdio.StateChange) error
|
||||
deleted []string
|
||||
}
|
||||
|
||||
func (f *fakeBounds) CreateTTL(_ string, _ any, ttl time.Duration) (uint64, error) {
|
||||
if f.exists {
|
||||
return 0, stdio.ErrChanged
|
||||
}
|
||||
f.created = append(f.created, ttl)
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
func (f *fakeBounds) Watch(_ string, handle func(stdio.StateChange) error) error {
|
||||
f.watch = handle
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeBounds) Delete(key string) error {
|
||||
f.deleted = append(f.deleted, key)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"mesh-recovery/internal/recovery"
|
||||
)
|
||||
|
||||
// Call is how the process calls a seat's verb (process.Conn.Call), and CallOn one machine's holder of a node seat.
|
||||
type (
|
||||
Call func(key string, body any) (json.RawMessage, error)
|
||||
CallOn func(node, key string, body any) (json.RawMessage, error)
|
||||
)
|
||||
|
||||
// liveSeats is what the engine asks of the rest of the mesh, each by its seat (design 47 G §554).
|
||||
type liveSeats struct {
|
||||
emit func(seat, event string, body any) error
|
||||
recorder meshhandactsclient.Recorder
|
||||
call Call
|
||||
callOn CallOn
|
||||
node string
|
||||
points string
|
||||
running *runningView
|
||||
}
|
||||
|
||||
// Say says one event of the seat mesh-recovery, each part filled in as one token, escaped as a key's name is (§523).
|
||||
func (s liveSeats) Say(event string, body any, parts ...string) error {
|
||||
subject := event
|
||||
for _, p := range parts {
|
||||
if p == "" || p == "." || p == ".." {
|
||||
return fmt.Errorf("the event %s cannot be said with the part %q", event, p)
|
||||
}
|
||||
subject += "." + keys.EscapeName(p)
|
||||
}
|
||||
return s.emit(recovery.Seat, subject, body)
|
||||
}
|
||||
|
||||
// HandAct writes the act in the seat mesh-hand-acts' log first, naming the account the bus proved and who asked (§554, §682).
|
||||
func (s liveSeats) HandAct(caller, asker, what, why string) error {
|
||||
line := what + " (the account " + orNone(caller)
|
||||
if asker != "" {
|
||||
line += ", asked by " + asker
|
||||
}
|
||||
_, err := s.recorder.Record(meshhandactsclient.Request{What: line + ")", Why: why, Verb: recovery.Seat})
|
||||
return err
|
||||
}
|
||||
|
||||
func orNone(s string) string {
|
||||
if s == "" {
|
||||
return "unproven"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Streams are the bus names of a module's kept data at a fingerprint, from the catalog graph entry `mesh-catalog`'s `entry`
|
||||
// answers (§554 Bus names, §443): a template entry's as mesh-sdk names them, an old section's as the bus composer of
|
||||
// mesh-sdk names it when it makes it (A-121).
|
||||
func (s liveSeats) Streams(_ context.Context, module, kept, fingerprint string) ([]string, error) {
|
||||
raw, err := s.call("seat:"+meshcatalog.Seat+"."+meshcatalog.VerbEntry, meshcatalog.EntryInput{Fingerprint: fingerprint})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the catalog entry of %s: %w", fingerprint, err)
|
||||
}
|
||||
var ans meshcatalog.EntryAnswer
|
||||
if err := json.Unmarshal(raw, &ans); err != nil {
|
||||
return nil, fmt.Errorf("the catalog entry of %s does not read: %w", fingerprint, err)
|
||||
}
|
||||
if ans.Entry.Module != module {
|
||||
return nil, fmt.Errorf("the catalog entry of %s is %s's, not %s's", fingerprint, ans.Entry.Module, module)
|
||||
}
|
||||
whole, err := json.Marshal(ans.Entry.Manifest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m, err := catalogue.Decode(whole)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the manifest of %s at %s: %w", module, fingerprint, err)
|
||||
}
|
||||
return KeptStreams(module, kept, m)
|
||||
}
|
||||
|
||||
// KeptStreams names the bus objects of one kept data a manifest declares (A-121).
|
||||
func KeptStreams(module, kept string, m catalogue.Manifest) ([]string, error) {
|
||||
for _, e := range m.BusState {
|
||||
if e.Name == kept {
|
||||
if names := catalogue.BusStateStreams(module, e); len(names) > 0 {
|
||||
return names, nil
|
||||
}
|
||||
return nil, fmt.Errorf("%s's %s has no bus name", module, kept)
|
||||
}
|
||||
}
|
||||
for _, d := range catalogue.KeptDeclarations(m) {
|
||||
if d.Name != kept {
|
||||
continue
|
||||
}
|
||||
switch d.Kind {
|
||||
case catalogue.KeptKindState:
|
||||
return []string{"KV_" + buscompose.BucketName(module, kept)}, nil
|
||||
case catalogue.KeptKindLog:
|
||||
return []string{buscompose.LogStreamName(module, kept)}, nil
|
||||
case catalogue.KeptKindStream:
|
||||
return []string{kept}, nil
|
||||
case catalogue.KeptKindObjects:
|
||||
return []string{"OBJ_" + buscompose.BucketName(module, kept)}, nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("%s's manifest declares no kept data %s", module, kept)
|
||||
}
|
||||
|
||||
// RunningFingerprint is the fingerprint the node-engine runs the module at, from the node-engine seat's running (A-123, A-132).
|
||||
func (s liveSeats) RunningFingerprint(_ context.Context, module string) (string, error) {
|
||||
return s.running.Fingerprint(module)
|
||||
}
|
||||
|
||||
// Backup brings the points directory back beside its path through the seat node-backup's restore on this node (§516),
|
||||
// answering the directory it was brought back into.
|
||||
func (s liveSeats) Backup(_ context.Context, module string) (string, error) {
|
||||
if _, err := s.callOn(s.node, "seat:node-backup.restore", map[string]string{"module": module, "path": s.points}); err != nil {
|
||||
return "", fmt.Errorf("the backup holder on %s brought nothing back: %w", s.node, err)
|
||||
}
|
||||
back, _ := filepath.Glob(s.points + ".restored-*")
|
||||
if len(back) == 0 {
|
||||
return "", fmt.Errorf("the backup holder on %s answered, and nothing lies beside %s", s.node, s.points)
|
||||
}
|
||||
sort.Strings(back)
|
||||
return back[len(back)-1], nil
|
||||
}
|
||||
|
||||
var _ recovery.Seats = liveSeats{}
|
||||
|
||||
// runningView is what the node-engine seat's running says of each node's module, read as §556 has it: the read of the last
|
||||
// of each subject first, the durable followed after it, an older one by `at` changing nothing.
|
||||
type runningView struct {
|
||||
mu sync.Mutex
|
||||
read bool
|
||||
by map[string]nodeengine.RunningEvent
|
||||
}
|
||||
|
||||
func newRunningView() *runningView { return &runningView{by: map[string]nodeengine.RunningEvent{}} }
|
||||
|
||||
func (v *runningView) hear(m jetstream.Msg) {
|
||||
var r nodeengine.RunningEvent
|
||||
if err := json.Unmarshal(m.Data(), &r); err == nil && r.Module != "" {
|
||||
k := r.Node + "\x00" + r.Module
|
||||
v.mu.Lock()
|
||||
if old, ok := v.by[k]; !ok || !r.At.Before(old.At) {
|
||||
v.by[k] = r
|
||||
}
|
||||
v.mu.Unlock()
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
|
||||
func (v *runningView) onRead(state string) {
|
||||
v.mu.Lock()
|
||||
v.read = state == process.ReadDone
|
||||
v.mu.Unlock()
|
||||
}
|
||||
|
||||
// Fingerprint is the one fingerprint the module runs at on the nodes that run it; two different ones are refused,
|
||||
// naming both, so a set-aside never takes the names of a build that does not run.
|
||||
func (v *runningView) Fingerprint(module string) (string, error) {
|
||||
v.mu.Lock()
|
||||
defer v.mu.Unlock()
|
||||
if !v.read {
|
||||
return "", errors.New("the node-engine seat's running is still being read")
|
||||
}
|
||||
fps := map[string][]string{}
|
||||
for _, r := range v.by {
|
||||
if r.Module == module && !r.Stopped && r.Fingerprint != "" {
|
||||
fps[r.Fingerprint] = append(fps[r.Fingerprint], r.Node)
|
||||
}
|
||||
}
|
||||
switch len(fps) {
|
||||
case 0:
|
||||
return "", fmt.Errorf("no node runs %s", module)
|
||||
case 1:
|
||||
for fp := range fps {
|
||||
return fp, nil
|
||||
}
|
||||
}
|
||||
var said []string
|
||||
for fp, nodes := range fps {
|
||||
sort.Strings(nodes)
|
||||
said = append(said, fp+" on "+strings.Join(nodes, ", "))
|
||||
}
|
||||
sort.Strings(said)
|
||||
return "", fmt.Errorf("%s runs at more than one fingerprint (%s)", module, strings.Join(said, "; "))
|
||||
}
|
||||
|
||||
// readMinutes reads a bound in minutes from its setting's file (§557, A-130, A-138); the default when missing.
|
||||
func readMinutes(path string, fallback time.Duration) (time.Duration, error) {
|
||||
n, err := readFraction(path, fallback.Minutes())
|
||||
return time.Duration(n * float64(time.Minute)), err
|
||||
}
|
||||
|
||||
// readFraction reads a bound's setting (minutes, hours, days), a fraction allowed so a whole test sets it in seconds
|
||||
// (§557, A-138); the default when missing.
|
||||
func readFraction(path string, fallback float64) (float64, error) {
|
||||
if path == "" {
|
||||
return fallback, nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return fallback, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var n float64
|
||||
if err := json.Unmarshal([]byte(strings.TrimSpace(string(raw))), &n); err != nil || n <= 0 {
|
||||
return 0, fmt.Errorf("the setting in %s is %q, not a positive number", path, strings.TrimSpace(string(raw)))
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// readNumber reads a setting's file (design 47 G A-112): its whole content a positive number, the default when missing.
|
||||
func readNumber(path string, fallback int64) (int64, error) {
|
||||
if path == "" {
|
||||
return fallback, nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return fallback, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var n float64
|
||||
if err := json.Unmarshal([]byte(strings.TrimSpace(string(raw))), &n); err != nil || n <= 0 {
|
||||
return 0, fmt.Errorf("the setting in %s is %q, not a positive number", path, strings.TrimSpace(string(raw)))
|
||||
}
|
||||
return int64(n), nil
|
||||
}
|
||||
@@ -0,0 +1,473 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
// The recovery module's one whole test (design 47 G §557, §554, §681, §562, A-138, A-143): its run() as the node starts it,
|
||||
// on its own account on a bus of this test alone, every bound a setting of a few seconds. A stand-in bus module answers
|
||||
// the seat mesh-broker's snapshot-get (in chunks), snapshot-restore and snapshot-restore-cancel and says snapshot-restored;
|
||||
// stand-ins answer the hand-act log, the assigner's assignments and the catalog's entry; the node-engine seat's running and
|
||||
// the backup holder's item events are said on their streams. The live data is a real bucket, set aside and restored by
|
||||
// the bus's own snapshots.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
|
||||
seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
"git.novox.be/novox/mesh-sdk/go/bussnap"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
|
||||
"mesh-recovery/internal/recovery"
|
||||
)
|
||||
|
||||
// standKeeper is the bus module's keeper as the test plays it: a snapshot by request, protected or not, and what it says
|
||||
// after a restore in place and after its cancel ("" says nothing).
|
||||
type standKeeper struct {
|
||||
mu sync.Mutex
|
||||
file []byte // the snapshot an unprotected request sends
|
||||
protected bool
|
||||
onRestore string
|
||||
onCancel string
|
||||
asked []string
|
||||
}
|
||||
|
||||
func (k *standKeeper) set(protected bool, onRestore, onCancel string) {
|
||||
k.mu.Lock()
|
||||
k.protected, k.onRestore, k.onCancel, k.asked = protected, onRestore, onCancel, nil
|
||||
k.mu.Unlock()
|
||||
}
|
||||
|
||||
func (k *standKeeper) said() string {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
return strings.Join(k.asked, "|")
|
||||
}
|
||||
|
||||
func TestRecoveryWhole(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
nc, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer nc.Close()
|
||||
js, _ := jetstream.New(nc)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
acct := process.Account{URL: url, Node: "srv", Module: "mesh-recovery", User: "srv.mesh-recovery", Password: "unused"}
|
||||
|
||||
// The module's own states (bounds takes keys with their own time to live), the app's live kept data, and the streams the
|
||||
// grants part and the bus composer make: the seat events it hears with its durables, its own events and its findings.
|
||||
for _, b := range []jetstream.KeyValueConfig{{Bucket: "mesh-recovery_bounds", LimitMarkerTTL: time.Second},
|
||||
{Bucket: "mesh-recovery_digests"}, {Bucket: "mesh-recovery_findings"}, {Bucket: "app_notes"}} {
|
||||
if _, err := js.CreateKeyValue(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, s := range []jetstream.StreamConfig{
|
||||
{Name: "RECOVERY_EVENTS", Subjects: []string{"mesh.seat.mesh-recovery.event.>"}},
|
||||
{Name: "RECOVERY_FINDINGS", Subjects: []string{"mesh.stream.mesh-recovery.event.>"}},
|
||||
{Name: "FINDINGS_ASKED", Subjects: []string{meshconditions.SubjectFindingsAsked}},
|
||||
} {
|
||||
if _, err := js.CreateStream(ctx, s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for stream, subject := range map[string]string{
|
||||
nodeengine.StreamRunning: "mesh.seat.node-engine.event.running.>",
|
||||
itemStartedSteam: "mesh.seat.node-backup.event.item-started.>",
|
||||
itemEndedStream: "mesh.seat.node-backup.event.item-ended.>",
|
||||
meshbroker.StreamSnapshotRestored: "mesh.seat.mesh-broker.event.snapshot-restored.>",
|
||||
} {
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{subject}, MaxMsgsPerSubject: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.CreateOrUpdateConsumer(ctx, stream, acct.SeatEventConsumerConfig([]string{subject}, true)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
notes, _ := js.KeyValue(ctx, "app_notes")
|
||||
live := func() string {
|
||||
e, err := notes.Get(ctx, "n")
|
||||
if err != nil {
|
||||
return "none: " + err.Error()
|
||||
}
|
||||
return string(e.Value())
|
||||
}
|
||||
if _, err := notes.Put(ctx, "n", []byte("v1")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
running, _ := json.Marshal(nodeengine.Running{V: 1, Node: "srv", Module: "app", Fingerprint: "src2:app1", At: time.Now()})
|
||||
if _, err := js.Publish(ctx, nodeengine.SubjectRunning("srv", "app"), running); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The stand-ins of the seats the module calls.
|
||||
answer := func(m *nats.Msg, result any) {
|
||||
b, _ := json.Marshal(map[string]any{"result": result, "node": "srv"})
|
||||
_ = m.Respond(b)
|
||||
}
|
||||
var logMu sync.Mutex
|
||||
var handActs []string
|
||||
sub := func(subject string, h func(*nats.Msg)) {
|
||||
if _, err := nc.Subscribe(subject, h); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
sub("mesh.seat.mesh-hand-acts.call.>", func(m *nats.Msg) {
|
||||
logMu.Lock()
|
||||
handActs = append(handActs, string(m.Data))
|
||||
logMu.Unlock()
|
||||
answer(m, map[string]any{"recorded": true})
|
||||
})
|
||||
sub("mesh.seat.mesh-module-assigner.call.assignments.*", func(m *nats.Msg) {
|
||||
answer(m, map[string]any{"seats": []any{map[string]any{"seat": meshbroker.Seat, "holders": []any{map[string]any{"module": "nats"}}}}})
|
||||
})
|
||||
sub("mesh.seat.mesh-catalog.call.entry.*", func(m *nats.Msg) {
|
||||
answer(m, map[string]any{"entry": map[string]any{"module": "app", "fingerprint": "src2:app1",
|
||||
"manifest": map[string]any{"module": "app", "version": "1", "state": []any{map[string]any{"name": "notes"}}}}})
|
||||
})
|
||||
keeper := &standKeeper{}
|
||||
restored := func(request, act, outcome string) {
|
||||
b, _ := json.Marshal(meshbroker.SnapshotRestoredEvent{Request: request, Act: act, Outcome: outcome, Error: "cancelled"})
|
||||
if _, err := js.Publish(ctx, meshbroker.SubjectSnapshotRestored(request, act), b); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
sub("mesh.seat.mesh-broker.call.snapshot-get.*", func(m *nats.Msg) {
|
||||
var in meshbroker.SnapshotGetInput
|
||||
_ = json.Unmarshal(m.Data, &in)
|
||||
keeper.mu.Lock()
|
||||
keeper.asked = append(keeper.asked, "get "+in.Request)
|
||||
file, protected := keeper.file, keeper.protected
|
||||
keeper.mu.Unlock()
|
||||
if protected {
|
||||
answer(m, meshbroker.SnapshotGetAnswer{Request: in.Request, Error: "snapshot protected"})
|
||||
return
|
||||
}
|
||||
sum := sha256.Sum256(file)
|
||||
answer(m, meshbroker.SnapshotGetAnswer{Request: in.Request, Module: "app", Local: "notes", Stream: "KV_app_notes",
|
||||
Size: int64(len(file)), Digest: "sha256:" + hex.EncodeToString(sum[:]), Chunks: 3})
|
||||
// Three chunks, each sent once the one before was acknowledged (§681).
|
||||
go func() {
|
||||
send := buscalls.SeatChunks{Seat: meshbroker.Seat, Verb: meshbroker.VerbSnapshotGet, Call: in.Call}
|
||||
third := (len(file) + 2) / 3
|
||||
for n := int64(1); n <= 3; n++ {
|
||||
acks, _ := nc.SubscribeSync(send.Ack(n))
|
||||
part := file[min(int(n-1)*third, len(file)):min(int(n)*third, len(file))]
|
||||
body, _ := json.Marshal(buscalls.SeatChunk{N: n, Of: 3, Data: part})
|
||||
_ = nc.Publish(send.Chunk(n), body)
|
||||
ack, err := acks.NextMsg(10 * time.Second)
|
||||
_ = acks.Unsubscribe()
|
||||
if err != nil {
|
||||
t.Errorf("chunk %d not acknowledged: %v", n, err)
|
||||
return
|
||||
}
|
||||
_ = ack.Respond([]byte("{}"))
|
||||
}
|
||||
}()
|
||||
})
|
||||
sub("mesh.seat.mesh-broker.call.snapshot-restore.*", func(m *nats.Msg) {
|
||||
var in meshbroker.SnapshotRestoreInput
|
||||
_ = json.Unmarshal(m.Data, &in)
|
||||
keeper.mu.Lock()
|
||||
if in.Verify {
|
||||
keeper.asked = append(keeper.asked, "verify "+in.Request)
|
||||
keeper.mu.Unlock()
|
||||
answer(m, meshbroker.SnapshotRestoreAnswer{Accepted: "keeper-verify"})
|
||||
return
|
||||
}
|
||||
keeper.asked = append(keeper.asked, "restore "+in.Request)
|
||||
then := keeper.onRestore
|
||||
keeper.mu.Unlock()
|
||||
answer(m, meshbroker.SnapshotRestoreAnswer{Accepted: "keeper-" + in.Request})
|
||||
if then != "" {
|
||||
restored(in.Request, "keeper-"+in.Request, then)
|
||||
}
|
||||
})
|
||||
sub("mesh.seat.mesh-broker.call.snapshot-restore-cancel.*", func(m *nats.Msg) {
|
||||
var in meshbroker.SnapshotRestoreCancelInput
|
||||
_ = json.Unmarshal(m.Data, &in)
|
||||
keeper.mu.Lock()
|
||||
keeper.asked = append(keeper.asked, "cancel "+in.Act)
|
||||
then := keeper.onCancel
|
||||
keeper.mu.Unlock()
|
||||
answer(m, meshbroker.SnapshotRestoreCancelAnswer{Accepted: in.Act})
|
||||
if then != "" {
|
||||
restored(strings.TrimPrefix(in.Act, "keeper-"), in.Act, then)
|
||||
}
|
||||
})
|
||||
|
||||
// Every event the module says, heard as it is said.
|
||||
type heard struct {
|
||||
subject string
|
||||
body map[string]any
|
||||
}
|
||||
events := make(chan heard, 256)
|
||||
sub("mesh.seat.mesh-recovery.event.>", func(m *nats.Msg) {
|
||||
var b map[string]any
|
||||
_ = json.Unmarshal(m.Data, &b)
|
||||
events <- heard{m.Subject, b}
|
||||
})
|
||||
var kept []heard
|
||||
waitEvent := func(t *testing.T, what string, ok func(heard) bool) map[string]any {
|
||||
t.Helper()
|
||||
for i, h := range kept {
|
||||
if ok(h) {
|
||||
kept = append(kept[:i:i], kept[i+1:]...)
|
||||
return h.body
|
||||
}
|
||||
}
|
||||
deadline := time.After(30 * time.Second)
|
||||
for {
|
||||
select {
|
||||
case h := <-events:
|
||||
if ok(h) {
|
||||
return h.body
|
||||
}
|
||||
kept = append(kept, h)
|
||||
case <-deadline:
|
||||
t.Fatalf("not said within 30s: %s", what)
|
||||
}
|
||||
}
|
||||
}
|
||||
endOf := func(t *testing.T, event, act string) map[string]any {
|
||||
t.Helper()
|
||||
return waitEvent(t, event+" of "+act, func(h heard) bool {
|
||||
return strings.HasPrefix(h.subject, "mesh.seat.mesh-recovery.event."+event+".") && h.body["act"] == act
|
||||
})
|
||||
}
|
||||
ask := func(t *testing.T, verb, caller string, body any) seat.SnapshotKeptAnswer {
|
||||
t.Helper()
|
||||
data, _ := json.Marshal(body)
|
||||
m, err := nc.Request("mesh.seat.mesh-recovery.call."+verb+"."+strings.ReplaceAll(caller, ".", "~"), data, 10*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", verb, err)
|
||||
}
|
||||
var r struct {
|
||||
Result seat.SnapshotKeptAnswer `json:"result"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Data, &r)
|
||||
if r.Error != "" {
|
||||
r.Result.Error = r.Error
|
||||
}
|
||||
return r.Result
|
||||
}
|
||||
// A caller told busy asks again when it hears the ending event of the act it was named (§554, §560): the expiry runs
|
||||
// on its own bound beside the test's acts.
|
||||
act := func(t *testing.T, verb, caller string, body any) string {
|
||||
t.Helper()
|
||||
for {
|
||||
a := ask(t, verb, caller, body)
|
||||
if a.Busy == "" {
|
||||
if a.Accepted == "" {
|
||||
t.Fatalf("%s answered %+v", verb, a)
|
||||
}
|
||||
return a.Accepted
|
||||
}
|
||||
waitEvent(t, "the ending of "+a.Busy, func(h heard) bool {
|
||||
return h.body["act"] == a.Busy && !strings.Contains(h.subject, ".restore-started.")
|
||||
})
|
||||
}
|
||||
}
|
||||
const engine, runner = "srv.node-engine", "srv.tool-runner"
|
||||
|
||||
// The process, every bound a setting in seconds (§557, A-138): the dump wait and the restore wait 1.8 s, the expiry
|
||||
// every 1.8 s, a point kept about 4 s.
|
||||
dir := t.TempDir()
|
||||
setting := func(name, v string) string {
|
||||
p := filepath.Join(dir, name)
|
||||
_ = os.WriteFile(p, []byte(v+"\n"), 0o600)
|
||||
return p
|
||||
}
|
||||
acctFile, _ := json.Marshal(map[string]any{"url": url, "node": acct.Node, "module": acct.Module, "user": acct.User, "password": acct.Password})
|
||||
env := map[string]string{
|
||||
process.BrokerFileEnv: setting("broker", string(acctFile)),
|
||||
envPoints: filepath.Join(dir, "points"),
|
||||
envDumpWait: setting("dump-wait-minutes", "0.03"),
|
||||
envRestore: setting("snapshot-restore-wait-minutes", "0.03"),
|
||||
envExpire: setting("expire-every-hours", "0.0005"),
|
||||
envKeepDays: setting("point-keep-days", "0.00005"),
|
||||
}
|
||||
runCtx, stop := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- run(runCtx, func(k string) string { return env[k] }, testLog{t}) }()
|
||||
defer func() { stop(); <-done }()
|
||||
waitEvent(t, "the start's removal of points past their keep days", func(h heard) bool {
|
||||
return h.subject == "mesh.seat.mesh-recovery.event."+seat.EventExpiryEnded
|
||||
})
|
||||
|
||||
var first string
|
||||
t.Run("a hand take's log names the account the bus proved and the asker (§554)", func(t *testing.T) {
|
||||
ev := endOf(t, seat.EventEnded, act(t, seat.VerbTake, runner, seat.TakeInput{Module: "app", Kept: "notes", Fingerprint: "src2:app1", Why: "before a test", Asker: "op@srv"}))
|
||||
if ev["outcome"] != "done" {
|
||||
t.Fatalf("ended %v", ev)
|
||||
}
|
||||
first, _ = ev["snapshot"].(string)
|
||||
logMu.Lock()
|
||||
defer logMu.Unlock()
|
||||
if len(handActs) == 0 || !strings.Contains(handActs[len(handActs)-1], runner) || !strings.Contains(handActs[len(handActs)-1], "op@srv") {
|
||||
t.Errorf("the hand-act log holds %v", handActs)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a restore waits on the dump past its bound key, a second ask is busy and asks again on its ended (§554, §563, A-143)", func(t *testing.T) {
|
||||
if a := ask(t, seat.VerbRestoreKept, engine, map[string]any{"module": "app", "kept": "notes", "snapshot": first, "why": "w", "aside-only": true}); !strings.Contains(a.Error, "aside-only") {
|
||||
t.Errorf("a node-engine's restore-kept with aside-only answered %+v", a)
|
||||
}
|
||||
item := func(event string) {
|
||||
b, _ := json.Marshal(map[string]any{"at": time.Now()})
|
||||
if _, err := js.Publish(ctx, itemSubject(event, "srv", "nats", "data"), b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
item(itemStarted)
|
||||
_, _ = notes.Put(ctx, "n", []byte("changed"))
|
||||
r := seat.SnapshotKeptAnswer{Accepted: act(t, seat.VerbRestoreKept, engine, seat.RestoreKeptInput{Module: "app", Kept: "notes", Form: 2, Fingerprint: "src2:app1", Snapshot: first, Why: "a form change"})}
|
||||
if b := ask(t, seat.VerbSnapshotKept, engine, seat.SnapshotKeptInput{Module: "app", Kept: "notes", Form: 2, Fingerprint: "src2:app1", Why: "x"}); b.Busy != r.Accepted {
|
||||
t.Errorf("a second ask answered %+v, want busy with %s", b, r.Accepted)
|
||||
}
|
||||
findings, _ := js.KeyValue(ctx, "mesh-recovery_findings")
|
||||
w, _ := findings.Watch(ctx, "app."+r.Accepted+"."+recovery.FindingRecoveryWaits)
|
||||
for e := range w.Updates() {
|
||||
if e != nil && e.Operation() == jetstream.KeyValuePut {
|
||||
break
|
||||
}
|
||||
}
|
||||
_ = w.Stop()
|
||||
item(itemEnded)
|
||||
if ev := endOf(t, seat.EventRestored, r.Accepted); ev["outcome"] != "done" || live() != "v1" {
|
||||
t.Fatalf("restored %v, live %q", ev, live())
|
||||
}
|
||||
if e, err := findings.Get(ctx, "app."+r.Accepted+"."+recovery.FindingRecoveryWaits); err == nil {
|
||||
t.Errorf("recovery-waits stands after the act ended: %s", e.Value())
|
||||
}
|
||||
again := act(t, seat.VerbSnapshotKept, engine, seat.SnapshotKeptInput{Module: "app", Kept: "notes", Form: 2, Fingerprint: "src2:app1", Why: "asked again"})
|
||||
if ev := endOf(t, seat.EventEnded, again); ev["outcome"] != "done" {
|
||||
t.Errorf("the ask again ended %v", ev)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a point past its keep days is removed on the expiry's bound key, never the newest (§551, A-138)", func(t *testing.T) {
|
||||
ev := waitEvent(t, "point-removed of "+first, func(h heard) bool {
|
||||
return strings.HasPrefix(h.subject, "mesh.seat.mesh-recovery.event."+seat.EventPointRemoved+".") && h.body["snapshot"] == first
|
||||
})
|
||||
if ev["module"] != "app" {
|
||||
t.Errorf("point-removed %v", ev)
|
||||
}
|
||||
var kp seat.KeptPointsAnswer
|
||||
m, _ := nc.Request("mesh.seat.mesh-recovery.call.kept-points."+strings.ReplaceAll(runner, ".", "~"), []byte(`{"module":"app","kept":"notes"}`), 10*time.Second)
|
||||
var r struct {
|
||||
Result seat.KeptPointsAnswer `json:"result"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Data, &r)
|
||||
kp = r.Result
|
||||
if len(kp.Points) != 1 {
|
||||
t.Errorf("the newest point is not kept alone: %+v", kp.Points)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a keeper's snapshot of three chunks is received and restored whole (§681)", func(t *testing.T) {
|
||||
_, _ = notes.Put(ctx, "n", []byte("from the keeper"))
|
||||
var snap bytes.Buffer
|
||||
if _, err := bussnap.Snapshot(ctx, nc, &snap, bussnap.SnapshotOptions{Streams: []string{"KV_app_notes"}, TempDir: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
keeper.mu.Lock()
|
||||
keeper.file = snap.Bytes()
|
||||
keeper.mu.Unlock()
|
||||
keeper.set(false, "", "")
|
||||
_, _ = notes.Put(ctx, "n", []byte("since the drop"))
|
||||
a := act(t, seat.VerbRestore, runner, seat.RestoreInput{Module: "app", Kept: "notes", Request: "req-1", Why: "test the drop's snapshot", Asker: "op@srv"})
|
||||
if ev := endOf(t, seat.EventRestored, a); ev["outcome"] != "done" || live() != "from the keeper" {
|
||||
t.Errorf("restored %v, live %q", ev, live())
|
||||
}
|
||||
if left, _ := os.ReadDir(filepath.Join(dir, "points", "incoming")); len(left) != 0 {
|
||||
t.Errorf("incoming holds %v", left)
|
||||
}
|
||||
})
|
||||
|
||||
inPlace := func(t *testing.T, request string) string {
|
||||
t.Helper()
|
||||
_, _ = notes.Put(ctx, "n", []byte("live before "+request))
|
||||
return act(t, seat.VerbRestore, runner, seat.RestoreInput{Module: "app", Kept: "notes", Request: request, Why: "restore in place", Asker: "op@srv"})
|
||||
}
|
||||
t.Run("a protected snapshot's bound asks the cancel first, then puts the aside back on its failed (§681, A-140, A-143)", func(t *testing.T) {
|
||||
keeper.set(true, "", meshbroker.SnapshotRestoredEventOutcomeFailed)
|
||||
id := inPlace(t, "req-2")
|
||||
if ev := endOf(t, seat.EventRestored, id); ev["outcome"] != "failed" || live() != "live before req-2" {
|
||||
t.Errorf("restored %v, live %q", ev, live())
|
||||
}
|
||||
if got := keeper.said(); got != "get req-2|verify req-2|restore req-2|cancel keeper-req-2" {
|
||||
t.Errorf("the keeper was asked %s", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a keeper that answers neither leaves the aside unrestored, which restore-aside puts back (§681, §554)", func(t *testing.T) {
|
||||
keeper.set(true, "", "")
|
||||
id := inPlace(t, "req-3")
|
||||
ev := endOf(t, seat.EventRestored, id)
|
||||
if ev["outcome"] != "failed" || ev["error"] != "snapshot-restore unanswered" {
|
||||
t.Fatalf("restored %v", ev)
|
||||
}
|
||||
var r struct {
|
||||
Result seat.KeptPointsAnswer `json:"result"`
|
||||
}
|
||||
m, _ := nc.Request("mesh.seat.mesh-recovery.call.kept-points."+strings.ReplaceAll(runner, ".", "~"), []byte(`{}`), 10*time.Second)
|
||||
_ = json.Unmarshal(m.Data, &r)
|
||||
if len(r.Result.Unrestored) != 1 {
|
||||
t.Fatalf("unrestored %+v", r.Result.Unrestored)
|
||||
}
|
||||
back := act(t, seat.VerbRestoreAside, runner, seat.RestoreAsideInput{Module: "app", Kept: "notes", ID: r.Result.Unrestored[0].ID, Why: "put it back", Asker: "op@srv"})
|
||||
if ev := endOf(t, seat.EventRestored, back); ev["outcome"] != "done" || live() != "live before req-3" {
|
||||
t.Errorf("restore-aside %v, live %q", ev, live())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a stop during a restore past its removal puts the aside back and ends it stopped (§562, A-141)", func(t *testing.T) {
|
||||
keeper.set(true, "", meshbroker.SnapshotRestoredEventOutcomeFailed)
|
||||
id := inPlace(t, "req-4")
|
||||
// The wait on snapshot-restored is a bound key: once it stands, the live data is removed and the keeper asked.
|
||||
bounds, _ := js.KeyValue(ctx, "mesh-recovery_bounds")
|
||||
w, _ := bounds.Watch(ctx, "restore-wait."+id)
|
||||
for e := range w.Updates() {
|
||||
if e != nil && e.Operation() == jetstream.KeyValuePut {
|
||||
break
|
||||
}
|
||||
}
|
||||
_ = w.Stop()
|
||||
stop()
|
||||
ev := endOf(t, seat.EventRestored, id)
|
||||
if err := <-done; err != nil {
|
||||
t.Errorf("the process ended %v", err)
|
||||
}
|
||||
done <- nil
|
||||
if ev["outcome"] != "failed" || ev["error"] != "stopped" || live() != "live before req-4" {
|
||||
t.Errorf("restored %v, live %q", ev, live())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
type testLog struct{ t *testing.T }
|
||||
|
||||
func (l testLog) Write(p []byte) (int, error) {
|
||||
l.t.Log(strings.TrimRight(string(p), "\n"))
|
||||
return len(p), nil
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
${config:dump-wait-minutes}
|
||||
@@ -0,0 +1 @@
|
||||
${config:expire-every-hours}
|
||||
@@ -0,0 +1 @@
|
||||
${config:point-keep-days}
|
||||
@@ -0,0 +1 @@
|
||||
${config:points-cap}
|
||||
@@ -0,0 +1 @@
|
||||
${config:points-filling-mark}
|
||||
@@ -0,0 +1 @@
|
||||
${config:snapshot-restore-wait-minutes}
|
||||
@@ -0,0 +1,28 @@
|
||||
module mesh-recovery
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-contracts/go/seats/meshbroker v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/go/seats/meshrecovery v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.1 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.2 // indirect
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.16.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,25 @@
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc=
|
||||
github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
|
||||
golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
|
||||
@@ -0,0 +1,189 @@
|
||||
package recovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
|
||||
)
|
||||
|
||||
// Cancel is the verb `cancel` (§566, §682): it ends an accepted act that has not begun its work (one waiting on the
|
||||
// dump, §563) with its ending event, outcome failed and error cancelled, the hand act's words as `why` (A-117); it
|
||||
// refuses an act that has begun, naming its step, since a restore cut off after its removal is put back only by its own
|
||||
// failure path (§516). It is written in the hand-act log first and answers in §691's one form.
|
||||
func (e *Engine) Cancel(act, why, caller, asker string) Answer {
|
||||
if stop := e.Stopping(); stop != "" {
|
||||
return Answer{Error: stop}
|
||||
}
|
||||
if err := e.Seats.HandAct(caller, asker, "cancel "+act, why); err != nil {
|
||||
return Answer{Error: "the hand-act log was not written, so nothing was done: " + err.Error()}
|
||||
}
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
switch {
|
||||
case e.running == nil || e.running.ID != act:
|
||||
return Answer{Error: "the act " + act + " is not running"}
|
||||
case e.running.Step != "waiting":
|
||||
return Answer{Error: fmt.Sprintf("the act %s has begun (its step: %q) and is put back only by its own failure path", act, e.running.Step)}
|
||||
}
|
||||
e.why = why
|
||||
e.cancel()
|
||||
return Answer{Accepted: act}
|
||||
}
|
||||
|
||||
// Retry is the verb `retry` (§682): it asks again the last failed act of a module and kept data, with its fields, under
|
||||
// an act of its own; `nothing to retry` when its last act did not fail.
|
||||
func (e *Engine) Retry(module, kept, why, caller, asker string) Answer {
|
||||
acts := e.readActs()
|
||||
var last *Act
|
||||
for i := range acts {
|
||||
a := acts[i]
|
||||
if a.Module == module && a.Kept == kept && a.Ended && a.Kind != KindExpire && (last == nil || !a.Since.Before(last.Since)) {
|
||||
last = &a
|
||||
}
|
||||
}
|
||||
if last == nil || last.Error == "" {
|
||||
return Answer{Error: "nothing to retry"}
|
||||
}
|
||||
again := *last
|
||||
again.ID, again.Ended, again.Error, again.Step, again.WaitsOn = "", false, "", "", ""
|
||||
again.Why, again.Caller, again.Asker = why, caller, asker
|
||||
return e.Accept(again)
|
||||
}
|
||||
|
||||
func (e *Engine) readActs() []Act {
|
||||
var out []Act
|
||||
files, _ := filepath.Glob(filepath.Join(e.Dir, "acts", "*.json"))
|
||||
for _, f := range files {
|
||||
var a Act
|
||||
if readJSON(f, &a) == nil {
|
||||
out = append(out, a)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// KeptPoints is the verb `kept-points` (§517): every restore point and aside with their sizes, narrowed to a module and a
|
||||
// kept data, and the act that runs, with since when and what it waits on.
|
||||
func (e *Engine) KeptPoints(module, kept string) (seat.KeptPointsAnswer, error) {
|
||||
pts, err := e.Points()
|
||||
if err != nil {
|
||||
return seat.KeptPointsAnswer{}, err
|
||||
}
|
||||
out := seat.KeptPointsAnswer{Points: []seat.Point{}, Asides: []seat.Aside{}}
|
||||
for _, p := range pts {
|
||||
if (module == "" || p.Module == module) && (kept == "" || p.Kept == kept) {
|
||||
out.Points = append(out.Points, p)
|
||||
}
|
||||
}
|
||||
_, recs, _ := e.Asides()
|
||||
for _, r := range recs {
|
||||
if (module == "" || r.Module == module) && (kept == "" || r.Kept == kept) {
|
||||
out.Asides = append(out.Asides, r.aside())
|
||||
}
|
||||
}
|
||||
e.mu.Lock()
|
||||
if e.running != nil {
|
||||
out.Running = &seat.Running{Act: e.running.ID, Step: e.running.Step, Since: e.running.Since, WaitsOn: e.running.WaitsOn}
|
||||
}
|
||||
e.mu.Unlock()
|
||||
// The asides only an aside holds, which the health check fails on, read from the engine (A-129).
|
||||
out.Unrestored = []seat.Aside{}
|
||||
for _, x := range e.Unrestored() {
|
||||
if (module == "" || x.Module == module) && (kept == "" || x.Kept == kept) {
|
||||
out.Unrestored = append(out.Unrestored, x.Aside)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Unrestored are the asides whose record ends at `removed` that no running act holds (§554 health, §561): the live data
|
||||
// of those names is gone and only the aside holds it.
|
||||
func (e *Engine) Unrestored() []UnrestoredAside {
|
||||
_, recs, _ := e.Asides()
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
var out []UnrestoredAside
|
||||
for _, r := range recs {
|
||||
held := e.running != nil && e.running.Module == r.Module && e.running.Kept == r.Kept
|
||||
if r.Step == stepRemoved && !held {
|
||||
out = append(out, UnrestoredAside{Module: r.Module, Kept: r.Kept, Aside: r.aside()})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// UnrestoredAside is an aside only it holds the data of, with the module and kept data it was taken of: the scope of the
|
||||
// finding aside-unrestored (§561).
|
||||
type UnrestoredAside struct {
|
||||
Module, Kept string
|
||||
Aside seat.Aside
|
||||
}
|
||||
|
||||
// Start repairs what a restart cut off (§516, §564, §560): points under a temporary name are removed, an aside whose
|
||||
// record ends at `set aside` (the live data never touched) is removed, an aside that ends at `removed` is put back under
|
||||
// the live names, and every accepted act that never ended ends failed with `stopped`, so its caller asks again. It
|
||||
// answers the asides it could not put back (the finding aside-unrestored, §561).
|
||||
func (e *Engine) Start(ctx context.Context) []UnrestoredAside {
|
||||
_ = filepath.WalkDir(filepath.Join(e.Dir, "points"), func(path string, d os.DirEntry, err error) error {
|
||||
if err == nil && d.IsDir() && strings.HasPrefix(d.Name(), ".tmp-") {
|
||||
_ = os.RemoveAll(path)
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
})
|
||||
// A keeper's snapshot a cut-off act left half or whole received is deleted (§681).
|
||||
_ = os.RemoveAll(e.incoming())
|
||||
_, recs, _ := e.Asides()
|
||||
for i := range recs {
|
||||
r := &recs[i]
|
||||
switch r.Step {
|
||||
case stepSetAside:
|
||||
_ = os.RemoveAll(e.asideDirOf(r))
|
||||
case stepRemoved:
|
||||
if live, err := e.Bus.Objects(ctx, r.Streams); err == nil && !anyExists(live) {
|
||||
if err := e.restoreFromAside(ctx, r, false); err != nil {
|
||||
e.logf("the aside %s is not put back: %v", r.ID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
e.endUnended()
|
||||
e.judge()
|
||||
return e.Unrestored()
|
||||
}
|
||||
|
||||
// endUnended ends every accepted act that never ended failed with `stopped` and its ending event (§554, §562), oldest first.
|
||||
func (e *Engine) endUnended() {
|
||||
acts := e.readActs()
|
||||
sort.Slice(acts, func(i, j int) bool { return acts[i].Since.Before(acts[j].Since) })
|
||||
for i := range acts {
|
||||
a := acts[i]
|
||||
if a.Ended {
|
||||
continue
|
||||
}
|
||||
stopped := errors.New(ErrStopped)
|
||||
switch a.Kind {
|
||||
case KindTake:
|
||||
_ = e.endedFailed(&a, stopped)
|
||||
case KindRestore, KindRestoreAside:
|
||||
_ = e.endedRestored(&a, stopped, nil, false)
|
||||
}
|
||||
a.Error = ErrStopped
|
||||
e.markEnded(&a)
|
||||
}
|
||||
}
|
||||
|
||||
func anyExists(os []Object) bool {
|
||||
for _, o := range os {
|
||||
if o.Exists {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,302 @@
|
||||
package recovery_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
|
||||
"mesh-recovery/internal/recovery"
|
||||
)
|
||||
|
||||
// fakeBus holds streams as strings; a snapshot file is "snap:<name>:<content>".
|
||||
type fakeBus struct {
|
||||
mu sync.Mutex
|
||||
streams map[string]string
|
||||
consumers map[string][]seat.Consumer
|
||||
setBack []seat.Consumer
|
||||
failRestores int
|
||||
}
|
||||
|
||||
func (b *fakeBus) Objects(_ context.Context, names []string) ([]recovery.Object, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
var out []recovery.Object
|
||||
for _, n := range names {
|
||||
c, ok := b.streams[n]
|
||||
out = append(out, recovery.Object{Stream: n, Exists: ok, Bytes: int64(len(c)), Consumers: b.consumers[n]})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (b *fakeBus) Take(_ context.Context, s string, w io.Writer) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
c, ok := b.streams[s]
|
||||
if !ok {
|
||||
return errors.New("no stream " + s)
|
||||
}
|
||||
_, err := io.WriteString(w, "snap:"+s+":"+c)
|
||||
return err
|
||||
}
|
||||
|
||||
func (b *fakeBus) Delete(_ context.Context, s string) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
delete(b.streams, s)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *fakeBus) Restore(_ context.Context, file string) error {
|
||||
raw, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.failRestores > 0 {
|
||||
b.failRestores--
|
||||
return errors.New("the server refused the archive")
|
||||
}
|
||||
var name, content string
|
||||
rest := string(raw[5:])
|
||||
for i := range rest {
|
||||
if rest[i] == ':' {
|
||||
name, content = rest[:i], rest[i+1:]
|
||||
break
|
||||
}
|
||||
}
|
||||
if _, live := b.streams[name]; live {
|
||||
return errors.New("stream name already in use")
|
||||
}
|
||||
b.streams[name] = content
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *fakeBus) SetConsumers(_ context.Context, cs []seat.Consumer) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.setBack = append(b.setBack, cs...)
|
||||
return nil
|
||||
}
|
||||
|
||||
type said struct {
|
||||
Event string
|
||||
Body any
|
||||
Parts []string
|
||||
}
|
||||
|
||||
type fakeSeats struct {
|
||||
mu sync.Mutex
|
||||
events []said
|
||||
handActs []string
|
||||
handFail error
|
||||
streams []string
|
||||
backup string
|
||||
sayFail map[string]error
|
||||
}
|
||||
|
||||
func (s *fakeSeats) Say(event string, body any, parts ...string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if err := s.sayFail[event]; err != nil {
|
||||
return err
|
||||
}
|
||||
s.events = append(s.events, said{event, body, parts})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *fakeSeats) HandAct(caller, asker, what, why string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.handFail != nil {
|
||||
return s.handFail
|
||||
}
|
||||
s.handActs = append(s.handActs, caller+"|"+asker+"|"+what+"|"+why)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *fakeSeats) Streams(context.Context, string, string, string) ([]string, error) {
|
||||
return s.streams, nil
|
||||
}
|
||||
|
||||
func (s *fakeSeats) RunningFingerprint(context.Context, string) (string, error) { return "fp-run", nil }
|
||||
|
||||
func (s *fakeSeats) Backup(context.Context, string) (string, error) {
|
||||
if s.backup == "" {
|
||||
return "", errors.New("no repository snapshot holds it")
|
||||
}
|
||||
return s.backup, nil
|
||||
}
|
||||
|
||||
func (s *fakeSeats) named(event string) []said {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var out []said
|
||||
for _, e := range s.events {
|
||||
if e.Event == event {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type memDigests struct {
|
||||
mu sync.Mutex
|
||||
m map[string][]byte
|
||||
}
|
||||
|
||||
func (d *memDigests) Put(k string, v []byte) error {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
d.m[k] = v
|
||||
return nil
|
||||
}
|
||||
func (d *memDigests) Get(k string) ([]byte, bool, error) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
v, ok := d.m[k]
|
||||
return v, ok, nil
|
||||
}
|
||||
func (d *memDigests) Delete(k string) error {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
delete(d.m, k)
|
||||
return nil
|
||||
}
|
||||
|
||||
type noDump struct{}
|
||||
|
||||
func (noDump) Wait(context.Context, func(string)) error { return nil }
|
||||
|
||||
// holdDump waits until ctx ends, saying it waits.
|
||||
type holdDump struct{ waiting chan string }
|
||||
|
||||
func (h holdDump) Wait(ctx context.Context, waiting func(string)) error {
|
||||
waiting("the dump of the bus")
|
||||
h.waiting <- "waiting"
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
type rig struct {
|
||||
e *recovery.Engine
|
||||
b *fakeBus
|
||||
s *fakeSeats
|
||||
d *memDigests
|
||||
t time.Time
|
||||
// bounds is the state bounds the act's waits are timed by (A-143).
|
||||
bounds *fakeBounds
|
||||
}
|
||||
|
||||
func newRig(t *testing.T) *rig {
|
||||
t.Helper()
|
||||
r := &rig{bounds: &fakeBounds{}, b: &fakeBus{streams: map[string]string{"app_notes": "hello", "KV_app_seen": "world"}, consumers: map[string][]seat.Consumer{}},
|
||||
s: &fakeSeats{streams: []string{"app_notes", "KV_app_seen"}}, d: &memDigests{m: map[string][]byte{}}, t: time.Date(2026, 11, 12, 1, 0, 0, 0, time.UTC)}
|
||||
r.b.consumers["app_notes"] = []seat.Consumer{{Name: "c1", Stream: "app_notes", Sequence: 7}}
|
||||
r.e = &recovery.Engine{Dir: filepath.Join(t.TempDir(), "points-dir"), Bus: r.b, Seats: r.s, Digests: r.d, Dump: noDump{}, Bounds: r.bounds,
|
||||
Settings: recovery.Settings{Cap: 1 << 20, KeepDays: 90}, Now: func() time.Time { return r.t }}
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *rig) do(a recovery.Act) recovery.Answer {
|
||||
ans := r.e.Accept(a)
|
||||
r.e.Wait()
|
||||
return ans
|
||||
}
|
||||
|
||||
var errTest = errors.New("the log is unreachable")
|
||||
|
||||
type gatedBus struct {
|
||||
*fakeBus
|
||||
gate chan struct{}
|
||||
}
|
||||
|
||||
func (g *gatedBus) Objects(ctx context.Context, n []string) ([]recovery.Object, error) {
|
||||
<-g.gate
|
||||
return g.fakeBus.Objects(ctx, n)
|
||||
}
|
||||
|
||||
// heldFindings is the module's finding writer as it holds them; written says each write.
|
||||
type heldFindings struct {
|
||||
mu sync.Mutex
|
||||
held map[string]meshconditions.Finding
|
||||
written chan meshconditions.Finding
|
||||
}
|
||||
|
||||
func newHeldFindings() *heldFindings {
|
||||
return &heldFindings{held: map[string]meshconditions.Finding{}, written: make(chan meshconditions.Finding, 64)}
|
||||
}
|
||||
|
||||
func (h *heldFindings) Write(_ context.Context, f meshconditions.Finding) error {
|
||||
h.mu.Lock()
|
||||
h.held[f.Scope+"."+f.ID+"."+f.Kind] = f
|
||||
h.mu.Unlock()
|
||||
h.written <- f
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *heldFindings) Clear(_ context.Context, scope, id, kind, _ string) error {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
delete(h.held, scope+"."+id+"."+kind)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *heldFindings) Held() []meshconditions.Finding {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
var out []meshconditions.Finding
|
||||
for _, f := range h.held {
|
||||
out = append(out, f)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (h *heldFindings) has(kind string) bool {
|
||||
for _, f := range h.Held() {
|
||||
if f.Kind == kind {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// fakeBounds is the state bounds: each key removed by the bus when its time to live passes, as the bus does, or by its
|
||||
// end; written and removed say which keys, with each key's time to live (A-143).
|
||||
type fakeBounds struct {
|
||||
mu sync.Mutex
|
||||
written []string
|
||||
removed []string
|
||||
}
|
||||
|
||||
func (f *fakeBounds) Bound(key string, ttl time.Duration, expired func()) (func(), error) {
|
||||
f.mu.Lock()
|
||||
f.written = append(f.written, key+" "+ttl.String())
|
||||
f.mu.Unlock()
|
||||
var once sync.Once
|
||||
gone := func(by string, then func()) {
|
||||
once.Do(func() {
|
||||
f.mu.Lock()
|
||||
f.removed = append(f.removed, key+" "+by)
|
||||
f.mu.Unlock()
|
||||
then()
|
||||
})
|
||||
}
|
||||
t := time.AfterFunc(ttl, func() { gone("expired", expired) })
|
||||
return func() { t.Stop(); gone("ended", func() {}) }, nil
|
||||
}
|
||||
|
||||
func (f *fakeBounds) keys() (written, removed string) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return strings.Join(f.written, "|"), strings.Join(f.removed, "|")
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
package recovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
)
|
||||
|
||||
// The recovery module's findings (design 47 G §561), each written by mesh-sdk's finding writer as the key
|
||||
// `<scope>.<id>.<kind>` and said on the module's finding stream.
|
||||
const (
|
||||
FindingPointsFilling = "points-filling"
|
||||
FindingRecoveryWaits = "recovery-waits"
|
||||
FindingAsideUnrestored = "aside-unrestored"
|
||||
)
|
||||
|
||||
// FindingKinds are the kinds the module's writer owns.
|
||||
var FindingKinds = []string{FindingPointsFilling, FindingRecoveryWaits, FindingAsideUnrestored}
|
||||
|
||||
// Findings is the module's finding writer (mesh-sdk's held writer): what it writes and clears, and what it holds.
|
||||
type Findings interface {
|
||||
Write(ctx context.Context, f meshconditions.Finding) error
|
||||
Clear(ctx context.Context, scope, id, kind, why string) error
|
||||
Held() []meshconditions.Finding
|
||||
}
|
||||
|
||||
// The defaults of the settings §557 and §564 make bounds of.
|
||||
const (
|
||||
DefaultFillingMark = 80
|
||||
DefaultDumpWait = 60 * time.Minute
|
||||
)
|
||||
|
||||
func (e *Engine) mark() int64 {
|
||||
if e.Settings.FillingMark > 0 {
|
||||
return int64(e.Settings.FillingMark)
|
||||
}
|
||||
return DefaultFillingMark
|
||||
}
|
||||
|
||||
func (e *Engine) dumpWait() time.Duration {
|
||||
if e.Settings.DumpWait > 0 {
|
||||
return e.Settings.DumpWait
|
||||
}
|
||||
return DefaultDumpWait
|
||||
}
|
||||
|
||||
func (e *Engine) finding(kind, scope, id, headline, summary, needs string) meshconditions.Finding {
|
||||
return meshconditions.Finding{Scope: scope, ID: id, Kind: kind, Subject: meshconditions.Subject{Scope: "module", ID: Module},
|
||||
Severity: meshconditions.FindingSeverityWarning, Headline: headline, Summary: summary, Explanation: summary, Needs: needs,
|
||||
Source: "points", Observer: Module, At: e.now()}
|
||||
}
|
||||
|
||||
func (e *Engine) write(f meshconditions.Finding) {
|
||||
if e.Findings == nil {
|
||||
return
|
||||
}
|
||||
if err := e.Findings.Write(context.Background(), f); err != nil {
|
||||
e.logf("the finding %s %s.%s cannot be written: %v", f.Kind, f.Scope, f.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) clear(scope, id, kind, why string) {
|
||||
if e.Findings == nil {
|
||||
return
|
||||
}
|
||||
if err := e.Findings.Clear(context.Background(), scope, id, kind, why); err != nil {
|
||||
e.logf("the finding %s %s.%s cannot be cleared: %v", kind, scope, id, err)
|
||||
}
|
||||
}
|
||||
|
||||
// judge measures the points directory against the fill mark of its cap (points-filling) and writes aside-unrestored for
|
||||
// each aside whose record ends at removed that no running act holds, clearing those that no longer are (§561). It runs at
|
||||
// the start, after each act and so after each expiry run.
|
||||
func (e *Engine) judge() {
|
||||
if e.Findings == nil {
|
||||
return
|
||||
}
|
||||
fill, limit := e.filled(), e.Settings.Cap*e.mark()/100
|
||||
if fill > limit {
|
||||
e.write(e.finding(FindingPointsFilling, Module, e.Node, "The recovery module's restore points are filling up",
|
||||
fmt.Sprintf("the points directory holds %d of %d bytes, past its fill mark of %d %%; the largest: %s", fill, e.Settings.Cap,
|
||||
e.mark(), strings.Join(e.largest(5), ", ")), "points removed or an aside discarded, or a larger points-cap"))
|
||||
} else {
|
||||
e.clear(Module, e.Node, FindingPointsFilling, fmt.Sprintf("the points directory holds %d bytes, under its fill mark", fill))
|
||||
}
|
||||
open := map[string]bool{}
|
||||
for _, u := range e.Unrestored() {
|
||||
open[u.Module+"\x00"+u.Aside.ID] = true
|
||||
e.write(e.finding(FindingAsideUnrestored, u.Module, u.Aside.ID, "An aside holds data the live bus lacks",
|
||||
fmt.Sprintf("the aside %s of %s's %s (%s) ended at removed and was not restored", u.Aside.ID, u.Module, u.Kept,
|
||||
strings.Join(u.Aside.Names, ", ")), "recovery_restore_aside, or recovery_discard_aside"))
|
||||
}
|
||||
for _, f := range e.Findings.Held() {
|
||||
if f.Kind == FindingAsideUnrestored && !open[f.Scope+"\x00"+f.ID] {
|
||||
e.clear(f.Scope, f.ID, f.Kind, "the aside was restored or discarded")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// largest are the n largest points and asides, each with its size.
|
||||
func (e *Engine) largest(n int) []string {
|
||||
type sized struct {
|
||||
what string
|
||||
size int64
|
||||
}
|
||||
var all []sized
|
||||
pts, _ := e.Points()
|
||||
for _, p := range pts {
|
||||
all = append(all, sized{"point " + p.Snapshot, p.Size})
|
||||
}
|
||||
_, recs, _ := e.Asides()
|
||||
for _, r := range recs {
|
||||
all = append(all, sized{"aside " + r.Module + "/" + r.Kept + "/" + r.ID, r.Size})
|
||||
}
|
||||
sort.SliceStable(all, func(i, j int) bool { return all[i].size > all[j].size })
|
||||
var out []string
|
||||
for i := 0; i < len(all) && i < n; i++ {
|
||||
out = append(out, fmt.Sprintf("%s (%d bytes)", all[i].what, all[i].size))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// waitsOn is the waiting of one act on the dump: past the bound, the key `dump-wait.<act>` of the state bounds expiring
|
||||
// (A-143), it writes recovery-waits, naming the item and since when; done removes the key and clears the finding once the
|
||||
// act starts its work or ends (§561, §563).
|
||||
func (e *Engine) waitsOn(a *Act) (waiting func(on string), done func()) {
|
||||
var end func()
|
||||
written := make(chan bool, 1)
|
||||
written <- false
|
||||
waiting = func(on string) {
|
||||
e.setStep(a, "waiting", on)
|
||||
since := e.now()
|
||||
key := BoundDumpWait + "." + a.ID
|
||||
var err error
|
||||
end, err = e.bound(key, e.dumpWait(), func() {
|
||||
if !<-written {
|
||||
e.write(e.finding(FindingRecoveryWaits, a.Module, a.ID, "A restore waits on the dump of the bus",
|
||||
fmt.Sprintf("the act %s waits on %s since %s", a.ID, on, since.Format(time.RFC3339)), "the dump to end, or recovery_cancel"))
|
||||
}
|
||||
written <- true
|
||||
})
|
||||
if err != nil {
|
||||
e.logf("the bound %s of the act's wait on the dump is not written, so recovery-waits is never written for it: %v", key, err)
|
||||
}
|
||||
}
|
||||
done = func() {
|
||||
if end != nil {
|
||||
end()
|
||||
}
|
||||
if <-written {
|
||||
e.clear(a.Module, a.ID, FindingRecoveryWaits, "the act no longer waits on the dump")
|
||||
}
|
||||
written <- false
|
||||
}
|
||||
return waiting, done
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
package recovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
|
||||
)
|
||||
|
||||
// Keeper is the bus module's keeper as the seat mesh-broker answers it (design 47 G §681): its snapshot taken before a
|
||||
// `drop` or `recreate`, sent by snapshot-get, or restored in place by snapshot-restore when it holds a protected stream.
|
||||
type Keeper interface {
|
||||
// Get asks snapshot-get of the request; when its answer names a file and fits accepts it, it receives the chunks into
|
||||
// w. A refusal of the keeper is the answer's Error, no error of Get.
|
||||
Get(ctx context.Context, request string, fits func(meshbroker.SnapshotGetAnswer) error, w io.Writer) (meshbroker.SnapshotGetAnswer, error)
|
||||
// Restore asks snapshot-restore; with verify the keeper checks the file and restores nothing.
|
||||
Restore(ctx context.Context, request string, verify bool, why string) (meshbroker.SnapshotRestoreAnswer, error)
|
||||
// CancelRestore asks snapshot-restore-cancel of a running restore.
|
||||
CancelRestore(ctx context.Context, act, why string) (meshbroker.SnapshotRestoreCancelAnswer, error)
|
||||
// Restored answers the keeper's snapshot-restored of that request and act once heard (read as §556 has it, so one said
|
||||
// before the wait counts), or ok false when ctx ends first.
|
||||
Restored(ctx context.Context, request, act string) (ev meshbroker.SnapshotRestoredEvent, ok bool)
|
||||
}
|
||||
|
||||
// The keeper's word for a snapshot it never hands out (§681).
|
||||
const snapshotProtected = "snapshot protected"
|
||||
|
||||
// ErrUnanswered is the end of a restore in place the keeper answered neither by its outcome nor by the cancel's (§681).
|
||||
const ErrUnanswered = "snapshot-restore unanswered"
|
||||
|
||||
// DefaultRestoreWait is how long a restore in place is waited on: past the keeper's own 60 minutes (§681, A-138).
|
||||
const DefaultRestoreWait = 90 * time.Minute
|
||||
|
||||
func (e *Engine) restoreWait() time.Duration {
|
||||
if e.Settings.RestoreWait > 0 {
|
||||
return e.Settings.RestoreWait
|
||||
}
|
||||
return DefaultRestoreWait
|
||||
}
|
||||
|
||||
func (e *Engine) incoming() string { return filepath.Join(e.Dir, "incoming") }
|
||||
|
||||
// restoreRequest is the hand verb `restore` with `request`, a keeper's snapshot (§681 Its use, §682): an unprotected one is
|
||||
// read through snapshot-get into incoming/<request>, refused before the first chunk when it would pass the cap, checked
|
||||
// against its digest before any aside, and restored by the module itself as a point is (§516), the file deleted when the
|
||||
// act ends; a protected one is restored in place by the keeper (restoreInPlace).
|
||||
func (e *Engine) restoreRequest(ctx context.Context, a *Act) error {
|
||||
if e.Keeper == nil {
|
||||
return e.endedRestored(a, errors.New("no keeper is reachable, so no snapshot of it is restored"), nil, false)
|
||||
}
|
||||
e.setStep(a, "receiving the keeper's snapshot", "")
|
||||
name, err := token(a.Request)
|
||||
if err != nil {
|
||||
return e.endedRestored(a, err, nil, false)
|
||||
}
|
||||
if err := os.MkdirAll(e.incoming(), 0o700); err != nil {
|
||||
return e.endedRestored(a, err, nil, false)
|
||||
}
|
||||
file := filepath.Join(e.incoming(), name)
|
||||
defer os.Remove(file)
|
||||
f, err := os.Create(file)
|
||||
if err != nil {
|
||||
return e.endedRestored(a, err, nil, false)
|
||||
}
|
||||
ans, err := e.Keeper.Get(ctx, a.Request, func(ans meshbroker.SnapshotGetAnswer) error {
|
||||
if fill := e.filled(); fill+ans.Size > e.Settings.Cap {
|
||||
return fmt.Errorf("%s: the cap is %d bytes, the snapshot of the request %s needs %d and the directory holds %d",
|
||||
ErrCap, e.Settings.Cap, a.Request, ans.Size, fill)
|
||||
}
|
||||
return nil
|
||||
}, f)
|
||||
if cerr := f.Close(); err == nil {
|
||||
err = cerr
|
||||
}
|
||||
if err != nil {
|
||||
return e.endedRestored(a, err, nil, false)
|
||||
}
|
||||
if ans.Error == snapshotProtected {
|
||||
_ = os.Remove(file)
|
||||
return e.restoreInPlace(ctx, a)
|
||||
}
|
||||
if ans.Error != "" {
|
||||
return e.endedRestored(a, errors.New(ans.Error), nil, false)
|
||||
}
|
||||
// The bytes are checked against the keeper's digest before any aside is taken (§567).
|
||||
if got, n, err := digestOf(file); err != nil || got != ans.Digest || n != ans.Size {
|
||||
return e.endedRestored(a, fmt.Errorf("snapshot damaged: the request %s's bytes received do not match its digest", a.Request), nil, false)
|
||||
}
|
||||
rec, err := e.setAside(ctx, a, []string{ans.Stream}, true)
|
||||
if err != nil {
|
||||
return e.endedRestored(a, err, nil, false)
|
||||
}
|
||||
e.setStep(a, "restoring", "")
|
||||
if err := e.Bus.Restore(ctx, file); err != nil {
|
||||
err = fmt.Errorf("stream %s: %w", ans.Stream, err)
|
||||
if back := e.restoreFromAside(ctx, rec, true); back != nil {
|
||||
err = fmt.Errorf("%w; and the aside is not put back: %v", err, back)
|
||||
}
|
||||
return e.endedRestored(a, err, asidePtr(rec), false)
|
||||
}
|
||||
rec.Step = stepRestored
|
||||
_ = writeJSON(filepath.Join(e.asideDirOf(rec), asideFile), rec)
|
||||
return e.endedRestored(a, nil, asidePtr(rec), false)
|
||||
}
|
||||
|
||||
// restoreInPlace is a protected snapshot's restore (§681, O-7): snapshot-restore with verify first, then the live data of
|
||||
// the restore's module and kept data set aside, its names from the catalog entry of the fingerprint the node runs
|
||||
// (A-137), then snapshot-restore, waited on by snapshot-restored of that request and the act it was answered for at most
|
||||
// snapshot-restore-wait-minutes (A-138). Its `failed` puts the aside back. At the bound it asks snapshot-restore-cancel and
|
||||
// waits one more bound for that act's outcome (A-140); when none comes it puts nothing back and ends failed with
|
||||
// `snapshot-restore unanswered`, the aside left at `removed` for aside-unrestored (§561). A keeper answering busy ends the
|
||||
// act failed before any aside, naming the keeper's running act (A-140).
|
||||
func (e *Engine) restoreInPlace(ctx context.Context, a *Act) error {
|
||||
e.setStep(a, "verifying the keeper's snapshot", "")
|
||||
v, err := e.Keeper.Restore(ctx, a.Request, true, a.Why)
|
||||
if err == nil {
|
||||
err = keeperAnswer(v)
|
||||
}
|
||||
if err != nil {
|
||||
return e.endedRestored(a, err, nil, false)
|
||||
}
|
||||
fp, err := e.Seats.RunningFingerprint(ctx, a.Module)
|
||||
if err != nil {
|
||||
return e.endedRestored(a, err, nil, false)
|
||||
}
|
||||
a.Fingerprint = fp
|
||||
names, err := e.Seats.Streams(ctx, a.Module, a.Kept, fp)
|
||||
if err != nil {
|
||||
return e.endedRestored(a, err, nil, false)
|
||||
}
|
||||
rec, err := e.setAside(ctx, a, names, true)
|
||||
if err != nil {
|
||||
return e.endedRestored(a, err, nil, false)
|
||||
}
|
||||
putBack := func(why error) error {
|
||||
if back := e.restoreFromAside(ctx, rec, true); back != nil {
|
||||
why = fmt.Errorf("%w; and the aside is not put back: %v", why, back)
|
||||
}
|
||||
return e.endedRestored(a, why, asidePtr(rec), false)
|
||||
}
|
||||
e.setStep(a, "restoring in place", "the keeper's snapshot-restored")
|
||||
// The wait's bound is the key `restore-wait.<act>` of the state bounds, written before the keeper is asked, so a bound
|
||||
// that cannot be written asks nothing of the keeper (A-143).
|
||||
wctx, passed, err := e.restoreBound(ctx, a.ID)
|
||||
if err != nil {
|
||||
return putBack(err)
|
||||
}
|
||||
r, err := e.Keeper.Restore(ctx, a.Request, false, a.Why)
|
||||
if err == nil {
|
||||
err = keeperAnswer(r)
|
||||
}
|
||||
if err != nil {
|
||||
passed()
|
||||
return putBack(err)
|
||||
}
|
||||
act := r.Accepted
|
||||
ev, ok := e.Keeper.Restored(wctx, a.Request, act)
|
||||
passed()
|
||||
if !ok {
|
||||
e.setStep(a, "cancelling the keeper's restore", "the keeper's snapshot-restored after the cancel")
|
||||
why := fmt.Sprintf("the recovery module's wait of %s on the restore in place passed (act %s)", e.restoreWait(), a.ID)
|
||||
if c, err := e.Keeper.CancelRestore(context.Background(), act, why); err != nil || c.Error != "" {
|
||||
e.logf("the cancel of the keeper's restore %s is not accepted (%v%s); its outcome is waited on once more", act, err, c.Error)
|
||||
}
|
||||
wctx, passed, err := e.restoreBound(context.Background(), a.ID)
|
||||
if err != nil {
|
||||
e.logf("the bound of the wait after the cancel of %s is not written: %v", act, err)
|
||||
return e.endedRestored(a, fmt.Errorf("%s: %v", ErrUnanswered, err), asidePtr(rec), false)
|
||||
}
|
||||
ev, ok = e.Keeper.Restored(wctx, a.Request, act)
|
||||
passed()
|
||||
}
|
||||
if !ok {
|
||||
return e.endedRestored(a, errors.New(ErrUnanswered), asidePtr(rec), false)
|
||||
}
|
||||
if ev.Outcome != meshbroker.SnapshotRestoredEventOutcomeDone {
|
||||
return putBack(fmt.Errorf("the keeper's restore in place failed: %s", ev.Error))
|
||||
}
|
||||
rec.Step = stepRestored
|
||||
_ = writeJSON(filepath.Join(e.asideDirOf(rec), asideFile), rec)
|
||||
return e.endedRestored(a, nil, asidePtr(rec), false)
|
||||
}
|
||||
|
||||
// restoreBound is the bound of one wait on snapshot-restored (§681, A-138, A-143): the key `restore-wait.<act>` with the time
|
||||
// to live snapshot-restore-wait-minutes; its expiry ends the context it answers. end removes the key.
|
||||
func (e *Engine) restoreBound(ctx context.Context, act string) (context.Context, func(), error) {
|
||||
wctx, cancel := context.WithCancel(ctx)
|
||||
end, err := e.bound(BoundRestoreWait+"."+act, e.restoreWait(), cancel)
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, nil, err
|
||||
}
|
||||
return wctx, func() { end(); cancel() }, nil
|
||||
}
|
||||
|
||||
// bound writes a bound key in the state bounds (A-77, A-143).
|
||||
func (e *Engine) bound(key string, ttl time.Duration, expired func()) (func(), error) {
|
||||
if e.Bounds == nil {
|
||||
return nil, fmt.Errorf("the bound %s is not written: the engine has no state bounds", key)
|
||||
}
|
||||
return e.Bounds.Bound(key, ttl, expired)
|
||||
}
|
||||
|
||||
// keeperAnswer reads snapshot-restore's one form (§691): accepted, or an error naming the keeper's refusal or its running act.
|
||||
func keeperAnswer(r meshbroker.SnapshotRestoreAnswer) error {
|
||||
switch {
|
||||
case r.Error != "":
|
||||
return errors.New(r.Error)
|
||||
case r.Busy != "":
|
||||
return fmt.Errorf("busy: the keeper restores its act %s", r.Busy)
|
||||
case r.Accepted == "":
|
||||
return errors.New("the keeper answered snapshot-restore with nothing")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
package recovery_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
|
||||
|
||||
"mesh-recovery/internal/recovery"
|
||||
)
|
||||
|
||||
// fakeKeeper is the bus module's keeper holding one snapshot of app_notes (the fake bus's file form). in place: how its
|
||||
// restore in place ends: "done", "failed", "silent" (it says nothing until cancelled, then failed) or "deaf" (never).
|
||||
type fakeKeeper struct {
|
||||
mu sync.Mutex
|
||||
b *fakeBus
|
||||
file string
|
||||
protected bool
|
||||
busy string
|
||||
damaged bool
|
||||
inPlace string
|
||||
asked []string
|
||||
wrote int
|
||||
cancelled chan struct{}
|
||||
}
|
||||
|
||||
func newKeeper(b *fakeBus) *fakeKeeper {
|
||||
return &fakeKeeper{b: b, file: "snap:app_notes:as dropped", inPlace: "done", cancelled: make(chan struct{})}
|
||||
}
|
||||
|
||||
func (k *fakeKeeper) note(s string) {
|
||||
k.mu.Lock()
|
||||
k.asked = append(k.asked, s)
|
||||
k.mu.Unlock()
|
||||
}
|
||||
|
||||
func (k *fakeKeeper) Get(_ context.Context, request string, fits func(meshbroker.SnapshotGetAnswer) error, w io.Writer) (meshbroker.SnapshotGetAnswer, error) {
|
||||
k.note("get " + request)
|
||||
if k.protected {
|
||||
return meshbroker.SnapshotGetAnswer{Error: "snapshot protected"}, nil
|
||||
}
|
||||
sum := sha256.Sum256([]byte(k.file))
|
||||
ans := meshbroker.SnapshotGetAnswer{Request: request, Module: "app", Local: "notes", Stream: "app_notes", Size: int64(len(k.file)),
|
||||
Digest: "sha256:" + hex.EncodeToString(sum[:]), Chunks: 1}
|
||||
if err := fits(ans); err != nil {
|
||||
return ans, err
|
||||
}
|
||||
body := k.file
|
||||
if k.damaged {
|
||||
body = "snap:app_notes:altered on its way"
|
||||
}
|
||||
n, err := io.WriteString(w, body)
|
||||
k.wrote += n
|
||||
return ans, err
|
||||
}
|
||||
|
||||
func (k *fakeKeeper) Restore(_ context.Context, request string, verify bool, _ string) (meshbroker.SnapshotRestoreAnswer, error) {
|
||||
if verify {
|
||||
k.note("verify " + request)
|
||||
return meshbroker.SnapshotRestoreAnswer{Accepted: request}, nil
|
||||
}
|
||||
k.note("restore " + request)
|
||||
if k.busy != "" {
|
||||
return meshbroker.SnapshotRestoreAnswer{Busy: k.busy}, nil
|
||||
}
|
||||
return meshbroker.SnapshotRestoreAnswer{Accepted: "keeper-act-1"}, nil
|
||||
}
|
||||
|
||||
func (k *fakeKeeper) CancelRestore(_ context.Context, act, _ string) (meshbroker.SnapshotRestoreCancelAnswer, error) {
|
||||
k.note("cancel " + act)
|
||||
close(k.cancelled)
|
||||
return meshbroker.SnapshotRestoreCancelAnswer{Accepted: act}, nil
|
||||
}
|
||||
|
||||
func (k *fakeKeeper) Restored(ctx context.Context, request, act string) (meshbroker.SnapshotRestoredEvent, bool) {
|
||||
ev := meshbroker.SnapshotRestoredEvent{Request: request, Act: act, Outcome: "failed", Error: "the server refused the archive"}
|
||||
switch k.inPlace {
|
||||
case "done":
|
||||
k.b.mu.Lock()
|
||||
if _, live := k.b.streams["app_notes"]; live {
|
||||
k.b.mu.Unlock()
|
||||
return meshbroker.SnapshotRestoredEvent{Request: request, Act: act, Outcome: "failed", Error: "names live"}, true
|
||||
}
|
||||
k.b.streams["app_notes"] = "as dropped"
|
||||
k.b.mu.Unlock()
|
||||
ev.Outcome, ev.Error = "done", ""
|
||||
return ev, true
|
||||
case "failed":
|
||||
return ev, true
|
||||
case "silent":
|
||||
select {
|
||||
case <-k.cancelled:
|
||||
ev.Error = "cancelled"
|
||||
return ev, true
|
||||
case <-ctx.Done():
|
||||
return ev, false
|
||||
}
|
||||
}
|
||||
<-ctx.Done()
|
||||
return ev, false
|
||||
}
|
||||
|
||||
func requestRig(t *testing.T) (*rig, *fakeKeeper) {
|
||||
r := newRig(t)
|
||||
k := newKeeper(r.b)
|
||||
r.e.Keeper = k
|
||||
r.e.Settings.RestoreWait = 20 * time.Millisecond
|
||||
return r, k
|
||||
}
|
||||
|
||||
func requestAct() recovery.Act {
|
||||
return recovery.Act{Kind: recovery.KindRestore, Module: "app", Kept: "notes", Request: "req-1", Why: "test the drop's snapshot", Caller: "tool-runner", Asker: "op@node"}
|
||||
}
|
||||
|
||||
// A keeper's snapshot by request is read through snapshot-get into incoming/<request>, refused before the first chunk past
|
||||
// the cap, checked against its digest before any aside, restored by the module itself, and its file deleted at the end
|
||||
// (design 47 G §681 Its use, §682).
|
||||
func TestARestoreByRequestReadsTheKeepersSnapshot(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
cap int64
|
||||
damaged bool
|
||||
err string
|
||||
}{
|
||||
{name: "received and restored", cap: 1 << 20},
|
||||
{name: "altered on its way", cap: 1 << 20, damaged: true, err: "snapshot damaged"},
|
||||
{name: "past the cap", cap: 10, err: "cap:"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
r, k := requestRig(t)
|
||||
r.e.Settings.Cap, k.damaged = tc.cap, tc.damaged
|
||||
ans := r.do(requestAct())
|
||||
ev := lastRestored(t, r)
|
||||
if ev.Act != ans.Accepted || !strings.HasPrefix(ev.Error, tc.err) || (tc.err == "") != (ev.Outcome == "done") {
|
||||
t.Fatalf("restored %+v, want error %q", ev, tc.err)
|
||||
}
|
||||
if left, _ := os.ReadDir(filepath.Join(r.e.Dir, "incoming")); len(left) != 0 {
|
||||
t.Errorf("incoming holds %v after the act", left)
|
||||
}
|
||||
kp, _ := r.e.KeptPoints("app", "notes")
|
||||
if tc.err != "" {
|
||||
if r.b.streams["app_notes"] != "hello" || len(kp.Asides) != 0 {
|
||||
t.Errorf("a refused restore touched the live data: %v, asides %+v", r.b.streams, kp.Asides)
|
||||
}
|
||||
if tc.name == "past the cap" && k.wrote != 0 {
|
||||
t.Errorf("%d bytes received past the cap", k.wrote)
|
||||
}
|
||||
return
|
||||
}
|
||||
if r.b.streams["app_notes"] != "as dropped" || r.b.streams["KV_app_seen"] != "world" || len(kp.Asides) != 1 ||
|
||||
ev.Aside == nil || len(ev.Aside.Names) != 1 || ev.Aside.Names[0] != "app_notes" {
|
||||
t.Errorf("streams %v, asides %+v, event aside %+v", r.b.streams, kp.Asides, ev.Aside)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A protected snapshot is verified, the running data of the module and kept data set aside (A-137), and restored in place
|
||||
// by the keeper; its failure puts the aside back; at the bound the module cancels first and puts the aside back on the
|
||||
// cancel's outcome, and with no outcome at all ends `snapshot-restore unanswered`, the aside left unrestored; a busy keeper
|
||||
// ends the act before any aside (§681, A-138, A-140).
|
||||
func TestAProtectedSnapshotIsRestoredInPlaceByTheKeeper(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
inPlace, busy string
|
||||
err string
|
||||
asked string
|
||||
live string
|
||||
unrestored int
|
||||
bounds string // how each bound key restore-wait.<act> was removed (A-143)
|
||||
}{
|
||||
{inPlace: "done", asked: "get req-1|verify req-1|restore req-1", live: "as dropped", bounds: "ended"},
|
||||
{inPlace: "failed", asked: "get req-1|verify req-1|restore req-1", live: "hello", err: "the keeper's restore in place failed", bounds: "ended"},
|
||||
{inPlace: "silent", asked: "get req-1|verify req-1|restore req-1|cancel keeper-act-1", live: "hello", err: "the keeper's restore in place failed: cancelled", bounds: "expired|ended"},
|
||||
{inPlace: "deaf", asked: "get req-1|verify req-1|restore req-1|cancel keeper-act-1", err: recovery.ErrUnanswered, unrestored: 1, bounds: "expired|expired"},
|
||||
{inPlace: "done", busy: "keeper-act-0", asked: "get req-1|verify req-1|restore req-1", live: "hello", err: "busy: the keeper restores its act keeper-act-0", bounds: "ended"},
|
||||
} {
|
||||
t.Run(tc.inPlace+tc.busy, func(t *testing.T) {
|
||||
r, k := requestRig(t)
|
||||
k.protected, k.inPlace, k.busy = true, tc.inPlace, tc.busy
|
||||
r.do(requestAct())
|
||||
ev := lastRestored(t, r)
|
||||
if !strings.HasPrefix(ev.Error, tc.err) || (tc.err == "") != (ev.Outcome == "done") || ev.Fingerprint != "fp-run" {
|
||||
t.Fatalf("restored %+v, want error %q", ev, tc.err)
|
||||
}
|
||||
if got := strings.Join(k.asked, "|"); got != tc.asked {
|
||||
t.Errorf("asked %s, want %s", got, tc.asked)
|
||||
}
|
||||
if r.b.streams["app_notes"] != tc.live {
|
||||
t.Errorf("app_notes is %q, want %q", r.b.streams["app_notes"], tc.live)
|
||||
}
|
||||
if tc.live == "hello" && r.b.streams["KV_app_seen"] != "world" {
|
||||
t.Errorf("the second name of the running data is not put back: %v", r.b.streams)
|
||||
}
|
||||
if n := len(r.e.Unrestored()); n != tc.unrestored {
|
||||
t.Errorf("%d unrestored asides, want %d", n, tc.unrestored)
|
||||
}
|
||||
w, rm := r.bounds.keys()
|
||||
var wantW, wantR []string
|
||||
for _, by := range strings.Split(tc.bounds, "|") {
|
||||
wantW = append(wantW, "restore-wait."+ev.Act+" 20ms")
|
||||
wantR = append(wantR, "restore-wait."+ev.Act+" "+by)
|
||||
}
|
||||
if w != strings.Join(wantW, "|") || rm != strings.Join(wantR, "|") {
|
||||
t.Errorf("bound keys written %q, removed %q; want %q, %q", w, rm, strings.Join(wantW, "|"), strings.Join(wantR, "|"))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package recovery_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
|
||||
)
|
||||
|
||||
type manifest struct {
|
||||
Module string `json:"module"`
|
||||
RunsAs string `json:"runs-as"`
|
||||
Claims []struct {
|
||||
Name string `json:"name"`
|
||||
Scope string `json:"scope"`
|
||||
Serves []string `json:"serves"`
|
||||
} `json:"claims"`
|
||||
Tools []string `json:"tools"`
|
||||
Consumes []string `json:"consumes"`
|
||||
Invokes []string `json:"invokes"`
|
||||
Config map[string]struct {
|
||||
Default float64 `json:"default"`
|
||||
} `json:"config"`
|
||||
Data struct {
|
||||
Own []struct{ ID string } `json:"own"`
|
||||
} `json:"data"`
|
||||
Resources []struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Serves []string `json:"serves"`
|
||||
Bus struct {
|
||||
State []string `json:"state"`
|
||||
Invokes []string `json:"invokes"`
|
||||
} `json:"bus"`
|
||||
Health struct {
|
||||
Kind string `json:"kind"`
|
||||
} `json:"health"`
|
||||
} `json:"resources"`
|
||||
RuntimeHealth struct {
|
||||
Kind string `json:"kind"`
|
||||
Tool string `json:"tool"`
|
||||
} `json:"runtime-health"`
|
||||
Requires []json.RawMessage `json:"requires"`
|
||||
}
|
||||
|
||||
// The manifest says what the module is (design 47 G §554, §656, §657, §633, §564, §567): it claims the seat once for the
|
||||
// mesh serving every verb of the seat's definition, its process answers each of them itself (A-115: `serves` beside its
|
||||
// `bus`), its health check is its tool beside the process's own check and its template states are its own by `requires`
|
||||
// alone, the digests state taking the template's fixed cap (A-133, A-135), its data item is `points`, its settings carry
|
||||
// the design's defaults, and what the process invokes the module declares.
|
||||
func TestTheManifestSaysWhatTheModuleIs(t *testing.T) {
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m manifest
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m.Module != "mesh-recovery" || m.RunsAs != "mesh-recovery" {
|
||||
t.Errorf("module %q runs as %q", m.Module, m.RunsAs)
|
||||
}
|
||||
if len(m.Claims) != 1 || m.Claims[0].Name != seat.Seat || m.Claims[0].Scope != "mesh" || !slices.Equal(m.Claims[0].Serves, seat.Verbs()) {
|
||||
t.Errorf("claims %+v, want %s at mesh scope serving %v", m.Claims, seat.Seat, seat.Verbs())
|
||||
}
|
||||
for _, id := range []string{"points"} {
|
||||
if !slices.ContainsFunc(m.Data.Own, func(o struct{ ID string }) bool { return o.ID == id }) {
|
||||
t.Errorf("no data item %q", id)
|
||||
}
|
||||
}
|
||||
for k, want := range map[string]float64{"points-cap": 10 << 30, "points-filling-mark": 80, "point-keep-days": 90, "dump-wait-minutes": 60, "snapshot-restore-wait-minutes": 90, "expire-every-hours": 24} {
|
||||
if m.Config[k].Default != want {
|
||||
t.Errorf("setting %s defaults to %v, want %v", k, m.Config[k].Default, want)
|
||||
}
|
||||
}
|
||||
// The keeper's snapshot verbs the recovery module asks, the cancel at its bound among them (§681).
|
||||
for _, inv := range []string{"seat:mesh-broker.snapshot-get", "seat:mesh-broker.snapshot-restore", "seat:mesh-broker.snapshot-restore-cancel"} {
|
||||
if !slices.Contains(m.Invokes, inv) {
|
||||
t.Errorf("does not invoke %s", inv)
|
||||
}
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
if r.Type != "process" {
|
||||
continue
|
||||
}
|
||||
var want []string
|
||||
for _, v := range seat.Verbs() {
|
||||
want = append(want, "seat:"+seat.Seat+"."+v)
|
||||
}
|
||||
if !slices.Equal(r.Serves, want) {
|
||||
t.Errorf("the process serves %v, want %v", r.Serves, want)
|
||||
}
|
||||
if r.Health.Kind != "unit" {
|
||||
t.Errorf("the process's own check is %q, want unit", r.Health.Kind)
|
||||
}
|
||||
if len(r.Bus.State) != 0 {
|
||||
t.Errorf("the process's bus lists the states %v again, which are the module's own by its requires", r.Bus.State)
|
||||
}
|
||||
for _, inv := range r.Bus.Invokes {
|
||||
if !slices.Contains(m.Invokes, inv) {
|
||||
t.Errorf("the process invokes %s, which the module does not declare", inv)
|
||||
}
|
||||
}
|
||||
}
|
||||
if m.RuntimeHealth.Kind != "tool" || m.RuntimeHealth.Tool != "recovery_kept_points" || !slices.Contains(m.Tools, m.RuntimeHealth.Tool) {
|
||||
t.Errorf("the tool check is %+v, want recovery_kept_points, a tool of %v", m.RuntimeHealth, m.Tools)
|
||||
}
|
||||
bounds := false
|
||||
for _, raw := range m.Requires {
|
||||
var q struct {
|
||||
Name string `json:"name"`
|
||||
Options map[string]any `json:"options"`
|
||||
}
|
||||
if json.Unmarshal(raw, &q) == nil && q.Name == "digests" && len(q.Options) != 0 {
|
||||
t.Errorf("digests gives the options %v beside the template state's fixed cap", q.Options)
|
||||
}
|
||||
// The expiry's bound is a bound key of its own state, its keys living a time of their own (A-77, A-138).
|
||||
bounds = bounds || q.Name == "bounds" && q.Options["key-ttl"] == true
|
||||
}
|
||||
if !bounds {
|
||||
t.Error("no state bounds with key-ttl for the expiry's bound key")
|
||||
}
|
||||
for _, c := range []string{"node-backup.item-started", "node-backup.item-ended", "mesh-broker.snapshot-restored"} {
|
||||
if !slices.Contains(m.Consumes, c) {
|
||||
t.Errorf("does not consume %s", c)
|
||||
}
|
||||
}
|
||||
for _, tool := range m.Tools {
|
||||
if len(tool) < 9 || tool[:9] != "recovery_" {
|
||||
t.Errorf("the tool %q is not named recovery_", tool)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
package recovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
const pointFile = "point.json"
|
||||
|
||||
// run is the work of one accepted act; it ends the act with its event under the act's id (§560).
|
||||
func (e *Engine) run(ctx context.Context, a *Act) {
|
||||
var err error
|
||||
switch a.Kind {
|
||||
case KindTake:
|
||||
err = e.take(ctx, a)
|
||||
case KindExpire:
|
||||
e.expire(a.ID, true)
|
||||
case KindRestore:
|
||||
err = e.restore(ctx, a)
|
||||
case KindAside:
|
||||
err = e.aside(ctx, a)
|
||||
case KindRestoreAside:
|
||||
err = e.restoreAside(ctx, a)
|
||||
case KindDiscardAside:
|
||||
err = e.discardAside(a)
|
||||
default:
|
||||
err = e.endedFailed(a, fmt.Errorf("no act of kind %q", a.Kind))
|
||||
}
|
||||
if err != nil {
|
||||
e.logf("act %s ended failed: %v", a.ID, err)
|
||||
a.Error = err.Error()
|
||||
}
|
||||
e.markEnded(a)
|
||||
}
|
||||
|
||||
// sayEnd says the ending event of the act id (§554, §560): the act is first no longer the running one and written ended, so
|
||||
// a caller told busy that asks again on hearing it is never told busy with the same act again.
|
||||
func (e *Engine) sayEnd(id, event string, body any, parts ...string) {
|
||||
e.mu.Lock()
|
||||
if e.running != nil && e.running.ID == id {
|
||||
e.markEnded(e.running)
|
||||
e.running, e.cancel = nil, nil
|
||||
}
|
||||
e.mu.Unlock()
|
||||
e.say(event, body, parts...)
|
||||
}
|
||||
|
||||
// say publishes an event and logs the failure; an event lost is never silent.
|
||||
func (e *Engine) say(event string, body any, parts ...string) {
|
||||
if err := e.Seats.Say(event, body, parts...); err != nil {
|
||||
e.logf("the event %s was not said: %v", event, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) endedFailed(a *Act, why error) error {
|
||||
why = e.stoppedOr(why)
|
||||
e.sayEnd(a.ID, seat.EventEnded, seat.EndedEvent{Act: a.ID, At: e.now(), Module: a.Module, Kept: a.Kept, Form: a.Form,
|
||||
Fingerprint: a.Fingerprint, Snapshot: a.Snapshot, Outcome: seat.EndedEventOutcomeFailed, Error: why.Error(), Why: cancelWhy(a, why)},
|
||||
a.Module, a.Kept)
|
||||
return why
|
||||
}
|
||||
|
||||
// cancelWhy is the hand act's words for a cancel (A-117).
|
||||
func cancelWhy(a *Act, why error) string {
|
||||
if why != nil && why.Error() == ErrCanceled {
|
||||
return a.Why
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// filled is the fill of the points directory without the temporary names of acts still writing.
|
||||
func (e *Engine) filled() int64 {
|
||||
return dirSize(filepath.Join(e.Dir, "points")) + dirSize(filepath.Join(e.Dir, "aside"))
|
||||
}
|
||||
|
||||
// take is `snapshot-kept` and the hand verb `take` (§516, §564, §567): one stream at a time under a temporary name,
|
||||
// renamed whole, its digests on the bus before `ended` is said.
|
||||
func (e *Engine) take(ctx context.Context, a *Act) error {
|
||||
dir, err := e.pointDir(a.Module, a.Kept, a.Form, a.Fingerprint)
|
||||
if err != nil {
|
||||
return e.endedFailed(a, err)
|
||||
}
|
||||
names, err := e.Seats.Streams(ctx, a.Module, a.Kept, a.Fingerprint)
|
||||
if err != nil || len(names) == 0 {
|
||||
if err == nil {
|
||||
err = fmt.Errorf("%s's %s has no bus name at %s", a.Module, a.Kept, a.Fingerprint)
|
||||
}
|
||||
return e.endedFailed(a, err)
|
||||
}
|
||||
e.setStep(a, "measuring", "")
|
||||
objs, err := e.Bus.Objects(ctx, names)
|
||||
if err != nil {
|
||||
return e.endedFailed(a, err)
|
||||
}
|
||||
var need int64
|
||||
for _, o := range objs {
|
||||
need += o.Bytes
|
||||
}
|
||||
if fill := e.filled(); fill+need > e.Settings.Cap {
|
||||
return e.endedFailed(a, fmt.Errorf("%s: the cap is %d bytes, the point needs %d and the directory holds %d", ErrCap, e.Settings.Cap, need, fill))
|
||||
}
|
||||
tmp := filepath.Join(filepath.Dir(dir), ".tmp-"+a.ID)
|
||||
_ = os.RemoveAll(tmp)
|
||||
if err := os.MkdirAll(tmp, 0o700); err != nil {
|
||||
return e.endedFailed(a, err)
|
||||
}
|
||||
defer os.RemoveAll(tmp)
|
||||
p := seat.Point{At: e.now(), CatchUp: a.CatchUp, Fingerprint: a.Fingerprint, Form: a.Form, Kept: a.Kept, Module: a.Module,
|
||||
Snapshot: strings.TrimPrefix(dir, filepath.Join(e.Dir, "points")+string(filepath.Separator))}
|
||||
e.setStep(a, "taking", "")
|
||||
for _, o := range objs {
|
||||
if !o.Exists {
|
||||
continue
|
||||
}
|
||||
file := filepath.Join(tmp, keys.EscapeName(o.Stream)+".snap")
|
||||
sum, n, err := e.takeOne(ctx, o.Stream, file)
|
||||
if err != nil {
|
||||
return e.endedFailed(a, err)
|
||||
}
|
||||
p.Digests = append(p.Digests, seat.Digest{Name: o.Stream, Digest: sum})
|
||||
p.Size += n
|
||||
p.Consumers = append(p.Consumers, o.Consumers...)
|
||||
if e.filled()+p.Size > e.Settings.Cap {
|
||||
return e.endedFailed(a, fmt.Errorf("%s: the stream grew while it was taken and passes the cap of %d bytes", ErrCap, e.Settings.Cap))
|
||||
}
|
||||
}
|
||||
if err := writeJSON(filepath.Join(tmp, pointFile), p); err != nil {
|
||||
return e.endedFailed(a, err)
|
||||
}
|
||||
_ = os.RemoveAll(dir)
|
||||
if err := os.Rename(tmp, dir); err != nil {
|
||||
return e.endedFailed(a, err)
|
||||
}
|
||||
raw, _ := jsonMarshal(p.Digests)
|
||||
if err := e.Digests.Put(keys.EscapeName(p.Snapshot), raw); err != nil {
|
||||
_ = os.RemoveAll(dir)
|
||||
return e.endedFailed(a, fmt.Errorf("the digests are not on the bus: %w", err))
|
||||
}
|
||||
// The removal at each snapshot-kept (§551) runs within the take, before its ended, so one act runs at a time.
|
||||
e.expire(a.ID, false)
|
||||
e.sayEnd(a.ID, seat.EventEnded, seat.EndedEvent{Act: a.ID, At: e.now(), Module: a.Module, Kept: a.Kept, Form: a.Form,
|
||||
Fingerprint: a.Fingerprint, Snapshot: p.Snapshot, Outcome: seat.EndedEventOutcomeDone, Consumers: p.Consumers,
|
||||
Digests: p.Digests, Size: p.Size, CatchUp: p.CatchUp}, a.Module, a.Kept)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *Engine) takeOne(ctx context.Context, stream, file string) (string, int64, error) {
|
||||
f, err := os.Create(file)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if err := e.Bus.Take(ctx, stream, f); err != nil {
|
||||
f.Close()
|
||||
return "", 0, fmt.Errorf("stream %s: %w", stream, err)
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
return digestOf(file)
|
||||
}
|
||||
|
||||
func digestOf(file string) (string, int64, error) {
|
||||
f, err := os.Open(file)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
defer f.Close()
|
||||
h := sha256.New()
|
||||
n, err := io.Copy(h, f)
|
||||
return "sha256:" + hex.EncodeToString(h.Sum(nil)), n, err
|
||||
}
|
||||
|
||||
// Points lists every restore point in the directory, newest first (§517): the one place they are kept.
|
||||
func (e *Engine) Points() ([]seat.Point, error) {
|
||||
var out []seat.Point
|
||||
root := filepath.Join(e.Dir, "points")
|
||||
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if d.IsDir() && strings.HasPrefix(d.Name(), ".") {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
if !d.IsDir() && d.Name() == pointFile {
|
||||
var p seat.Point
|
||||
if err := readJSON(path, &p); err != nil {
|
||||
return fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
out = append(out, p)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].At.After(out[j].At) })
|
||||
return out, err
|
||||
}
|
||||
|
||||
// expire is §551's removal: a point 90 days after its `at` (the setting), never the newest of its kept data, each removal
|
||||
// said as point-removed and the run as expiry-ended, also when it removes nothing (§560). It runs inside an act: own when
|
||||
// it is the act expire-points, whose ending event its expiry-ended is.
|
||||
func (e *Engine) expire(act string, own bool) {
|
||||
pts, err := e.Points()
|
||||
if err != nil {
|
||||
e.logf("expiry: %v", err)
|
||||
}
|
||||
newest := map[string]string{}
|
||||
for _, p := range pts { // newest first
|
||||
k := p.Module + "\x00" + p.Kept
|
||||
if _, ok := newest[k]; !ok {
|
||||
newest[k] = p.Snapshot
|
||||
}
|
||||
}
|
||||
cutoff := e.now().Add(-time.Duration(e.Settings.KeepDays * float64(24*time.Hour)))
|
||||
removed := int64(0)
|
||||
for _, p := range pts {
|
||||
if !p.At.Before(cutoff) || newest[p.Module+"\x00"+p.Kept] == p.Snapshot {
|
||||
continue
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(e.Dir, "points", p.Snapshot)); err != nil {
|
||||
e.logf("the point %s is not removed: %v", p.Snapshot, err)
|
||||
continue
|
||||
}
|
||||
_ = e.Digests.Delete(keys.EscapeName(p.Snapshot))
|
||||
if err := e.Seats.HandAct(Module, "", "point-removed "+p.Snapshot, "older than the keep days"); err != nil {
|
||||
e.logf("the removal of %s is not in the hand-act log: %v", p.Snapshot, err)
|
||||
}
|
||||
e.say(seat.EventPointRemoved, seat.PointRemovedEvent{Act: act, At: e.now(), Module: p.Module, Kept: p.Kept, Form: p.Form,
|
||||
Fingerprint: p.Fingerprint, Snapshot: p.Snapshot}, p.Module, p.Kept)
|
||||
removed++
|
||||
}
|
||||
ev := seat.ExpiryEndedEvent{Act: act, At: e.now(), Removed: removed}
|
||||
if own {
|
||||
e.sayEnd(act, seat.EventExpiryEnded, ev)
|
||||
return
|
||||
}
|
||||
e.say(seat.EventExpiryEnded, ev)
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package recovery_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
|
||||
|
||||
"mesh-recovery/internal/recovery"
|
||||
)
|
||||
|
||||
func takeAct() recovery.Act {
|
||||
return recovery.Act{Kind: recovery.KindTake, Module: "app", Kept: "notes", Form: 2, Fingerprint: "fp1", Why: "form change", Caller: "node.engine"}
|
||||
}
|
||||
|
||||
// A point is taken stream by stream under a temporary name, renamed whole, its digests put on the bus before `ended` is
|
||||
// said under the act's id with the point whole, and the act is written in the hand-act log with the caller the bus proved
|
||||
// (design 47 G §516, §554, §560, §567).
|
||||
func TestATakenPointIsListedWithItsDigestsAndEndedUnderItsAct(t *testing.T) {
|
||||
r := newRig(t)
|
||||
ans := r.do(takeAct())
|
||||
if ans.Accepted == "" {
|
||||
t.Fatalf("answer %+v", ans)
|
||||
}
|
||||
ended := r.s.named(seat.EventEnded)
|
||||
if len(ended) != 1 {
|
||||
t.Fatalf("events %+v", r.s.events)
|
||||
}
|
||||
ev := ended[0].Body.(seat.EndedEvent)
|
||||
if ev.Act != ans.Accepted || ev.Outcome != "done" || len(ev.Digests) != 2 || ev.Size == 0 || ev.Snapshot != "app/notes/2-fp1" {
|
||||
t.Errorf("ended %+v", ev)
|
||||
}
|
||||
if got := strings.Join(ended[0].Parts, "."); got != "app.notes" {
|
||||
t.Errorf("parts %q", got)
|
||||
}
|
||||
if len(ev.Consumers) != 1 || ev.Consumers[0].Sequence != 7 {
|
||||
t.Errorf("consumers %+v", ev.Consumers)
|
||||
}
|
||||
pts, err := r.e.Points()
|
||||
if err != nil || len(pts) != 1 || pts[0].Snapshot != ev.Snapshot {
|
||||
t.Fatalf("points %+v %v", pts, err)
|
||||
}
|
||||
if _, ok, _ := r.d.Get("app/notes/2-fp1"); !ok {
|
||||
t.Errorf("no digests on the bus: %v", r.d.m)
|
||||
}
|
||||
if len(r.s.handActs) != 1 || !strings.HasPrefix(r.s.handActs[0], "node.engine||take app notes|form change") {
|
||||
t.Errorf("hand acts %v", r.s.handActs)
|
||||
}
|
||||
}
|
||||
|
||||
// One act at a time and no queue: a second ask while one runs is answered busy with the running act (§554).
|
||||
func TestASecondAskWhileOneRunsIsAnsweredBusy(t *testing.T) {
|
||||
r := newRig(t)
|
||||
gate := make(chan struct{})
|
||||
r.e.Dump = nil
|
||||
r.e.Bus = &gatedBus{fakeBus: r.b, gate: gate}
|
||||
first := r.e.Accept(takeAct())
|
||||
second := r.e.Accept(takeAct())
|
||||
close(gate)
|
||||
r.e.Wait()
|
||||
if first.Accepted == "" || second.Busy != first.Accepted || second.Accepted != "" || second.Error != "" {
|
||||
t.Errorf("first %+v second %+v", first, second)
|
||||
}
|
||||
}
|
||||
|
||||
// An act whose hand-act log write fails does nothing and answers an error naming the log (§554).
|
||||
func TestAnActWhoseHandActLogWriteFailsDoesNothing(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.s.handFail = errTest
|
||||
ans := r.do(takeAct())
|
||||
if !strings.Contains(ans.Error, "hand-act log") || ans.Accepted != "" || len(r.s.events) != 0 {
|
||||
t.Errorf("answer %+v events %+v", ans, r.s.events)
|
||||
}
|
||||
if pts, _ := r.e.Points(); len(pts) != 0 {
|
||||
t.Errorf("points %+v", pts)
|
||||
}
|
||||
}
|
||||
|
||||
// An act that would pass the cap ends failed naming it, before anything is written (§564).
|
||||
func TestAnActThatWouldPassTheCapEndsFailedBeforeItWritesAnything(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.e.Settings.Cap = 4
|
||||
r.do(takeAct())
|
||||
ev := r.s.named(seat.EventEnded)[0].Body.(seat.EndedEvent)
|
||||
if ev.Outcome != "failed" || !strings.HasPrefix(ev.Error, recovery.ErrCap) {
|
||||
t.Errorf("ended %+v", ev)
|
||||
}
|
||||
if pts, _ := r.e.Points(); len(pts) != 0 || len(r.d.m) != 0 {
|
||||
t.Errorf("points %+v digests %v", pts, r.d.m)
|
||||
}
|
||||
}
|
||||
|
||||
// A point older than the keep days is removed and said, the newest of its kept data stays however old, and the run says
|
||||
// expiry-ended with the count, 0 included (§551, §560).
|
||||
func TestExpiryRemovesOldPointsButNeverTheNewest(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.t = time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC)
|
||||
r.do(takeAct())
|
||||
a := takeAct()
|
||||
a.Fingerprint, a.Form = "fp2", 3
|
||||
r.t = time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)
|
||||
r.do(a)
|
||||
r.s.events = nil
|
||||
r.t = time.Date(2026, 11, 12, 0, 0, 0, 0, time.UTC)
|
||||
r.do(recovery.Act{Kind: recovery.KindExpire, Why: "by hand", Caller: "tools"})
|
||||
pts, _ := r.e.Points()
|
||||
if len(pts) != 1 || pts[0].Fingerprint != "fp2" {
|
||||
t.Fatalf("points %+v", pts)
|
||||
}
|
||||
rem := r.s.named(seat.EventPointRemoved)
|
||||
if len(rem) != 1 || rem[0].Body.(seat.PointRemovedEvent).Fingerprint != "fp1" {
|
||||
t.Errorf("removed %+v", rem)
|
||||
}
|
||||
if ex := r.s.named(seat.EventExpiryEnded); len(ex) != 1 || ex[0].Body.(seat.ExpiryEndedEvent).Removed != 1 {
|
||||
t.Errorf("expiry %+v", ex)
|
||||
}
|
||||
if _, ok, _ := r.d.Get("app/notes/2-fp1"); ok {
|
||||
t.Errorf("its digests stay on the bus")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,343 @@
|
||||
// Package recovery is the recovery module's engine (design 47 G §554, §516): it takes, sets aside and restores a
|
||||
// module's kept data on the live bus, one act at a time, each accepted act written in its directory before the caller is
|
||||
// answered, the result said as an event under the act's id (§560). Every dependence on the bus, the seat list, the
|
||||
// hand-act log and the backup holder is an interface here, so the one whole test runs the engine against the real bus
|
||||
// and a unit test against a fake.
|
||||
package recovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// Module and Seat are the module's name and the seat it claims.
|
||||
const (
|
||||
Module = "mesh-recovery"
|
||||
Seat = seat.Seat
|
||||
)
|
||||
|
||||
// The kinds of act, one per verb that starts one.
|
||||
const (
|
||||
KindTake = "take"
|
||||
KindRestore = "restore"
|
||||
KindAside = "aside"
|
||||
KindDiscardAside = "discard-aside"
|
||||
KindRestoreAside = "restore-aside"
|
||||
KindExpire = "expire-points"
|
||||
)
|
||||
|
||||
// Errors an act ends with: the words the node agent's table reads (§560).
|
||||
const (
|
||||
ErrCap = "cap"
|
||||
ErrStopped = "stopped"
|
||||
ErrCanceled = "cancelled"
|
||||
ErrMissing = "restore point missing"
|
||||
ErrDamaged = "restore point damaged"
|
||||
)
|
||||
|
||||
// Object is one bus object of a kept data: a stream, by its name, with what the bus states of it.
|
||||
type Object struct {
|
||||
Stream string
|
||||
Exists bool
|
||||
Bytes int64
|
||||
Consumers []seat.Consumer
|
||||
}
|
||||
|
||||
// Bus is the bus as the module's account sees it (§554): a stream's snapshot and restore, its deletion, the
|
||||
// durable consumers' positions.
|
||||
type Bus interface {
|
||||
// Objects states each stream of names: whether it exists, its bytes, its durable consumers' acknowledged sequences.
|
||||
Objects(ctx context.Context, streams []string) ([]Object, error)
|
||||
// Take writes one stream's snapshot file to w (ADR 0235 §10's pacing and §11's bounds are the implementation's).
|
||||
Take(ctx context.Context, stream string, w io.Writer) error
|
||||
// Delete removes a stream; one that is not there is no error.
|
||||
Delete(ctx context.Context, stream string) error
|
||||
// Restore puts the stream of a snapshot file back under its own name, where no stream of that name exists.
|
||||
Restore(ctx context.Context, file string) error
|
||||
// SetConsumers creates each durable consumer at its recorded sequence.
|
||||
SetConsumers(ctx context.Context, cs []seat.Consumer) error
|
||||
}
|
||||
|
||||
// Seats is what the engine asks of the rest of the mesh.
|
||||
type Seats interface {
|
||||
// Say publishes one event of the seat under its subject parts (§560); an event of no parts has none.
|
||||
Say(event string, body any, parts ...string) error
|
||||
// HandAct writes the act in the hand-act log first (§554): the account the bus proved, who asked, what, why.
|
||||
HandAct(caller, asker, what, why string) error
|
||||
// Streams are the bus names of a module's kept data at a fingerprint, from its catalog graph entry (§554 Bus names).
|
||||
Streams(ctx context.Context, module, kept, fingerprint string) ([]string, error)
|
||||
// RunningFingerprint is the fingerprint the node-engine's placement runs for the module (A-123).
|
||||
RunningFingerprint(ctx context.Context, module string) (string, error)
|
||||
// Backup brings the points directory of the module's item back beside its path through the backup holder and
|
||||
// answers where (§516); an error when no repository snapshot holds it.
|
||||
Backup(ctx context.Context, module string) (dir string, err error)
|
||||
}
|
||||
|
||||
// Digests keeps each point's digests on the bus, apart from its bytes (§567): the state digests.
|
||||
type Digests interface {
|
||||
Put(key string, v []byte) error
|
||||
Get(key string) ([]byte, bool, error)
|
||||
Delete(key string) error
|
||||
}
|
||||
|
||||
// Dump is what the act waits on while the bus's item is dumped (§563).
|
||||
type Dump interface {
|
||||
// Wait returns once no dump of the bus's item runs; it calls waiting(on) once if it has to wait.
|
||||
Wait(ctx context.Context, waiting func(on string)) error
|
||||
}
|
||||
|
||||
// Bounds is the module's state bounds (design 47 G A-77, A-143): a wait's bound is a key with its own time to live, its
|
||||
// expiry heard on the key's watch as the bus's own change, so no timer runs in the process.
|
||||
type Bounds interface {
|
||||
// Bound writes key with the time to live ttl and calls expired once when the bus removes it; end removes it first, and
|
||||
// expired is then never called.
|
||||
Bound(key string, ttl time.Duration, expired func()) (end func(), err error)
|
||||
}
|
||||
|
||||
// The bound keys of an act's waits (A-143): `<kind>.<act>`.
|
||||
const (
|
||||
BoundDumpWait = "dump-wait"
|
||||
BoundRestoreWait = "restore-wait"
|
||||
)
|
||||
|
||||
// Settings are the bounds a whole test sets in seconds (§557, §564, §633).
|
||||
type Settings struct {
|
||||
Cap int64
|
||||
KeepDays float64
|
||||
// FillingMark is the percent of the cap past which points-filling is written (points-filling-mark, 80 unless set).
|
||||
FillingMark int
|
||||
// DumpWait is how long an act waits on the dump before recovery-waits is written (dump-wait-minutes, 60 unless set).
|
||||
DumpWait time.Duration
|
||||
// RestoreWait is how long a restore in place is waited on, and once more after its cancel (snapshot-restore-wait-minutes,
|
||||
// 90 unless set; §681, A-138, A-140).
|
||||
RestoreWait time.Duration
|
||||
}
|
||||
|
||||
// Engine is the recovery module's one engine.
|
||||
type Engine struct {
|
||||
Dir string
|
||||
Bus Bus
|
||||
Seats Seats
|
||||
Digests Digests
|
||||
Dump Dump
|
||||
// Keeper is the bus module's keeper, whose snapshots the hand verb restore takes by request (§681).
|
||||
Keeper Keeper
|
||||
// Bounds times the dump wait and the snapshot-restored wait (A-143).
|
||||
Bounds Bounds
|
||||
Settings Settings
|
||||
// Findings is the module's finding writer (§561); Node the machine it runs on, the id of points-filling.
|
||||
Findings Findings
|
||||
Node string
|
||||
Now func() time.Time
|
||||
Logf func(string, ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
running *Act
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
seq int
|
||||
why string // the hand act words of a cancel (A-117)
|
||||
stop string // why the process stops, once it does (§562, A-141)
|
||||
}
|
||||
|
||||
// Act is one accepted act, written in the directory before its caller is answered (§554).
|
||||
type Act struct {
|
||||
ID string `json:"act"`
|
||||
Kind string `json:"kind"`
|
||||
Module string `json:"module,omitempty"`
|
||||
Kept string `json:"kept,omitempty"`
|
||||
Form int64 `json:"form,omitempty"`
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
Snapshot string `json:"snapshot,omitempty"`
|
||||
// Request is a keeper's snapshot by its request, the hand verb restore's other source (§681, §682).
|
||||
Request string `json:"request,omitempty"`
|
||||
AsideID string `json:"aside-id,omitempty"`
|
||||
FromBackup bool `json:"from-backup,omitempty"`
|
||||
CatchUp *seat.CatchUp `json:"catch-up,omitempty"`
|
||||
Why string `json:"why"`
|
||||
Caller string `json:"caller"`
|
||||
Asker string `json:"asker,omitempty"`
|
||||
Since time.Time `json:"since"`
|
||||
Step string `json:"step,omitempty"`
|
||||
WaitsOn string `json:"waits-on,omitempty"`
|
||||
Ended bool `json:"ended,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// Answer is the one form every acting verb answers (§691): exactly one field set.
|
||||
type Answer struct {
|
||||
Accepted string `json:"accepted,omitempty"`
|
||||
Busy string `json:"busy,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func (e *Engine) now() time.Time {
|
||||
if e.Now != nil {
|
||||
return e.Now().UTC()
|
||||
}
|
||||
return time.Now().UTC()
|
||||
}
|
||||
|
||||
func (e *Engine) logf(f string, a ...any) {
|
||||
if e.Logf != nil {
|
||||
e.Logf(f, a...)
|
||||
}
|
||||
}
|
||||
|
||||
// Tokens are escaped as a key's name is (§73); a token that is empty, `.` or `..` is refused.
|
||||
func token(s string) (string, error) {
|
||||
if s == "" || s == "." || s == ".." {
|
||||
return "", fmt.Errorf("%q is not a name", s)
|
||||
}
|
||||
return keys.EscapeName(s), nil
|
||||
}
|
||||
|
||||
func (e *Engine) pointDir(module, kept string, form int64, fp string) (string, error) {
|
||||
m, err := token(module)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
k, err := token(kept)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
f, err := token(fp)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filepath.Join(e.Dir, "points", m, k, fmt.Sprintf("%d-%s", form, f)), nil
|
||||
}
|
||||
|
||||
func writeJSON(path string, v any) error {
|
||||
raw, err := json.MarshalIndent(v, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := path + ".partial"
|
||||
if err := os.WriteFile(tmp, raw, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, path)
|
||||
}
|
||||
|
||||
func readJSON(path string, v any) error {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.Unmarshal(raw, v)
|
||||
}
|
||||
|
||||
func dirSize(dir string) int64 {
|
||||
var n int64
|
||||
_ = filepath.WalkDir(dir, func(_ string, d os.DirEntry, err error) error {
|
||||
if err == nil && !d.IsDir() {
|
||||
if info, err := d.Info(); err == nil {
|
||||
n += info.Size()
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return n
|
||||
}
|
||||
|
||||
var errBusy = errors.New("busy")
|
||||
|
||||
// Accept is the one door of every acting verb (§554, §682): it writes the act in the hand-act log first (a failed
|
||||
// write does nothing and answers an error naming the log), then answers busy with the running act's id, or writes the
|
||||
// act in its directory and answers accepted; the work runs on, and its result is the event.
|
||||
func (e *Engine) Accept(a Act) Answer {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
if e.stop != "" {
|
||||
return Answer{Error: e.stop}
|
||||
}
|
||||
if e.running != nil {
|
||||
return Answer{Busy: e.running.ID}
|
||||
}
|
||||
if err := e.Seats.HandAct(a.Caller, a.Asker, a.Kind+" "+a.Module+" "+a.Kept, a.Why); err != nil {
|
||||
return Answer{Error: "the hand-act log was not written, so nothing was done: " + err.Error()}
|
||||
}
|
||||
e.seq++
|
||||
a.ID = fmt.Sprintf("act-%s-%06d", e.now().Format("20060102T150405Z"), e.seq)
|
||||
a.Since = e.now()
|
||||
if err := os.MkdirAll(filepath.Join(e.Dir, "acts"), 0o700); err != nil {
|
||||
return Answer{Error: err.Error()}
|
||||
}
|
||||
if err := writeJSON(filepath.Join(e.Dir, "acts", a.ID+".json"), a); err != nil {
|
||||
return Answer{Error: "the act cannot be written: " + err.Error()}
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
e.running, e.cancel = &a, cancel
|
||||
e.wg.Add(1)
|
||||
go func() {
|
||||
defer e.wg.Done()
|
||||
e.run(ctx, &a)
|
||||
e.mu.Lock()
|
||||
if e.running == &a { // an act that said its ending event is no longer the running one (sayEnd)
|
||||
e.running, e.cancel = nil, nil
|
||||
}
|
||||
e.mu.Unlock()
|
||||
cancel()
|
||||
e.judge()
|
||||
}()
|
||||
return Answer{Accepted: a.ID}
|
||||
}
|
||||
|
||||
// Wait returns once the running act, if any, has ended (a whole test's and a stop's join).
|
||||
func (e *Engine) Wait() { e.wg.Wait() }
|
||||
|
||||
// Stop is the process's stop, a move's or a restart's (§562, A-141): from now on every call is answered an error naming the
|
||||
// stop, the running act's context is cancelled so it ends through its own failure path (a restore past its removal puts
|
||||
// the aside back and sets the consumers back), and once it has ended every accepted act not ended is ended failed with
|
||||
// `stopped` and its ending event, so its caller asks again.
|
||||
func (e *Engine) Stop(why string) {
|
||||
e.mu.Lock()
|
||||
e.stop = "the recovery module is stopping (" + why + "), so it takes no call; ask again once it runs"
|
||||
if e.cancel != nil {
|
||||
e.cancel()
|
||||
}
|
||||
e.mu.Unlock()
|
||||
e.wg.Wait()
|
||||
e.endUnended()
|
||||
}
|
||||
|
||||
// Stopping is the error a call is answered while the process stops, empty while it runs.
|
||||
func (e *Engine) Stopping() string {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
return e.stop
|
||||
}
|
||||
|
||||
// stoppedOr is an act's failure as its ending event says it: `stopped` once the process stops (§562).
|
||||
func (e *Engine) stoppedOr(err error) error {
|
||||
if err != nil && e.Stopping() != "" {
|
||||
return errors.New(ErrStopped)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (e *Engine) setStep(a *Act, step, waitsOn string) {
|
||||
e.mu.Lock()
|
||||
a.Step, a.WaitsOn = step, waitsOn
|
||||
e.mu.Unlock()
|
||||
}
|
||||
|
||||
func (e *Engine) markEnded(a *Act) {
|
||||
a.Ended = true
|
||||
_ = writeJSON(filepath.Join(e.Dir, "acts", a.ID+".json"), a)
|
||||
}
|
||||
|
||||
func jsonMarshal(v any) ([]byte, error) { return json.Marshal(v) }
|
||||
|
||||
func unmarshal(raw []byte, v any) error { return json.Unmarshal(raw, v) }
|
||||
@@ -0,0 +1,370 @@
|
||||
package recovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
const asideFile = "aside.json"
|
||||
|
||||
// The steps an aside's record says, in their order (§516): a restore that fails after `removed` puts the aside back.
|
||||
const (
|
||||
stepSetAside = "set aside"
|
||||
stepRemoved = "removed"
|
||||
stepRestored = "restored"
|
||||
)
|
||||
|
||||
// asideRecord is the record in an aside's directory (§516); nothing of an aside is on the bus.
|
||||
type asideRecord struct {
|
||||
ID string `json:"id"`
|
||||
Module string `json:"module"`
|
||||
Kept string `json:"kept"`
|
||||
Streams []string `json:"streams"`
|
||||
Consumers []seat.Consumer `json:"consumers,omitempty"`
|
||||
Digests []seat.Digest `json:"digests"`
|
||||
Size int64 `json:"size"`
|
||||
Step string `json:"step"`
|
||||
}
|
||||
|
||||
func (r asideRecord) aside() seat.Aside {
|
||||
return seat.Aside{ID: r.ID, Names: r.Streams, Size: r.Size}
|
||||
}
|
||||
|
||||
func (e *Engine) asideRoot(module, kept string) (string, error) {
|
||||
m, err := token(module)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
k, err := token(kept)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filepath.Join(e.Dir, "aside", m, k), nil
|
||||
}
|
||||
|
||||
// endedRestored says the end of a restore, or of a restore-aside, under the act's id (§560).
|
||||
func (e *Engine) endedRestored(a *Act, outcomeErr error, aside *seat.Aside, missing bool) error {
|
||||
ev := seat.RestoredEvent{Act: a.ID, At: e.now(), Module: a.Module, Kept: a.Kept, Form: a.Form, Fingerprint: a.Fingerprint,
|
||||
Snapshot: a.Snapshot, Outcome: seat.RestoredEventOutcomeDone, Aside: aside, Missing: missing}
|
||||
outcomeErr = e.stoppedOr(outcomeErr)
|
||||
if outcomeErr != nil {
|
||||
ev.Outcome, ev.Error, ev.Why = seat.RestoredEventOutcomeFailed, outcomeErr.Error(), e.cancelledWhy(outcomeErr)
|
||||
}
|
||||
e.sayEnd(a.ID, seat.EventRestored, ev, a.Module, a.Kept)
|
||||
return outcomeErr
|
||||
}
|
||||
|
||||
func (e *Engine) cancelledWhy(err error) string {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
if err != nil && err.Error() == ErrCanceled {
|
||||
return e.why
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// restore is `restore-kept` and the hand verb `restore` with a point (or, past the dump, a keeper's snapshot by request) (§516, §554, §563, §567): it says restore-started,
|
||||
// waits for the dump of the bus's item, finds the point (or has the backup holder bring it back), checks its digests
|
||||
// against the bus, sets the live data aside whole and checked, removes it, restores the point under its names and
|
||||
// sets each consumer to its recorded sequence; a step after the removal that fails puts the aside back.
|
||||
func (e *Engine) restore(ctx context.Context, a *Act) error {
|
||||
// Said means acknowledged: the dump's item events are read only once the bus took restore-started (§563).
|
||||
if err := e.Seats.Say(seat.EventRestoreStarted, seat.RestoreStartedEvent{Act: a.ID, At: e.now(), Module: a.Module, Kept: a.Kept, Form: a.Form,
|
||||
Fingerprint: a.Fingerprint, Snapshot: a.Snapshot}, a.Module, a.Kept); err != nil {
|
||||
return e.endedRestored(a, fmt.Errorf("restore-started was not acknowledged by the bus, so nothing was read or set aside: %w", err), nil, false)
|
||||
}
|
||||
if e.Dump != nil {
|
||||
waiting, done := e.waitsOn(a)
|
||||
err := e.Dump.Wait(ctx, waiting)
|
||||
done()
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
err = errors.New(ErrCanceled)
|
||||
}
|
||||
return e.endedRestored(a, err, nil, false)
|
||||
}
|
||||
}
|
||||
if a.Request != "" {
|
||||
return e.restoreRequest(ctx, a)
|
||||
}
|
||||
e.setStep(a, "finding the point", "")
|
||||
pointPath := filepath.Join(e.Dir, "points", filepath.FromSlash(a.Snapshot))
|
||||
if _, err := os.Stat(filepath.Join(pointPath, pointFile)); err != nil {
|
||||
if !a.FromBackup {
|
||||
return e.endedRestored(a, errors.New(ErrMissing), nil, true)
|
||||
}
|
||||
dir, err := e.Seats.Backup(ctx, Module)
|
||||
if err != nil {
|
||||
return e.endedRestored(a, errors.New(ErrMissing), nil, true)
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
pointPath = filepath.Join(dir, filepath.FromSlash(a.Snapshot))
|
||||
if _, err := os.Stat(filepath.Join(pointPath, pointFile)); err != nil {
|
||||
return e.endedRestored(a, errors.New(ErrMissing), nil, true)
|
||||
}
|
||||
}
|
||||
var p seat.Point
|
||||
if err := readJSON(filepath.Join(pointPath, pointFile), &p); err != nil {
|
||||
return e.endedRestored(a, fmt.Errorf("%s: %w", ErrDamaged, err), nil, false)
|
||||
}
|
||||
if err := e.checkDigests(pointPath, a.Snapshot, p); err != nil {
|
||||
return e.endedRestored(a, err, nil, false)
|
||||
}
|
||||
names := make([]string, 0, len(p.Digests))
|
||||
for _, d := range p.Digests {
|
||||
names = append(names, d.Name)
|
||||
}
|
||||
rec, err := e.setAside(ctx, a, names, true)
|
||||
if err != nil {
|
||||
return e.endedRestored(a, err, nil, false)
|
||||
}
|
||||
// Past the removal, a failure puts the aside back (§516).
|
||||
e.setStep(a, "restoring", "")
|
||||
if err := e.putBack(ctx, pointPath, p.Digests, p.Consumers); err != nil {
|
||||
if back := e.restoreFromAside(ctx, rec, true); back != nil {
|
||||
err = fmt.Errorf("%w; and the aside is not put back: %v", err, back)
|
||||
}
|
||||
return e.endedRestored(a, err, asidePtr(rec), false)
|
||||
}
|
||||
rec.Step = stepRestored
|
||||
_ = writeJSON(filepath.Join(e.asideDirOf(rec), asideFile), rec)
|
||||
return e.endedRestored(a, nil, asidePtr(rec), false)
|
||||
}
|
||||
|
||||
func asidePtr(r *asideRecord) *seat.Aside {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
x := r.aside()
|
||||
return &x
|
||||
}
|
||||
|
||||
func (e *Engine) asideDirOf(r *asideRecord) string {
|
||||
root, _ := e.asideRoot(r.Module, r.Kept)
|
||||
return filepath.Join(root, r.ID)
|
||||
}
|
||||
|
||||
// checkDigests computes the digest of the bytes about to be restored and compares it with the key on the bus; the
|
||||
// record in the directory is never the reference (§567).
|
||||
func (e *Engine) checkDigests(pointPath, snapshot string, p seat.Point) error {
|
||||
raw, ok, err := e.Digests.Get(keys.EscapeName(snapshot))
|
||||
if err != nil {
|
||||
return fmt.Errorf("the digests of %s cannot be read: %w", snapshot, err)
|
||||
}
|
||||
if !ok {
|
||||
return fmt.Errorf("%s: %s has no digests on the bus", ErrDamaged, snapshot)
|
||||
}
|
||||
var want []seat.Digest
|
||||
if err := unmarshal(raw, &want); err != nil {
|
||||
return fmt.Errorf("%s: %s: %w", ErrDamaged, snapshot, err)
|
||||
}
|
||||
if len(want) != len(p.Digests) {
|
||||
return fmt.Errorf("%s: %s holds %d object(s), the bus says %d", ErrDamaged, snapshot, len(p.Digests), len(want))
|
||||
}
|
||||
for _, w := range want {
|
||||
got, _, err := digestOf(filepath.Join(pointPath, keys.EscapeName(w.Name)+".snap"))
|
||||
if err != nil || got != w.Digest {
|
||||
return fmt.Errorf("%s: %s, bus object %s", ErrDamaged, snapshot, w.Name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// setAside takes a snapshot of each live object into aside/<module>/<kept>/<at>, reads it back to check it, and only then
|
||||
// (when remove) removes the live objects, each step written first in the record (§516). Past the cap it refuses before
|
||||
// any object is removed (§564).
|
||||
func (e *Engine) setAside(ctx context.Context, a *Act, names []string, remove bool) (*asideRecord, error) {
|
||||
e.setStep(a, "setting aside", "")
|
||||
objs, err := e.Bus.Objects(ctx, names)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var need int64
|
||||
for _, o := range objs {
|
||||
need += o.Bytes
|
||||
}
|
||||
if fill := e.filled(); fill+need > e.Settings.Cap {
|
||||
return nil, fmt.Errorf("%s: the cap is %d bytes, the aside needs %d and the directory holds %d", ErrCap, e.Settings.Cap, need, fill)
|
||||
}
|
||||
root, err := e.asideRoot(a.Module, a.Kept)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
id := e.now().Format("20060102T150405Z")
|
||||
for n := 2; ; n++ {
|
||||
if _, err := os.Stat(filepath.Join(root, id)); err != nil {
|
||||
break
|
||||
}
|
||||
id = fmt.Sprintf("%s-%d", e.now().Format("20060102T150405Z"), n)
|
||||
}
|
||||
dir := filepath.Join(root, id)
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rec := &asideRecord{ID: id, Module: a.Module, Kept: a.Kept, Step: stepSetAside}
|
||||
fail := func(err error) (*asideRecord, error) { _ = os.RemoveAll(dir); return nil, err }
|
||||
for _, o := range objs {
|
||||
if !o.Exists {
|
||||
continue
|
||||
}
|
||||
sum, n, err := e.takeOne(ctx, o.Stream, filepath.Join(dir, keys.EscapeName(o.Stream)+".snap"))
|
||||
if err != nil {
|
||||
return fail(err)
|
||||
}
|
||||
if again, _, err := digestOf(filepath.Join(dir, keys.EscapeName(o.Stream)+".snap")); err != nil || again != sum {
|
||||
return fail(fmt.Errorf("the aside of %s does not read back whole", o.Stream))
|
||||
}
|
||||
rec.Streams = append(rec.Streams, o.Stream)
|
||||
rec.Digests = append(rec.Digests, seat.Digest{Name: o.Stream, Digest: sum})
|
||||
rec.Consumers = append(rec.Consumers, o.Consumers...)
|
||||
rec.Size += n
|
||||
}
|
||||
if err := writeJSON(filepath.Join(dir, asideFile), rec); err != nil {
|
||||
return fail(err)
|
||||
}
|
||||
if !remove {
|
||||
return rec, nil
|
||||
}
|
||||
rec.Step = stepRemoved
|
||||
if err := writeJSON(filepath.Join(dir, asideFile), rec); err != nil {
|
||||
return fail(err)
|
||||
}
|
||||
for _, s := range rec.Streams {
|
||||
if err := e.Bus.Delete(ctx, s); err != nil {
|
||||
if back := e.restoreFromAside(ctx, rec, false); back != nil {
|
||||
return rec, fmt.Errorf("%w; and the aside is not put back: %v", err, back)
|
||||
}
|
||||
return rec, err
|
||||
}
|
||||
}
|
||||
return rec, nil
|
||||
}
|
||||
|
||||
// putBack restores each stream of a point under its own name and sets each consumer to its recorded sequence.
|
||||
func (e *Engine) putBack(ctx context.Context, dir string, ds []seat.Digest, cs []seat.Consumer) error {
|
||||
for _, d := range ds {
|
||||
if err := e.Bus.Restore(ctx, filepath.Join(dir, keys.EscapeName(d.Name)+".snap")); err != nil {
|
||||
return fmt.Errorf("stream %s: %w", d.Name, err)
|
||||
}
|
||||
}
|
||||
return e.Bus.SetConsumers(ctx, cs)
|
||||
}
|
||||
|
||||
// restoreFromAside puts an aside back under the live names: any partly restored stream is removed first. It is a failure
|
||||
// path, so it runs whole also when the act's context ended, as at the process's stop (§562).
|
||||
func (e *Engine) restoreFromAside(ctx context.Context, rec *asideRecord, clear bool) error {
|
||||
ctx = context.WithoutCancel(ctx)
|
||||
if clear {
|
||||
for _, s := range rec.Streams {
|
||||
if err := e.Bus.Delete(ctx, s); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := e.putBack(ctx, e.asideDirOf(rec), rec.Digests, rec.Consumers); err != nil {
|
||||
return err
|
||||
}
|
||||
rec.Step = stepRestored
|
||||
return writeJSON(filepath.Join(e.asideDirOf(rec), asideFile), rec)
|
||||
}
|
||||
|
||||
func (e *Engine) readAside(module, kept, id string) (*asideRecord, error) {
|
||||
root, err := e.asideRoot(module, kept)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if id == "" || id == "." || id == ".." || strings.ContainsAny(id, "/") {
|
||||
return nil, fmt.Errorf("%q is not an aside", id)
|
||||
}
|
||||
var rec asideRecord
|
||||
if err := readJSON(filepath.Join(root, id, asideFile), &rec); err != nil {
|
||||
return nil, fmt.Errorf("no aside %s of %s's %s", id, module, kept)
|
||||
}
|
||||
return &rec, nil
|
||||
}
|
||||
|
||||
// aside is the hand verb `aside` (§516, §554, A-123): it sets the live kept data aside without restoring it, taking the bus
|
||||
// names from the catalog entry of the fingerprint the node-engine's placement runs for the module, and removes the live
|
||||
// objects once the aside is whole and checked (a move, not a removal: the data stays in the aside until it is discarded or
|
||||
// put back by `restore-aside`); it ends with `set-aside`.
|
||||
func (e *Engine) aside(ctx context.Context, a *Act) error {
|
||||
fp, err := e.Seats.RunningFingerprint(ctx, a.Module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
a.Fingerprint = fp
|
||||
names, err := e.Seats.Streams(ctx, a.Module, a.Kept, fp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rec, err := e.setAside(ctx, a, names, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
x := rec.aside()
|
||||
e.sayEnd(a.ID, seat.EventSetAside, seat.SetAsideEvent{Act: a.ID, At: e.now(), ID: x.ID, Names: x.Names, Size: x.Size}, a.Module, a.Kept)
|
||||
return nil
|
||||
}
|
||||
|
||||
// restoreAside is `restore-aside` (§566): it puts an aside whose record ends at `removed` back under the live names it
|
||||
// was taken from, refusing while a live object of those names exists, and ends with `restored`.
|
||||
func (e *Engine) restoreAside(ctx context.Context, a *Act) error {
|
||||
rec, err := e.readAside(a.Module, a.Kept, a.AsideID)
|
||||
if err != nil {
|
||||
return e.endedRestored(a, err, nil, false)
|
||||
}
|
||||
objs, err := e.Bus.Objects(ctx, rec.Streams)
|
||||
if err != nil {
|
||||
return e.endedRestored(a, err, nil, false)
|
||||
}
|
||||
for _, o := range objs {
|
||||
if o.Exists {
|
||||
return e.endedRestored(a, fmt.Errorf("the live stream %s exists; the aside %s is not put back over it", o.Stream, rec.ID), asidePtr(rec), false)
|
||||
}
|
||||
}
|
||||
if err := e.restoreFromAside(ctx, rec, false); err != nil {
|
||||
return e.endedRestored(a, err, asidePtr(rec), false)
|
||||
}
|
||||
return e.endedRestored(a, nil, asidePtr(rec), false)
|
||||
}
|
||||
|
||||
// discardAside is `discard-aside` (§554): it removes that directory and nothing else, an id holding `.`, `..` or `/`
|
||||
// refused; it ends with `aside-discarded`.
|
||||
func (e *Engine) discardAside(a *Act) error {
|
||||
rec, err := e.readAside(a.Module, a.Kept, a.AsideID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.RemoveAll(e.asideDirOf(rec)); err != nil {
|
||||
return err
|
||||
}
|
||||
x := rec.aside()
|
||||
e.sayEnd(a.ID, seat.EventAsideDiscarded, seat.AsideDiscardedEvent{Act: a.ID, At: e.now(), ID: x.ID, Names: x.Names, Size: x.Size}, a.Module, a.Kept)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Asides lists every aside in the directory (§517).
|
||||
func (e *Engine) Asides() ([]seat.Aside, []asideRecord, error) {
|
||||
var out []seat.Aside
|
||||
var recs []asideRecord
|
||||
_ = filepath.WalkDir(filepath.Join(e.Dir, "aside"), func(path string, d os.DirEntry, err error) error {
|
||||
if err == nil && !d.IsDir() && d.Name() == asideFile {
|
||||
var r asideRecord
|
||||
if readJSON(path, &r) == nil {
|
||||
out = append(out, r.aside())
|
||||
recs = append(recs, r)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
|
||||
return out, recs, nil
|
||||
}
|
||||
@@ -0,0 +1,349 @@
|
||||
package recovery_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
|
||||
|
||||
"mesh-recovery/internal/recovery"
|
||||
)
|
||||
|
||||
func restoreAct(snapshot string) recovery.Act {
|
||||
return recovery.Act{Kind: recovery.KindRestore, Module: "app", Kept: "notes", Form: 2, Fingerprint: "fp1", Snapshot: snapshot, Why: "revert", Caller: "node.engine"}
|
||||
}
|
||||
|
||||
func takenRig(t *testing.T) *rig {
|
||||
r := newRig(t)
|
||||
r.do(takeAct())
|
||||
r.s.events = nil
|
||||
return r
|
||||
}
|
||||
|
||||
func lastRestored(t *testing.T, r *rig) seat.RestoredEvent {
|
||||
t.Helper()
|
||||
ev := r.s.named(seat.EventRestored)
|
||||
if len(ev) != 1 {
|
||||
t.Fatalf("restored events %+v", r.s.events)
|
||||
}
|
||||
return ev[0].Body.(seat.RestoredEvent)
|
||||
}
|
||||
|
||||
// A restore says restore-started, sets the live data aside whole, removes it, restores the point under its names and sets
|
||||
// each consumer to its recorded sequence, and says restored with the aside (design 47 G §516, §560).
|
||||
func TestARestoreSetsTheLiveDataAsideAndRestoresThePoint(t *testing.T) {
|
||||
r := takenRig(t)
|
||||
r.b.streams["app_notes"] = "changed since"
|
||||
ans := r.do(restoreAct("app/notes/2-fp1"))
|
||||
ev := lastRestored(t, r)
|
||||
if ev.Outcome != "done" || ev.Act != ans.Accepted || ev.Aside == nil || ev.Aside.Size == 0 {
|
||||
t.Fatalf("restored %+v", ev)
|
||||
}
|
||||
if r.b.streams["app_notes"] != "hello" || r.b.streams["KV_app_seen"] != "world" {
|
||||
t.Errorf("streams %v", r.b.streams)
|
||||
}
|
||||
if len(r.b.setBack) != 1 || r.b.setBack[0].Sequence != 7 {
|
||||
t.Errorf("consumers %+v", r.b.setBack)
|
||||
}
|
||||
if len(r.s.named(seat.EventRestoreStarted)) != 1 {
|
||||
t.Errorf("no restore-started")
|
||||
}
|
||||
kp, _ := r.e.KeptPoints("app", "notes")
|
||||
if len(kp.Asides) != 1 || len(kp.Points) != 1 || kp.Running != nil {
|
||||
t.Errorf("kept points %+v", kp)
|
||||
}
|
||||
}
|
||||
|
||||
// A point missing from the directory says missing; with from-backup the backup holder brings it back and it is restored
|
||||
// once its digest is checked; when no repository snapshot holds it, restore point missing (§516, §567).
|
||||
func TestAMissingPointIsBroughtBackOrSaidMissing(t *testing.T) {
|
||||
r := takenRig(t)
|
||||
back := filepath.Join(t.TempDir(), "back")
|
||||
if err := os.Rename(filepath.Join(r.e.Dir, "points"), back); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.do(restoreAct("app/notes/2-fp1"))
|
||||
if ev := lastRestored(t, r); !ev.Missing || ev.Outcome != "failed" || ev.Error != recovery.ErrMissing {
|
||||
t.Errorf("no backup asked: %+v", ev)
|
||||
}
|
||||
r.s.events = nil
|
||||
a := restoreAct("app/notes/2-fp1")
|
||||
a.FromBackup = true
|
||||
r.do(a)
|
||||
if ev := lastRestored(t, r); !ev.Missing || ev.Error != recovery.ErrMissing {
|
||||
t.Errorf("backup holder holds none: %+v", ev)
|
||||
}
|
||||
r.s.events, r.s.backup = nil, back
|
||||
r.do(a)
|
||||
if ev := lastRestored(t, r); ev.Outcome != "done" {
|
||||
t.Errorf("brought back: %+v", ev)
|
||||
}
|
||||
}
|
||||
|
||||
// A point whose bytes differ from the digests on the bus is refused before any aside is taken, the record in the
|
||||
// directory never being the reference (§567).
|
||||
func TestAPointWhoseBytesWereAlteredIsRefusedBeforeAnyAside(t *testing.T) {
|
||||
r := takenRig(t)
|
||||
snap := filepath.Join(r.e.Dir, "points", "app", "notes", "2-fp1", "app_notes.snap")
|
||||
if err := os.WriteFile(snap, []byte("snap:app_notes:evil"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.do(restoreAct("app/notes/2-fp1"))
|
||||
if ev := lastRestored(t, r); ev.Outcome != "failed" || !strings.HasPrefix(ev.Error, recovery.ErrDamaged) {
|
||||
t.Errorf("restored %+v", ev)
|
||||
}
|
||||
if kp, _ := r.e.KeptPoints("", ""); len(kp.Asides) != 0 || r.b.streams["app_notes"] != "hello" {
|
||||
t.Errorf("an aside was taken or the live data changed: %+v %v", kp, r.b.streams)
|
||||
}
|
||||
}
|
||||
|
||||
// A step after the removal that fails puts the aside back under the live names and the act ends failed, so the kept data
|
||||
// is never left gone (§516).
|
||||
func TestARestoreThatFailsAfterTheRemovalPutsTheAsideBack(t *testing.T) {
|
||||
r := takenRig(t)
|
||||
r.b.streams["app_notes"] = "live now"
|
||||
r.b.failRestores = 1
|
||||
r.do(restoreAct("app/notes/2-fp1"))
|
||||
if ev := lastRestored(t, r); ev.Outcome != "failed" || ev.Aside == nil {
|
||||
t.Fatalf("restored %+v", ev)
|
||||
}
|
||||
if r.b.streams["app_notes"] != "live now" || r.b.streams["KV_app_seen"] != "world" {
|
||||
t.Errorf("live data not as before: %v", r.b.streams)
|
||||
}
|
||||
if n := len(r.e.Unrestored()); n != 0 {
|
||||
t.Errorf("%d asides unrestored", n)
|
||||
}
|
||||
}
|
||||
|
||||
// A restore past the cap is refused with the live data untouched (§564).
|
||||
func TestARestoreWhoseAsideWouldPassTheCapIsRefusedBeforeTheRemoval(t *testing.T) {
|
||||
r := takenRig(t)
|
||||
r.e.Settings.Cap = 1
|
||||
r.do(restoreAct("app/notes/2-fp1"))
|
||||
if ev := lastRestored(t, r); ev.Outcome != "failed" || !strings.HasPrefix(ev.Error, recovery.ErrCap) {
|
||||
t.Errorf("restored %+v", ev)
|
||||
}
|
||||
if r.b.streams["app_notes"] != "hello" {
|
||||
t.Errorf("live data touched: %v", r.b.streams)
|
||||
}
|
||||
}
|
||||
|
||||
// An act waiting on the dump is cancelled with its ending event failed/cancelled and the hand act's words as why; an act
|
||||
// that has begun is refused, naming its step (§566, A-117).
|
||||
func TestAWaitingActIsCancelledAndABegunOneRefused(t *testing.T) {
|
||||
r := takenRig(t)
|
||||
hd := holdDump{waiting: make(chan string, 1)}
|
||||
r.e.Dump = hd
|
||||
// Past the bound (a setting, §557) the waiting act writes recovery-waits, cleared once it ends (§561).
|
||||
fs := newHeldFindings()
|
||||
r.e.Findings, r.e.Settings.DumpWait = fs, time.Millisecond
|
||||
ans := r.e.Accept(restoreAct("app/notes/2-fp1"))
|
||||
<-hd.waiting
|
||||
if f := <-fs.written; f.Kind != recovery.FindingRecoveryWaits || f.Scope != "app" || f.ID != ans.Accepted {
|
||||
t.Errorf("written %+v", f)
|
||||
}
|
||||
kp, _ := r.e.KeptPoints("", "")
|
||||
if kp.Running == nil || kp.Running.WaitsOn == "" || kp.Running.Act != ans.Accepted {
|
||||
t.Errorf("running %+v", kp.Running)
|
||||
}
|
||||
if c := r.e.Cancel("act-nothing", "x", "tools", "op"); c.Error == "" {
|
||||
t.Errorf("an unknown act cancelled: %+v", c)
|
||||
}
|
||||
if c := r.e.Cancel(ans.Accepted, "not now", "tools", "op"); c.Accepted != ans.Accepted {
|
||||
t.Errorf("cancel %+v", c)
|
||||
}
|
||||
r.e.Wait()
|
||||
ev := lastRestored(t, r)
|
||||
if ev.Outcome != "failed" || ev.Error != recovery.ErrCanceled || ev.Why != "not now" || ev.Act != ans.Accepted {
|
||||
t.Errorf("restored %+v", ev)
|
||||
}
|
||||
if fs.has(recovery.FindingRecoveryWaits) {
|
||||
t.Errorf("recovery-waits stands after the act ended: %+v", fs.Held())
|
||||
}
|
||||
// The wait's bound is a key of the state bounds, its expiry what wrote recovery-waits (A-143).
|
||||
if w, rm := r.bounds.keys(); w != "dump-wait."+ans.Accepted+" 1ms" || rm != "dump-wait."+ans.Accepted+" expired" {
|
||||
t.Errorf("bound keys written %q, removed %q", w, rm)
|
||||
}
|
||||
}
|
||||
|
||||
// retry asks again the last failed act with its fields under an act of its own, and answers `nothing to retry` when its
|
||||
// last act did not fail (§682).
|
||||
func TestRetryAsksTheLastFailedActAgain(t *testing.T) {
|
||||
r := takenRig(t)
|
||||
r.e.Settings.Cap = 1
|
||||
r.do(restoreAct("app/notes/2-fp1"))
|
||||
r.e.Settings.Cap = 1 << 20
|
||||
r.s.events = nil
|
||||
ans := r.e.Retry("app", "notes", "again", "tools", "op")
|
||||
r.e.Wait()
|
||||
if ev := lastRestored(t, r); ans.Accepted == "" || ev.Outcome != "done" || ev.Act != ans.Accepted {
|
||||
t.Errorf("answer %+v restored %+v", ans, ev)
|
||||
}
|
||||
if again := r.e.Retry("app", "notes", "again", "tools", "op"); again.Error != "nothing to retry" {
|
||||
t.Errorf("answer %+v", again)
|
||||
}
|
||||
}
|
||||
|
||||
// An aside whose record ends at removed is put back at the start, an act cut off by a restart ends failed with stopped,
|
||||
// and a point left under a temporary name is removed (§516, §560, §564).
|
||||
func TestAStartPutsBackAnAsideEndsACutOffActAndClearsTemporaryPoints(t *testing.T) {
|
||||
r := takenRig(t)
|
||||
r.do(restoreAct("app/notes/2-fp1")) // leaves an aside at "restored"
|
||||
_, recs, _ := r.e.Asides()
|
||||
if len(recs) != 1 {
|
||||
t.Fatal("no aside")
|
||||
}
|
||||
// Cut a restore off after its removal: the record says removed, the live data is gone, an accepted act never ended.
|
||||
dir := filepath.Join(r.e.Dir, "aside", "app", "notes", recs[0].ID)
|
||||
raw, _ := os.ReadFile(filepath.Join(dir, "aside.json"))
|
||||
_ = os.WriteFile(filepath.Join(dir, "aside.json"), []byte(strings.Replace(string(raw), `"step": "restored"`, `"step": "removed"`, 1)), 0o600)
|
||||
delete(r.b.streams, "app_notes")
|
||||
delete(r.b.streams, "KV_app_seen")
|
||||
tmp := filepath.Join(r.e.Dir, "points", "app", "notes", ".tmp-act-x")
|
||||
_ = os.MkdirAll(tmp, 0o700)
|
||||
_ = os.WriteFile(filepath.Join(r.e.Dir, "acts", "act-cut.json"), []byte(`{"act":"act-cut","kind":"restore","module":"app","kept":"notes","why":"w","caller":"c","since":"2026-11-12T00:00:00Z"}`), 0o600)
|
||||
r.s.events = nil
|
||||
if left := r.e.Start(context.Background()); len(left) != 0 {
|
||||
t.Errorf("unrestored %+v", left)
|
||||
}
|
||||
if r.b.streams["app_notes"] != "hello" {
|
||||
t.Errorf("the aside was not put back: %v", r.b.streams)
|
||||
}
|
||||
if _, err := os.Stat(tmp); err == nil {
|
||||
t.Errorf("the temporary point stays")
|
||||
}
|
||||
if ev := lastRestored(t, r); ev.Act != "act-cut" || ev.Error != recovery.ErrStopped {
|
||||
t.Errorf("restored %+v", ev)
|
||||
}
|
||||
}
|
||||
|
||||
// restore-aside puts an aside back under the live names, refused while live objects of those names exist; discard-aside
|
||||
// removes the aside's directory and says aside-discarded (§554, §566).
|
||||
func TestRestoreAsideRefusesLiveNamesAndDiscardAsideRemovesTheDirectory(t *testing.T) {
|
||||
r := takenRig(t)
|
||||
r.do(restoreAct("app/notes/2-fp1"))
|
||||
_, recs, _ := r.e.Asides()
|
||||
id := recs[0].ID
|
||||
r.s.events = nil
|
||||
r.do(recovery.Act{Kind: recovery.KindRestoreAside, Module: "app", Kept: "notes", AsideID: id, Why: "x", Caller: "tools"})
|
||||
if ev := lastRestored(t, r); ev.Outcome != "failed" || !strings.Contains(ev.Error, "live stream") {
|
||||
t.Errorf("restored %+v", ev)
|
||||
}
|
||||
r.s.events = nil
|
||||
r.do(recovery.Act{Kind: recovery.KindDiscardAside, Module: "app", Kept: "notes", AsideID: "../x", Why: "x", Caller: "tools"})
|
||||
if len(r.s.named(seat.EventAsideDiscarded)) != 0 {
|
||||
t.Errorf("a path outside was discarded")
|
||||
}
|
||||
r.do(recovery.Act{Kind: recovery.KindDiscardAside, Module: "app", Kept: "notes", AsideID: id, Why: "x", Caller: "tools"})
|
||||
if ev := r.s.named(seat.EventAsideDiscarded); len(ev) != 1 || ev[0].Body.(seat.AsideDiscardedEvent).ID != id {
|
||||
t.Errorf("events %+v", r.s.events)
|
||||
}
|
||||
if kp, _ := r.e.KeptPoints("", ""); len(kp.Asides) != 0 {
|
||||
t.Errorf("asides %+v", kp.Asides)
|
||||
}
|
||||
}
|
||||
|
||||
// The hand verb aside sets the live data aside whole and removes it, naming the streams as a list, and restore-aside puts
|
||||
// it back once no live name exists (§516, §554, §566, A-123, A-124).
|
||||
func TestAsideMovesTheLiveDataIntoAnAsideAndRestoreAsidePutsItBack(t *testing.T) {
|
||||
r := newRig(t)
|
||||
// The findings of §561: aside-unrestored while the aside alone holds the data, points-filling past the fill mark.
|
||||
fs := newHeldFindings()
|
||||
r.e.Findings, r.e.Node, r.e.Settings.FillingMark, r.e.Settings.Cap = fs, "srv", 1, 4096
|
||||
ans := r.do(recovery.Act{Kind: recovery.KindAside, Module: "app", Kept: "notes", Why: "look at it", Caller: "tools", Asker: "op"})
|
||||
ev := r.s.named(seat.EventSetAside)
|
||||
if ans.Accepted == "" || len(ev) != 1 {
|
||||
t.Fatalf("answer %+v events %+v", ans, r.s.events)
|
||||
}
|
||||
sa := ev[0].Body.(seat.SetAsideEvent)
|
||||
if sa.Act != ans.Accepted || len(sa.Names) != 2 || sa.Size == 0 || len(r.b.streams) != 0 {
|
||||
t.Errorf("set-aside %+v streams %v", sa, r.b.streams)
|
||||
}
|
||||
if got := r.e.Unrestored(); len(got) != 1 {
|
||||
t.Errorf("unrestored %+v", got)
|
||||
}
|
||||
// kept-points says it, which the tool and the health check read (A-129).
|
||||
if kp, _ := r.e.KeptPoints("app", ""); len(kp.Unrestored) != 1 || kp.Unrestored[0].ID != sa.ID {
|
||||
t.Errorf("kept-points unrestored %+v", kp.Unrestored)
|
||||
}
|
||||
if !fs.has(recovery.FindingAsideUnrestored) || !fs.has(recovery.FindingPointsFilling) {
|
||||
t.Errorf("findings %+v", fs.Held())
|
||||
}
|
||||
r.s.events = nil
|
||||
r.do(recovery.Act{Kind: recovery.KindRestoreAside, Module: "app", Kept: "notes", AsideID: sa.ID, Why: "done looking", Caller: "tools"})
|
||||
if lastRestored(t, r).Outcome != "done" || r.b.streams["app_notes"] != "hello" || len(r.e.Unrestored()) != 0 {
|
||||
t.Errorf("streams %v", r.b.streams)
|
||||
}
|
||||
r.e.Settings.FillingMark = 100
|
||||
r.do(recovery.Act{Kind: recovery.KindDiscardAside, Module: "app", Kept: "notes", AsideID: sa.ID, Why: "not needed", Caller: "tools"})
|
||||
if fs.has(recovery.FindingAsideUnrestored) || fs.has(recovery.FindingPointsFilling) {
|
||||
t.Errorf("findings stand after the aside was restored and discarded: %+v", fs.Held())
|
||||
}
|
||||
}
|
||||
|
||||
// Said means acknowledged: a restore whose restore-started the bus did not take reads nothing and sets nothing aside, and
|
||||
// ends failed (§563).
|
||||
func TestARestoreWhoseStartIsNotAcknowledgedTouchesNothing(t *testing.T) {
|
||||
r := takenRig(t)
|
||||
r.s.sayFail = map[string]error{seat.EventRestoreStarted: errTest}
|
||||
r.do(restoreAct("app/notes/2-fp1"))
|
||||
if ev := lastRestored(t, r); ev.Outcome != "failed" || !strings.Contains(ev.Error, "restore-started") || r.b.streams["app_notes"] != "hello" {
|
||||
t.Errorf("restored %+v, streams %v", ev, r.b.streams)
|
||||
}
|
||||
if kp, _ := r.e.KeptPoints("", ""); len(kp.Asides) != 0 {
|
||||
t.Errorf("an aside was taken: %+v", kp.Asides)
|
||||
}
|
||||
}
|
||||
|
||||
// stallBus blocks its first Restore until the act's context ends, as a restore under way when the process stops.
|
||||
type stallBus struct {
|
||||
*fakeBus
|
||||
in chan struct{}
|
||||
stalled bool
|
||||
}
|
||||
|
||||
func (s *stallBus) Restore(ctx context.Context, file string) error {
|
||||
if !s.stalled {
|
||||
s.stalled = true
|
||||
close(s.in)
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}
|
||||
return s.fakeBus.Restore(ctx, file)
|
||||
}
|
||||
|
||||
// A stop of the process (a move's or a restart's) answers every new call an error naming it, ends the running restore
|
||||
// through its failure path, putting the aside back, and ends every accepted act with `stopped`, so its caller asks again
|
||||
// (design 47 G §562, A-141).
|
||||
func TestAStopEndsTheRunningActThroughItsFailurePathAndStopped(t *testing.T) {
|
||||
r := takenRig(t)
|
||||
r.b.streams["app_notes"] = "changed since"
|
||||
stall := &stallBus{fakeBus: r.b, in: make(chan struct{})}
|
||||
r.e.Bus = stall
|
||||
ans := r.e.Accept(restoreAct("app/notes/2-fp1"))
|
||||
<-stall.in
|
||||
// An accepted act a crash left unended is ended by the stop too.
|
||||
_ = os.WriteFile(filepath.Join(r.e.Dir, "acts", "act-0.json"), []byte(`{"act":"act-0","kind":"take","module":"app","kept":"seen","why":"w","caller":"c","since":"2026-11-11T00:00:00Z"}`), 0o600)
|
||||
r.e.Stop("a move")
|
||||
if r.b.streams["app_notes"] != "changed since" || r.b.streams["KV_app_seen"] != "world" {
|
||||
t.Errorf("the aside is not put back: %v", r.b.streams)
|
||||
}
|
||||
if ev := lastRestored(t, r); ev.Act != ans.Accepted || ev.Outcome != "failed" || ev.Error != recovery.ErrStopped {
|
||||
t.Errorf("restored %+v", ev)
|
||||
}
|
||||
ended := r.s.named(seat.EventEnded)
|
||||
if len(ended) != 1 || ended[0].Body.(seat.EndedEvent).Act != "act-0" || ended[0].Body.(seat.EndedEvent).Error != recovery.ErrStopped {
|
||||
t.Errorf("ended %+v", ended)
|
||||
}
|
||||
if got := r.e.Accept(takeAct()); !strings.Contains(got.Error, "stopping (a move)") {
|
||||
t.Errorf("a call during the stop answered %+v", got)
|
||||
}
|
||||
verb := r.e.Handlers()[seat.VerbKeptPoints]
|
||||
if got, _ := verb("c", []byte(`{}`)); !strings.Contains(got.(recovery.Answer).Error, "stopping") {
|
||||
t.Errorf("kept-points during the stop answered %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package recovery
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// Handlers are the seat's verbs as the process answers them over the bus with the caller the bus proved (§554, §656, A-107):
|
||||
// every acting verb answers one of accepted, busy or error and nothing more (§691); the result is the event.
|
||||
func (e *Engine) Handlers() map[string]process.SeatHandler {
|
||||
h := map[string]process.SeatHandler{}
|
||||
h[seat.VerbSnapshotKept] = actOf(e, func(sk seat.SnapshotKeptInput) (Act, error) {
|
||||
return Act{Kind: KindTake, Module: sk.Module, Kept: sk.Kept, Form: sk.Form, Fingerprint: sk.Fingerprint, CatchUp: sk.CatchUp, Why: sk.Why}, nil
|
||||
})
|
||||
restoreKept := actOf(e, func(rk seat.RestoreKeptInput) (Act, error) {
|
||||
return Act{Kind: KindRestore, Module: rk.Module, Kept: rk.Kept, Form: rk.Form, Fingerprint: rk.Fingerprint, Snapshot: rk.Snapshot, FromBackup: rk.FromBackup, Why: rk.Why}, nil
|
||||
})
|
||||
h[seat.VerbRestoreKept] = func(caller string, body json.RawMessage) (any, error) {
|
||||
// aside-only and discard-aside are the operator's hand tools alone (§516, §554).
|
||||
var loose map[string]json.RawMessage
|
||||
_ = json.Unmarshal(body, &loose)
|
||||
for _, k := range []string{"aside-only", "discard-aside"} {
|
||||
if _, ok := loose[k]; ok {
|
||||
return Answer{Error: k + " is the operator's hand tool alone, not an argument of restore-kept"}, nil
|
||||
}
|
||||
}
|
||||
return restoreKept(caller, body)
|
||||
}
|
||||
h[seat.VerbKeptPoints] = func(_ string, body json.RawMessage) (any, error) {
|
||||
var in seat.KeptPointsInput
|
||||
if err := strict(body, &in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return e.KeptPoints(in.Module, in.Kept)
|
||||
}
|
||||
h[seat.VerbTake] = actOf(e, func(tk seat.TakeInput) (Act, error) {
|
||||
return Act{Kind: KindTake, Module: tk.Module, Kept: tk.Kept, Fingerprint: tk.Fingerprint, Why: tk.Why, Asker: tk.Asker}, nil
|
||||
})
|
||||
h[seat.VerbRestore] = actOf(e, func(rs seat.RestoreInput) (Act, error) {
|
||||
if (rs.Snapshot == "") == (rs.Request == "") {
|
||||
return Act{}, errors.New("restore takes exactly one of snapshot and request")
|
||||
}
|
||||
if rs.Request != "" && rs.FromBackup {
|
||||
return Act{}, errors.New("from-backup is taken only with snapshot, a restore point (design 47 G §682)")
|
||||
}
|
||||
return Act{Kind: KindRestore, Module: rs.Module, Kept: rs.Kept, Snapshot: rs.Snapshot, Request: rs.Request, FromBackup: rs.FromBackup, Why: rs.Why, Asker: rs.Asker}, nil
|
||||
})
|
||||
h[seat.VerbAside] = actOf(e, func(ad seat.AsideInput) (Act, error) {
|
||||
return Act{Kind: KindAside, Module: ad.Module, Kept: ad.Kept, Why: ad.Why, Asker: ad.Asker}, nil
|
||||
})
|
||||
h[seat.VerbDiscardAside] = actOf(e, func(da seat.DiscardAsideInput) (Act, error) {
|
||||
return Act{Kind: KindDiscardAside, Module: da.Module, Kept: da.Kept, AsideID: da.ID, Why: da.Why, Asker: da.Asker}, nil
|
||||
})
|
||||
h[seat.VerbRestoreAside] = actOf(e, func(ra seat.RestoreAsideInput) (Act, error) {
|
||||
return Act{Kind: KindRestoreAside, Module: ra.Module, Kept: ra.Kept, AsideID: ra.ID, Why: ra.Why, Asker: ra.Asker}, nil
|
||||
})
|
||||
h[seat.VerbExpirePoints] = actOf(e, func(ex seat.ExpirePointsInput) (Act, error) {
|
||||
return Act{Kind: KindExpire, Why: ex.Why, Asker: ex.Asker}, nil
|
||||
})
|
||||
h[seat.VerbRetry] = func(caller string, body json.RawMessage) (any, error) {
|
||||
var in seat.RetryInput
|
||||
if err := strict(body, &in); err != nil {
|
||||
return Answer{Error: err.Error()}, nil
|
||||
}
|
||||
return e.Retry(in.Module, in.Kept, in.Why, caller, in.Asker), nil
|
||||
}
|
||||
h[seat.VerbCancel] = func(caller string, body json.RawMessage) (any, error) {
|
||||
var in seat.CancelInput
|
||||
if err := strict(body, &in); err != nil {
|
||||
return Answer{Error: err.Error()}, nil
|
||||
}
|
||||
return e.Cancel(in.Act, in.Why, caller, in.Asker), nil
|
||||
}
|
||||
// A stopping process takes no call, each answered the error naming the stop (§562, A-141).
|
||||
for verb, handle := range h {
|
||||
h[verb] = func(caller string, body json.RawMessage) (any, error) {
|
||||
if stop := e.Stopping(); stop != "" {
|
||||
return Answer{Error: stop}, nil
|
||||
}
|
||||
return handle(caller, body)
|
||||
}
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// actOf is one acting verb: its arguments are read afresh for each call (calls come on concurrent bus callbacks), a call
|
||||
// without the `why` the hand-act log keeps is refused, and the act is accepted as the engine says.
|
||||
func actOf[T any](e *Engine, build func(T) (Act, error)) process.SeatHandler {
|
||||
return func(caller string, body json.RawMessage) (any, error) {
|
||||
var in T
|
||||
if err := strict(body, &in); err != nil {
|
||||
return Answer{Error: err.Error()}, nil
|
||||
}
|
||||
a, err := build(in)
|
||||
if err != nil {
|
||||
return Answer{Error: err.Error()}, nil
|
||||
}
|
||||
if a.Why == "" {
|
||||
return Answer{Error: "why is required: the hand-act log keeps it"}, nil
|
||||
}
|
||||
a.Caller = caller
|
||||
return e.Accept(a), nil
|
||||
}
|
||||
}
|
||||
|
||||
// strict reads a call's arguments, refusing a field the verb does not take (no `local-name`, no `from`: §516) and a call
|
||||
// that leaves out the `why` the hand-act log keeps (§682).
|
||||
func strict(body json.RawMessage, into any) error {
|
||||
dec := json.NewDecoder(bytes.NewReader(body))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(into); err != nil {
|
||||
return fmt.Errorf("the arguments are refused: %s", strings.TrimPrefix(err.Error(), "json: "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package recovery_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
|
||||
|
||||
"mesh-recovery/internal/recovery"
|
||||
)
|
||||
|
||||
func call(t *testing.T, r *rig, verb, body string) recovery.Answer {
|
||||
t.Helper()
|
||||
out, err := r.e.Handlers()[verb]("node.engine", json.RawMessage(body))
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", verb, err)
|
||||
}
|
||||
r.e.Wait()
|
||||
return out.(recovery.Answer)
|
||||
}
|
||||
|
||||
// Every verb of the seat's definition is answered; an acting verb answers one of accepted, busy and error and nothing more
|
||||
// (§691); an argument the verb does not take (`local-name`, `from`), `aside-only` and `discard-aside` on restore-kept, a
|
||||
// restore naming both a snapshot and a request, and a call without `why` are refused naming the fault (§516, §682, §554).
|
||||
func TestEveryVerbIsAnsweredAndAnActingVerbAnswersOneForm(t *testing.T) {
|
||||
r := newRig(t)
|
||||
h := r.e.Handlers()
|
||||
for _, v := range seat.Verbs() {
|
||||
if h[v] == nil {
|
||||
t.Errorf("verb %s has no handler", v)
|
||||
}
|
||||
}
|
||||
taken := call(t, r, "snapshot-kept", `{"module":"app","kept":"notes","form":2,"fingerprint":"fp1","why":"form change"}`)
|
||||
if taken.Accepted == "" || taken.Busy != "" || taken.Error != "" {
|
||||
t.Fatalf("answer %+v", taken)
|
||||
}
|
||||
for name, c := range map[string]struct{ verb, body, want string }{
|
||||
"local-name": {"snapshot-kept", `{"module":"app","kept":"notes","form":2,"fingerprint":"fp1","why":"w","local-name":"x"}`, "local-name"},
|
||||
"from": {"restore-kept", `{"module":"app","kept":"notes","form":2,"fingerprint":"fp1","why":"w","from":"x"}`, "from"},
|
||||
"aside-only": {"restore-kept", `{"module":"app","kept":"notes","form":2,"fingerprint":"fp1","why":"w","aside-only":true}`, "aside-only"},
|
||||
"discard": {"restore-kept", `{"module":"app","kept":"notes","form":2,"fingerprint":"fp1","why":"w","discard-aside":true}`, "discard-aside"},
|
||||
"both": {"restore", `{"module":"app","kept":"notes","snapshot":"a","request":"b","why":"w"}`, "exactly one"},
|
||||
"no why": {"restore", `{"module":"app","kept":"notes","snapshot":"a"}`, "why"},
|
||||
} {
|
||||
if got := call(t, r, c.verb, c.body); !strings.Contains(got.Error, c.want) || got.Accepted != "" {
|
||||
t.Errorf("%s: answer %+v", name, got)
|
||||
}
|
||||
}
|
||||
if len(r.s.named(seat.EventEnded)) != 1 {
|
||||
t.Errorf("a refused call acted: %+v", r.s.events)
|
||||
}
|
||||
out, err := h["kept-points"]("tools", json.RawMessage(`{"module":"app"}`))
|
||||
if kp, ok := out.(seat.KeptPointsAnswer); err != nil || !ok || len(kp.Points) != 1 {
|
||||
t.Errorf("kept-points %+v %v", out, err)
|
||||
}
|
||||
if got := call(t, r, "take", `{"module":"app","kept":"notes","fingerprint":"fp9","why":"by hand","asker":"op@shanks"}`); got.Accepted == "" {
|
||||
t.Errorf("take %+v", got)
|
||||
}
|
||||
if !strings.Contains(r.s.handActs[len(r.s.handActs)-1], "node.engine|op@shanks|take app notes|by hand") {
|
||||
t.Errorf("hand acts %v: the account the bus proved and the asker are both logged", r.s.handActs)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,313 @@
|
||||
{
|
||||
"module": "mesh-recovery",
|
||||
"version": "0.1",
|
||||
"runs-as": "mesh-recovery",
|
||||
"claims": [
|
||||
{
|
||||
"name": "mesh-recovery",
|
||||
"scope": "mesh",
|
||||
"serves": [
|
||||
"snapshot-kept",
|
||||
"restore-kept",
|
||||
"kept-points",
|
||||
"restore-aside",
|
||||
"cancel",
|
||||
"take",
|
||||
"restore",
|
||||
"retry",
|
||||
"aside",
|
||||
"discard-aside",
|
||||
"expire-points"
|
||||
]
|
||||
}
|
||||
],
|
||||
"tools": [
|
||||
"recovery_kept_points",
|
||||
"recovery_take",
|
||||
"recovery_restore",
|
||||
"recovery_retry",
|
||||
"recovery_aside",
|
||||
"recovery_discard_aside",
|
||||
"recovery_restore_aside",
|
||||
"recovery_cancel",
|
||||
"recovery_expire_points"
|
||||
],
|
||||
"consumes": [
|
||||
"node-backup.item-started",
|
||||
"node-backup.item-ended",
|
||||
"node-engine.running",
|
||||
"mesh-broker.snapshot-restored",
|
||||
"mesh-conditions.findings-asked"
|
||||
],
|
||||
"invokes": [
|
||||
"seat:mesh-catalog.entry",
|
||||
"seat:mesh-module-assigner.assignments",
|
||||
"seat:mesh-hand-acts.hand-act",
|
||||
"seat:mesh-broker.snapshot-get",
|
||||
"seat:mesh-broker.snapshot-restore",
|
||||
"seat:mesh-broker.snapshot-restore-cancel",
|
||||
"seat:node-backup.restore",
|
||||
"seat:mesh-recovery.kept-points",
|
||||
"seat:mesh-recovery.restore-aside",
|
||||
"seat:mesh-recovery.cancel",
|
||||
"seat:mesh-recovery.take",
|
||||
"seat:mesh-recovery.restore",
|
||||
"seat:mesh-recovery.retry",
|
||||
"seat:mesh-recovery.aside",
|
||||
"seat:mesh-recovery.discard-aside",
|
||||
"seat:mesh-recovery.expire-points"
|
||||
],
|
||||
"requires": [
|
||||
{
|
||||
"name": "digests",
|
||||
"storage": "key-value",
|
||||
"template": "state"
|
||||
},
|
||||
{
|
||||
"name": "findings",
|
||||
"storage": "key-value",
|
||||
"template": "state"
|
||||
},
|
||||
{
|
||||
"name": "bounds",
|
||||
"storage": "key-value",
|
||||
"template": "state",
|
||||
"options": {
|
||||
"key-ttl": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "finding-events",
|
||||
"storage": "stream",
|
||||
"template": "events",
|
||||
"options": {
|
||||
"events": [
|
||||
"finding-written.*",
|
||||
"finding-cleared.*",
|
||||
"findings-stated.*"
|
||||
],
|
||||
"max-age-days": 7,
|
||||
"max-mib": 16
|
||||
}
|
||||
}
|
||||
],
|
||||
"config": {
|
||||
"points-cap": {
|
||||
"type": "number",
|
||||
"description": "bytes the points directory may hold; an act that would pass it ends failed, naming it, before it removes anything (design 47 G section 564).",
|
||||
"minimum": 1,
|
||||
"trusted": false,
|
||||
"default": 10737418240
|
||||
},
|
||||
"points-filling-mark": {
|
||||
"type": "number",
|
||||
"description": "percent of the cap past which the finding points-filling is written (design 47 G section 564).",
|
||||
"minimum": 1,
|
||||
"maximum": 100,
|
||||
"trusted": false,
|
||||
"default": 80
|
||||
},
|
||||
"point-keep-days": {
|
||||
"type": "number",
|
||||
"description": "days after which a restore point is removed, the newest of its kept data never (design 47 G sections 551, 557).",
|
||||
"minimum": 0,
|
||||
"maximum": 3650,
|
||||
"trusted": false,
|
||||
"default": 90
|
||||
},
|
||||
"dump-wait-minutes": {
|
||||
"type": "number",
|
||||
"description": "minutes an act waits on the dump of the bus's item before recovery-waits is written (design 47 G sections 561, 557).",
|
||||
"minimum": 0,
|
||||
"maximum": 10080,
|
||||
"trusted": false,
|
||||
"default": 60
|
||||
},
|
||||
"snapshot-restore-wait-minutes": {
|
||||
"type": "number",
|
||||
"description": "minutes a restore in place of a protected keeper's snapshot is waited on, and once more after its cancel (design 47 G section 681, A-138, A-140).",
|
||||
"minimum": 0,
|
||||
"maximum": 10080,
|
||||
"trusted": false,
|
||||
"default": 90
|
||||
},
|
||||
"expire-every-hours": {
|
||||
"type": "number",
|
||||
"description": "hours between two removals of the points past their keep days, beside the one at the start and at each snapshot-kept; timed by a bound key of the state bounds (design 47 G sections 551, 557, A-138).",
|
||||
"minimum": 0,
|
||||
"maximum": 8760,
|
||||
"trusted": false,
|
||||
"default": 24
|
||||
}
|
||||
},
|
||||
"data": {
|
||||
"own": [
|
||||
{
|
||||
"id": "points",
|
||||
"path": "${dir:points}",
|
||||
"class": "valuable",
|
||||
"why": "the restore points taken of modules' kept data before a form change, and the asides of live data set aside before a restore (design 47 G sections 516, 554)"
|
||||
}
|
||||
]
|
||||
},
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
},
|
||||
"secrets-owner": "mesh-recovery",
|
||||
"resources": [
|
||||
{
|
||||
"id": "account",
|
||||
"type": "user",
|
||||
"name": "mesh-recovery",
|
||||
"shell": "/usr/bin/nologin",
|
||||
"home": "/var/lib/mesh-recovery"
|
||||
},
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"owner": "mesh-recovery",
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "points",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/mesh-recovery/points",
|
||||
"mode": "0700",
|
||||
"owner": "mesh-recovery",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "cap",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/points-cap",
|
||||
"owner": "mesh-recovery",
|
||||
"mode": "0600",
|
||||
"template": "files/points-cap"
|
||||
},
|
||||
{
|
||||
"id": "mark",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/points-filling-mark",
|
||||
"owner": "mesh-recovery",
|
||||
"mode": "0600",
|
||||
"template": "files/points-filling-mark"
|
||||
},
|
||||
{
|
||||
"id": "keep-days",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/point-keep-days",
|
||||
"owner": "mesh-recovery",
|
||||
"mode": "0600",
|
||||
"template": "files/point-keep-days"
|
||||
},
|
||||
{
|
||||
"id": "dump-wait",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/dump-wait-minutes",
|
||||
"owner": "mesh-recovery",
|
||||
"mode": "0600",
|
||||
"template": "files/dump-wait-minutes"
|
||||
},
|
||||
{
|
||||
"id": "restore-wait",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/snapshot-restore-wait-minutes",
|
||||
"owner": "mesh-recovery",
|
||||
"mode": "0600",
|
||||
"template": "files/snapshot-restore-wait-minutes"
|
||||
},
|
||||
{
|
||||
"id": "expire-every",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/expire-every-hours",
|
||||
"owner": "mesh-recovery",
|
||||
"mode": "0600",
|
||||
"template": "files/expire-every-hours"
|
||||
},
|
||||
{
|
||||
"id": "recovery",
|
||||
"type": "process",
|
||||
"name": "mesh-recovery",
|
||||
"artifact": "program",
|
||||
"run": [
|
||||
"./mesh-recovery"
|
||||
],
|
||||
"env": {
|
||||
"MESH_BROKER_FILE": "${dir:state}/broker",
|
||||
"MESH_RECOVERY_POINTS": "${dir:points}",
|
||||
"MESH_RECOVERY_POINTS_CAP_FILE": "${dir:state}/points-cap",
|
||||
"MESH_RECOVERY_FILLING_MARK_FILE": "${dir:state}/points-filling-mark",
|
||||
"MESH_RECOVERY_KEEP_DAYS_FILE": "${dir:state}/point-keep-days",
|
||||
"MESH_RECOVERY_DUMP_WAIT_FILE": "${dir:state}/dump-wait-minutes",
|
||||
"MESH_RECOVERY_RESTORE_WAIT_FILE": "${dir:state}/snapshot-restore-wait-minutes",
|
||||
"MESH_RECOVERY_EXPIRE_EVERY_FILE": "${dir:state}/expire-every-hours"
|
||||
},
|
||||
"user": "mesh-recovery",
|
||||
"serves": [
|
||||
"seat:mesh-recovery.snapshot-kept",
|
||||
"seat:mesh-recovery.restore-kept",
|
||||
"seat:mesh-recovery.kept-points",
|
||||
"seat:mesh-recovery.restore-aside",
|
||||
"seat:mesh-recovery.cancel",
|
||||
"seat:mesh-recovery.take",
|
||||
"seat:mesh-recovery.restore",
|
||||
"seat:mesh-recovery.retry",
|
||||
"seat:mesh-recovery.aside",
|
||||
"seat:mesh-recovery.discard-aside",
|
||||
"seat:mesh-recovery.expire-points"
|
||||
],
|
||||
"bus": {
|
||||
"invokes": [
|
||||
"seat:mesh-catalog.entry",
|
||||
"seat:mesh-module-assigner.assignments",
|
||||
"seat:mesh-hand-acts.hand-act",
|
||||
"seat:mesh-broker.snapshot-get",
|
||||
"seat:mesh-broker.snapshot-restore",
|
||||
"seat:mesh-broker.snapshot-restore-cancel",
|
||||
"seat:node-backup.restore"
|
||||
]
|
||||
},
|
||||
"health": {
|
||||
"kind": "unit"
|
||||
}
|
||||
}
|
||||
],
|
||||
"runtime-health": {
|
||||
"kind": "tool",
|
||||
"tool": "recovery_kept_points",
|
||||
"interval": "60s",
|
||||
"timeout": "10s",
|
||||
"looks": 2,
|
||||
"grace": "180s"
|
||||
},
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "program",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-recovery",
|
||||
"binary": "mesh-recovery"
|
||||
},
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-recovery-tools",
|
||||
"binary": "mesh-recovery-tools",
|
||||
"loads": [
|
||||
"mesh-recovery-tools"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "files",
|
||||
"kind": "archive",
|
||||
"from": "files"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
package main
|
||||
|
||||
import "nats-tools/internal/bussnap"
|
||||
import "git.novox.be/novox/mesh-sdk/go/bussnap"
|
||||
|
||||
// The snapshot itself is bussnap's, which the keeper takes one entry's with too (design 25 § 12.3).
|
||||
type (
|
||||
|
||||
@@ -19,8 +19,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/busarchive"
|
||||
"github.com/nats-io/nats.go"
|
||||
"nats-tools/internal/busarchive"
|
||||
)
|
||||
|
||||
// manifestOf is the nats module's own definition, beside this program.
|
||||
|
||||
@@ -34,8 +34,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/busarchive"
|
||||
"github.com/nats-io/nats.go"
|
||||
"nats-tools/internal/busarchive"
|
||||
)
|
||||
|
||||
// image is the bus's release (the manifest's server artifact names the same release in its tag).
|
||||
|
||||
@@ -30,10 +30,11 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"nats-tools/internal/busarchive"
|
||||
"nats-tools/internal/bussnap"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/busarchive"
|
||||
"git.novox.be/novox/mesh-sdk/go/bussnap"
|
||||
)
|
||||
|
||||
func main() {
|
||||
|
||||
@@ -16,8 +16,8 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/busarchive"
|
||||
"github.com/nats-io/nats.go"
|
||||
"nats-tools/internal/busarchive"
|
||||
)
|
||||
|
||||
// restoreChunk is what is sent at a time; the server answers each before the next goes.
|
||||
|
||||
@@ -29,7 +29,7 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"nats-tools/internal/bussnap"
|
||||
"git.novox.be/novox/mesh-sdk/go/bussnap"
|
||||
"nats-tools/internal/keeper"
|
||||
)
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"nats-tools/internal/bussnap"
|
||||
"git.novox.be/novox/mesh-sdk/go/bussnap"
|
||||
"nats-tools/internal/keeper"
|
||||
)
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"nats-tools/internal/busarchive"
|
||||
"git.novox.be/novox/mesh-sdk/go/busarchive"
|
||||
"nats-tools/internal/keeper"
|
||||
)
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"nats-tools/internal/keeper"
|
||||
@@ -30,34 +31,32 @@ import (
|
||||
// snapshot-restore, snapshot-restore-cancel and snapshot-remove are acted on by the keeper at once, with no hand-over through the
|
||||
// state requests. The tools bundle keeps nats_snapshots, which lists them.
|
||||
|
||||
// ChunkSize is at most what one chunk carries: 512 KiB.
|
||||
const ChunkSize = 512 << 10
|
||||
|
||||
// AckWithin is how long the next chunk waits for its acknowledgement.
|
||||
const AckWithin = 30 * time.Second
|
||||
// The form of the chunks is mesh-sdk's (buscalls.SeatChunks, A-139): the caller receives with the same one.
|
||||
const (
|
||||
// ChunkSize is at most what one chunk carries: 512 KiB.
|
||||
ChunkSize = buscalls.SeatChunkBytes
|
||||
// AckWithin is how long the next chunk waits for its acknowledgement.
|
||||
AckWithin = buscalls.SeatAckWithin
|
||||
)
|
||||
|
||||
// callToken is the shape of the subject token a caller chooses for a send.
|
||||
var callToken = regexp.MustCompile(`^[A-Za-z0-9_-]{16,64}$`)
|
||||
|
||||
// ChunkSubject and AckSubject are the subjects of the nth chunk of a send and of its acknowledgement.
|
||||
func ChunkSubject(call string, n int64) string {
|
||||
return "mesh.seat." + meshbroker.Seat + ".chunk." + meshbroker.VerbSnapshotGet + "." + call + "." + strconv.FormatInt(n, 10)
|
||||
func sendOf(call string) buscalls.SeatChunks {
|
||||
return buscalls.SeatChunks{Seat: meshbroker.Seat, Verb: meshbroker.VerbSnapshotGet, Call: call}
|
||||
}
|
||||
|
||||
// ChunkSubject and AckSubject are the subjects of the nth chunk of a send and of its acknowledgement.
|
||||
func ChunkSubject(call string, n int64) string { return sendOf(call).Chunk(n) }
|
||||
|
||||
// AckSubject is the subject the caller acknowledges the nth chunk on.
|
||||
func AckSubject(call string, n int64) string {
|
||||
return "mesh.seat." + meshbroker.Seat + ".ack." + meshbroker.VerbSnapshotGet + "." + call + "." + strconv.FormatInt(n, 10)
|
||||
}
|
||||
func AckSubject(call string, n int64) string { return sendOf(call).Ack(n) }
|
||||
|
||||
// AckPattern is every acknowledgement of snapshot-get.
|
||||
const AckPattern = "mesh.seat.mesh-broker.ack.snapshot-get.*.*"
|
||||
var AckPattern = buscalls.SeatAcks(meshbroker.Seat, meshbroker.VerbSnapshotGet)
|
||||
|
||||
// Chunk is the body of one chunk: its number, how many there are, and its bytes.
|
||||
type Chunk struct {
|
||||
N int64 `json:"n"`
|
||||
Of int64 `json:"of"`
|
||||
Data []byte `json:"data"`
|
||||
}
|
||||
type Chunk = buscalls.SeatChunk
|
||||
|
||||
// snapshotWorld is what the keeper's snapshot verbs reach: the file, the process's own publish and subscription, and the bus's users.
|
||||
type snapshotWorld struct {
|
||||
|
||||
@@ -1,162 +0,0 @@
|
||||
package busarchive
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// Archive is a snapshot read back: its manifest, and each file beside it unpacked into a scratch
|
||||
// directory of its own, removed by Close.
|
||||
type Archive struct {
|
||||
Manifest Manifest
|
||||
dir string
|
||||
files map[string]string
|
||||
}
|
||||
|
||||
// ReadArchive unpacks a snapshot. Only the files the manifest names are kept, and only under names
|
||||
// that are one stream's directory: an archive is data, and a path in it is never trusted to say where
|
||||
// on the machine a file goes.
|
||||
func ReadArchive(r io.Reader, scratch string) (*Archive, error) {
|
||||
dir, err := os.MkdirTemp(scratch, "mesh-nats-restore-*")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a := &Archive{dir: dir, files: map[string]string{}}
|
||||
tr := tar.NewReader(r)
|
||||
first := true
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
a.Close()
|
||||
return nil, fmt.Errorf("the archive does not read: %w", err)
|
||||
}
|
||||
if first {
|
||||
if hdr.Name != ManifestName {
|
||||
a.Close()
|
||||
return nil, fmt.Errorf("the archive starts with %q, not its manifest: it is not a snapshot of the bus", hdr.Name)
|
||||
}
|
||||
raw, err := io.ReadAll(io.LimitReader(tr, 64<<20))
|
||||
if err != nil {
|
||||
a.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a.Manifest); err != nil {
|
||||
a.Close()
|
||||
return nil, fmt.Errorf("its manifest: %w", err)
|
||||
}
|
||||
if a.Manifest.Format != 1 {
|
||||
a.Close()
|
||||
return nil, fmt.Errorf("its manifest is format %d; this program reads format 1", a.Manifest.Format)
|
||||
}
|
||||
first = false
|
||||
continue
|
||||
}
|
||||
if !a.named(hdr.Name) {
|
||||
a.Close()
|
||||
return nil, fmt.Errorf("the archive holds %q, which its manifest does not name", hdr.Name)
|
||||
}
|
||||
local := filepath.Join(dir, fmt.Sprintf("%04d", len(a.files)))
|
||||
f, err := os.OpenFile(local, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
a.Close()
|
||||
return nil, err
|
||||
}
|
||||
_, err = io.Copy(f, tr)
|
||||
f.Close()
|
||||
if err != nil {
|
||||
a.Close()
|
||||
return nil, err
|
||||
}
|
||||
a.files[hdr.Name] = local
|
||||
}
|
||||
if first {
|
||||
a.Close()
|
||||
return nil, errors.New("the archive is empty")
|
||||
}
|
||||
for _, s := range a.Manifest.Streams {
|
||||
if !ValidName(s.Name) || s.Meta != "streams/"+s.Name+"/backup.json" || s.Archive != "streams/"+s.Name+"/stream.tar.s2" {
|
||||
a.Close()
|
||||
return nil, fmt.Errorf("the manifest names a stream %q at paths that are not its own", s.Name)
|
||||
}
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
func (a *Archive) named(name string) bool {
|
||||
for _, s := range a.Manifest.Streams {
|
||||
if name == s.Meta || name == s.Archive {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Close removes the unpacked files.
|
||||
func (a *Archive) Close() { os.RemoveAll(a.dir) }
|
||||
|
||||
// Verify holds every file to the manifest's size and checksum, and reads every stream's archive to
|
||||
// its end.
|
||||
func (a *Archive) Verify() error {
|
||||
for _, s := range a.Manifest.Streams {
|
||||
for _, f := range []struct{ name, sum string }{{s.Meta, s.MetaSHA256}, {s.Archive, s.ArchiveSHA256}} {
|
||||
local, ok := a.files[f.name]
|
||||
if !ok {
|
||||
return fmt.Errorf("%s is named in the manifest and missing from the archive", f.name)
|
||||
}
|
||||
got, size, err := sha256Of(local)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if got != f.sum {
|
||||
return fmt.Errorf("%s does not match the checksum its manifest gives", f.name)
|
||||
}
|
||||
if f.name == s.Archive && size != s.ArchiveBytes {
|
||||
return fmt.Errorf("%s is %d bytes; its manifest says %d", f.name, size, s.ArchiveBytes)
|
||||
}
|
||||
}
|
||||
af, err := os.Open(a.files[s.Archive])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = ReadsThrough(af)
|
||||
af.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s does not read: %w", s.Archive, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sha256Of(path string) (string, int64, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
defer f.Close()
|
||||
h := sha256.New()
|
||||
n, err := io.Copy(h, f)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), n, nil
|
||||
}
|
||||
|
||||
// Open is one stream's meta and archive.
|
||||
func (a *Archive) Open(s StreamEntry) (meta []byte, archive *os.File, err error) {
|
||||
meta, err = os.ReadFile(a.files[s.Meta])
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
archive, err = os.Open(a.files[s.Archive])
|
||||
return meta, archive, err
|
||||
}
|
||||
@@ -1,86 +0,0 @@
|
||||
// Package busarchive is the bus's snapshot archive: its manifest, reading one back and holding it to
|
||||
// its checksums. The snapshot program (cmd/mesh-nats-snapshot) writes and restores with it and the bus's
|
||||
// tool nats_snapshot_verify (cmd/nats-tools) checks with it, so the two never differ on what a good
|
||||
// snapshot is.
|
||||
package busarchive
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/klauspost/compress/s2"
|
||||
)
|
||||
|
||||
// Manifest is the first entry of an archive: what was taken, when, how long it took, and the checksum
|
||||
// of every file beside it. It is what `verify` and `restore` hold the archive to.
|
||||
type Manifest struct {
|
||||
Format int `json:"format"`
|
||||
Server string `json:"server_version"`
|
||||
Started time.Time `json:"started"`
|
||||
Finished time.Time `json:"finished"`
|
||||
Seconds float64 `json:"seconds"`
|
||||
// Messages and StreamBytes are the streams' own totals at their snapshots; ArchiveBytes what the
|
||||
// archives take, compressed.
|
||||
Messages uint64 `json:"messages"`
|
||||
StreamBytes int64 `json:"stream_bytes"`
|
||||
ArchiveBytes int64 `json:"archive_bytes"`
|
||||
Streams []StreamEntry `json:"streams"`
|
||||
// Skipped are streams that hold nothing across a restart (memory storage), said rather than
|
||||
// silently left out.
|
||||
Skipped []Skipped `json:"skipped,omitempty"`
|
||||
}
|
||||
|
||||
// StreamEntry is one stream as it was snapshotted.
|
||||
type StreamEntry struct {
|
||||
Name string `json:"name"`
|
||||
Subjects []string `json:"subjects,omitempty"`
|
||||
Retention string `json:"retention,omitempty"`
|
||||
Messages uint64 `json:"messages"`
|
||||
Bytes uint64 `json:"bytes"`
|
||||
FirstSeq uint64 `json:"first_seq"`
|
||||
LastSeq uint64 `json:"last_seq"`
|
||||
Deleted int `json:"deleted,omitempty"`
|
||||
Consumers int `json:"consumers"`
|
||||
// Meta is the server's own description of the stream (its configuration and state), in the
|
||||
// shape the restore API and the nats CLI take back: backup.json.
|
||||
Meta string `json:"meta"`
|
||||
MetaSHA256 string `json:"meta_sha256"`
|
||||
// Archive is the server's snapshot of the stream, as the server sent it: stream.tar.s2.
|
||||
Archive string `json:"archive"`
|
||||
ArchiveBytes int64 `json:"archive_bytes"`
|
||||
ArchiveSHA256 string `json:"archive_sha256"`
|
||||
Seconds float64 `json:"seconds"`
|
||||
}
|
||||
|
||||
// Skipped is a stream not snapshotted, and why.
|
||||
type Skipped struct {
|
||||
Name string `json:"name"`
|
||||
Why string `json:"why"`
|
||||
}
|
||||
|
||||
// ManifestName is the archive's first entry.
|
||||
const ManifestName = "manifest.json"
|
||||
|
||||
// ReadsThrough is whether a stream's archive decompresses and lists to its end.
|
||||
func ReadsThrough(r io.Reader) error {
|
||||
tr := tar.NewReader(s2.NewReader(r))
|
||||
for {
|
||||
_, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(io.Discard, tr); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ValidName is a stream name that is one safe path component.
|
||||
func ValidName(name string) bool {
|
||||
return name != "" && name != "." && name != ".." && !strings.ContainsAny(name, "/\\\x00 *>")
|
||||
}
|
||||
@@ -1,109 +0,0 @@
|
||||
package busarchive
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// StreamReport is one stream of a verified snapshot.
|
||||
type StreamReport struct {
|
||||
Name string `json:"name"`
|
||||
LastSeq uint64 `json:"last_seq"`
|
||||
}
|
||||
|
||||
// Report is the answer of VerifyFile: which file was held to its manifest, and whether it passed.
|
||||
type Report struct {
|
||||
File string `json:"file"`
|
||||
Modified time.Time `json:"modified"`
|
||||
Size int64 `json:"size"`
|
||||
Taken *time.Time `json:"taken,omitempty"`
|
||||
Streams []StreamReport `json:"streams"`
|
||||
Skipped []Skipped `json:"skipped,omitempty"`
|
||||
Passed bool `json:"passed"`
|
||||
// Fault is the first thing wrong, empty when the snapshot passed.
|
||||
Fault string `json:"fault,omitempty"`
|
||||
}
|
||||
|
||||
// VerifyFile verifies one snapshot inside dir: the file called name, or the newest *.tar when name is
|
||||
// empty. Nothing outside dir is read and nothing is written there; a snapshot that fails is an answer
|
||||
// (Passed false, the first Fault), and only a directory without a snapshot is an error.
|
||||
func VerifyFile(dir, name string) (*Report, error) {
|
||||
path, info, err := pick(dir, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r := &Report{File: filepath.Base(path), Modified: info.ModTime().UTC(), Size: info.Size(), Streams: []StreamReport{}}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
r.Fault = err.Error()
|
||||
return r, nil
|
||||
}
|
||||
defer f.Close()
|
||||
a, err := ReadArchive(f, "")
|
||||
if err != nil {
|
||||
r.Fault = err.Error()
|
||||
return r, nil
|
||||
}
|
||||
defer a.Close()
|
||||
taken := a.Manifest.Finished
|
||||
r.Taken = &taken
|
||||
r.Skipped = a.Manifest.Skipped
|
||||
for _, s := range a.Manifest.Streams {
|
||||
r.Streams = append(r.Streams, StreamReport{Name: s.Name, LastSeq: s.LastSeq})
|
||||
}
|
||||
if err := a.Verify(); err != nil {
|
||||
r.Fault = err.Error()
|
||||
return r, nil
|
||||
}
|
||||
r.Passed = true
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// pick names the regular file to verify, and nothing outside dir: a name is one path component, and a
|
||||
// symbolic link is not followed out of the directory.
|
||||
func pick(dir, name string) (string, fs.FileInfo, error) {
|
||||
if dir == "" {
|
||||
return "", nil, errors.New("the snapshots directory is not set")
|
||||
}
|
||||
if name != "" {
|
||||
if name != filepath.Base(name) || name == "." || name == ".." || strings.ContainsRune(name, 0) {
|
||||
return "", nil, fmt.Errorf("%q is not a file name inside the snapshots directory", name)
|
||||
}
|
||||
p := filepath.Join(dir, name)
|
||||
info, err := os.Lstat(p)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("no snapshot %s in the snapshots directory: %w", name, err)
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return "", nil, fmt.Errorf("%s is not a regular file", name)
|
||||
}
|
||||
return p, info, nil
|
||||
}
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("the snapshots directory does not read: %w", err)
|
||||
}
|
||||
var best string
|
||||
var bestInfo fs.FileInfo
|
||||
for _, e := range entries {
|
||||
if !strings.HasSuffix(e.Name(), ".tar") {
|
||||
continue
|
||||
}
|
||||
info, err := e.Info()
|
||||
if err != nil || !info.Mode().IsRegular() {
|
||||
continue
|
||||
}
|
||||
if bestInfo == nil || info.ModTime().After(bestInfo.ModTime()) {
|
||||
best, bestInfo = e.Name(), info
|
||||
}
|
||||
}
|
||||
if bestInfo == nil {
|
||||
return "", nil, errors.New("the snapshots directory holds no .tar snapshot")
|
||||
}
|
||||
return filepath.Join(dir, best), bestInfo, nil
|
||||
}
|
||||
@@ -1,173 +0,0 @@
|
||||
package busarchive
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/klauspost/compress/s2"
|
||||
)
|
||||
|
||||
func sum(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) }
|
||||
|
||||
// streamArchive is a stream's archive as the server sends it: a tar of one file, s2-compressed.
|
||||
func streamArchive(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
var raw bytes.Buffer
|
||||
tw := tar.NewWriter(&raw)
|
||||
body := []byte(strings.Repeat("a message of the stream\n", 50))
|
||||
if err := tw.WriteHeader(&tar.Header{Name: "msgs.blk", Mode: 0o600, Size: int64(len(body))}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tw.Write(body)
|
||||
tw.Close()
|
||||
var out bytes.Buffer
|
||||
w := s2.NewWriter(&out)
|
||||
w.Write(raw.Bytes())
|
||||
w.Close()
|
||||
return out.Bytes()
|
||||
}
|
||||
|
||||
// snapshotTar builds a snapshot of two streams; the mutations change what the manifest says or what is
|
||||
// stored, so each fault is made in the one place it is found.
|
||||
func snapshotTar(t *testing.T, mutate func(m *Manifest, files map[string][]byte)) []byte {
|
||||
t.Helper()
|
||||
files := map[string][]byte{}
|
||||
m := Manifest{Format: 1, Server: "2.11.17", Finished: time.Date(2026, 10, 22, 3, 0, 0, 0, time.UTC)}
|
||||
for i, name := range []string{"EVENTS", "KV_conditions"} {
|
||||
meta := []byte(`{"config":{"name":"` + name + `"}}`)
|
||||
arc := streamArchive(t)
|
||||
e := StreamEntry{Name: name, LastSeq: uint64(10 * (i + 1)),
|
||||
Meta: "streams/" + name + "/backup.json", MetaSHA256: sum(meta),
|
||||
Archive: "streams/" + name + "/stream.tar.s2", ArchiveBytes: int64(len(arc)), ArchiveSHA256: sum(arc)}
|
||||
m.Streams = append(m.Streams, e)
|
||||
files[e.Meta], files[e.Archive] = meta, arc
|
||||
}
|
||||
if mutate != nil {
|
||||
mutate(&m, files)
|
||||
}
|
||||
var out bytes.Buffer
|
||||
tw := tar.NewWriter(&out)
|
||||
put := func(name string, b []byte) {
|
||||
if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(b))}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tw.Write(b)
|
||||
}
|
||||
raw, _ := json.Marshal(m)
|
||||
put(ManifestName, raw)
|
||||
for _, s := range m.Streams {
|
||||
put(s.Meta, files[s.Meta])
|
||||
put(s.Archive, files[s.Archive])
|
||||
}
|
||||
tw.Close()
|
||||
return out.Bytes()
|
||||
}
|
||||
|
||||
func put(t *testing.T, dir, name string, b []byte, at time.Time) {
|
||||
t.Helper()
|
||||
p := filepath.Join(dir, name)
|
||||
if err := os.WriteFile(p, b, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
os.Chtimes(p, at, at)
|
||||
}
|
||||
|
||||
func TestAGoodSnapshotPassesAndSaysItsStreams(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
good := snapshotTar(t, nil)
|
||||
put(t, dir, "bus.tar", good, time.Now())
|
||||
r, err := VerifyFile(dir, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !r.Passed || r.Fault != "" || r.File != "bus.tar" || r.Size != int64(len(good)) {
|
||||
t.Fatalf("%+v", r)
|
||||
}
|
||||
if r.Taken == nil || !r.Taken.Equal(time.Date(2026, 10, 22, 3, 0, 0, 0, time.UTC)) {
|
||||
t.Errorf("taken %v", r.Taken)
|
||||
}
|
||||
if len(r.Streams) != 2 || r.Streams[0] != (StreamReport{"EVENTS", 10}) || r.Streams[1] != (StreamReport{"KV_conditions", 20}) {
|
||||
t.Errorf("streams %+v", r.Streams)
|
||||
}
|
||||
}
|
||||
|
||||
func TestACorruptedChecksumFailsNamingTheFile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
put(t, dir, "bus.tar", snapshotTar(t, func(m *Manifest, files map[string][]byte) {
|
||||
m.Streams[1].ArchiveSHA256 = sum([]byte("something else"))
|
||||
}), time.Now())
|
||||
r, err := VerifyFile(dir, "bus.tar")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r.Passed || !strings.Contains(r.Fault, "streams/KV_conditions/stream.tar.s2") || !strings.Contains(r.Fault, "checksum") {
|
||||
t.Fatalf("%+v", r)
|
||||
}
|
||||
if len(r.Streams) != 2 {
|
||||
t.Errorf("a failed snapshot still says its streams: %+v", r.Streams)
|
||||
}
|
||||
}
|
||||
|
||||
func TestATruncatedSnapshotFails(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
good := snapshotTar(t, nil)
|
||||
put(t, dir, "bus.tar", good[:len(good)*2/3], time.Now())
|
||||
r, err := VerifyFile(dir, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r.Passed || r.Fault == "" {
|
||||
t.Fatalf("a truncated snapshot passed: %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnArchiveThatDoesNotReadToItsEndFailsEvenWithMatchingChecksums(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
put(t, dir, "bus.tar", snapshotTar(t, func(m *Manifest, files map[string][]byte) {
|
||||
bad := []byte("not an s2 stream at all")
|
||||
s := &m.Streams[0]
|
||||
files[s.Archive] = bad
|
||||
s.ArchiveSHA256, s.ArchiveBytes = sum(bad), int64(len(bad))
|
||||
}), time.Now())
|
||||
r, _ := VerifyFile(dir, "")
|
||||
if r.Passed || !strings.Contains(r.Fault, "does not read") {
|
||||
t.Fatalf("%+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheNewestTarIsVerifiedAndOnlyInsideTheDirectory(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
now := time.Now()
|
||||
put(t, dir, "old.tar", []byte("junk"), now.Add(-48*time.Hour))
|
||||
put(t, dir, "bus.tar", snapshotTar(t, nil), now)
|
||||
put(t, dir, "bus.tar.partial", []byte("half"), now.Add(time.Hour))
|
||||
r, err := VerifyFile(dir, "")
|
||||
if err != nil || r.File != "bus.tar" || !r.Passed {
|
||||
t.Fatalf("%+v %v", r, err)
|
||||
}
|
||||
if r, _ := VerifyFile(dir, "old.tar"); r.Passed || r.File != "old.tar" {
|
||||
t.Errorf("a named file is the one checked: %+v", r)
|
||||
}
|
||||
outside := t.TempDir()
|
||||
put(t, outside, "x.tar", snapshotTar(t, nil), now)
|
||||
for _, name := range []string{"../" + filepath.Base(outside) + "/x.tar", outside + "/x.tar", ".."} {
|
||||
if _, err := VerifyFile(dir, name); err == nil {
|
||||
t.Errorf("%q was read", name)
|
||||
}
|
||||
}
|
||||
os.Symlink(filepath.Join(outside, "x.tar"), filepath.Join(dir, "link.tar"))
|
||||
if _, err := VerifyFile(dir, "link.tar"); err == nil {
|
||||
t.Error("a symbolic link out of the directory was followed")
|
||||
}
|
||||
if _, err := VerifyFile(t.TempDir(), ""); err == nil {
|
||||
t.Error("an empty directory is not a snapshot")
|
||||
}
|
||||
}
|
||||
@@ -1,158 +0,0 @@
|
||||
package bussnap
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"nats-tools/internal/busarchive"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// The keeper's restore in place of a snapshot it took (design 47 G §681): each stream of the file put back under its own
|
||||
// name, only where no stream of that name is live.
|
||||
|
||||
// APIStreamDelete is the subject the keeper removes its own partial stream with.
|
||||
const APIStreamDelete = "$JS.API.STREAM.DELETE."
|
||||
|
||||
const restoreChunk = 64 * 1024
|
||||
|
||||
// ManifestOf reads the manifest of a snapshot file: the streams it holds, without unpacking them.
|
||||
func ManifestOf(path string) (*busarchive.Manifest, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
tr := tar.NewReader(f)
|
||||
hdr, err := tr.Next()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the snapshot does not read: %w", err)
|
||||
}
|
||||
if hdr.Name != busarchive.ManifestName {
|
||||
return nil, fmt.Errorf("the snapshot starts with %q, not its manifest", hdr.Name)
|
||||
}
|
||||
raw, err := io.ReadAll(io.LimitReader(tr, 64<<20))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m busarchive.Manifest
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, fmt.Errorf("its manifest: %w", err)
|
||||
}
|
||||
return &m, nil
|
||||
}
|
||||
|
||||
// Live says whether a stream of that name exists.
|
||||
func Live(ctx context.Context, nc *nats.Conn, name string) (bool, error) {
|
||||
err := Request(ctx, nc, APIStreamInfo+name, nil, nil)
|
||||
var api *APIError
|
||||
switch {
|
||||
case err == nil:
|
||||
return true, nil
|
||||
case errors.As(err, &api) && api.Code == 404:
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
// DeleteStream removes a stream; one that is not there is no error.
|
||||
func DeleteStream(ctx context.Context, nc *nats.Conn, name string) error {
|
||||
err := Request(ctx, nc, APIStreamDelete+name, nil, nil)
|
||||
var api *APIError
|
||||
if errors.As(err, &api) && api.Code == 404 {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// RestoreFile puts every stream of the snapshot file back under its own name, each sent in chunks the server answers; it
|
||||
// ends with the context's bound. A stream already there is the caller's to have refused beforehand.
|
||||
func RestoreFile(ctx context.Context, nc *nats.Conn, path, scratch string, log func(string, ...any)) error {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
a, err := busarchive.ReadArchive(f, scratch)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer a.Close()
|
||||
if err := a.Verify(); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, s := range a.Manifest.Streams {
|
||||
started := time.Now()
|
||||
if err := restoreStream(ctx, nc, a, s); err != nil {
|
||||
return fmt.Errorf("stream %s: %w", s.Name, err)
|
||||
}
|
||||
log("%s: restored in %.2fs", s.Name, time.Since(started).Seconds())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func restoreStream(ctx context.Context, nc *nats.Conn, a *busarchive.Archive, s busarchive.StreamEntry) error {
|
||||
meta, archive, err := a.Open(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer archive.Close()
|
||||
var req map[string]json.RawMessage
|
||||
if err := json.Unmarshal(meta, &req); err != nil || req["config"] == nil {
|
||||
return errors.New("its meta is not a stream's configuration and state")
|
||||
}
|
||||
var resp struct {
|
||||
DeliverSubject string `json:"deliver_subject"`
|
||||
}
|
||||
if err := Request(ctx, nc, APIStreamRestore+s.Name, req, &resp); err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.DeliverSubject == "" {
|
||||
return errors.New("the server gave nowhere to send the archive")
|
||||
}
|
||||
buf := make([]byte, restoreChunk)
|
||||
for {
|
||||
n, err := archive.Read(buf)
|
||||
if n > 0 {
|
||||
reply, rerr := nc.RequestWithContext(ctx, resp.DeliverSubject, buf[:n])
|
||||
if rerr != nil {
|
||||
return fmt.Errorf("sending the archive: %w", rerr)
|
||||
}
|
||||
if len(reply.Data) > 0 {
|
||||
return fmt.Errorf("the server refused the archive: %s", Truncate(reply.Data))
|
||||
}
|
||||
}
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
final, err := nc.RequestWithContext(ctx, resp.DeliverSubject, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("finishing the restore: %w", err)
|
||||
}
|
||||
var created struct {
|
||||
Error *APIError `json:"error"`
|
||||
State StreamState `json:"state"`
|
||||
}
|
||||
if err := json.Unmarshal(final.Data, &created); err != nil {
|
||||
return fmt.Errorf("the server's last answer is not JSON: %q", Truncate(final.Data))
|
||||
}
|
||||
if created.Error != nil {
|
||||
return created.Error
|
||||
}
|
||||
if created.State.LastSeq < s.LastSeq || (s.Messages > 0 && created.State.Messages == 0) {
|
||||
return fmt.Errorf("restored with %d message(s) up to sequence %d; the snapshot held %d up to %d",
|
||||
created.State.Messages, created.State.LastSeq, s.Messages, s.LastSeq)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,433 +0,0 @@
|
||||
// Package bussnap takes a snapshot of the bus's streams with the server's own snapshot API (novox/hq ADR 0235): the
|
||||
// nightly dump of every stream, and the bus module's keeper's snapshot of one entry before it is rebuilt or removed
|
||||
// (design 25 § 12.3).
|
||||
package bussnap
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"nats-tools/internal/busarchive"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// The JetStream API's subjects this program speaks. Each is granted to the bus's own module and
|
||||
// nothing else is (mesh-controller internal/broker, BusSnapshotGrants); a test holds the two lists
|
||||
// equal against a real server.
|
||||
const (
|
||||
APIStreamNames = "$JS.API.STREAM.NAMES"
|
||||
APIStreamInfo = "$JS.API.STREAM.INFO."
|
||||
APIStreamSnapshot = "$JS.API.STREAM.SNAPSHOT."
|
||||
APIStreamRestore = "$JS.API.STREAM.RESTORE."
|
||||
)
|
||||
|
||||
// ChunkSize is what the server is asked to send at a time: its own default. The server sends up to
|
||||
// an 8 MiB window ahead of the acknowledgements and waits for them past it, so one stream costs the
|
||||
// server at most that much in flight, whatever its size.
|
||||
const ChunkSize = 128 * 1024
|
||||
|
||||
// SnapshotOptions bounds a snapshot.
|
||||
type SnapshotOptions struct {
|
||||
// StreamTimeout is the longest one stream may take; past it the snapshot fails, naming it.
|
||||
StreamTimeout time.Duration
|
||||
// Pause is a rest between two streams, so a night's snapshot is never one long burst.
|
||||
Pause time.Duration
|
||||
// TempDir is where one stream's archive waits until it is written out; the system's when empty.
|
||||
TempDir string
|
||||
Log func(format string, args ...any)
|
||||
// Streams, when set, are the only streams taken (the keeper's snapshot of one entry); every stream when empty.
|
||||
Streams []string
|
||||
}
|
||||
|
||||
// APIError is the error every JetStream API answer may carry.
|
||||
type APIError struct {
|
||||
Code int `json:"code"`
|
||||
ErrCode int `json:"err_code"`
|
||||
Description string `json:"description"`
|
||||
}
|
||||
|
||||
func (e *APIError) Error() string { return fmt.Sprintf("%s (%d)", e.Description, e.Code) }
|
||||
|
||||
// StreamState is what this program reads of a stream's state; the whole of it is kept as the server
|
||||
// gave it, in the stream's meta.
|
||||
type StreamState struct {
|
||||
Messages uint64 `json:"messages"`
|
||||
Bytes uint64 `json:"bytes"`
|
||||
FirstSeq uint64 `json:"first_seq"`
|
||||
LastSeq uint64 `json:"last_seq"`
|
||||
NumDelete int `json:"num_deleted"`
|
||||
Consumers int `json:"consumer_count"`
|
||||
}
|
||||
|
||||
type streamConfig struct {
|
||||
Name string `json:"name"`
|
||||
Subjects []string `json:"subjects"`
|
||||
Retention string `json:"retention"`
|
||||
Storage string `json:"storage"`
|
||||
}
|
||||
|
||||
// Request asks the JetStream API and decodes its answer, refusing an answer that carries an error.
|
||||
func Request(ctx context.Context, nc *nats.Conn, subject string, body any, into any) error {
|
||||
var payload []byte
|
||||
if body != nil {
|
||||
var err error
|
||||
if payload, err = json.Marshal(body); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
rctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
msg, err := nc.RequestWithContext(rctx, subject, payload)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, nats.ErrTimeout) || errors.Is(err, nats.ErrNoResponders) {
|
||||
return fmt.Errorf("%s was not answered (%v): the bus refuses a subject this user is not granted by not answering it", subject, err)
|
||||
}
|
||||
return fmt.Errorf("%s: %w", subject, err)
|
||||
}
|
||||
var wrapped struct {
|
||||
Error *APIError `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(msg.Data, &wrapped); err != nil {
|
||||
return fmt.Errorf("%s answered something that is not JSON: %q", subject, Truncate(msg.Data))
|
||||
}
|
||||
if wrapped.Error != nil {
|
||||
return fmt.Errorf("%s: %w", subject, wrapped.Error)
|
||||
}
|
||||
if into != nil {
|
||||
return json.Unmarshal(msg.Data, into)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func Truncate(b []byte) string {
|
||||
if len(b) > 200 {
|
||||
return string(b[:200]) + "…"
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// StreamNames is every stream in the account, in name order, all pages of them.
|
||||
func StreamNames(ctx context.Context, nc *nats.Conn) ([]string, error) {
|
||||
var names []string
|
||||
for {
|
||||
var page struct {
|
||||
Total int `json:"total"`
|
||||
Offset int `json:"offset"`
|
||||
Limit int `json:"limit"`
|
||||
Streams []string `json:"streams"`
|
||||
}
|
||||
if err := Request(ctx, nc, APIStreamNames, map[string]int{"offset": len(names)}, &page); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names = append(names, page.Streams...)
|
||||
if len(page.Streams) == 0 || len(names) >= page.Total {
|
||||
break
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// Snapshot writes every stream, one at a time, as a tar: the manifest first, then each stream's
|
||||
// meta and archive under streams/<name>/. Any stream failing fails the whole: a partial copy of the
|
||||
// bus written where a whole one is expected is the silent failure this exists to prevent.
|
||||
func Snapshot(ctx context.Context, nc *nats.Conn, out io.Writer, o SnapshotOptions) (*busarchive.Manifest, error) {
|
||||
if o.StreamTimeout <= 0 {
|
||||
o.StreamTimeout = 10 * time.Minute
|
||||
}
|
||||
if o.Log == nil {
|
||||
o.Log = func(string, ...any) {}
|
||||
}
|
||||
m := &busarchive.Manifest{Format: 1, Server: nc.ConnectedServerVersion(), Started: time.Now().UTC()}
|
||||
names, err := StreamNames(ctx, nc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(o.Streams) > 0 {
|
||||
var only []string
|
||||
for _, n := range names {
|
||||
if slices.Contains(o.Streams, n) {
|
||||
only = append(only, n)
|
||||
}
|
||||
}
|
||||
if len(only) != len(o.Streams) {
|
||||
return nil, fmt.Errorf("the bus holds %v of the streams %v asked for: a snapshot missing one would pass for a whole one", only, o.Streams)
|
||||
}
|
||||
names = only
|
||||
}
|
||||
if len(names) == 0 {
|
||||
// The mesh's own streams always exist once the controller has run; a bus with none is not
|
||||
// one to call backed up.
|
||||
return nil, errors.New("the bus holds no streams; a snapshot of nothing would pass for a backup")
|
||||
}
|
||||
|
||||
type taken struct {
|
||||
entry busarchive.StreamEntry
|
||||
meta []byte
|
||||
archive *os.File
|
||||
}
|
||||
var files []taken
|
||||
defer func() {
|
||||
for _, t := range files {
|
||||
t.archive.Close()
|
||||
os.Remove(t.archive.Name())
|
||||
}
|
||||
}()
|
||||
|
||||
for i, name := range names {
|
||||
if i > 0 && o.Pause > 0 {
|
||||
select {
|
||||
case <-time.After(o.Pause):
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
var info struct {
|
||||
Config streamConfig `json:"config"`
|
||||
}
|
||||
if err := Request(ctx, nc, APIStreamInfo+name, nil, &info); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if info.Config.Storage == "memory" {
|
||||
m.Skipped = append(m.Skipped, busarchive.Skipped{Name: name, Why: "memory storage: it holds nothing across a restart, and the server cannot snapshot it"})
|
||||
o.Log("%s: skipped, memory storage", name)
|
||||
continue
|
||||
}
|
||||
f, err := os.CreateTemp(o.TempDir, "mesh-nats-snapshot-*")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
files = append(files, taken{archive: f})
|
||||
sctx, cancel := context.WithTimeout(ctx, o.StreamTimeout)
|
||||
entry, meta, err := snapshotStream(sctx, nc, name, f)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stream %s: %w", name, err)
|
||||
}
|
||||
files[len(files)-1].entry, files[len(files)-1].meta = entry, meta
|
||||
m.Messages += entry.Messages
|
||||
m.StreamBytes += int64(entry.Bytes)
|
||||
m.ArchiveBytes += entry.ArchiveBytes
|
||||
o.Log("%s: %d message(s), %s, last sequence %d, %d consumer(s) → %s in %.2fs",
|
||||
name, entry.Messages, Human(int64(entry.Bytes)), entry.LastSeq, entry.Consumers, Human(entry.ArchiveBytes), entry.Seconds)
|
||||
}
|
||||
m.Finished = time.Now().UTC()
|
||||
m.Seconds = m.Finished.Sub(m.Started).Seconds()
|
||||
for _, t := range files {
|
||||
m.Streams = append(m.Streams, t.entry)
|
||||
}
|
||||
|
||||
tw := tar.NewWriter(out)
|
||||
manifest, err := json.MarshalIndent(m, "", " ")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := writeEntry(tw, busarchive.ManifestName, append(manifest, '\n'), m.Finished); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, t := range files {
|
||||
if err := writeEntry(tw, t.entry.Meta, t.meta, m.Finished); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := t.archive.Seek(0, io.SeekStart); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hdr := &tar.Header{Name: t.entry.Archive, Mode: 0o600, Size: t.entry.ArchiveBytes, ModTime: m.Finished, Format: tar.FormatPAX}
|
||||
if err := tw.WriteHeader(hdr); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := io.Copy(tw, t.archive); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func writeEntry(tw *tar.Writer, name string, body []byte, at time.Time) error {
|
||||
if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(body)), ModTime: at, Format: tar.FormatPAX}); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := tw.Write(body)
|
||||
return err
|
||||
}
|
||||
|
||||
// snapshotStream takes one stream's snapshot into f: the server's chunks, each acknowledged as it
|
||||
// arrives so the server's window keeps moving, until the server's last, empty message. It answers
|
||||
// the stream's entry (the archive's size and checksum included) and its meta.
|
||||
//
|
||||
// The protocol is the server's (nats-server jetstream_api.go, streamSnapshot): the request names a
|
||||
// subject to deliver to; the server answers with the stream's configuration and its state at the
|
||||
// snapshot, then sends the archive in chunks whose reply subject is the acknowledgement it waits for
|
||||
// past its window — two seconds without one and it gives up with "408 No Flow Response". The end is
|
||||
// an empty message whose status header says 204, or the error that ended it.
|
||||
func snapshotStream(ctx context.Context, nc *nats.Conn, name string, f *os.File) (busarchive.StreamEntry, []byte, error) {
|
||||
started := time.Now()
|
||||
deliver := nc.NewRespInbox()
|
||||
|
||||
hash := sha256.New()
|
||||
var written int64
|
||||
var mu sync.Mutex
|
||||
var failed error
|
||||
done := make(chan struct{})
|
||||
var once sync.Once
|
||||
finish := func(err error) {
|
||||
mu.Lock()
|
||||
if failed == nil {
|
||||
failed = err
|
||||
}
|
||||
mu.Unlock()
|
||||
once.Do(func() { close(done) })
|
||||
}
|
||||
sub, err := nc.Subscribe(deliver, func(msg *nats.Msg) {
|
||||
if len(msg.Data) == 0 {
|
||||
if status := msg.Header.Get("Status"); status != "" && status != "204" {
|
||||
finish(fmt.Errorf("the server ended the snapshot: %s %s", status, msg.Header.Get("Description")))
|
||||
return
|
||||
}
|
||||
finish(nil)
|
||||
return
|
||||
}
|
||||
// Acknowledged before it is written: the chunk is already in memory, and a slow disk must not
|
||||
// read to the server as a reader that went away.
|
||||
if msg.Reply != "" {
|
||||
if err := nc.Publish(msg.Reply, nil); err != nil {
|
||||
finish(fmt.Errorf("acknowledging a chunk: %w", err))
|
||||
return
|
||||
}
|
||||
}
|
||||
if _, err := f.Write(msg.Data); err != nil {
|
||||
finish(fmt.Errorf("writing the archive: %w", err))
|
||||
return
|
||||
}
|
||||
hash.Write(msg.Data)
|
||||
written += int64(len(msg.Data))
|
||||
})
|
||||
if err != nil {
|
||||
return busarchive.StreamEntry{}, nil, err
|
||||
}
|
||||
defer sub.Unsubscribe()
|
||||
// Every chunk is held until the callback has it; the window bounds how many there can be.
|
||||
if err := sub.SetPendingLimits(-1, -1); err != nil {
|
||||
return busarchive.StreamEntry{}, nil, err
|
||||
}
|
||||
if err := nc.FlushWithContext(ctx); err != nil {
|
||||
return busarchive.StreamEntry{}, nil, err
|
||||
}
|
||||
|
||||
var resp struct {
|
||||
Config json.RawMessage `json:"config"`
|
||||
State json.RawMessage `json:"state"`
|
||||
}
|
||||
req := map[string]any{"deliver_subject": deliver, "chunk_size": ChunkSize}
|
||||
if err := Request(ctx, nc, APIStreamSnapshot+name, req, &resp); err != nil {
|
||||
return busarchive.StreamEntry{}, nil, err
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
case <-ctx.Done():
|
||||
return busarchive.StreamEntry{}, nil, fmt.Errorf("not finished within its bound: %w", ctx.Err())
|
||||
}
|
||||
// The subscription is drained before the archive is read back, so no callback is still writing.
|
||||
_ = sub.Unsubscribe()
|
||||
mu.Lock()
|
||||
err = failed
|
||||
mu.Unlock()
|
||||
if err != nil {
|
||||
return busarchive.StreamEntry{}, nil, err
|
||||
}
|
||||
if written == 0 {
|
||||
return busarchive.StreamEntry{}, nil, errors.New("the server sent an empty archive")
|
||||
}
|
||||
if err := f.Sync(); err != nil {
|
||||
return busarchive.StreamEntry{}, nil, err
|
||||
}
|
||||
|
||||
var cfg streamConfig
|
||||
var state StreamState
|
||||
if err := json.Unmarshal(resp.Config, &cfg); err != nil {
|
||||
return busarchive.StreamEntry{}, nil, fmt.Errorf("its configuration: %w", err)
|
||||
}
|
||||
if err := json.Unmarshal(resp.State, &state); err != nil {
|
||||
return busarchive.StreamEntry{}, nil, fmt.Errorf("its state: %w", err)
|
||||
}
|
||||
// The archive reads to its end — s2 frames, then a tar — before it is called a snapshot.
|
||||
if _, err := f.Seek(0, io.SeekStart); err != nil {
|
||||
return busarchive.StreamEntry{}, nil, err
|
||||
}
|
||||
if err := busarchive.ReadsThrough(f); err != nil {
|
||||
return busarchive.StreamEntry{}, nil, fmt.Errorf("the archive the server sent does not read: %w", err)
|
||||
}
|
||||
|
||||
// The meta, in the shape the restore API takes and the nats CLI writes (backup.json): the
|
||||
// server's own configuration and state, kept as raw JSON so a field this program does not know
|
||||
// is never dropped on the way back.
|
||||
meta, err := json.MarshalIndent(map[string]json.RawMessage{"config": resp.Config, "state": resp.State}, "", " ")
|
||||
if err != nil {
|
||||
return busarchive.StreamEntry{}, nil, err
|
||||
}
|
||||
metaSum := sha256.Sum256(meta)
|
||||
return busarchive.StreamEntry{
|
||||
Name: name, Subjects: cfg.Subjects, Retention: cfg.Retention,
|
||||
Messages: state.Messages, Bytes: state.Bytes, FirstSeq: state.FirstSeq, LastSeq: state.LastSeq,
|
||||
Deleted: state.NumDelete, Consumers: state.Consumers,
|
||||
Meta: "streams/" + name + "/backup.json", MetaSHA256: hex.EncodeToString(metaSum[:]),
|
||||
Archive: "streams/" + name + "/stream.tar.s2", ArchiveBytes: written,
|
||||
ArchiveSHA256: hex.EncodeToString(hash.Sum(nil)), Seconds: time.Since(started).Seconds(),
|
||||
}, meta, nil
|
||||
}
|
||||
|
||||
// Human is a size in bytes as a person reads it: 1.5 MiB.
|
||||
func Human(n int64) string {
|
||||
const unit = 1024
|
||||
if n < unit {
|
||||
return fmt.Sprintf("%d B", n)
|
||||
}
|
||||
div, exp := int64(unit), 0
|
||||
for v := n / unit; v >= unit; v /= unit {
|
||||
div *= unit
|
||||
exp++
|
||||
}
|
||||
return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp])
|
||||
}
|
||||
|
||||
// Into is the keeper's snapshot of some streams before it rebuilds or removes them (design 25 § 12.3): one tar under
|
||||
// dir, named for the streams and the time, written whole before its name is given, so a partial one is never taken
|
||||
// for it. It answers the file's path.
|
||||
func Into(nc *nats.Conn, dir string) func(ctx context.Context, streams []string) (string, error) {
|
||||
return func(ctx context.Context, streams []string) (string, error) {
|
||||
if dir == "" {
|
||||
return "", errors.New("no snapshots directory is set (MESH_NATS_SNAPSHOTS_DIR), so no snapshot is taken and nothing is removed")
|
||||
}
|
||||
name := filepath.Join(dir, "entry-"+strings.Join(streams, "+")+"-"+time.Now().UTC().Format("20060102T150405.000000Z")+".tar")
|
||||
f, err := os.CreateTemp(dir, ".entry-*.partial")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
if _, err := Snapshot(ctx, nc, f, SnapshotOptions{Streams: streams, TempDir: dir}); err != nil {
|
||||
f.Close()
|
||||
return "", err
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return name, os.Rename(f.Name(), name)
|
||||
}
|
||||
}
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"nats-tools/internal/bussnap"
|
||||
"git.novox.be/novox/mesh-sdk/go/bussnap"
|
||||
)
|
||||
|
||||
// JetStream is the bus as the keeper's own account reaches it (Bus): each object a stream, a key-value bucket or an
|
||||
|
||||
Reference in New Issue
Block a user