diff --git a/modules/mesh-recovery/.gitignore b/modules/mesh-recovery/.gitignore new file mode 100644 index 000000000..0135b2410 --- /dev/null +++ b/modules/mesh-recovery/.gitignore @@ -0,0 +1,4 @@ +/mesh-recovery +/cmd/mesh-recovery/mesh-recovery +/cmd/mesh-recovery-tools/mesh-recovery-tools +/mesh-recovery-tools diff --git a/modules/mesh-recovery/README.md b/modules/mesh-recovery/README.md new file mode 100644 index 000000000..a1156d343 --- /dev/null +++ b/modules/mesh-recovery/README.md @@ -0,0 +1,9 @@ +# mesh-recovery + +Holds the seat `mesh-recovery` (design 47 G §554): the recovery module takes, sets aside and restores a module's kept +data on the live bus, one act at a time, each act in the hand-act log, its result said as an event under the act's id. +Go: one process (`cmd/mesh-recovery`) that answers the seat's verbs itself (A-107, A-115) and a tools bundle +(`cmd/mesh-recovery-tools`) whose `recovery_*` tools call those verbs and hold no grant of their own (§656). + +Its data item `points` holds the restore points and the asides; the digests of the points are on the bus in its state +`digests` (§567). Its settings are `points-cap`, `points-filling-mark`, `point-keep-days` and `dump-wait-minutes`. diff --git a/modules/mesh-recovery/cmd/mesh-recovery-tools/main.go b/modules/mesh-recovery/cmd/mesh-recovery-tools/main.go new file mode 100644 index 000000000..efb4a2ddd --- /dev/null +++ b/modules/mesh-recovery/cmd/mesh-recovery-tools/main.go @@ -0,0 +1,129 @@ +// mesh-recovery-tools: the recovery module's tools, its command-line interface (design 47 G §554, §656, §657). Each hand +// tool calls one verb of the seat mesh-recovery with the `asker` the tool runner proved for the call and holds no grant of +// its own: the bus proves the tool runner's account, the verb logs both in the hand-act log. stdout is the MCP channel. +package main + +import ( + "encoding/json" + "fmt" + "os" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/health" +) + +func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } + +func main() { + if err := stdio.Serve("", Tools(stdio.Ask, stdio.CallerOf)); err != nil { + logf("%v", err) + os.Exit(1) + } +} + +// Ask is how a bundle calls a seat's verb: the key `seat:.` and the arguments. +type Ask func(key string, body any) (json.RawMessage, error) + +func obj(props map[string]any, required ...string) map[string]any { + m := map[string]any{"type": "object", "properties": props} + if len(required) > 0 { + m["required"] = required + } + return m +} + +func str(d string) map[string]any { return map[string]any{"type": "string", "description": d} } + +var ( + module = str("the module") + kept = str("the kept data, its local name in the module's manifest") + why = str("why, which the hand-act log keeps") +) + +// hand is a tool that calls a verb of the seat: the arguments go through as they are, with the asker the tool runner proved. +func hand(ask Ask, callerOf func(map[string]any) string, name, verb, description string, props map[string]any, required ...string) stdio.Tool { + props["why"] = why + return stdio.Tool{Name: name, Description: description + " Class act: written in the hand-act log first with the account the bus proved and who asked; answers accepted, busy or error at once and the result comes as the act's ending event.", + Input: obj(props, append(required, "why")...), + Run: func(a map[string]any) (any, error) { + body := map[string]any{} + for k, v := range a { + body[k] = v + } + body["asker"] = callerOf(a) + raw, err := ask("seat:mesh-recovery."+verb, body) + if err != nil { + return nil, err + } + var out any + return out, json.Unmarshal(raw, &out) + }} +} + +// Tools are what the runtime serves, named recovery_ (§554); each acting tool is one verb of the seat (§656). +func Tools(ask Ask, callerOf func(map[string]any) string) []stdio.Tool { + return []stdio.Tool{ + {Name: "recovery_kept_points", + Description: "Every restore point and aside the recovery module holds, with their sizes, and the act that runs with since when and what it waits on; narrowed to a module and a kept data. Changes nothing (also the module's declared health check, §657).", + Input: obj(map[string]any{"module": module, "kept": kept}), + Run: func(a map[string]any) (any, error) { + body := map[string]any{} + for _, k := range []string{"module", "kept"} { + if s, _ := a[k].(string); strings.TrimSpace(s) != "" { + body[k] = s + } + } + raw, err := ask("seat:mesh-recovery.kept-points", body) + if err != nil { + return nil, err + } + return keptPointsHealth(raw) + }}, + hand(ask, callerOf, "recovery_take", "take", "Take a restore point of one module's kept data now, at the form of the given fingerprint.", + map[string]any{"module": module, "kept": kept, "fingerprint": str("the fingerprint whose form the point is taken at")}, "module", "kept", "fingerprint"), + hand(ask, callerOf, "recovery_restore", "restore", "Restore one point (snapshot, a point's name from recovery_kept_points) or a keeper's snapshot (request), exactly one of them.", + map[string]any{"module": module, "kept": kept, "snapshot": str("the restore point's name"), "request": str("the keeper's snapshot by its request"), + "from-backup": map[string]any{"type": "boolean", "description": "ask the backup holder for the points directory when the point is missing, only with snapshot"}}, "module", "kept"), + hand(ask, callerOf, "recovery_retry", "retry", "Ask again the last failed act of a module and kept data, with its fields.", + map[string]any{"module": module, "kept": kept}, "module", "kept"), + hand(ask, callerOf, "recovery_aside", "aside", "Set the live kept data aside without restoring: a move, the data stays in the aside until it is put back or discarded.", + map[string]any{"module": module, "kept": kept}, "module", "kept"), + hand(ask, callerOf, "recovery_discard_aside", "discard-aside", "Remove one aside's directory and nothing else.", + map[string]any{"module": module, "kept": kept, "id": str("the aside's id")}, "module", "kept", "id"), + hand(ask, callerOf, "recovery_restore_aside", "restore-aside", "Put an aside back under the live names it was taken from, refused while live objects of those names exist.", + map[string]any{"module": module, "kept": kept, "id": str("the aside's id")}, "module", "kept", "id"), + hand(ask, callerOf, "recovery_cancel", "cancel", "End an accepted act that has not begun its work (one waiting on the dump) failed, cancelled; refused naming the step of an act that has begun.", + map[string]any{"act": str("the act's id")}, "act"), + hand(ask, callerOf, "recovery_expire_points", "expire-points", "Remove the restore points past their keep days now, never the newest of a kept data.", map[string]any{}), + } +} + +// keptPointsHealth is what kept-points answers with the health answer beside it, which the module's declared health +// check reads (§554, §657, A-129): unhealthy while an aside only it holds the data of is listed as unrestored. +func keptPointsHealth(raw json.RawMessage) (map[string]any, error) { + var out map[string]any + if err := json.Unmarshal(raw, &out); err != nil { + return nil, err + } + var kp struct { + Unrestored []struct { + ID string `json:"id"` + Names []string `json:"names"` + } `json:"unrestored"` + } + if err := json.Unmarshal(raw, &kp); err != nil { + return nil, err + } + h := health.Healthy("No aside holds data the live bus lacks.") + if len(kp.Unrestored) > 0 { + var said []string + for _, a := range kp.Unrestored { + said = append(said, a.ID+" ("+strings.Join(a.Names, ", ")+")") + } + h = health.Unhealthy(fmt.Sprintf("%d aside(s) hold data whose live objects were removed and not restored: %s; recovery_restore_aside puts one back.", + len(kp.Unrestored), strings.Join(said, "; "))) + } + out["state"], out["healthy"], out["why"] = h.State, h.Healthy, h.Why + return out, nil +} diff --git a/modules/mesh-recovery/cmd/mesh-recovery-tools/main_test.go b/modules/mesh-recovery/cmd/mesh-recovery-tools/main_test.go new file mode 100644 index 000000000..ded8d73f7 --- /dev/null +++ b/modules/mesh-recovery/cmd/mesh-recovery-tools/main_test.go @@ -0,0 +1,78 @@ +package main + +import ( + "encoding/json" + "os" + "slices" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// The bundle serves exactly the tools the manifest names, each named recovery_, and each acting tool calls its verb of the +// seat with the asker the tool runner proved, written into the call, and nothing of its own (design 47 G §554, §656, §682). +func TestEachToolCallsItsVerbWithTheProvedAsker(t *testing.T) { + var key string + var sent map[string]any + ask := func(k string, b any) (json.RawMessage, error) { + key, sent = k, b.(map[string]any) + return json.RawMessage(`{"accepted":"act-1"}`), nil + } + tools := Tools(ask, func(map[string]any) string { return "claude@shanks" }) + raw, _ := os.ReadFile("../../module.json") + var m struct { + Tools []string `json:"tools"` + Invokes []string `json:"invokes"` + } + _ = json.Unmarshal(raw, &m) + var names []string + verbs := map[string]string{"recovery_take": "take", "recovery_restore": "restore", "recovery_retry": "retry", "recovery_aside": "aside", + "recovery_discard_aside": "discard-aside", "recovery_restore_aside": "restore-aside", "recovery_cancel": "cancel", "recovery_expire_points": "expire-points"} + for _, tool := range tools { + names = append(names, tool.Name) + if !strings.HasPrefix(tool.Name, "recovery_") { + t.Errorf("tool %s is not named recovery_", tool.Name) + } + if verb, acts := verbs[tool.Name]; acts { + if _, err := tool.Run(map[string]any{"module": "app", "why": "w"}); err != nil { + t.Fatal(err) + } + if key != "seat:mesh-recovery."+verb || sent["asker"] != "claude@shanks" || sent["module"] != "app" { + t.Errorf("%s called %s with %v", tool.Name, key, sent) + } + if !slices.Contains(m.Invokes, key) { + t.Errorf("the manifest does not invoke %s", key) + } + } + } + slices.Sort(names) + want := slices.Clone(m.Tools) + slices.Sort(want) + if !slices.Equal(names, want) { + t.Errorf("tools %v, the manifest names %v", names, want) + } +} + +// recovery_kept_points answers what kept-points answers with the health answer beside it: unhealthy while an aside is +// unrestored, healthy otherwise, as the declared health check reads it (§657, A-129). +func TestKeptPointsIsTheHealthCheck(t *testing.T) { + for _, c := range []struct { + answer string + state string + }{ + {`{"points":[],"asides":[],"unrestored":[]}`, health.StateHealthy}, + {`{"points":[],"asides":[{"id":"x","names":["app_notes"],"size":1}],"unrestored":[{"id":"x","names":["app_notes"],"size":1}]}`, health.StateUnhealthy}, + } { + tools := Tools(func(string, any) (json.RawMessage, error) { return json.RawMessage(c.answer), nil }, func(map[string]any) string { return "" }) + out, err := tools[0].Run(map[string]any{}) + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(out) + a, err := health.Read(raw) + if err != nil || a.State != c.state || !strings.Contains(string(raw), `"unrestored"`) { + t.Errorf("%s answered %s: %+v, %v", c.answer, raw, a, err) + } + } +} diff --git a/modules/mesh-recovery/cmd/mesh-recovery/bus.go b/modules/mesh-recovery/cmd/mesh-recovery/bus.go new file mode 100644 index 000000000..10dbac97c --- /dev/null +++ b/modules/mesh-recovery/cmd/mesh-recovery/bus.go @@ -0,0 +1,98 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io" + "os" + + seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" + "git.novox.be/novox/mesh-sdk/go/bussnap" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "mesh-recovery/internal/recovery" +) + +// liveBus is the bus as the recovery module's own account sees it (design 47 G §554): a stream's snapshot and restore +// through mesh-sdk's bussnap (§655, A-125), its deletion, and its durable consumers' positions. +type liveBus struct { + nc *nats.Conn + js jetstream.JetStream + scratch string + logf func(string, ...any) +} + +func (b liveBus) Objects(ctx context.Context, streams []string) ([]recovery.Object, error) { + var out []recovery.Object + for _, name := range streams { + s, err := b.js.Stream(ctx, name) + if errors.Is(err, jetstream.ErrStreamNotFound) { + out = append(out, recovery.Object{Stream: name}) + continue + } + if err != nil { + return nil, fmt.Errorf("stream %s: %w", name, err) + } + info, err := s.Info(ctx) + if err != nil { + return nil, fmt.Errorf("stream %s: %w", name, err) + } + o := recovery.Object{Stream: name, Exists: true, Bytes: int64(info.State.Bytes)} + names := s.ListConsumers(ctx) + for ci := range names.Info() { + if ci.Config.Durable == "" { + continue + } + o.Consumers = append(o.Consumers, seat.Consumer{Stream: name, Name: ci.Name, Sequence: int64(ci.AckFloor.Stream)}) + } + if err := names.Err(); err != nil { + return nil, fmt.Errorf("the consumers of %s: %w", name, err) + } + out = append(out, o) + } + return out, nil +} + +func (b liveBus) Take(ctx context.Context, stream string, w io.Writer) error { + _, err := bussnap.Snapshot(ctx, b.nc, w, bussnap.SnapshotOptions{Streams: []string{stream}, TempDir: b.scratch, Log: b.logf}) + return err +} + +func (b liveBus) Delete(ctx context.Context, stream string) error { + return bussnap.DeleteStream(ctx, b.nc, stream) +} + +func (b liveBus) Restore(ctx context.Context, file string) error { + scratch, err := os.MkdirTemp(b.scratch, ".restore-") + if err != nil { + return err + } + defer os.RemoveAll(scratch) + return bussnap.RestoreFile(ctx, b.nc, file, scratch, b.logf) +} + +// SetConsumers sets each durable consumer to its recorded sequence: the server's snapshot carries its consumers, so one +// restored with its stream stands as it was taken; one missing is made again from the sequence after its acknowledged one. +func (b liveBus) SetConsumers(ctx context.Context, cs []seat.Consumer) error { + for _, c := range cs { + s, err := b.js.Stream(ctx, c.Stream) + if err != nil { + return fmt.Errorf("stream %s: %w", c.Stream, err) + } + if _, err := s.Consumer(ctx, c.Name); err == nil { + continue + } else if !errors.Is(err, jetstream.ErrConsumerNotFound) { + return fmt.Errorf("the consumer %s of %s: %w", c.Name, c.Stream, err) + } + if _, err := s.CreateConsumer(ctx, jetstream.ConsumerConfig{Durable: c.Name, AckPolicy: jetstream.AckExplicitPolicy, + DeliverPolicy: jetstream.DeliverByStartSequencePolicy, OptStartSeq: uint64(c.Sequence) + 1}); err != nil { + return fmt.Errorf("the consumer %s of %s at %d: %w", c.Name, c.Stream, c.Sequence, err) + } + } + return nil +} + +var _ recovery.Bus = liveBus{} diff --git a/modules/mesh-recovery/cmd/mesh-recovery/dump.go b/modules/mesh-recovery/cmd/mesh-recovery/dump.go new file mode 100644 index 000000000..3fb2d9b83 --- /dev/null +++ b/modules/mesh-recovery/cmd/mesh-recovery/dump.go @@ -0,0 +1,163 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/keys" + + "github.com/nats-io/nats.go/jetstream" + + "mesh-recovery/internal/recovery" +) + +// The seat node-backup's item events (design 47 G §554, §563), each kept last per subject on a stream of its own. +const ( + backupSeat = "node-backup" + itemStarted = "item-started" + itemEnded = "item-ended" + itemStartedSteam = "SEAT_node-backup_item-started" + itemEndedStream = "SEAT_node-backup_item-ended" +) + +func itemSubject(event, node, module, item string) string { + return "mesh.seat." + backupSeat + ".event." + event + "." + node + "." + module + "." + item +} + +// dumpView is what the backup holders say of each item, `..` to the `at` of its last start and of its +// last end, read as §556 has it. A dump of an item runs while its last start is newer than its last end. +type dumpView struct { + mu sync.Mutex + started map[string]time.Time + ended map[string]time.Time + changed chan struct{} + // busModule is the module that holds the seat mesh-broker now (A-126), whose items are the bus's. + busModule func() (string, error) + js jetstream.JetStream +} + +func newDumpView(js jetstream.JetStream, busModule func() (string, error)) *dumpView { + return &dumpView{started: map[string]time.Time{}, ended: map[string]time.Time{}, changed: make(chan struct{}), + busModule: busModule, js: js} +} + +// hear takes one item event, an older one by `at` than the one held changing nothing, and wakes every wait. +func (d *dumpView) hear(m jetstream.Msg) { + d.take(m.Subject(), m.Data()) + _ = m.Ack() +} + +func (d *dumpView) take(subject string, data []byte) { + var body struct { + At time.Time `json:"at"` + } + rest, ok := strings.CutPrefix(subject, "mesh.seat."+backupSeat+".event.") + if !ok || json.Unmarshal(data, &body) != nil { + return + } + event, item, ok := strings.Cut(rest, ".") + if !ok { + return + } + d.mu.Lock() + defer d.mu.Unlock() + into := d.started + if event == itemEnded { + into = d.ended + } else if event != itemStarted { + return + } + if old, ok := into[item]; ok && body.At.Before(old) { + return + } + into[item] = body.At + close(d.changed) + d.changed = make(chan struct{}) +} + +// runs is the first item of the module whose dump runs, or "". +func (d *dumpView) runs(module string) string { + token := keys.EscapeName(module) + var on []string + for item, at := range d.started { + parts := strings.Split(item, ".") + if len(parts) != 3 || parts[1] != token { + continue + } + if end, ok := d.ended[item]; !ok || end.Before(at) { + on = append(on, item) + } + } + sort.Strings(on) + if len(on) == 0 { + return "" + } + return on[0] +} + +// readNow reads the last start and end of each of the bus module's items once more, through an ordered consumer made for +// that read (§556): the restore said its restore-started, acknowledged, before it reads (§563). +func (d *dumpView) readNow(ctx context.Context, module string) error { + for _, c := range []struct{ stream, event string }{{itemStartedSteam, itemStarted}, {itemEndedStream, itemEnded}} { + cons, err := d.js.OrderedConsumer(ctx, c.stream, jetstream.OrderedConsumerConfig{ + FilterSubjects: []string{itemSubject(c.event, "*", keys.EscapeName(module), "*")}, + DeliverPolicy: jetstream.DeliverLastPerSubjectPolicy}) + if err != nil { + return fmt.Errorf("the read of %s: %w", c.stream, err) + } + info, err := cons.Info(ctx) + if err != nil { + return fmt.Errorf("the read of %s: %w", c.stream, err) + } + for left := info.NumPending; left > 0; { + m, err := cons.Next(jetstream.FetchMaxWait(5 * time.Second)) + if err != nil { + return fmt.Errorf("the read of %s: %w", c.stream, err) + } + d.take(m.Subject(), m.Data()) + meta, err := m.Metadata() + if err != nil { + return err + } + left = meta.NumPending + } + } + return nil +} + +// Wait returns once no dump of the bus module's items runs; it calls waiting once, naming the item, when it has to wait +// (§554, §563). It wakes on the item events, never on a timer. +func (d *dumpView) Wait(ctx context.Context, waiting func(on string)) error { + module, err := d.busModule() + if err != nil { + return err + } + if err := d.readNow(ctx, module); err != nil { + return err + } + said := false + for { + d.mu.Lock() + on, ch := d.runs(module), d.changed + d.mu.Unlock() + if on == "" { + return nil + } + if !said { + waiting("the item-ended of the dump of " + on) + said = true + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-ch: + } + } +} + +var _ recovery.Dump = (*dumpView)(nil) diff --git a/modules/mesh-recovery/cmd/mesh-recovery/expiry.go b/modules/mesh-recovery/cmd/mesh-recovery/expiry.go new file mode 100644 index 000000000..9dfb5b96a --- /dev/null +++ b/modules/mesh-recovery/cmd/mesh-recovery/expiry.go @@ -0,0 +1,82 @@ +package main + +import ( + "errors" + "sync/atomic" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + + "mesh-recovery/internal/recovery" +) + +// boundState is the module's own state bounds, whose keys live a time of their own (key-ttl): a bound key's expiry is +// heard as the bus's own change, so nothing in the process waits on a timer (design 47 G A-77, A-138). +type boundState interface { + CreateTTL(key string, value any, ttl time.Duration) (uint64, error) + Watch(key string, handle func(stdio.StateChange) error) error + Delete(key string) error +} + +// errBoundEnded ends a bound key's watch once the key is gone. +var errBoundEnded = errors.New("the bound key is removed") + +// stateBounds times an act's waits by bound keys of the state bounds (A-77, A-143): the key is watched before it is +// written, so its expiry is never missed, and the watch ends with the key's removal. +type stateBounds struct{ s boundState } + +func (b stateBounds) Bound(key string, ttl time.Duration, expired func()) (func(), error) { + var ended atomic.Bool + if err := b.s.Watch(key, func(c stdio.StateChange) error { + if c.Op == "put" || c.Current { + return nil + } + if !ended.Swap(true) { + expired() + } + return errBoundEnded + }); err != nil { + return nil, err + } + if _, err := b.s.CreateTTL(key, map[string]any{"ttl": ttl.String()}, ttl); err != nil { + ended.Store(true) + _ = b.s.Delete(key) + return nil, err + } + return func() { + if !ended.Swap(true) { + _ = b.s.Delete(key) + } + }, nil +} + +var _ recovery.Bounds = stateBounds{} + +// expiryKey is the bound key of §551's daily removal. +const expiryKey = "expiry" + +// expireOnBound runs due each time the bound key `expiry`, written with the time to live `every` (expire-every-hours), is +// removed by the bus, and writes it again (§551, A-138). A key a run before this start wrote is kept, so a restart does not +// put the next removal off; its removal at the start is the start's own run. +func expireOnBound(s boundState, every time.Duration, due func(), logf func(string, ...any)) error { + arm := func() error { + _, err := s.CreateTTL(expiryKey, map[string]any{"every": every.String()}, every) + if errors.Is(err, stdio.ErrChanged) { + return nil + } + return err + } + if err := s.Watch(expiryKey, func(c stdio.StateChange) error { + if c.Op == "put" || c.Current { + return nil + } + due() + if err := arm(); err != nil { + logf("the bound of the next removal of points past their keep days is not written: %v", err) + } + return nil + }); err != nil { + return err + } + return arm() +} diff --git a/modules/mesh-recovery/cmd/mesh-recovery/keeper.go b/modules/mesh-recovery/cmd/mesh-recovery/keeper.go new file mode 100644 index 000000000..1d60219c4 --- /dev/null +++ b/modules/mesh-recovery/cmd/mesh-recovery/keeper.go @@ -0,0 +1,151 @@ +package main + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "sync" + "time" + + "git.novox.be/novox/mesh-contracts/go/seats/meshbroker" + "git.novox.be/novox/mesh-sdk/go/buscalls" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "mesh-recovery/internal/recovery" +) + +// liveKeeper is the bus module's keeper reached by the seat mesh-broker's verbs (design 47 G §681): snapshot-get's chunks +// received in mesh-sdk's form (A-139), snapshot-restore and its cancel called, and snapshot-restored heard. +type liveKeeper struct { + nc *nats.Conn + call Call + restored *restoredView +} + +func (k liveKeeper) verb(verb string, in, out any) error { + raw, err := k.call("seat:"+meshbroker.Seat+"."+verb, in) + if err != nil { + return fmt.Errorf("%s's %s: %w", meshbroker.Seat, verb, err) + } + if err := json.Unmarshal(raw, out); err != nil { + return fmt.Errorf("%s's %s answered what does not read: %w", meshbroker.Seat, verb, err) + } + return nil +} + +// Get subscribes to the chunks of a call of its own before it asks snapshot-get, so none is missed, then receives them in +// order, acknowledging each (§681, A-139). +func (k liveKeeper) Get(ctx context.Context, request string, fits func(meshbroker.SnapshotGetAnswer) error, w io.Writer) (meshbroker.SnapshotGetAnswer, error) { + var bits [16]byte + if _, err := rand.Read(bits[:]); err != nil { + return meshbroker.SnapshotGetAnswer{}, err + } + send := buscalls.SeatChunks{Seat: meshbroker.Seat, Verb: meshbroker.VerbSnapshotGet, Call: hex.EncodeToString(bits[:])} + chunks := make(chan *nats.Msg, 4) + sub, err := k.nc.ChanSubscribe(send.Chunks(), chunks) + if err != nil { + return meshbroker.SnapshotGetAnswer{}, fmt.Errorf("the chunks of snapshot-get cannot be heard: %w", err) + } + defer func() { _ = sub.Unsubscribe() }() + var ans meshbroker.SnapshotGetAnswer + if err := k.verb(meshbroker.VerbSnapshotGet, meshbroker.SnapshotGetInput{Call: send.Call, Request: request}, &ans); err != nil { + return ans, err + } + if ans.Error != "" { + return ans, nil + } + if err := fits(ans); err != nil { + return ans, err + } + if ans.Size > 0 && ans.Chunks == 0 { + return ans, fmt.Errorf("the keeper answered the request %s's %d bytes with no chunk: this account is granted no chunk subjects of snapshot-get", request, ans.Size) + } + next := func() (string, []byte, error) { + bound := time.NewTimer(buscalls.SeatAckWithin) + defer bound.Stop() + select { + case m := <-chunks: + return m.Subject, m.Data, nil + case <-bound.C: + return "", nil, buscalls.ErrSeatChunkMissed + case <-ctx.Done(): + return "", nil, ctx.Err() + } + } + ack := func(subject string) error { + _, err := k.nc.Request(subject, []byte("{}"), buscalls.SeatAckWithin) + return err + } + if _, err := send.Receive(ans.Chunks, next, ack, w); err != nil { + return ans, fmt.Errorf("the snapshot of the request %s: %w", request, err) + } + return ans, nil +} + +func (k liveKeeper) Restore(_ context.Context, request string, verify bool, why string) (meshbroker.SnapshotRestoreAnswer, error) { + var ans meshbroker.SnapshotRestoreAnswer + err := k.verb(meshbroker.VerbSnapshotRestore, meshbroker.SnapshotRestoreInput{Request: request, Verify: verify, Why: why}, &ans) + return ans, err +} + +func (k liveKeeper) CancelRestore(_ context.Context, act, why string) (meshbroker.SnapshotRestoreCancelAnswer, error) { + var ans meshbroker.SnapshotRestoreCancelAnswer + err := k.verb(meshbroker.VerbSnapshotRestoreCancel, meshbroker.SnapshotRestoreCancelInput{Act: act, Why: why}, &ans) + return ans, err +} + +func (k liveKeeper) Restored(ctx context.Context, request, act string) (meshbroker.SnapshotRestoredEvent, bool) { + return k.restored.wait(ctx, request, act) +} + +var _ recovery.Keeper = liveKeeper{} + +// restoredView is the keeper's snapshot-restored as heard, each request and act kept (§681): read as §556 has it, the last +// of each subject first, so an outcome said before a wait began wakes it too. +type restoredView struct { + mu sync.Mutex + by map[string]meshbroker.SnapshotRestoredEvent + changed chan struct{} +} + +func newRestoredView() *restoredView { + return &restoredView{by: map[string]meshbroker.SnapshotRestoredEvent{}, changed: make(chan struct{})} +} + +func (v *restoredView) hear(m jetstream.Msg) { + var ev meshbroker.SnapshotRestoredEvent + if err := json.Unmarshal(m.Data(), &ev); err == nil && ev.Request != "" && ev.Act != "" { + v.take(ev) + } + _ = m.Ack() +} + +func (v *restoredView) take(ev meshbroker.SnapshotRestoredEvent) { + v.mu.Lock() + v.by[ev.Request+"\x00"+ev.Act] = ev + close(v.changed) + v.changed = make(chan struct{}) + v.mu.Unlock() +} + +func (v *restoredView) wait(ctx context.Context, request, act string) (meshbroker.SnapshotRestoredEvent, bool) { + for { + v.mu.Lock() + ev, ok := v.by[request+"\x00"+act] + changed := v.changed + v.mu.Unlock() + if ok { + return ev, true + } + select { + case <-changed: + case <-ctx.Done(): + return meshbroker.SnapshotRestoredEvent{}, false + } + } +} diff --git a/modules/mesh-recovery/cmd/mesh-recovery/main.go b/modules/mesh-recovery/cmd/mesh-recovery/main.go new file mode 100644 index 000000000..3c50f938c --- /dev/null +++ b/modules/mesh-recovery/cmd/mesh-recovery/main.go @@ -0,0 +1,195 @@ +// mesh-recovery: the recovery module's process (design 47 G §554). One process on an account of its own holds the seat +// mesh-recovery and answers its verbs itself over the bus with the caller the bus proved (A-107); it takes, sets aside and +// restores modules' kept data on the live bus, one act at a time, each act's result said as the seat's event. +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "os" + "os/signal" + "syscall" + "time" + + "git.novox.be/novox/mesh-contracts/go/seats/meshbroker" + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/seatlist" + "git.novox.be/novox/mesh-sdk/go/views" + + "github.com/nats-io/nats.go" + + "mesh-recovery/internal/recovery" +) + +// The settings' files and the points directory, as the manifest's process section sets them (A-112, §633). +const ( + envPoints = "MESH_RECOVERY_POINTS" + envCap = "MESH_RECOVERY_POINTS_CAP_FILE" + envKeepDays = "MESH_RECOVERY_KEEP_DAYS_FILE" + envMark = "MESH_RECOVERY_FILLING_MARK_FILE" + envDumpWait = "MESH_RECOVERY_DUMP_WAIT_FILE" + envRestore = "MESH_RECOVERY_RESTORE_WAIT_FILE" + envExpire = "MESH_RECOVERY_EXPIRE_EVERY_FILE" +) + +// The settings' defaults: the design's values (§551, §564). +const ( + defaultCap = 10 << 30 + defaultKeepDays = 90 +) + +// defaultExpireEvery is how often §551's removal runs beside its start and each snapshot-kept: once a day (expire-every-hours). +const defaultExpireEvery = 24 * time.Hour + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + if err := run(ctx, os.Getenv, os.Stderr); err != nil { + fmt.Fprintln(os.Stderr, "mesh-recovery:", err) + os.Exit(1) + } +} + +// digests is the state digests, the points' digests kept on the bus apart from their bytes (§567). +type digests struct{ s process.State } + +func (d digests) Put(key string, v []byte) error { + _, err := d.s.Put(key, json.RawMessage(v)) + return err +} + +func (d digests) Get(key string) ([]byte, bool, error) { + e, err := d.s.Get(key) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} + +func (d digests) Delete(key string) error { return d.s.Delete(key) } + +// run is the process: it returns when ctx ends, after the running act ended. +func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error { + logger := log.New(logOut, "mesh-recovery: ", log.LstdFlags) + dir := getenv(envPoints) + if dir == "" { + return errors.New(envPoints + " names no points directory, so nothing can be kept") + } + cap, err := readNumber(getenv(envCap), defaultCap) + if err != nil { + return err + } + keepDays, err := readFraction(getenv(envKeepDays), defaultKeepDays) + if err != nil { + return err + } + mark, err := readNumber(getenv(envMark), recovery.DefaultFillingMark) + if err != nil { + return err + } + dumpWait, err := readMinutes(getenv(envDumpWait), recovery.DefaultDumpWait) + if err != nil { + return err + } + restoreWait, err := readMinutes(getenv(envRestore), recovery.DefaultRestoreWait) + if err != nil { + return err + } + expireEvery, err := readFraction(getenv(envExpire), defaultExpireEvery.Hours()) + if err != nil { + return err + } + account, err := process.ReadBrokerAccount(getenv) + if err != nil { + return err + } + conn, err := process.Connect(account) + if err != nil { + return err + } + defer conn.Close() + nc, js, err := account.Dial(nats.Name(account.Node + "." + account.Module)) + if err != nil { + return err + } + defer nc.Close() + + // What the engine reads of the bus, each read as §556 has it before it acts on it: the node-engine seat's running + // (A-132) and the backup holders' item events of the bus module (§563). + running := newRunningView() + if _, err := account.HearSeatEvents(ctx, js, nodeengine.StreamRunning, running.hear, logger.Printf, process.OnRead(running.onRead)); err != nil { + return err + } + dump := newDumpView(js, func() (string, error) { return seatlist.HolderOf(conn.Call, meshbroker.Seat) }) + for _, s := range []string{itemStartedSteam, itemEndedStream} { + if _, err := account.HearSeatEvents(ctx, js, s, dump.hear, logger.Printf); err != nil { + return err + } + } + // The keeper's snapshot-restored, each request and act, for a restore in place (§681). + restored := newRestoredView() + if _, err := account.HearSeatEvents(ctx, js, meshbroker.StreamSnapshotRestored, restored.hear, logger.Printf); err != nil { + return err + } + // The module's findings go through mesh-sdk's held writer over its own state findings (§561), started before the + // engine's start, so what the start writes is held and written. + writer, err := views.NewHeldWriter(views.WriterConfig[meshconditions.Finding, meshconditions.FindingCleared, meshconditions.FindingsStated, meshconditions.FindingsAskedEvent]{ + Asked: meshconditions.SubjectFindingsAsked, Module: account.Module, Node: account.Node, State: conn.State("findings"), + Kinds: recovery.FindingKinds, JS: js, Conn: nc, Logf: logger.Printf}) + if err != nil { + return err + } + if err := writer.Start(ctx); err != nil { + return err + } + defer writer.Close() + e := &recovery.Engine{ + Dir: dir, + Bus: liveBus{nc: nc, js: js, scratch: dir, logf: logger.Printf}, + Seats: liveSeats{emit: func(seat, event string, body any) error { return conn.EmitSeat(seat, event, body) }, + recorder: meshhandactsclient.Recorder{Ask: conn.Call}, call: conn.Call, callOn: conn.CallOn, + node: account.Node, points: dir, running: running}, + Digests: digests{conn.State("digests")}, + Dump: dump, + Keeper: liveKeeper{nc: nc, call: conn.Call, restored: restored}, + Bounds: stateBounds{conn.State("bounds")}, + Settings: recovery.Settings{Cap: cap, KeepDays: keepDays, FillingMark: int(mark), DumpWait: dumpWait, RestoreWait: restoreWait}, + Findings: writer, + Node: account.Node, + Logf: logger.Printf, + } + for _, x := range e.Start(ctx) { + logger.Printf("the aside %s of %s's %s holds data the live bus lacks: recovery_restore_aside puts it back", x.Aside.ID, x.Module, x.Kept) + } + stopAnswers, err := conn.AnswerSeat(recovery.Seat, e.Handlers()) + if err != nil { + return fmt.Errorf("the seat %s's verbs cannot be answered: %w", recovery.Seat, err) + } + defer stopAnswers() + logger.Printf("started on the account %s, holding the seat %s", account.User, recovery.Seat) + expire := func(why string) { + if a := e.Accept(recovery.Act{Kind: recovery.KindExpire, Why: why, Caller: account.User}); a.Accepted == "" { + logger.Printf("the removal of points past their keep days is not run now (%s%s); it runs at the next snapshot-kept or day", a.Busy, a.Error) + } + } + expire("the removal of points past their keep days at the start (design 47 G section 551)") + every := time.Duration(expireEvery * float64(time.Hour)) + if err := expireOnBound(conn.State("bounds"), every, func() { + expire("the removal of points past their keep days on its bound of " + every.String() + " (design 47 G section 551)") + }, logger.Printf); err != nil { + return fmt.Errorf("the bound of the removal of points past their keep days: %w", err) + } + <-ctx.Done() + // A move or a restart stops the process: it takes no new call, the running act ends through its own failure path, and + // every accepted act ends stopped, so its caller asks again (§562, A-141). + e.Stop("its stop signal") + stopAnswers() + return nil +} diff --git a/modules/mesh-recovery/cmd/mesh-recovery/main_test.go b/modules/mesh-recovery/cmd/mesh-recovery/main_test.go new file mode 100644 index 000000000..e0e4482d8 --- /dev/null +++ b/modules/mesh-recovery/cmd/mesh-recovery/main_test.go @@ -0,0 +1,181 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-contracts/go/seats/meshbroker" + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/process" +) + +// The process's own readings of the mesh (design 47 G §554, §563, A-121, A-132): the bus names of a kept data from its +// manifest, the fingerprint a module runs at from the node-engine seat's running, and whether a dump of the bus module runs. +func TestWhatTheProcessReads(t *testing.T) { + t.Run("bus names of a kept data (A-121)", func(t *testing.T) { + m, err := catalogue.Decode([]byte(`{"module":"app","version":"1","state":[{"name":"old"}],"logs":[{"name":"journal"}], + "streams":[{"name":"APP_EVENTS"}],"objects":[{"name":"files"}], + "requires":[{"name":"notes","storage":"key-value","template":"state"}]}`)) + if err != nil { + t.Fatal(err) + } + for kept, want := range map[string]string{"notes": "KV_app_notes OBJ_app_notes-objects", "old": "KV_app_old", + "journal": "LOG_app_journal", "APP_EVENTS": "APP_EVENTS", "files": "OBJ_app_files"} { + got, err := KeptStreams("app", kept, m) + if err != nil || strings.Join(got, " ") != want { + t.Errorf("%s: %v, %v; want %s", kept, got, err, want) + } + } + if _, err := KeptStreams("app", "none", m); err == nil { + t.Error("a kept data the manifest does not declare was named") + } + }) + t.Run("the running fingerprint (A-132)", func(t *testing.T) { + v := newRunningView() + if _, err := v.Fingerprint("app"); err == nil { + t.Error("answered before the read of running ended") + } + v.onRead(process.ReadDone) + at := time.Date(2026, 11, 12, 1, 0, 0, 0, time.UTC) + v.by["a\x00app"] = nodeengine.RunningEvent{Node: "a", Module: "app", Fingerprint: "src2:1", At: at} + v.by["b\x00app"] = nodeengine.RunningEvent{Node: "b", Module: "app", Stopped: true, At: at} + if fp, err := v.Fingerprint("app"); err != nil || fp != "src2:1" { + t.Errorf("%q, %v", fp, err) + } + v.by["b\x00app"] = nodeengine.RunningEvent{Node: "b", Module: "app", Fingerprint: "src2:2", At: at} + if _, err := v.Fingerprint("app"); err == nil || !strings.Contains(err.Error(), "src2:2 on b") { + t.Errorf("two fingerprints answered %v", err) + } + }) + t.Run("the keeper's snapshot-restored of a request and act (§681)", func(t *testing.T) { + v := newRestoredView() + v.take(meshbroker.SnapshotRestoredEvent{Request: "req-1", Act: "act-0", Outcome: "failed"}) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, ok := v.wait(ctx, "req-1", "act-1"); ok { + t.Error("another act's outcome woke the wait") + } + woken := make(chan meshbroker.SnapshotRestoredEvent, 1) + go func() { + ev, _ := v.wait(context.Background(), "req-1", "act-1") + woken <- ev + }() + v.take(meshbroker.SnapshotRestoredEvent{Request: "req-1", Act: "act-1", Outcome: "done"}) + if ev := <-woken; ev.Outcome != "done" { + t.Errorf("woken by %+v", ev) + } + if ev, ok := v.wait(ctx, "req-1", "act-1"); !ok || ev.Outcome != "done" { + t.Errorf("an outcome said before the wait is not taken: %+v %v", ev, ok) + } + }) + t.Run("every bound is a setting, a fraction allowed (§557, A-138)", func(t *testing.T) { + dir := t.TempDir() + write := func(name, v string) string { + p := filepath.Join(dir, name) + _ = os.WriteFile(p, []byte(v+"\n"), 0o600) + return p + } + if d, err := readMinutes(write("wait", "0.05"), time.Hour); err != nil || d != 3*time.Second { + t.Errorf("0.05 minutes read %s, %v", d, err) + } + if n, err := readFraction(write("days", "0.00002"), 90); err != nil || n != 0.00002 { + t.Errorf("a fraction of a day read %v, %v", n, err) + } + if n, err := readFraction(filepath.Join(dir, "none"), 24); err != nil || n != 24 { + t.Errorf("a missing setting read %v, %v", n, err) + } + if _, err := readFraction(write("zero", "0"), 24); err == nil { + t.Error("a bound of 0 was taken") + } + }) + t.Run("the expiry runs on its bound key's expiry (§551, A-77, A-138)", func(t *testing.T) { + s := &fakeBounds{} + runs := 0 + if err := expireOnBound(s, 2*time.Second, func() { runs++ }, t.Logf); err != nil { + t.Fatal(err) + } + _ = s.watch(stdio.StateChange{Key: expiryKey, Op: "put"}) + _ = s.watch(stdio.StateChange{Key: expiryKey, Op: "delete"}) + if runs != 1 || len(s.created) != 2 || s.created[1] != 2*time.Second { + t.Errorf("%d runs, keys written %v", runs, s.created) + } + s.exists = true // a key a run before this start wrote stands + if err := expireOnBound(s, time.Hour, func() {}, t.Logf); err != nil { + t.Errorf("a standing bound key is refused: %v", err) + } + }) + t.Run("an act's wait is a bound key, its expiry heard on its watch (A-143)", func(t *testing.T) { + s := &fakeBounds{} + expired := 0 + end, err := stateBounds{s}.Bound("dump-wait.act-1", 3*time.Second, func() { expired++ }) + if err != nil || len(s.created) != 1 || s.created[0] != 3*time.Second { + t.Fatalf("%v, keys written %v", err, s.created) + } + if s.watch(stdio.StateChange{Key: "dump-wait.act-1", Op: "put"}) != nil || expired != 0 { + t.Error("the key's own write ended the wait") + } + if s.watch(stdio.StateChange{Key: "dump-wait.act-1", Op: "delete"}) == nil || expired != 1 { + t.Errorf("the key's expiry: %d calls, its watch not ended", expired) + } + end() + if expired != 1 || len(s.deleted) != 0 { + t.Errorf("an expired key's end: %d calls, deleted %v", expired, s.deleted) + } + s = &fakeBounds{} + end, _ = stateBounds{s}.Bound("restore-wait.act-2", time.Minute, func() { expired++ }) + end() + if s.watch(stdio.StateChange{Key: "restore-wait.act-2", Op: "delete"}) == nil || expired != 1 || + strings.Join(s.deleted, " ") != "restore-wait.act-2" { + t.Errorf("an ended key's removal: %d calls, deleted %v", expired, s.deleted) + } + }) + t.Run("a dump of the bus module runs (§563)", func(t *testing.T) { + d := newDumpView(nil, nil) + start := time.Date(2026, 11, 12, 2, 0, 0, 0, time.UTC) + d.take(itemSubject(itemStarted, "srv", "nats", "data"), []byte(`{"at":"`+start.Format(time.RFC3339)+`"}`)) + d.take(itemSubject(itemStarted, "srv", "app", "data"), []byte(`{"at":"`+start.Format(time.RFC3339)+`"}`)) + if on := d.runs("nats"); on != "srv.nats.data" { + t.Errorf("runs %q", on) + } + d.take(itemSubject(itemEnded, "srv", "nats", "data"), []byte(`{"at":"`+start.Add(time.Minute).Format(time.RFC3339)+`","outcome":"done"}`)) + if on := d.runs("nats"); on != "" { + t.Errorf("an ended dump runs: %q", on) + } + d.take(itemSubject(itemEnded, "srv", "nats", "data"), []byte(`{"at":"`+start.Add(-time.Hour).Format(time.RFC3339)+`","outcome":"done"}`)) + if on := d.runs("nats"); on != "" { + t.Errorf("an older end changed what runs: %q", on) + } + }) +} + +// fakeBounds is the state bounds: the keys written with their time to live, and the watch's handler. +type fakeBounds struct { + created []time.Duration + exists bool + watch func(stdio.StateChange) error + deleted []string +} + +func (f *fakeBounds) CreateTTL(_ string, _ any, ttl time.Duration) (uint64, error) { + if f.exists { + return 0, stdio.ErrChanged + } + f.created = append(f.created, ttl) + return 1, nil +} + +func (f *fakeBounds) Watch(_ string, handle func(stdio.StateChange) error) error { + f.watch = handle + return nil +} + +func (f *fakeBounds) Delete(key string) error { + f.deleted = append(f.deleted, key) + return nil +} diff --git a/modules/mesh-recovery/cmd/mesh-recovery/seats.go b/modules/mesh-recovery/cmd/mesh-recovery/seats.go new file mode 100644 index 000000000..688c596fb --- /dev/null +++ b/modules/mesh-recovery/cmd/mesh-recovery/seats.go @@ -0,0 +1,252 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/process" + + "github.com/nats-io/nats.go/jetstream" + + "mesh-recovery/internal/recovery" +) + +// Call is how the process calls a seat's verb (process.Conn.Call), and CallOn one machine's holder of a node seat. +type ( + Call func(key string, body any) (json.RawMessage, error) + CallOn func(node, key string, body any) (json.RawMessage, error) +) + +// liveSeats is what the engine asks of the rest of the mesh, each by its seat (design 47 G §554). +type liveSeats struct { + emit func(seat, event string, body any) error + recorder meshhandactsclient.Recorder + call Call + callOn CallOn + node string + points string + running *runningView +} + +// Say says one event of the seat mesh-recovery, each part filled in as one token, escaped as a key's name is (§523). +func (s liveSeats) Say(event string, body any, parts ...string) error { + subject := event + for _, p := range parts { + if p == "" || p == "." || p == ".." { + return fmt.Errorf("the event %s cannot be said with the part %q", event, p) + } + subject += "." + keys.EscapeName(p) + } + return s.emit(recovery.Seat, subject, body) +} + +// HandAct writes the act in the seat mesh-hand-acts' log first, naming the account the bus proved and who asked (§554, §682). +func (s liveSeats) HandAct(caller, asker, what, why string) error { + line := what + " (the account " + orNone(caller) + if asker != "" { + line += ", asked by " + asker + } + _, err := s.recorder.Record(meshhandactsclient.Request{What: line + ")", Why: why, Verb: recovery.Seat}) + return err +} + +func orNone(s string) string { + if s == "" { + return "unproven" + } + return s +} + +// Streams are the bus names of a module's kept data at a fingerprint, from the catalog graph entry `mesh-catalog`'s `entry` +// answers (§554 Bus names, §443): a template entry's as mesh-sdk names them, an old section's as the bus composer of +// mesh-sdk names it when it makes it (A-121). +func (s liveSeats) Streams(_ context.Context, module, kept, fingerprint string) ([]string, error) { + raw, err := s.call("seat:"+meshcatalog.Seat+"."+meshcatalog.VerbEntry, meshcatalog.EntryInput{Fingerprint: fingerprint}) + if err != nil { + return nil, fmt.Errorf("the catalog entry of %s: %w", fingerprint, err) + } + var ans meshcatalog.EntryAnswer + if err := json.Unmarshal(raw, &ans); err != nil { + return nil, fmt.Errorf("the catalog entry of %s does not read: %w", fingerprint, err) + } + if ans.Entry.Module != module { + return nil, fmt.Errorf("the catalog entry of %s is %s's, not %s's", fingerprint, ans.Entry.Module, module) + } + whole, err := json.Marshal(ans.Entry.Manifest) + if err != nil { + return nil, err + } + m, err := catalogue.Decode(whole) + if err != nil { + return nil, fmt.Errorf("the manifest of %s at %s: %w", module, fingerprint, err) + } + return KeptStreams(module, kept, m) +} + +// KeptStreams names the bus objects of one kept data a manifest declares (A-121). +func KeptStreams(module, kept string, m catalogue.Manifest) ([]string, error) { + for _, e := range m.BusState { + if e.Name == kept { + if names := catalogue.BusStateStreams(module, e); len(names) > 0 { + return names, nil + } + return nil, fmt.Errorf("%s's %s has no bus name", module, kept) + } + } + for _, d := range catalogue.KeptDeclarations(m) { + if d.Name != kept { + continue + } + switch d.Kind { + case catalogue.KeptKindState: + return []string{"KV_" + buscompose.BucketName(module, kept)}, nil + case catalogue.KeptKindLog: + return []string{buscompose.LogStreamName(module, kept)}, nil + case catalogue.KeptKindStream: + return []string{kept}, nil + case catalogue.KeptKindObjects: + return []string{"OBJ_" + buscompose.BucketName(module, kept)}, nil + } + } + return nil, fmt.Errorf("%s's manifest declares no kept data %s", module, kept) +} + +// RunningFingerprint is the fingerprint the node-engine runs the module at, from the node-engine seat's running (A-123, A-132). +func (s liveSeats) RunningFingerprint(_ context.Context, module string) (string, error) { + return s.running.Fingerprint(module) +} + +// Backup brings the points directory back beside its path through the seat node-backup's restore on this node (§516), +// answering the directory it was brought back into. +func (s liveSeats) Backup(_ context.Context, module string) (string, error) { + if _, err := s.callOn(s.node, "seat:node-backup.restore", map[string]string{"module": module, "path": s.points}); err != nil { + return "", fmt.Errorf("the backup holder on %s brought nothing back: %w", s.node, err) + } + back, _ := filepath.Glob(s.points + ".restored-*") + if len(back) == 0 { + return "", fmt.Errorf("the backup holder on %s answered, and nothing lies beside %s", s.node, s.points) + } + sort.Strings(back) + return back[len(back)-1], nil +} + +var _ recovery.Seats = liveSeats{} + +// runningView is what the node-engine seat's running says of each node's module, read as §556 has it: the read of the last +// of each subject first, the durable followed after it, an older one by `at` changing nothing. +type runningView struct { + mu sync.Mutex + read bool + by map[string]nodeengine.RunningEvent +} + +func newRunningView() *runningView { return &runningView{by: map[string]nodeengine.RunningEvent{}} } + +func (v *runningView) hear(m jetstream.Msg) { + var r nodeengine.RunningEvent + if err := json.Unmarshal(m.Data(), &r); err == nil && r.Module != "" { + k := r.Node + "\x00" + r.Module + v.mu.Lock() + if old, ok := v.by[k]; !ok || !r.At.Before(old.At) { + v.by[k] = r + } + v.mu.Unlock() + } + _ = m.Ack() +} + +func (v *runningView) onRead(state string) { + v.mu.Lock() + v.read = state == process.ReadDone + v.mu.Unlock() +} + +// Fingerprint is the one fingerprint the module runs at on the nodes that run it; two different ones are refused, +// naming both, so a set-aside never takes the names of a build that does not run. +func (v *runningView) Fingerprint(module string) (string, error) { + v.mu.Lock() + defer v.mu.Unlock() + if !v.read { + return "", errors.New("the node-engine seat's running is still being read") + } + fps := map[string][]string{} + for _, r := range v.by { + if r.Module == module && !r.Stopped && r.Fingerprint != "" { + fps[r.Fingerprint] = append(fps[r.Fingerprint], r.Node) + } + } + switch len(fps) { + case 0: + return "", fmt.Errorf("no node runs %s", module) + case 1: + for fp := range fps { + return fp, nil + } + } + var said []string + for fp, nodes := range fps { + sort.Strings(nodes) + said = append(said, fp+" on "+strings.Join(nodes, ", ")) + } + sort.Strings(said) + return "", fmt.Errorf("%s runs at more than one fingerprint (%s)", module, strings.Join(said, "; ")) +} + +// readMinutes reads a bound in minutes from its setting's file (§557, A-130, A-138); the default when missing. +func readMinutes(path string, fallback time.Duration) (time.Duration, error) { + n, err := readFraction(path, fallback.Minutes()) + return time.Duration(n * float64(time.Minute)), err +} + +// readFraction reads a bound's setting (minutes, hours, days), a fraction allowed so a whole test sets it in seconds +// (§557, A-138); the default when missing. +func readFraction(path string, fallback float64) (float64, error) { + if path == "" { + return fallback, nil + } + raw, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return fallback, nil + } + if err != nil { + return 0, err + } + var n float64 + if err := json.Unmarshal([]byte(strings.TrimSpace(string(raw))), &n); err != nil || n <= 0 { + return 0, fmt.Errorf("the setting in %s is %q, not a positive number", path, strings.TrimSpace(string(raw))) + } + return n, nil +} + +// readNumber reads a setting's file (design 47 G A-112): its whole content a positive number, the default when missing. +func readNumber(path string, fallback int64) (int64, error) { + if path == "" { + return fallback, nil + } + raw, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return fallback, nil + } + if err != nil { + return 0, err + } + var n float64 + if err := json.Unmarshal([]byte(strings.TrimSpace(string(raw))), &n); err != nil || n <= 0 { + return 0, fmt.Errorf("the setting in %s is %q, not a positive number", path, strings.TrimSpace(string(raw))) + } + return int64(n), nil +} diff --git a/modules/mesh-recovery/cmd/mesh-recovery/whole_integration_test.go b/modules/mesh-recovery/cmd/mesh-recovery/whole_integration_test.go new file mode 100644 index 000000000..5cbaf0e31 --- /dev/null +++ b/modules/mesh-recovery/cmd/mesh-recovery/whole_integration_test.go @@ -0,0 +1,473 @@ +//go:build integration + +package main + +// The recovery module's one whole test (design 47 G §557, §554, §681, §562, A-138, A-143): its run() as the node starts it, +// on its own account on a bus of this test alone, every bound a setting of a few seconds. A stand-in bus module answers +// the seat mesh-broker's snapshot-get (in chunks), snapshot-restore and snapshot-restore-cancel and says snapshot-restored; +// stand-ins answer the hand-act log, the assigner's assignments and the catalog's entry; the node-engine seat's running and +// the backup holder's item events are said on their streams. The live data is a real bucket, set aside and restored by +// the bus's own snapshots. + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-contracts/go/seats/meshbroker" + seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + "git.novox.be/novox/mesh-sdk/go/buscalls" + "git.novox.be/novox/mesh-sdk/go/bussnap" + "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/testbus" + + "mesh-recovery/internal/recovery" +) + +// standKeeper is the bus module's keeper as the test plays it: a snapshot by request, protected or not, and what it says +// after a restore in place and after its cancel ("" says nothing). +type standKeeper struct { + mu sync.Mutex + file []byte // the snapshot an unprotected request sends + protected bool + onRestore string + onCancel string + asked []string +} + +func (k *standKeeper) set(protected bool, onRestore, onCancel string) { + k.mu.Lock() + k.protected, k.onRestore, k.onCancel, k.asked = protected, onRestore, onCancel, nil + k.mu.Unlock() +} + +func (k *standKeeper) said() string { + k.mu.Lock() + defer k.mu.Unlock() + return strings.Join(k.asked, "|") +} + +func TestRecoveryWhole(t *testing.T) { + url := testbus.URL(t) + nc, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + js, _ := jetstream.New(nc) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + acct := process.Account{URL: url, Node: "srv", Module: "mesh-recovery", User: "srv.mesh-recovery", Password: "unused"} + + // The module's own states (bounds takes keys with their own time to live), the app's live kept data, and the streams the + // grants part and the bus composer make: the seat events it hears with its durables, its own events and its findings. + for _, b := range []jetstream.KeyValueConfig{{Bucket: "mesh-recovery_bounds", LimitMarkerTTL: time.Second}, + {Bucket: "mesh-recovery_digests"}, {Bucket: "mesh-recovery_findings"}, {Bucket: "app_notes"}} { + if _, err := js.CreateKeyValue(ctx, b); err != nil { + t.Fatal(err) + } + } + for _, s := range []jetstream.StreamConfig{ + {Name: "RECOVERY_EVENTS", Subjects: []string{"mesh.seat.mesh-recovery.event.>"}}, + {Name: "RECOVERY_FINDINGS", Subjects: []string{"mesh.stream.mesh-recovery.event.>"}}, + {Name: "FINDINGS_ASKED", Subjects: []string{meshconditions.SubjectFindingsAsked}}, + } { + if _, err := js.CreateStream(ctx, s); err != nil { + t.Fatal(err) + } + } + for stream, subject := range map[string]string{ + nodeengine.StreamRunning: "mesh.seat.node-engine.event.running.>", + itemStartedSteam: "mesh.seat.node-backup.event.item-started.>", + itemEndedStream: "mesh.seat.node-backup.event.item-ended.>", + meshbroker.StreamSnapshotRestored: "mesh.seat.mesh-broker.event.snapshot-restored.>", + } { + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{subject}, MaxMsgsPerSubject: 1}); err != nil { + t.Fatal(err) + } + if _, err := js.CreateOrUpdateConsumer(ctx, stream, acct.SeatEventConsumerConfig([]string{subject}, true)); err != nil { + t.Fatal(err) + } + } + notes, _ := js.KeyValue(ctx, "app_notes") + live := func() string { + e, err := notes.Get(ctx, "n") + if err != nil { + return "none: " + err.Error() + } + return string(e.Value()) + } + if _, err := notes.Put(ctx, "n", []byte("v1")); err != nil { + t.Fatal(err) + } + running, _ := json.Marshal(nodeengine.Running{V: 1, Node: "srv", Module: "app", Fingerprint: "src2:app1", At: time.Now()}) + if _, err := js.Publish(ctx, nodeengine.SubjectRunning("srv", "app"), running); err != nil { + t.Fatal(err) + } + + // The stand-ins of the seats the module calls. + answer := func(m *nats.Msg, result any) { + b, _ := json.Marshal(map[string]any{"result": result, "node": "srv"}) + _ = m.Respond(b) + } + var logMu sync.Mutex + var handActs []string + sub := func(subject string, h func(*nats.Msg)) { + if _, err := nc.Subscribe(subject, h); err != nil { + t.Fatal(err) + } + } + sub("mesh.seat.mesh-hand-acts.call.>", func(m *nats.Msg) { + logMu.Lock() + handActs = append(handActs, string(m.Data)) + logMu.Unlock() + answer(m, map[string]any{"recorded": true}) + }) + sub("mesh.seat.mesh-module-assigner.call.assignments.*", func(m *nats.Msg) { + answer(m, map[string]any{"seats": []any{map[string]any{"seat": meshbroker.Seat, "holders": []any{map[string]any{"module": "nats"}}}}}) + }) + sub("mesh.seat.mesh-catalog.call.entry.*", func(m *nats.Msg) { + answer(m, map[string]any{"entry": map[string]any{"module": "app", "fingerprint": "src2:app1", + "manifest": map[string]any{"module": "app", "version": "1", "state": []any{map[string]any{"name": "notes"}}}}}) + }) + keeper := &standKeeper{} + restored := func(request, act, outcome string) { + b, _ := json.Marshal(meshbroker.SnapshotRestoredEvent{Request: request, Act: act, Outcome: outcome, Error: "cancelled"}) + if _, err := js.Publish(ctx, meshbroker.SubjectSnapshotRestored(request, act), b); err != nil { + t.Error(err) + } + } + sub("mesh.seat.mesh-broker.call.snapshot-get.*", func(m *nats.Msg) { + var in meshbroker.SnapshotGetInput + _ = json.Unmarshal(m.Data, &in) + keeper.mu.Lock() + keeper.asked = append(keeper.asked, "get "+in.Request) + file, protected := keeper.file, keeper.protected + keeper.mu.Unlock() + if protected { + answer(m, meshbroker.SnapshotGetAnswer{Request: in.Request, Error: "snapshot protected"}) + return + } + sum := sha256.Sum256(file) + answer(m, meshbroker.SnapshotGetAnswer{Request: in.Request, Module: "app", Local: "notes", Stream: "KV_app_notes", + Size: int64(len(file)), Digest: "sha256:" + hex.EncodeToString(sum[:]), Chunks: 3}) + // Three chunks, each sent once the one before was acknowledged (§681). + go func() { + send := buscalls.SeatChunks{Seat: meshbroker.Seat, Verb: meshbroker.VerbSnapshotGet, Call: in.Call} + third := (len(file) + 2) / 3 + for n := int64(1); n <= 3; n++ { + acks, _ := nc.SubscribeSync(send.Ack(n)) + part := file[min(int(n-1)*third, len(file)):min(int(n)*third, len(file))] + body, _ := json.Marshal(buscalls.SeatChunk{N: n, Of: 3, Data: part}) + _ = nc.Publish(send.Chunk(n), body) + ack, err := acks.NextMsg(10 * time.Second) + _ = acks.Unsubscribe() + if err != nil { + t.Errorf("chunk %d not acknowledged: %v", n, err) + return + } + _ = ack.Respond([]byte("{}")) + } + }() + }) + sub("mesh.seat.mesh-broker.call.snapshot-restore.*", func(m *nats.Msg) { + var in meshbroker.SnapshotRestoreInput + _ = json.Unmarshal(m.Data, &in) + keeper.mu.Lock() + if in.Verify { + keeper.asked = append(keeper.asked, "verify "+in.Request) + keeper.mu.Unlock() + answer(m, meshbroker.SnapshotRestoreAnswer{Accepted: "keeper-verify"}) + return + } + keeper.asked = append(keeper.asked, "restore "+in.Request) + then := keeper.onRestore + keeper.mu.Unlock() + answer(m, meshbroker.SnapshotRestoreAnswer{Accepted: "keeper-" + in.Request}) + if then != "" { + restored(in.Request, "keeper-"+in.Request, then) + } + }) + sub("mesh.seat.mesh-broker.call.snapshot-restore-cancel.*", func(m *nats.Msg) { + var in meshbroker.SnapshotRestoreCancelInput + _ = json.Unmarshal(m.Data, &in) + keeper.mu.Lock() + keeper.asked = append(keeper.asked, "cancel "+in.Act) + then := keeper.onCancel + keeper.mu.Unlock() + answer(m, meshbroker.SnapshotRestoreCancelAnswer{Accepted: in.Act}) + if then != "" { + restored(strings.TrimPrefix(in.Act, "keeper-"), in.Act, then) + } + }) + + // Every event the module says, heard as it is said. + type heard struct { + subject string + body map[string]any + } + events := make(chan heard, 256) + sub("mesh.seat.mesh-recovery.event.>", func(m *nats.Msg) { + var b map[string]any + _ = json.Unmarshal(m.Data, &b) + events <- heard{m.Subject, b} + }) + var kept []heard + waitEvent := func(t *testing.T, what string, ok func(heard) bool) map[string]any { + t.Helper() + for i, h := range kept { + if ok(h) { + kept = append(kept[:i:i], kept[i+1:]...) + return h.body + } + } + deadline := time.After(30 * time.Second) + for { + select { + case h := <-events: + if ok(h) { + return h.body + } + kept = append(kept, h) + case <-deadline: + t.Fatalf("not said within 30s: %s", what) + } + } + } + endOf := func(t *testing.T, event, act string) map[string]any { + t.Helper() + return waitEvent(t, event+" of "+act, func(h heard) bool { + return strings.HasPrefix(h.subject, "mesh.seat.mesh-recovery.event."+event+".") && h.body["act"] == act + }) + } + ask := func(t *testing.T, verb, caller string, body any) seat.SnapshotKeptAnswer { + t.Helper() + data, _ := json.Marshal(body) + m, err := nc.Request("mesh.seat.mesh-recovery.call."+verb+"."+strings.ReplaceAll(caller, ".", "~"), data, 10*time.Second) + if err != nil { + t.Fatalf("%s: %v", verb, err) + } + var r struct { + Result seat.SnapshotKeptAnswer `json:"result"` + Error string `json:"error"` + } + _ = json.Unmarshal(m.Data, &r) + if r.Error != "" { + r.Result.Error = r.Error + } + return r.Result + } + // A caller told busy asks again when it hears the ending event of the act it was named (§554, §560): the expiry runs + // on its own bound beside the test's acts. + act := func(t *testing.T, verb, caller string, body any) string { + t.Helper() + for { + a := ask(t, verb, caller, body) + if a.Busy == "" { + if a.Accepted == "" { + t.Fatalf("%s answered %+v", verb, a) + } + return a.Accepted + } + waitEvent(t, "the ending of "+a.Busy, func(h heard) bool { + return h.body["act"] == a.Busy && !strings.Contains(h.subject, ".restore-started.") + }) + } + } + const engine, runner = "srv.node-engine", "srv.tool-runner" + + // The process, every bound a setting in seconds (§557, A-138): the dump wait and the restore wait 1.8 s, the expiry + // every 1.8 s, a point kept about 4 s. + dir := t.TempDir() + setting := func(name, v string) string { + p := filepath.Join(dir, name) + _ = os.WriteFile(p, []byte(v+"\n"), 0o600) + return p + } + acctFile, _ := json.Marshal(map[string]any{"url": url, "node": acct.Node, "module": acct.Module, "user": acct.User, "password": acct.Password}) + env := map[string]string{ + process.BrokerFileEnv: setting("broker", string(acctFile)), + envPoints: filepath.Join(dir, "points"), + envDumpWait: setting("dump-wait-minutes", "0.03"), + envRestore: setting("snapshot-restore-wait-minutes", "0.03"), + envExpire: setting("expire-every-hours", "0.0005"), + envKeepDays: setting("point-keep-days", "0.00005"), + } + runCtx, stop := context.WithCancel(ctx) + done := make(chan error, 1) + go func() { done <- run(runCtx, func(k string) string { return env[k] }, testLog{t}) }() + defer func() { stop(); <-done }() + waitEvent(t, "the start's removal of points past their keep days", func(h heard) bool { + return h.subject == "mesh.seat.mesh-recovery.event."+seat.EventExpiryEnded + }) + + var first string + t.Run("a hand take's log names the account the bus proved and the asker (§554)", func(t *testing.T) { + ev := endOf(t, seat.EventEnded, act(t, seat.VerbTake, runner, seat.TakeInput{Module: "app", Kept: "notes", Fingerprint: "src2:app1", Why: "before a test", Asker: "op@srv"})) + if ev["outcome"] != "done" { + t.Fatalf("ended %v", ev) + } + first, _ = ev["snapshot"].(string) + logMu.Lock() + defer logMu.Unlock() + if len(handActs) == 0 || !strings.Contains(handActs[len(handActs)-1], runner) || !strings.Contains(handActs[len(handActs)-1], "op@srv") { + t.Errorf("the hand-act log holds %v", handActs) + } + }) + + t.Run("a restore waits on the dump past its bound key, a second ask is busy and asks again on its ended (§554, §563, A-143)", func(t *testing.T) { + if a := ask(t, seat.VerbRestoreKept, engine, map[string]any{"module": "app", "kept": "notes", "snapshot": first, "why": "w", "aside-only": true}); !strings.Contains(a.Error, "aside-only") { + t.Errorf("a node-engine's restore-kept with aside-only answered %+v", a) + } + item := func(event string) { + b, _ := json.Marshal(map[string]any{"at": time.Now()}) + if _, err := js.Publish(ctx, itemSubject(event, "srv", "nats", "data"), b); err != nil { + t.Fatal(err) + } + } + item(itemStarted) + _, _ = notes.Put(ctx, "n", []byte("changed")) + r := seat.SnapshotKeptAnswer{Accepted: act(t, seat.VerbRestoreKept, engine, seat.RestoreKeptInput{Module: "app", Kept: "notes", Form: 2, Fingerprint: "src2:app1", Snapshot: first, Why: "a form change"})} + if b := ask(t, seat.VerbSnapshotKept, engine, seat.SnapshotKeptInput{Module: "app", Kept: "notes", Form: 2, Fingerprint: "src2:app1", Why: "x"}); b.Busy != r.Accepted { + t.Errorf("a second ask answered %+v, want busy with %s", b, r.Accepted) + } + findings, _ := js.KeyValue(ctx, "mesh-recovery_findings") + w, _ := findings.Watch(ctx, "app."+r.Accepted+"."+recovery.FindingRecoveryWaits) + for e := range w.Updates() { + if e != nil && e.Operation() == jetstream.KeyValuePut { + break + } + } + _ = w.Stop() + item(itemEnded) + if ev := endOf(t, seat.EventRestored, r.Accepted); ev["outcome"] != "done" || live() != "v1" { + t.Fatalf("restored %v, live %q", ev, live()) + } + if e, err := findings.Get(ctx, "app."+r.Accepted+"."+recovery.FindingRecoveryWaits); err == nil { + t.Errorf("recovery-waits stands after the act ended: %s", e.Value()) + } + again := act(t, seat.VerbSnapshotKept, engine, seat.SnapshotKeptInput{Module: "app", Kept: "notes", Form: 2, Fingerprint: "src2:app1", Why: "asked again"}) + if ev := endOf(t, seat.EventEnded, again); ev["outcome"] != "done" { + t.Errorf("the ask again ended %v", ev) + } + }) + + t.Run("a point past its keep days is removed on the expiry's bound key, never the newest (§551, A-138)", func(t *testing.T) { + ev := waitEvent(t, "point-removed of "+first, func(h heard) bool { + return strings.HasPrefix(h.subject, "mesh.seat.mesh-recovery.event."+seat.EventPointRemoved+".") && h.body["snapshot"] == first + }) + if ev["module"] != "app" { + t.Errorf("point-removed %v", ev) + } + var kp seat.KeptPointsAnswer + m, _ := nc.Request("mesh.seat.mesh-recovery.call.kept-points."+strings.ReplaceAll(runner, ".", "~"), []byte(`{"module":"app","kept":"notes"}`), 10*time.Second) + var r struct { + Result seat.KeptPointsAnswer `json:"result"` + } + _ = json.Unmarshal(m.Data, &r) + kp = r.Result + if len(kp.Points) != 1 { + t.Errorf("the newest point is not kept alone: %+v", kp.Points) + } + }) + + t.Run("a keeper's snapshot of three chunks is received and restored whole (§681)", func(t *testing.T) { + _, _ = notes.Put(ctx, "n", []byte("from the keeper")) + var snap bytes.Buffer + if _, err := bussnap.Snapshot(ctx, nc, &snap, bussnap.SnapshotOptions{Streams: []string{"KV_app_notes"}, TempDir: t.TempDir()}); err != nil { + t.Fatal(err) + } + keeper.mu.Lock() + keeper.file = snap.Bytes() + keeper.mu.Unlock() + keeper.set(false, "", "") + _, _ = notes.Put(ctx, "n", []byte("since the drop")) + a := act(t, seat.VerbRestore, runner, seat.RestoreInput{Module: "app", Kept: "notes", Request: "req-1", Why: "test the drop's snapshot", Asker: "op@srv"}) + if ev := endOf(t, seat.EventRestored, a); ev["outcome"] != "done" || live() != "from the keeper" { + t.Errorf("restored %v, live %q", ev, live()) + } + if left, _ := os.ReadDir(filepath.Join(dir, "points", "incoming")); len(left) != 0 { + t.Errorf("incoming holds %v", left) + } + }) + + inPlace := func(t *testing.T, request string) string { + t.Helper() + _, _ = notes.Put(ctx, "n", []byte("live before "+request)) + return act(t, seat.VerbRestore, runner, seat.RestoreInput{Module: "app", Kept: "notes", Request: request, Why: "restore in place", Asker: "op@srv"}) + } + t.Run("a protected snapshot's bound asks the cancel first, then puts the aside back on its failed (§681, A-140, A-143)", func(t *testing.T) { + keeper.set(true, "", meshbroker.SnapshotRestoredEventOutcomeFailed) + id := inPlace(t, "req-2") + if ev := endOf(t, seat.EventRestored, id); ev["outcome"] != "failed" || live() != "live before req-2" { + t.Errorf("restored %v, live %q", ev, live()) + } + if got := keeper.said(); got != "get req-2|verify req-2|restore req-2|cancel keeper-req-2" { + t.Errorf("the keeper was asked %s", got) + } + }) + + t.Run("a keeper that answers neither leaves the aside unrestored, which restore-aside puts back (§681, §554)", func(t *testing.T) { + keeper.set(true, "", "") + id := inPlace(t, "req-3") + ev := endOf(t, seat.EventRestored, id) + if ev["outcome"] != "failed" || ev["error"] != "snapshot-restore unanswered" { + t.Fatalf("restored %v", ev) + } + var r struct { + Result seat.KeptPointsAnswer `json:"result"` + } + m, _ := nc.Request("mesh.seat.mesh-recovery.call.kept-points."+strings.ReplaceAll(runner, ".", "~"), []byte(`{}`), 10*time.Second) + _ = json.Unmarshal(m.Data, &r) + if len(r.Result.Unrestored) != 1 { + t.Fatalf("unrestored %+v", r.Result.Unrestored) + } + back := act(t, seat.VerbRestoreAside, runner, seat.RestoreAsideInput{Module: "app", Kept: "notes", ID: r.Result.Unrestored[0].ID, Why: "put it back", Asker: "op@srv"}) + if ev := endOf(t, seat.EventRestored, back); ev["outcome"] != "done" || live() != "live before req-3" { + t.Errorf("restore-aside %v, live %q", ev, live()) + } + }) + + t.Run("a stop during a restore past its removal puts the aside back and ends it stopped (§562, A-141)", func(t *testing.T) { + keeper.set(true, "", meshbroker.SnapshotRestoredEventOutcomeFailed) + id := inPlace(t, "req-4") + // The wait on snapshot-restored is a bound key: once it stands, the live data is removed and the keeper asked. + bounds, _ := js.KeyValue(ctx, "mesh-recovery_bounds") + w, _ := bounds.Watch(ctx, "restore-wait."+id) + for e := range w.Updates() { + if e != nil && e.Operation() == jetstream.KeyValuePut { + break + } + } + _ = w.Stop() + stop() + ev := endOf(t, seat.EventRestored, id) + if err := <-done; err != nil { + t.Errorf("the process ended %v", err) + } + done <- nil + if ev["outcome"] != "failed" || ev["error"] != "stopped" || live() != "live before req-4" { + t.Errorf("restored %v, live %q", ev, live()) + } + }) +} + +type testLog struct{ t *testing.T } + +func (l testLog) Write(p []byte) (int, error) { + l.t.Log(strings.TrimRight(string(p), "\n")) + return len(p), nil +} diff --git a/modules/mesh-recovery/files/dump-wait-minutes b/modules/mesh-recovery/files/dump-wait-minutes new file mode 100644 index 000000000..bcb00821f --- /dev/null +++ b/modules/mesh-recovery/files/dump-wait-minutes @@ -0,0 +1 @@ +${config:dump-wait-minutes} diff --git a/modules/mesh-recovery/files/expire-every-hours b/modules/mesh-recovery/files/expire-every-hours new file mode 100644 index 000000000..68cafde19 --- /dev/null +++ b/modules/mesh-recovery/files/expire-every-hours @@ -0,0 +1 @@ +${config:expire-every-hours} diff --git a/modules/mesh-recovery/files/point-keep-days b/modules/mesh-recovery/files/point-keep-days new file mode 100644 index 000000000..02af64917 --- /dev/null +++ b/modules/mesh-recovery/files/point-keep-days @@ -0,0 +1 @@ +${config:point-keep-days} diff --git a/modules/mesh-recovery/files/points-cap b/modules/mesh-recovery/files/points-cap new file mode 100644 index 000000000..cbe82190d --- /dev/null +++ b/modules/mesh-recovery/files/points-cap @@ -0,0 +1 @@ +${config:points-cap} diff --git a/modules/mesh-recovery/files/points-filling-mark b/modules/mesh-recovery/files/points-filling-mark new file mode 100644 index 000000000..da03c7e78 --- /dev/null +++ b/modules/mesh-recovery/files/points-filling-mark @@ -0,0 +1 @@ +${config:points-filling-mark} diff --git a/modules/mesh-recovery/files/snapshot-restore-wait-minutes b/modules/mesh-recovery/files/snapshot-restore-wait-minutes new file mode 100644 index 000000000..89cd4bdf5 --- /dev/null +++ b/modules/mesh-recovery/files/snapshot-restore-wait-minutes @@ -0,0 +1 @@ +${config:snapshot-restore-wait-minutes} diff --git a/modules/mesh-recovery/go.mod b/modules/mesh-recovery/go.mod new file mode 100644 index 000000000..87e7e287a --- /dev/null +++ b/modules/mesh-recovery/go.mod @@ -0,0 +1,28 @@ +module mesh-recovery + +go 1.26.0 + +require ( + git.novox.be/novox/mesh-contracts/go/seats/meshbroker v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/meshrecovery v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 + github.com/nats-io/nats.go v1.54.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.1 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.2 // indirect + github.com/nats-io/nats-server/v2 v2.11.17 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.16.0 // indirect +) diff --git a/modules/mesh-recovery/go.sum b/modules/mesh-recovery/go.sum new file mode 100644 index 000000000..8ca93dfd0 --- /dev/null +++ b/modules/mesh-recovery/go.sum @@ -0,0 +1,25 @@ +github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E= +github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ= +github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc= +github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE= +golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s= diff --git a/modules/mesh-recovery/internal/recovery/control.go b/modules/mesh-recovery/internal/recovery/control.go new file mode 100644 index 000000000..d4841a96c --- /dev/null +++ b/modules/mesh-recovery/internal/recovery/control.go @@ -0,0 +1,189 @@ +package recovery + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" +) + +// Cancel is the verb `cancel` (§566, §682): it ends an accepted act that has not begun its work (one waiting on the +// dump, §563) with its ending event, outcome failed and error cancelled, the hand act's words as `why` (A-117); it +// refuses an act that has begun, naming its step, since a restore cut off after its removal is put back only by its own +// failure path (§516). It is written in the hand-act log first and answers in §691's one form. +func (e *Engine) Cancel(act, why, caller, asker string) Answer { + if stop := e.Stopping(); stop != "" { + return Answer{Error: stop} + } + if err := e.Seats.HandAct(caller, asker, "cancel "+act, why); err != nil { + return Answer{Error: "the hand-act log was not written, so nothing was done: " + err.Error()} + } + e.mu.Lock() + defer e.mu.Unlock() + switch { + case e.running == nil || e.running.ID != act: + return Answer{Error: "the act " + act + " is not running"} + case e.running.Step != "waiting": + return Answer{Error: fmt.Sprintf("the act %s has begun (its step: %q) and is put back only by its own failure path", act, e.running.Step)} + } + e.why = why + e.cancel() + return Answer{Accepted: act} +} + +// Retry is the verb `retry` (§682): it asks again the last failed act of a module and kept data, with its fields, under +// an act of its own; `nothing to retry` when its last act did not fail. +func (e *Engine) Retry(module, kept, why, caller, asker string) Answer { + acts := e.readActs() + var last *Act + for i := range acts { + a := acts[i] + if a.Module == module && a.Kept == kept && a.Ended && a.Kind != KindExpire && (last == nil || !a.Since.Before(last.Since)) { + last = &a + } + } + if last == nil || last.Error == "" { + return Answer{Error: "nothing to retry"} + } + again := *last + again.ID, again.Ended, again.Error, again.Step, again.WaitsOn = "", false, "", "", "" + again.Why, again.Caller, again.Asker = why, caller, asker + return e.Accept(again) +} + +func (e *Engine) readActs() []Act { + var out []Act + files, _ := filepath.Glob(filepath.Join(e.Dir, "acts", "*.json")) + for _, f := range files { + var a Act + if readJSON(f, &a) == nil { + out = append(out, a) + } + } + return out +} + +// KeptPoints is the verb `kept-points` (§517): every restore point and aside with their sizes, narrowed to a module and a +// kept data, and the act that runs, with since when and what it waits on. +func (e *Engine) KeptPoints(module, kept string) (seat.KeptPointsAnswer, error) { + pts, err := e.Points() + if err != nil { + return seat.KeptPointsAnswer{}, err + } + out := seat.KeptPointsAnswer{Points: []seat.Point{}, Asides: []seat.Aside{}} + for _, p := range pts { + if (module == "" || p.Module == module) && (kept == "" || p.Kept == kept) { + out.Points = append(out.Points, p) + } + } + _, recs, _ := e.Asides() + for _, r := range recs { + if (module == "" || r.Module == module) && (kept == "" || r.Kept == kept) { + out.Asides = append(out.Asides, r.aside()) + } + } + e.mu.Lock() + if e.running != nil { + out.Running = &seat.Running{Act: e.running.ID, Step: e.running.Step, Since: e.running.Since, WaitsOn: e.running.WaitsOn} + } + e.mu.Unlock() + // The asides only an aside holds, which the health check fails on, read from the engine (A-129). + out.Unrestored = []seat.Aside{} + for _, x := range e.Unrestored() { + if (module == "" || x.Module == module) && (kept == "" || x.Kept == kept) { + out.Unrestored = append(out.Unrestored, x.Aside) + } + } + return out, nil +} + +// Unrestored are the asides whose record ends at `removed` that no running act holds (§554 health, §561): the live data +// of those names is gone and only the aside holds it. +func (e *Engine) Unrestored() []UnrestoredAside { + _, recs, _ := e.Asides() + e.mu.Lock() + defer e.mu.Unlock() + var out []UnrestoredAside + for _, r := range recs { + held := e.running != nil && e.running.Module == r.Module && e.running.Kept == r.Kept + if r.Step == stepRemoved && !held { + out = append(out, UnrestoredAside{Module: r.Module, Kept: r.Kept, Aside: r.aside()}) + } + } + return out +} + +// UnrestoredAside is an aside only it holds the data of, with the module and kept data it was taken of: the scope of the +// finding aside-unrestored (§561). +type UnrestoredAside struct { + Module, Kept string + Aside seat.Aside +} + +// Start repairs what a restart cut off (§516, §564, §560): points under a temporary name are removed, an aside whose +// record ends at `set aside` (the live data never touched) is removed, an aside that ends at `removed` is put back under +// the live names, and every accepted act that never ended ends failed with `stopped`, so its caller asks again. It +// answers the asides it could not put back (the finding aside-unrestored, §561). +func (e *Engine) Start(ctx context.Context) []UnrestoredAside { + _ = filepath.WalkDir(filepath.Join(e.Dir, "points"), func(path string, d os.DirEntry, err error) error { + if err == nil && d.IsDir() && strings.HasPrefix(d.Name(), ".tmp-") { + _ = os.RemoveAll(path) + return filepath.SkipDir + } + return nil + }) + // A keeper's snapshot a cut-off act left half or whole received is deleted (§681). + _ = os.RemoveAll(e.incoming()) + _, recs, _ := e.Asides() + for i := range recs { + r := &recs[i] + switch r.Step { + case stepSetAside: + _ = os.RemoveAll(e.asideDirOf(r)) + case stepRemoved: + if live, err := e.Bus.Objects(ctx, r.Streams); err == nil && !anyExists(live) { + if err := e.restoreFromAside(ctx, r, false); err != nil { + e.logf("the aside %s is not put back: %v", r.ID, err) + } + } + } + } + e.endUnended() + e.judge() + return e.Unrestored() +} + +// endUnended ends every accepted act that never ended failed with `stopped` and its ending event (§554, §562), oldest first. +func (e *Engine) endUnended() { + acts := e.readActs() + sort.Slice(acts, func(i, j int) bool { return acts[i].Since.Before(acts[j].Since) }) + for i := range acts { + a := acts[i] + if a.Ended { + continue + } + stopped := errors.New(ErrStopped) + switch a.Kind { + case KindTake: + _ = e.endedFailed(&a, stopped) + case KindRestore, KindRestoreAside: + _ = e.endedRestored(&a, stopped, nil, false) + } + a.Error = ErrStopped + e.markEnded(&a) + } +} + +func anyExists(os []Object) bool { + for _, o := range os { + if o.Exists { + return true + } + } + return false +} diff --git a/modules/mesh-recovery/internal/recovery/fakes_test.go b/modules/mesh-recovery/internal/recovery/fakes_test.go new file mode 100644 index 000000000..254956aa7 --- /dev/null +++ b/modules/mesh-recovery/internal/recovery/fakes_test.go @@ -0,0 +1,302 @@ +package recovery_test + +import ( + "context" + "errors" + "io" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + + "mesh-recovery/internal/recovery" +) + +// fakeBus holds streams as strings; a snapshot file is "snap::". +type fakeBus struct { + mu sync.Mutex + streams map[string]string + consumers map[string][]seat.Consumer + setBack []seat.Consumer + failRestores int +} + +func (b *fakeBus) Objects(_ context.Context, names []string) ([]recovery.Object, error) { + b.mu.Lock() + defer b.mu.Unlock() + var out []recovery.Object + for _, n := range names { + c, ok := b.streams[n] + out = append(out, recovery.Object{Stream: n, Exists: ok, Bytes: int64(len(c)), Consumers: b.consumers[n]}) + } + return out, nil +} + +func (b *fakeBus) Take(_ context.Context, s string, w io.Writer) error { + b.mu.Lock() + defer b.mu.Unlock() + c, ok := b.streams[s] + if !ok { + return errors.New("no stream " + s) + } + _, err := io.WriteString(w, "snap:"+s+":"+c) + return err +} + +func (b *fakeBus) Delete(_ context.Context, s string) error { + b.mu.Lock() + defer b.mu.Unlock() + delete(b.streams, s) + return nil +} + +func (b *fakeBus) Restore(_ context.Context, file string) error { + raw, err := os.ReadFile(file) + if err != nil { + return err + } + b.mu.Lock() + defer b.mu.Unlock() + if b.failRestores > 0 { + b.failRestores-- + return errors.New("the server refused the archive") + } + var name, content string + rest := string(raw[5:]) + for i := range rest { + if rest[i] == ':' { + name, content = rest[:i], rest[i+1:] + break + } + } + if _, live := b.streams[name]; live { + return errors.New("stream name already in use") + } + b.streams[name] = content + return nil +} + +func (b *fakeBus) SetConsumers(_ context.Context, cs []seat.Consumer) error { + b.mu.Lock() + defer b.mu.Unlock() + b.setBack = append(b.setBack, cs...) + return nil +} + +type said struct { + Event string + Body any + Parts []string +} + +type fakeSeats struct { + mu sync.Mutex + events []said + handActs []string + handFail error + streams []string + backup string + sayFail map[string]error +} + +func (s *fakeSeats) Say(event string, body any, parts ...string) error { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.sayFail[event]; err != nil { + return err + } + s.events = append(s.events, said{event, body, parts}) + return nil +} + +func (s *fakeSeats) HandAct(caller, asker, what, why string) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.handFail != nil { + return s.handFail + } + s.handActs = append(s.handActs, caller+"|"+asker+"|"+what+"|"+why) + return nil +} + +func (s *fakeSeats) Streams(context.Context, string, string, string) ([]string, error) { + return s.streams, nil +} + +func (s *fakeSeats) RunningFingerprint(context.Context, string) (string, error) { return "fp-run", nil } + +func (s *fakeSeats) Backup(context.Context, string) (string, error) { + if s.backup == "" { + return "", errors.New("no repository snapshot holds it") + } + return s.backup, nil +} + +func (s *fakeSeats) named(event string) []said { + s.mu.Lock() + defer s.mu.Unlock() + var out []said + for _, e := range s.events { + if e.Event == event { + out = append(out, e) + } + } + return out +} + +type memDigests struct { + mu sync.Mutex + m map[string][]byte +} + +func (d *memDigests) Put(k string, v []byte) error { + d.mu.Lock() + defer d.mu.Unlock() + d.m[k] = v + return nil +} +func (d *memDigests) Get(k string) ([]byte, bool, error) { + d.mu.Lock() + defer d.mu.Unlock() + v, ok := d.m[k] + return v, ok, nil +} +func (d *memDigests) Delete(k string) error { + d.mu.Lock() + defer d.mu.Unlock() + delete(d.m, k) + return nil +} + +type noDump struct{} + +func (noDump) Wait(context.Context, func(string)) error { return nil } + +// holdDump waits until ctx ends, saying it waits. +type holdDump struct{ waiting chan string } + +func (h holdDump) Wait(ctx context.Context, waiting func(string)) error { + waiting("the dump of the bus") + h.waiting <- "waiting" + <-ctx.Done() + return ctx.Err() +} + +type rig struct { + e *recovery.Engine + b *fakeBus + s *fakeSeats + d *memDigests + t time.Time + // bounds is the state bounds the act's waits are timed by (A-143). + bounds *fakeBounds +} + +func newRig(t *testing.T) *rig { + t.Helper() + r := &rig{bounds: &fakeBounds{}, b: &fakeBus{streams: map[string]string{"app_notes": "hello", "KV_app_seen": "world"}, consumers: map[string][]seat.Consumer{}}, + s: &fakeSeats{streams: []string{"app_notes", "KV_app_seen"}}, d: &memDigests{m: map[string][]byte{}}, t: time.Date(2026, 11, 12, 1, 0, 0, 0, time.UTC)} + r.b.consumers["app_notes"] = []seat.Consumer{{Name: "c1", Stream: "app_notes", Sequence: 7}} + r.e = &recovery.Engine{Dir: filepath.Join(t.TempDir(), "points-dir"), Bus: r.b, Seats: r.s, Digests: r.d, Dump: noDump{}, Bounds: r.bounds, + Settings: recovery.Settings{Cap: 1 << 20, KeepDays: 90}, Now: func() time.Time { return r.t }} + return r +} + +func (r *rig) do(a recovery.Act) recovery.Answer { + ans := r.e.Accept(a) + r.e.Wait() + return ans +} + +var errTest = errors.New("the log is unreachable") + +type gatedBus struct { + *fakeBus + gate chan struct{} +} + +func (g *gatedBus) Objects(ctx context.Context, n []string) ([]recovery.Object, error) { + <-g.gate + return g.fakeBus.Objects(ctx, n) +} + +// heldFindings is the module's finding writer as it holds them; written says each write. +type heldFindings struct { + mu sync.Mutex + held map[string]meshconditions.Finding + written chan meshconditions.Finding +} + +func newHeldFindings() *heldFindings { + return &heldFindings{held: map[string]meshconditions.Finding{}, written: make(chan meshconditions.Finding, 64)} +} + +func (h *heldFindings) Write(_ context.Context, f meshconditions.Finding) error { + h.mu.Lock() + h.held[f.Scope+"."+f.ID+"."+f.Kind] = f + h.mu.Unlock() + h.written <- f + return nil +} + +func (h *heldFindings) Clear(_ context.Context, scope, id, kind, _ string) error { + h.mu.Lock() + defer h.mu.Unlock() + delete(h.held, scope+"."+id+"."+kind) + return nil +} + +func (h *heldFindings) Held() []meshconditions.Finding { + h.mu.Lock() + defer h.mu.Unlock() + var out []meshconditions.Finding + for _, f := range h.held { + out = append(out, f) + } + return out +} + +func (h *heldFindings) has(kind string) bool { + for _, f := range h.Held() { + if f.Kind == kind { + return true + } + } + return false +} + +// fakeBounds is the state bounds: each key removed by the bus when its time to live passes, as the bus does, or by its +// end; written and removed say which keys, with each key's time to live (A-143). +type fakeBounds struct { + mu sync.Mutex + written []string + removed []string +} + +func (f *fakeBounds) Bound(key string, ttl time.Duration, expired func()) (func(), error) { + f.mu.Lock() + f.written = append(f.written, key+" "+ttl.String()) + f.mu.Unlock() + var once sync.Once + gone := func(by string, then func()) { + once.Do(func() { + f.mu.Lock() + f.removed = append(f.removed, key+" "+by) + f.mu.Unlock() + then() + }) + } + t := time.AfterFunc(ttl, func() { gone("expired", expired) }) + return func() { t.Stop(); gone("ended", func() {}) }, nil +} + +func (f *fakeBounds) keys() (written, removed string) { + f.mu.Lock() + defer f.mu.Unlock() + return strings.Join(f.written, "|"), strings.Join(f.removed, "|") +} diff --git a/modules/mesh-recovery/internal/recovery/findings.go b/modules/mesh-recovery/internal/recovery/findings.go new file mode 100644 index 000000000..ca4be0c11 --- /dev/null +++ b/modules/mesh-recovery/internal/recovery/findings.go @@ -0,0 +1,160 @@ +package recovery + +import ( + "context" + "fmt" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" +) + +// The recovery module's findings (design 47 G §561), each written by mesh-sdk's finding writer as the key +// `..` and said on the module's finding stream. +const ( + FindingPointsFilling = "points-filling" + FindingRecoveryWaits = "recovery-waits" + FindingAsideUnrestored = "aside-unrestored" +) + +// FindingKinds are the kinds the module's writer owns. +var FindingKinds = []string{FindingPointsFilling, FindingRecoveryWaits, FindingAsideUnrestored} + +// Findings is the module's finding writer (mesh-sdk's held writer): what it writes and clears, and what it holds. +type Findings interface { + Write(ctx context.Context, f meshconditions.Finding) error + Clear(ctx context.Context, scope, id, kind, why string) error + Held() []meshconditions.Finding +} + +// The defaults of the settings §557 and §564 make bounds of. +const ( + DefaultFillingMark = 80 + DefaultDumpWait = 60 * time.Minute +) + +func (e *Engine) mark() int64 { + if e.Settings.FillingMark > 0 { + return int64(e.Settings.FillingMark) + } + return DefaultFillingMark +} + +func (e *Engine) dumpWait() time.Duration { + if e.Settings.DumpWait > 0 { + return e.Settings.DumpWait + } + return DefaultDumpWait +} + +func (e *Engine) finding(kind, scope, id, headline, summary, needs string) meshconditions.Finding { + return meshconditions.Finding{Scope: scope, ID: id, Kind: kind, Subject: meshconditions.Subject{Scope: "module", ID: Module}, + Severity: meshconditions.FindingSeverityWarning, Headline: headline, Summary: summary, Explanation: summary, Needs: needs, + Source: "points", Observer: Module, At: e.now()} +} + +func (e *Engine) write(f meshconditions.Finding) { + if e.Findings == nil { + return + } + if err := e.Findings.Write(context.Background(), f); err != nil { + e.logf("the finding %s %s.%s cannot be written: %v", f.Kind, f.Scope, f.ID, err) + } +} + +func (e *Engine) clear(scope, id, kind, why string) { + if e.Findings == nil { + return + } + if err := e.Findings.Clear(context.Background(), scope, id, kind, why); err != nil { + e.logf("the finding %s %s.%s cannot be cleared: %v", kind, scope, id, err) + } +} + +// judge measures the points directory against the fill mark of its cap (points-filling) and writes aside-unrestored for +// each aside whose record ends at removed that no running act holds, clearing those that no longer are (§561). It runs at +// the start, after each act and so after each expiry run. +func (e *Engine) judge() { + if e.Findings == nil { + return + } + fill, limit := e.filled(), e.Settings.Cap*e.mark()/100 + if fill > limit { + e.write(e.finding(FindingPointsFilling, Module, e.Node, "The recovery module's restore points are filling up", + fmt.Sprintf("the points directory holds %d of %d bytes, past its fill mark of %d %%; the largest: %s", fill, e.Settings.Cap, + e.mark(), strings.Join(e.largest(5), ", ")), "points removed or an aside discarded, or a larger points-cap")) + } else { + e.clear(Module, e.Node, FindingPointsFilling, fmt.Sprintf("the points directory holds %d bytes, under its fill mark", fill)) + } + open := map[string]bool{} + for _, u := range e.Unrestored() { + open[u.Module+"\x00"+u.Aside.ID] = true + e.write(e.finding(FindingAsideUnrestored, u.Module, u.Aside.ID, "An aside holds data the live bus lacks", + fmt.Sprintf("the aside %s of %s's %s (%s) ended at removed and was not restored", u.Aside.ID, u.Module, u.Kept, + strings.Join(u.Aside.Names, ", ")), "recovery_restore_aside, or recovery_discard_aside")) + } + for _, f := range e.Findings.Held() { + if f.Kind == FindingAsideUnrestored && !open[f.Scope+"\x00"+f.ID] { + e.clear(f.Scope, f.ID, f.Kind, "the aside was restored or discarded") + } + } +} + +// largest are the n largest points and asides, each with its size. +func (e *Engine) largest(n int) []string { + type sized struct { + what string + size int64 + } + var all []sized + pts, _ := e.Points() + for _, p := range pts { + all = append(all, sized{"point " + p.Snapshot, p.Size}) + } + _, recs, _ := e.Asides() + for _, r := range recs { + all = append(all, sized{"aside " + r.Module + "/" + r.Kept + "/" + r.ID, r.Size}) + } + sort.SliceStable(all, func(i, j int) bool { return all[i].size > all[j].size }) + var out []string + for i := 0; i < len(all) && i < n; i++ { + out = append(out, fmt.Sprintf("%s (%d bytes)", all[i].what, all[i].size)) + } + return out +} + +// waitsOn is the waiting of one act on the dump: past the bound, the key `dump-wait.` of the state bounds expiring +// (A-143), it writes recovery-waits, naming the item and since when; done removes the key and clears the finding once the +// act starts its work or ends (§561, §563). +func (e *Engine) waitsOn(a *Act) (waiting func(on string), done func()) { + var end func() + written := make(chan bool, 1) + written <- false + waiting = func(on string) { + e.setStep(a, "waiting", on) + since := e.now() + key := BoundDumpWait + "." + a.ID + var err error + end, err = e.bound(key, e.dumpWait(), func() { + if !<-written { + e.write(e.finding(FindingRecoveryWaits, a.Module, a.ID, "A restore waits on the dump of the bus", + fmt.Sprintf("the act %s waits on %s since %s", a.ID, on, since.Format(time.RFC3339)), "the dump to end, or recovery_cancel")) + } + written <- true + }) + if err != nil { + e.logf("the bound %s of the act's wait on the dump is not written, so recovery-waits is never written for it: %v", key, err) + } + } + done = func() { + if end != nil { + end() + } + if <-written { + e.clear(a.Module, a.ID, FindingRecoveryWaits, "the act no longer waits on the dump") + } + written <- false + } + return waiting, done +} diff --git a/modules/mesh-recovery/internal/recovery/keeper.go b/modules/mesh-recovery/internal/recovery/keeper.go new file mode 100644 index 000000000..38dde2e82 --- /dev/null +++ b/modules/mesh-recovery/internal/recovery/keeper.go @@ -0,0 +1,220 @@ +package recovery + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "git.novox.be/novox/mesh-contracts/go/seats/meshbroker" +) + +// Keeper is the bus module's keeper as the seat mesh-broker answers it (design 47 G §681): its snapshot taken before a +// `drop` or `recreate`, sent by snapshot-get, or restored in place by snapshot-restore when it holds a protected stream. +type Keeper interface { + // Get asks snapshot-get of the request; when its answer names a file and fits accepts it, it receives the chunks into + // w. A refusal of the keeper is the answer's Error, no error of Get. + Get(ctx context.Context, request string, fits func(meshbroker.SnapshotGetAnswer) error, w io.Writer) (meshbroker.SnapshotGetAnswer, error) + // Restore asks snapshot-restore; with verify the keeper checks the file and restores nothing. + Restore(ctx context.Context, request string, verify bool, why string) (meshbroker.SnapshotRestoreAnswer, error) + // CancelRestore asks snapshot-restore-cancel of a running restore. + CancelRestore(ctx context.Context, act, why string) (meshbroker.SnapshotRestoreCancelAnswer, error) + // Restored answers the keeper's snapshot-restored of that request and act once heard (read as §556 has it, so one said + // before the wait counts), or ok false when ctx ends first. + Restored(ctx context.Context, request, act string) (ev meshbroker.SnapshotRestoredEvent, ok bool) +} + +// The keeper's word for a snapshot it never hands out (§681). +const snapshotProtected = "snapshot protected" + +// ErrUnanswered is the end of a restore in place the keeper answered neither by its outcome nor by the cancel's (§681). +const ErrUnanswered = "snapshot-restore unanswered" + +// DefaultRestoreWait is how long a restore in place is waited on: past the keeper's own 60 minutes (§681, A-138). +const DefaultRestoreWait = 90 * time.Minute + +func (e *Engine) restoreWait() time.Duration { + if e.Settings.RestoreWait > 0 { + return e.Settings.RestoreWait + } + return DefaultRestoreWait +} + +func (e *Engine) incoming() string { return filepath.Join(e.Dir, "incoming") } + +// restoreRequest is the hand verb `restore` with `request`, a keeper's snapshot (§681 Its use, §682): an unprotected one is +// read through snapshot-get into incoming/, refused before the first chunk when it would pass the cap, checked +// against its digest before any aside, and restored by the module itself as a point is (§516), the file deleted when the +// act ends; a protected one is restored in place by the keeper (restoreInPlace). +func (e *Engine) restoreRequest(ctx context.Context, a *Act) error { + if e.Keeper == nil { + return e.endedRestored(a, errors.New("no keeper is reachable, so no snapshot of it is restored"), nil, false) + } + e.setStep(a, "receiving the keeper's snapshot", "") + name, err := token(a.Request) + if err != nil { + return e.endedRestored(a, err, nil, false) + } + if err := os.MkdirAll(e.incoming(), 0o700); err != nil { + return e.endedRestored(a, err, nil, false) + } + file := filepath.Join(e.incoming(), name) + defer os.Remove(file) + f, err := os.Create(file) + if err != nil { + return e.endedRestored(a, err, nil, false) + } + ans, err := e.Keeper.Get(ctx, a.Request, func(ans meshbroker.SnapshotGetAnswer) error { + if fill := e.filled(); fill+ans.Size > e.Settings.Cap { + return fmt.Errorf("%s: the cap is %d bytes, the snapshot of the request %s needs %d and the directory holds %d", + ErrCap, e.Settings.Cap, a.Request, ans.Size, fill) + } + return nil + }, f) + if cerr := f.Close(); err == nil { + err = cerr + } + if err != nil { + return e.endedRestored(a, err, nil, false) + } + if ans.Error == snapshotProtected { + _ = os.Remove(file) + return e.restoreInPlace(ctx, a) + } + if ans.Error != "" { + return e.endedRestored(a, errors.New(ans.Error), nil, false) + } + // The bytes are checked against the keeper's digest before any aside is taken (§567). + if got, n, err := digestOf(file); err != nil || got != ans.Digest || n != ans.Size { + return e.endedRestored(a, fmt.Errorf("snapshot damaged: the request %s's bytes received do not match its digest", a.Request), nil, false) + } + rec, err := e.setAside(ctx, a, []string{ans.Stream}, true) + if err != nil { + return e.endedRestored(a, err, nil, false) + } + e.setStep(a, "restoring", "") + if err := e.Bus.Restore(ctx, file); err != nil { + err = fmt.Errorf("stream %s: %w", ans.Stream, err) + if back := e.restoreFromAside(ctx, rec, true); back != nil { + err = fmt.Errorf("%w; and the aside is not put back: %v", err, back) + } + return e.endedRestored(a, err, asidePtr(rec), false) + } + rec.Step = stepRestored + _ = writeJSON(filepath.Join(e.asideDirOf(rec), asideFile), rec) + return e.endedRestored(a, nil, asidePtr(rec), false) +} + +// restoreInPlace is a protected snapshot's restore (§681, O-7): snapshot-restore with verify first, then the live data of +// the restore's module and kept data set aside, its names from the catalog entry of the fingerprint the node runs +// (A-137), then snapshot-restore, waited on by snapshot-restored of that request and the act it was answered for at most +// snapshot-restore-wait-minutes (A-138). Its `failed` puts the aside back. At the bound it asks snapshot-restore-cancel and +// waits one more bound for that act's outcome (A-140); when none comes it puts nothing back and ends failed with +// `snapshot-restore unanswered`, the aside left at `removed` for aside-unrestored (§561). A keeper answering busy ends the +// act failed before any aside, naming the keeper's running act (A-140). +func (e *Engine) restoreInPlace(ctx context.Context, a *Act) error { + e.setStep(a, "verifying the keeper's snapshot", "") + v, err := e.Keeper.Restore(ctx, a.Request, true, a.Why) + if err == nil { + err = keeperAnswer(v) + } + if err != nil { + return e.endedRestored(a, err, nil, false) + } + fp, err := e.Seats.RunningFingerprint(ctx, a.Module) + if err != nil { + return e.endedRestored(a, err, nil, false) + } + a.Fingerprint = fp + names, err := e.Seats.Streams(ctx, a.Module, a.Kept, fp) + if err != nil { + return e.endedRestored(a, err, nil, false) + } + rec, err := e.setAside(ctx, a, names, true) + if err != nil { + return e.endedRestored(a, err, nil, false) + } + putBack := func(why error) error { + if back := e.restoreFromAside(ctx, rec, true); back != nil { + why = fmt.Errorf("%w; and the aside is not put back: %v", why, back) + } + return e.endedRestored(a, why, asidePtr(rec), false) + } + e.setStep(a, "restoring in place", "the keeper's snapshot-restored") + // The wait's bound is the key `restore-wait.` of the state bounds, written before the keeper is asked, so a bound + // that cannot be written asks nothing of the keeper (A-143). + wctx, passed, err := e.restoreBound(ctx, a.ID) + if err != nil { + return putBack(err) + } + r, err := e.Keeper.Restore(ctx, a.Request, false, a.Why) + if err == nil { + err = keeperAnswer(r) + } + if err != nil { + passed() + return putBack(err) + } + act := r.Accepted + ev, ok := e.Keeper.Restored(wctx, a.Request, act) + passed() + if !ok { + e.setStep(a, "cancelling the keeper's restore", "the keeper's snapshot-restored after the cancel") + why := fmt.Sprintf("the recovery module's wait of %s on the restore in place passed (act %s)", e.restoreWait(), a.ID) + if c, err := e.Keeper.CancelRestore(context.Background(), act, why); err != nil || c.Error != "" { + e.logf("the cancel of the keeper's restore %s is not accepted (%v%s); its outcome is waited on once more", act, err, c.Error) + } + wctx, passed, err := e.restoreBound(context.Background(), a.ID) + if err != nil { + e.logf("the bound of the wait after the cancel of %s is not written: %v", act, err) + return e.endedRestored(a, fmt.Errorf("%s: %v", ErrUnanswered, err), asidePtr(rec), false) + } + ev, ok = e.Keeper.Restored(wctx, a.Request, act) + passed() + } + if !ok { + return e.endedRestored(a, errors.New(ErrUnanswered), asidePtr(rec), false) + } + if ev.Outcome != meshbroker.SnapshotRestoredEventOutcomeDone { + return putBack(fmt.Errorf("the keeper's restore in place failed: %s", ev.Error)) + } + rec.Step = stepRestored + _ = writeJSON(filepath.Join(e.asideDirOf(rec), asideFile), rec) + return e.endedRestored(a, nil, asidePtr(rec), false) +} + +// restoreBound is the bound of one wait on snapshot-restored (§681, A-138, A-143): the key `restore-wait.` with the time +// to live snapshot-restore-wait-minutes; its expiry ends the context it answers. end removes the key. +func (e *Engine) restoreBound(ctx context.Context, act string) (context.Context, func(), error) { + wctx, cancel := context.WithCancel(ctx) + end, err := e.bound(BoundRestoreWait+"."+act, e.restoreWait(), cancel) + if err != nil { + cancel() + return nil, nil, err + } + return wctx, func() { end(); cancel() }, nil +} + +// bound writes a bound key in the state bounds (A-77, A-143). +func (e *Engine) bound(key string, ttl time.Duration, expired func()) (func(), error) { + if e.Bounds == nil { + return nil, fmt.Errorf("the bound %s is not written: the engine has no state bounds", key) + } + return e.Bounds.Bound(key, ttl, expired) +} + +// keeperAnswer reads snapshot-restore's one form (§691): accepted, or an error naming the keeper's refusal or its running act. +func keeperAnswer(r meshbroker.SnapshotRestoreAnswer) error { + switch { + case r.Error != "": + return errors.New(r.Error) + case r.Busy != "": + return fmt.Errorf("busy: the keeper restores its act %s", r.Busy) + case r.Accepted == "": + return errors.New("the keeper answered snapshot-restore with nothing") + } + return nil +} diff --git a/modules/mesh-recovery/internal/recovery/keeper_test.go b/modules/mesh-recovery/internal/recovery/keeper_test.go new file mode 100644 index 000000000..f816966ab --- /dev/null +++ b/modules/mesh-recovery/internal/recovery/keeper_test.go @@ -0,0 +1,216 @@ +package recovery_test + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "io" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-contracts/go/seats/meshbroker" + + "mesh-recovery/internal/recovery" +) + +// fakeKeeper is the bus module's keeper holding one snapshot of app_notes (the fake bus's file form). in place: how its +// restore in place ends: "done", "failed", "silent" (it says nothing until cancelled, then failed) or "deaf" (never). +type fakeKeeper struct { + mu sync.Mutex + b *fakeBus + file string + protected bool + busy string + damaged bool + inPlace string + asked []string + wrote int + cancelled chan struct{} +} + +func newKeeper(b *fakeBus) *fakeKeeper { + return &fakeKeeper{b: b, file: "snap:app_notes:as dropped", inPlace: "done", cancelled: make(chan struct{})} +} + +func (k *fakeKeeper) note(s string) { + k.mu.Lock() + k.asked = append(k.asked, s) + k.mu.Unlock() +} + +func (k *fakeKeeper) Get(_ context.Context, request string, fits func(meshbroker.SnapshotGetAnswer) error, w io.Writer) (meshbroker.SnapshotGetAnswer, error) { + k.note("get " + request) + if k.protected { + return meshbroker.SnapshotGetAnswer{Error: "snapshot protected"}, nil + } + sum := sha256.Sum256([]byte(k.file)) + ans := meshbroker.SnapshotGetAnswer{Request: request, Module: "app", Local: "notes", Stream: "app_notes", Size: int64(len(k.file)), + Digest: "sha256:" + hex.EncodeToString(sum[:]), Chunks: 1} + if err := fits(ans); err != nil { + return ans, err + } + body := k.file + if k.damaged { + body = "snap:app_notes:altered on its way" + } + n, err := io.WriteString(w, body) + k.wrote += n + return ans, err +} + +func (k *fakeKeeper) Restore(_ context.Context, request string, verify bool, _ string) (meshbroker.SnapshotRestoreAnswer, error) { + if verify { + k.note("verify " + request) + return meshbroker.SnapshotRestoreAnswer{Accepted: request}, nil + } + k.note("restore " + request) + if k.busy != "" { + return meshbroker.SnapshotRestoreAnswer{Busy: k.busy}, nil + } + return meshbroker.SnapshotRestoreAnswer{Accepted: "keeper-act-1"}, nil +} + +func (k *fakeKeeper) CancelRestore(_ context.Context, act, _ string) (meshbroker.SnapshotRestoreCancelAnswer, error) { + k.note("cancel " + act) + close(k.cancelled) + return meshbroker.SnapshotRestoreCancelAnswer{Accepted: act}, nil +} + +func (k *fakeKeeper) Restored(ctx context.Context, request, act string) (meshbroker.SnapshotRestoredEvent, bool) { + ev := meshbroker.SnapshotRestoredEvent{Request: request, Act: act, Outcome: "failed", Error: "the server refused the archive"} + switch k.inPlace { + case "done": + k.b.mu.Lock() + if _, live := k.b.streams["app_notes"]; live { + k.b.mu.Unlock() + return meshbroker.SnapshotRestoredEvent{Request: request, Act: act, Outcome: "failed", Error: "names live"}, true + } + k.b.streams["app_notes"] = "as dropped" + k.b.mu.Unlock() + ev.Outcome, ev.Error = "done", "" + return ev, true + case "failed": + return ev, true + case "silent": + select { + case <-k.cancelled: + ev.Error = "cancelled" + return ev, true + case <-ctx.Done(): + return ev, false + } + } + <-ctx.Done() + return ev, false +} + +func requestRig(t *testing.T) (*rig, *fakeKeeper) { + r := newRig(t) + k := newKeeper(r.b) + r.e.Keeper = k + r.e.Settings.RestoreWait = 20 * time.Millisecond + return r, k +} + +func requestAct() recovery.Act { + return recovery.Act{Kind: recovery.KindRestore, Module: "app", Kept: "notes", Request: "req-1", Why: "test the drop's snapshot", Caller: "tool-runner", Asker: "op@node"} +} + +// A keeper's snapshot by request is read through snapshot-get into incoming/, refused before the first chunk past +// the cap, checked against its digest before any aside, restored by the module itself, and its file deleted at the end +// (design 47 G §681 Its use, §682). +func TestARestoreByRequestReadsTheKeepersSnapshot(t *testing.T) { + for _, tc := range []struct { + name string + cap int64 + damaged bool + err string + }{ + {name: "received and restored", cap: 1 << 20}, + {name: "altered on its way", cap: 1 << 20, damaged: true, err: "snapshot damaged"}, + {name: "past the cap", cap: 10, err: "cap:"}, + } { + t.Run(tc.name, func(t *testing.T) { + r, k := requestRig(t) + r.e.Settings.Cap, k.damaged = tc.cap, tc.damaged + ans := r.do(requestAct()) + ev := lastRestored(t, r) + if ev.Act != ans.Accepted || !strings.HasPrefix(ev.Error, tc.err) || (tc.err == "") != (ev.Outcome == "done") { + t.Fatalf("restored %+v, want error %q", ev, tc.err) + } + if left, _ := os.ReadDir(filepath.Join(r.e.Dir, "incoming")); len(left) != 0 { + t.Errorf("incoming holds %v after the act", left) + } + kp, _ := r.e.KeptPoints("app", "notes") + if tc.err != "" { + if r.b.streams["app_notes"] != "hello" || len(kp.Asides) != 0 { + t.Errorf("a refused restore touched the live data: %v, asides %+v", r.b.streams, kp.Asides) + } + if tc.name == "past the cap" && k.wrote != 0 { + t.Errorf("%d bytes received past the cap", k.wrote) + } + return + } + if r.b.streams["app_notes"] != "as dropped" || r.b.streams["KV_app_seen"] != "world" || len(kp.Asides) != 1 || + ev.Aside == nil || len(ev.Aside.Names) != 1 || ev.Aside.Names[0] != "app_notes" { + t.Errorf("streams %v, asides %+v, event aside %+v", r.b.streams, kp.Asides, ev.Aside) + } + }) + } +} + +// A protected snapshot is verified, the running data of the module and kept data set aside (A-137), and restored in place +// by the keeper; its failure puts the aside back; at the bound the module cancels first and puts the aside back on the +// cancel's outcome, and with no outcome at all ends `snapshot-restore unanswered`, the aside left unrestored; a busy keeper +// ends the act before any aside (§681, A-138, A-140). +func TestAProtectedSnapshotIsRestoredInPlaceByTheKeeper(t *testing.T) { + for _, tc := range []struct { + inPlace, busy string + err string + asked string + live string + unrestored int + bounds string // how each bound key restore-wait. was removed (A-143) + }{ + {inPlace: "done", asked: "get req-1|verify req-1|restore req-1", live: "as dropped", bounds: "ended"}, + {inPlace: "failed", asked: "get req-1|verify req-1|restore req-1", live: "hello", err: "the keeper's restore in place failed", bounds: "ended"}, + {inPlace: "silent", asked: "get req-1|verify req-1|restore req-1|cancel keeper-act-1", live: "hello", err: "the keeper's restore in place failed: cancelled", bounds: "expired|ended"}, + {inPlace: "deaf", asked: "get req-1|verify req-1|restore req-1|cancel keeper-act-1", err: recovery.ErrUnanswered, unrestored: 1, bounds: "expired|expired"}, + {inPlace: "done", busy: "keeper-act-0", asked: "get req-1|verify req-1|restore req-1", live: "hello", err: "busy: the keeper restores its act keeper-act-0", bounds: "ended"}, + } { + t.Run(tc.inPlace+tc.busy, func(t *testing.T) { + r, k := requestRig(t) + k.protected, k.inPlace, k.busy = true, tc.inPlace, tc.busy + r.do(requestAct()) + ev := lastRestored(t, r) + if !strings.HasPrefix(ev.Error, tc.err) || (tc.err == "") != (ev.Outcome == "done") || ev.Fingerprint != "fp-run" { + t.Fatalf("restored %+v, want error %q", ev, tc.err) + } + if got := strings.Join(k.asked, "|"); got != tc.asked { + t.Errorf("asked %s, want %s", got, tc.asked) + } + if r.b.streams["app_notes"] != tc.live { + t.Errorf("app_notes is %q, want %q", r.b.streams["app_notes"], tc.live) + } + if tc.live == "hello" && r.b.streams["KV_app_seen"] != "world" { + t.Errorf("the second name of the running data is not put back: %v", r.b.streams) + } + if n := len(r.e.Unrestored()); n != tc.unrestored { + t.Errorf("%d unrestored asides, want %d", n, tc.unrestored) + } + w, rm := r.bounds.keys() + var wantW, wantR []string + for _, by := range strings.Split(tc.bounds, "|") { + wantW = append(wantW, "restore-wait."+ev.Act+" 20ms") + wantR = append(wantR, "restore-wait."+ev.Act+" "+by) + } + if w != strings.Join(wantW, "|") || rm != strings.Join(wantR, "|") { + t.Errorf("bound keys written %q, removed %q; want %q, %q", w, rm, strings.Join(wantW, "|"), strings.Join(wantR, "|")) + } + }) + } +} diff --git a/modules/mesh-recovery/internal/recovery/manifest_test.go b/modules/mesh-recovery/internal/recovery/manifest_test.go new file mode 100644 index 000000000..185840f3d --- /dev/null +++ b/modules/mesh-recovery/internal/recovery/manifest_test.go @@ -0,0 +1,135 @@ +package recovery_test + +import ( + "encoding/json" + "os" + "slices" + "testing" + + seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" +) + +type manifest struct { + Module string `json:"module"` + RunsAs string `json:"runs-as"` + Claims []struct { + Name string `json:"name"` + Scope string `json:"scope"` + Serves []string `json:"serves"` + } `json:"claims"` + Tools []string `json:"tools"` + Consumes []string `json:"consumes"` + Invokes []string `json:"invokes"` + Config map[string]struct { + Default float64 `json:"default"` + } `json:"config"` + Data struct { + Own []struct{ ID string } `json:"own"` + } `json:"data"` + Resources []struct { + ID string `json:"id"` + Type string `json:"type"` + Serves []string `json:"serves"` + Bus struct { + State []string `json:"state"` + Invokes []string `json:"invokes"` + } `json:"bus"` + Health struct { + Kind string `json:"kind"` + } `json:"health"` + } `json:"resources"` + RuntimeHealth struct { + Kind string `json:"kind"` + Tool string `json:"tool"` + } `json:"runtime-health"` + Requires []json.RawMessage `json:"requires"` +} + +// The manifest says what the module is (design 47 G §554, §656, §657, §633, §564, §567): it claims the seat once for the +// mesh serving every verb of the seat's definition, its process answers each of them itself (A-115: `serves` beside its +// `bus`), its health check is its tool beside the process's own check and its template states are its own by `requires` +// alone, the digests state taking the template's fixed cap (A-133, A-135), its data item is `points`, its settings carry +// the design's defaults, and what the process invokes the module declares. +func TestTheManifestSaysWhatTheModuleIs(t *testing.T) { + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + var m manifest + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + if m.Module != "mesh-recovery" || m.RunsAs != "mesh-recovery" { + t.Errorf("module %q runs as %q", m.Module, m.RunsAs) + } + if len(m.Claims) != 1 || m.Claims[0].Name != seat.Seat || m.Claims[0].Scope != "mesh" || !slices.Equal(m.Claims[0].Serves, seat.Verbs()) { + t.Errorf("claims %+v, want %s at mesh scope serving %v", m.Claims, seat.Seat, seat.Verbs()) + } + for _, id := range []string{"points"} { + if !slices.ContainsFunc(m.Data.Own, func(o struct{ ID string }) bool { return o.ID == id }) { + t.Errorf("no data item %q", id) + } + } + for k, want := range map[string]float64{"points-cap": 10 << 30, "points-filling-mark": 80, "point-keep-days": 90, "dump-wait-minutes": 60, "snapshot-restore-wait-minutes": 90, "expire-every-hours": 24} { + if m.Config[k].Default != want { + t.Errorf("setting %s defaults to %v, want %v", k, m.Config[k].Default, want) + } + } + // The keeper's snapshot verbs the recovery module asks, the cancel at its bound among them (§681). + for _, inv := range []string{"seat:mesh-broker.snapshot-get", "seat:mesh-broker.snapshot-restore", "seat:mesh-broker.snapshot-restore-cancel"} { + if !slices.Contains(m.Invokes, inv) { + t.Errorf("does not invoke %s", inv) + } + } + for _, r := range m.Resources { + if r.Type != "process" { + continue + } + var want []string + for _, v := range seat.Verbs() { + want = append(want, "seat:"+seat.Seat+"."+v) + } + if !slices.Equal(r.Serves, want) { + t.Errorf("the process serves %v, want %v", r.Serves, want) + } + if r.Health.Kind != "unit" { + t.Errorf("the process's own check is %q, want unit", r.Health.Kind) + } + if len(r.Bus.State) != 0 { + t.Errorf("the process's bus lists the states %v again, which are the module's own by its requires", r.Bus.State) + } + for _, inv := range r.Bus.Invokes { + if !slices.Contains(m.Invokes, inv) { + t.Errorf("the process invokes %s, which the module does not declare", inv) + } + } + } + if m.RuntimeHealth.Kind != "tool" || m.RuntimeHealth.Tool != "recovery_kept_points" || !slices.Contains(m.Tools, m.RuntimeHealth.Tool) { + t.Errorf("the tool check is %+v, want recovery_kept_points, a tool of %v", m.RuntimeHealth, m.Tools) + } + bounds := false + for _, raw := range m.Requires { + var q struct { + Name string `json:"name"` + Options map[string]any `json:"options"` + } + if json.Unmarshal(raw, &q) == nil && q.Name == "digests" && len(q.Options) != 0 { + t.Errorf("digests gives the options %v beside the template state's fixed cap", q.Options) + } + // The expiry's bound is a bound key of its own state, its keys living a time of their own (A-77, A-138). + bounds = bounds || q.Name == "bounds" && q.Options["key-ttl"] == true + } + if !bounds { + t.Error("no state bounds with key-ttl for the expiry's bound key") + } + for _, c := range []string{"node-backup.item-started", "node-backup.item-ended", "mesh-broker.snapshot-restored"} { + if !slices.Contains(m.Consumes, c) { + t.Errorf("does not consume %s", c) + } + } + for _, tool := range m.Tools { + if len(tool) < 9 || tool[:9] != "recovery_" { + t.Errorf("the tool %q is not named recovery_", tool) + } + } +} diff --git a/modules/mesh-recovery/internal/recovery/points.go b/modules/mesh-recovery/internal/recovery/points.go new file mode 100644 index 000000000..aa7207074 --- /dev/null +++ b/modules/mesh-recovery/internal/recovery/points.go @@ -0,0 +1,251 @@ +package recovery + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "time" + + seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +const pointFile = "point.json" + +// run is the work of one accepted act; it ends the act with its event under the act's id (§560). +func (e *Engine) run(ctx context.Context, a *Act) { + var err error + switch a.Kind { + case KindTake: + err = e.take(ctx, a) + case KindExpire: + e.expire(a.ID, true) + case KindRestore: + err = e.restore(ctx, a) + case KindAside: + err = e.aside(ctx, a) + case KindRestoreAside: + err = e.restoreAside(ctx, a) + case KindDiscardAside: + err = e.discardAside(a) + default: + err = e.endedFailed(a, fmt.Errorf("no act of kind %q", a.Kind)) + } + if err != nil { + e.logf("act %s ended failed: %v", a.ID, err) + a.Error = err.Error() + } + e.markEnded(a) +} + +// sayEnd says the ending event of the act id (§554, §560): the act is first no longer the running one and written ended, so +// a caller told busy that asks again on hearing it is never told busy with the same act again. +func (e *Engine) sayEnd(id, event string, body any, parts ...string) { + e.mu.Lock() + if e.running != nil && e.running.ID == id { + e.markEnded(e.running) + e.running, e.cancel = nil, nil + } + e.mu.Unlock() + e.say(event, body, parts...) +} + +// say publishes an event and logs the failure; an event lost is never silent. +func (e *Engine) say(event string, body any, parts ...string) { + if err := e.Seats.Say(event, body, parts...); err != nil { + e.logf("the event %s was not said: %v", event, err) + } +} + +func (e *Engine) endedFailed(a *Act, why error) error { + why = e.stoppedOr(why) + e.sayEnd(a.ID, seat.EventEnded, seat.EndedEvent{Act: a.ID, At: e.now(), Module: a.Module, Kept: a.Kept, Form: a.Form, + Fingerprint: a.Fingerprint, Snapshot: a.Snapshot, Outcome: seat.EndedEventOutcomeFailed, Error: why.Error(), Why: cancelWhy(a, why)}, + a.Module, a.Kept) + return why +} + +// cancelWhy is the hand act's words for a cancel (A-117). +func cancelWhy(a *Act, why error) string { + if why != nil && why.Error() == ErrCanceled { + return a.Why + } + return "" +} + +// filled is the fill of the points directory without the temporary names of acts still writing. +func (e *Engine) filled() int64 { + return dirSize(filepath.Join(e.Dir, "points")) + dirSize(filepath.Join(e.Dir, "aside")) +} + +// take is `snapshot-kept` and the hand verb `take` (§516, §564, §567): one stream at a time under a temporary name, +// renamed whole, its digests on the bus before `ended` is said. +func (e *Engine) take(ctx context.Context, a *Act) error { + dir, err := e.pointDir(a.Module, a.Kept, a.Form, a.Fingerprint) + if err != nil { + return e.endedFailed(a, err) + } + names, err := e.Seats.Streams(ctx, a.Module, a.Kept, a.Fingerprint) + if err != nil || len(names) == 0 { + if err == nil { + err = fmt.Errorf("%s's %s has no bus name at %s", a.Module, a.Kept, a.Fingerprint) + } + return e.endedFailed(a, err) + } + e.setStep(a, "measuring", "") + objs, err := e.Bus.Objects(ctx, names) + if err != nil { + return e.endedFailed(a, err) + } + var need int64 + for _, o := range objs { + need += o.Bytes + } + if fill := e.filled(); fill+need > e.Settings.Cap { + return e.endedFailed(a, fmt.Errorf("%s: the cap is %d bytes, the point needs %d and the directory holds %d", ErrCap, e.Settings.Cap, need, fill)) + } + tmp := filepath.Join(filepath.Dir(dir), ".tmp-"+a.ID) + _ = os.RemoveAll(tmp) + if err := os.MkdirAll(tmp, 0o700); err != nil { + return e.endedFailed(a, err) + } + defer os.RemoveAll(tmp) + p := seat.Point{At: e.now(), CatchUp: a.CatchUp, Fingerprint: a.Fingerprint, Form: a.Form, Kept: a.Kept, Module: a.Module, + Snapshot: strings.TrimPrefix(dir, filepath.Join(e.Dir, "points")+string(filepath.Separator))} + e.setStep(a, "taking", "") + for _, o := range objs { + if !o.Exists { + continue + } + file := filepath.Join(tmp, keys.EscapeName(o.Stream)+".snap") + sum, n, err := e.takeOne(ctx, o.Stream, file) + if err != nil { + return e.endedFailed(a, err) + } + p.Digests = append(p.Digests, seat.Digest{Name: o.Stream, Digest: sum}) + p.Size += n + p.Consumers = append(p.Consumers, o.Consumers...) + if e.filled()+p.Size > e.Settings.Cap { + return e.endedFailed(a, fmt.Errorf("%s: the stream grew while it was taken and passes the cap of %d bytes", ErrCap, e.Settings.Cap)) + } + } + if err := writeJSON(filepath.Join(tmp, pointFile), p); err != nil { + return e.endedFailed(a, err) + } + _ = os.RemoveAll(dir) + if err := os.Rename(tmp, dir); err != nil { + return e.endedFailed(a, err) + } + raw, _ := jsonMarshal(p.Digests) + if err := e.Digests.Put(keys.EscapeName(p.Snapshot), raw); err != nil { + _ = os.RemoveAll(dir) + return e.endedFailed(a, fmt.Errorf("the digests are not on the bus: %w", err)) + } + // The removal at each snapshot-kept (§551) runs within the take, before its ended, so one act runs at a time. + e.expire(a.ID, false) + e.sayEnd(a.ID, seat.EventEnded, seat.EndedEvent{Act: a.ID, At: e.now(), Module: a.Module, Kept: a.Kept, Form: a.Form, + Fingerprint: a.Fingerprint, Snapshot: p.Snapshot, Outcome: seat.EndedEventOutcomeDone, Consumers: p.Consumers, + Digests: p.Digests, Size: p.Size, CatchUp: p.CatchUp}, a.Module, a.Kept) + return nil +} + +func (e *Engine) takeOne(ctx context.Context, stream, file string) (string, int64, error) { + f, err := os.Create(file) + if err != nil { + return "", 0, err + } + if err := e.Bus.Take(ctx, stream, f); err != nil { + f.Close() + return "", 0, fmt.Errorf("stream %s: %w", stream, err) + } + if err := f.Close(); err != nil { + return "", 0, err + } + return digestOf(file) +} + +func digestOf(file string) (string, int64, error) { + f, err := os.Open(file) + if err != nil { + return "", 0, err + } + defer f.Close() + h := sha256.New() + n, err := io.Copy(h, f) + return "sha256:" + hex.EncodeToString(h.Sum(nil)), n, err +} + +// Points lists every restore point in the directory, newest first (§517): the one place they are kept. +func (e *Engine) Points() ([]seat.Point, error) { + var out []seat.Point + root := filepath.Join(e.Dir, "points") + err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err + } + if d.IsDir() && strings.HasPrefix(d.Name(), ".") { + return filepath.SkipDir + } + if !d.IsDir() && d.Name() == pointFile { + var p seat.Point + if err := readJSON(path, &p); err != nil { + return fmt.Errorf("%s: %w", path, err) + } + out = append(out, p) + } + return nil + }) + sort.Slice(out, func(i, j int) bool { return out[i].At.After(out[j].At) }) + return out, err +} + +// expire is §551's removal: a point 90 days after its `at` (the setting), never the newest of its kept data, each removal +// said as point-removed and the run as expiry-ended, also when it removes nothing (§560). It runs inside an act: own when +// it is the act expire-points, whose ending event its expiry-ended is. +func (e *Engine) expire(act string, own bool) { + pts, err := e.Points() + if err != nil { + e.logf("expiry: %v", err) + } + newest := map[string]string{} + for _, p := range pts { // newest first + k := p.Module + "\x00" + p.Kept + if _, ok := newest[k]; !ok { + newest[k] = p.Snapshot + } + } + cutoff := e.now().Add(-time.Duration(e.Settings.KeepDays * float64(24*time.Hour))) + removed := int64(0) + for _, p := range pts { + if !p.At.Before(cutoff) || newest[p.Module+"\x00"+p.Kept] == p.Snapshot { + continue + } + if err := os.RemoveAll(filepath.Join(e.Dir, "points", p.Snapshot)); err != nil { + e.logf("the point %s is not removed: %v", p.Snapshot, err) + continue + } + _ = e.Digests.Delete(keys.EscapeName(p.Snapshot)) + if err := e.Seats.HandAct(Module, "", "point-removed "+p.Snapshot, "older than the keep days"); err != nil { + e.logf("the removal of %s is not in the hand-act log: %v", p.Snapshot, err) + } + e.say(seat.EventPointRemoved, seat.PointRemovedEvent{Act: act, At: e.now(), Module: p.Module, Kept: p.Kept, Form: p.Form, + Fingerprint: p.Fingerprint, Snapshot: p.Snapshot}, p.Module, p.Kept) + removed++ + } + ev := seat.ExpiryEndedEvent{Act: act, At: e.now(), Removed: removed} + if own { + e.sayEnd(act, seat.EventExpiryEnded, ev) + return + } + e.say(seat.EventExpiryEnded, ev) +} diff --git a/modules/mesh-recovery/internal/recovery/points_test.go b/modules/mesh-recovery/internal/recovery/points_test.go new file mode 100644 index 000000000..adef20c49 --- /dev/null +++ b/modules/mesh-recovery/internal/recovery/points_test.go @@ -0,0 +1,121 @@ +package recovery_test + +import ( + "strings" + "testing" + "time" + + seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" + + "mesh-recovery/internal/recovery" +) + +func takeAct() recovery.Act { + return recovery.Act{Kind: recovery.KindTake, Module: "app", Kept: "notes", Form: 2, Fingerprint: "fp1", Why: "form change", Caller: "node.engine"} +} + +// A point is taken stream by stream under a temporary name, renamed whole, its digests put on the bus before `ended` is +// said under the act's id with the point whole, and the act is written in the hand-act log with the caller the bus proved +// (design 47 G §516, §554, §560, §567). +func TestATakenPointIsListedWithItsDigestsAndEndedUnderItsAct(t *testing.T) { + r := newRig(t) + ans := r.do(takeAct()) + if ans.Accepted == "" { + t.Fatalf("answer %+v", ans) + } + ended := r.s.named(seat.EventEnded) + if len(ended) != 1 { + t.Fatalf("events %+v", r.s.events) + } + ev := ended[0].Body.(seat.EndedEvent) + if ev.Act != ans.Accepted || ev.Outcome != "done" || len(ev.Digests) != 2 || ev.Size == 0 || ev.Snapshot != "app/notes/2-fp1" { + t.Errorf("ended %+v", ev) + } + if got := strings.Join(ended[0].Parts, "."); got != "app.notes" { + t.Errorf("parts %q", got) + } + if len(ev.Consumers) != 1 || ev.Consumers[0].Sequence != 7 { + t.Errorf("consumers %+v", ev.Consumers) + } + pts, err := r.e.Points() + if err != nil || len(pts) != 1 || pts[0].Snapshot != ev.Snapshot { + t.Fatalf("points %+v %v", pts, err) + } + if _, ok, _ := r.d.Get("app/notes/2-fp1"); !ok { + t.Errorf("no digests on the bus: %v", r.d.m) + } + if len(r.s.handActs) != 1 || !strings.HasPrefix(r.s.handActs[0], "node.engine||take app notes|form change") { + t.Errorf("hand acts %v", r.s.handActs) + } +} + +// One act at a time and no queue: a second ask while one runs is answered busy with the running act (§554). +func TestASecondAskWhileOneRunsIsAnsweredBusy(t *testing.T) { + r := newRig(t) + gate := make(chan struct{}) + r.e.Dump = nil + r.e.Bus = &gatedBus{fakeBus: r.b, gate: gate} + first := r.e.Accept(takeAct()) + second := r.e.Accept(takeAct()) + close(gate) + r.e.Wait() + if first.Accepted == "" || second.Busy != first.Accepted || second.Accepted != "" || second.Error != "" { + t.Errorf("first %+v second %+v", first, second) + } +} + +// An act whose hand-act log write fails does nothing and answers an error naming the log (§554). +func TestAnActWhoseHandActLogWriteFailsDoesNothing(t *testing.T) { + r := newRig(t) + r.s.handFail = errTest + ans := r.do(takeAct()) + if !strings.Contains(ans.Error, "hand-act log") || ans.Accepted != "" || len(r.s.events) != 0 { + t.Errorf("answer %+v events %+v", ans, r.s.events) + } + if pts, _ := r.e.Points(); len(pts) != 0 { + t.Errorf("points %+v", pts) + } +} + +// An act that would pass the cap ends failed naming it, before anything is written (§564). +func TestAnActThatWouldPassTheCapEndsFailedBeforeItWritesAnything(t *testing.T) { + r := newRig(t) + r.e.Settings.Cap = 4 + r.do(takeAct()) + ev := r.s.named(seat.EventEnded)[0].Body.(seat.EndedEvent) + if ev.Outcome != "failed" || !strings.HasPrefix(ev.Error, recovery.ErrCap) { + t.Errorf("ended %+v", ev) + } + if pts, _ := r.e.Points(); len(pts) != 0 || len(r.d.m) != 0 { + t.Errorf("points %+v digests %v", pts, r.d.m) + } +} + +// A point older than the keep days is removed and said, the newest of its kept data stays however old, and the run says +// expiry-ended with the count, 0 included (§551, §560). +func TestExpiryRemovesOldPointsButNeverTheNewest(t *testing.T) { + r := newRig(t) + r.t = time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC) + r.do(takeAct()) + a := takeAct() + a.Fingerprint, a.Form = "fp2", 3 + r.t = time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC) + r.do(a) + r.s.events = nil + r.t = time.Date(2026, 11, 12, 0, 0, 0, 0, time.UTC) + r.do(recovery.Act{Kind: recovery.KindExpire, Why: "by hand", Caller: "tools"}) + pts, _ := r.e.Points() + if len(pts) != 1 || pts[0].Fingerprint != "fp2" { + t.Fatalf("points %+v", pts) + } + rem := r.s.named(seat.EventPointRemoved) + if len(rem) != 1 || rem[0].Body.(seat.PointRemovedEvent).Fingerprint != "fp1" { + t.Errorf("removed %+v", rem) + } + if ex := r.s.named(seat.EventExpiryEnded); len(ex) != 1 || ex[0].Body.(seat.ExpiryEndedEvent).Removed != 1 { + t.Errorf("expiry %+v", ex) + } + if _, ok, _ := r.d.Get("app/notes/2-fp1"); ok { + t.Errorf("its digests stay on the bus") + } +} diff --git a/modules/mesh-recovery/internal/recovery/recovery.go b/modules/mesh-recovery/internal/recovery/recovery.go new file mode 100644 index 000000000..aa3568b3a --- /dev/null +++ b/modules/mesh-recovery/internal/recovery/recovery.go @@ -0,0 +1,343 @@ +// Package recovery is the recovery module's engine (design 47 G §554, §516): it takes, sets aside and restores a +// module's kept data on the live bus, one act at a time, each accepted act written in its directory before the caller is +// answered, the result said as an event under the act's id (§560). Every dependence on the bus, the seat list, the +// hand-act log and the backup holder is an interface here, so the one whole test runs the engine against the real bus +// and a unit test against a fake. +package recovery + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sync" + "time" + + seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// Module and Seat are the module's name and the seat it claims. +const ( + Module = "mesh-recovery" + Seat = seat.Seat +) + +// The kinds of act, one per verb that starts one. +const ( + KindTake = "take" + KindRestore = "restore" + KindAside = "aside" + KindDiscardAside = "discard-aside" + KindRestoreAside = "restore-aside" + KindExpire = "expire-points" +) + +// Errors an act ends with: the words the node agent's table reads (§560). +const ( + ErrCap = "cap" + ErrStopped = "stopped" + ErrCanceled = "cancelled" + ErrMissing = "restore point missing" + ErrDamaged = "restore point damaged" +) + +// Object is one bus object of a kept data: a stream, by its name, with what the bus states of it. +type Object struct { + Stream string + Exists bool + Bytes int64 + Consumers []seat.Consumer +} + +// Bus is the bus as the module's account sees it (§554): a stream's snapshot and restore, its deletion, the +// durable consumers' positions. +type Bus interface { + // Objects states each stream of names: whether it exists, its bytes, its durable consumers' acknowledged sequences. + Objects(ctx context.Context, streams []string) ([]Object, error) + // Take writes one stream's snapshot file to w (ADR 0235 §10's pacing and §11's bounds are the implementation's). + Take(ctx context.Context, stream string, w io.Writer) error + // Delete removes a stream; one that is not there is no error. + Delete(ctx context.Context, stream string) error + // Restore puts the stream of a snapshot file back under its own name, where no stream of that name exists. + Restore(ctx context.Context, file string) error + // SetConsumers creates each durable consumer at its recorded sequence. + SetConsumers(ctx context.Context, cs []seat.Consumer) error +} + +// Seats is what the engine asks of the rest of the mesh. +type Seats interface { + // Say publishes one event of the seat under its subject parts (§560); an event of no parts has none. + Say(event string, body any, parts ...string) error + // HandAct writes the act in the hand-act log first (§554): the account the bus proved, who asked, what, why. + HandAct(caller, asker, what, why string) error + // Streams are the bus names of a module's kept data at a fingerprint, from its catalog graph entry (§554 Bus names). + Streams(ctx context.Context, module, kept, fingerprint string) ([]string, error) + // RunningFingerprint is the fingerprint the node-engine's placement runs for the module (A-123). + RunningFingerprint(ctx context.Context, module string) (string, error) + // Backup brings the points directory of the module's item back beside its path through the backup holder and + // answers where (§516); an error when no repository snapshot holds it. + Backup(ctx context.Context, module string) (dir string, err error) +} + +// Digests keeps each point's digests on the bus, apart from its bytes (§567): the state digests. +type Digests interface { + Put(key string, v []byte) error + Get(key string) ([]byte, bool, error) + Delete(key string) error +} + +// Dump is what the act waits on while the bus's item is dumped (§563). +type Dump interface { + // Wait returns once no dump of the bus's item runs; it calls waiting(on) once if it has to wait. + Wait(ctx context.Context, waiting func(on string)) error +} + +// Bounds is the module's state bounds (design 47 G A-77, A-143): a wait's bound is a key with its own time to live, its +// expiry heard on the key's watch as the bus's own change, so no timer runs in the process. +type Bounds interface { + // Bound writes key with the time to live ttl and calls expired once when the bus removes it; end removes it first, and + // expired is then never called. + Bound(key string, ttl time.Duration, expired func()) (end func(), err error) +} + +// The bound keys of an act's waits (A-143): `.`. +const ( + BoundDumpWait = "dump-wait" + BoundRestoreWait = "restore-wait" +) + +// Settings are the bounds a whole test sets in seconds (§557, §564, §633). +type Settings struct { + Cap int64 + KeepDays float64 + // FillingMark is the percent of the cap past which points-filling is written (points-filling-mark, 80 unless set). + FillingMark int + // DumpWait is how long an act waits on the dump before recovery-waits is written (dump-wait-minutes, 60 unless set). + DumpWait time.Duration + // RestoreWait is how long a restore in place is waited on, and once more after its cancel (snapshot-restore-wait-minutes, + // 90 unless set; §681, A-138, A-140). + RestoreWait time.Duration +} + +// Engine is the recovery module's one engine. +type Engine struct { + Dir string + Bus Bus + Seats Seats + Digests Digests + Dump Dump + // Keeper is the bus module's keeper, whose snapshots the hand verb restore takes by request (§681). + Keeper Keeper + // Bounds times the dump wait and the snapshot-restored wait (A-143). + Bounds Bounds + Settings Settings + // Findings is the module's finding writer (§561); Node the machine it runs on, the id of points-filling. + Findings Findings + Node string + Now func() time.Time + Logf func(string, ...any) + + mu sync.Mutex + running *Act + cancel context.CancelFunc + wg sync.WaitGroup + seq int + why string // the hand act words of a cancel (A-117) + stop string // why the process stops, once it does (§562, A-141) +} + +// Act is one accepted act, written in the directory before its caller is answered (§554). +type Act struct { + ID string `json:"act"` + Kind string `json:"kind"` + Module string `json:"module,omitempty"` + Kept string `json:"kept,omitempty"` + Form int64 `json:"form,omitempty"` + Fingerprint string `json:"fingerprint,omitempty"` + Snapshot string `json:"snapshot,omitempty"` + // Request is a keeper's snapshot by its request, the hand verb restore's other source (§681, §682). + Request string `json:"request,omitempty"` + AsideID string `json:"aside-id,omitempty"` + FromBackup bool `json:"from-backup,omitempty"` + CatchUp *seat.CatchUp `json:"catch-up,omitempty"` + Why string `json:"why"` + Caller string `json:"caller"` + Asker string `json:"asker,omitempty"` + Since time.Time `json:"since"` + Step string `json:"step,omitempty"` + WaitsOn string `json:"waits-on,omitempty"` + Ended bool `json:"ended,omitempty"` + Error string `json:"error,omitempty"` +} + +// Answer is the one form every acting verb answers (§691): exactly one field set. +type Answer struct { + Accepted string `json:"accepted,omitempty"` + Busy string `json:"busy,omitempty"` + Error string `json:"error,omitempty"` +} + +func (e *Engine) now() time.Time { + if e.Now != nil { + return e.Now().UTC() + } + return time.Now().UTC() +} + +func (e *Engine) logf(f string, a ...any) { + if e.Logf != nil { + e.Logf(f, a...) + } +} + +// Tokens are escaped as a key's name is (§73); a token that is empty, `.` or `..` is refused. +func token(s string) (string, error) { + if s == "" || s == "." || s == ".." { + return "", fmt.Errorf("%q is not a name", s) + } + return keys.EscapeName(s), nil +} + +func (e *Engine) pointDir(module, kept string, form int64, fp string) (string, error) { + m, err := token(module) + if err != nil { + return "", err + } + k, err := token(kept) + if err != nil { + return "", err + } + f, err := token(fp) + if err != nil { + return "", err + } + return filepath.Join(e.Dir, "points", m, k, fmt.Sprintf("%d-%s", form, f)), nil +} + +func writeJSON(path string, v any) error { + raw, err := json.MarshalIndent(v, "", " ") + if err != nil { + return err + } + tmp := path + ".partial" + if err := os.WriteFile(tmp, raw, 0o600); err != nil { + return err + } + return os.Rename(tmp, path) +} + +func readJSON(path string, v any) error { + raw, err := os.ReadFile(path) + if err != nil { + return err + } + return json.Unmarshal(raw, v) +} + +func dirSize(dir string) int64 { + var n int64 + _ = filepath.WalkDir(dir, func(_ string, d os.DirEntry, err error) error { + if err == nil && !d.IsDir() { + if info, err := d.Info(); err == nil { + n += info.Size() + } + } + return nil + }) + return n +} + +var errBusy = errors.New("busy") + +// Accept is the one door of every acting verb (§554, §682): it writes the act in the hand-act log first (a failed +// write does nothing and answers an error naming the log), then answers busy with the running act's id, or writes the +// act in its directory and answers accepted; the work runs on, and its result is the event. +func (e *Engine) Accept(a Act) Answer { + e.mu.Lock() + defer e.mu.Unlock() + if e.stop != "" { + return Answer{Error: e.stop} + } + if e.running != nil { + return Answer{Busy: e.running.ID} + } + if err := e.Seats.HandAct(a.Caller, a.Asker, a.Kind+" "+a.Module+" "+a.Kept, a.Why); err != nil { + return Answer{Error: "the hand-act log was not written, so nothing was done: " + err.Error()} + } + e.seq++ + a.ID = fmt.Sprintf("act-%s-%06d", e.now().Format("20060102T150405Z"), e.seq) + a.Since = e.now() + if err := os.MkdirAll(filepath.Join(e.Dir, "acts"), 0o700); err != nil { + return Answer{Error: err.Error()} + } + if err := writeJSON(filepath.Join(e.Dir, "acts", a.ID+".json"), a); err != nil { + return Answer{Error: "the act cannot be written: " + err.Error()} + } + ctx, cancel := context.WithCancel(context.Background()) + e.running, e.cancel = &a, cancel + e.wg.Add(1) + go func() { + defer e.wg.Done() + e.run(ctx, &a) + e.mu.Lock() + if e.running == &a { // an act that said its ending event is no longer the running one (sayEnd) + e.running, e.cancel = nil, nil + } + e.mu.Unlock() + cancel() + e.judge() + }() + return Answer{Accepted: a.ID} +} + +// Wait returns once the running act, if any, has ended (a whole test's and a stop's join). +func (e *Engine) Wait() { e.wg.Wait() } + +// Stop is the process's stop, a move's or a restart's (§562, A-141): from now on every call is answered an error naming the +// stop, the running act's context is cancelled so it ends through its own failure path (a restore past its removal puts +// the aside back and sets the consumers back), and once it has ended every accepted act not ended is ended failed with +// `stopped` and its ending event, so its caller asks again. +func (e *Engine) Stop(why string) { + e.mu.Lock() + e.stop = "the recovery module is stopping (" + why + "), so it takes no call; ask again once it runs" + if e.cancel != nil { + e.cancel() + } + e.mu.Unlock() + e.wg.Wait() + e.endUnended() +} + +// Stopping is the error a call is answered while the process stops, empty while it runs. +func (e *Engine) Stopping() string { + e.mu.Lock() + defer e.mu.Unlock() + return e.stop +} + +// stoppedOr is an act's failure as its ending event says it: `stopped` once the process stops (§562). +func (e *Engine) stoppedOr(err error) error { + if err != nil && e.Stopping() != "" { + return errors.New(ErrStopped) + } + return err +} + +func (e *Engine) setStep(a *Act, step, waitsOn string) { + e.mu.Lock() + a.Step, a.WaitsOn = step, waitsOn + e.mu.Unlock() +} + +func (e *Engine) markEnded(a *Act) { + a.Ended = true + _ = writeJSON(filepath.Join(e.Dir, "acts", a.ID+".json"), a) +} + +func jsonMarshal(v any) ([]byte, error) { return json.Marshal(v) } + +func unmarshal(raw []byte, v any) error { return json.Unmarshal(raw, v) } diff --git a/modules/mesh-recovery/internal/recovery/restore.go b/modules/mesh-recovery/internal/recovery/restore.go new file mode 100644 index 000000000..73f067e45 --- /dev/null +++ b/modules/mesh-recovery/internal/recovery/restore.go @@ -0,0 +1,370 @@ +package recovery + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +const asideFile = "aside.json" + +// The steps an aside's record says, in their order (§516): a restore that fails after `removed` puts the aside back. +const ( + stepSetAside = "set aside" + stepRemoved = "removed" + stepRestored = "restored" +) + +// asideRecord is the record in an aside's directory (§516); nothing of an aside is on the bus. +type asideRecord struct { + ID string `json:"id"` + Module string `json:"module"` + Kept string `json:"kept"` + Streams []string `json:"streams"` + Consumers []seat.Consumer `json:"consumers,omitempty"` + Digests []seat.Digest `json:"digests"` + Size int64 `json:"size"` + Step string `json:"step"` +} + +func (r asideRecord) aside() seat.Aside { + return seat.Aside{ID: r.ID, Names: r.Streams, Size: r.Size} +} + +func (e *Engine) asideRoot(module, kept string) (string, error) { + m, err := token(module) + if err != nil { + return "", err + } + k, err := token(kept) + if err != nil { + return "", err + } + return filepath.Join(e.Dir, "aside", m, k), nil +} + +// endedRestored says the end of a restore, or of a restore-aside, under the act's id (§560). +func (e *Engine) endedRestored(a *Act, outcomeErr error, aside *seat.Aside, missing bool) error { + ev := seat.RestoredEvent{Act: a.ID, At: e.now(), Module: a.Module, Kept: a.Kept, Form: a.Form, Fingerprint: a.Fingerprint, + Snapshot: a.Snapshot, Outcome: seat.RestoredEventOutcomeDone, Aside: aside, Missing: missing} + outcomeErr = e.stoppedOr(outcomeErr) + if outcomeErr != nil { + ev.Outcome, ev.Error, ev.Why = seat.RestoredEventOutcomeFailed, outcomeErr.Error(), e.cancelledWhy(outcomeErr) + } + e.sayEnd(a.ID, seat.EventRestored, ev, a.Module, a.Kept) + return outcomeErr +} + +func (e *Engine) cancelledWhy(err error) string { + e.mu.Lock() + defer e.mu.Unlock() + if err != nil && err.Error() == ErrCanceled { + return e.why + } + return "" +} + +// restore is `restore-kept` and the hand verb `restore` with a point (or, past the dump, a keeper's snapshot by request) (§516, §554, §563, §567): it says restore-started, +// waits for the dump of the bus's item, finds the point (or has the backup holder bring it back), checks its digests +// against the bus, sets the live data aside whole and checked, removes it, restores the point under its names and +// sets each consumer to its recorded sequence; a step after the removal that fails puts the aside back. +func (e *Engine) restore(ctx context.Context, a *Act) error { + // Said means acknowledged: the dump's item events are read only once the bus took restore-started (§563). + if err := e.Seats.Say(seat.EventRestoreStarted, seat.RestoreStartedEvent{Act: a.ID, At: e.now(), Module: a.Module, Kept: a.Kept, Form: a.Form, + Fingerprint: a.Fingerprint, Snapshot: a.Snapshot}, a.Module, a.Kept); err != nil { + return e.endedRestored(a, fmt.Errorf("restore-started was not acknowledged by the bus, so nothing was read or set aside: %w", err), nil, false) + } + if e.Dump != nil { + waiting, done := e.waitsOn(a) + err := e.Dump.Wait(ctx, waiting) + done() + if err != nil { + if ctx.Err() != nil { + err = errors.New(ErrCanceled) + } + return e.endedRestored(a, err, nil, false) + } + } + if a.Request != "" { + return e.restoreRequest(ctx, a) + } + e.setStep(a, "finding the point", "") + pointPath := filepath.Join(e.Dir, "points", filepath.FromSlash(a.Snapshot)) + if _, err := os.Stat(filepath.Join(pointPath, pointFile)); err != nil { + if !a.FromBackup { + return e.endedRestored(a, errors.New(ErrMissing), nil, true) + } + dir, err := e.Seats.Backup(ctx, Module) + if err != nil { + return e.endedRestored(a, errors.New(ErrMissing), nil, true) + } + defer os.RemoveAll(dir) + pointPath = filepath.Join(dir, filepath.FromSlash(a.Snapshot)) + if _, err := os.Stat(filepath.Join(pointPath, pointFile)); err != nil { + return e.endedRestored(a, errors.New(ErrMissing), nil, true) + } + } + var p seat.Point + if err := readJSON(filepath.Join(pointPath, pointFile), &p); err != nil { + return e.endedRestored(a, fmt.Errorf("%s: %w", ErrDamaged, err), nil, false) + } + if err := e.checkDigests(pointPath, a.Snapshot, p); err != nil { + return e.endedRestored(a, err, nil, false) + } + names := make([]string, 0, len(p.Digests)) + for _, d := range p.Digests { + names = append(names, d.Name) + } + rec, err := e.setAside(ctx, a, names, true) + if err != nil { + return e.endedRestored(a, err, nil, false) + } + // Past the removal, a failure puts the aside back (§516). + e.setStep(a, "restoring", "") + if err := e.putBack(ctx, pointPath, p.Digests, p.Consumers); err != nil { + if back := e.restoreFromAside(ctx, rec, true); back != nil { + err = fmt.Errorf("%w; and the aside is not put back: %v", err, back) + } + return e.endedRestored(a, err, asidePtr(rec), false) + } + rec.Step = stepRestored + _ = writeJSON(filepath.Join(e.asideDirOf(rec), asideFile), rec) + return e.endedRestored(a, nil, asidePtr(rec), false) +} + +func asidePtr(r *asideRecord) *seat.Aside { + if r == nil { + return nil + } + x := r.aside() + return &x +} + +func (e *Engine) asideDirOf(r *asideRecord) string { + root, _ := e.asideRoot(r.Module, r.Kept) + return filepath.Join(root, r.ID) +} + +// checkDigests computes the digest of the bytes about to be restored and compares it with the key on the bus; the +// record in the directory is never the reference (§567). +func (e *Engine) checkDigests(pointPath, snapshot string, p seat.Point) error { + raw, ok, err := e.Digests.Get(keys.EscapeName(snapshot)) + if err != nil { + return fmt.Errorf("the digests of %s cannot be read: %w", snapshot, err) + } + if !ok { + return fmt.Errorf("%s: %s has no digests on the bus", ErrDamaged, snapshot) + } + var want []seat.Digest + if err := unmarshal(raw, &want); err != nil { + return fmt.Errorf("%s: %s: %w", ErrDamaged, snapshot, err) + } + if len(want) != len(p.Digests) { + return fmt.Errorf("%s: %s holds %d object(s), the bus says %d", ErrDamaged, snapshot, len(p.Digests), len(want)) + } + for _, w := range want { + got, _, err := digestOf(filepath.Join(pointPath, keys.EscapeName(w.Name)+".snap")) + if err != nil || got != w.Digest { + return fmt.Errorf("%s: %s, bus object %s", ErrDamaged, snapshot, w.Name) + } + } + return nil +} + +// setAside takes a snapshot of each live object into aside///, reads it back to check it, and only then +// (when remove) removes the live objects, each step written first in the record (§516). Past the cap it refuses before +// any object is removed (§564). +func (e *Engine) setAside(ctx context.Context, a *Act, names []string, remove bool) (*asideRecord, error) { + e.setStep(a, "setting aside", "") + objs, err := e.Bus.Objects(ctx, names) + if err != nil { + return nil, err + } + var need int64 + for _, o := range objs { + need += o.Bytes + } + if fill := e.filled(); fill+need > e.Settings.Cap { + return nil, fmt.Errorf("%s: the cap is %d bytes, the aside needs %d and the directory holds %d", ErrCap, e.Settings.Cap, need, fill) + } + root, err := e.asideRoot(a.Module, a.Kept) + if err != nil { + return nil, err + } + id := e.now().Format("20060102T150405Z") + for n := 2; ; n++ { + if _, err := os.Stat(filepath.Join(root, id)); err != nil { + break + } + id = fmt.Sprintf("%s-%d", e.now().Format("20060102T150405Z"), n) + } + dir := filepath.Join(root, id) + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, err + } + rec := &asideRecord{ID: id, Module: a.Module, Kept: a.Kept, Step: stepSetAside} + fail := func(err error) (*asideRecord, error) { _ = os.RemoveAll(dir); return nil, err } + for _, o := range objs { + if !o.Exists { + continue + } + sum, n, err := e.takeOne(ctx, o.Stream, filepath.Join(dir, keys.EscapeName(o.Stream)+".snap")) + if err != nil { + return fail(err) + } + if again, _, err := digestOf(filepath.Join(dir, keys.EscapeName(o.Stream)+".snap")); err != nil || again != sum { + return fail(fmt.Errorf("the aside of %s does not read back whole", o.Stream)) + } + rec.Streams = append(rec.Streams, o.Stream) + rec.Digests = append(rec.Digests, seat.Digest{Name: o.Stream, Digest: sum}) + rec.Consumers = append(rec.Consumers, o.Consumers...) + rec.Size += n + } + if err := writeJSON(filepath.Join(dir, asideFile), rec); err != nil { + return fail(err) + } + if !remove { + return rec, nil + } + rec.Step = stepRemoved + if err := writeJSON(filepath.Join(dir, asideFile), rec); err != nil { + return fail(err) + } + for _, s := range rec.Streams { + if err := e.Bus.Delete(ctx, s); err != nil { + if back := e.restoreFromAside(ctx, rec, false); back != nil { + return rec, fmt.Errorf("%w; and the aside is not put back: %v", err, back) + } + return rec, err + } + } + return rec, nil +} + +// putBack restores each stream of a point under its own name and sets each consumer to its recorded sequence. +func (e *Engine) putBack(ctx context.Context, dir string, ds []seat.Digest, cs []seat.Consumer) error { + for _, d := range ds { + if err := e.Bus.Restore(ctx, filepath.Join(dir, keys.EscapeName(d.Name)+".snap")); err != nil { + return fmt.Errorf("stream %s: %w", d.Name, err) + } + } + return e.Bus.SetConsumers(ctx, cs) +} + +// restoreFromAside puts an aside back under the live names: any partly restored stream is removed first. It is a failure +// path, so it runs whole also when the act's context ended, as at the process's stop (§562). +func (e *Engine) restoreFromAside(ctx context.Context, rec *asideRecord, clear bool) error { + ctx = context.WithoutCancel(ctx) + if clear { + for _, s := range rec.Streams { + if err := e.Bus.Delete(ctx, s); err != nil { + return err + } + } + } + if err := e.putBack(ctx, e.asideDirOf(rec), rec.Digests, rec.Consumers); err != nil { + return err + } + rec.Step = stepRestored + return writeJSON(filepath.Join(e.asideDirOf(rec), asideFile), rec) +} + +func (e *Engine) readAside(module, kept, id string) (*asideRecord, error) { + root, err := e.asideRoot(module, kept) + if err != nil { + return nil, err + } + if id == "" || id == "." || id == ".." || strings.ContainsAny(id, "/") { + return nil, fmt.Errorf("%q is not an aside", id) + } + var rec asideRecord + if err := readJSON(filepath.Join(root, id, asideFile), &rec); err != nil { + return nil, fmt.Errorf("no aside %s of %s's %s", id, module, kept) + } + return &rec, nil +} + +// aside is the hand verb `aside` (§516, §554, A-123): it sets the live kept data aside without restoring it, taking the bus +// names from the catalog entry of the fingerprint the node-engine's placement runs for the module, and removes the live +// objects once the aside is whole and checked (a move, not a removal: the data stays in the aside until it is discarded or +// put back by `restore-aside`); it ends with `set-aside`. +func (e *Engine) aside(ctx context.Context, a *Act) error { + fp, err := e.Seats.RunningFingerprint(ctx, a.Module) + if err != nil { + return err + } + a.Fingerprint = fp + names, err := e.Seats.Streams(ctx, a.Module, a.Kept, fp) + if err != nil { + return err + } + rec, err := e.setAside(ctx, a, names, true) + if err != nil { + return err + } + x := rec.aside() + e.sayEnd(a.ID, seat.EventSetAside, seat.SetAsideEvent{Act: a.ID, At: e.now(), ID: x.ID, Names: x.Names, Size: x.Size}, a.Module, a.Kept) + return nil +} + +// restoreAside is `restore-aside` (§566): it puts an aside whose record ends at `removed` back under the live names it +// was taken from, refusing while a live object of those names exists, and ends with `restored`. +func (e *Engine) restoreAside(ctx context.Context, a *Act) error { + rec, err := e.readAside(a.Module, a.Kept, a.AsideID) + if err != nil { + return e.endedRestored(a, err, nil, false) + } + objs, err := e.Bus.Objects(ctx, rec.Streams) + if err != nil { + return e.endedRestored(a, err, nil, false) + } + for _, o := range objs { + if o.Exists { + return e.endedRestored(a, fmt.Errorf("the live stream %s exists; the aside %s is not put back over it", o.Stream, rec.ID), asidePtr(rec), false) + } + } + if err := e.restoreFromAside(ctx, rec, false); err != nil { + return e.endedRestored(a, err, asidePtr(rec), false) + } + return e.endedRestored(a, nil, asidePtr(rec), false) +} + +// discardAside is `discard-aside` (§554): it removes that directory and nothing else, an id holding `.`, `..` or `/` +// refused; it ends with `aside-discarded`. +func (e *Engine) discardAside(a *Act) error { + rec, err := e.readAside(a.Module, a.Kept, a.AsideID) + if err != nil { + return err + } + if err := os.RemoveAll(e.asideDirOf(rec)); err != nil { + return err + } + x := rec.aside() + e.sayEnd(a.ID, seat.EventAsideDiscarded, seat.AsideDiscardedEvent{Act: a.ID, At: e.now(), ID: x.ID, Names: x.Names, Size: x.Size}, a.Module, a.Kept) + return nil +} + +// Asides lists every aside in the directory (§517). +func (e *Engine) Asides() ([]seat.Aside, []asideRecord, error) { + var out []seat.Aside + var recs []asideRecord + _ = filepath.WalkDir(filepath.Join(e.Dir, "aside"), func(path string, d os.DirEntry, err error) error { + if err == nil && !d.IsDir() && d.Name() == asideFile { + var r asideRecord + if readJSON(path, &r) == nil { + out = append(out, r.aside()) + recs = append(recs, r) + } + } + return nil + }) + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out, recs, nil +} diff --git a/modules/mesh-recovery/internal/recovery/restore_test.go b/modules/mesh-recovery/internal/recovery/restore_test.go new file mode 100644 index 000000000..b63c44fe7 --- /dev/null +++ b/modules/mesh-recovery/internal/recovery/restore_test.go @@ -0,0 +1,349 @@ +package recovery_test + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" + + "mesh-recovery/internal/recovery" +) + +func restoreAct(snapshot string) recovery.Act { + return recovery.Act{Kind: recovery.KindRestore, Module: "app", Kept: "notes", Form: 2, Fingerprint: "fp1", Snapshot: snapshot, Why: "revert", Caller: "node.engine"} +} + +func takenRig(t *testing.T) *rig { + r := newRig(t) + r.do(takeAct()) + r.s.events = nil + return r +} + +func lastRestored(t *testing.T, r *rig) seat.RestoredEvent { + t.Helper() + ev := r.s.named(seat.EventRestored) + if len(ev) != 1 { + t.Fatalf("restored events %+v", r.s.events) + } + return ev[0].Body.(seat.RestoredEvent) +} + +// A restore says restore-started, sets the live data aside whole, removes it, restores the point under its names and sets +// each consumer to its recorded sequence, and says restored with the aside (design 47 G §516, §560). +func TestARestoreSetsTheLiveDataAsideAndRestoresThePoint(t *testing.T) { + r := takenRig(t) + r.b.streams["app_notes"] = "changed since" + ans := r.do(restoreAct("app/notes/2-fp1")) + ev := lastRestored(t, r) + if ev.Outcome != "done" || ev.Act != ans.Accepted || ev.Aside == nil || ev.Aside.Size == 0 { + t.Fatalf("restored %+v", ev) + } + if r.b.streams["app_notes"] != "hello" || r.b.streams["KV_app_seen"] != "world" { + t.Errorf("streams %v", r.b.streams) + } + if len(r.b.setBack) != 1 || r.b.setBack[0].Sequence != 7 { + t.Errorf("consumers %+v", r.b.setBack) + } + if len(r.s.named(seat.EventRestoreStarted)) != 1 { + t.Errorf("no restore-started") + } + kp, _ := r.e.KeptPoints("app", "notes") + if len(kp.Asides) != 1 || len(kp.Points) != 1 || kp.Running != nil { + t.Errorf("kept points %+v", kp) + } +} + +// A point missing from the directory says missing; with from-backup the backup holder brings it back and it is restored +// once its digest is checked; when no repository snapshot holds it, restore point missing (§516, §567). +func TestAMissingPointIsBroughtBackOrSaidMissing(t *testing.T) { + r := takenRig(t) + back := filepath.Join(t.TempDir(), "back") + if err := os.Rename(filepath.Join(r.e.Dir, "points"), back); err != nil { + t.Fatal(err) + } + r.do(restoreAct("app/notes/2-fp1")) + if ev := lastRestored(t, r); !ev.Missing || ev.Outcome != "failed" || ev.Error != recovery.ErrMissing { + t.Errorf("no backup asked: %+v", ev) + } + r.s.events = nil + a := restoreAct("app/notes/2-fp1") + a.FromBackup = true + r.do(a) + if ev := lastRestored(t, r); !ev.Missing || ev.Error != recovery.ErrMissing { + t.Errorf("backup holder holds none: %+v", ev) + } + r.s.events, r.s.backup = nil, back + r.do(a) + if ev := lastRestored(t, r); ev.Outcome != "done" { + t.Errorf("brought back: %+v", ev) + } +} + +// A point whose bytes differ from the digests on the bus is refused before any aside is taken, the record in the +// directory never being the reference (§567). +func TestAPointWhoseBytesWereAlteredIsRefusedBeforeAnyAside(t *testing.T) { + r := takenRig(t) + snap := filepath.Join(r.e.Dir, "points", "app", "notes", "2-fp1", "app_notes.snap") + if err := os.WriteFile(snap, []byte("snap:app_notes:evil"), 0o600); err != nil { + t.Fatal(err) + } + r.do(restoreAct("app/notes/2-fp1")) + if ev := lastRestored(t, r); ev.Outcome != "failed" || !strings.HasPrefix(ev.Error, recovery.ErrDamaged) { + t.Errorf("restored %+v", ev) + } + if kp, _ := r.e.KeptPoints("", ""); len(kp.Asides) != 0 || r.b.streams["app_notes"] != "hello" { + t.Errorf("an aside was taken or the live data changed: %+v %v", kp, r.b.streams) + } +} + +// A step after the removal that fails puts the aside back under the live names and the act ends failed, so the kept data +// is never left gone (§516). +func TestARestoreThatFailsAfterTheRemovalPutsTheAsideBack(t *testing.T) { + r := takenRig(t) + r.b.streams["app_notes"] = "live now" + r.b.failRestores = 1 + r.do(restoreAct("app/notes/2-fp1")) + if ev := lastRestored(t, r); ev.Outcome != "failed" || ev.Aside == nil { + t.Fatalf("restored %+v", ev) + } + if r.b.streams["app_notes"] != "live now" || r.b.streams["KV_app_seen"] != "world" { + t.Errorf("live data not as before: %v", r.b.streams) + } + if n := len(r.e.Unrestored()); n != 0 { + t.Errorf("%d asides unrestored", n) + } +} + +// A restore past the cap is refused with the live data untouched (§564). +func TestARestoreWhoseAsideWouldPassTheCapIsRefusedBeforeTheRemoval(t *testing.T) { + r := takenRig(t) + r.e.Settings.Cap = 1 + r.do(restoreAct("app/notes/2-fp1")) + if ev := lastRestored(t, r); ev.Outcome != "failed" || !strings.HasPrefix(ev.Error, recovery.ErrCap) { + t.Errorf("restored %+v", ev) + } + if r.b.streams["app_notes"] != "hello" { + t.Errorf("live data touched: %v", r.b.streams) + } +} + +// An act waiting on the dump is cancelled with its ending event failed/cancelled and the hand act's words as why; an act +// that has begun is refused, naming its step (§566, A-117). +func TestAWaitingActIsCancelledAndABegunOneRefused(t *testing.T) { + r := takenRig(t) + hd := holdDump{waiting: make(chan string, 1)} + r.e.Dump = hd + // Past the bound (a setting, §557) the waiting act writes recovery-waits, cleared once it ends (§561). + fs := newHeldFindings() + r.e.Findings, r.e.Settings.DumpWait = fs, time.Millisecond + ans := r.e.Accept(restoreAct("app/notes/2-fp1")) + <-hd.waiting + if f := <-fs.written; f.Kind != recovery.FindingRecoveryWaits || f.Scope != "app" || f.ID != ans.Accepted { + t.Errorf("written %+v", f) + } + kp, _ := r.e.KeptPoints("", "") + if kp.Running == nil || kp.Running.WaitsOn == "" || kp.Running.Act != ans.Accepted { + t.Errorf("running %+v", kp.Running) + } + if c := r.e.Cancel("act-nothing", "x", "tools", "op"); c.Error == "" { + t.Errorf("an unknown act cancelled: %+v", c) + } + if c := r.e.Cancel(ans.Accepted, "not now", "tools", "op"); c.Accepted != ans.Accepted { + t.Errorf("cancel %+v", c) + } + r.e.Wait() + ev := lastRestored(t, r) + if ev.Outcome != "failed" || ev.Error != recovery.ErrCanceled || ev.Why != "not now" || ev.Act != ans.Accepted { + t.Errorf("restored %+v", ev) + } + if fs.has(recovery.FindingRecoveryWaits) { + t.Errorf("recovery-waits stands after the act ended: %+v", fs.Held()) + } + // The wait's bound is a key of the state bounds, its expiry what wrote recovery-waits (A-143). + if w, rm := r.bounds.keys(); w != "dump-wait."+ans.Accepted+" 1ms" || rm != "dump-wait."+ans.Accepted+" expired" { + t.Errorf("bound keys written %q, removed %q", w, rm) + } +} + +// retry asks again the last failed act with its fields under an act of its own, and answers `nothing to retry` when its +// last act did not fail (§682). +func TestRetryAsksTheLastFailedActAgain(t *testing.T) { + r := takenRig(t) + r.e.Settings.Cap = 1 + r.do(restoreAct("app/notes/2-fp1")) + r.e.Settings.Cap = 1 << 20 + r.s.events = nil + ans := r.e.Retry("app", "notes", "again", "tools", "op") + r.e.Wait() + if ev := lastRestored(t, r); ans.Accepted == "" || ev.Outcome != "done" || ev.Act != ans.Accepted { + t.Errorf("answer %+v restored %+v", ans, ev) + } + if again := r.e.Retry("app", "notes", "again", "tools", "op"); again.Error != "nothing to retry" { + t.Errorf("answer %+v", again) + } +} + +// An aside whose record ends at removed is put back at the start, an act cut off by a restart ends failed with stopped, +// and a point left under a temporary name is removed (§516, §560, §564). +func TestAStartPutsBackAnAsideEndsACutOffActAndClearsTemporaryPoints(t *testing.T) { + r := takenRig(t) + r.do(restoreAct("app/notes/2-fp1")) // leaves an aside at "restored" + _, recs, _ := r.e.Asides() + if len(recs) != 1 { + t.Fatal("no aside") + } + // Cut a restore off after its removal: the record says removed, the live data is gone, an accepted act never ended. + dir := filepath.Join(r.e.Dir, "aside", "app", "notes", recs[0].ID) + raw, _ := os.ReadFile(filepath.Join(dir, "aside.json")) + _ = os.WriteFile(filepath.Join(dir, "aside.json"), []byte(strings.Replace(string(raw), `"step": "restored"`, `"step": "removed"`, 1)), 0o600) + delete(r.b.streams, "app_notes") + delete(r.b.streams, "KV_app_seen") + tmp := filepath.Join(r.e.Dir, "points", "app", "notes", ".tmp-act-x") + _ = os.MkdirAll(tmp, 0o700) + _ = os.WriteFile(filepath.Join(r.e.Dir, "acts", "act-cut.json"), []byte(`{"act":"act-cut","kind":"restore","module":"app","kept":"notes","why":"w","caller":"c","since":"2026-11-12T00:00:00Z"}`), 0o600) + r.s.events = nil + if left := r.e.Start(context.Background()); len(left) != 0 { + t.Errorf("unrestored %+v", left) + } + if r.b.streams["app_notes"] != "hello" { + t.Errorf("the aside was not put back: %v", r.b.streams) + } + if _, err := os.Stat(tmp); err == nil { + t.Errorf("the temporary point stays") + } + if ev := lastRestored(t, r); ev.Act != "act-cut" || ev.Error != recovery.ErrStopped { + t.Errorf("restored %+v", ev) + } +} + +// restore-aside puts an aside back under the live names, refused while live objects of those names exist; discard-aside +// removes the aside's directory and says aside-discarded (§554, §566). +func TestRestoreAsideRefusesLiveNamesAndDiscardAsideRemovesTheDirectory(t *testing.T) { + r := takenRig(t) + r.do(restoreAct("app/notes/2-fp1")) + _, recs, _ := r.e.Asides() + id := recs[0].ID + r.s.events = nil + r.do(recovery.Act{Kind: recovery.KindRestoreAside, Module: "app", Kept: "notes", AsideID: id, Why: "x", Caller: "tools"}) + if ev := lastRestored(t, r); ev.Outcome != "failed" || !strings.Contains(ev.Error, "live stream") { + t.Errorf("restored %+v", ev) + } + r.s.events = nil + r.do(recovery.Act{Kind: recovery.KindDiscardAside, Module: "app", Kept: "notes", AsideID: "../x", Why: "x", Caller: "tools"}) + if len(r.s.named(seat.EventAsideDiscarded)) != 0 { + t.Errorf("a path outside was discarded") + } + r.do(recovery.Act{Kind: recovery.KindDiscardAside, Module: "app", Kept: "notes", AsideID: id, Why: "x", Caller: "tools"}) + if ev := r.s.named(seat.EventAsideDiscarded); len(ev) != 1 || ev[0].Body.(seat.AsideDiscardedEvent).ID != id { + t.Errorf("events %+v", r.s.events) + } + if kp, _ := r.e.KeptPoints("", ""); len(kp.Asides) != 0 { + t.Errorf("asides %+v", kp.Asides) + } +} + +// The hand verb aside sets the live data aside whole and removes it, naming the streams as a list, and restore-aside puts +// it back once no live name exists (§516, §554, §566, A-123, A-124). +func TestAsideMovesTheLiveDataIntoAnAsideAndRestoreAsidePutsItBack(t *testing.T) { + r := newRig(t) + // The findings of §561: aside-unrestored while the aside alone holds the data, points-filling past the fill mark. + fs := newHeldFindings() + r.e.Findings, r.e.Node, r.e.Settings.FillingMark, r.e.Settings.Cap = fs, "srv", 1, 4096 + ans := r.do(recovery.Act{Kind: recovery.KindAside, Module: "app", Kept: "notes", Why: "look at it", Caller: "tools", Asker: "op"}) + ev := r.s.named(seat.EventSetAside) + if ans.Accepted == "" || len(ev) != 1 { + t.Fatalf("answer %+v events %+v", ans, r.s.events) + } + sa := ev[0].Body.(seat.SetAsideEvent) + if sa.Act != ans.Accepted || len(sa.Names) != 2 || sa.Size == 0 || len(r.b.streams) != 0 { + t.Errorf("set-aside %+v streams %v", sa, r.b.streams) + } + if got := r.e.Unrestored(); len(got) != 1 { + t.Errorf("unrestored %+v", got) + } + // kept-points says it, which the tool and the health check read (A-129). + if kp, _ := r.e.KeptPoints("app", ""); len(kp.Unrestored) != 1 || kp.Unrestored[0].ID != sa.ID { + t.Errorf("kept-points unrestored %+v", kp.Unrestored) + } + if !fs.has(recovery.FindingAsideUnrestored) || !fs.has(recovery.FindingPointsFilling) { + t.Errorf("findings %+v", fs.Held()) + } + r.s.events = nil + r.do(recovery.Act{Kind: recovery.KindRestoreAside, Module: "app", Kept: "notes", AsideID: sa.ID, Why: "done looking", Caller: "tools"}) + if lastRestored(t, r).Outcome != "done" || r.b.streams["app_notes"] != "hello" || len(r.e.Unrestored()) != 0 { + t.Errorf("streams %v", r.b.streams) + } + r.e.Settings.FillingMark = 100 + r.do(recovery.Act{Kind: recovery.KindDiscardAside, Module: "app", Kept: "notes", AsideID: sa.ID, Why: "not needed", Caller: "tools"}) + if fs.has(recovery.FindingAsideUnrestored) || fs.has(recovery.FindingPointsFilling) { + t.Errorf("findings stand after the aside was restored and discarded: %+v", fs.Held()) + } +} + +// Said means acknowledged: a restore whose restore-started the bus did not take reads nothing and sets nothing aside, and +// ends failed (§563). +func TestARestoreWhoseStartIsNotAcknowledgedTouchesNothing(t *testing.T) { + r := takenRig(t) + r.s.sayFail = map[string]error{seat.EventRestoreStarted: errTest} + r.do(restoreAct("app/notes/2-fp1")) + if ev := lastRestored(t, r); ev.Outcome != "failed" || !strings.Contains(ev.Error, "restore-started") || r.b.streams["app_notes"] != "hello" { + t.Errorf("restored %+v, streams %v", ev, r.b.streams) + } + if kp, _ := r.e.KeptPoints("", ""); len(kp.Asides) != 0 { + t.Errorf("an aside was taken: %+v", kp.Asides) + } +} + +// stallBus blocks its first Restore until the act's context ends, as a restore under way when the process stops. +type stallBus struct { + *fakeBus + in chan struct{} + stalled bool +} + +func (s *stallBus) Restore(ctx context.Context, file string) error { + if !s.stalled { + s.stalled = true + close(s.in) + <-ctx.Done() + return ctx.Err() + } + return s.fakeBus.Restore(ctx, file) +} + +// A stop of the process (a move's or a restart's) answers every new call an error naming it, ends the running restore +// through its failure path, putting the aside back, and ends every accepted act with `stopped`, so its caller asks again +// (design 47 G §562, A-141). +func TestAStopEndsTheRunningActThroughItsFailurePathAndStopped(t *testing.T) { + r := takenRig(t) + r.b.streams["app_notes"] = "changed since" + stall := &stallBus{fakeBus: r.b, in: make(chan struct{})} + r.e.Bus = stall + ans := r.e.Accept(restoreAct("app/notes/2-fp1")) + <-stall.in + // An accepted act a crash left unended is ended by the stop too. + _ = os.WriteFile(filepath.Join(r.e.Dir, "acts", "act-0.json"), []byte(`{"act":"act-0","kind":"take","module":"app","kept":"seen","why":"w","caller":"c","since":"2026-11-11T00:00:00Z"}`), 0o600) + r.e.Stop("a move") + if r.b.streams["app_notes"] != "changed since" || r.b.streams["KV_app_seen"] != "world" { + t.Errorf("the aside is not put back: %v", r.b.streams) + } + if ev := lastRestored(t, r); ev.Act != ans.Accepted || ev.Outcome != "failed" || ev.Error != recovery.ErrStopped { + t.Errorf("restored %+v", ev) + } + ended := r.s.named(seat.EventEnded) + if len(ended) != 1 || ended[0].Body.(seat.EndedEvent).Act != "act-0" || ended[0].Body.(seat.EndedEvent).Error != recovery.ErrStopped { + t.Errorf("ended %+v", ended) + } + if got := r.e.Accept(takeAct()); !strings.Contains(got.Error, "stopping (a move)") { + t.Errorf("a call during the stop answered %+v", got) + } + verb := r.e.Handlers()[seat.VerbKeptPoints] + if got, _ := verb("c", []byte(`{}`)); !strings.Contains(got.(recovery.Answer).Error, "stopping") { + t.Errorf("kept-points during the stop answered %+v", got) + } +} diff --git a/modules/mesh-recovery/internal/recovery/verbs.go b/modules/mesh-recovery/internal/recovery/verbs.go new file mode 100644 index 000000000..bd053a9ca --- /dev/null +++ b/modules/mesh-recovery/internal/recovery/verbs.go @@ -0,0 +1,121 @@ +package recovery + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "strings" + + seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" + "git.novox.be/novox/mesh-sdk/go/process" +) + +// Handlers are the seat's verbs as the process answers them over the bus with the caller the bus proved (§554, §656, A-107): +// every acting verb answers one of accepted, busy or error and nothing more (§691); the result is the event. +func (e *Engine) Handlers() map[string]process.SeatHandler { + h := map[string]process.SeatHandler{} + h[seat.VerbSnapshotKept] = actOf(e, func(sk seat.SnapshotKeptInput) (Act, error) { + return Act{Kind: KindTake, Module: sk.Module, Kept: sk.Kept, Form: sk.Form, Fingerprint: sk.Fingerprint, CatchUp: sk.CatchUp, Why: sk.Why}, nil + }) + restoreKept := actOf(e, func(rk seat.RestoreKeptInput) (Act, error) { + return Act{Kind: KindRestore, Module: rk.Module, Kept: rk.Kept, Form: rk.Form, Fingerprint: rk.Fingerprint, Snapshot: rk.Snapshot, FromBackup: rk.FromBackup, Why: rk.Why}, nil + }) + h[seat.VerbRestoreKept] = func(caller string, body json.RawMessage) (any, error) { + // aside-only and discard-aside are the operator's hand tools alone (§516, §554). + var loose map[string]json.RawMessage + _ = json.Unmarshal(body, &loose) + for _, k := range []string{"aside-only", "discard-aside"} { + if _, ok := loose[k]; ok { + return Answer{Error: k + " is the operator's hand tool alone, not an argument of restore-kept"}, nil + } + } + return restoreKept(caller, body) + } + h[seat.VerbKeptPoints] = func(_ string, body json.RawMessage) (any, error) { + var in seat.KeptPointsInput + if err := strict(body, &in); err != nil { + return nil, err + } + return e.KeptPoints(in.Module, in.Kept) + } + h[seat.VerbTake] = actOf(e, func(tk seat.TakeInput) (Act, error) { + return Act{Kind: KindTake, Module: tk.Module, Kept: tk.Kept, Fingerprint: tk.Fingerprint, Why: tk.Why, Asker: tk.Asker}, nil + }) + h[seat.VerbRestore] = actOf(e, func(rs seat.RestoreInput) (Act, error) { + if (rs.Snapshot == "") == (rs.Request == "") { + return Act{}, errors.New("restore takes exactly one of snapshot and request") + } + if rs.Request != "" && rs.FromBackup { + return Act{}, errors.New("from-backup is taken only with snapshot, a restore point (design 47 G §682)") + } + return Act{Kind: KindRestore, Module: rs.Module, Kept: rs.Kept, Snapshot: rs.Snapshot, Request: rs.Request, FromBackup: rs.FromBackup, Why: rs.Why, Asker: rs.Asker}, nil + }) + h[seat.VerbAside] = actOf(e, func(ad seat.AsideInput) (Act, error) { + return Act{Kind: KindAside, Module: ad.Module, Kept: ad.Kept, Why: ad.Why, Asker: ad.Asker}, nil + }) + h[seat.VerbDiscardAside] = actOf(e, func(da seat.DiscardAsideInput) (Act, error) { + return Act{Kind: KindDiscardAside, Module: da.Module, Kept: da.Kept, AsideID: da.ID, Why: da.Why, Asker: da.Asker}, nil + }) + h[seat.VerbRestoreAside] = actOf(e, func(ra seat.RestoreAsideInput) (Act, error) { + return Act{Kind: KindRestoreAside, Module: ra.Module, Kept: ra.Kept, AsideID: ra.ID, Why: ra.Why, Asker: ra.Asker}, nil + }) + h[seat.VerbExpirePoints] = actOf(e, func(ex seat.ExpirePointsInput) (Act, error) { + return Act{Kind: KindExpire, Why: ex.Why, Asker: ex.Asker}, nil + }) + h[seat.VerbRetry] = func(caller string, body json.RawMessage) (any, error) { + var in seat.RetryInput + if err := strict(body, &in); err != nil { + return Answer{Error: err.Error()}, nil + } + return e.Retry(in.Module, in.Kept, in.Why, caller, in.Asker), nil + } + h[seat.VerbCancel] = func(caller string, body json.RawMessage) (any, error) { + var in seat.CancelInput + if err := strict(body, &in); err != nil { + return Answer{Error: err.Error()}, nil + } + return e.Cancel(in.Act, in.Why, caller, in.Asker), nil + } + // A stopping process takes no call, each answered the error naming the stop (§562, A-141). + for verb, handle := range h { + h[verb] = func(caller string, body json.RawMessage) (any, error) { + if stop := e.Stopping(); stop != "" { + return Answer{Error: stop}, nil + } + return handle(caller, body) + } + } + return h +} + +// actOf is one acting verb: its arguments are read afresh for each call (calls come on concurrent bus callbacks), a call +// without the `why` the hand-act log keeps is refused, and the act is accepted as the engine says. +func actOf[T any](e *Engine, build func(T) (Act, error)) process.SeatHandler { + return func(caller string, body json.RawMessage) (any, error) { + var in T + if err := strict(body, &in); err != nil { + return Answer{Error: err.Error()}, nil + } + a, err := build(in) + if err != nil { + return Answer{Error: err.Error()}, nil + } + if a.Why == "" { + return Answer{Error: "why is required: the hand-act log keeps it"}, nil + } + a.Caller = caller + return e.Accept(a), nil + } +} + +// strict reads a call's arguments, refusing a field the verb does not take (no `local-name`, no `from`: §516) and a call +// that leaves out the `why` the hand-act log keeps (§682). +func strict(body json.RawMessage, into any) error { + dec := json.NewDecoder(bytes.NewReader(body)) + dec.DisallowUnknownFields() + if err := dec.Decode(into); err != nil { + return fmt.Errorf("the arguments are refused: %s", strings.TrimPrefix(err.Error(), "json: ")) + } + return nil +} diff --git a/modules/mesh-recovery/internal/recovery/verbs_test.go b/modules/mesh-recovery/internal/recovery/verbs_test.go new file mode 100644 index 000000000..9b2ca8860 --- /dev/null +++ b/modules/mesh-recovery/internal/recovery/verbs_test.go @@ -0,0 +1,63 @@ +package recovery_test + +import ( + "encoding/json" + "strings" + "testing" + + seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" + + "mesh-recovery/internal/recovery" +) + +func call(t *testing.T, r *rig, verb, body string) recovery.Answer { + t.Helper() + out, err := r.e.Handlers()[verb]("node.engine", json.RawMessage(body)) + if err != nil { + t.Fatalf("%s: %v", verb, err) + } + r.e.Wait() + return out.(recovery.Answer) +} + +// Every verb of the seat's definition is answered; an acting verb answers one of accepted, busy and error and nothing more +// (§691); an argument the verb does not take (`local-name`, `from`), `aside-only` and `discard-aside` on restore-kept, a +// restore naming both a snapshot and a request, and a call without `why` are refused naming the fault (§516, §682, §554). +func TestEveryVerbIsAnsweredAndAnActingVerbAnswersOneForm(t *testing.T) { + r := newRig(t) + h := r.e.Handlers() + for _, v := range seat.Verbs() { + if h[v] == nil { + t.Errorf("verb %s has no handler", v) + } + } + taken := call(t, r, "snapshot-kept", `{"module":"app","kept":"notes","form":2,"fingerprint":"fp1","why":"form change"}`) + if taken.Accepted == "" || taken.Busy != "" || taken.Error != "" { + t.Fatalf("answer %+v", taken) + } + for name, c := range map[string]struct{ verb, body, want string }{ + "local-name": {"snapshot-kept", `{"module":"app","kept":"notes","form":2,"fingerprint":"fp1","why":"w","local-name":"x"}`, "local-name"}, + "from": {"restore-kept", `{"module":"app","kept":"notes","form":2,"fingerprint":"fp1","why":"w","from":"x"}`, "from"}, + "aside-only": {"restore-kept", `{"module":"app","kept":"notes","form":2,"fingerprint":"fp1","why":"w","aside-only":true}`, "aside-only"}, + "discard": {"restore-kept", `{"module":"app","kept":"notes","form":2,"fingerprint":"fp1","why":"w","discard-aside":true}`, "discard-aside"}, + "both": {"restore", `{"module":"app","kept":"notes","snapshot":"a","request":"b","why":"w"}`, "exactly one"}, + "no why": {"restore", `{"module":"app","kept":"notes","snapshot":"a"}`, "why"}, + } { + if got := call(t, r, c.verb, c.body); !strings.Contains(got.Error, c.want) || got.Accepted != "" { + t.Errorf("%s: answer %+v", name, got) + } + } + if len(r.s.named(seat.EventEnded)) != 1 { + t.Errorf("a refused call acted: %+v", r.s.events) + } + out, err := h["kept-points"]("tools", json.RawMessage(`{"module":"app"}`)) + if kp, ok := out.(seat.KeptPointsAnswer); err != nil || !ok || len(kp.Points) != 1 { + t.Errorf("kept-points %+v %v", out, err) + } + if got := call(t, r, "take", `{"module":"app","kept":"notes","fingerprint":"fp9","why":"by hand","asker":"op@shanks"}`); got.Accepted == "" { + t.Errorf("take %+v", got) + } + if !strings.Contains(r.s.handActs[len(r.s.handActs)-1], "node.engine|op@shanks|take app notes|by hand") { + t.Errorf("hand acts %v: the account the bus proved and the asker are both logged", r.s.handActs) + } +} diff --git a/modules/mesh-recovery/module.json b/modules/mesh-recovery/module.json new file mode 100644 index 000000000..bbfe35474 --- /dev/null +++ b/modules/mesh-recovery/module.json @@ -0,0 +1,313 @@ +{ + "module": "mesh-recovery", + "version": "0.1", + "runs-as": "mesh-recovery", + "claims": [ + { + "name": "mesh-recovery", + "scope": "mesh", + "serves": [ + "snapshot-kept", + "restore-kept", + "kept-points", + "restore-aside", + "cancel", + "take", + "restore", + "retry", + "aside", + "discard-aside", + "expire-points" + ] + } + ], + "tools": [ + "recovery_kept_points", + "recovery_take", + "recovery_restore", + "recovery_retry", + "recovery_aside", + "recovery_discard_aside", + "recovery_restore_aside", + "recovery_cancel", + "recovery_expire_points" + ], + "consumes": [ + "node-backup.item-started", + "node-backup.item-ended", + "node-engine.running", + "mesh-broker.snapshot-restored", + "mesh-conditions.findings-asked" + ], + "invokes": [ + "seat:mesh-catalog.entry", + "seat:mesh-module-assigner.assignments", + "seat:mesh-hand-acts.hand-act", + "seat:mesh-broker.snapshot-get", + "seat:mesh-broker.snapshot-restore", + "seat:mesh-broker.snapshot-restore-cancel", + "seat:node-backup.restore", + "seat:mesh-recovery.kept-points", + "seat:mesh-recovery.restore-aside", + "seat:mesh-recovery.cancel", + "seat:mesh-recovery.take", + "seat:mesh-recovery.restore", + "seat:mesh-recovery.retry", + "seat:mesh-recovery.aside", + "seat:mesh-recovery.discard-aside", + "seat:mesh-recovery.expire-points" + ], + "requires": [ + { + "name": "digests", + "storage": "key-value", + "template": "state" + }, + { + "name": "findings", + "storage": "key-value", + "template": "state" + }, + { + "name": "bounds", + "storage": "key-value", + "template": "state", + "options": { + "key-ttl": true + } + }, + { + "name": "finding-events", + "storage": "stream", + "template": "events", + "options": { + "events": [ + "finding-written.*", + "finding-cleared.*", + "findings-stated.*" + ], + "max-age-days": 7, + "max-mib": 16 + } + } + ], + "config": { + "points-cap": { + "type": "number", + "description": "bytes the points directory may hold; an act that would pass it ends failed, naming it, before it removes anything (design 47 G section 564).", + "minimum": 1, + "trusted": false, + "default": 10737418240 + }, + "points-filling-mark": { + "type": "number", + "description": "percent of the cap past which the finding points-filling is written (design 47 G section 564).", + "minimum": 1, + "maximum": 100, + "trusted": false, + "default": 80 + }, + "point-keep-days": { + "type": "number", + "description": "days after which a restore point is removed, the newest of its kept data never (design 47 G sections 551, 557).", + "minimum": 0, + "maximum": 3650, + "trusted": false, + "default": 90 + }, + "dump-wait-minutes": { + "type": "number", + "description": "minutes an act waits on the dump of the bus's item before recovery-waits is written (design 47 G sections 561, 557).", + "minimum": 0, + "maximum": 10080, + "trusted": false, + "default": 60 + }, + "snapshot-restore-wait-minutes": { + "type": "number", + "description": "minutes a restore in place of a protected keeper's snapshot is waited on, and once more after its cancel (design 47 G section 681, A-138, A-140).", + "minimum": 0, + "maximum": 10080, + "trusted": false, + "default": 90 + }, + "expire-every-hours": { + "type": "number", + "description": "hours between two removals of the points past their keep days, beside the one at the start and at each snapshot-kept; timed by a bound key of the state bounds (design 47 G sections 551, 557, A-138).", + "minimum": 0, + "maximum": 8760, + "trusted": false, + "default": 24 + } + }, + "data": { + "own": [ + { + "id": "points", + "path": "${dir:points}", + "class": "valuable", + "why": "the restore points taken of modules' kept data before a form change, and the asides of live data set aside before a restore (design 47 G sections 516, 554)" + } + ] + }, + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-recovery", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-recovery", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-recovery" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-recovery", + "place": ".", + "unknown": "program" + }, + { + "id": "points", + "type": "directory", + "path": "/var/lib/mesh-recovery/points", + "mode": "0700", + "owner": "mesh-recovery", + "unknown": "program" + }, + { + "id": "cap", + "type": "file", + "path": "${dir:state}/points-cap", + "owner": "mesh-recovery", + "mode": "0600", + "template": "files/points-cap" + }, + { + "id": "mark", + "type": "file", + "path": "${dir:state}/points-filling-mark", + "owner": "mesh-recovery", + "mode": "0600", + "template": "files/points-filling-mark" + }, + { + "id": "keep-days", + "type": "file", + "path": "${dir:state}/point-keep-days", + "owner": "mesh-recovery", + "mode": "0600", + "template": "files/point-keep-days" + }, + { + "id": "dump-wait", + "type": "file", + "path": "${dir:state}/dump-wait-minutes", + "owner": "mesh-recovery", + "mode": "0600", + "template": "files/dump-wait-minutes" + }, + { + "id": "restore-wait", + "type": "file", + "path": "${dir:state}/snapshot-restore-wait-minutes", + "owner": "mesh-recovery", + "mode": "0600", + "template": "files/snapshot-restore-wait-minutes" + }, + { + "id": "expire-every", + "type": "file", + "path": "${dir:state}/expire-every-hours", + "owner": "mesh-recovery", + "mode": "0600", + "template": "files/expire-every-hours" + }, + { + "id": "recovery", + "type": "process", + "name": "mesh-recovery", + "artifact": "program", + "run": [ + "./mesh-recovery" + ], + "env": { + "MESH_BROKER_FILE": "${dir:state}/broker", + "MESH_RECOVERY_POINTS": "${dir:points}", + "MESH_RECOVERY_POINTS_CAP_FILE": "${dir:state}/points-cap", + "MESH_RECOVERY_FILLING_MARK_FILE": "${dir:state}/points-filling-mark", + "MESH_RECOVERY_KEEP_DAYS_FILE": "${dir:state}/point-keep-days", + "MESH_RECOVERY_DUMP_WAIT_FILE": "${dir:state}/dump-wait-minutes", + "MESH_RECOVERY_RESTORE_WAIT_FILE": "${dir:state}/snapshot-restore-wait-minutes", + "MESH_RECOVERY_EXPIRE_EVERY_FILE": "${dir:state}/expire-every-hours" + }, + "user": "mesh-recovery", + "serves": [ + "seat:mesh-recovery.snapshot-kept", + "seat:mesh-recovery.restore-kept", + "seat:mesh-recovery.kept-points", + "seat:mesh-recovery.restore-aside", + "seat:mesh-recovery.cancel", + "seat:mesh-recovery.take", + "seat:mesh-recovery.restore", + "seat:mesh-recovery.retry", + "seat:mesh-recovery.aside", + "seat:mesh-recovery.discard-aside", + "seat:mesh-recovery.expire-points" + ], + "bus": { + "invokes": [ + "seat:mesh-catalog.entry", + "seat:mesh-module-assigner.assignments", + "seat:mesh-hand-acts.hand-act", + "seat:mesh-broker.snapshot-get", + "seat:mesh-broker.snapshot-restore", + "seat:mesh-broker.snapshot-restore-cancel", + "seat:node-backup.restore" + ] + }, + "health": { + "kind": "unit" + } + } + ], + "runtime-health": { + "kind": "tool", + "tool": "recovery_kept_points", + "interval": "60s", + "timeout": "10s", + "looks": 2, + "grace": "180s" + }, + "build": { + "artifacts": [ + { + "name": "program", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-recovery", + "binary": "mesh-recovery" + }, + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-recovery-tools", + "binary": "mesh-recovery-tools", + "loads": [ + "mesh-recovery-tools" + ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + } +} diff --git a/modules/nats/cmd/mesh-nats-snapshot/bussnap.go b/modules/nats/cmd/mesh-nats-snapshot/bussnap.go index e4d8a8f59..0c205ed83 100644 --- a/modules/nats/cmd/mesh-nats-snapshot/bussnap.go +++ b/modules/nats/cmd/mesh-nats-snapshot/bussnap.go @@ -1,6 +1,6 @@ package main -import "nats-tools/internal/bussnap" +import "git.novox.be/novox/mesh-sdk/go/bussnap" // The snapshot itself is bussnap's, which the keeper takes one entry's with too (design 25 § 12.3). type ( diff --git a/modules/nats/cmd/mesh-nats-snapshot/image_test.go b/modules/nats/cmd/mesh-nats-snapshot/image_test.go index d53fcee36..ddf3b86ca 100644 --- a/modules/nats/cmd/mesh-nats-snapshot/image_test.go +++ b/modules/nats/cmd/mesh-nats-snapshot/image_test.go @@ -19,8 +19,8 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-sdk/go/busarchive" "github.com/nats-io/nats.go" - "nats-tools/internal/busarchive" ) // manifestOf is the nats module's own definition, beside this program. diff --git a/modules/nats/cmd/mesh-nats-snapshot/live_test.go b/modules/nats/cmd/mesh-nats-snapshot/live_test.go index cd64c0fc7..cbea74248 100644 --- a/modules/nats/cmd/mesh-nats-snapshot/live_test.go +++ b/modules/nats/cmd/mesh-nats-snapshot/live_test.go @@ -34,8 +34,8 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-sdk/go/busarchive" "github.com/nats-io/nats.go" - "nats-tools/internal/busarchive" ) // image is the bus's release (the manifest's server artifact names the same release in its tag). diff --git a/modules/nats/cmd/mesh-nats-snapshot/main.go b/modules/nats/cmd/mesh-nats-snapshot/main.go index 983b33393..4ff65226b 100644 --- a/modules/nats/cmd/mesh-nats-snapshot/main.go +++ b/modules/nats/cmd/mesh-nats-snapshot/main.go @@ -30,10 +30,11 @@ import ( "flag" "fmt" "io" - "nats-tools/internal/busarchive" - "nats-tools/internal/bussnap" "os" "time" + + "git.novox.be/novox/mesh-sdk/go/busarchive" + "git.novox.be/novox/mesh-sdk/go/bussnap" ) func main() { diff --git a/modules/nats/cmd/mesh-nats-snapshot/restore.go b/modules/nats/cmd/mesh-nats-snapshot/restore.go index 5d5866094..f56c80d05 100644 --- a/modules/nats/cmd/mesh-nats-snapshot/restore.go +++ b/modules/nats/cmd/mesh-nats-snapshot/restore.go @@ -16,8 +16,8 @@ import ( "syscall" "time" + "git.novox.be/novox/mesh-sdk/go/busarchive" "github.com/nats-io/nats.go" - "nats-tools/internal/busarchive" ) // restoreChunk is what is sent at a time; the server answers each before the next goes. diff --git a/modules/nats/cmd/nats-tools/keep.go b/modules/nats/cmd/nats-tools/keep.go index 37c18e506..2547bf5b1 100644 --- a/modules/nats/cmd/nats-tools/keep.go +++ b/modules/nats/cmd/nats-tools/keep.go @@ -29,7 +29,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/seatdef" "git.novox.be/novox/mesh-sdk/go/views" - "nats-tools/internal/bussnap" + "git.novox.be/novox/mesh-sdk/go/bussnap" "nats-tools/internal/keeper" ) diff --git a/modules/nats/cmd/nats-tools/keeper_integration_test.go b/modules/nats/cmd/nats-tools/keeper_integration_test.go index bb06a2a05..63e38779f 100644 --- a/modules/nats/cmd/nats-tools/keeper_integration_test.go +++ b/modules/nats/cmd/nats-tools/keeper_integration_test.go @@ -28,7 +28,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/views" - "nats-tools/internal/bussnap" + "git.novox.be/novox/mesh-sdk/go/bussnap" "nats-tools/internal/keeper" ) diff --git a/modules/nats/cmd/nats-tools/main.go b/modules/nats/cmd/nats-tools/main.go index 535eeb75b..b20eefcba 100644 --- a/modules/nats/cmd/nats-tools/main.go +++ b/modules/nats/cmd/nats-tools/main.go @@ -11,7 +11,7 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" - "nats-tools/internal/busarchive" + "git.novox.be/novox/mesh-sdk/go/busarchive" "nats-tools/internal/keeper" ) diff --git a/modules/nats/cmd/nats-tools/snapshots.go b/modules/nats/cmd/nats-tools/snapshots.go index 01f2e03f8..a1d455588 100644 --- a/modules/nats/cmd/nats-tools/snapshots.go +++ b/modules/nats/cmd/nats-tools/snapshots.go @@ -19,6 +19,7 @@ import ( "git.novox.be/novox/mesh-contracts/go/seats/meshbroker" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscalls" "git.novox.be/novox/mesh-sdk/go/process" "nats-tools/internal/keeper" @@ -30,34 +31,32 @@ import ( // snapshot-restore, snapshot-restore-cancel and snapshot-remove are acted on by the keeper at once, with no hand-over through the // state requests. The tools bundle keeps nats_snapshots, which lists them. -// ChunkSize is at most what one chunk carries: 512 KiB. -const ChunkSize = 512 << 10 - -// AckWithin is how long the next chunk waits for its acknowledgement. -const AckWithin = 30 * time.Second +// The form of the chunks is mesh-sdk's (buscalls.SeatChunks, A-139): the caller receives with the same one. +const ( + // ChunkSize is at most what one chunk carries: 512 KiB. + ChunkSize = buscalls.SeatChunkBytes + // AckWithin is how long the next chunk waits for its acknowledgement. + AckWithin = buscalls.SeatAckWithin +) // callToken is the shape of the subject token a caller chooses for a send. var callToken = regexp.MustCompile(`^[A-Za-z0-9_-]{16,64}$`) -// ChunkSubject and AckSubject are the subjects of the nth chunk of a send and of its acknowledgement. -func ChunkSubject(call string, n int64) string { - return "mesh.seat." + meshbroker.Seat + ".chunk." + meshbroker.VerbSnapshotGet + "." + call + "." + strconv.FormatInt(n, 10) +func sendOf(call string) buscalls.SeatChunks { + return buscalls.SeatChunks{Seat: meshbroker.Seat, Verb: meshbroker.VerbSnapshotGet, Call: call} } +// ChunkSubject and AckSubject are the subjects of the nth chunk of a send and of its acknowledgement. +func ChunkSubject(call string, n int64) string { return sendOf(call).Chunk(n) } + // AckSubject is the subject the caller acknowledges the nth chunk on. -func AckSubject(call string, n int64) string { - return "mesh.seat." + meshbroker.Seat + ".ack." + meshbroker.VerbSnapshotGet + "." + call + "." + strconv.FormatInt(n, 10) -} +func AckSubject(call string, n int64) string { return sendOf(call).Ack(n) } // AckPattern is every acknowledgement of snapshot-get. -const AckPattern = "mesh.seat.mesh-broker.ack.snapshot-get.*.*" +var AckPattern = buscalls.SeatAcks(meshbroker.Seat, meshbroker.VerbSnapshotGet) // Chunk is the body of one chunk: its number, how many there are, and its bytes. -type Chunk struct { - N int64 `json:"n"` - Of int64 `json:"of"` - Data []byte `json:"data"` -} +type Chunk = buscalls.SeatChunk // snapshotWorld is what the keeper's snapshot verbs reach: the file, the process's own publish and subscription, and the bus's users. type snapshotWorld struct { diff --git a/modules/nats/internal/busarchive/archive.go b/modules/nats/internal/busarchive/archive.go deleted file mode 100644 index 0d14f545a..000000000 --- a/modules/nats/internal/busarchive/archive.go +++ /dev/null @@ -1,162 +0,0 @@ -package busarchive - -import ( - "archive/tar" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "io" - "os" - "path/filepath" -) - -// Archive is a snapshot read back: its manifest, and each file beside it unpacked into a scratch -// directory of its own, removed by Close. -type Archive struct { - Manifest Manifest - dir string - files map[string]string -} - -// ReadArchive unpacks a snapshot. Only the files the manifest names are kept, and only under names -// that are one stream's directory: an archive is data, and a path in it is never trusted to say where -// on the machine a file goes. -func ReadArchive(r io.Reader, scratch string) (*Archive, error) { - dir, err := os.MkdirTemp(scratch, "mesh-nats-restore-*") - if err != nil { - return nil, err - } - a := &Archive{dir: dir, files: map[string]string{}} - tr := tar.NewReader(r) - first := true - for { - hdr, err := tr.Next() - if err == io.EOF { - break - } - if err != nil { - a.Close() - return nil, fmt.Errorf("the archive does not read: %w", err) - } - if first { - if hdr.Name != ManifestName { - a.Close() - return nil, fmt.Errorf("the archive starts with %q, not its manifest: it is not a snapshot of the bus", hdr.Name) - } - raw, err := io.ReadAll(io.LimitReader(tr, 64<<20)) - if err != nil { - a.Close() - return nil, err - } - if err := json.Unmarshal(raw, &a.Manifest); err != nil { - a.Close() - return nil, fmt.Errorf("its manifest: %w", err) - } - if a.Manifest.Format != 1 { - a.Close() - return nil, fmt.Errorf("its manifest is format %d; this program reads format 1", a.Manifest.Format) - } - first = false - continue - } - if !a.named(hdr.Name) { - a.Close() - return nil, fmt.Errorf("the archive holds %q, which its manifest does not name", hdr.Name) - } - local := filepath.Join(dir, fmt.Sprintf("%04d", len(a.files))) - f, err := os.OpenFile(local, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) - if err != nil { - a.Close() - return nil, err - } - _, err = io.Copy(f, tr) - f.Close() - if err != nil { - a.Close() - return nil, err - } - a.files[hdr.Name] = local - } - if first { - a.Close() - return nil, errors.New("the archive is empty") - } - for _, s := range a.Manifest.Streams { - if !ValidName(s.Name) || s.Meta != "streams/"+s.Name+"/backup.json" || s.Archive != "streams/"+s.Name+"/stream.tar.s2" { - a.Close() - return nil, fmt.Errorf("the manifest names a stream %q at paths that are not its own", s.Name) - } - } - return a, nil -} - -func (a *Archive) named(name string) bool { - for _, s := range a.Manifest.Streams { - if name == s.Meta || name == s.Archive { - return true - } - } - return false -} - -// Close removes the unpacked files. -func (a *Archive) Close() { os.RemoveAll(a.dir) } - -// Verify holds every file to the manifest's size and checksum, and reads every stream's archive to -// its end. -func (a *Archive) Verify() error { - for _, s := range a.Manifest.Streams { - for _, f := range []struct{ name, sum string }{{s.Meta, s.MetaSHA256}, {s.Archive, s.ArchiveSHA256}} { - local, ok := a.files[f.name] - if !ok { - return fmt.Errorf("%s is named in the manifest and missing from the archive", f.name) - } - got, size, err := sha256Of(local) - if err != nil { - return err - } - if got != f.sum { - return fmt.Errorf("%s does not match the checksum its manifest gives", f.name) - } - if f.name == s.Archive && size != s.ArchiveBytes { - return fmt.Errorf("%s is %d bytes; its manifest says %d", f.name, size, s.ArchiveBytes) - } - } - af, err := os.Open(a.files[s.Archive]) - if err != nil { - return err - } - err = ReadsThrough(af) - af.Close() - if err != nil { - return fmt.Errorf("%s does not read: %w", s.Archive, err) - } - } - return nil -} - -func sha256Of(path string) (string, int64, error) { - f, err := os.Open(path) - if err != nil { - return "", 0, err - } - defer f.Close() - h := sha256.New() - n, err := io.Copy(h, f) - if err != nil { - return "", 0, err - } - return hex.EncodeToString(h.Sum(nil)), n, nil -} - -// Open is one stream's meta and archive. -func (a *Archive) Open(s StreamEntry) (meta []byte, archive *os.File, err error) { - meta, err = os.ReadFile(a.files[s.Meta]) - if err != nil { - return nil, nil, err - } - archive, err = os.Open(a.files[s.Archive]) - return meta, archive, err -} diff --git a/modules/nats/internal/busarchive/types.go b/modules/nats/internal/busarchive/types.go deleted file mode 100644 index 12ca5eba3..000000000 --- a/modules/nats/internal/busarchive/types.go +++ /dev/null @@ -1,86 +0,0 @@ -// Package busarchive is the bus's snapshot archive: its manifest, reading one back and holding it to -// its checksums. The snapshot program (cmd/mesh-nats-snapshot) writes and restores with it and the bus's -// tool nats_snapshot_verify (cmd/nats-tools) checks with it, so the two never differ on what a good -// snapshot is. -package busarchive - -import ( - "archive/tar" - "io" - "strings" - "time" - - "github.com/klauspost/compress/s2" -) - -// Manifest is the first entry of an archive: what was taken, when, how long it took, and the checksum -// of every file beside it. It is what `verify` and `restore` hold the archive to. -type Manifest struct { - Format int `json:"format"` - Server string `json:"server_version"` - Started time.Time `json:"started"` - Finished time.Time `json:"finished"` - Seconds float64 `json:"seconds"` - // Messages and StreamBytes are the streams' own totals at their snapshots; ArchiveBytes what the - // archives take, compressed. - Messages uint64 `json:"messages"` - StreamBytes int64 `json:"stream_bytes"` - ArchiveBytes int64 `json:"archive_bytes"` - Streams []StreamEntry `json:"streams"` - // Skipped are streams that hold nothing across a restart (memory storage), said rather than - // silently left out. - Skipped []Skipped `json:"skipped,omitempty"` -} - -// StreamEntry is one stream as it was snapshotted. -type StreamEntry struct { - Name string `json:"name"` - Subjects []string `json:"subjects,omitempty"` - Retention string `json:"retention,omitempty"` - Messages uint64 `json:"messages"` - Bytes uint64 `json:"bytes"` - FirstSeq uint64 `json:"first_seq"` - LastSeq uint64 `json:"last_seq"` - Deleted int `json:"deleted,omitempty"` - Consumers int `json:"consumers"` - // Meta is the server's own description of the stream (its configuration and state), in the - // shape the restore API and the nats CLI take back: backup.json. - Meta string `json:"meta"` - MetaSHA256 string `json:"meta_sha256"` - // Archive is the server's snapshot of the stream, as the server sent it: stream.tar.s2. - Archive string `json:"archive"` - ArchiveBytes int64 `json:"archive_bytes"` - ArchiveSHA256 string `json:"archive_sha256"` - Seconds float64 `json:"seconds"` -} - -// Skipped is a stream not snapshotted, and why. -type Skipped struct { - Name string `json:"name"` - Why string `json:"why"` -} - -// ManifestName is the archive's first entry. -const ManifestName = "manifest.json" - -// ReadsThrough is whether a stream's archive decompresses and lists to its end. -func ReadsThrough(r io.Reader) error { - tr := tar.NewReader(s2.NewReader(r)) - for { - _, err := tr.Next() - if err == io.EOF { - return nil - } - if err != nil { - return err - } - if _, err := io.Copy(io.Discard, tr); err != nil { - return err - } - } -} - -// ValidName is a stream name that is one safe path component. -func ValidName(name string) bool { - return name != "" && name != "." && name != ".." && !strings.ContainsAny(name, "/\\\x00 *>") -} diff --git a/modules/nats/internal/busarchive/verifyfile.go b/modules/nats/internal/busarchive/verifyfile.go deleted file mode 100644 index fdb4a2fcf..000000000 --- a/modules/nats/internal/busarchive/verifyfile.go +++ /dev/null @@ -1,109 +0,0 @@ -package busarchive - -import ( - "errors" - "fmt" - "io/fs" - "os" - "path/filepath" - "strings" - "time" -) - -// StreamReport is one stream of a verified snapshot. -type StreamReport struct { - Name string `json:"name"` - LastSeq uint64 `json:"last_seq"` -} - -// Report is the answer of VerifyFile: which file was held to its manifest, and whether it passed. -type Report struct { - File string `json:"file"` - Modified time.Time `json:"modified"` - Size int64 `json:"size"` - Taken *time.Time `json:"taken,omitempty"` - Streams []StreamReport `json:"streams"` - Skipped []Skipped `json:"skipped,omitempty"` - Passed bool `json:"passed"` - // Fault is the first thing wrong, empty when the snapshot passed. - Fault string `json:"fault,omitempty"` -} - -// VerifyFile verifies one snapshot inside dir: the file called name, or the newest *.tar when name is -// empty. Nothing outside dir is read and nothing is written there; a snapshot that fails is an answer -// (Passed false, the first Fault), and only a directory without a snapshot is an error. -func VerifyFile(dir, name string) (*Report, error) { - path, info, err := pick(dir, name) - if err != nil { - return nil, err - } - r := &Report{File: filepath.Base(path), Modified: info.ModTime().UTC(), Size: info.Size(), Streams: []StreamReport{}} - f, err := os.Open(path) - if err != nil { - r.Fault = err.Error() - return r, nil - } - defer f.Close() - a, err := ReadArchive(f, "") - if err != nil { - r.Fault = err.Error() - return r, nil - } - defer a.Close() - taken := a.Manifest.Finished - r.Taken = &taken - r.Skipped = a.Manifest.Skipped - for _, s := range a.Manifest.Streams { - r.Streams = append(r.Streams, StreamReport{Name: s.Name, LastSeq: s.LastSeq}) - } - if err := a.Verify(); err != nil { - r.Fault = err.Error() - return r, nil - } - r.Passed = true - return r, nil -} - -// pick names the regular file to verify, and nothing outside dir: a name is one path component, and a -// symbolic link is not followed out of the directory. -func pick(dir, name string) (string, fs.FileInfo, error) { - if dir == "" { - return "", nil, errors.New("the snapshots directory is not set") - } - if name != "" { - if name != filepath.Base(name) || name == "." || name == ".." || strings.ContainsRune(name, 0) { - return "", nil, fmt.Errorf("%q is not a file name inside the snapshots directory", name) - } - p := filepath.Join(dir, name) - info, err := os.Lstat(p) - if err != nil { - return "", nil, fmt.Errorf("no snapshot %s in the snapshots directory: %w", name, err) - } - if !info.Mode().IsRegular() { - return "", nil, fmt.Errorf("%s is not a regular file", name) - } - return p, info, nil - } - entries, err := os.ReadDir(dir) - if err != nil { - return "", nil, fmt.Errorf("the snapshots directory does not read: %w", err) - } - var best string - var bestInfo fs.FileInfo - for _, e := range entries { - if !strings.HasSuffix(e.Name(), ".tar") { - continue - } - info, err := e.Info() - if err != nil || !info.Mode().IsRegular() { - continue - } - if bestInfo == nil || info.ModTime().After(bestInfo.ModTime()) { - best, bestInfo = e.Name(), info - } - } - if bestInfo == nil { - return "", nil, errors.New("the snapshots directory holds no .tar snapshot") - } - return filepath.Join(dir, best), bestInfo, nil -} diff --git a/modules/nats/internal/busarchive/verifyfile_test.go b/modules/nats/internal/busarchive/verifyfile_test.go deleted file mode 100644 index be62737fb..000000000 --- a/modules/nats/internal/busarchive/verifyfile_test.go +++ /dev/null @@ -1,173 +0,0 @@ -package busarchive - -import ( - "archive/tar" - "bytes" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/klauspost/compress/s2" -) - -func sum(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) } - -// streamArchive is a stream's archive as the server sends it: a tar of one file, s2-compressed. -func streamArchive(t *testing.T) []byte { - t.Helper() - var raw bytes.Buffer - tw := tar.NewWriter(&raw) - body := []byte(strings.Repeat("a message of the stream\n", 50)) - if err := tw.WriteHeader(&tar.Header{Name: "msgs.blk", Mode: 0o600, Size: int64(len(body))}); err != nil { - t.Fatal(err) - } - tw.Write(body) - tw.Close() - var out bytes.Buffer - w := s2.NewWriter(&out) - w.Write(raw.Bytes()) - w.Close() - return out.Bytes() -} - -// snapshotTar builds a snapshot of two streams; the mutations change what the manifest says or what is -// stored, so each fault is made in the one place it is found. -func snapshotTar(t *testing.T, mutate func(m *Manifest, files map[string][]byte)) []byte { - t.Helper() - files := map[string][]byte{} - m := Manifest{Format: 1, Server: "2.11.17", Finished: time.Date(2026, 10, 22, 3, 0, 0, 0, time.UTC)} - for i, name := range []string{"EVENTS", "KV_conditions"} { - meta := []byte(`{"config":{"name":"` + name + `"}}`) - arc := streamArchive(t) - e := StreamEntry{Name: name, LastSeq: uint64(10 * (i + 1)), - Meta: "streams/" + name + "/backup.json", MetaSHA256: sum(meta), - Archive: "streams/" + name + "/stream.tar.s2", ArchiveBytes: int64(len(arc)), ArchiveSHA256: sum(arc)} - m.Streams = append(m.Streams, e) - files[e.Meta], files[e.Archive] = meta, arc - } - if mutate != nil { - mutate(&m, files) - } - var out bytes.Buffer - tw := tar.NewWriter(&out) - put := func(name string, b []byte) { - if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(b))}); err != nil { - t.Fatal(err) - } - tw.Write(b) - } - raw, _ := json.Marshal(m) - put(ManifestName, raw) - for _, s := range m.Streams { - put(s.Meta, files[s.Meta]) - put(s.Archive, files[s.Archive]) - } - tw.Close() - return out.Bytes() -} - -func put(t *testing.T, dir, name string, b []byte, at time.Time) { - t.Helper() - p := filepath.Join(dir, name) - if err := os.WriteFile(p, b, 0o600); err != nil { - t.Fatal(err) - } - os.Chtimes(p, at, at) -} - -func TestAGoodSnapshotPassesAndSaysItsStreams(t *testing.T) { - dir := t.TempDir() - good := snapshotTar(t, nil) - put(t, dir, "bus.tar", good, time.Now()) - r, err := VerifyFile(dir, "") - if err != nil { - t.Fatal(err) - } - if !r.Passed || r.Fault != "" || r.File != "bus.tar" || r.Size != int64(len(good)) { - t.Fatalf("%+v", r) - } - if r.Taken == nil || !r.Taken.Equal(time.Date(2026, 10, 22, 3, 0, 0, 0, time.UTC)) { - t.Errorf("taken %v", r.Taken) - } - if len(r.Streams) != 2 || r.Streams[0] != (StreamReport{"EVENTS", 10}) || r.Streams[1] != (StreamReport{"KV_conditions", 20}) { - t.Errorf("streams %+v", r.Streams) - } -} - -func TestACorruptedChecksumFailsNamingTheFile(t *testing.T) { - dir := t.TempDir() - put(t, dir, "bus.tar", snapshotTar(t, func(m *Manifest, files map[string][]byte) { - m.Streams[1].ArchiveSHA256 = sum([]byte("something else")) - }), time.Now()) - r, err := VerifyFile(dir, "bus.tar") - if err != nil { - t.Fatal(err) - } - if r.Passed || !strings.Contains(r.Fault, "streams/KV_conditions/stream.tar.s2") || !strings.Contains(r.Fault, "checksum") { - t.Fatalf("%+v", r) - } - if len(r.Streams) != 2 { - t.Errorf("a failed snapshot still says its streams: %+v", r.Streams) - } -} - -func TestATruncatedSnapshotFails(t *testing.T) { - dir := t.TempDir() - good := snapshotTar(t, nil) - put(t, dir, "bus.tar", good[:len(good)*2/3], time.Now()) - r, err := VerifyFile(dir, "") - if err != nil { - t.Fatal(err) - } - if r.Passed || r.Fault == "" { - t.Fatalf("a truncated snapshot passed: %+v", r) - } -} - -func TestAnArchiveThatDoesNotReadToItsEndFailsEvenWithMatchingChecksums(t *testing.T) { - dir := t.TempDir() - put(t, dir, "bus.tar", snapshotTar(t, func(m *Manifest, files map[string][]byte) { - bad := []byte("not an s2 stream at all") - s := &m.Streams[0] - files[s.Archive] = bad - s.ArchiveSHA256, s.ArchiveBytes = sum(bad), int64(len(bad)) - }), time.Now()) - r, _ := VerifyFile(dir, "") - if r.Passed || !strings.Contains(r.Fault, "does not read") { - t.Fatalf("%+v", r) - } -} - -func TestTheNewestTarIsVerifiedAndOnlyInsideTheDirectory(t *testing.T) { - dir := t.TempDir() - now := time.Now() - put(t, dir, "old.tar", []byte("junk"), now.Add(-48*time.Hour)) - put(t, dir, "bus.tar", snapshotTar(t, nil), now) - put(t, dir, "bus.tar.partial", []byte("half"), now.Add(time.Hour)) - r, err := VerifyFile(dir, "") - if err != nil || r.File != "bus.tar" || !r.Passed { - t.Fatalf("%+v %v", r, err) - } - if r, _ := VerifyFile(dir, "old.tar"); r.Passed || r.File != "old.tar" { - t.Errorf("a named file is the one checked: %+v", r) - } - outside := t.TempDir() - put(t, outside, "x.tar", snapshotTar(t, nil), now) - for _, name := range []string{"../" + filepath.Base(outside) + "/x.tar", outside + "/x.tar", ".."} { - if _, err := VerifyFile(dir, name); err == nil { - t.Errorf("%q was read", name) - } - } - os.Symlink(filepath.Join(outside, "x.tar"), filepath.Join(dir, "link.tar")) - if _, err := VerifyFile(dir, "link.tar"); err == nil { - t.Error("a symbolic link out of the directory was followed") - } - if _, err := VerifyFile(t.TempDir(), ""); err == nil { - t.Error("an empty directory is not a snapshot") - } -} diff --git a/modules/nats/internal/bussnap/restore.go b/modules/nats/internal/bussnap/restore.go deleted file mode 100644 index 030f8d7b1..000000000 --- a/modules/nats/internal/bussnap/restore.go +++ /dev/null @@ -1,158 +0,0 @@ -package bussnap - -import ( - "archive/tar" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "os" - "time" - - "nats-tools/internal/busarchive" - - "github.com/nats-io/nats.go" -) - -// The keeper's restore in place of a snapshot it took (design 47 G §681): each stream of the file put back under its own -// name, only where no stream of that name is live. - -// APIStreamDelete is the subject the keeper removes its own partial stream with. -const APIStreamDelete = "$JS.API.STREAM.DELETE." - -const restoreChunk = 64 * 1024 - -// ManifestOf reads the manifest of a snapshot file: the streams it holds, without unpacking them. -func ManifestOf(path string) (*busarchive.Manifest, error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - defer f.Close() - tr := tar.NewReader(f) - hdr, err := tr.Next() - if err != nil { - return nil, fmt.Errorf("the snapshot does not read: %w", err) - } - if hdr.Name != busarchive.ManifestName { - return nil, fmt.Errorf("the snapshot starts with %q, not its manifest", hdr.Name) - } - raw, err := io.ReadAll(io.LimitReader(tr, 64<<20)) - if err != nil { - return nil, err - } - var m busarchive.Manifest - if err := json.Unmarshal(raw, &m); err != nil { - return nil, fmt.Errorf("its manifest: %w", err) - } - return &m, nil -} - -// Live says whether a stream of that name exists. -func Live(ctx context.Context, nc *nats.Conn, name string) (bool, error) { - err := Request(ctx, nc, APIStreamInfo+name, nil, nil) - var api *APIError - switch { - case err == nil: - return true, nil - case errors.As(err, &api) && api.Code == 404: - return false, nil - } - return false, err -} - -// DeleteStream removes a stream; one that is not there is no error. -func DeleteStream(ctx context.Context, nc *nats.Conn, name string) error { - err := Request(ctx, nc, APIStreamDelete+name, nil, nil) - var api *APIError - if errors.As(err, &api) && api.Code == 404 { - return nil - } - return err -} - -// RestoreFile puts every stream of the snapshot file back under its own name, each sent in chunks the server answers; it -// ends with the context's bound. A stream already there is the caller's to have refused beforehand. -func RestoreFile(ctx context.Context, nc *nats.Conn, path, scratch string, log func(string, ...any)) error { - f, err := os.Open(path) - if err != nil { - return err - } - defer f.Close() - a, err := busarchive.ReadArchive(f, scratch) - if err != nil { - return err - } - defer a.Close() - if err := a.Verify(); err != nil { - return err - } - for _, s := range a.Manifest.Streams { - started := time.Now() - if err := restoreStream(ctx, nc, a, s); err != nil { - return fmt.Errorf("stream %s: %w", s.Name, err) - } - log("%s: restored in %.2fs", s.Name, time.Since(started).Seconds()) - } - return nil -} - -func restoreStream(ctx context.Context, nc *nats.Conn, a *busarchive.Archive, s busarchive.StreamEntry) error { - meta, archive, err := a.Open(s) - if err != nil { - return err - } - defer archive.Close() - var req map[string]json.RawMessage - if err := json.Unmarshal(meta, &req); err != nil || req["config"] == nil { - return errors.New("its meta is not a stream's configuration and state") - } - var resp struct { - DeliverSubject string `json:"deliver_subject"` - } - if err := Request(ctx, nc, APIStreamRestore+s.Name, req, &resp); err != nil { - return err - } - if resp.DeliverSubject == "" { - return errors.New("the server gave nowhere to send the archive") - } - buf := make([]byte, restoreChunk) - for { - n, err := archive.Read(buf) - if n > 0 { - reply, rerr := nc.RequestWithContext(ctx, resp.DeliverSubject, buf[:n]) - if rerr != nil { - return fmt.Errorf("sending the archive: %w", rerr) - } - if len(reply.Data) > 0 { - return fmt.Errorf("the server refused the archive: %s", Truncate(reply.Data)) - } - } - if err == io.EOF { - break - } - if err != nil { - return err - } - } - final, err := nc.RequestWithContext(ctx, resp.DeliverSubject, nil) - if err != nil { - return fmt.Errorf("finishing the restore: %w", err) - } - var created struct { - Error *APIError `json:"error"` - State StreamState `json:"state"` - } - if err := json.Unmarshal(final.Data, &created); err != nil { - return fmt.Errorf("the server's last answer is not JSON: %q", Truncate(final.Data)) - } - if created.Error != nil { - return created.Error - } - if created.State.LastSeq < s.LastSeq || (s.Messages > 0 && created.State.Messages == 0) { - return fmt.Errorf("restored with %d message(s) up to sequence %d; the snapshot held %d up to %d", - created.State.Messages, created.State.LastSeq, s.Messages, s.LastSeq) - } - return nil -} diff --git a/modules/nats/internal/bussnap/snapshot.go b/modules/nats/internal/bussnap/snapshot.go deleted file mode 100644 index dd4517d40..000000000 --- a/modules/nats/internal/bussnap/snapshot.go +++ /dev/null @@ -1,433 +0,0 @@ -// Package bussnap takes a snapshot of the bus's streams with the server's own snapshot API (novox/hq ADR 0235): the -// nightly dump of every stream, and the bus module's keeper's snapshot of one entry before it is rebuilt or removed -// (design 25 § 12.3). -package bussnap - -import ( - "archive/tar" - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "io" - "os" - "path/filepath" - "slices" - "sort" - "strings" - "sync" - "time" - - "nats-tools/internal/busarchive" - - "github.com/nats-io/nats.go" -) - -// The JetStream API's subjects this program speaks. Each is granted to the bus's own module and -// nothing else is (mesh-controller internal/broker, BusSnapshotGrants); a test holds the two lists -// equal against a real server. -const ( - APIStreamNames = "$JS.API.STREAM.NAMES" - APIStreamInfo = "$JS.API.STREAM.INFO." - APIStreamSnapshot = "$JS.API.STREAM.SNAPSHOT." - APIStreamRestore = "$JS.API.STREAM.RESTORE." -) - -// ChunkSize is what the server is asked to send at a time: its own default. The server sends up to -// an 8 MiB window ahead of the acknowledgements and waits for them past it, so one stream costs the -// server at most that much in flight, whatever its size. -const ChunkSize = 128 * 1024 - -// SnapshotOptions bounds a snapshot. -type SnapshotOptions struct { - // StreamTimeout is the longest one stream may take; past it the snapshot fails, naming it. - StreamTimeout time.Duration - // Pause is a rest between two streams, so a night's snapshot is never one long burst. - Pause time.Duration - // TempDir is where one stream's archive waits until it is written out; the system's when empty. - TempDir string - Log func(format string, args ...any) - // Streams, when set, are the only streams taken (the keeper's snapshot of one entry); every stream when empty. - Streams []string -} - -// APIError is the error every JetStream API answer may carry. -type APIError struct { - Code int `json:"code"` - ErrCode int `json:"err_code"` - Description string `json:"description"` -} - -func (e *APIError) Error() string { return fmt.Sprintf("%s (%d)", e.Description, e.Code) } - -// StreamState is what this program reads of a stream's state; the whole of it is kept as the server -// gave it, in the stream's meta. -type StreamState struct { - Messages uint64 `json:"messages"` - Bytes uint64 `json:"bytes"` - FirstSeq uint64 `json:"first_seq"` - LastSeq uint64 `json:"last_seq"` - NumDelete int `json:"num_deleted"` - Consumers int `json:"consumer_count"` -} - -type streamConfig struct { - Name string `json:"name"` - Subjects []string `json:"subjects"` - Retention string `json:"retention"` - Storage string `json:"storage"` -} - -// Request asks the JetStream API and decodes its answer, refusing an answer that carries an error. -func Request(ctx context.Context, nc *nats.Conn, subject string, body any, into any) error { - var payload []byte - if body != nil { - var err error - if payload, err = json.Marshal(body); err != nil { - return err - } - } - rctx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - msg, err := nc.RequestWithContext(rctx, subject, payload) - if err != nil { - if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, nats.ErrTimeout) || errors.Is(err, nats.ErrNoResponders) { - return fmt.Errorf("%s was not answered (%v): the bus refuses a subject this user is not granted by not answering it", subject, err) - } - return fmt.Errorf("%s: %w", subject, err) - } - var wrapped struct { - Error *APIError `json:"error"` - } - if err := json.Unmarshal(msg.Data, &wrapped); err != nil { - return fmt.Errorf("%s answered something that is not JSON: %q", subject, Truncate(msg.Data)) - } - if wrapped.Error != nil { - return fmt.Errorf("%s: %w", subject, wrapped.Error) - } - if into != nil { - return json.Unmarshal(msg.Data, into) - } - return nil -} - -func Truncate(b []byte) string { - if len(b) > 200 { - return string(b[:200]) + "…" - } - return string(b) -} - -// StreamNames is every stream in the account, in name order, all pages of them. -func StreamNames(ctx context.Context, nc *nats.Conn) ([]string, error) { - var names []string - for { - var page struct { - Total int `json:"total"` - Offset int `json:"offset"` - Limit int `json:"limit"` - Streams []string `json:"streams"` - } - if err := Request(ctx, nc, APIStreamNames, map[string]int{"offset": len(names)}, &page); err != nil { - return nil, err - } - names = append(names, page.Streams...) - if len(page.Streams) == 0 || len(names) >= page.Total { - break - } - } - sort.Strings(names) - return names, nil -} - -// Snapshot writes every stream, one at a time, as a tar: the manifest first, then each stream's -// meta and archive under streams//. Any stream failing fails the whole: a partial copy of the -// bus written where a whole one is expected is the silent failure this exists to prevent. -func Snapshot(ctx context.Context, nc *nats.Conn, out io.Writer, o SnapshotOptions) (*busarchive.Manifest, error) { - if o.StreamTimeout <= 0 { - o.StreamTimeout = 10 * time.Minute - } - if o.Log == nil { - o.Log = func(string, ...any) {} - } - m := &busarchive.Manifest{Format: 1, Server: nc.ConnectedServerVersion(), Started: time.Now().UTC()} - names, err := StreamNames(ctx, nc) - if err != nil { - return nil, err - } - if len(o.Streams) > 0 { - var only []string - for _, n := range names { - if slices.Contains(o.Streams, n) { - only = append(only, n) - } - } - if len(only) != len(o.Streams) { - return nil, fmt.Errorf("the bus holds %v of the streams %v asked for: a snapshot missing one would pass for a whole one", only, o.Streams) - } - names = only - } - if len(names) == 0 { - // The mesh's own streams always exist once the controller has run; a bus with none is not - // one to call backed up. - return nil, errors.New("the bus holds no streams; a snapshot of nothing would pass for a backup") - } - - type taken struct { - entry busarchive.StreamEntry - meta []byte - archive *os.File - } - var files []taken - defer func() { - for _, t := range files { - t.archive.Close() - os.Remove(t.archive.Name()) - } - }() - - for i, name := range names { - if i > 0 && o.Pause > 0 { - select { - case <-time.After(o.Pause): - case <-ctx.Done(): - return nil, ctx.Err() - } - } - var info struct { - Config streamConfig `json:"config"` - } - if err := Request(ctx, nc, APIStreamInfo+name, nil, &info); err != nil { - return nil, err - } - if info.Config.Storage == "memory" { - m.Skipped = append(m.Skipped, busarchive.Skipped{Name: name, Why: "memory storage: it holds nothing across a restart, and the server cannot snapshot it"}) - o.Log("%s: skipped, memory storage", name) - continue - } - f, err := os.CreateTemp(o.TempDir, "mesh-nats-snapshot-*") - if err != nil { - return nil, err - } - files = append(files, taken{archive: f}) - sctx, cancel := context.WithTimeout(ctx, o.StreamTimeout) - entry, meta, err := snapshotStream(sctx, nc, name, f) - cancel() - if err != nil { - return nil, fmt.Errorf("stream %s: %w", name, err) - } - files[len(files)-1].entry, files[len(files)-1].meta = entry, meta - m.Messages += entry.Messages - m.StreamBytes += int64(entry.Bytes) - m.ArchiveBytes += entry.ArchiveBytes - o.Log("%s: %d message(s), %s, last sequence %d, %d consumer(s) → %s in %.2fs", - name, entry.Messages, Human(int64(entry.Bytes)), entry.LastSeq, entry.Consumers, Human(entry.ArchiveBytes), entry.Seconds) - } - m.Finished = time.Now().UTC() - m.Seconds = m.Finished.Sub(m.Started).Seconds() - for _, t := range files { - m.Streams = append(m.Streams, t.entry) - } - - tw := tar.NewWriter(out) - manifest, err := json.MarshalIndent(m, "", " ") - if err != nil { - return nil, err - } - if err := writeEntry(tw, busarchive.ManifestName, append(manifest, '\n'), m.Finished); err != nil { - return nil, err - } - for _, t := range files { - if err := writeEntry(tw, t.entry.Meta, t.meta, m.Finished); err != nil { - return nil, err - } - if _, err := t.archive.Seek(0, io.SeekStart); err != nil { - return nil, err - } - hdr := &tar.Header{Name: t.entry.Archive, Mode: 0o600, Size: t.entry.ArchiveBytes, ModTime: m.Finished, Format: tar.FormatPAX} - if err := tw.WriteHeader(hdr); err != nil { - return nil, err - } - if _, err := io.Copy(tw, t.archive); err != nil { - return nil, err - } - } - if err := tw.Close(); err != nil { - return nil, err - } - return m, nil -} - -func writeEntry(tw *tar.Writer, name string, body []byte, at time.Time) error { - if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(body)), ModTime: at, Format: tar.FormatPAX}); err != nil { - return err - } - _, err := tw.Write(body) - return err -} - -// snapshotStream takes one stream's snapshot into f: the server's chunks, each acknowledged as it -// arrives so the server's window keeps moving, until the server's last, empty message. It answers -// the stream's entry (the archive's size and checksum included) and its meta. -// -// The protocol is the server's (nats-server jetstream_api.go, streamSnapshot): the request names a -// subject to deliver to; the server answers with the stream's configuration and its state at the -// snapshot, then sends the archive in chunks whose reply subject is the acknowledgement it waits for -// past its window — two seconds without one and it gives up with "408 No Flow Response". The end is -// an empty message whose status header says 204, or the error that ended it. -func snapshotStream(ctx context.Context, nc *nats.Conn, name string, f *os.File) (busarchive.StreamEntry, []byte, error) { - started := time.Now() - deliver := nc.NewRespInbox() - - hash := sha256.New() - var written int64 - var mu sync.Mutex - var failed error - done := make(chan struct{}) - var once sync.Once - finish := func(err error) { - mu.Lock() - if failed == nil { - failed = err - } - mu.Unlock() - once.Do(func() { close(done) }) - } - sub, err := nc.Subscribe(deliver, func(msg *nats.Msg) { - if len(msg.Data) == 0 { - if status := msg.Header.Get("Status"); status != "" && status != "204" { - finish(fmt.Errorf("the server ended the snapshot: %s %s", status, msg.Header.Get("Description"))) - return - } - finish(nil) - return - } - // Acknowledged before it is written: the chunk is already in memory, and a slow disk must not - // read to the server as a reader that went away. - if msg.Reply != "" { - if err := nc.Publish(msg.Reply, nil); err != nil { - finish(fmt.Errorf("acknowledging a chunk: %w", err)) - return - } - } - if _, err := f.Write(msg.Data); err != nil { - finish(fmt.Errorf("writing the archive: %w", err)) - return - } - hash.Write(msg.Data) - written += int64(len(msg.Data)) - }) - if err != nil { - return busarchive.StreamEntry{}, nil, err - } - defer sub.Unsubscribe() - // Every chunk is held until the callback has it; the window bounds how many there can be. - if err := sub.SetPendingLimits(-1, -1); err != nil { - return busarchive.StreamEntry{}, nil, err - } - if err := nc.FlushWithContext(ctx); err != nil { - return busarchive.StreamEntry{}, nil, err - } - - var resp struct { - Config json.RawMessage `json:"config"` - State json.RawMessage `json:"state"` - } - req := map[string]any{"deliver_subject": deliver, "chunk_size": ChunkSize} - if err := Request(ctx, nc, APIStreamSnapshot+name, req, &resp); err != nil { - return busarchive.StreamEntry{}, nil, err - } - select { - case <-done: - case <-ctx.Done(): - return busarchive.StreamEntry{}, nil, fmt.Errorf("not finished within its bound: %w", ctx.Err()) - } - // The subscription is drained before the archive is read back, so no callback is still writing. - _ = sub.Unsubscribe() - mu.Lock() - err = failed - mu.Unlock() - if err != nil { - return busarchive.StreamEntry{}, nil, err - } - if written == 0 { - return busarchive.StreamEntry{}, nil, errors.New("the server sent an empty archive") - } - if err := f.Sync(); err != nil { - return busarchive.StreamEntry{}, nil, err - } - - var cfg streamConfig - var state StreamState - if err := json.Unmarshal(resp.Config, &cfg); err != nil { - return busarchive.StreamEntry{}, nil, fmt.Errorf("its configuration: %w", err) - } - if err := json.Unmarshal(resp.State, &state); err != nil { - return busarchive.StreamEntry{}, nil, fmt.Errorf("its state: %w", err) - } - // The archive reads to its end — s2 frames, then a tar — before it is called a snapshot. - if _, err := f.Seek(0, io.SeekStart); err != nil { - return busarchive.StreamEntry{}, nil, err - } - if err := busarchive.ReadsThrough(f); err != nil { - return busarchive.StreamEntry{}, nil, fmt.Errorf("the archive the server sent does not read: %w", err) - } - - // The meta, in the shape the restore API takes and the nats CLI writes (backup.json): the - // server's own configuration and state, kept as raw JSON so a field this program does not know - // is never dropped on the way back. - meta, err := json.MarshalIndent(map[string]json.RawMessage{"config": resp.Config, "state": resp.State}, "", " ") - if err != nil { - return busarchive.StreamEntry{}, nil, err - } - metaSum := sha256.Sum256(meta) - return busarchive.StreamEntry{ - Name: name, Subjects: cfg.Subjects, Retention: cfg.Retention, - Messages: state.Messages, Bytes: state.Bytes, FirstSeq: state.FirstSeq, LastSeq: state.LastSeq, - Deleted: state.NumDelete, Consumers: state.Consumers, - Meta: "streams/" + name + "/backup.json", MetaSHA256: hex.EncodeToString(metaSum[:]), - Archive: "streams/" + name + "/stream.tar.s2", ArchiveBytes: written, - ArchiveSHA256: hex.EncodeToString(hash.Sum(nil)), Seconds: time.Since(started).Seconds(), - }, meta, nil -} - -// Human is a size in bytes as a person reads it: 1.5 MiB. -func Human(n int64) string { - const unit = 1024 - if n < unit { - return fmt.Sprintf("%d B", n) - } - div, exp := int64(unit), 0 - for v := n / unit; v >= unit; v /= unit { - div *= unit - exp++ - } - return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp]) -} - -// Into is the keeper's snapshot of some streams before it rebuilds or removes them (design 25 § 12.3): one tar under -// dir, named for the streams and the time, written whole before its name is given, so a partial one is never taken -// for it. It answers the file's path. -func Into(nc *nats.Conn, dir string) func(ctx context.Context, streams []string) (string, error) { - return func(ctx context.Context, streams []string) (string, error) { - if dir == "" { - return "", errors.New("no snapshots directory is set (MESH_NATS_SNAPSHOTS_DIR), so no snapshot is taken and nothing is removed") - } - name := filepath.Join(dir, "entry-"+strings.Join(streams, "+")+"-"+time.Now().UTC().Format("20060102T150405.000000Z")+".tar") - f, err := os.CreateTemp(dir, ".entry-*.partial") - if err != nil { - return "", err - } - defer os.Remove(f.Name()) - if _, err := Snapshot(ctx, nc, f, SnapshotOptions{Streams: streams, TempDir: dir}); err != nil { - f.Close() - return "", err - } - if err := f.Close(); err != nil { - return "", err - } - return name, os.Rename(f.Name(), name) - } -} diff --git a/modules/nats/internal/keeper/jetstream.go b/modules/nats/internal/keeper/jetstream.go index 5ff2d51c0..c3e20079f 100644 --- a/modules/nats/internal/keeper/jetstream.go +++ b/modules/nats/internal/keeper/jetstream.go @@ -10,7 +10,7 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" - "nats-tools/internal/bussnap" + "git.novox.be/novox/mesh-sdk/go/bussnap" ) // JetStream is the bus as the keeper's own account reaches it (Bus): each object a stream, a key-value bucket or an