From b43c57f43b37b153cac878dc200e4892a2ac5e49 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 01:13:13 +0100 Subject: [PATCH 01/17] mesh-recovery: the module, its manifest and its claim of the seat mesh-recovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §554, §656, §657, §633, §564, §557, A-107, A-115 The seat claimed once for the mesh with every verb of its definition; a process answering them (serves beside bus), a tools bundle, the data item points, the state digests (§567), the findings of §561, the four settings. Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- modules/mesh-recovery/.gitignore | 3 + modules/mesh-recovery/README.md | 9 + modules/mesh-recovery/go.mod | 24 ++ modules/mesh-recovery/go.sum | 25 ++ .../internal/recovery/manifest_test.go | 101 +++++++ modules/mesh-recovery/module.json | 265 ++++++++++++++++++ 6 files changed, 427 insertions(+) create mode 100644 modules/mesh-recovery/.gitignore create mode 100644 modules/mesh-recovery/README.md create mode 100644 modules/mesh-recovery/go.mod create mode 100644 modules/mesh-recovery/go.sum create mode 100644 modules/mesh-recovery/internal/recovery/manifest_test.go create mode 100644 modules/mesh-recovery/module.json diff --git a/modules/mesh-recovery/.gitignore b/modules/mesh-recovery/.gitignore new file mode 100644 index 000000000..1b8d4339c --- /dev/null +++ b/modules/mesh-recovery/.gitignore @@ -0,0 +1,3 @@ +/mesh-recovery +/cmd/mesh-recovery/mesh-recovery +/cmd/mesh-recovery-tools/mesh-recovery-tools diff --git a/modules/mesh-recovery/README.md b/modules/mesh-recovery/README.md new file mode 100644 index 000000000..a1156d343 --- /dev/null +++ b/modules/mesh-recovery/README.md @@ -0,0 +1,9 @@ +# mesh-recovery + +Holds the seat `mesh-recovery` (design 47 G §554): the recovery module takes, sets aside and restores a module's kept +data on the live bus, one act at a time, each act in the hand-act log, its result said as an event under the act's id. +Go: one process (`cmd/mesh-recovery`) that answers the seat's verbs itself (A-107, A-115) and a tools bundle +(`cmd/mesh-recovery-tools`) whose `recovery_*` tools call those verbs and hold no grant of their own (§656). + +Its data item `points` holds the restore points and the asides; the digests of the points are on the bus in its state +`digests` (§567). Its settings are `points-cap`, `points-filling-mark`, `point-keep-days` and `dump-wait-minutes`. diff --git a/modules/mesh-recovery/go.mod b/modules/mesh-recovery/go.mod new file mode 100644 index 000000000..ee3bd0cc0 --- /dev/null +++ b/modules/mesh-recovery/go.mod @@ -0,0 +1,24 @@ +module mesh-recovery + +go 1.26.0 + +require ( + git.novox.be/novox/mesh-contracts/go/seats/meshbroker v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/meshrecovery v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 + github.com/nats-io/nats.go v1.54.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.1 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.2 // indirect + github.com/nats-io/nats-server/v2 v2.11.17 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.16.0 // indirect +) diff --git a/modules/mesh-recovery/go.sum b/modules/mesh-recovery/go.sum new file mode 100644 index 000000000..8ca93dfd0 --- /dev/null +++ b/modules/mesh-recovery/go.sum @@ -0,0 +1,25 @@ +github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E= +github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ= +github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc= +github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE= +golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s= diff --git a/modules/mesh-recovery/internal/recovery/manifest_test.go b/modules/mesh-recovery/internal/recovery/manifest_test.go new file mode 100644 index 000000000..61da8aa86 --- /dev/null +++ b/modules/mesh-recovery/internal/recovery/manifest_test.go @@ -0,0 +1,101 @@ +package recovery_test + +import ( + "encoding/json" + "os" + "slices" + "testing" + + seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" +) + +type manifest struct { + Module string `json:"module"` + RunsAs string `json:"runs-as"` + Claims []struct { + Name string `json:"name"` + Scope string `json:"scope"` + Serves []string `json:"serves"` + } `json:"claims"` + Tools []string `json:"tools"` + Consumes []string `json:"consumes"` + Invokes []string `json:"invokes"` + Config map[string]struct { + Default float64 `json:"default"` + } `json:"config"` + Data struct { + Own []struct{ ID string } `json:"own"` + } `json:"data"` + Resources []struct { + ID string `json:"id"` + Type string `json:"type"` + Serves []string `json:"serves"` + Bus struct { + Invokes []string `json:"invokes"` + } `json:"bus"` + Health struct { + Tool string `json:"tool"` + } `json:"health"` + } `json:"resources"` +} + +// The manifest says what the module is (design 47 G §554, §656, §657, §633, §564, §567): it claims the seat once for the +// mesh serving every verb of the seat's definition, its process answers each of them itself (A-115: `serves` beside its +// `bus`), its health check names a tool it serves, its data item is `points`, its settings carry the design's defaults, +// and what the process invokes the module declares. +func TestTheManifestSaysWhatTheModuleIs(t *testing.T) { + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + var m manifest + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + if m.Module != "mesh-recovery" || m.RunsAs != "mesh-recovery" { + t.Errorf("module %q runs as %q", m.Module, m.RunsAs) + } + if len(m.Claims) != 1 || m.Claims[0].Name != seat.Seat || m.Claims[0].Scope != "mesh" || !slices.Equal(m.Claims[0].Serves, seat.Verbs()) { + t.Errorf("claims %+v, want %s at mesh scope serving %v", m.Claims, seat.Seat, seat.Verbs()) + } + for _, id := range []string{"points"} { + if !slices.ContainsFunc(m.Data.Own, func(o struct{ ID string }) bool { return o.ID == id }) { + t.Errorf("no data item %q", id) + } + } + for k, want := range map[string]float64{"points-cap": 10 << 30, "points-filling-mark": 80, "point-keep-days": 90, "dump-wait-minutes": 60} { + if m.Config[k].Default != want { + t.Errorf("setting %s defaults to %v, want %v", k, m.Config[k].Default, want) + } + } + for _, r := range m.Resources { + if r.Type != "process" { + continue + } + var want []string + for _, v := range seat.Verbs() { + want = append(want, "seat:"+seat.Seat+"."+v) + } + if !slices.Equal(r.Serves, want) { + t.Errorf("the process serves %v, want %v", r.Serves, want) + } + if !slices.Contains(m.Tools, r.Health.Tool) || r.Health.Tool != "recovery_kept_points" { + t.Errorf("the health check names %q, a tool of %v", r.Health.Tool, m.Tools) + } + for _, inv := range r.Bus.Invokes { + if !slices.Contains(m.Invokes, inv) { + t.Errorf("the process invokes %s, which the module does not declare", inv) + } + } + } + for _, c := range []string{"node-backup.item-started", "node-backup.item-ended", "mesh-broker.snapshot-restored"} { + if !slices.Contains(m.Consumes, c) { + t.Errorf("does not consume %s", c) + } + } + for _, tool := range m.Tools { + if len(tool) < 9 || tool[:9] != "recovery_" { + t.Errorf("the tool %q is not named recovery_", tool) + } + } +} diff --git a/modules/mesh-recovery/module.json b/modules/mesh-recovery/module.json new file mode 100644 index 000000000..ac477062c --- /dev/null +++ b/modules/mesh-recovery/module.json @@ -0,0 +1,265 @@ +{ + "module": "mesh-recovery", + "version": "0.1", + "runs-as": "mesh-recovery", + "claims": [ + { + "name": "mesh-recovery", + "scope": "mesh", + "serves": [ + "snapshot-kept", + "restore-kept", + "kept-points", + "restore-aside", + "cancel", + "take", + "restore", + "retry", + "aside", + "discard-aside", + "expire-points" + ] + } + ], + "tools": [ + "recovery_kept_points", + "recovery_take", + "recovery_restore", + "recovery_retry", + "recovery_aside", + "recovery_discard_aside", + "recovery_restore_aside", + "recovery_cancel", + "recovery_expire_points" + ], + "consumes": [ + "node-backup.item-started", + "node-backup.item-ended", + "mesh-broker.snapshot-restored", + "mesh-conditions.findings-asked" + ], + "invokes": [ + "seat:mesh-catalog.entry", + "seat:mesh-hand-acts.hand-act", + "seat:mesh-broker.snapshot-get", + "seat:mesh-broker.snapshot-restore", + "seat:node-backup.restore", + "seat:mesh-recovery.kept-points", + "seat:mesh-recovery.restore-aside", + "seat:mesh-recovery.cancel", + "seat:mesh-recovery.take", + "seat:mesh-recovery.restore", + "seat:mesh-recovery.retry", + "seat:mesh-recovery.aside", + "seat:mesh-recovery.discard-aside", + "seat:mesh-recovery.expire-points" + ], + "requires": [ + { + "name": "digests", + "storage": "key-value", + "template": "state", + "options": { + "max-mib": 4 + } + }, + { + "name": "findings", + "storage": "key-value", + "template": "state" + }, + { + "name": "finding-events", + "storage": "stream", + "template": "events", + "options": { + "events": [ + "finding-written.*", + "finding-cleared.*", + "findings-stated.*" + ], + "max-age-days": 7, + "max-mib": 16 + } + } + ], + "config": { + "points-cap": { + "type": "number", + "description": "bytes the points directory may hold; an act that would pass it ends failed, naming it, before it removes anything (design 47 G section 564).", + "minimum": 1, + "trusted": false, + "default": 10737418240 + }, + "points-filling-mark": { + "type": "number", + "description": "percent of the cap past which the finding points-filling is written (design 47 G section 564).", + "minimum": 1, + "maximum": 100, + "trusted": false, + "default": 80 + }, + "point-keep-days": { + "type": "number", + "description": "days after which a restore point is removed, the newest of its kept data never (design 47 G sections 551, 557).", + "minimum": 1, + "maximum": 3650, + "trusted": false, + "default": 90 + }, + "dump-wait-minutes": { + "type": "number", + "description": "minutes an act waits on the dump of the bus's item before recovery-waits is written (design 47 G sections 561, 557).", + "minimum": 1, + "maximum": 10080, + "trusted": false, + "default": 60 + } + }, + "data": { + "own": [ + { + "id": "points", + "path": "${dir:points}", + "class": "valuable", + "why": "the restore points taken of modules' kept data before a form change, and the asides of live data set aside before a restore (design 47 G sections 516, 554)" + } + ] + }, + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-recovery", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-recovery", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-recovery" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-recovery", + "place": ".", + "unknown": "program" + }, + { + "id": "points", + "type": "directory", + "path": "/var/lib/mesh-recovery/points", + "mode": "0700", + "owner": "mesh-recovery", + "unknown": "program" + }, + { + "id": "cap", + "type": "file", + "path": "${dir:state}/points-cap", + "owner": "mesh-recovery", + "mode": "0600", + "content": "${config:points-cap}" + }, + { + "id": "mark", + "type": "file", + "path": "${dir:state}/points-filling-mark", + "owner": "mesh-recovery", + "mode": "0600", + "content": "${config:points-filling-mark}" + }, + { + "id": "keep-days", + "type": "file", + "path": "${dir:state}/point-keep-days", + "owner": "mesh-recovery", + "mode": "0600", + "content": "${config:point-keep-days}" + }, + { + "id": "dump-wait", + "type": "file", + "path": "${dir:state}/dump-wait-minutes", + "owner": "mesh-recovery", + "mode": "0600", + "content": "${config:dump-wait-minutes}" + }, + { + "id": "recovery", + "type": "process", + "name": "mesh-recovery", + "artifact": "program", + "run": [ + "./mesh-recovery" + ], + "env": { + "MESH_BROKER_FILE": "${dir:state}/broker", + "MESH_RECOVERY_POINTS": "${dir:points}", + "MESH_RECOVERY_POINTS_CAP_FILE": "${dir:state}/points-cap", + "MESH_RECOVERY_FILLING_MARK_FILE": "${dir:state}/points-filling-mark", + "MESH_RECOVERY_KEEP_DAYS_FILE": "${dir:state}/point-keep-days", + "MESH_RECOVERY_DUMP_WAIT_FILE": "${dir:state}/dump-wait-minutes" + }, + "user": "mesh-recovery", + "serves": [ + "seat:mesh-recovery.snapshot-kept", + "seat:mesh-recovery.restore-kept", + "seat:mesh-recovery.kept-points", + "seat:mesh-recovery.restore-aside", + "seat:mesh-recovery.cancel", + "seat:mesh-recovery.take", + "seat:mesh-recovery.restore", + "seat:mesh-recovery.retry", + "seat:mesh-recovery.aside", + "seat:mesh-recovery.discard-aside", + "seat:mesh-recovery.expire-points" + ], + "bus": { + "state": [ + "digests", + "findings" + ], + "invokes": [ + "seat:mesh-catalog.entry", + "seat:mesh-hand-acts.hand-act", + "seat:mesh-broker.snapshot-get", + "seat:mesh-broker.snapshot-restore", + "seat:node-backup.restore" + ] + }, + "health": { + "kind": "tool", + "tool": "recovery_kept_points", + "interval": "60s", + "timeout": "10s", + "looks": 2, + "grace": "180s" + } + } + ], + "build": { + "artifacts": [ + { + "name": "program", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-recovery", + "binary": "mesh-recovery" + }, + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-recovery-tools", + "binary": "mesh-recovery-tools", + "loads": [ + "mesh-recovery-tools" + ] + } + ] + } +} From 7d755c8f086c35fcde00cfb848f54c5f5d7f9650 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 01:13:58 +0100 Subject: [PATCH 02/17] mesh-recovery: take a restore point, list the points, expire them, one act at a time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §516, §551, §554, §560, §564, §567, §517 Accept writes the hand-act log first, answers accepted/busy/error at once and writes the act before it answers; take writes each stream under a temporary name, its digests on the bus before ended; the cap is refused before anything is written; expiry never removes the newest point. Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../internal/recovery/fakes_test.go | 214 ++++++++++++++ .../mesh-recovery/internal/recovery/points.go | 221 ++++++++++++++ .../internal/recovery/points_test.go | 121 ++++++++ .../internal/recovery/recovery.go | 270 ++++++++++++++++++ 4 files changed, 826 insertions(+) create mode 100644 modules/mesh-recovery/internal/recovery/fakes_test.go create mode 100644 modules/mesh-recovery/internal/recovery/points.go create mode 100644 modules/mesh-recovery/internal/recovery/points_test.go create mode 100644 modules/mesh-recovery/internal/recovery/recovery.go diff --git a/modules/mesh-recovery/internal/recovery/fakes_test.go b/modules/mesh-recovery/internal/recovery/fakes_test.go new file mode 100644 index 000000000..58ccdbe2a --- /dev/null +++ b/modules/mesh-recovery/internal/recovery/fakes_test.go @@ -0,0 +1,214 @@ +package recovery_test + +import ( + "context" + "errors" + "io" + "os" + "path/filepath" + "sync" + "testing" + "time" + + seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" + + "mesh-recovery/internal/recovery" +) + +// fakeBus holds streams as strings; a snapshot file is "snap::". +type fakeBus struct { + mu sync.Mutex + streams map[string]string + consumers map[string][]seat.Consumer + setBack []seat.Consumer + failRestoreOnce bool +} + +func (b *fakeBus) Objects(_ context.Context, names []string) ([]recovery.Object, error) { + b.mu.Lock() + defer b.mu.Unlock() + var out []recovery.Object + for _, n := range names { + c, ok := b.streams[n] + out = append(out, recovery.Object{Stream: n, Exists: ok, Bytes: int64(len(c)), Consumers: b.consumers[n]}) + } + return out, nil +} + +func (b *fakeBus) Take(_ context.Context, s string, w io.Writer) error { + b.mu.Lock() + defer b.mu.Unlock() + c, ok := b.streams[s] + if !ok { + return errors.New("no stream " + s) + } + _, err := io.WriteString(w, "snap:"+s+":"+c) + return err +} + +func (b *fakeBus) Delete(_ context.Context, s string) error { + b.mu.Lock() + defer b.mu.Unlock() + delete(b.streams, s) + return nil +} + +func (b *fakeBus) Restore(_ context.Context, file string) error { + raw, err := os.ReadFile(file) + if err != nil { + return err + } + b.mu.Lock() + defer b.mu.Unlock() + if b.failRestoreOnce && len(raw) > 0 && b.streams != nil { + if string(raw[:5]) == "snap:" && !b.restoredOnce(raw) { + return errors.New("the server refused the archive") + } + } + var name, content string + rest := string(raw[5:]) + for i := range rest { + if rest[i] == ':' { + name, content = rest[:i], rest[i+1:] + break + } + } + if _, live := b.streams[name]; live { + return errors.New("stream name already in use") + } + b.streams[name] = content + return nil +} + +var once sync.Map + +func (b *fakeBus) restoredOnce(raw []byte) bool { + _, loaded := once.LoadOrStore(string(raw), true) + return loaded +} + +func (b *fakeBus) SetConsumers(_ context.Context, cs []seat.Consumer) error { + b.mu.Lock() + defer b.mu.Unlock() + b.setBack = append(b.setBack, cs...) + return nil +} + +type said struct { + Event string + Body any + Parts []string +} + +type fakeSeats struct { + mu sync.Mutex + events []said + handActs []string + handFail error + streams []string + backup string +} + +func (s *fakeSeats) Say(event string, body any, parts ...string) error { + s.mu.Lock() + defer s.mu.Unlock() + s.events = append(s.events, said{event, body, parts}) + return nil +} + +func (s *fakeSeats) HandAct(caller, asker, what, why string) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.handFail != nil { + return s.handFail + } + s.handActs = append(s.handActs, caller+"|"+asker+"|"+what+"|"+why) + return nil +} + +func (s *fakeSeats) Streams(context.Context, string, string, string) ([]string, error) { + return s.streams, nil +} + +func (s *fakeSeats) Backup(context.Context, string) (string, error) { + if s.backup == "" { + return "", errors.New("no repository snapshot holds it") + } + return s.backup, nil +} + +func (s *fakeSeats) named(event string) []said { + s.mu.Lock() + defer s.mu.Unlock() + var out []said + for _, e := range s.events { + if e.Event == event { + out = append(out, e) + } + } + return out +} + +type memDigests struct { + mu sync.Mutex + m map[string][]byte +} + +func (d *memDigests) Put(k string, v []byte) error { + d.mu.Lock() + defer d.mu.Unlock() + d.m[k] = v + return nil +} +func (d *memDigests) Get(k string) ([]byte, bool, error) { + d.mu.Lock() + defer d.mu.Unlock() + v, ok := d.m[k] + return v, ok, nil +} +func (d *memDigests) Delete(k string) error { + d.mu.Lock() + defer d.mu.Unlock() + delete(d.m, k) + return nil +} + +type noDump struct{} + +func (noDump) Wait(context.Context, func(string)) error { return nil } + +type rig struct { + e *recovery.Engine + b *fakeBus + s *fakeSeats + d *memDigests + t time.Time +} + +func newRig(t *testing.T) *rig { + t.Helper() + r := &rig{b: &fakeBus{streams: map[string]string{"app_notes": "hello", "KV_app_seen": "world"}, consumers: map[string][]seat.Consumer{}}, + s: &fakeSeats{streams: []string{"app_notes", "KV_app_seen"}}, d: &memDigests{m: map[string][]byte{}}, t: time.Date(2026, 11, 12, 1, 0, 0, 0, time.UTC)} + r.b.consumers["app_notes"] = []seat.Consumer{{Name: "c1", Stream: "app_notes", Sequence: 7}} + r.e = &recovery.Engine{Dir: filepath.Join(t.TempDir(), "points-dir"), Bus: r.b, Seats: r.s, Digests: r.d, Dump: noDump{}, + Settings: recovery.Settings{Cap: 1 << 20, KeepDays: 90}, Now: func() time.Time { return r.t }} + return r +} + +func (r *rig) do(a recovery.Act) recovery.Answer { + ans := r.e.Accept(a) + r.e.Wait() + return ans +} + +var errTest = errors.New("the log is unreachable") + +type gatedBus struct { + *fakeBus + gate chan struct{} +} + +func (g *gatedBus) Objects(ctx context.Context, n []string) ([]recovery.Object, error) { + <-g.gate + return g.fakeBus.Objects(ctx, n) +} diff --git a/modules/mesh-recovery/internal/recovery/points.go b/modules/mesh-recovery/internal/recovery/points.go new file mode 100644 index 000000000..beb3688b1 --- /dev/null +++ b/modules/mesh-recovery/internal/recovery/points.go @@ -0,0 +1,221 @@ +package recovery + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + + seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +const pointFile = "point.json" + +// run is the work of one accepted act; it ends the act with its event under the act's id (§560). +func (e *Engine) run(ctx context.Context, a *Act) { + var err error + switch a.Kind { + case KindTake: + err = e.take(ctx, a) + case KindExpire: + e.expire(a.ID) + default: + err = e.endedFailed(a, fmt.Errorf("no act of kind %q", a.Kind)) + } + if err != nil { + e.logf("act %s ended failed: %v", a.ID, err) + } + e.markEnded(a) +} + +// say publishes an event and logs the failure; an event lost is never silent. +func (e *Engine) say(event string, body any, parts ...string) { + if err := e.Seats.Say(event, body, parts...); err != nil { + e.logf("the event %s was not said: %v", event, err) + } +} + +func (e *Engine) endedFailed(a *Act, why error) error { + e.say(seat.EventEnded, seat.EndedEvent{Act: a.ID, At: e.now(), Module: a.Module, Kept: a.Kept, Form: a.Form, + Fingerprint: a.Fingerprint, Snapshot: a.Snapshot, Outcome: seat.EndedEventOutcomeFailed, Error: why.Error(), Why: cancelWhy(a, why)}, + a.Module, a.Kept) + return why +} + +// cancelWhy is the hand act's words for a cancel (A-117). +func cancelWhy(a *Act, why error) string { + if why != nil && why.Error() == ErrCanceled { + return a.Why + } + return "" +} + +// filled is the fill of the points directory without the temporary names of acts still writing. +func (e *Engine) filled() int64 { + return dirSize(filepath.Join(e.Dir, "points")) + dirSize(filepath.Join(e.Dir, "aside")) +} + +// take is `snapshot-kept` and the hand verb `take` (§516, §564, §567): one stream at a time under a temporary name, +// renamed whole, its digests on the bus before `ended` is said. +func (e *Engine) take(ctx context.Context, a *Act) error { + dir, err := e.pointDir(a.Module, a.Kept, a.Form, a.Fingerprint) + if err != nil { + return e.endedFailed(a, err) + } + names, err := e.Seats.Streams(ctx, a.Module, a.Kept, a.Fingerprint) + if err != nil || len(names) == 0 { + if err == nil { + err = fmt.Errorf("%s's %s has no bus name at %s", a.Module, a.Kept, a.Fingerprint) + } + return e.endedFailed(a, err) + } + e.setStep(a, "measuring", "") + objs, err := e.Bus.Objects(ctx, names) + if err != nil { + return e.endedFailed(a, err) + } + var need int64 + for _, o := range objs { + need += o.Bytes + } + if fill := e.filled(); fill+need > e.Settings.Cap { + return e.endedFailed(a, fmt.Errorf("%s: the cap is %d bytes, the point needs %d and the directory holds %d", ErrCap, e.Settings.Cap, need, fill)) + } + tmp := filepath.Join(filepath.Dir(dir), ".tmp-"+a.ID) + _ = os.RemoveAll(tmp) + if err := os.MkdirAll(tmp, 0o700); err != nil { + return e.endedFailed(a, err) + } + defer os.RemoveAll(tmp) + p := seat.Point{At: e.now(), CatchUp: a.CatchUp, Fingerprint: a.Fingerprint, Form: a.Form, Kept: a.Kept, Module: a.Module, + Snapshot: strings.TrimPrefix(dir, filepath.Join(e.Dir, "points")+string(filepath.Separator))} + e.setStep(a, "taking", "") + for _, o := range objs { + if !o.Exists { + continue + } + file := filepath.Join(tmp, keys.EscapeName(o.Stream)+".snap") + sum, n, err := e.takeOne(ctx, o.Stream, file) + if err != nil { + return e.endedFailed(a, err) + } + p.Digests = append(p.Digests, seat.Digest{Name: o.Stream, Digest: sum}) + p.Size += n + p.Consumers = append(p.Consumers, o.Consumers...) + if e.filled()+p.Size > e.Settings.Cap { + return e.endedFailed(a, fmt.Errorf("%s: the stream grew while it was taken and passes the cap of %d bytes", ErrCap, e.Settings.Cap)) + } + } + if err := writeJSON(filepath.Join(tmp, pointFile), p); err != nil { + return e.endedFailed(a, err) + } + _ = os.RemoveAll(dir) + if err := os.Rename(tmp, dir); err != nil { + return e.endedFailed(a, err) + } + raw, _ := jsonMarshal(p.Digests) + if err := e.Digests.Put(keys.EscapeName(p.Snapshot), raw); err != nil { + _ = os.RemoveAll(dir) + return e.endedFailed(a, fmt.Errorf("the digests are not on the bus: %w", err)) + } + e.say(seat.EventEnded, seat.EndedEvent{Act: a.ID, At: e.now(), Module: a.Module, Kept: a.Kept, Form: a.Form, + Fingerprint: a.Fingerprint, Snapshot: p.Snapshot, Outcome: seat.EndedEventOutcomeDone, Consumers: p.Consumers, + Digests: p.Digests, Size: p.Size, CatchUp: p.CatchUp}, a.Module, a.Kept) + e.expire(a.ID) + return nil +} + +func (e *Engine) takeOne(ctx context.Context, stream, file string) (string, int64, error) { + f, err := os.Create(file) + if err != nil { + return "", 0, err + } + if err := e.Bus.Take(ctx, stream, f); err != nil { + f.Close() + return "", 0, fmt.Errorf("stream %s: %w", stream, err) + } + if err := f.Close(); err != nil { + return "", 0, err + } + return digestOf(file) +} + +func digestOf(file string) (string, int64, error) { + f, err := os.Open(file) + if err != nil { + return "", 0, err + } + defer f.Close() + h := sha256.New() + n, err := io.Copy(h, f) + return "sha256:" + hex.EncodeToString(h.Sum(nil)), n, err +} + +// Points lists every restore point in the directory, newest first (§517): the one place they are kept. +func (e *Engine) Points() ([]seat.Point, error) { + var out []seat.Point + root := filepath.Join(e.Dir, "points") + err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err + } + if d.IsDir() && strings.HasPrefix(d.Name(), ".") { + return filepath.SkipDir + } + if !d.IsDir() && d.Name() == pointFile { + var p seat.Point + if err := readJSON(path, &p); err != nil { + return fmt.Errorf("%s: %w", path, err) + } + out = append(out, p) + } + return nil + }) + sort.Slice(out, func(i, j int) bool { return out[i].At.After(out[j].At) }) + return out, err +} + +// expire is §551's removal: a point 90 days after its `at` (the setting), never the newest of its kept data, each removal +// said as point-removed and the run as expiry-ended, also when it removes nothing (§560). It runs inside an act. +func (e *Engine) expire(act string) { + pts, err := e.Points() + if err != nil { + e.logf("expiry: %v", err) + } + newest := map[string]string{} + for _, p := range pts { // newest first + k := p.Module + "\x00" + p.Kept + if _, ok := newest[k]; !ok { + newest[k] = p.Snapshot + } + } + cutoff := e.now().AddDate(0, 0, -e.Settings.KeepDays) + removed := int64(0) + for _, p := range pts { + if !p.At.Before(cutoff) || newest[p.Module+"\x00"+p.Kept] == p.Snapshot { + continue + } + if err := os.RemoveAll(filepath.Join(e.Dir, "points", p.Snapshot)); err != nil { + e.logf("the point %s is not removed: %v", p.Snapshot, err) + continue + } + _ = e.Digests.Delete(keys.EscapeName(p.Snapshot)) + if err := e.Seats.HandAct(Module, "", "point-removed "+p.Snapshot, "older than the keep days"); err != nil { + e.logf("the removal of %s is not in the hand-act log: %v", p.Snapshot, err) + } + e.say(seat.EventPointRemoved, seat.PointRemovedEvent{Act: act, At: e.now(), Module: p.Module, Kept: p.Kept, Form: p.Form, + Fingerprint: p.Fingerprint, Snapshot: p.Snapshot}, p.Module, p.Kept) + removed++ + } + e.say(seat.EventExpiryEnded, seat.ExpiryEndedEvent{Act: act, At: e.now(), Removed: removed}) +} diff --git a/modules/mesh-recovery/internal/recovery/points_test.go b/modules/mesh-recovery/internal/recovery/points_test.go new file mode 100644 index 000000000..adef20c49 --- /dev/null +++ b/modules/mesh-recovery/internal/recovery/points_test.go @@ -0,0 +1,121 @@ +package recovery_test + +import ( + "strings" + "testing" + "time" + + seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" + + "mesh-recovery/internal/recovery" +) + +func takeAct() recovery.Act { + return recovery.Act{Kind: recovery.KindTake, Module: "app", Kept: "notes", Form: 2, Fingerprint: "fp1", Why: "form change", Caller: "node.engine"} +} + +// A point is taken stream by stream under a temporary name, renamed whole, its digests put on the bus before `ended` is +// said under the act's id with the point whole, and the act is written in the hand-act log with the caller the bus proved +// (design 47 G §516, §554, §560, §567). +func TestATakenPointIsListedWithItsDigestsAndEndedUnderItsAct(t *testing.T) { + r := newRig(t) + ans := r.do(takeAct()) + if ans.Accepted == "" { + t.Fatalf("answer %+v", ans) + } + ended := r.s.named(seat.EventEnded) + if len(ended) != 1 { + t.Fatalf("events %+v", r.s.events) + } + ev := ended[0].Body.(seat.EndedEvent) + if ev.Act != ans.Accepted || ev.Outcome != "done" || len(ev.Digests) != 2 || ev.Size == 0 || ev.Snapshot != "app/notes/2-fp1" { + t.Errorf("ended %+v", ev) + } + if got := strings.Join(ended[0].Parts, "."); got != "app.notes" { + t.Errorf("parts %q", got) + } + if len(ev.Consumers) != 1 || ev.Consumers[0].Sequence != 7 { + t.Errorf("consumers %+v", ev.Consumers) + } + pts, err := r.e.Points() + if err != nil || len(pts) != 1 || pts[0].Snapshot != ev.Snapshot { + t.Fatalf("points %+v %v", pts, err) + } + if _, ok, _ := r.d.Get("app/notes/2-fp1"); !ok { + t.Errorf("no digests on the bus: %v", r.d.m) + } + if len(r.s.handActs) != 1 || !strings.HasPrefix(r.s.handActs[0], "node.engine||take app notes|form change") { + t.Errorf("hand acts %v", r.s.handActs) + } +} + +// One act at a time and no queue: a second ask while one runs is answered busy with the running act (§554). +func TestASecondAskWhileOneRunsIsAnsweredBusy(t *testing.T) { + r := newRig(t) + gate := make(chan struct{}) + r.e.Dump = nil + r.e.Bus = &gatedBus{fakeBus: r.b, gate: gate} + first := r.e.Accept(takeAct()) + second := r.e.Accept(takeAct()) + close(gate) + r.e.Wait() + if first.Accepted == "" || second.Busy != first.Accepted || second.Accepted != "" || second.Error != "" { + t.Errorf("first %+v second %+v", first, second) + } +} + +// An act whose hand-act log write fails does nothing and answers an error naming the log (§554). +func TestAnActWhoseHandActLogWriteFailsDoesNothing(t *testing.T) { + r := newRig(t) + r.s.handFail = errTest + ans := r.do(takeAct()) + if !strings.Contains(ans.Error, "hand-act log") || ans.Accepted != "" || len(r.s.events) != 0 { + t.Errorf("answer %+v events %+v", ans, r.s.events) + } + if pts, _ := r.e.Points(); len(pts) != 0 { + t.Errorf("points %+v", pts) + } +} + +// An act that would pass the cap ends failed naming it, before anything is written (§564). +func TestAnActThatWouldPassTheCapEndsFailedBeforeItWritesAnything(t *testing.T) { + r := newRig(t) + r.e.Settings.Cap = 4 + r.do(takeAct()) + ev := r.s.named(seat.EventEnded)[0].Body.(seat.EndedEvent) + if ev.Outcome != "failed" || !strings.HasPrefix(ev.Error, recovery.ErrCap) { + t.Errorf("ended %+v", ev) + } + if pts, _ := r.e.Points(); len(pts) != 0 || len(r.d.m) != 0 { + t.Errorf("points %+v digests %v", pts, r.d.m) + } +} + +// A point older than the keep days is removed and said, the newest of its kept data stays however old, and the run says +// expiry-ended with the count, 0 included (§551, §560). +func TestExpiryRemovesOldPointsButNeverTheNewest(t *testing.T) { + r := newRig(t) + r.t = time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC) + r.do(takeAct()) + a := takeAct() + a.Fingerprint, a.Form = "fp2", 3 + r.t = time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC) + r.do(a) + r.s.events = nil + r.t = time.Date(2026, 11, 12, 0, 0, 0, 0, time.UTC) + r.do(recovery.Act{Kind: recovery.KindExpire, Why: "by hand", Caller: "tools"}) + pts, _ := r.e.Points() + if len(pts) != 1 || pts[0].Fingerprint != "fp2" { + t.Fatalf("points %+v", pts) + } + rem := r.s.named(seat.EventPointRemoved) + if len(rem) != 1 || rem[0].Body.(seat.PointRemovedEvent).Fingerprint != "fp1" { + t.Errorf("removed %+v", rem) + } + if ex := r.s.named(seat.EventExpiryEnded); len(ex) != 1 || ex[0].Body.(seat.ExpiryEndedEvent).Removed != 1 { + t.Errorf("expiry %+v", ex) + } + if _, ok, _ := r.d.Get("app/notes/2-fp1"); ok { + t.Errorf("its digests stay on the bus") + } +} diff --git a/modules/mesh-recovery/internal/recovery/recovery.go b/modules/mesh-recovery/internal/recovery/recovery.go new file mode 100644 index 000000000..153a18f6d --- /dev/null +++ b/modules/mesh-recovery/internal/recovery/recovery.go @@ -0,0 +1,270 @@ +// Package recovery is the recovery module's engine (design 47 G §554, §516): it takes, sets aside and restores a +// module's kept data on the live bus, one act at a time, each accepted act written in its directory before the caller is +// answered, the result said as an event under the act's id (§560). Every dependence on the bus, the seat list, the +// hand-act log and the backup holder is an interface here, so the one whole test runs the engine against the real bus +// and a unit test against a fake. +package recovery + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sync" + "time" + + seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// Module and Seat are the module's name and the seat it claims. +const ( + Module = "mesh-recovery" + Seat = seat.Seat +) + +// The kinds of act, one per verb that starts one. +const ( + KindTake = "take" + KindRestore = "restore" + KindAside = "aside" + KindDiscardAside = "discard-aside" + KindRestoreAside = "restore-aside" + KindExpire = "expire-points" +) + +// Errors an act ends with: the words the node agent's table reads (§560). +const ( + ErrCap = "cap" + ErrStopped = "stopped" + ErrCanceled = "cancelled" + ErrMissing = "restore point missing" + ErrDamaged = "restore point damaged" +) + +// Object is one bus object of a kept data: a stream, by its name, with what the bus states of it. +type Object struct { + Stream string + Exists bool + Bytes int64 + Consumers []seat.Consumer +} + +// Bus is the bus as the module's account sees it (§554): a stream's snapshot and restore, its deletion, the +// durable consumers' positions. +type Bus interface { + // Objects states each stream of names: whether it exists, its bytes, its durable consumers' acknowledged sequences. + Objects(ctx context.Context, streams []string) ([]Object, error) + // Take writes one stream's snapshot file to w (ADR 0235 §10's pacing and §11's bounds are the implementation's). + Take(ctx context.Context, stream string, w io.Writer) error + // Delete removes a stream; one that is not there is no error. + Delete(ctx context.Context, stream string) error + // Restore puts the stream of a snapshot file back under its own name, where no stream of that name exists. + Restore(ctx context.Context, file string) error + // SetConsumers creates each durable consumer at its recorded sequence. + SetConsumers(ctx context.Context, cs []seat.Consumer) error +} + +// Seats is what the engine asks of the rest of the mesh. +type Seats interface { + // Say publishes one event of the seat under its subject parts (§560); an event of no parts has none. + Say(event string, body any, parts ...string) error + // HandAct writes the act in the hand-act log first (§554): the account the bus proved, who asked, what, why. + HandAct(caller, asker, what, why string) error + // Streams are the bus names of a module's kept data at a fingerprint, from its catalog graph entry (§554 Bus names). + Streams(ctx context.Context, module, kept, fingerprint string) ([]string, error) + // Backup brings the points directory of the module's item back beside its path through the backup holder and + // answers where (§516); an error when no repository snapshot holds it. + Backup(ctx context.Context, module string) (dir string, err error) +} + +// Digests keeps each point's digests on the bus, apart from its bytes (§567): the state digests. +type Digests interface { + Put(key string, v []byte) error + Get(key string) ([]byte, bool, error) + Delete(key string) error +} + +// Dump is what the act waits on while the bus's item is dumped (§563). +type Dump interface { + // Wait returns once no dump of the bus's item runs; it calls waiting(on) once if it has to wait. + Wait(ctx context.Context, waiting func(on string)) error +} + +// Settings are the bounds a whole test sets in seconds (§557, §564, §633). +type Settings struct { + Cap int64 + KeepDays int +} + +// Engine is the recovery module's one engine. +type Engine struct { + Dir string + Bus Bus + Seats Seats + Digests Digests + Dump Dump + Settings Settings + Now func() time.Time + Logf func(string, ...any) + + mu sync.Mutex + running *Act + cancel context.CancelFunc + wg sync.WaitGroup + seq int +} + +// Act is one accepted act, written in the directory before its caller is answered (§554). +type Act struct { + ID string `json:"act"` + Kind string `json:"kind"` + Module string `json:"module,omitempty"` + Kept string `json:"kept,omitempty"` + Form int64 `json:"form,omitempty"` + Fingerprint string `json:"fingerprint,omitempty"` + Snapshot string `json:"snapshot,omitempty"` + AsideID string `json:"aside-id,omitempty"` + FromBackup bool `json:"from-backup,omitempty"` + CatchUp *seat.CatchUp `json:"catch-up,omitempty"` + Why string `json:"why"` + Caller string `json:"caller"` + Asker string `json:"asker,omitempty"` + Since time.Time `json:"since"` + Step string `json:"step,omitempty"` + WaitsOn string `json:"waits-on,omitempty"` + Ended bool `json:"ended,omitempty"` +} + +// Answer is the one form every acting verb answers (§691): exactly one field set. +type Answer struct { + Accepted string `json:"accepted,omitempty"` + Busy string `json:"busy,omitempty"` + Error string `json:"error,omitempty"` +} + +func (e *Engine) now() time.Time { + if e.Now != nil { + return e.Now().UTC() + } + return time.Now().UTC() +} + +func (e *Engine) logf(f string, a ...any) { + if e.Logf != nil { + e.Logf(f, a...) + } +} + +// Tokens are escaped as a key's name is (§73); a token that is empty, `.` or `..` is refused. +func token(s string) (string, error) { + if s == "" || s == "." || s == ".." { + return "", fmt.Errorf("%q is not a name", s) + } + return keys.EscapeName(s), nil +} + +func (e *Engine) pointDir(module, kept string, form int64, fp string) (string, error) { + m, err := token(module) + if err != nil { + return "", err + } + k, err := token(kept) + if err != nil { + return "", err + } + f, err := token(fp) + if err != nil { + return "", err + } + return filepath.Join(e.Dir, "points", m, k, fmt.Sprintf("%d-%s", form, f)), nil +} + +func writeJSON(path string, v any) error { + raw, err := json.MarshalIndent(v, "", " ") + if err != nil { + return err + } + tmp := path + ".partial" + if err := os.WriteFile(tmp, raw, 0o600); err != nil { + return err + } + return os.Rename(tmp, path) +} + +func readJSON(path string, v any) error { + raw, err := os.ReadFile(path) + if err != nil { + return err + } + return json.Unmarshal(raw, v) +} + +func dirSize(dir string) int64 { + var n int64 + _ = filepath.WalkDir(dir, func(_ string, d os.DirEntry, err error) error { + if err == nil && !d.IsDir() { + if info, err := d.Info(); err == nil { + n += info.Size() + } + } + return nil + }) + return n +} + +var errBusy = errors.New("busy") + +// Accept is the one door of every acting verb (§554, §682): it writes the act in the hand-act log first (a failed +// write does nothing and answers an error naming the log), then answers busy with the running act's id, or writes the +// act in its directory and answers accepted; the work runs on, and its result is the event. +func (e *Engine) Accept(a Act) Answer { + e.mu.Lock() + defer e.mu.Unlock() + if e.running != nil { + return Answer{Busy: e.running.ID} + } + if err := e.Seats.HandAct(a.Caller, a.Asker, a.Kind+" "+a.Module+" "+a.Kept, a.Why); err != nil { + return Answer{Error: "the hand-act log was not written, so nothing was done: " + err.Error()} + } + e.seq++ + a.ID = fmt.Sprintf("act-%s-%d", e.now().Format("20060102T150405Z"), e.seq) + a.Since = e.now() + if err := os.MkdirAll(filepath.Join(e.Dir, "acts"), 0o700); err != nil { + return Answer{Error: err.Error()} + } + if err := writeJSON(filepath.Join(e.Dir, "acts", a.ID+".json"), a); err != nil { + return Answer{Error: "the act cannot be written: " + err.Error()} + } + ctx, cancel := context.WithCancel(context.Background()) + e.running, e.cancel = &a, cancel + e.wg.Add(1) + go func() { + defer e.wg.Done() + e.run(ctx, &a) + e.mu.Lock() + e.running, e.cancel = nil, nil + e.mu.Unlock() + cancel() + }() + return Answer{Accepted: a.ID} +} + +// Wait returns once the running act, if any, has ended (a whole test's and a stop's join). +func (e *Engine) Wait() { e.wg.Wait() } + +func (e *Engine) setStep(a *Act, step, waitsOn string) { + e.mu.Lock() + a.Step, a.WaitsOn = step, waitsOn + e.mu.Unlock() +} + +func (e *Engine) markEnded(a *Act) { + a.Ended = true + _ = writeJSON(filepath.Join(e.Dir, "acts", a.ID+".json"), a) +} + +func jsonMarshal(v any) ([]byte, error) { return json.Marshal(v) } From 0ebabf7174953ca9de5338aea286136dc34f6a37 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 01:16:29 +0100 Subject: [PATCH 03/17] mesh-recovery: restore, set aside, put an aside back, cancel, retry, kept-points, the repair at a start MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §516, §554, §560, §562, §563, §564, §566, §567, §682, §691, A-117 restore says restore-started, waits for the dump, finds the point (or has the backup holder bring it back), checks its digests against the bus, sets the live data aside whole and checked, removes it, restores the point and its consumers, and puts the aside back when a step after the removal fails; cancel ends a waiting act failed/cancelled with the hand act's words; at a start it clears temporary points, puts an aside back and ends a cut-off act failed/stopped. Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../internal/recovery/control.go | 164 +++++++++ .../internal/recovery/fakes_test.go | 34 +- .../mesh-recovery/internal/recovery/points.go | 7 + .../internal/recovery/recovery.go | 6 +- .../internal/recovery/restore.go | 335 ++++++++++++++++++ .../internal/recovery/restore_test.go | 234 ++++++++++++ 6 files changed, 763 insertions(+), 17 deletions(-) create mode 100644 modules/mesh-recovery/internal/recovery/control.go create mode 100644 modules/mesh-recovery/internal/recovery/restore.go create mode 100644 modules/mesh-recovery/internal/recovery/restore_test.go diff --git a/modules/mesh-recovery/internal/recovery/control.go b/modules/mesh-recovery/internal/recovery/control.go new file mode 100644 index 000000000..868fd9f9e --- /dev/null +++ b/modules/mesh-recovery/internal/recovery/control.go @@ -0,0 +1,164 @@ +package recovery + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" +) + +// Cancel is the verb `cancel` (§566, §682): it ends an accepted act that has not begun its work (one waiting on the +// dump, §563) with its ending event, outcome failed and error cancelled, the hand act's words as `why` (A-117); it +// refuses an act that has begun, naming its step, since a restore cut off after its removal is put back only by its own +// failure path (§516). It is written in the hand-act log first and answers in §691's one form. +func (e *Engine) Cancel(act, why, caller, asker string) Answer { + if err := e.Seats.HandAct(caller, asker, "cancel "+act, why); err != nil { + return Answer{Error: "the hand-act log was not written, so nothing was done: " + err.Error()} + } + e.mu.Lock() + defer e.mu.Unlock() + switch { + case e.running == nil || e.running.ID != act: + return Answer{Error: "the act " + act + " is not running"} + case e.running.Step != "waiting": + return Answer{Error: fmt.Sprintf("the act %s has begun (its step: %q) and is put back only by its own failure path", act, e.running.Step)} + } + e.why = why + e.cancel() + return Answer{Accepted: act} +} + +// Retry is the verb `retry` (§682): it asks again the last failed act of a module and kept data, with its fields, under +// an act of its own; `nothing to retry` when its last act did not fail. +func (e *Engine) Retry(module, kept, why, caller, asker string) Answer { + acts := e.readActs() + var last *Act + for i := range acts { + a := acts[i] + if a.Module == module && a.Kept == kept && a.Ended && a.Kind != KindExpire && (last == nil || !a.Since.Before(last.Since)) { + last = &a + } + } + if last == nil || last.Error == "" { + return Answer{Error: "nothing to retry"} + } + again := *last + again.ID, again.Ended, again.Error, again.Step, again.WaitsOn = "", false, "", "", "" + again.Why, again.Caller, again.Asker = why, caller, asker + return e.Accept(again) +} + +func (e *Engine) readActs() []Act { + var out []Act + files, _ := filepath.Glob(filepath.Join(e.Dir, "acts", "*.json")) + for _, f := range files { + var a Act + if readJSON(f, &a) == nil { + out = append(out, a) + } + } + return out +} + +// KeptPoints is the verb `kept-points` (§517): every restore point and aside with their sizes, narrowed to a module and a +// kept data, and the act that runs, with since when and what it waits on. +func (e *Engine) KeptPoints(module, kept string) (seat.KeptPointsAnswer, error) { + pts, err := e.Points() + if err != nil { + return seat.KeptPointsAnswer{}, err + } + out := seat.KeptPointsAnswer{Points: []seat.Point{}, Asides: []seat.Aside{}} + for _, p := range pts { + if (module == "" || p.Module == module) && (kept == "" || p.Kept == kept) { + out.Points = append(out.Points, p) + } + } + _, recs, _ := e.Asides() + for _, r := range recs { + if (module == "" || r.Module == module) && (kept == "" || r.Kept == kept) { + out.Asides = append(out.Asides, r.aside()) + } + } + e.mu.Lock() + if e.running != nil { + out.Running = &seat.Running{Act: e.running.ID, Step: e.running.Step, Since: e.running.Since, WaitsOn: e.running.WaitsOn} + } + e.mu.Unlock() + return out, nil +} + +// Unrestored are the asides whose record ends at `removed` that no running act holds (§554 health, §561): the live data +// of those names is gone and only the aside holds it. +func (e *Engine) Unrestored() []seat.Aside { + _, recs, _ := e.Asides() + e.mu.Lock() + defer e.mu.Unlock() + var out []seat.Aside + for _, r := range recs { + held := e.running != nil && e.running.Module == r.Module && e.running.Kept == r.Kept + if r.Step == stepRemoved && !held { + out = append(out, r.aside()) + } + } + return out +} + +// Start repairs what a restart cut off (§516, §564, §560): points under a temporary name are removed, an aside whose +// record ends at `set aside` (the live data never touched) is removed, an aside that ends at `removed` is put back under +// the live names, and every accepted act that never ended ends failed with `stopped`, so its caller asks again. It +// answers the asides it could not put back (the finding aside-unrestored, §561). +func (e *Engine) Start(ctx context.Context) []seat.Aside { + _ = filepath.WalkDir(filepath.Join(e.Dir, "points"), func(path string, d os.DirEntry, err error) error { + if err == nil && d.IsDir() && strings.HasPrefix(d.Name(), ".tmp-") { + _ = os.RemoveAll(path) + return filepath.SkipDir + } + return nil + }) + _, recs, _ := e.Asides() + for i := range recs { + r := &recs[i] + switch r.Step { + case stepSetAside: + _ = os.RemoveAll(e.asideDirOf(r)) + case stepRemoved: + if live, err := e.Bus.Objects(ctx, r.Streams); err == nil && !anyExists(live) { + if err := e.restoreFromAside(ctx, r, false); err != nil { + e.logf("the aside %s is not put back: %v", r.ID, err) + } + } + } + } + acts := e.readActs() + sort.Slice(acts, func(i, j int) bool { return acts[i].Since.Before(acts[j].Since) }) + for i := range acts { + a := acts[i] + if a.Ended { + continue + } + stopped := errors.New(ErrStopped) + switch a.Kind { + case KindTake: + _ = e.endedFailed(&a, stopped) + case KindRestore, KindRestoreAside: + _ = e.endedRestored(&a, stopped, nil, false) + } + a.Error = ErrStopped + e.markEnded(&a) + } + return e.Unrestored() +} + +func anyExists(os []Object) bool { + for _, o := range os { + if o.Exists { + return true + } + } + return false +} diff --git a/modules/mesh-recovery/internal/recovery/fakes_test.go b/modules/mesh-recovery/internal/recovery/fakes_test.go index 58ccdbe2a..a38f086f8 100644 --- a/modules/mesh-recovery/internal/recovery/fakes_test.go +++ b/modules/mesh-recovery/internal/recovery/fakes_test.go @@ -17,11 +17,11 @@ import ( // fakeBus holds streams as strings; a snapshot file is "snap::". type fakeBus struct { - mu sync.Mutex - streams map[string]string - consumers map[string][]seat.Consumer - setBack []seat.Consumer - failRestoreOnce bool + mu sync.Mutex + streams map[string]string + consumers map[string][]seat.Consumer + setBack []seat.Consumer + failRestores int } func (b *fakeBus) Objects(_ context.Context, names []string) ([]recovery.Object, error) { @@ -60,10 +60,9 @@ func (b *fakeBus) Restore(_ context.Context, file string) error { } b.mu.Lock() defer b.mu.Unlock() - if b.failRestoreOnce && len(raw) > 0 && b.streams != nil { - if string(raw[:5]) == "snap:" && !b.restoredOnce(raw) { - return errors.New("the server refused the archive") - } + if b.failRestores > 0 { + b.failRestores-- + return errors.New("the server refused the archive") } var name, content string rest := string(raw[5:]) @@ -80,13 +79,6 @@ func (b *fakeBus) Restore(_ context.Context, file string) error { return nil } -var once sync.Map - -func (b *fakeBus) restoredOnce(raw []byte) bool { - _, loaded := once.LoadOrStore(string(raw), true) - return loaded -} - func (b *fakeBus) SetConsumers(_ context.Context, cs []seat.Consumer) error { b.mu.Lock() defer b.mu.Unlock() @@ -177,6 +169,16 @@ type noDump struct{} func (noDump) Wait(context.Context, func(string)) error { return nil } +// holdDump waits until ctx ends, saying it waits. +type holdDump struct{ waiting chan string } + +func (h holdDump) Wait(ctx context.Context, waiting func(string)) error { + waiting("the dump of the bus") + h.waiting <- "waiting" + <-ctx.Done() + return ctx.Err() +} + type rig struct { e *recovery.Engine b *fakeBus diff --git a/modules/mesh-recovery/internal/recovery/points.go b/modules/mesh-recovery/internal/recovery/points.go index beb3688b1..9256efd47 100644 --- a/modules/mesh-recovery/internal/recovery/points.go +++ b/modules/mesh-recovery/internal/recovery/points.go @@ -26,11 +26,18 @@ func (e *Engine) run(ctx context.Context, a *Act) { err = e.take(ctx, a) case KindExpire: e.expire(a.ID) + case KindRestore: + err = e.restore(ctx, a) + case KindRestoreAside: + err = e.restoreAside(ctx, a) + case KindDiscardAside: + err = e.discardAside(a) default: err = e.endedFailed(a, fmt.Errorf("no act of kind %q", a.Kind)) } if err != nil { e.logf("act %s ended failed: %v", a.ID, err) + a.Error = err.Error() } e.markEnded(a) } diff --git a/modules/mesh-recovery/internal/recovery/recovery.go b/modules/mesh-recovery/internal/recovery/recovery.go index 153a18f6d..bdd0d05d8 100644 --- a/modules/mesh-recovery/internal/recovery/recovery.go +++ b/modules/mesh-recovery/internal/recovery/recovery.go @@ -116,6 +116,7 @@ type Engine struct { cancel context.CancelFunc wg sync.WaitGroup seq int + why string // the hand act words of a cancel (A-117) } // Act is one accepted act, written in the directory before its caller is answered (§554). @@ -137,6 +138,7 @@ type Act struct { Step string `json:"step,omitempty"` WaitsOn string `json:"waits-on,omitempty"` Ended bool `json:"ended,omitempty"` + Error string `json:"error,omitempty"` } // Answer is the one form every acting verb answers (§691): exactly one field set. @@ -231,7 +233,7 @@ func (e *Engine) Accept(a Act) Answer { return Answer{Error: "the hand-act log was not written, so nothing was done: " + err.Error()} } e.seq++ - a.ID = fmt.Sprintf("act-%s-%d", e.now().Format("20060102T150405Z"), e.seq) + a.ID = fmt.Sprintf("act-%s-%06d", e.now().Format("20060102T150405Z"), e.seq) a.Since = e.now() if err := os.MkdirAll(filepath.Join(e.Dir, "acts"), 0o700); err != nil { return Answer{Error: err.Error()} @@ -268,3 +270,5 @@ func (e *Engine) markEnded(a *Act) { } func jsonMarshal(v any) ([]byte, error) { return json.Marshal(v) } + +func unmarshal(raw []byte, v any) error { return json.Unmarshal(raw, v) } diff --git a/modules/mesh-recovery/internal/recovery/restore.go b/modules/mesh-recovery/internal/recovery/restore.go new file mode 100644 index 000000000..cfec63109 --- /dev/null +++ b/modules/mesh-recovery/internal/recovery/restore.go @@ -0,0 +1,335 @@ +package recovery + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +const asideFile = "aside.json" + +// The steps an aside's record says, in their order (§516): a restore that fails after `removed` puts the aside back. +const ( + stepSetAside = "set aside" + stepRemoved = "removed" + stepRestored = "restored" +) + +// asideRecord is the record in an aside's directory (§516); nothing of an aside is on the bus. +type asideRecord struct { + ID string `json:"id"` + Module string `json:"module"` + Kept string `json:"kept"` + Streams []string `json:"streams"` + Consumers []seat.Consumer `json:"consumers,omitempty"` + Digests []seat.Digest `json:"digests"` + Size int64 `json:"size"` + Step string `json:"step"` +} + +func (r asideRecord) aside() seat.Aside { + return seat.Aside{ID: r.ID, Name: strings.Join(r.Streams, ","), Size: r.Size} +} + +func (e *Engine) asideRoot(module, kept string) (string, error) { + m, err := token(module) + if err != nil { + return "", err + } + k, err := token(kept) + if err != nil { + return "", err + } + return filepath.Join(e.Dir, "aside", m, k), nil +} + +// endedRestored says the end of a restore, or of a restore-aside, under the act's id (§560). +func (e *Engine) endedRestored(a *Act, outcomeErr error, aside *seat.Aside, missing bool) error { + ev := seat.RestoredEvent{Act: a.ID, At: e.now(), Module: a.Module, Kept: a.Kept, Form: a.Form, Fingerprint: a.Fingerprint, + Snapshot: a.Snapshot, Outcome: seat.RestoredEventOutcomeDone, Aside: aside, Missing: missing} + if outcomeErr != nil { + ev.Outcome, ev.Error, ev.Why = seat.RestoredEventOutcomeFailed, outcomeErr.Error(), e.cancelledWhy(outcomeErr) + } + e.say(seat.EventRestored, ev, a.Module, a.Kept) + return outcomeErr +} + +func (e *Engine) cancelledWhy(err error) string { + e.mu.Lock() + defer e.mu.Unlock() + if err != nil && err.Error() == ErrCanceled { + return e.why + } + return "" +} + +// restore is `restore-kept` and the hand verb `restore` with a point (§516, §554, §563, §567): it says restore-started, +// waits for the dump of the bus's item, finds the point (or has the backup holder bring it back), checks its digests +// against the bus, sets the live data aside whole and checked, removes it, restores the point under its names and +// sets each consumer to its recorded sequence; a step after the removal that fails puts the aside back. +func (e *Engine) restore(ctx context.Context, a *Act) error { + e.say(seat.EventRestoreStarted, seat.RestoreStartedEvent{Act: a.ID, At: e.now(), Module: a.Module, Kept: a.Kept, Form: a.Form, + Fingerprint: a.Fingerprint, Snapshot: a.Snapshot}, a.Module, a.Kept) + if e.Dump != nil { + if err := e.Dump.Wait(ctx, func(on string) { e.setStep(a, "waiting", on) }); err != nil { + if ctx.Err() != nil { + err = errors.New(ErrCanceled) + } + return e.endedRestored(a, err, nil, false) + } + } + e.setStep(a, "finding the point", "") + pointPath := filepath.Join(e.Dir, "points", filepath.FromSlash(a.Snapshot)) + if _, err := os.Stat(filepath.Join(pointPath, pointFile)); err != nil { + if !a.FromBackup { + return e.endedRestored(a, errors.New(ErrMissing), nil, true) + } + dir, err := e.Seats.Backup(ctx, Module) + if err != nil { + return e.endedRestored(a, errors.New(ErrMissing), nil, true) + } + defer os.RemoveAll(dir) + pointPath = filepath.Join(dir, filepath.FromSlash(a.Snapshot)) + if _, err := os.Stat(filepath.Join(pointPath, pointFile)); err != nil { + return e.endedRestored(a, errors.New(ErrMissing), nil, true) + } + } + var p seat.Point + if err := readJSON(filepath.Join(pointPath, pointFile), &p); err != nil { + return e.endedRestored(a, fmt.Errorf("%s: %w", ErrDamaged, err), nil, false) + } + if err := e.checkDigests(pointPath, a.Snapshot, p); err != nil { + return e.endedRestored(a, err, nil, false) + } + names := make([]string, 0, len(p.Digests)) + for _, d := range p.Digests { + names = append(names, d.Name) + } + rec, err := e.setAside(ctx, a, names, true) + if err != nil { + return e.endedRestored(a, err, nil, false) + } + // Past the removal, a failure puts the aside back (§516). + e.setStep(a, "restoring", "") + if err := e.putBack(ctx, pointPath, p.Digests, p.Consumers); err != nil { + if back := e.restoreFromAside(ctx, rec, true); back != nil { + err = fmt.Errorf("%w; and the aside is not put back: %v", err, back) + } + return e.endedRestored(a, err, asidePtr(rec), false) + } + rec.Step = stepRestored + _ = writeJSON(filepath.Join(e.asideDirOf(rec), asideFile), rec) + return e.endedRestored(a, nil, asidePtr(rec), false) +} + +func asidePtr(r *asideRecord) *seat.Aside { + if r == nil { + return nil + } + x := r.aside() + return &x +} + +func (e *Engine) asideDirOf(r *asideRecord) string { + root, _ := e.asideRoot(r.Module, r.Kept) + return filepath.Join(root, r.ID) +} + +// checkDigests computes the digest of the bytes about to be restored and compares it with the key on the bus; the +// record in the directory is never the reference (§567). +func (e *Engine) checkDigests(pointPath, snapshot string, p seat.Point) error { + raw, ok, err := e.Digests.Get(keys.EscapeName(snapshot)) + if err != nil { + return fmt.Errorf("the digests of %s cannot be read: %w", snapshot, err) + } + if !ok { + return fmt.Errorf("%s: %s has no digests on the bus", ErrDamaged, snapshot) + } + var want []seat.Digest + if err := unmarshal(raw, &want); err != nil { + return fmt.Errorf("%s: %s: %w", ErrDamaged, snapshot, err) + } + if len(want) != len(p.Digests) { + return fmt.Errorf("%s: %s holds %d object(s), the bus says %d", ErrDamaged, snapshot, len(p.Digests), len(want)) + } + for _, w := range want { + got, _, err := digestOf(filepath.Join(pointPath, keys.EscapeName(w.Name)+".snap")) + if err != nil || got != w.Digest { + return fmt.Errorf("%s: %s, bus object %s", ErrDamaged, snapshot, w.Name) + } + } + return nil +} + +// setAside takes a snapshot of each live object into aside///, reads it back to check it, and only then +// (when remove) removes the live objects, each step written first in the record (§516). Past the cap it refuses before +// any object is removed (§564). +func (e *Engine) setAside(ctx context.Context, a *Act, names []string, remove bool) (*asideRecord, error) { + e.setStep(a, "setting aside", "") + objs, err := e.Bus.Objects(ctx, names) + if err != nil { + return nil, err + } + var need int64 + for _, o := range objs { + need += o.Bytes + } + if fill := e.filled(); fill+need > e.Settings.Cap { + return nil, fmt.Errorf("%s: the cap is %d bytes, the aside needs %d and the directory holds %d", ErrCap, e.Settings.Cap, need, fill) + } + root, err := e.asideRoot(a.Module, a.Kept) + if err != nil { + return nil, err + } + id := e.now().Format("20060102T150405Z") + for n := 2; ; n++ { + if _, err := os.Stat(filepath.Join(root, id)); err != nil { + break + } + id = fmt.Sprintf("%s-%d", e.now().Format("20060102T150405Z"), n) + } + dir := filepath.Join(root, id) + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, err + } + rec := &asideRecord{ID: id, Module: a.Module, Kept: a.Kept, Step: stepSetAside} + fail := func(err error) (*asideRecord, error) { _ = os.RemoveAll(dir); return nil, err } + for _, o := range objs { + if !o.Exists { + continue + } + sum, n, err := e.takeOne(ctx, o.Stream, filepath.Join(dir, keys.EscapeName(o.Stream)+".snap")) + if err != nil { + return fail(err) + } + if again, _, err := digestOf(filepath.Join(dir, keys.EscapeName(o.Stream)+".snap")); err != nil || again != sum { + return fail(fmt.Errorf("the aside of %s does not read back whole", o.Stream)) + } + rec.Streams = append(rec.Streams, o.Stream) + rec.Digests = append(rec.Digests, seat.Digest{Name: o.Stream, Digest: sum}) + rec.Consumers = append(rec.Consumers, o.Consumers...) + rec.Size += n + } + if err := writeJSON(filepath.Join(dir, asideFile), rec); err != nil { + return fail(err) + } + if !remove { + return rec, nil + } + rec.Step = stepRemoved + if err := writeJSON(filepath.Join(dir, asideFile), rec); err != nil { + return fail(err) + } + for _, s := range rec.Streams { + if err := e.Bus.Delete(ctx, s); err != nil { + if back := e.restoreFromAside(ctx, rec, false); back != nil { + return rec, fmt.Errorf("%w; and the aside is not put back: %v", err, back) + } + return rec, err + } + } + return rec, nil +} + +// putBack restores each stream of a point under its own name and sets each consumer to its recorded sequence. +func (e *Engine) putBack(ctx context.Context, dir string, ds []seat.Digest, cs []seat.Consumer) error { + for _, d := range ds { + if err := e.Bus.Restore(ctx, filepath.Join(dir, keys.EscapeName(d.Name)+".snap")); err != nil { + return fmt.Errorf("stream %s: %w", d.Name, err) + } + } + return e.Bus.SetConsumers(ctx, cs) +} + +// restoreFromAside puts an aside back under the live names: any partly restored stream is removed first. +func (e *Engine) restoreFromAside(ctx context.Context, rec *asideRecord, clear bool) error { + if clear { + for _, s := range rec.Streams { + if err := e.Bus.Delete(ctx, s); err != nil { + return err + } + } + } + if err := e.putBack(ctx, e.asideDirOf(rec), rec.Digests, rec.Consumers); err != nil { + return err + } + rec.Step = stepRestored + return writeJSON(filepath.Join(e.asideDirOf(rec), asideFile), rec) +} + +func (e *Engine) readAside(module, kept, id string) (*asideRecord, error) { + root, err := e.asideRoot(module, kept) + if err != nil { + return nil, err + } + if id == "" || id == "." || id == ".." || strings.ContainsAny(id, "/") { + return nil, fmt.Errorf("%q is not an aside", id) + } + var rec asideRecord + if err := readJSON(filepath.Join(root, id, asideFile), &rec); err != nil { + return nil, fmt.Errorf("no aside %s of %s's %s", id, module, kept) + } + return &rec, nil +} + +// restoreAside is `restore-aside` (§566): it puts an aside whose record ends at `removed` back under the live names it +// was taken from, refusing while a live object of those names exists, and ends with `restored`. +func (e *Engine) restoreAside(ctx context.Context, a *Act) error { + rec, err := e.readAside(a.Module, a.Kept, a.AsideID) + if err != nil { + return e.endedRestored(a, err, nil, false) + } + objs, err := e.Bus.Objects(ctx, rec.Streams) + if err != nil { + return e.endedRestored(a, err, nil, false) + } + for _, o := range objs { + if o.Exists { + return e.endedRestored(a, fmt.Errorf("the live stream %s exists; the aside %s is not put back over it", o.Stream, rec.ID), asidePtr(rec), false) + } + } + if err := e.restoreFromAside(ctx, rec, false); err != nil { + return e.endedRestored(a, err, asidePtr(rec), false) + } + return e.endedRestored(a, nil, asidePtr(rec), false) +} + +// discardAside is `discard-aside` (§554): it removes that directory and nothing else, an id holding `.`, `..` or `/` +// refused; it ends with `aside-discarded`. +func (e *Engine) discardAside(a *Act) error { + rec, err := e.readAside(a.Module, a.Kept, a.AsideID) + if err != nil { + return err + } + if err := os.RemoveAll(e.asideDirOf(rec)); err != nil { + return err + } + x := rec.aside() + e.say(seat.EventAsideDiscarded, seat.AsideDiscardedEvent{Act: a.ID, At: e.now(), ID: x.ID, Name: x.Name, Size: x.Size}, a.Module, a.Kept) + return nil +} + +// Asides lists every aside in the directory (§517). +func (e *Engine) Asides() ([]seat.Aside, []asideRecord, error) { + var out []seat.Aside + var recs []asideRecord + _ = filepath.WalkDir(filepath.Join(e.Dir, "aside"), func(path string, d os.DirEntry, err error) error { + if err == nil && !d.IsDir() && d.Name() == asideFile { + var r asideRecord + if readJSON(path, &r) == nil { + out = append(out, r.aside()) + recs = append(recs, r) + } + } + return nil + }) + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out, recs, nil +} diff --git a/modules/mesh-recovery/internal/recovery/restore_test.go b/modules/mesh-recovery/internal/recovery/restore_test.go new file mode 100644 index 000000000..1945a1823 --- /dev/null +++ b/modules/mesh-recovery/internal/recovery/restore_test.go @@ -0,0 +1,234 @@ +package recovery_test + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" + + "mesh-recovery/internal/recovery" +) + +func restoreAct(snapshot string) recovery.Act { + return recovery.Act{Kind: recovery.KindRestore, Module: "app", Kept: "notes", Form: 2, Fingerprint: "fp1", Snapshot: snapshot, Why: "revert", Caller: "node.engine"} +} + +func takenRig(t *testing.T) *rig { + r := newRig(t) + r.do(takeAct()) + r.s.events = nil + return r +} + +func lastRestored(t *testing.T, r *rig) seat.RestoredEvent { + t.Helper() + ev := r.s.named(seat.EventRestored) + if len(ev) != 1 { + t.Fatalf("restored events %+v", r.s.events) + } + return ev[0].Body.(seat.RestoredEvent) +} + +// A restore says restore-started, sets the live data aside whole, removes it, restores the point under its names and sets +// each consumer to its recorded sequence, and says restored with the aside (design 47 G §516, §560). +func TestARestoreSetsTheLiveDataAsideAndRestoresThePoint(t *testing.T) { + r := takenRig(t) + r.b.streams["app_notes"] = "changed since" + ans := r.do(restoreAct("app/notes/2-fp1")) + ev := lastRestored(t, r) + if ev.Outcome != "done" || ev.Act != ans.Accepted || ev.Aside == nil || ev.Aside.Size == 0 { + t.Fatalf("restored %+v", ev) + } + if r.b.streams["app_notes"] != "hello" || r.b.streams["KV_app_seen"] != "world" { + t.Errorf("streams %v", r.b.streams) + } + if len(r.b.setBack) != 1 || r.b.setBack[0].Sequence != 7 { + t.Errorf("consumers %+v", r.b.setBack) + } + if len(r.s.named(seat.EventRestoreStarted)) != 1 { + t.Errorf("no restore-started") + } + kp, _ := r.e.KeptPoints("app", "notes") + if len(kp.Asides) != 1 || len(kp.Points) != 1 || kp.Running != nil { + t.Errorf("kept points %+v", kp) + } +} + +// A point missing from the directory says missing; with from-backup the backup holder brings it back and it is restored +// once its digest is checked; when no repository snapshot holds it, restore point missing (§516, §567). +func TestAMissingPointIsBroughtBackOrSaidMissing(t *testing.T) { + r := takenRig(t) + back := filepath.Join(t.TempDir(), "back") + if err := os.Rename(filepath.Join(r.e.Dir, "points"), back); err != nil { + t.Fatal(err) + } + r.do(restoreAct("app/notes/2-fp1")) + if ev := lastRestored(t, r); !ev.Missing || ev.Outcome != "failed" || ev.Error != recovery.ErrMissing { + t.Errorf("no backup asked: %+v", ev) + } + r.s.events = nil + a := restoreAct("app/notes/2-fp1") + a.FromBackup = true + r.do(a) + if ev := lastRestored(t, r); !ev.Missing || ev.Error != recovery.ErrMissing { + t.Errorf("backup holder holds none: %+v", ev) + } + r.s.events, r.s.backup = nil, back + r.do(a) + if ev := lastRestored(t, r); ev.Outcome != "done" { + t.Errorf("brought back: %+v", ev) + } +} + +// A point whose bytes differ from the digests on the bus is refused before any aside is taken, the record in the +// directory never being the reference (§567). +func TestAPointWhoseBytesWereAlteredIsRefusedBeforeAnyAside(t *testing.T) { + r := takenRig(t) + snap := filepath.Join(r.e.Dir, "points", "app", "notes", "2-fp1", "app_notes.snap") + if err := os.WriteFile(snap, []byte("snap:app_notes:evil"), 0o600); err != nil { + t.Fatal(err) + } + r.do(restoreAct("app/notes/2-fp1")) + if ev := lastRestored(t, r); ev.Outcome != "failed" || !strings.HasPrefix(ev.Error, recovery.ErrDamaged) { + t.Errorf("restored %+v", ev) + } + if kp, _ := r.e.KeptPoints("", ""); len(kp.Asides) != 0 || r.b.streams["app_notes"] != "hello" { + t.Errorf("an aside was taken or the live data changed: %+v %v", kp, r.b.streams) + } +} + +// A step after the removal that fails puts the aside back under the live names and the act ends failed, so the kept data +// is never left gone (§516). +func TestARestoreThatFailsAfterTheRemovalPutsTheAsideBack(t *testing.T) { + r := takenRig(t) + r.b.streams["app_notes"] = "live now" + r.b.failRestores = 1 + r.do(restoreAct("app/notes/2-fp1")) + if ev := lastRestored(t, r); ev.Outcome != "failed" || ev.Aside == nil { + t.Fatalf("restored %+v", ev) + } + if r.b.streams["app_notes"] != "live now" || r.b.streams["KV_app_seen"] != "world" { + t.Errorf("live data not as before: %v", r.b.streams) + } + if n := len(r.e.Unrestored()); n != 0 { + t.Errorf("%d asides unrestored", n) + } +} + +// A restore past the cap is refused with the live data untouched (§564). +func TestARestoreWhoseAsideWouldPassTheCapIsRefusedBeforeTheRemoval(t *testing.T) { + r := takenRig(t) + r.e.Settings.Cap = 1 + r.do(restoreAct("app/notes/2-fp1")) + if ev := lastRestored(t, r); ev.Outcome != "failed" || !strings.HasPrefix(ev.Error, recovery.ErrCap) { + t.Errorf("restored %+v", ev) + } + if r.b.streams["app_notes"] != "hello" { + t.Errorf("live data touched: %v", r.b.streams) + } +} + +// An act waiting on the dump is cancelled with its ending event failed/cancelled and the hand act's words as why; an act +// that has begun is refused, naming its step (§566, A-117). +func TestAWaitingActIsCancelledAndABegunOneRefused(t *testing.T) { + r := takenRig(t) + hd := holdDump{waiting: make(chan string, 1)} + r.e.Dump = hd + ans := r.e.Accept(restoreAct("app/notes/2-fp1")) + <-hd.waiting + kp, _ := r.e.KeptPoints("", "") + if kp.Running == nil || kp.Running.WaitsOn == "" || kp.Running.Act != ans.Accepted { + t.Errorf("running %+v", kp.Running) + } + if c := r.e.Cancel("act-nothing", "x", "tools", "op"); c.Error == "" { + t.Errorf("an unknown act cancelled: %+v", c) + } + if c := r.e.Cancel(ans.Accepted, "not now", "tools", "op"); c.Accepted != ans.Accepted { + t.Errorf("cancel %+v", c) + } + r.e.Wait() + ev := lastRestored(t, r) + if ev.Outcome != "failed" || ev.Error != recovery.ErrCanceled || ev.Why != "not now" || ev.Act != ans.Accepted { + t.Errorf("restored %+v", ev) + } +} + +// retry asks again the last failed act with its fields under an act of its own, and answers `nothing to retry` when its +// last act did not fail (§682). +func TestRetryAsksTheLastFailedActAgain(t *testing.T) { + r := takenRig(t) + r.e.Settings.Cap = 1 + r.do(restoreAct("app/notes/2-fp1")) + r.e.Settings.Cap = 1 << 20 + r.s.events = nil + ans := r.e.Retry("app", "notes", "again", "tools", "op") + r.e.Wait() + if ev := lastRestored(t, r); ans.Accepted == "" || ev.Outcome != "done" || ev.Act != ans.Accepted { + t.Errorf("answer %+v restored %+v", ans, ev) + } + if again := r.e.Retry("app", "notes", "again", "tools", "op"); again.Error != "nothing to retry" { + t.Errorf("answer %+v", again) + } +} + +// An aside whose record ends at removed is put back at the start, an act cut off by a restart ends failed with stopped, +// and a point left under a temporary name is removed (§516, §560, §564). +func TestAStartPutsBackAnAsideEndsACutOffActAndClearsTemporaryPoints(t *testing.T) { + r := takenRig(t) + r.do(restoreAct("app/notes/2-fp1")) // leaves an aside at "restored" + _, recs, _ := r.e.Asides() + if len(recs) != 1 { + t.Fatal("no aside") + } + // Cut a restore off after its removal: the record says removed, the live data is gone, an accepted act never ended. + dir := filepath.Join(r.e.Dir, "aside", "app", "notes", recs[0].ID) + raw, _ := os.ReadFile(filepath.Join(dir, "aside.json")) + _ = os.WriteFile(filepath.Join(dir, "aside.json"), []byte(strings.Replace(string(raw), `"step": "restored"`, `"step": "removed"`, 1)), 0o600) + delete(r.b.streams, "app_notes") + delete(r.b.streams, "KV_app_seen") + tmp := filepath.Join(r.e.Dir, "points", "app", "notes", ".tmp-act-x") + _ = os.MkdirAll(tmp, 0o700) + _ = os.WriteFile(filepath.Join(r.e.Dir, "acts", "act-cut.json"), []byte(`{"act":"act-cut","kind":"restore","module":"app","kept":"notes","why":"w","caller":"c","since":"2026-11-12T00:00:00Z"}`), 0o600) + r.s.events = nil + if left := r.e.Start(context.Background()); len(left) != 0 { + t.Errorf("unrestored %+v", left) + } + if r.b.streams["app_notes"] != "hello" { + t.Errorf("the aside was not put back: %v", r.b.streams) + } + if _, err := os.Stat(tmp); err == nil { + t.Errorf("the temporary point stays") + } + if ev := lastRestored(t, r); ev.Act != "act-cut" || ev.Error != recovery.ErrStopped { + t.Errorf("restored %+v", ev) + } +} + +// restore-aside puts an aside back under the live names, refused while live objects of those names exist; discard-aside +// removes the aside's directory and says aside-discarded (§554, §566). +func TestRestoreAsideRefusesLiveNamesAndDiscardAsideRemovesTheDirectory(t *testing.T) { + r := takenRig(t) + r.do(restoreAct("app/notes/2-fp1")) + _, recs, _ := r.e.Asides() + id := recs[0].ID + r.s.events = nil + r.do(recovery.Act{Kind: recovery.KindRestoreAside, Module: "app", Kept: "notes", AsideID: id, Why: "x", Caller: "tools"}) + if ev := lastRestored(t, r); ev.Outcome != "failed" || !strings.Contains(ev.Error, "live stream") { + t.Errorf("restored %+v", ev) + } + r.s.events = nil + r.do(recovery.Act{Kind: recovery.KindDiscardAside, Module: "app", Kept: "notes", AsideID: "../x", Why: "x", Caller: "tools"}) + if len(r.s.named(seat.EventAsideDiscarded)) != 0 { + t.Errorf("a path outside was discarded") + } + r.do(recovery.Act{Kind: recovery.KindDiscardAside, Module: "app", Kept: "notes", AsideID: id, Why: "x", Caller: "tools"}) + if ev := r.s.named(seat.EventAsideDiscarded); len(ev) != 1 || ev[0].Body.(seat.AsideDiscardedEvent).ID != id { + t.Errorf("events %+v", r.s.events) + } + if kp, _ := r.e.KeptPoints("", ""); len(kp.Asides) != 0 { + t.Errorf("asides %+v", kp.Asides) + } +} From 2dd02da76e0e6cd591b84de713f8a32ec2182a42 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 01:17:41 +0100 Subject: [PATCH 04/17] mesh-recovery: aside sets the live data aside from the placement's entry, names as a list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §516, §554, §566, A-123, A-124 aside takes the bus names from the catalog entry of the fingerprint the node-engine's placement runs, takes the aside whole and checked, removes the live objects, says set-aside; restore-aside puts it back. Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../internal/recovery/fakes_test.go | 2 ++ .../mesh-recovery/internal/recovery/points.go | 2 ++ .../internal/recovery/recovery.go | 2 ++ .../internal/recovery/restore.go | 27 +++++++++++++++++-- .../internal/recovery/restore_test.go | 23 ++++++++++++++++ 5 files changed, 54 insertions(+), 2 deletions(-) diff --git a/modules/mesh-recovery/internal/recovery/fakes_test.go b/modules/mesh-recovery/internal/recovery/fakes_test.go index a38f086f8..48796d2cb 100644 --- a/modules/mesh-recovery/internal/recovery/fakes_test.go +++ b/modules/mesh-recovery/internal/recovery/fakes_test.go @@ -122,6 +122,8 @@ func (s *fakeSeats) Streams(context.Context, string, string, string) ([]string, return s.streams, nil } +func (s *fakeSeats) RunningFingerprint(context.Context, string) (string, error) { return "fp-run", nil } + func (s *fakeSeats) Backup(context.Context, string) (string, error) { if s.backup == "" { return "", errors.New("no repository snapshot holds it") diff --git a/modules/mesh-recovery/internal/recovery/points.go b/modules/mesh-recovery/internal/recovery/points.go index 9256efd47..18e5ec14c 100644 --- a/modules/mesh-recovery/internal/recovery/points.go +++ b/modules/mesh-recovery/internal/recovery/points.go @@ -28,6 +28,8 @@ func (e *Engine) run(ctx context.Context, a *Act) { e.expire(a.ID) case KindRestore: err = e.restore(ctx, a) + case KindAside: + err = e.aside(ctx, a) case KindRestoreAside: err = e.restoreAside(ctx, a) case KindDiscardAside: diff --git a/modules/mesh-recovery/internal/recovery/recovery.go b/modules/mesh-recovery/internal/recovery/recovery.go index bdd0d05d8..91c1ab8f7 100644 --- a/modules/mesh-recovery/internal/recovery/recovery.go +++ b/modules/mesh-recovery/internal/recovery/recovery.go @@ -76,6 +76,8 @@ type Seats interface { HandAct(caller, asker, what, why string) error // Streams are the bus names of a module's kept data at a fingerprint, from its catalog graph entry (§554 Bus names). Streams(ctx context.Context, module, kept, fingerprint string) ([]string, error) + // RunningFingerprint is the fingerprint the node-engine's placement runs for the module (A-123). + RunningFingerprint(ctx context.Context, module string) (string, error) // Backup brings the points directory of the module's item back beside its path through the backup holder and // answers where (§516); an error when no repository snapshot holds it. Backup(ctx context.Context, module string) (dir string, err error) diff --git a/modules/mesh-recovery/internal/recovery/restore.go b/modules/mesh-recovery/internal/recovery/restore.go index cfec63109..744a2eaf6 100644 --- a/modules/mesh-recovery/internal/recovery/restore.go +++ b/modules/mesh-recovery/internal/recovery/restore.go @@ -35,7 +35,7 @@ type asideRecord struct { } func (r asideRecord) aside() seat.Aside { - return seat.Aside{ID: r.ID, Name: strings.Join(r.Streams, ","), Size: r.Size} + return seat.Aside{ID: r.ID, Names: r.Streams, Size: r.Size} } func (e *Engine) asideRoot(module, kept string) (string, error) { @@ -279,6 +279,29 @@ func (e *Engine) readAside(module, kept, id string) (*asideRecord, error) { return &rec, nil } +// aside is the hand verb `aside` (§516, §554, A-123): it sets the live kept data aside without restoring it, taking the bus +// names from the catalog entry of the fingerprint the node-engine's placement runs for the module, and removes the live +// objects once the aside is whole and checked (a move, not a removal: the data stays in the aside until it is discarded or +// put back by `restore-aside`); it ends with `set-aside`. +func (e *Engine) aside(ctx context.Context, a *Act) error { + fp, err := e.Seats.RunningFingerprint(ctx, a.Module) + if err != nil { + return err + } + a.Fingerprint = fp + names, err := e.Seats.Streams(ctx, a.Module, a.Kept, fp) + if err != nil { + return err + } + rec, err := e.setAside(ctx, a, names, true) + if err != nil { + return err + } + x := rec.aside() + e.say(seat.EventSetAside, seat.SetAsideEvent{Act: a.ID, At: e.now(), ID: x.ID, Names: x.Names, Size: x.Size}, a.Module, a.Kept) + return nil +} + // restoreAside is `restore-aside` (§566): it puts an aside whose record ends at `removed` back under the live names it // was taken from, refusing while a live object of those names exists, and ends with `restored`. func (e *Engine) restoreAside(ctx context.Context, a *Act) error { @@ -312,7 +335,7 @@ func (e *Engine) discardAside(a *Act) error { return err } x := rec.aside() - e.say(seat.EventAsideDiscarded, seat.AsideDiscardedEvent{Act: a.ID, At: e.now(), ID: x.ID, Name: x.Name, Size: x.Size}, a.Module, a.Kept) + e.say(seat.EventAsideDiscarded, seat.AsideDiscardedEvent{Act: a.ID, At: e.now(), ID: x.ID, Names: x.Names, Size: x.Size}, a.Module, a.Kept) return nil } diff --git a/modules/mesh-recovery/internal/recovery/restore_test.go b/modules/mesh-recovery/internal/recovery/restore_test.go index 1945a1823..2242705b1 100644 --- a/modules/mesh-recovery/internal/recovery/restore_test.go +++ b/modules/mesh-recovery/internal/recovery/restore_test.go @@ -232,3 +232,26 @@ func TestRestoreAsideRefusesLiveNamesAndDiscardAsideRemovesTheDirectory(t *testi t.Errorf("asides %+v", kp.Asides) } } + +// The hand verb aside sets the live data aside whole and removes it, naming the streams as a list, and restore-aside puts +// it back once no live name exists (§516, §554, §566, A-123, A-124). +func TestAsideMovesTheLiveDataIntoAnAsideAndRestoreAsidePutsItBack(t *testing.T) { + r := newRig(t) + ans := r.do(recovery.Act{Kind: recovery.KindAside, Module: "app", Kept: "notes", Why: "look at it", Caller: "tools", Asker: "op"}) + ev := r.s.named(seat.EventSetAside) + if ans.Accepted == "" || len(ev) != 1 { + t.Fatalf("answer %+v events %+v", ans, r.s.events) + } + sa := ev[0].Body.(seat.SetAsideEvent) + if sa.Act != ans.Accepted || len(sa.Names) != 2 || sa.Size == 0 || len(r.b.streams) != 0 { + t.Errorf("set-aside %+v streams %v", sa, r.b.streams) + } + if got := r.e.Unrestored(); len(got) != 1 { + t.Errorf("unrestored %+v", got) + } + r.s.events = nil + r.do(recovery.Act{Kind: recovery.KindRestoreAside, Module: "app", Kept: "notes", AsideID: sa.ID, Why: "done looking", Caller: "tools"}) + if lastRestored(t, r).Outcome != "done" || r.b.streams["app_notes"] != "hello" || len(r.e.Unrestored()) != 0 { + t.Errorf("streams %v", r.b.streams) + } +} From 572a2b3a4b3be6c0659d85c259f242da236abe7b Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 01:18:43 +0100 Subject: [PATCH 05/17] mesh-recovery: the seat's verbs, answered by the process with the proved caller MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §516, §554, §656, §682, §691, A-107, A-115 Each acting verb answers one of accepted, busy or error; arguments are read afresh per call and refused when unknown (local-name, from, aside-only, discard-aside) or without why; restore takes exactly one of snapshot and request (request is refused until built); the hand-act log names the proved account and the asker. Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../mesh-recovery/internal/recovery/verbs.go | 112 ++++++++++++++++++ .../internal/recovery/verbs_test.go | 63 ++++++++++ 2 files changed, 175 insertions(+) create mode 100644 modules/mesh-recovery/internal/recovery/verbs.go create mode 100644 modules/mesh-recovery/internal/recovery/verbs_test.go diff --git a/modules/mesh-recovery/internal/recovery/verbs.go b/modules/mesh-recovery/internal/recovery/verbs.go new file mode 100644 index 000000000..105868eb7 --- /dev/null +++ b/modules/mesh-recovery/internal/recovery/verbs.go @@ -0,0 +1,112 @@ +package recovery + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "strings" + + seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" + "git.novox.be/novox/mesh-sdk/go/process" +) + +// Handlers are the seat's verbs as the process answers them over the bus with the caller the bus proved (§554, §656, A-107): +// every acting verb answers one of accepted, busy or error and nothing more (§691); the result is the event. +func (e *Engine) Handlers() map[string]process.SeatHandler { + h := map[string]process.SeatHandler{} + h[seat.VerbSnapshotKept] = actOf(e, func(sk seat.SnapshotKeptInput) (Act, error) { + return Act{Kind: KindTake, Module: sk.Module, Kept: sk.Kept, Form: sk.Form, Fingerprint: sk.Fingerprint, CatchUp: sk.CatchUp, Why: sk.Why}, nil + }) + restoreKept := actOf(e, func(rk seat.RestoreKeptInput) (Act, error) { + return Act{Kind: KindRestore, Module: rk.Module, Kept: rk.Kept, Form: rk.Form, Fingerprint: rk.Fingerprint, Snapshot: rk.Snapshot, FromBackup: rk.FromBackup, Why: rk.Why}, nil + }) + h[seat.VerbRestoreKept] = func(caller string, body json.RawMessage) (any, error) { + // aside-only and discard-aside are the operator's hand tools alone (§516, §554). + var loose map[string]json.RawMessage + _ = json.Unmarshal(body, &loose) + for _, k := range []string{"aside-only", "discard-aside"} { + if _, ok := loose[k]; ok { + return Answer{Error: k + " is the operator's hand tool alone, not an argument of restore-kept"}, nil + } + } + return restoreKept(caller, body) + } + h[seat.VerbKeptPoints] = func(_ string, body json.RawMessage) (any, error) { + var in seat.KeptPointsInput + if err := strict(body, &in); err != nil { + return nil, err + } + return e.KeptPoints(in.Module, in.Kept) + } + h[seat.VerbTake] = actOf(e, func(tk seat.TakeInput) (Act, error) { + return Act{Kind: KindTake, Module: tk.Module, Kept: tk.Kept, Fingerprint: tk.Fingerprint, Why: tk.Why, Asker: tk.Asker}, nil + }) + h[seat.VerbRestore] = actOf(e, func(rs seat.RestoreInput) (Act, error) { + if (rs.Snapshot == "") == (rs.Request == "") { + return Act{}, errors.New("restore takes exactly one of snapshot and request") + } + if rs.Request != "" { + return Act{}, errors.New("the restore of a keeper's snapshot by request is not built yet (design 47 G §681)") + } + return Act{Kind: KindRestore, Module: rs.Module, Kept: rs.Kept, Snapshot: rs.Snapshot, FromBackup: rs.FromBackup, Why: rs.Why, Asker: rs.Asker}, nil + }) + h[seat.VerbAside] = actOf(e, func(ad seat.AsideInput) (Act, error) { + return Act{Kind: KindAside, Module: ad.Module, Kept: ad.Kept, Why: ad.Why, Asker: ad.Asker}, nil + }) + h[seat.VerbDiscardAside] = actOf(e, func(da seat.DiscardAsideInput) (Act, error) { + return Act{Kind: KindDiscardAside, Module: da.Module, Kept: da.Kept, AsideID: da.ID, Why: da.Why, Asker: da.Asker}, nil + }) + h[seat.VerbRestoreAside] = actOf(e, func(ra seat.RestoreAsideInput) (Act, error) { + return Act{Kind: KindRestoreAside, Module: ra.Module, Kept: ra.Kept, AsideID: ra.ID, Why: ra.Why, Asker: ra.Asker}, nil + }) + h[seat.VerbExpirePoints] = actOf(e, func(ex seat.ExpirePointsInput) (Act, error) { + return Act{Kind: KindExpire, Why: ex.Why, Asker: ex.Asker}, nil + }) + h[seat.VerbRetry] = func(caller string, body json.RawMessage) (any, error) { + var in seat.RetryInput + if err := strict(body, &in); err != nil { + return Answer{Error: err.Error()}, nil + } + return e.Retry(in.Module, in.Kept, in.Why, caller, in.Asker), nil + } + h[seat.VerbCancel] = func(caller string, body json.RawMessage) (any, error) { + var in seat.CancelInput + if err := strict(body, &in); err != nil { + return Answer{Error: err.Error()}, nil + } + return e.Cancel(in.Act, in.Why, caller, in.Asker), nil + } + return h +} + +// actOf is one acting verb: its arguments are read afresh for each call (calls come on concurrent bus callbacks), a call +// without the `why` the hand-act log keeps is refused, and the act is accepted as the engine says. +func actOf[T any](e *Engine, build func(T) (Act, error)) process.SeatHandler { + return func(caller string, body json.RawMessage) (any, error) { + var in T + if err := strict(body, &in); err != nil { + return Answer{Error: err.Error()}, nil + } + a, err := build(in) + if err != nil { + return Answer{Error: err.Error()}, nil + } + if a.Why == "" { + return Answer{Error: "why is required: the hand-act log keeps it"}, nil + } + a.Caller = caller + return e.Accept(a), nil + } +} + +// strict reads a call's arguments, refusing a field the verb does not take (no `local-name`, no `from`: §516) and a call +// that leaves out the `why` the hand-act log keeps (§682). +func strict(body json.RawMessage, into any) error { + dec := json.NewDecoder(bytes.NewReader(body)) + dec.DisallowUnknownFields() + if err := dec.Decode(into); err != nil { + return fmt.Errorf("the arguments are refused: %s", strings.TrimPrefix(err.Error(), "json: ")) + } + return nil +} diff --git a/modules/mesh-recovery/internal/recovery/verbs_test.go b/modules/mesh-recovery/internal/recovery/verbs_test.go new file mode 100644 index 000000000..9b2ca8860 --- /dev/null +++ b/modules/mesh-recovery/internal/recovery/verbs_test.go @@ -0,0 +1,63 @@ +package recovery_test + +import ( + "encoding/json" + "strings" + "testing" + + seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" + + "mesh-recovery/internal/recovery" +) + +func call(t *testing.T, r *rig, verb, body string) recovery.Answer { + t.Helper() + out, err := r.e.Handlers()[verb]("node.engine", json.RawMessage(body)) + if err != nil { + t.Fatalf("%s: %v", verb, err) + } + r.e.Wait() + return out.(recovery.Answer) +} + +// Every verb of the seat's definition is answered; an acting verb answers one of accepted, busy and error and nothing more +// (§691); an argument the verb does not take (`local-name`, `from`), `aside-only` and `discard-aside` on restore-kept, a +// restore naming both a snapshot and a request, and a call without `why` are refused naming the fault (§516, §682, §554). +func TestEveryVerbIsAnsweredAndAnActingVerbAnswersOneForm(t *testing.T) { + r := newRig(t) + h := r.e.Handlers() + for _, v := range seat.Verbs() { + if h[v] == nil { + t.Errorf("verb %s has no handler", v) + } + } + taken := call(t, r, "snapshot-kept", `{"module":"app","kept":"notes","form":2,"fingerprint":"fp1","why":"form change"}`) + if taken.Accepted == "" || taken.Busy != "" || taken.Error != "" { + t.Fatalf("answer %+v", taken) + } + for name, c := range map[string]struct{ verb, body, want string }{ + "local-name": {"snapshot-kept", `{"module":"app","kept":"notes","form":2,"fingerprint":"fp1","why":"w","local-name":"x"}`, "local-name"}, + "from": {"restore-kept", `{"module":"app","kept":"notes","form":2,"fingerprint":"fp1","why":"w","from":"x"}`, "from"}, + "aside-only": {"restore-kept", `{"module":"app","kept":"notes","form":2,"fingerprint":"fp1","why":"w","aside-only":true}`, "aside-only"}, + "discard": {"restore-kept", `{"module":"app","kept":"notes","form":2,"fingerprint":"fp1","why":"w","discard-aside":true}`, "discard-aside"}, + "both": {"restore", `{"module":"app","kept":"notes","snapshot":"a","request":"b","why":"w"}`, "exactly one"}, + "no why": {"restore", `{"module":"app","kept":"notes","snapshot":"a"}`, "why"}, + } { + if got := call(t, r, c.verb, c.body); !strings.Contains(got.Error, c.want) || got.Accepted != "" { + t.Errorf("%s: answer %+v", name, got) + } + } + if len(r.s.named(seat.EventEnded)) != 1 { + t.Errorf("a refused call acted: %+v", r.s.events) + } + out, err := h["kept-points"]("tools", json.RawMessage(`{"module":"app"}`)) + if kp, ok := out.(seat.KeptPointsAnswer); err != nil || !ok || len(kp.Points) != 1 { + t.Errorf("kept-points %+v %v", out, err) + } + if got := call(t, r, "take", `{"module":"app","kept":"notes","fingerprint":"fp9","why":"by hand","asker":"op@shanks"}`); got.Accepted == "" { + t.Errorf("take %+v", got) + } + if !strings.Contains(r.s.handActs[len(r.s.handActs)-1], "node.engine|op@shanks|take app notes|by hand") { + t.Errorf("hand acts %v: the account the bus proved and the asker are both logged", r.s.handActs) + } +} From 968a0aac7ab9a6d1d029a7422e46bc8922526454 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 01:19:32 +0100 Subject: [PATCH 06/17] mesh-recovery: the tools bundle, recovery_ tools calling the seat's verbs with the proved asker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §554, §656, §657, §682 recovery_kept_points reads kept-points; every hand tool (take, restore, retry, aside, discard_aside, restore_aside, cancel, expire_points) is one verb of the seat called with the asker the tool runner proved; the bundle holds no grant of its own. Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-recovery-tools/main.go | 100 ++++++++++++++++++ .../cmd/mesh-recovery-tools/main_test.go | 53 ++++++++++ 2 files changed, 153 insertions(+) create mode 100644 modules/mesh-recovery/cmd/mesh-recovery-tools/main.go create mode 100644 modules/mesh-recovery/cmd/mesh-recovery-tools/main_test.go diff --git a/modules/mesh-recovery/cmd/mesh-recovery-tools/main.go b/modules/mesh-recovery/cmd/mesh-recovery-tools/main.go new file mode 100644 index 000000000..6bd188dc5 --- /dev/null +++ b/modules/mesh-recovery/cmd/mesh-recovery-tools/main.go @@ -0,0 +1,100 @@ +// mesh-recovery-tools: the recovery module's tools, its command-line interface (design 47 G §554, §656, §657). Each hand +// tool calls one verb of the seat mesh-recovery with the `asker` the tool runner proved for the call and holds no grant of +// its own: the bus proves the tool runner's account, the verb logs both in the hand-act log. stdout is the MCP channel. +package main + +import ( + "encoding/json" + "fmt" + "os" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } + +func main() { + if err := stdio.Serve("", Tools(stdio.Ask, stdio.CallerOf)); err != nil { + logf("%v", err) + os.Exit(1) + } +} + +// Ask is how a bundle calls a seat's verb: the key `seat:.` and the arguments. +type Ask func(key string, body any) (json.RawMessage, error) + +func obj(props map[string]any, required ...string) map[string]any { + m := map[string]any{"type": "object", "properties": props} + if len(required) > 0 { + m["required"] = required + } + return m +} + +func str(d string) map[string]any { return map[string]any{"type": "string", "description": d} } + +var ( + module = str("the module") + kept = str("the kept data, its local name in the module's manifest") + why = str("why, which the hand-act log keeps") +) + +// hand is a tool that calls a verb of the seat: the arguments go through as they are, with the asker the tool runner proved. +func hand(ask Ask, callerOf func(map[string]any) string, name, verb, description string, props map[string]any, required ...string) stdio.Tool { + props["why"] = why + return stdio.Tool{Name: name, Description: description + " Class act: written in the hand-act log first with the account the bus proved and who asked; answers accepted, busy or error at once and the result comes as the act's ending event.", + Input: obj(props, append(required, "why")...), + Run: func(a map[string]any) (any, error) { + body := map[string]any{} + for k, v := range a { + body[k] = v + } + body["asker"] = callerOf(a) + raw, err := ask("seat:mesh-recovery."+verb, body) + if err != nil { + return nil, err + } + var out any + return out, json.Unmarshal(raw, &out) + }} +} + +// Tools are what the runtime serves, named recovery_ (§554); each acting tool is one verb of the seat (§656). +func Tools(ask Ask, callerOf func(map[string]any) string) []stdio.Tool { + return []stdio.Tool{ + {Name: "recovery_kept_points", + Description: "Every restore point and aside the recovery module holds, with their sizes, and the act that runs with since when and what it waits on; narrowed to a module and a kept data. Changes nothing (also the module's declared health check, §657).", + Input: obj(map[string]any{"module": module, "kept": kept}), + Run: func(a map[string]any) (any, error) { + body := map[string]any{} + for _, k := range []string{"module", "kept"} { + if s, _ := a[k].(string); strings.TrimSpace(s) != "" { + body[k] = s + } + } + raw, err := ask("seat:mesh-recovery.kept-points", body) + if err != nil { + return nil, err + } + var out any + return out, json.Unmarshal(raw, &out) + }}, + hand(ask, callerOf, "recovery_take", "take", "Take a restore point of one module's kept data now, at the form of the given fingerprint.", + map[string]any{"module": module, "kept": kept, "fingerprint": str("the fingerprint whose form the point is taken at")}, "module", "kept", "fingerprint"), + hand(ask, callerOf, "recovery_restore", "restore", "Restore one point (snapshot, a point's name from recovery_kept_points) or a keeper's snapshot (request), exactly one of them.", + map[string]any{"module": module, "kept": kept, "snapshot": str("the restore point's name"), "request": str("the keeper's snapshot by its request"), + "from-backup": map[string]any{"type": "boolean", "description": "ask the backup holder for the points directory when the point is missing, only with snapshot"}}, "module", "kept"), + hand(ask, callerOf, "recovery_retry", "retry", "Ask again the last failed act of a module and kept data, with its fields.", + map[string]any{"module": module, "kept": kept}, "module", "kept"), + hand(ask, callerOf, "recovery_aside", "aside", "Set the live kept data aside without restoring: a move, the data stays in the aside until it is put back or discarded.", + map[string]any{"module": module, "kept": kept}, "module", "kept"), + hand(ask, callerOf, "recovery_discard_aside", "discard-aside", "Remove one aside's directory and nothing else.", + map[string]any{"module": module, "kept": kept, "id": str("the aside's id")}, "module", "kept", "id"), + hand(ask, callerOf, "recovery_restore_aside", "restore-aside", "Put an aside back under the live names it was taken from, refused while live objects of those names exist.", + map[string]any{"module": module, "kept": kept, "id": str("the aside's id")}, "module", "kept", "id"), + hand(ask, callerOf, "recovery_cancel", "cancel", "End an accepted act that has not begun its work (one waiting on the dump) failed, cancelled; refused naming the step of an act that has begun.", + map[string]any{"act": str("the act's id")}, "act"), + hand(ask, callerOf, "recovery_expire_points", "expire-points", "Remove the restore points past their keep days now, never the newest of a kept data.", map[string]any{}), + } +} diff --git a/modules/mesh-recovery/cmd/mesh-recovery-tools/main_test.go b/modules/mesh-recovery/cmd/mesh-recovery-tools/main_test.go new file mode 100644 index 000000000..fe28938ea --- /dev/null +++ b/modules/mesh-recovery/cmd/mesh-recovery-tools/main_test.go @@ -0,0 +1,53 @@ +package main + +import ( + "encoding/json" + "os" + "slices" + "strings" + "testing" +) + +// The bundle serves exactly the tools the manifest names, each named recovery_, and each acting tool calls its verb of the +// seat with the asker the tool runner proved, written into the call, and nothing of its own (design 47 G §554, §656, §682). +func TestEachToolCallsItsVerbWithTheProvedAsker(t *testing.T) { + var key string + var sent map[string]any + ask := func(k string, b any) (json.RawMessage, error) { + key, sent = k, b.(map[string]any) + return json.RawMessage(`{"accepted":"act-1"}`), nil + } + tools := Tools(ask, func(map[string]any) string { return "claude@shanks" }) + raw, _ := os.ReadFile("../../module.json") + var m struct { + Tools []string `json:"tools"` + Invokes []string `json:"invokes"` + } + _ = json.Unmarshal(raw, &m) + var names []string + verbs := map[string]string{"recovery_take": "take", "recovery_restore": "restore", "recovery_retry": "retry", "recovery_aside": "aside", + "recovery_discard_aside": "discard-aside", "recovery_restore_aside": "restore-aside", "recovery_cancel": "cancel", "recovery_expire_points": "expire-points"} + for _, tool := range tools { + names = append(names, tool.Name) + if !strings.HasPrefix(tool.Name, "recovery_") { + t.Errorf("tool %s is not named recovery_", tool.Name) + } + if verb, acts := verbs[tool.Name]; acts { + if _, err := tool.Run(map[string]any{"module": "app", "why": "w"}); err != nil { + t.Fatal(err) + } + if key != "seat:mesh-recovery."+verb || sent["asker"] != "claude@shanks" || sent["module"] != "app" { + t.Errorf("%s called %s with %v", tool.Name, key, sent) + } + if !slices.Contains(m.Invokes, key) { + t.Errorf("the manifest does not invoke %s", key) + } + } + } + slices.Sort(names) + want := slices.Clone(m.Tools) + slices.Sort(want) + if !slices.Equal(names, want) { + t.Errorf("tools %v, the manifest names %v", names, want) + } +} From 6d45f3ef5849838dcb115e90fc8d9cdc573709e4 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 01:26:38 +0100 Subject: [PATCH 07/17] A-125: the bus module imports mesh-sdk's bussnap and busarchive, its copies deleted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The take and restore of one stream is shared code in mesh-sdk; the bus module's keeper, its nightly dump and its tools import it from there and keep no copy. Builds: design 47 G §655, A-125 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../nats/cmd/mesh-nats-snapshot/bussnap.go | 2 +- .../nats/cmd/mesh-nats-snapshot/image_test.go | 2 +- .../nats/cmd/mesh-nats-snapshot/live_test.go | 2 +- modules/nats/cmd/mesh-nats-snapshot/main.go | 5 +- .../nats/cmd/mesh-nats-snapshot/restore.go | 2 +- modules/nats/cmd/nats-tools/keep.go | 2 +- .../cmd/nats-tools/keeper_integration_test.go | 2 +- modules/nats/cmd/nats-tools/main.go | 2 +- modules/nats/internal/busarchive/archive.go | 162 ------- modules/nats/internal/busarchive/types.go | 86 ---- .../nats/internal/busarchive/verifyfile.go | 109 ----- .../internal/busarchive/verifyfile_test.go | 173 ------- modules/nats/internal/bussnap/restore.go | 158 ------- modules/nats/internal/bussnap/snapshot.go | 433 ------------------ modules/nats/internal/keeper/jetstream.go | 2 +- 15 files changed, 11 insertions(+), 1131 deletions(-) delete mode 100644 modules/nats/internal/busarchive/archive.go delete mode 100644 modules/nats/internal/busarchive/types.go delete mode 100644 modules/nats/internal/busarchive/verifyfile.go delete mode 100644 modules/nats/internal/busarchive/verifyfile_test.go delete mode 100644 modules/nats/internal/bussnap/restore.go delete mode 100644 modules/nats/internal/bussnap/snapshot.go diff --git a/modules/nats/cmd/mesh-nats-snapshot/bussnap.go b/modules/nats/cmd/mesh-nats-snapshot/bussnap.go index e4d8a8f59..0c205ed83 100644 --- a/modules/nats/cmd/mesh-nats-snapshot/bussnap.go +++ b/modules/nats/cmd/mesh-nats-snapshot/bussnap.go @@ -1,6 +1,6 @@ package main -import "nats-tools/internal/bussnap" +import "git.novox.be/novox/mesh-sdk/go/bussnap" // The snapshot itself is bussnap's, which the keeper takes one entry's with too (design 25 § 12.3). type ( diff --git a/modules/nats/cmd/mesh-nats-snapshot/image_test.go b/modules/nats/cmd/mesh-nats-snapshot/image_test.go index d53fcee36..ddf3b86ca 100644 --- a/modules/nats/cmd/mesh-nats-snapshot/image_test.go +++ b/modules/nats/cmd/mesh-nats-snapshot/image_test.go @@ -19,8 +19,8 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-sdk/go/busarchive" "github.com/nats-io/nats.go" - "nats-tools/internal/busarchive" ) // manifestOf is the nats module's own definition, beside this program. diff --git a/modules/nats/cmd/mesh-nats-snapshot/live_test.go b/modules/nats/cmd/mesh-nats-snapshot/live_test.go index cd64c0fc7..cbea74248 100644 --- a/modules/nats/cmd/mesh-nats-snapshot/live_test.go +++ b/modules/nats/cmd/mesh-nats-snapshot/live_test.go @@ -34,8 +34,8 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-sdk/go/busarchive" "github.com/nats-io/nats.go" - "nats-tools/internal/busarchive" ) // image is the bus's release (the manifest's server artifact names the same release in its tag). diff --git a/modules/nats/cmd/mesh-nats-snapshot/main.go b/modules/nats/cmd/mesh-nats-snapshot/main.go index 983b33393..4ff65226b 100644 --- a/modules/nats/cmd/mesh-nats-snapshot/main.go +++ b/modules/nats/cmd/mesh-nats-snapshot/main.go @@ -30,10 +30,11 @@ import ( "flag" "fmt" "io" - "nats-tools/internal/busarchive" - "nats-tools/internal/bussnap" "os" "time" + + "git.novox.be/novox/mesh-sdk/go/busarchive" + "git.novox.be/novox/mesh-sdk/go/bussnap" ) func main() { diff --git a/modules/nats/cmd/mesh-nats-snapshot/restore.go b/modules/nats/cmd/mesh-nats-snapshot/restore.go index 5d5866094..f56c80d05 100644 --- a/modules/nats/cmd/mesh-nats-snapshot/restore.go +++ b/modules/nats/cmd/mesh-nats-snapshot/restore.go @@ -16,8 +16,8 @@ import ( "syscall" "time" + "git.novox.be/novox/mesh-sdk/go/busarchive" "github.com/nats-io/nats.go" - "nats-tools/internal/busarchive" ) // restoreChunk is what is sent at a time; the server answers each before the next goes. diff --git a/modules/nats/cmd/nats-tools/keep.go b/modules/nats/cmd/nats-tools/keep.go index 37c18e506..2547bf5b1 100644 --- a/modules/nats/cmd/nats-tools/keep.go +++ b/modules/nats/cmd/nats-tools/keep.go @@ -29,7 +29,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/seatdef" "git.novox.be/novox/mesh-sdk/go/views" - "nats-tools/internal/bussnap" + "git.novox.be/novox/mesh-sdk/go/bussnap" "nats-tools/internal/keeper" ) diff --git a/modules/nats/cmd/nats-tools/keeper_integration_test.go b/modules/nats/cmd/nats-tools/keeper_integration_test.go index bb06a2a05..63e38779f 100644 --- a/modules/nats/cmd/nats-tools/keeper_integration_test.go +++ b/modules/nats/cmd/nats-tools/keeper_integration_test.go @@ -28,7 +28,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/views" - "nats-tools/internal/bussnap" + "git.novox.be/novox/mesh-sdk/go/bussnap" "nats-tools/internal/keeper" ) diff --git a/modules/nats/cmd/nats-tools/main.go b/modules/nats/cmd/nats-tools/main.go index 535eeb75b..b20eefcba 100644 --- a/modules/nats/cmd/nats-tools/main.go +++ b/modules/nats/cmd/nats-tools/main.go @@ -11,7 +11,7 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" - "nats-tools/internal/busarchive" + "git.novox.be/novox/mesh-sdk/go/busarchive" "nats-tools/internal/keeper" ) diff --git a/modules/nats/internal/busarchive/archive.go b/modules/nats/internal/busarchive/archive.go deleted file mode 100644 index 0d14f545a..000000000 --- a/modules/nats/internal/busarchive/archive.go +++ /dev/null @@ -1,162 +0,0 @@ -package busarchive - -import ( - "archive/tar" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "io" - "os" - "path/filepath" -) - -// Archive is a snapshot read back: its manifest, and each file beside it unpacked into a scratch -// directory of its own, removed by Close. -type Archive struct { - Manifest Manifest - dir string - files map[string]string -} - -// ReadArchive unpacks a snapshot. Only the files the manifest names are kept, and only under names -// that are one stream's directory: an archive is data, and a path in it is never trusted to say where -// on the machine a file goes. -func ReadArchive(r io.Reader, scratch string) (*Archive, error) { - dir, err := os.MkdirTemp(scratch, "mesh-nats-restore-*") - if err != nil { - return nil, err - } - a := &Archive{dir: dir, files: map[string]string{}} - tr := tar.NewReader(r) - first := true - for { - hdr, err := tr.Next() - if err == io.EOF { - break - } - if err != nil { - a.Close() - return nil, fmt.Errorf("the archive does not read: %w", err) - } - if first { - if hdr.Name != ManifestName { - a.Close() - return nil, fmt.Errorf("the archive starts with %q, not its manifest: it is not a snapshot of the bus", hdr.Name) - } - raw, err := io.ReadAll(io.LimitReader(tr, 64<<20)) - if err != nil { - a.Close() - return nil, err - } - if err := json.Unmarshal(raw, &a.Manifest); err != nil { - a.Close() - return nil, fmt.Errorf("its manifest: %w", err) - } - if a.Manifest.Format != 1 { - a.Close() - return nil, fmt.Errorf("its manifest is format %d; this program reads format 1", a.Manifest.Format) - } - first = false - continue - } - if !a.named(hdr.Name) { - a.Close() - return nil, fmt.Errorf("the archive holds %q, which its manifest does not name", hdr.Name) - } - local := filepath.Join(dir, fmt.Sprintf("%04d", len(a.files))) - f, err := os.OpenFile(local, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) - if err != nil { - a.Close() - return nil, err - } - _, err = io.Copy(f, tr) - f.Close() - if err != nil { - a.Close() - return nil, err - } - a.files[hdr.Name] = local - } - if first { - a.Close() - return nil, errors.New("the archive is empty") - } - for _, s := range a.Manifest.Streams { - if !ValidName(s.Name) || s.Meta != "streams/"+s.Name+"/backup.json" || s.Archive != "streams/"+s.Name+"/stream.tar.s2" { - a.Close() - return nil, fmt.Errorf("the manifest names a stream %q at paths that are not its own", s.Name) - } - } - return a, nil -} - -func (a *Archive) named(name string) bool { - for _, s := range a.Manifest.Streams { - if name == s.Meta || name == s.Archive { - return true - } - } - return false -} - -// Close removes the unpacked files. -func (a *Archive) Close() { os.RemoveAll(a.dir) } - -// Verify holds every file to the manifest's size and checksum, and reads every stream's archive to -// its end. -func (a *Archive) Verify() error { - for _, s := range a.Manifest.Streams { - for _, f := range []struct{ name, sum string }{{s.Meta, s.MetaSHA256}, {s.Archive, s.ArchiveSHA256}} { - local, ok := a.files[f.name] - if !ok { - return fmt.Errorf("%s is named in the manifest and missing from the archive", f.name) - } - got, size, err := sha256Of(local) - if err != nil { - return err - } - if got != f.sum { - return fmt.Errorf("%s does not match the checksum its manifest gives", f.name) - } - if f.name == s.Archive && size != s.ArchiveBytes { - return fmt.Errorf("%s is %d bytes; its manifest says %d", f.name, size, s.ArchiveBytes) - } - } - af, err := os.Open(a.files[s.Archive]) - if err != nil { - return err - } - err = ReadsThrough(af) - af.Close() - if err != nil { - return fmt.Errorf("%s does not read: %w", s.Archive, err) - } - } - return nil -} - -func sha256Of(path string) (string, int64, error) { - f, err := os.Open(path) - if err != nil { - return "", 0, err - } - defer f.Close() - h := sha256.New() - n, err := io.Copy(h, f) - if err != nil { - return "", 0, err - } - return hex.EncodeToString(h.Sum(nil)), n, nil -} - -// Open is one stream's meta and archive. -func (a *Archive) Open(s StreamEntry) (meta []byte, archive *os.File, err error) { - meta, err = os.ReadFile(a.files[s.Meta]) - if err != nil { - return nil, nil, err - } - archive, err = os.Open(a.files[s.Archive]) - return meta, archive, err -} diff --git a/modules/nats/internal/busarchive/types.go b/modules/nats/internal/busarchive/types.go deleted file mode 100644 index 12ca5eba3..000000000 --- a/modules/nats/internal/busarchive/types.go +++ /dev/null @@ -1,86 +0,0 @@ -// Package busarchive is the bus's snapshot archive: its manifest, reading one back and holding it to -// its checksums. The snapshot program (cmd/mesh-nats-snapshot) writes and restores with it and the bus's -// tool nats_snapshot_verify (cmd/nats-tools) checks with it, so the two never differ on what a good -// snapshot is. -package busarchive - -import ( - "archive/tar" - "io" - "strings" - "time" - - "github.com/klauspost/compress/s2" -) - -// Manifest is the first entry of an archive: what was taken, when, how long it took, and the checksum -// of every file beside it. It is what `verify` and `restore` hold the archive to. -type Manifest struct { - Format int `json:"format"` - Server string `json:"server_version"` - Started time.Time `json:"started"` - Finished time.Time `json:"finished"` - Seconds float64 `json:"seconds"` - // Messages and StreamBytes are the streams' own totals at their snapshots; ArchiveBytes what the - // archives take, compressed. - Messages uint64 `json:"messages"` - StreamBytes int64 `json:"stream_bytes"` - ArchiveBytes int64 `json:"archive_bytes"` - Streams []StreamEntry `json:"streams"` - // Skipped are streams that hold nothing across a restart (memory storage), said rather than - // silently left out. - Skipped []Skipped `json:"skipped,omitempty"` -} - -// StreamEntry is one stream as it was snapshotted. -type StreamEntry struct { - Name string `json:"name"` - Subjects []string `json:"subjects,omitempty"` - Retention string `json:"retention,omitempty"` - Messages uint64 `json:"messages"` - Bytes uint64 `json:"bytes"` - FirstSeq uint64 `json:"first_seq"` - LastSeq uint64 `json:"last_seq"` - Deleted int `json:"deleted,omitempty"` - Consumers int `json:"consumers"` - // Meta is the server's own description of the stream (its configuration and state), in the - // shape the restore API and the nats CLI take back: backup.json. - Meta string `json:"meta"` - MetaSHA256 string `json:"meta_sha256"` - // Archive is the server's snapshot of the stream, as the server sent it: stream.tar.s2. - Archive string `json:"archive"` - ArchiveBytes int64 `json:"archive_bytes"` - ArchiveSHA256 string `json:"archive_sha256"` - Seconds float64 `json:"seconds"` -} - -// Skipped is a stream not snapshotted, and why. -type Skipped struct { - Name string `json:"name"` - Why string `json:"why"` -} - -// ManifestName is the archive's first entry. -const ManifestName = "manifest.json" - -// ReadsThrough is whether a stream's archive decompresses and lists to its end. -func ReadsThrough(r io.Reader) error { - tr := tar.NewReader(s2.NewReader(r)) - for { - _, err := tr.Next() - if err == io.EOF { - return nil - } - if err != nil { - return err - } - if _, err := io.Copy(io.Discard, tr); err != nil { - return err - } - } -} - -// ValidName is a stream name that is one safe path component. -func ValidName(name string) bool { - return name != "" && name != "." && name != ".." && !strings.ContainsAny(name, "/\\\x00 *>") -} diff --git a/modules/nats/internal/busarchive/verifyfile.go b/modules/nats/internal/busarchive/verifyfile.go deleted file mode 100644 index fdb4a2fcf..000000000 --- a/modules/nats/internal/busarchive/verifyfile.go +++ /dev/null @@ -1,109 +0,0 @@ -package busarchive - -import ( - "errors" - "fmt" - "io/fs" - "os" - "path/filepath" - "strings" - "time" -) - -// StreamReport is one stream of a verified snapshot. -type StreamReport struct { - Name string `json:"name"` - LastSeq uint64 `json:"last_seq"` -} - -// Report is the answer of VerifyFile: which file was held to its manifest, and whether it passed. -type Report struct { - File string `json:"file"` - Modified time.Time `json:"modified"` - Size int64 `json:"size"` - Taken *time.Time `json:"taken,omitempty"` - Streams []StreamReport `json:"streams"` - Skipped []Skipped `json:"skipped,omitempty"` - Passed bool `json:"passed"` - // Fault is the first thing wrong, empty when the snapshot passed. - Fault string `json:"fault,omitempty"` -} - -// VerifyFile verifies one snapshot inside dir: the file called name, or the newest *.tar when name is -// empty. Nothing outside dir is read and nothing is written there; a snapshot that fails is an answer -// (Passed false, the first Fault), and only a directory without a snapshot is an error. -func VerifyFile(dir, name string) (*Report, error) { - path, info, err := pick(dir, name) - if err != nil { - return nil, err - } - r := &Report{File: filepath.Base(path), Modified: info.ModTime().UTC(), Size: info.Size(), Streams: []StreamReport{}} - f, err := os.Open(path) - if err != nil { - r.Fault = err.Error() - return r, nil - } - defer f.Close() - a, err := ReadArchive(f, "") - if err != nil { - r.Fault = err.Error() - return r, nil - } - defer a.Close() - taken := a.Manifest.Finished - r.Taken = &taken - r.Skipped = a.Manifest.Skipped - for _, s := range a.Manifest.Streams { - r.Streams = append(r.Streams, StreamReport{Name: s.Name, LastSeq: s.LastSeq}) - } - if err := a.Verify(); err != nil { - r.Fault = err.Error() - return r, nil - } - r.Passed = true - return r, nil -} - -// pick names the regular file to verify, and nothing outside dir: a name is one path component, and a -// symbolic link is not followed out of the directory. -func pick(dir, name string) (string, fs.FileInfo, error) { - if dir == "" { - return "", nil, errors.New("the snapshots directory is not set") - } - if name != "" { - if name != filepath.Base(name) || name == "." || name == ".." || strings.ContainsRune(name, 0) { - return "", nil, fmt.Errorf("%q is not a file name inside the snapshots directory", name) - } - p := filepath.Join(dir, name) - info, err := os.Lstat(p) - if err != nil { - return "", nil, fmt.Errorf("no snapshot %s in the snapshots directory: %w", name, err) - } - if !info.Mode().IsRegular() { - return "", nil, fmt.Errorf("%s is not a regular file", name) - } - return p, info, nil - } - entries, err := os.ReadDir(dir) - if err != nil { - return "", nil, fmt.Errorf("the snapshots directory does not read: %w", err) - } - var best string - var bestInfo fs.FileInfo - for _, e := range entries { - if !strings.HasSuffix(e.Name(), ".tar") { - continue - } - info, err := e.Info() - if err != nil || !info.Mode().IsRegular() { - continue - } - if bestInfo == nil || info.ModTime().After(bestInfo.ModTime()) { - best, bestInfo = e.Name(), info - } - } - if bestInfo == nil { - return "", nil, errors.New("the snapshots directory holds no .tar snapshot") - } - return filepath.Join(dir, best), bestInfo, nil -} diff --git a/modules/nats/internal/busarchive/verifyfile_test.go b/modules/nats/internal/busarchive/verifyfile_test.go deleted file mode 100644 index be62737fb..000000000 --- a/modules/nats/internal/busarchive/verifyfile_test.go +++ /dev/null @@ -1,173 +0,0 @@ -package busarchive - -import ( - "archive/tar" - "bytes" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/klauspost/compress/s2" -) - -func sum(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) } - -// streamArchive is a stream's archive as the server sends it: a tar of one file, s2-compressed. -func streamArchive(t *testing.T) []byte { - t.Helper() - var raw bytes.Buffer - tw := tar.NewWriter(&raw) - body := []byte(strings.Repeat("a message of the stream\n", 50)) - if err := tw.WriteHeader(&tar.Header{Name: "msgs.blk", Mode: 0o600, Size: int64(len(body))}); err != nil { - t.Fatal(err) - } - tw.Write(body) - tw.Close() - var out bytes.Buffer - w := s2.NewWriter(&out) - w.Write(raw.Bytes()) - w.Close() - return out.Bytes() -} - -// snapshotTar builds a snapshot of two streams; the mutations change what the manifest says or what is -// stored, so each fault is made in the one place it is found. -func snapshotTar(t *testing.T, mutate func(m *Manifest, files map[string][]byte)) []byte { - t.Helper() - files := map[string][]byte{} - m := Manifest{Format: 1, Server: "2.11.17", Finished: time.Date(2026, 10, 22, 3, 0, 0, 0, time.UTC)} - for i, name := range []string{"EVENTS", "KV_conditions"} { - meta := []byte(`{"config":{"name":"` + name + `"}}`) - arc := streamArchive(t) - e := StreamEntry{Name: name, LastSeq: uint64(10 * (i + 1)), - Meta: "streams/" + name + "/backup.json", MetaSHA256: sum(meta), - Archive: "streams/" + name + "/stream.tar.s2", ArchiveBytes: int64(len(arc)), ArchiveSHA256: sum(arc)} - m.Streams = append(m.Streams, e) - files[e.Meta], files[e.Archive] = meta, arc - } - if mutate != nil { - mutate(&m, files) - } - var out bytes.Buffer - tw := tar.NewWriter(&out) - put := func(name string, b []byte) { - if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(b))}); err != nil { - t.Fatal(err) - } - tw.Write(b) - } - raw, _ := json.Marshal(m) - put(ManifestName, raw) - for _, s := range m.Streams { - put(s.Meta, files[s.Meta]) - put(s.Archive, files[s.Archive]) - } - tw.Close() - return out.Bytes() -} - -func put(t *testing.T, dir, name string, b []byte, at time.Time) { - t.Helper() - p := filepath.Join(dir, name) - if err := os.WriteFile(p, b, 0o600); err != nil { - t.Fatal(err) - } - os.Chtimes(p, at, at) -} - -func TestAGoodSnapshotPassesAndSaysItsStreams(t *testing.T) { - dir := t.TempDir() - good := snapshotTar(t, nil) - put(t, dir, "bus.tar", good, time.Now()) - r, err := VerifyFile(dir, "") - if err != nil { - t.Fatal(err) - } - if !r.Passed || r.Fault != "" || r.File != "bus.tar" || r.Size != int64(len(good)) { - t.Fatalf("%+v", r) - } - if r.Taken == nil || !r.Taken.Equal(time.Date(2026, 10, 22, 3, 0, 0, 0, time.UTC)) { - t.Errorf("taken %v", r.Taken) - } - if len(r.Streams) != 2 || r.Streams[0] != (StreamReport{"EVENTS", 10}) || r.Streams[1] != (StreamReport{"KV_conditions", 20}) { - t.Errorf("streams %+v", r.Streams) - } -} - -func TestACorruptedChecksumFailsNamingTheFile(t *testing.T) { - dir := t.TempDir() - put(t, dir, "bus.tar", snapshotTar(t, func(m *Manifest, files map[string][]byte) { - m.Streams[1].ArchiveSHA256 = sum([]byte("something else")) - }), time.Now()) - r, err := VerifyFile(dir, "bus.tar") - if err != nil { - t.Fatal(err) - } - if r.Passed || !strings.Contains(r.Fault, "streams/KV_conditions/stream.tar.s2") || !strings.Contains(r.Fault, "checksum") { - t.Fatalf("%+v", r) - } - if len(r.Streams) != 2 { - t.Errorf("a failed snapshot still says its streams: %+v", r.Streams) - } -} - -func TestATruncatedSnapshotFails(t *testing.T) { - dir := t.TempDir() - good := snapshotTar(t, nil) - put(t, dir, "bus.tar", good[:len(good)*2/3], time.Now()) - r, err := VerifyFile(dir, "") - if err != nil { - t.Fatal(err) - } - if r.Passed || r.Fault == "" { - t.Fatalf("a truncated snapshot passed: %+v", r) - } -} - -func TestAnArchiveThatDoesNotReadToItsEndFailsEvenWithMatchingChecksums(t *testing.T) { - dir := t.TempDir() - put(t, dir, "bus.tar", snapshotTar(t, func(m *Manifest, files map[string][]byte) { - bad := []byte("not an s2 stream at all") - s := &m.Streams[0] - files[s.Archive] = bad - s.ArchiveSHA256, s.ArchiveBytes = sum(bad), int64(len(bad)) - }), time.Now()) - r, _ := VerifyFile(dir, "") - if r.Passed || !strings.Contains(r.Fault, "does not read") { - t.Fatalf("%+v", r) - } -} - -func TestTheNewestTarIsVerifiedAndOnlyInsideTheDirectory(t *testing.T) { - dir := t.TempDir() - now := time.Now() - put(t, dir, "old.tar", []byte("junk"), now.Add(-48*time.Hour)) - put(t, dir, "bus.tar", snapshotTar(t, nil), now) - put(t, dir, "bus.tar.partial", []byte("half"), now.Add(time.Hour)) - r, err := VerifyFile(dir, "") - if err != nil || r.File != "bus.tar" || !r.Passed { - t.Fatalf("%+v %v", r, err) - } - if r, _ := VerifyFile(dir, "old.tar"); r.Passed || r.File != "old.tar" { - t.Errorf("a named file is the one checked: %+v", r) - } - outside := t.TempDir() - put(t, outside, "x.tar", snapshotTar(t, nil), now) - for _, name := range []string{"../" + filepath.Base(outside) + "/x.tar", outside + "/x.tar", ".."} { - if _, err := VerifyFile(dir, name); err == nil { - t.Errorf("%q was read", name) - } - } - os.Symlink(filepath.Join(outside, "x.tar"), filepath.Join(dir, "link.tar")) - if _, err := VerifyFile(dir, "link.tar"); err == nil { - t.Error("a symbolic link out of the directory was followed") - } - if _, err := VerifyFile(t.TempDir(), ""); err == nil { - t.Error("an empty directory is not a snapshot") - } -} diff --git a/modules/nats/internal/bussnap/restore.go b/modules/nats/internal/bussnap/restore.go deleted file mode 100644 index 030f8d7b1..000000000 --- a/modules/nats/internal/bussnap/restore.go +++ /dev/null @@ -1,158 +0,0 @@ -package bussnap - -import ( - "archive/tar" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "os" - "time" - - "nats-tools/internal/busarchive" - - "github.com/nats-io/nats.go" -) - -// The keeper's restore in place of a snapshot it took (design 47 G §681): each stream of the file put back under its own -// name, only where no stream of that name is live. - -// APIStreamDelete is the subject the keeper removes its own partial stream with. -const APIStreamDelete = "$JS.API.STREAM.DELETE." - -const restoreChunk = 64 * 1024 - -// ManifestOf reads the manifest of a snapshot file: the streams it holds, without unpacking them. -func ManifestOf(path string) (*busarchive.Manifest, error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - defer f.Close() - tr := tar.NewReader(f) - hdr, err := tr.Next() - if err != nil { - return nil, fmt.Errorf("the snapshot does not read: %w", err) - } - if hdr.Name != busarchive.ManifestName { - return nil, fmt.Errorf("the snapshot starts with %q, not its manifest", hdr.Name) - } - raw, err := io.ReadAll(io.LimitReader(tr, 64<<20)) - if err != nil { - return nil, err - } - var m busarchive.Manifest - if err := json.Unmarshal(raw, &m); err != nil { - return nil, fmt.Errorf("its manifest: %w", err) - } - return &m, nil -} - -// Live says whether a stream of that name exists. -func Live(ctx context.Context, nc *nats.Conn, name string) (bool, error) { - err := Request(ctx, nc, APIStreamInfo+name, nil, nil) - var api *APIError - switch { - case err == nil: - return true, nil - case errors.As(err, &api) && api.Code == 404: - return false, nil - } - return false, err -} - -// DeleteStream removes a stream; one that is not there is no error. -func DeleteStream(ctx context.Context, nc *nats.Conn, name string) error { - err := Request(ctx, nc, APIStreamDelete+name, nil, nil) - var api *APIError - if errors.As(err, &api) && api.Code == 404 { - return nil - } - return err -} - -// RestoreFile puts every stream of the snapshot file back under its own name, each sent in chunks the server answers; it -// ends with the context's bound. A stream already there is the caller's to have refused beforehand. -func RestoreFile(ctx context.Context, nc *nats.Conn, path, scratch string, log func(string, ...any)) error { - f, err := os.Open(path) - if err != nil { - return err - } - defer f.Close() - a, err := busarchive.ReadArchive(f, scratch) - if err != nil { - return err - } - defer a.Close() - if err := a.Verify(); err != nil { - return err - } - for _, s := range a.Manifest.Streams { - started := time.Now() - if err := restoreStream(ctx, nc, a, s); err != nil { - return fmt.Errorf("stream %s: %w", s.Name, err) - } - log("%s: restored in %.2fs", s.Name, time.Since(started).Seconds()) - } - return nil -} - -func restoreStream(ctx context.Context, nc *nats.Conn, a *busarchive.Archive, s busarchive.StreamEntry) error { - meta, archive, err := a.Open(s) - if err != nil { - return err - } - defer archive.Close() - var req map[string]json.RawMessage - if err := json.Unmarshal(meta, &req); err != nil || req["config"] == nil { - return errors.New("its meta is not a stream's configuration and state") - } - var resp struct { - DeliverSubject string `json:"deliver_subject"` - } - if err := Request(ctx, nc, APIStreamRestore+s.Name, req, &resp); err != nil { - return err - } - if resp.DeliverSubject == "" { - return errors.New("the server gave nowhere to send the archive") - } - buf := make([]byte, restoreChunk) - for { - n, err := archive.Read(buf) - if n > 0 { - reply, rerr := nc.RequestWithContext(ctx, resp.DeliverSubject, buf[:n]) - if rerr != nil { - return fmt.Errorf("sending the archive: %w", rerr) - } - if len(reply.Data) > 0 { - return fmt.Errorf("the server refused the archive: %s", Truncate(reply.Data)) - } - } - if err == io.EOF { - break - } - if err != nil { - return err - } - } - final, err := nc.RequestWithContext(ctx, resp.DeliverSubject, nil) - if err != nil { - return fmt.Errorf("finishing the restore: %w", err) - } - var created struct { - Error *APIError `json:"error"` - State StreamState `json:"state"` - } - if err := json.Unmarshal(final.Data, &created); err != nil { - return fmt.Errorf("the server's last answer is not JSON: %q", Truncate(final.Data)) - } - if created.Error != nil { - return created.Error - } - if created.State.LastSeq < s.LastSeq || (s.Messages > 0 && created.State.Messages == 0) { - return fmt.Errorf("restored with %d message(s) up to sequence %d; the snapshot held %d up to %d", - created.State.Messages, created.State.LastSeq, s.Messages, s.LastSeq) - } - return nil -} diff --git a/modules/nats/internal/bussnap/snapshot.go b/modules/nats/internal/bussnap/snapshot.go deleted file mode 100644 index dd4517d40..000000000 --- a/modules/nats/internal/bussnap/snapshot.go +++ /dev/null @@ -1,433 +0,0 @@ -// Package bussnap takes a snapshot of the bus's streams with the server's own snapshot API (novox/hq ADR 0235): the -// nightly dump of every stream, and the bus module's keeper's snapshot of one entry before it is rebuilt or removed -// (design 25 § 12.3). -package bussnap - -import ( - "archive/tar" - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "io" - "os" - "path/filepath" - "slices" - "sort" - "strings" - "sync" - "time" - - "nats-tools/internal/busarchive" - - "github.com/nats-io/nats.go" -) - -// The JetStream API's subjects this program speaks. Each is granted to the bus's own module and -// nothing else is (mesh-controller internal/broker, BusSnapshotGrants); a test holds the two lists -// equal against a real server. -const ( - APIStreamNames = "$JS.API.STREAM.NAMES" - APIStreamInfo = "$JS.API.STREAM.INFO." - APIStreamSnapshot = "$JS.API.STREAM.SNAPSHOT." - APIStreamRestore = "$JS.API.STREAM.RESTORE." -) - -// ChunkSize is what the server is asked to send at a time: its own default. The server sends up to -// an 8 MiB window ahead of the acknowledgements and waits for them past it, so one stream costs the -// server at most that much in flight, whatever its size. -const ChunkSize = 128 * 1024 - -// SnapshotOptions bounds a snapshot. -type SnapshotOptions struct { - // StreamTimeout is the longest one stream may take; past it the snapshot fails, naming it. - StreamTimeout time.Duration - // Pause is a rest between two streams, so a night's snapshot is never one long burst. - Pause time.Duration - // TempDir is where one stream's archive waits until it is written out; the system's when empty. - TempDir string - Log func(format string, args ...any) - // Streams, when set, are the only streams taken (the keeper's snapshot of one entry); every stream when empty. - Streams []string -} - -// APIError is the error every JetStream API answer may carry. -type APIError struct { - Code int `json:"code"` - ErrCode int `json:"err_code"` - Description string `json:"description"` -} - -func (e *APIError) Error() string { return fmt.Sprintf("%s (%d)", e.Description, e.Code) } - -// StreamState is what this program reads of a stream's state; the whole of it is kept as the server -// gave it, in the stream's meta. -type StreamState struct { - Messages uint64 `json:"messages"` - Bytes uint64 `json:"bytes"` - FirstSeq uint64 `json:"first_seq"` - LastSeq uint64 `json:"last_seq"` - NumDelete int `json:"num_deleted"` - Consumers int `json:"consumer_count"` -} - -type streamConfig struct { - Name string `json:"name"` - Subjects []string `json:"subjects"` - Retention string `json:"retention"` - Storage string `json:"storage"` -} - -// Request asks the JetStream API and decodes its answer, refusing an answer that carries an error. -func Request(ctx context.Context, nc *nats.Conn, subject string, body any, into any) error { - var payload []byte - if body != nil { - var err error - if payload, err = json.Marshal(body); err != nil { - return err - } - } - rctx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - msg, err := nc.RequestWithContext(rctx, subject, payload) - if err != nil { - if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, nats.ErrTimeout) || errors.Is(err, nats.ErrNoResponders) { - return fmt.Errorf("%s was not answered (%v): the bus refuses a subject this user is not granted by not answering it", subject, err) - } - return fmt.Errorf("%s: %w", subject, err) - } - var wrapped struct { - Error *APIError `json:"error"` - } - if err := json.Unmarshal(msg.Data, &wrapped); err != nil { - return fmt.Errorf("%s answered something that is not JSON: %q", subject, Truncate(msg.Data)) - } - if wrapped.Error != nil { - return fmt.Errorf("%s: %w", subject, wrapped.Error) - } - if into != nil { - return json.Unmarshal(msg.Data, into) - } - return nil -} - -func Truncate(b []byte) string { - if len(b) > 200 { - return string(b[:200]) + "…" - } - return string(b) -} - -// StreamNames is every stream in the account, in name order, all pages of them. -func StreamNames(ctx context.Context, nc *nats.Conn) ([]string, error) { - var names []string - for { - var page struct { - Total int `json:"total"` - Offset int `json:"offset"` - Limit int `json:"limit"` - Streams []string `json:"streams"` - } - if err := Request(ctx, nc, APIStreamNames, map[string]int{"offset": len(names)}, &page); err != nil { - return nil, err - } - names = append(names, page.Streams...) - if len(page.Streams) == 0 || len(names) >= page.Total { - break - } - } - sort.Strings(names) - return names, nil -} - -// Snapshot writes every stream, one at a time, as a tar: the manifest first, then each stream's -// meta and archive under streams//. Any stream failing fails the whole: a partial copy of the -// bus written where a whole one is expected is the silent failure this exists to prevent. -func Snapshot(ctx context.Context, nc *nats.Conn, out io.Writer, o SnapshotOptions) (*busarchive.Manifest, error) { - if o.StreamTimeout <= 0 { - o.StreamTimeout = 10 * time.Minute - } - if o.Log == nil { - o.Log = func(string, ...any) {} - } - m := &busarchive.Manifest{Format: 1, Server: nc.ConnectedServerVersion(), Started: time.Now().UTC()} - names, err := StreamNames(ctx, nc) - if err != nil { - return nil, err - } - if len(o.Streams) > 0 { - var only []string - for _, n := range names { - if slices.Contains(o.Streams, n) { - only = append(only, n) - } - } - if len(only) != len(o.Streams) { - return nil, fmt.Errorf("the bus holds %v of the streams %v asked for: a snapshot missing one would pass for a whole one", only, o.Streams) - } - names = only - } - if len(names) == 0 { - // The mesh's own streams always exist once the controller has run; a bus with none is not - // one to call backed up. - return nil, errors.New("the bus holds no streams; a snapshot of nothing would pass for a backup") - } - - type taken struct { - entry busarchive.StreamEntry - meta []byte - archive *os.File - } - var files []taken - defer func() { - for _, t := range files { - t.archive.Close() - os.Remove(t.archive.Name()) - } - }() - - for i, name := range names { - if i > 0 && o.Pause > 0 { - select { - case <-time.After(o.Pause): - case <-ctx.Done(): - return nil, ctx.Err() - } - } - var info struct { - Config streamConfig `json:"config"` - } - if err := Request(ctx, nc, APIStreamInfo+name, nil, &info); err != nil { - return nil, err - } - if info.Config.Storage == "memory" { - m.Skipped = append(m.Skipped, busarchive.Skipped{Name: name, Why: "memory storage: it holds nothing across a restart, and the server cannot snapshot it"}) - o.Log("%s: skipped, memory storage", name) - continue - } - f, err := os.CreateTemp(o.TempDir, "mesh-nats-snapshot-*") - if err != nil { - return nil, err - } - files = append(files, taken{archive: f}) - sctx, cancel := context.WithTimeout(ctx, o.StreamTimeout) - entry, meta, err := snapshotStream(sctx, nc, name, f) - cancel() - if err != nil { - return nil, fmt.Errorf("stream %s: %w", name, err) - } - files[len(files)-1].entry, files[len(files)-1].meta = entry, meta - m.Messages += entry.Messages - m.StreamBytes += int64(entry.Bytes) - m.ArchiveBytes += entry.ArchiveBytes - o.Log("%s: %d message(s), %s, last sequence %d, %d consumer(s) → %s in %.2fs", - name, entry.Messages, Human(int64(entry.Bytes)), entry.LastSeq, entry.Consumers, Human(entry.ArchiveBytes), entry.Seconds) - } - m.Finished = time.Now().UTC() - m.Seconds = m.Finished.Sub(m.Started).Seconds() - for _, t := range files { - m.Streams = append(m.Streams, t.entry) - } - - tw := tar.NewWriter(out) - manifest, err := json.MarshalIndent(m, "", " ") - if err != nil { - return nil, err - } - if err := writeEntry(tw, busarchive.ManifestName, append(manifest, '\n'), m.Finished); err != nil { - return nil, err - } - for _, t := range files { - if err := writeEntry(tw, t.entry.Meta, t.meta, m.Finished); err != nil { - return nil, err - } - if _, err := t.archive.Seek(0, io.SeekStart); err != nil { - return nil, err - } - hdr := &tar.Header{Name: t.entry.Archive, Mode: 0o600, Size: t.entry.ArchiveBytes, ModTime: m.Finished, Format: tar.FormatPAX} - if err := tw.WriteHeader(hdr); err != nil { - return nil, err - } - if _, err := io.Copy(tw, t.archive); err != nil { - return nil, err - } - } - if err := tw.Close(); err != nil { - return nil, err - } - return m, nil -} - -func writeEntry(tw *tar.Writer, name string, body []byte, at time.Time) error { - if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(body)), ModTime: at, Format: tar.FormatPAX}); err != nil { - return err - } - _, err := tw.Write(body) - return err -} - -// snapshotStream takes one stream's snapshot into f: the server's chunks, each acknowledged as it -// arrives so the server's window keeps moving, until the server's last, empty message. It answers -// the stream's entry (the archive's size and checksum included) and its meta. -// -// The protocol is the server's (nats-server jetstream_api.go, streamSnapshot): the request names a -// subject to deliver to; the server answers with the stream's configuration and its state at the -// snapshot, then sends the archive in chunks whose reply subject is the acknowledgement it waits for -// past its window — two seconds without one and it gives up with "408 No Flow Response". The end is -// an empty message whose status header says 204, or the error that ended it. -func snapshotStream(ctx context.Context, nc *nats.Conn, name string, f *os.File) (busarchive.StreamEntry, []byte, error) { - started := time.Now() - deliver := nc.NewRespInbox() - - hash := sha256.New() - var written int64 - var mu sync.Mutex - var failed error - done := make(chan struct{}) - var once sync.Once - finish := func(err error) { - mu.Lock() - if failed == nil { - failed = err - } - mu.Unlock() - once.Do(func() { close(done) }) - } - sub, err := nc.Subscribe(deliver, func(msg *nats.Msg) { - if len(msg.Data) == 0 { - if status := msg.Header.Get("Status"); status != "" && status != "204" { - finish(fmt.Errorf("the server ended the snapshot: %s %s", status, msg.Header.Get("Description"))) - return - } - finish(nil) - return - } - // Acknowledged before it is written: the chunk is already in memory, and a slow disk must not - // read to the server as a reader that went away. - if msg.Reply != "" { - if err := nc.Publish(msg.Reply, nil); err != nil { - finish(fmt.Errorf("acknowledging a chunk: %w", err)) - return - } - } - if _, err := f.Write(msg.Data); err != nil { - finish(fmt.Errorf("writing the archive: %w", err)) - return - } - hash.Write(msg.Data) - written += int64(len(msg.Data)) - }) - if err != nil { - return busarchive.StreamEntry{}, nil, err - } - defer sub.Unsubscribe() - // Every chunk is held until the callback has it; the window bounds how many there can be. - if err := sub.SetPendingLimits(-1, -1); err != nil { - return busarchive.StreamEntry{}, nil, err - } - if err := nc.FlushWithContext(ctx); err != nil { - return busarchive.StreamEntry{}, nil, err - } - - var resp struct { - Config json.RawMessage `json:"config"` - State json.RawMessage `json:"state"` - } - req := map[string]any{"deliver_subject": deliver, "chunk_size": ChunkSize} - if err := Request(ctx, nc, APIStreamSnapshot+name, req, &resp); err != nil { - return busarchive.StreamEntry{}, nil, err - } - select { - case <-done: - case <-ctx.Done(): - return busarchive.StreamEntry{}, nil, fmt.Errorf("not finished within its bound: %w", ctx.Err()) - } - // The subscription is drained before the archive is read back, so no callback is still writing. - _ = sub.Unsubscribe() - mu.Lock() - err = failed - mu.Unlock() - if err != nil { - return busarchive.StreamEntry{}, nil, err - } - if written == 0 { - return busarchive.StreamEntry{}, nil, errors.New("the server sent an empty archive") - } - if err := f.Sync(); err != nil { - return busarchive.StreamEntry{}, nil, err - } - - var cfg streamConfig - var state StreamState - if err := json.Unmarshal(resp.Config, &cfg); err != nil { - return busarchive.StreamEntry{}, nil, fmt.Errorf("its configuration: %w", err) - } - if err := json.Unmarshal(resp.State, &state); err != nil { - return busarchive.StreamEntry{}, nil, fmt.Errorf("its state: %w", err) - } - // The archive reads to its end — s2 frames, then a tar — before it is called a snapshot. - if _, err := f.Seek(0, io.SeekStart); err != nil { - return busarchive.StreamEntry{}, nil, err - } - if err := busarchive.ReadsThrough(f); err != nil { - return busarchive.StreamEntry{}, nil, fmt.Errorf("the archive the server sent does not read: %w", err) - } - - // The meta, in the shape the restore API takes and the nats CLI writes (backup.json): the - // server's own configuration and state, kept as raw JSON so a field this program does not know - // is never dropped on the way back. - meta, err := json.MarshalIndent(map[string]json.RawMessage{"config": resp.Config, "state": resp.State}, "", " ") - if err != nil { - return busarchive.StreamEntry{}, nil, err - } - metaSum := sha256.Sum256(meta) - return busarchive.StreamEntry{ - Name: name, Subjects: cfg.Subjects, Retention: cfg.Retention, - Messages: state.Messages, Bytes: state.Bytes, FirstSeq: state.FirstSeq, LastSeq: state.LastSeq, - Deleted: state.NumDelete, Consumers: state.Consumers, - Meta: "streams/" + name + "/backup.json", MetaSHA256: hex.EncodeToString(metaSum[:]), - Archive: "streams/" + name + "/stream.tar.s2", ArchiveBytes: written, - ArchiveSHA256: hex.EncodeToString(hash.Sum(nil)), Seconds: time.Since(started).Seconds(), - }, meta, nil -} - -// Human is a size in bytes as a person reads it: 1.5 MiB. -func Human(n int64) string { - const unit = 1024 - if n < unit { - return fmt.Sprintf("%d B", n) - } - div, exp := int64(unit), 0 - for v := n / unit; v >= unit; v /= unit { - div *= unit - exp++ - } - return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp]) -} - -// Into is the keeper's snapshot of some streams before it rebuilds or removes them (design 25 § 12.3): one tar under -// dir, named for the streams and the time, written whole before its name is given, so a partial one is never taken -// for it. It answers the file's path. -func Into(nc *nats.Conn, dir string) func(ctx context.Context, streams []string) (string, error) { - return func(ctx context.Context, streams []string) (string, error) { - if dir == "" { - return "", errors.New("no snapshots directory is set (MESH_NATS_SNAPSHOTS_DIR), so no snapshot is taken and nothing is removed") - } - name := filepath.Join(dir, "entry-"+strings.Join(streams, "+")+"-"+time.Now().UTC().Format("20060102T150405.000000Z")+".tar") - f, err := os.CreateTemp(dir, ".entry-*.partial") - if err != nil { - return "", err - } - defer os.Remove(f.Name()) - if _, err := Snapshot(ctx, nc, f, SnapshotOptions{Streams: streams, TempDir: dir}); err != nil { - f.Close() - return "", err - } - if err := f.Close(); err != nil { - return "", err - } - return name, os.Rename(f.Name(), name) - } -} diff --git a/modules/nats/internal/keeper/jetstream.go b/modules/nats/internal/keeper/jetstream.go index 5ff2d51c0..c3e20079f 100644 --- a/modules/nats/internal/keeper/jetstream.go +++ b/modules/nats/internal/keeper/jetstream.go @@ -10,7 +10,7 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" - "nats-tools/internal/bussnap" + "git.novox.be/novox/mesh-sdk/go/bussnap" ) // JetStream is the bus as the keeper's own account reaches it (Bus): each object a stream, a key-value bucket or an From bd2997cf3666e851c35bc45d92175ffcc93fd351 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 01:30:00 +0100 Subject: [PATCH 08/17] A-129: kept-points says the unrestored asides; recovery_kept_points answers them as the health check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The module's declared health check names recovery_kept_points: it answers what kept-points answers with the health answer beside it, unhealthy while an aside only it holds the data of is listed. Builds: design 47 G A-129, §554, §657 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-recovery-tools/main.go | 33 +++++++++++++++++-- .../cmd/mesh-recovery-tools/main_test.go | 25 ++++++++++++++ .../internal/recovery/control.go | 22 ++++++++++--- .../internal/recovery/restore_test.go | 4 +++ 4 files changed, 78 insertions(+), 6 deletions(-) diff --git a/modules/mesh-recovery/cmd/mesh-recovery-tools/main.go b/modules/mesh-recovery/cmd/mesh-recovery-tools/main.go index 6bd188dc5..efb4a2ddd 100644 --- a/modules/mesh-recovery/cmd/mesh-recovery-tools/main.go +++ b/modules/mesh-recovery/cmd/mesh-recovery-tools/main.go @@ -10,6 +10,7 @@ import ( "strings" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/health" ) func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } @@ -77,8 +78,7 @@ func Tools(ask Ask, callerOf func(map[string]any) string) []stdio.Tool { if err != nil { return nil, err } - var out any - return out, json.Unmarshal(raw, &out) + return keptPointsHealth(raw) }}, hand(ask, callerOf, "recovery_take", "take", "Take a restore point of one module's kept data now, at the form of the given fingerprint.", map[string]any{"module": module, "kept": kept, "fingerprint": str("the fingerprint whose form the point is taken at")}, "module", "kept", "fingerprint"), @@ -98,3 +98,32 @@ func Tools(ask Ask, callerOf func(map[string]any) string) []stdio.Tool { hand(ask, callerOf, "recovery_expire_points", "expire-points", "Remove the restore points past their keep days now, never the newest of a kept data.", map[string]any{}), } } + +// keptPointsHealth is what kept-points answers with the health answer beside it, which the module's declared health +// check reads (§554, §657, A-129): unhealthy while an aside only it holds the data of is listed as unrestored. +func keptPointsHealth(raw json.RawMessage) (map[string]any, error) { + var out map[string]any + if err := json.Unmarshal(raw, &out); err != nil { + return nil, err + } + var kp struct { + Unrestored []struct { + ID string `json:"id"` + Names []string `json:"names"` + } `json:"unrestored"` + } + if err := json.Unmarshal(raw, &kp); err != nil { + return nil, err + } + h := health.Healthy("No aside holds data the live bus lacks.") + if len(kp.Unrestored) > 0 { + var said []string + for _, a := range kp.Unrestored { + said = append(said, a.ID+" ("+strings.Join(a.Names, ", ")+")") + } + h = health.Unhealthy(fmt.Sprintf("%d aside(s) hold data whose live objects were removed and not restored: %s; recovery_restore_aside puts one back.", + len(kp.Unrestored), strings.Join(said, "; "))) + } + out["state"], out["healthy"], out["why"] = h.State, h.Healthy, h.Why + return out, nil +} diff --git a/modules/mesh-recovery/cmd/mesh-recovery-tools/main_test.go b/modules/mesh-recovery/cmd/mesh-recovery-tools/main_test.go index fe28938ea..ded8d73f7 100644 --- a/modules/mesh-recovery/cmd/mesh-recovery-tools/main_test.go +++ b/modules/mesh-recovery/cmd/mesh-recovery-tools/main_test.go @@ -6,6 +6,8 @@ import ( "slices" "strings" "testing" + + "git.novox.be/novox/mesh-sdk/go/health" ) // The bundle serves exactly the tools the manifest names, each named recovery_, and each acting tool calls its verb of the @@ -51,3 +53,26 @@ func TestEachToolCallsItsVerbWithTheProvedAsker(t *testing.T) { t.Errorf("tools %v, the manifest names %v", names, want) } } + +// recovery_kept_points answers what kept-points answers with the health answer beside it: unhealthy while an aside is +// unrestored, healthy otherwise, as the declared health check reads it (§657, A-129). +func TestKeptPointsIsTheHealthCheck(t *testing.T) { + for _, c := range []struct { + answer string + state string + }{ + {`{"points":[],"asides":[],"unrestored":[]}`, health.StateHealthy}, + {`{"points":[],"asides":[{"id":"x","names":["app_notes"],"size":1}],"unrestored":[{"id":"x","names":["app_notes"],"size":1}]}`, health.StateUnhealthy}, + } { + tools := Tools(func(string, any) (json.RawMessage, error) { return json.RawMessage(c.answer), nil }, func(map[string]any) string { return "" }) + out, err := tools[0].Run(map[string]any{}) + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(out) + a, err := health.Read(raw) + if err != nil || a.State != c.state || !strings.Contains(string(raw), `"unrestored"`) { + t.Errorf("%s answered %s: %+v, %v", c.answer, raw, a, err) + } + } +} diff --git a/modules/mesh-recovery/internal/recovery/control.go b/modules/mesh-recovery/internal/recovery/control.go index 868fd9f9e..66e725d4e 100644 --- a/modules/mesh-recovery/internal/recovery/control.go +++ b/modules/mesh-recovery/internal/recovery/control.go @@ -89,30 +89,44 @@ func (e *Engine) KeptPoints(module, kept string) (seat.KeptPointsAnswer, error) out.Running = &seat.Running{Act: e.running.ID, Step: e.running.Step, Since: e.running.Since, WaitsOn: e.running.WaitsOn} } e.mu.Unlock() + // The asides only an aside holds, which the health check fails on, read from the engine (A-129). + out.Unrestored = []seat.Aside{} + for _, x := range e.Unrestored() { + if (module == "" || x.Module == module) && (kept == "" || x.Kept == kept) { + out.Unrestored = append(out.Unrestored, x.Aside) + } + } return out, nil } // Unrestored are the asides whose record ends at `removed` that no running act holds (§554 health, §561): the live data // of those names is gone and only the aside holds it. -func (e *Engine) Unrestored() []seat.Aside { +func (e *Engine) Unrestored() []UnrestoredAside { _, recs, _ := e.Asides() e.mu.Lock() defer e.mu.Unlock() - var out []seat.Aside + var out []UnrestoredAside for _, r := range recs { held := e.running != nil && e.running.Module == r.Module && e.running.Kept == r.Kept if r.Step == stepRemoved && !held { - out = append(out, r.aside()) + out = append(out, UnrestoredAside{Module: r.Module, Kept: r.Kept, Aside: r.aside()}) } } return out } +// UnrestoredAside is an aside only it holds the data of, with the module and kept data it was taken of: the scope of the +// finding aside-unrestored (§561). +type UnrestoredAside struct { + Module, Kept string + Aside seat.Aside +} + // Start repairs what a restart cut off (§516, §564, §560): points under a temporary name are removed, an aside whose // record ends at `set aside` (the live data never touched) is removed, an aside that ends at `removed` is put back under // the live names, and every accepted act that never ended ends failed with `stopped`, so its caller asks again. It // answers the asides it could not put back (the finding aside-unrestored, §561). -func (e *Engine) Start(ctx context.Context) []seat.Aside { +func (e *Engine) Start(ctx context.Context) []UnrestoredAside { _ = filepath.WalkDir(filepath.Join(e.Dir, "points"), func(path string, d os.DirEntry, err error) error { if err == nil && d.IsDir() && strings.HasPrefix(d.Name(), ".tmp-") { _ = os.RemoveAll(path) diff --git a/modules/mesh-recovery/internal/recovery/restore_test.go b/modules/mesh-recovery/internal/recovery/restore_test.go index 2242705b1..853a68645 100644 --- a/modules/mesh-recovery/internal/recovery/restore_test.go +++ b/modules/mesh-recovery/internal/recovery/restore_test.go @@ -249,6 +249,10 @@ func TestAsideMovesTheLiveDataIntoAnAsideAndRestoreAsidePutsItBack(t *testing.T) if got := r.e.Unrestored(); len(got) != 1 { t.Errorf("unrestored %+v", got) } + // kept-points says it, which the tool and the health check read (A-129). + if kp, _ := r.e.KeptPoints("app", ""); len(kp.Unrestored) != 1 || kp.Unrestored[0].ID != sa.ID { + t.Errorf("kept-points unrestored %+v", kp.Unrestored) + } r.s.events = nil r.do(recovery.Act{Kind: recovery.KindRestoreAside, Module: "app", Kept: "notes", AsideID: sa.ID, Why: "done looking", Caller: "tools"}) if lastRestored(t, r).Outcome != "done" || r.b.streams["app_notes"] != "hello" || len(r.e.Unrestored()) != 0 { From 9e41f6380a1fc664fe2655d53f36ec594ee84125 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 01:37:55 +0100 Subject: [PATCH 09/17] mesh-recovery: the process, on its own account, answering the seat's verbs on the live bus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The process reads its broker account, answers the seat mesh-recovery's verbs itself with the caller the bus proved, takes and restores streams through mesh-sdk's bussnap, keeps the digests in its state, names a kept data's bus objects from its catalog entry as mesh-sdk names them, learns the fingerprint a module runs at from the node-engine seat's running and the bus module's dumps from the backup holders' item events, each read whole at its start, and asks the backup holder on its node to bring the points directory back. A restore whose restore-started the bus did not acknowledge reads and sets aside nothing. Builds: design 47 G §554, §516, §551, §556, §563, §567, §655, A-107, A-112, A-121, A-123, A-126, A-132 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../mesh-recovery/cmd/mesh-recovery/bus.go | 98 ++++++++ .../mesh-recovery/cmd/mesh-recovery/dump.go | 163 +++++++++++++ .../mesh-recovery/cmd/mesh-recovery/main.go | 152 ++++++++++++ .../cmd/mesh-recovery/main_test.go | 68 ++++++ .../mesh-recovery/cmd/mesh-recovery/seats.go | 225 ++++++++++++++++++ modules/mesh-recovery/go.mod | 3 + .../internal/recovery/fakes_test.go | 4 + .../internal/recovery/restore.go | 7 +- .../internal/recovery/restore_test.go | 14 ++ modules/mesh-recovery/module.json | 3 + 10 files changed, 735 insertions(+), 2 deletions(-) create mode 100644 modules/mesh-recovery/cmd/mesh-recovery/bus.go create mode 100644 modules/mesh-recovery/cmd/mesh-recovery/dump.go create mode 100644 modules/mesh-recovery/cmd/mesh-recovery/main.go create mode 100644 modules/mesh-recovery/cmd/mesh-recovery/main_test.go create mode 100644 modules/mesh-recovery/cmd/mesh-recovery/seats.go diff --git a/modules/mesh-recovery/cmd/mesh-recovery/bus.go b/modules/mesh-recovery/cmd/mesh-recovery/bus.go new file mode 100644 index 000000000..10dbac97c --- /dev/null +++ b/modules/mesh-recovery/cmd/mesh-recovery/bus.go @@ -0,0 +1,98 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io" + "os" + + seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" + "git.novox.be/novox/mesh-sdk/go/bussnap" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "mesh-recovery/internal/recovery" +) + +// liveBus is the bus as the recovery module's own account sees it (design 47 G §554): a stream's snapshot and restore +// through mesh-sdk's bussnap (§655, A-125), its deletion, and its durable consumers' positions. +type liveBus struct { + nc *nats.Conn + js jetstream.JetStream + scratch string + logf func(string, ...any) +} + +func (b liveBus) Objects(ctx context.Context, streams []string) ([]recovery.Object, error) { + var out []recovery.Object + for _, name := range streams { + s, err := b.js.Stream(ctx, name) + if errors.Is(err, jetstream.ErrStreamNotFound) { + out = append(out, recovery.Object{Stream: name}) + continue + } + if err != nil { + return nil, fmt.Errorf("stream %s: %w", name, err) + } + info, err := s.Info(ctx) + if err != nil { + return nil, fmt.Errorf("stream %s: %w", name, err) + } + o := recovery.Object{Stream: name, Exists: true, Bytes: int64(info.State.Bytes)} + names := s.ListConsumers(ctx) + for ci := range names.Info() { + if ci.Config.Durable == "" { + continue + } + o.Consumers = append(o.Consumers, seat.Consumer{Stream: name, Name: ci.Name, Sequence: int64(ci.AckFloor.Stream)}) + } + if err := names.Err(); err != nil { + return nil, fmt.Errorf("the consumers of %s: %w", name, err) + } + out = append(out, o) + } + return out, nil +} + +func (b liveBus) Take(ctx context.Context, stream string, w io.Writer) error { + _, err := bussnap.Snapshot(ctx, b.nc, w, bussnap.SnapshotOptions{Streams: []string{stream}, TempDir: b.scratch, Log: b.logf}) + return err +} + +func (b liveBus) Delete(ctx context.Context, stream string) error { + return bussnap.DeleteStream(ctx, b.nc, stream) +} + +func (b liveBus) Restore(ctx context.Context, file string) error { + scratch, err := os.MkdirTemp(b.scratch, ".restore-") + if err != nil { + return err + } + defer os.RemoveAll(scratch) + return bussnap.RestoreFile(ctx, b.nc, file, scratch, b.logf) +} + +// SetConsumers sets each durable consumer to its recorded sequence: the server's snapshot carries its consumers, so one +// restored with its stream stands as it was taken; one missing is made again from the sequence after its acknowledged one. +func (b liveBus) SetConsumers(ctx context.Context, cs []seat.Consumer) error { + for _, c := range cs { + s, err := b.js.Stream(ctx, c.Stream) + if err != nil { + return fmt.Errorf("stream %s: %w", c.Stream, err) + } + if _, err := s.Consumer(ctx, c.Name); err == nil { + continue + } else if !errors.Is(err, jetstream.ErrConsumerNotFound) { + return fmt.Errorf("the consumer %s of %s: %w", c.Name, c.Stream, err) + } + if _, err := s.CreateConsumer(ctx, jetstream.ConsumerConfig{Durable: c.Name, AckPolicy: jetstream.AckExplicitPolicy, + DeliverPolicy: jetstream.DeliverByStartSequencePolicy, OptStartSeq: uint64(c.Sequence) + 1}); err != nil { + return fmt.Errorf("the consumer %s of %s at %d: %w", c.Name, c.Stream, c.Sequence, err) + } + } + return nil +} + +var _ recovery.Bus = liveBus{} diff --git a/modules/mesh-recovery/cmd/mesh-recovery/dump.go b/modules/mesh-recovery/cmd/mesh-recovery/dump.go new file mode 100644 index 000000000..3fb2d9b83 --- /dev/null +++ b/modules/mesh-recovery/cmd/mesh-recovery/dump.go @@ -0,0 +1,163 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/keys" + + "github.com/nats-io/nats.go/jetstream" + + "mesh-recovery/internal/recovery" +) + +// The seat node-backup's item events (design 47 G §554, §563), each kept last per subject on a stream of its own. +const ( + backupSeat = "node-backup" + itemStarted = "item-started" + itemEnded = "item-ended" + itemStartedSteam = "SEAT_node-backup_item-started" + itemEndedStream = "SEAT_node-backup_item-ended" +) + +func itemSubject(event, node, module, item string) string { + return "mesh.seat." + backupSeat + ".event." + event + "." + node + "." + module + "." + item +} + +// dumpView is what the backup holders say of each item, `..` to the `at` of its last start and of its +// last end, read as §556 has it. A dump of an item runs while its last start is newer than its last end. +type dumpView struct { + mu sync.Mutex + started map[string]time.Time + ended map[string]time.Time + changed chan struct{} + // busModule is the module that holds the seat mesh-broker now (A-126), whose items are the bus's. + busModule func() (string, error) + js jetstream.JetStream +} + +func newDumpView(js jetstream.JetStream, busModule func() (string, error)) *dumpView { + return &dumpView{started: map[string]time.Time{}, ended: map[string]time.Time{}, changed: make(chan struct{}), + busModule: busModule, js: js} +} + +// hear takes one item event, an older one by `at` than the one held changing nothing, and wakes every wait. +func (d *dumpView) hear(m jetstream.Msg) { + d.take(m.Subject(), m.Data()) + _ = m.Ack() +} + +func (d *dumpView) take(subject string, data []byte) { + var body struct { + At time.Time `json:"at"` + } + rest, ok := strings.CutPrefix(subject, "mesh.seat."+backupSeat+".event.") + if !ok || json.Unmarshal(data, &body) != nil { + return + } + event, item, ok := strings.Cut(rest, ".") + if !ok { + return + } + d.mu.Lock() + defer d.mu.Unlock() + into := d.started + if event == itemEnded { + into = d.ended + } else if event != itemStarted { + return + } + if old, ok := into[item]; ok && body.At.Before(old) { + return + } + into[item] = body.At + close(d.changed) + d.changed = make(chan struct{}) +} + +// runs is the first item of the module whose dump runs, or "". +func (d *dumpView) runs(module string) string { + token := keys.EscapeName(module) + var on []string + for item, at := range d.started { + parts := strings.Split(item, ".") + if len(parts) != 3 || parts[1] != token { + continue + } + if end, ok := d.ended[item]; !ok || end.Before(at) { + on = append(on, item) + } + } + sort.Strings(on) + if len(on) == 0 { + return "" + } + return on[0] +} + +// readNow reads the last start and end of each of the bus module's items once more, through an ordered consumer made for +// that read (§556): the restore said its restore-started, acknowledged, before it reads (§563). +func (d *dumpView) readNow(ctx context.Context, module string) error { + for _, c := range []struct{ stream, event string }{{itemStartedSteam, itemStarted}, {itemEndedStream, itemEnded}} { + cons, err := d.js.OrderedConsumer(ctx, c.stream, jetstream.OrderedConsumerConfig{ + FilterSubjects: []string{itemSubject(c.event, "*", keys.EscapeName(module), "*")}, + DeliverPolicy: jetstream.DeliverLastPerSubjectPolicy}) + if err != nil { + return fmt.Errorf("the read of %s: %w", c.stream, err) + } + info, err := cons.Info(ctx) + if err != nil { + return fmt.Errorf("the read of %s: %w", c.stream, err) + } + for left := info.NumPending; left > 0; { + m, err := cons.Next(jetstream.FetchMaxWait(5 * time.Second)) + if err != nil { + return fmt.Errorf("the read of %s: %w", c.stream, err) + } + d.take(m.Subject(), m.Data()) + meta, err := m.Metadata() + if err != nil { + return err + } + left = meta.NumPending + } + } + return nil +} + +// Wait returns once no dump of the bus module's items runs; it calls waiting once, naming the item, when it has to wait +// (§554, §563). It wakes on the item events, never on a timer. +func (d *dumpView) Wait(ctx context.Context, waiting func(on string)) error { + module, err := d.busModule() + if err != nil { + return err + } + if err := d.readNow(ctx, module); err != nil { + return err + } + said := false + for { + d.mu.Lock() + on, ch := d.runs(module), d.changed + d.mu.Unlock() + if on == "" { + return nil + } + if !said { + waiting("the item-ended of the dump of " + on) + said = true + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-ch: + } + } +} + +var _ recovery.Dump = (*dumpView)(nil) diff --git a/modules/mesh-recovery/cmd/mesh-recovery/main.go b/modules/mesh-recovery/cmd/mesh-recovery/main.go new file mode 100644 index 000000000..91293e1fc --- /dev/null +++ b/modules/mesh-recovery/cmd/mesh-recovery/main.go @@ -0,0 +1,152 @@ +// mesh-recovery: the recovery module's process (design 47 G §554). One process on an account of its own holds the seat +// mesh-recovery and answers its verbs itself over the bus with the caller the bus proved (A-107); it takes, sets aside and +// restores modules' kept data on the live bus, one act at a time, each act's result said as the seat's event. +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "os" + "os/signal" + "syscall" + "time" + + "git.novox.be/novox/mesh-contracts/go/seats/meshbroker" + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/seatlist" + + "github.com/nats-io/nats.go" + + "mesh-recovery/internal/recovery" +) + +// The settings' files and the points directory, as the manifest's process section sets them (A-112, §633). +const ( + envPoints = "MESH_RECOVERY_POINTS" + envCap = "MESH_RECOVERY_POINTS_CAP_FILE" + envKeepDays = "MESH_RECOVERY_KEEP_DAYS_FILE" +) + +// The settings' defaults: the design's values (§551, §564). +const ( + defaultCap = 10 << 30 + defaultKeepDays = 90 +) + +// ExpireEvery is how often §551's removal runs beside its start and each snapshot-kept: once a day. +var ExpireEvery = 24 * time.Hour + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + if err := run(ctx, os.Getenv, os.Stderr); err != nil { + fmt.Fprintln(os.Stderr, "mesh-recovery:", err) + os.Exit(1) + } +} + +// digests is the state digests, the points' digests kept on the bus apart from their bytes (§567). +type digests struct{ s process.State } + +func (d digests) Put(key string, v []byte) error { + _, err := d.s.Put(key, json.RawMessage(v)) + return err +} + +func (d digests) Get(key string) ([]byte, bool, error) { + e, err := d.s.Get(key) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} + +func (d digests) Delete(key string) error { return d.s.Delete(key) } + +// run is the process: it returns when ctx ends, after the running act ended. +func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error { + logger := log.New(logOut, "mesh-recovery: ", log.LstdFlags) + dir := getenv(envPoints) + if dir == "" { + return errors.New(envPoints + " names no points directory, so nothing can be kept") + } + cap, err := readNumber(getenv(envCap), defaultCap) + if err != nil { + return err + } + keepDays, err := readNumber(getenv(envKeepDays), defaultKeepDays) + if err != nil { + return err + } + account, err := process.ReadBrokerAccount(getenv) + if err != nil { + return err + } + conn, err := process.Connect(account) + if err != nil { + return err + } + defer conn.Close() + nc, js, err := account.Dial(nats.Name(account.Node + "." + account.Module)) + if err != nil { + return err + } + defer nc.Close() + + // What the engine reads of the bus, each read as §556 has it before it acts on it: the node-engine seat's running + // (A-132) and the backup holders' item events of the bus module (§563). + running := newRunningView() + if _, err := account.HearSeatEvents(ctx, js, nodeengine.StreamRunning, running.hear, logger.Printf, process.OnRead(running.onRead)); err != nil { + return err + } + dump := newDumpView(js, func() (string, error) { return seatlist.HolderOf(conn.Call, meshbroker.Seat) }) + for _, s := range []string{itemStartedSteam, itemEndedStream} { + if _, err := account.HearSeatEvents(ctx, js, s, dump.hear, logger.Printf); err != nil { + return err + } + } + e := &recovery.Engine{ + Dir: dir, + Bus: liveBus{nc: nc, js: js, scratch: dir, logf: logger.Printf}, + Seats: liveSeats{emit: func(seat, event string, body any) error { return conn.EmitSeat(seat, event, body) }, + recorder: meshhandactsclient.Recorder{Ask: conn.Call}, call: conn.Call, callOn: conn.CallOn, + node: account.Node, points: dir, running: running}, + Digests: digests{conn.State("digests")}, + Dump: dump, + Settings: recovery.Settings{Cap: cap, KeepDays: int(keepDays)}, + Logf: logger.Printf, + } + for _, x := range e.Start(ctx) { + logger.Printf("the aside %s of %s's %s holds data the live bus lacks: recovery_restore_aside puts it back", x.Aside.ID, x.Module, x.Kept) + } + stopAnswers, err := conn.AnswerSeat(recovery.Seat, e.Handlers()) + if err != nil { + return fmt.Errorf("the seat %s's verbs cannot be answered: %w", recovery.Seat, err) + } + defer stopAnswers() + logger.Printf("started on the account %s, holding the seat %s", account.User, recovery.Seat) + expire := func(why string) { + if a := e.Accept(recovery.Act{Kind: recovery.KindExpire, Why: why, Caller: account.User}); a.Accepted == "" { + logger.Printf("the removal of points past their keep days is not run now (%s%s); it runs at the next snapshot-kept or day", a.Busy, a.Error) + } + } + expire("the removal of points past their keep days at the start (design 47 G section 551)") + tick := time.NewTicker(ExpireEvery) + defer tick.Stop() + for { + select { + case <-ctx.Done(): + stopAnswers() + e.Wait() + return nil + case <-tick.C: + expire("the daily removal of points past their keep days (design 47 G section 551)") + } + } +} diff --git a/modules/mesh-recovery/cmd/mesh-recovery/main_test.go b/modules/mesh-recovery/cmd/mesh-recovery/main_test.go new file mode 100644 index 000000000..7f1d18dce --- /dev/null +++ b/modules/mesh-recovery/cmd/mesh-recovery/main_test.go @@ -0,0 +1,68 @@ +package main + +import ( + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/process" +) + +// The process's own readings of the mesh (design 47 G §554, §563, A-121, A-132): the bus names of a kept data from its +// manifest, the fingerprint a module runs at from the node-engine seat's running, and whether a dump of the bus module runs. +func TestWhatTheProcessReads(t *testing.T) { + t.Run("bus names of a kept data (A-121)", func(t *testing.T) { + m, err := catalogue.Decode([]byte(`{"module":"app","version":"1","state":[{"name":"old"}],"logs":[{"name":"journal"}], + "streams":[{"name":"APP_EVENTS"}],"objects":[{"name":"files"}], + "requires":[{"name":"notes","storage":"key-value","template":"state"}]}`)) + if err != nil { + t.Fatal(err) + } + for kept, want := range map[string]string{"notes": "KV_app_notes OBJ_app_notes-objects", "old": "KV_app_old", + "journal": "LOG_app_journal", "APP_EVENTS": "APP_EVENTS", "files": "OBJ_app_files"} { + got, err := KeptStreams("app", kept, m) + if err != nil || strings.Join(got, " ") != want { + t.Errorf("%s: %v, %v; want %s", kept, got, err, want) + } + } + if _, err := KeptStreams("app", "none", m); err == nil { + t.Error("a kept data the manifest does not declare was named") + } + }) + t.Run("the running fingerprint (A-132)", func(t *testing.T) { + v := newRunningView() + if _, err := v.Fingerprint("app"); err == nil { + t.Error("answered before the read of running ended") + } + v.onRead(process.ReadDone) + at := time.Date(2026, 11, 12, 1, 0, 0, 0, time.UTC) + v.by["a\x00app"] = nodeengine.RunningEvent{Node: "a", Module: "app", Fingerprint: "src2:1", At: at} + v.by["b\x00app"] = nodeengine.RunningEvent{Node: "b", Module: "app", Stopped: true, At: at} + if fp, err := v.Fingerprint("app"); err != nil || fp != "src2:1" { + t.Errorf("%q, %v", fp, err) + } + v.by["b\x00app"] = nodeengine.RunningEvent{Node: "b", Module: "app", Fingerprint: "src2:2", At: at} + if _, err := v.Fingerprint("app"); err == nil || !strings.Contains(err.Error(), "src2:2 on b") { + t.Errorf("two fingerprints answered %v", err) + } + }) + t.Run("a dump of the bus module runs (§563)", func(t *testing.T) { + d := newDumpView(nil, nil) + start := time.Date(2026, 11, 12, 2, 0, 0, 0, time.UTC) + d.take(itemSubject(itemStarted, "srv", "nats", "data"), []byte(`{"at":"`+start.Format(time.RFC3339)+`"}`)) + d.take(itemSubject(itemStarted, "srv", "app", "data"), []byte(`{"at":"`+start.Format(time.RFC3339)+`"}`)) + if on := d.runs("nats"); on != "srv.nats.data" { + t.Errorf("runs %q", on) + } + d.take(itemSubject(itemEnded, "srv", "nats", "data"), []byte(`{"at":"`+start.Add(time.Minute).Format(time.RFC3339)+`","outcome":"done"}`)) + if on := d.runs("nats"); on != "" { + t.Errorf("an ended dump runs: %q", on) + } + d.take(itemSubject(itemEnded, "srv", "nats", "data"), []byte(`{"at":"`+start.Add(-time.Hour).Format(time.RFC3339)+`","outcome":"done"}`)) + if on := d.runs("nats"); on != "" { + t.Errorf("an older end changed what runs: %q", on) + } + }) +} diff --git a/modules/mesh-recovery/cmd/mesh-recovery/seats.go b/modules/mesh-recovery/cmd/mesh-recovery/seats.go new file mode 100644 index 000000000..58937e038 --- /dev/null +++ b/modules/mesh-recovery/cmd/mesh-recovery/seats.go @@ -0,0 +1,225 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync" + + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/process" + + "github.com/nats-io/nats.go/jetstream" + + "mesh-recovery/internal/recovery" +) + +// Call is how the process calls a seat's verb (process.Conn.Call), and CallOn one machine's holder of a node seat. +type ( + Call func(key string, body any) (json.RawMessage, error) + CallOn func(node, key string, body any) (json.RawMessage, error) +) + +// liveSeats is what the engine asks of the rest of the mesh, each by its seat (design 47 G §554). +type liveSeats struct { + emit func(seat, event string, body any) error + recorder meshhandactsclient.Recorder + call Call + callOn CallOn + node string + points string + running *runningView +} + +// Say says one event of the seat mesh-recovery, each part filled in as one token, escaped as a key's name is (§523). +func (s liveSeats) Say(event string, body any, parts ...string) error { + subject := event + for _, p := range parts { + if p == "" || p == "." || p == ".." { + return fmt.Errorf("the event %s cannot be said with the part %q", event, p) + } + subject += "." + keys.EscapeName(p) + } + return s.emit(recovery.Seat, subject, body) +} + +// HandAct writes the act in the seat mesh-hand-acts' log first, naming the account the bus proved and who asked (§554, §682). +func (s liveSeats) HandAct(caller, asker, what, why string) error { + line := what + " (the account " + orNone(caller) + if asker != "" { + line += ", asked by " + asker + } + _, err := s.recorder.Record(meshhandactsclient.Request{What: line + ")", Why: why, Verb: recovery.Seat}) + return err +} + +func orNone(s string) string { + if s == "" { + return "unproven" + } + return s +} + +// Streams are the bus names of a module's kept data at a fingerprint, from the catalog graph entry `mesh-catalog`'s `entry` +// answers (§554 Bus names, §443): a template entry's as mesh-sdk names them, an old section's as the bus composer of +// mesh-sdk names it when it makes it (A-121). +func (s liveSeats) Streams(_ context.Context, module, kept, fingerprint string) ([]string, error) { + raw, err := s.call("seat:"+meshcatalog.Seat+"."+meshcatalog.VerbEntry, meshcatalog.EntryInput{Fingerprint: fingerprint}) + if err != nil { + return nil, fmt.Errorf("the catalog entry of %s: %w", fingerprint, err) + } + var ans meshcatalog.EntryAnswer + if err := json.Unmarshal(raw, &ans); err != nil { + return nil, fmt.Errorf("the catalog entry of %s does not read: %w", fingerprint, err) + } + if ans.Entry.Module != module { + return nil, fmt.Errorf("the catalog entry of %s is %s's, not %s's", fingerprint, ans.Entry.Module, module) + } + whole, err := json.Marshal(ans.Entry.Manifest) + if err != nil { + return nil, err + } + m, err := catalogue.Decode(whole) + if err != nil { + return nil, fmt.Errorf("the manifest of %s at %s: %w", module, fingerprint, err) + } + return KeptStreams(module, kept, m) +} + +// KeptStreams names the bus objects of one kept data a manifest declares (A-121). +func KeptStreams(module, kept string, m catalogue.Manifest) ([]string, error) { + for _, e := range m.BusState { + if e.Name == kept { + if names := catalogue.BusStateStreams(module, e); len(names) > 0 { + return names, nil + } + return nil, fmt.Errorf("%s's %s has no bus name", module, kept) + } + } + for _, d := range catalogue.KeptDeclarations(m) { + if d.Name != kept { + continue + } + switch d.Kind { + case catalogue.KeptKindState: + return []string{"KV_" + buscompose.BucketName(module, kept)}, nil + case catalogue.KeptKindLog: + return []string{buscompose.LogStreamName(module, kept)}, nil + case catalogue.KeptKindStream: + return []string{kept}, nil + case catalogue.KeptKindObjects: + return []string{"OBJ_" + buscompose.BucketName(module, kept)}, nil + } + } + return nil, fmt.Errorf("%s's manifest declares no kept data %s", module, kept) +} + +// RunningFingerprint is the fingerprint the node-engine runs the module at, from the node-engine seat's running (A-123, A-132). +func (s liveSeats) RunningFingerprint(_ context.Context, module string) (string, error) { + return s.running.Fingerprint(module) +} + +// Backup brings the points directory back beside its path through the seat node-backup's restore on this node (§516), +// answering the directory it was brought back into. +func (s liveSeats) Backup(_ context.Context, module string) (string, error) { + if _, err := s.callOn(s.node, "seat:node-backup.restore", map[string]string{"module": module, "path": s.points}); err != nil { + return "", fmt.Errorf("the backup holder on %s brought nothing back: %w", s.node, err) + } + back, _ := filepath.Glob(s.points + ".restored-*") + if len(back) == 0 { + return "", fmt.Errorf("the backup holder on %s answered, and nothing lies beside %s", s.node, s.points) + } + sort.Strings(back) + return back[len(back)-1], nil +} + +var _ recovery.Seats = liveSeats{} + +// runningView is what the node-engine seat's running says of each node's module, read as §556 has it: the read of the last +// of each subject first, the durable followed after it, an older one by `at` changing nothing. +type runningView struct { + mu sync.Mutex + read bool + by map[string]nodeengine.RunningEvent +} + +func newRunningView() *runningView { return &runningView{by: map[string]nodeengine.RunningEvent{}} } + +func (v *runningView) hear(m jetstream.Msg) { + var r nodeengine.RunningEvent + if err := json.Unmarshal(m.Data(), &r); err == nil && r.Module != "" { + k := r.Node + "\x00" + r.Module + v.mu.Lock() + if old, ok := v.by[k]; !ok || !r.At.Before(old.At) { + v.by[k] = r + } + v.mu.Unlock() + } + _ = m.Ack() +} + +func (v *runningView) onRead(state string) { + v.mu.Lock() + v.read = state == process.ReadDone + v.mu.Unlock() +} + +// Fingerprint is the one fingerprint the module runs at on the nodes that run it; two different ones are refused, +// naming both, so a set-aside never takes the names of a build that does not run. +func (v *runningView) Fingerprint(module string) (string, error) { + v.mu.Lock() + defer v.mu.Unlock() + if !v.read { + return "", errors.New("the node-engine seat's running is still being read") + } + fps := map[string][]string{} + for _, r := range v.by { + if r.Module == module && !r.Stopped && r.Fingerprint != "" { + fps[r.Fingerprint] = append(fps[r.Fingerprint], r.Node) + } + } + switch len(fps) { + case 0: + return "", fmt.Errorf("no node runs %s", module) + case 1: + for fp := range fps { + return fp, nil + } + } + var said []string + for fp, nodes := range fps { + sort.Strings(nodes) + said = append(said, fp+" on "+strings.Join(nodes, ", ")) + } + sort.Strings(said) + return "", fmt.Errorf("%s runs at more than one fingerprint (%s)", module, strings.Join(said, "; ")) +} + +// readNumber reads a setting's file (design 47 G A-112): its whole content a positive number, the default when missing. +func readNumber(path string, fallback int64) (int64, error) { + if path == "" { + return fallback, nil + } + raw, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return fallback, nil + } + if err != nil { + return 0, err + } + var n float64 + if err := json.Unmarshal([]byte(strings.TrimSpace(string(raw))), &n); err != nil || n <= 0 { + return 0, fmt.Errorf("the setting in %s is %q, not a positive number", path, strings.TrimSpace(string(raw))) + } + return int64(n), nil +} diff --git a/modules/mesh-recovery/go.mod b/modules/mesh-recovery/go.mod index ee3bd0cc0..8056787cb 100644 --- a/modules/mesh-recovery/go.mod +++ b/modules/mesh-recovery/go.mod @@ -5,6 +5,9 @@ go 1.26.0 require ( git.novox.be/novox/mesh-contracts/go/seats/meshbroker v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/go/seats/meshrecovery v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-recovery/internal/recovery/fakes_test.go b/modules/mesh-recovery/internal/recovery/fakes_test.go index 48796d2cb..69c471955 100644 --- a/modules/mesh-recovery/internal/recovery/fakes_test.go +++ b/modules/mesh-recovery/internal/recovery/fakes_test.go @@ -99,11 +99,15 @@ type fakeSeats struct { handFail error streams []string backup string + sayFail map[string]error } func (s *fakeSeats) Say(event string, body any, parts ...string) error { s.mu.Lock() defer s.mu.Unlock() + if err := s.sayFail[event]; err != nil { + return err + } s.events = append(s.events, said{event, body, parts}) return nil } diff --git a/modules/mesh-recovery/internal/recovery/restore.go b/modules/mesh-recovery/internal/recovery/restore.go index 744a2eaf6..fb6803b55 100644 --- a/modules/mesh-recovery/internal/recovery/restore.go +++ b/modules/mesh-recovery/internal/recovery/restore.go @@ -75,8 +75,11 @@ func (e *Engine) cancelledWhy(err error) string { // against the bus, sets the live data aside whole and checked, removes it, restores the point under its names and // sets each consumer to its recorded sequence; a step after the removal that fails puts the aside back. func (e *Engine) restore(ctx context.Context, a *Act) error { - e.say(seat.EventRestoreStarted, seat.RestoreStartedEvent{Act: a.ID, At: e.now(), Module: a.Module, Kept: a.Kept, Form: a.Form, - Fingerprint: a.Fingerprint, Snapshot: a.Snapshot}, a.Module, a.Kept) + // Said means acknowledged: the dump's item events are read only once the bus took restore-started (§563). + if err := e.Seats.Say(seat.EventRestoreStarted, seat.RestoreStartedEvent{Act: a.ID, At: e.now(), Module: a.Module, Kept: a.Kept, Form: a.Form, + Fingerprint: a.Fingerprint, Snapshot: a.Snapshot}, a.Module, a.Kept); err != nil { + return e.endedRestored(a, fmt.Errorf("restore-started was not acknowledged by the bus, so nothing was read or set aside: %w", err), nil, false) + } if e.Dump != nil { if err := e.Dump.Wait(ctx, func(on string) { e.setStep(a, "waiting", on) }); err != nil { if ctx.Err() != nil { diff --git a/modules/mesh-recovery/internal/recovery/restore_test.go b/modules/mesh-recovery/internal/recovery/restore_test.go index 853a68645..de2cbe0ff 100644 --- a/modules/mesh-recovery/internal/recovery/restore_test.go +++ b/modules/mesh-recovery/internal/recovery/restore_test.go @@ -259,3 +259,17 @@ func TestAsideMovesTheLiveDataIntoAnAsideAndRestoreAsidePutsItBack(t *testing.T) t.Errorf("streams %v", r.b.streams) } } + +// Said means acknowledged: a restore whose restore-started the bus did not take reads nothing and sets nothing aside, and +// ends failed (§563). +func TestARestoreWhoseStartIsNotAcknowledgedTouchesNothing(t *testing.T) { + r := takenRig(t) + r.s.sayFail = map[string]error{seat.EventRestoreStarted: errTest} + r.do(restoreAct("app/notes/2-fp1")) + if ev := lastRestored(t, r); ev.Outcome != "failed" || !strings.Contains(ev.Error, "restore-started") || r.b.streams["app_notes"] != "hello" { + t.Errorf("restored %+v, streams %v", ev, r.b.streams) + } + if kp, _ := r.e.KeptPoints("", ""); len(kp.Asides) != 0 { + t.Errorf("an aside was taken: %+v", kp.Asides) + } +} diff --git a/modules/mesh-recovery/module.json b/modules/mesh-recovery/module.json index ac477062c..5f502377f 100644 --- a/modules/mesh-recovery/module.json +++ b/modules/mesh-recovery/module.json @@ -35,11 +35,13 @@ "consumes": [ "node-backup.item-started", "node-backup.item-ended", + "node-engine.running", "mesh-broker.snapshot-restored", "mesh-conditions.findings-asked" ], "invokes": [ "seat:mesh-catalog.entry", + "seat:mesh-module-assigner.assignments", "seat:mesh-hand-acts.hand-act", "seat:mesh-broker.snapshot-get", "seat:mesh-broker.snapshot-restore", @@ -223,6 +225,7 @@ ], "invokes": [ "seat:mesh-catalog.entry", + "seat:mesh-module-assigner.assignments", "seat:mesh-hand-acts.hand-act", "seat:mesh-broker.snapshot-get", "seat:mesh-broker.snapshot-restore", From d854f936f95383d62bb0acfe7734d445479f6273 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 01:40:49 +0100 Subject: [PATCH 10/17] mesh-recovery: points-filling, recovery-waits and aside-unrestored through mesh-sdk's finding writer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The module measures its points directory against the fill mark of its cap at its start and after each act, writes recovery-waits for an act that waits on the dump past its bound and clears it once the act starts its work or ends, and writes aside-unrestored for each aside only it holds the data of, cleared once the aside is restored or discarded. The bounds are the settings points-filling-mark and dump-wait-minutes, read at the start. Builds: design 47 G §561, §557, §564, §633, A-112, A-130 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../mesh-recovery/cmd/mesh-recovery/main.go | 28 +++- .../mesh-recovery/cmd/mesh-recovery/seats.go | 21 +++ modules/mesh-recovery/go.mod | 1 + .../internal/recovery/control.go | 1 + .../internal/recovery/fakes_test.go | 46 ++++++ .../internal/recovery/findings.go | 153 ++++++++++++++++++ .../internal/recovery/recovery.go | 8 + .../internal/recovery/restore.go | 5 +- .../internal/recovery/restore_test.go | 21 +++ 9 files changed, 282 insertions(+), 2 deletions(-) create mode 100644 modules/mesh-recovery/internal/recovery/findings.go diff --git a/modules/mesh-recovery/cmd/mesh-recovery/main.go b/modules/mesh-recovery/cmd/mesh-recovery/main.go index 91293e1fc..8f48f11a5 100644 --- a/modules/mesh-recovery/cmd/mesh-recovery/main.go +++ b/modules/mesh-recovery/cmd/mesh-recovery/main.go @@ -18,8 +18,10 @@ import ( "git.novox.be/novox/mesh-contracts/go/seats/meshbroker" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/seatlist" + "git.novox.be/novox/mesh-sdk/go/views" "github.com/nats-io/nats.go" @@ -31,6 +33,8 @@ const ( envPoints = "MESH_RECOVERY_POINTS" envCap = "MESH_RECOVERY_POINTS_CAP_FILE" envKeepDays = "MESH_RECOVERY_KEEP_DAYS_FILE" + envMark = "MESH_RECOVERY_FILLING_MARK_FILE" + envDumpWait = "MESH_RECOVERY_DUMP_WAIT_FILE" ) // The settings' defaults: the design's values (§551, §564). @@ -84,6 +88,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } + mark, err := readNumber(getenv(envMark), recovery.DefaultFillingMark) + if err != nil { + return err + } + dumpWait, err := readMinutes(getenv(envDumpWait), recovery.DefaultDumpWait) + if err != nil { + return err + } account, err := process.ReadBrokerAccount(getenv) if err != nil { return err @@ -111,6 +123,18 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } } + // The module's findings go through mesh-sdk's held writer over its own state findings (§561), started before the + // engine's start, so what the start writes is held and written. + writer, err := views.NewHeldWriter(views.WriterConfig[meshconditions.Finding, meshconditions.FindingCleared, meshconditions.FindingsStated, meshconditions.FindingsAskedEvent]{ + Asked: meshconditions.SubjectFindingsAsked, Module: account.Module, Node: account.Node, State: conn.State("findings"), + Kinds: recovery.FindingKinds, JS: js, Conn: nc, Logf: logger.Printf}) + if err != nil { + return err + } + if err := writer.Start(ctx); err != nil { + return err + } + defer writer.Close() e := &recovery.Engine{ Dir: dir, Bus: liveBus{nc: nc, js: js, scratch: dir, logf: logger.Printf}, @@ -119,7 +143,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro node: account.Node, points: dir, running: running}, Digests: digests{conn.State("digests")}, Dump: dump, - Settings: recovery.Settings{Cap: cap, KeepDays: int(keepDays)}, + Settings: recovery.Settings{Cap: cap, KeepDays: int(keepDays), FillingMark: int(mark), DumpWait: dumpWait}, + Findings: writer, + Node: account.Node, Logf: logger.Printf, } for _, x := range e.Start(ctx) { diff --git a/modules/mesh-recovery/cmd/mesh-recovery/seats.go b/modules/mesh-recovery/cmd/mesh-recovery/seats.go index 58937e038..26f0e06c1 100644 --- a/modules/mesh-recovery/cmd/mesh-recovery/seats.go +++ b/modules/mesh-recovery/cmd/mesh-recovery/seats.go @@ -10,6 +10,7 @@ import ( "sort" "strings" "sync" + "time" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" @@ -205,6 +206,26 @@ func (v *runningView) Fingerprint(module string) (string, error) { return "", fmt.Errorf("%s runs at more than one fingerprint (%s)", module, strings.Join(said, "; ")) } +// readMinutes reads a bound in minutes from its setting's file, a fraction allowed so a whole test sets it in seconds +// (§557, A-130); the default when missing. +func readMinutes(path string, fallback time.Duration) (time.Duration, error) { + if path == "" { + return fallback, nil + } + raw, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return fallback, nil + } + if err != nil { + return 0, err + } + var n float64 + if err := json.Unmarshal([]byte(strings.TrimSpace(string(raw))), &n); err != nil || n <= 0 { + return 0, fmt.Errorf("the setting in %s is %q, not a positive number of minutes", path, strings.TrimSpace(string(raw))) + } + return time.Duration(n * float64(time.Minute)), nil +} + // readNumber reads a setting's file (design 47 G A-112): its whole content a positive number, the default when missing. func readNumber(path string, fallback int64) (int64, error) { if path == "" { diff --git a/modules/mesh-recovery/go.mod b/modules/mesh-recovery/go.mod index 8056787cb..87e7e287a 100644 --- a/modules/mesh-recovery/go.mod +++ b/modules/mesh-recovery/go.mod @@ -7,6 +7,7 @@ require ( git.novox.be/novox/mesh-contracts/go/seats/meshrecovery v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 github.com/nats-io/nats.go v1.54.0 diff --git a/modules/mesh-recovery/internal/recovery/control.go b/modules/mesh-recovery/internal/recovery/control.go index 66e725d4e..941b22ccf 100644 --- a/modules/mesh-recovery/internal/recovery/control.go +++ b/modules/mesh-recovery/internal/recovery/control.go @@ -165,6 +165,7 @@ func (e *Engine) Start(ctx context.Context) []UnrestoredAside { a.Error = ErrStopped e.markEnded(&a) } + e.judge() return e.Unrestored() } diff --git a/modules/mesh-recovery/internal/recovery/fakes_test.go b/modules/mesh-recovery/internal/recovery/fakes_test.go index 69c471955..38fca691b 100644 --- a/modules/mesh-recovery/internal/recovery/fakes_test.go +++ b/modules/mesh-recovery/internal/recovery/fakes_test.go @@ -11,6 +11,7 @@ import ( "time" seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" "mesh-recovery/internal/recovery" ) @@ -220,3 +221,48 @@ func (g *gatedBus) Objects(ctx context.Context, n []string) ([]recovery.Object, <-g.gate return g.fakeBus.Objects(ctx, n) } + +// heldFindings is the module's finding writer as it holds them; written says each write. +type heldFindings struct { + mu sync.Mutex + held map[string]meshconditions.Finding + written chan meshconditions.Finding +} + +func newHeldFindings() *heldFindings { + return &heldFindings{held: map[string]meshconditions.Finding{}, written: make(chan meshconditions.Finding, 64)} +} + +func (h *heldFindings) Write(_ context.Context, f meshconditions.Finding) error { + h.mu.Lock() + h.held[f.Scope+"."+f.ID+"."+f.Kind] = f + h.mu.Unlock() + h.written <- f + return nil +} + +func (h *heldFindings) Clear(_ context.Context, scope, id, kind, _ string) error { + h.mu.Lock() + defer h.mu.Unlock() + delete(h.held, scope+"."+id+"."+kind) + return nil +} + +func (h *heldFindings) Held() []meshconditions.Finding { + h.mu.Lock() + defer h.mu.Unlock() + var out []meshconditions.Finding + for _, f := range h.held { + out = append(out, f) + } + return out +} + +func (h *heldFindings) has(kind string) bool { + for _, f := range h.Held() { + if f.Kind == kind { + return true + } + } + return false +} diff --git a/modules/mesh-recovery/internal/recovery/findings.go b/modules/mesh-recovery/internal/recovery/findings.go new file mode 100644 index 000000000..09fa7310a --- /dev/null +++ b/modules/mesh-recovery/internal/recovery/findings.go @@ -0,0 +1,153 @@ +package recovery + +import ( + "context" + "fmt" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" +) + +// The recovery module's findings (design 47 G §561), each written by mesh-sdk's finding writer as the key +// `..` and said on the module's finding stream. +const ( + FindingPointsFilling = "points-filling" + FindingRecoveryWaits = "recovery-waits" + FindingAsideUnrestored = "aside-unrestored" +) + +// FindingKinds are the kinds the module's writer owns. +var FindingKinds = []string{FindingPointsFilling, FindingRecoveryWaits, FindingAsideUnrestored} + +// Findings is the module's finding writer (mesh-sdk's held writer): what it writes and clears, and what it holds. +type Findings interface { + Write(ctx context.Context, f meshconditions.Finding) error + Clear(ctx context.Context, scope, id, kind, why string) error + Held() []meshconditions.Finding +} + +// The defaults of the settings §557 and §564 make bounds of. +const ( + DefaultFillingMark = 80 + DefaultDumpWait = 60 * time.Minute +) + +func (e *Engine) mark() int64 { + if e.Settings.FillingMark > 0 { + return int64(e.Settings.FillingMark) + } + return DefaultFillingMark +} + +func (e *Engine) dumpWait() time.Duration { + if e.Settings.DumpWait > 0 { + return e.Settings.DumpWait + } + return DefaultDumpWait +} + +func (e *Engine) finding(kind, scope, id, headline, summary, needs string) meshconditions.Finding { + return meshconditions.Finding{Scope: scope, ID: id, Kind: kind, Subject: meshconditions.Subject{Scope: "module", ID: Module}, + Severity: meshconditions.FindingSeverityWarning, Headline: headline, Summary: summary, Explanation: summary, Needs: needs, + Source: "points", Observer: Module, At: e.now()} +} + +func (e *Engine) write(f meshconditions.Finding) { + if e.Findings == nil { + return + } + if err := e.Findings.Write(context.Background(), f); err != nil { + e.logf("the finding %s %s.%s cannot be written: %v", f.Kind, f.Scope, f.ID, err) + } +} + +func (e *Engine) clear(scope, id, kind, why string) { + if e.Findings == nil { + return + } + if err := e.Findings.Clear(context.Background(), scope, id, kind, why); err != nil { + e.logf("the finding %s %s.%s cannot be cleared: %v", kind, scope, id, err) + } +} + +// judge measures the points directory against the fill mark of its cap (points-filling) and writes aside-unrestored for +// each aside whose record ends at removed that no running act holds, clearing those that no longer are (§561). It runs at +// the start, after each act and so after each expiry run. +func (e *Engine) judge() { + if e.Findings == nil { + return + } + fill, limit := e.filled(), e.Settings.Cap*e.mark()/100 + if fill > limit { + e.write(e.finding(FindingPointsFilling, Module, e.Node, "The recovery module's restore points are filling up", + fmt.Sprintf("the points directory holds %d of %d bytes, past its fill mark of %d %%; the largest: %s", fill, e.Settings.Cap, + e.mark(), strings.Join(e.largest(5), ", ")), "points removed or an aside discarded, or a larger points-cap")) + } else { + e.clear(Module, e.Node, FindingPointsFilling, fmt.Sprintf("the points directory holds %d bytes, under its fill mark", fill)) + } + open := map[string]bool{} + for _, u := range e.Unrestored() { + open[u.Module+"\x00"+u.Aside.ID] = true + e.write(e.finding(FindingAsideUnrestored, u.Module, u.Aside.ID, "An aside holds data the live bus lacks", + fmt.Sprintf("the aside %s of %s's %s (%s) ended at removed and was not restored", u.Aside.ID, u.Module, u.Kept, + strings.Join(u.Aside.Names, ", ")), "recovery_restore_aside, or recovery_discard_aside")) + } + for _, f := range e.Findings.Held() { + if f.Kind == FindingAsideUnrestored && !open[f.Scope+"\x00"+f.ID] { + e.clear(f.Scope, f.ID, f.Kind, "the aside was restored or discarded") + } + } +} + +// largest are the n largest points and asides, each with its size. +func (e *Engine) largest(n int) []string { + type sized struct { + what string + size int64 + } + var all []sized + pts, _ := e.Points() + for _, p := range pts { + all = append(all, sized{"point " + p.Snapshot, p.Size}) + } + _, recs, _ := e.Asides() + for _, r := range recs { + all = append(all, sized{"aside " + r.Module + "/" + r.Kept + "/" + r.ID, r.Size}) + } + sort.SliceStable(all, func(i, j int) bool { return all[i].size > all[j].size }) + var out []string + for i := 0; i < len(all) && i < n; i++ { + out = append(out, fmt.Sprintf("%s (%d bytes)", all[i].what, all[i].size)) + } + return out +} + +// waitsOn is the waiting of one act on the dump: past the bound it writes recovery-waits, naming the item and since when; +// done clears it once the act starts its work or ends (§561, §563). +func (e *Engine) waitsOn(a *Act) (waiting func(on string), done func()) { + var timer *time.Timer + written := make(chan bool, 1) + written <- false + waiting = func(on string) { + e.setStep(a, "waiting", on) + since := e.now() + timer = time.AfterFunc(e.dumpWait(), func() { + <-written + e.write(e.finding(FindingRecoveryWaits, a.Module, a.ID, "A restore waits on the dump of the bus", + fmt.Sprintf("the act %s waits on %s since %s", a.ID, on, since.Format(time.RFC3339)), "the dump to end, or recovery_cancel")) + written <- true + }) + } + done = func() { + if timer != nil { + timer.Stop() + } + if <-written { + e.clear(a.Module, a.ID, FindingRecoveryWaits, "the act no longer waits on the dump") + } + written <- false + } + return waiting, done +} diff --git a/modules/mesh-recovery/internal/recovery/recovery.go b/modules/mesh-recovery/internal/recovery/recovery.go index 91c1ab8f7..f37c7b704 100644 --- a/modules/mesh-recovery/internal/recovery/recovery.go +++ b/modules/mesh-recovery/internal/recovery/recovery.go @@ -100,6 +100,10 @@ type Dump interface { type Settings struct { Cap int64 KeepDays int + // FillingMark is the percent of the cap past which points-filling is written (points-filling-mark, 80 unless set). + FillingMark int + // DumpWait is how long an act waits on the dump before recovery-waits is written (dump-wait-minutes, 60 unless set). + DumpWait time.Duration } // Engine is the recovery module's one engine. @@ -110,6 +114,9 @@ type Engine struct { Digests Digests Dump Dump Settings Settings + // Findings is the module's finding writer (§561); Node the machine it runs on, the id of points-filling. + Findings Findings + Node string Now func() time.Time Logf func(string, ...any) @@ -253,6 +260,7 @@ func (e *Engine) Accept(a Act) Answer { e.running, e.cancel = nil, nil e.mu.Unlock() cancel() + e.judge() }() return Answer{Accepted: a.ID} } diff --git a/modules/mesh-recovery/internal/recovery/restore.go b/modules/mesh-recovery/internal/recovery/restore.go index fb6803b55..4fdbdfd20 100644 --- a/modules/mesh-recovery/internal/recovery/restore.go +++ b/modules/mesh-recovery/internal/recovery/restore.go @@ -81,7 +81,10 @@ func (e *Engine) restore(ctx context.Context, a *Act) error { return e.endedRestored(a, fmt.Errorf("restore-started was not acknowledged by the bus, so nothing was read or set aside: %w", err), nil, false) } if e.Dump != nil { - if err := e.Dump.Wait(ctx, func(on string) { e.setStep(a, "waiting", on) }); err != nil { + waiting, done := e.waitsOn(a) + err := e.Dump.Wait(ctx, waiting) + done() + if err != nil { if ctx.Err() != nil { err = errors.New(ErrCanceled) } diff --git a/modules/mesh-recovery/internal/recovery/restore_test.go b/modules/mesh-recovery/internal/recovery/restore_test.go index de2cbe0ff..88031c462 100644 --- a/modules/mesh-recovery/internal/recovery/restore_test.go +++ b/modules/mesh-recovery/internal/recovery/restore_test.go @@ -6,6 +6,7 @@ import ( "path/filepath" "strings" "testing" + "time" seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" @@ -137,8 +138,14 @@ func TestAWaitingActIsCancelledAndABegunOneRefused(t *testing.T) { r := takenRig(t) hd := holdDump{waiting: make(chan string, 1)} r.e.Dump = hd + // Past the bound (a setting, §557) the waiting act writes recovery-waits, cleared once it ends (§561). + fs := newHeldFindings() + r.e.Findings, r.e.Settings.DumpWait = fs, time.Millisecond ans := r.e.Accept(restoreAct("app/notes/2-fp1")) <-hd.waiting + if f := <-fs.written; f.Kind != recovery.FindingRecoveryWaits || f.Scope != "app" || f.ID != ans.Accepted { + t.Errorf("written %+v", f) + } kp, _ := r.e.KeptPoints("", "") if kp.Running == nil || kp.Running.WaitsOn == "" || kp.Running.Act != ans.Accepted { t.Errorf("running %+v", kp.Running) @@ -154,6 +161,9 @@ func TestAWaitingActIsCancelledAndABegunOneRefused(t *testing.T) { if ev.Outcome != "failed" || ev.Error != recovery.ErrCanceled || ev.Why != "not now" || ev.Act != ans.Accepted { t.Errorf("restored %+v", ev) } + if fs.has(recovery.FindingRecoveryWaits) { + t.Errorf("recovery-waits stands after the act ended: %+v", fs.Held()) + } } // retry asks again the last failed act with its fields under an act of its own, and answers `nothing to retry` when its @@ -237,6 +247,9 @@ func TestRestoreAsideRefusesLiveNamesAndDiscardAsideRemovesTheDirectory(t *testi // it back once no live name exists (§516, §554, §566, A-123, A-124). func TestAsideMovesTheLiveDataIntoAnAsideAndRestoreAsidePutsItBack(t *testing.T) { r := newRig(t) + // The findings of §561: aside-unrestored while the aside alone holds the data, points-filling past the fill mark. + fs := newHeldFindings() + r.e.Findings, r.e.Node, r.e.Settings.FillingMark, r.e.Settings.Cap = fs, "srv", 1, 4096 ans := r.do(recovery.Act{Kind: recovery.KindAside, Module: "app", Kept: "notes", Why: "look at it", Caller: "tools", Asker: "op"}) ev := r.s.named(seat.EventSetAside) if ans.Accepted == "" || len(ev) != 1 { @@ -253,11 +266,19 @@ func TestAsideMovesTheLiveDataIntoAnAsideAndRestoreAsidePutsItBack(t *testing.T) if kp, _ := r.e.KeptPoints("app", ""); len(kp.Unrestored) != 1 || kp.Unrestored[0].ID != sa.ID { t.Errorf("kept-points unrestored %+v", kp.Unrestored) } + if !fs.has(recovery.FindingAsideUnrestored) || !fs.has(recovery.FindingPointsFilling) { + t.Errorf("findings %+v", fs.Held()) + } r.s.events = nil r.do(recovery.Act{Kind: recovery.KindRestoreAside, Module: "app", Kept: "notes", AsideID: sa.ID, Why: "done looking", Caller: "tools"}) if lastRestored(t, r).Outcome != "done" || r.b.streams["app_notes"] != "hello" || len(r.e.Unrestored()) != 0 { t.Errorf("streams %v", r.b.streams) } + r.e.Settings.FillingMark = 100 + r.do(recovery.Act{Kind: recovery.KindDiscardAside, Module: "app", Kept: "notes", AsideID: sa.ID, Why: "not needed", Caller: "tools"}) + if fs.has(recovery.FindingAsideUnrestored) || fs.has(recovery.FindingPointsFilling) { + t.Errorf("findings stand after the aside was restored and discarded: %+v", fs.Held()) + } } // Said means acknowledged: a restore whose restore-started the bus did not take reads nothing and sets nothing aside, and From 3a6a127c38dab8fa986dcd823685934603043f20 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 01:45:44 +0100 Subject: [PATCH 11/17] mesh-recovery: its settings' files from templates under files/, the tools binary ignored MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The catalogue keeps no file text inline for a new module: each setting's file is a template under files/, packed by the module's archive artifact. A local build of the tools bundle leaves a binary the module's ignore file now names. Builds: design 47 G §633, A-112, A-130 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- modules/mesh-recovery/.gitignore | 1 + modules/mesh-recovery/files/dump-wait-minutes | 1 + modules/mesh-recovery/files/point-keep-days | 1 + modules/mesh-recovery/files/points-cap | 1 + modules/mesh-recovery/files/points-filling-mark | 1 + modules/mesh-recovery/module.json | 13 +++++++++---- 6 files changed, 14 insertions(+), 4 deletions(-) create mode 100644 modules/mesh-recovery/files/dump-wait-minutes create mode 100644 modules/mesh-recovery/files/point-keep-days create mode 100644 modules/mesh-recovery/files/points-cap create mode 100644 modules/mesh-recovery/files/points-filling-mark diff --git a/modules/mesh-recovery/.gitignore b/modules/mesh-recovery/.gitignore index 1b8d4339c..0135b2410 100644 --- a/modules/mesh-recovery/.gitignore +++ b/modules/mesh-recovery/.gitignore @@ -1,3 +1,4 @@ /mesh-recovery /cmd/mesh-recovery/mesh-recovery /cmd/mesh-recovery-tools/mesh-recovery-tools +/mesh-recovery-tools diff --git a/modules/mesh-recovery/files/dump-wait-minutes b/modules/mesh-recovery/files/dump-wait-minutes new file mode 100644 index 000000000..bcb00821f --- /dev/null +++ b/modules/mesh-recovery/files/dump-wait-minutes @@ -0,0 +1 @@ +${config:dump-wait-minutes} diff --git a/modules/mesh-recovery/files/point-keep-days b/modules/mesh-recovery/files/point-keep-days new file mode 100644 index 000000000..02af64917 --- /dev/null +++ b/modules/mesh-recovery/files/point-keep-days @@ -0,0 +1 @@ +${config:point-keep-days} diff --git a/modules/mesh-recovery/files/points-cap b/modules/mesh-recovery/files/points-cap new file mode 100644 index 000000000..cbe82190d --- /dev/null +++ b/modules/mesh-recovery/files/points-cap @@ -0,0 +1 @@ +${config:points-cap} diff --git a/modules/mesh-recovery/files/points-filling-mark b/modules/mesh-recovery/files/points-filling-mark new file mode 100644 index 000000000..da03c7e78 --- /dev/null +++ b/modules/mesh-recovery/files/points-filling-mark @@ -0,0 +1 @@ +${config:points-filling-mark} diff --git a/modules/mesh-recovery/module.json b/modules/mesh-recovery/module.json index 5f502377f..fb68370b5 100644 --- a/modules/mesh-recovery/module.json +++ b/modules/mesh-recovery/module.json @@ -162,7 +162,7 @@ "path": "${dir:state}/points-cap", "owner": "mesh-recovery", "mode": "0600", - "content": "${config:points-cap}" + "template": "files/points-cap" }, { "id": "mark", @@ -170,7 +170,7 @@ "path": "${dir:state}/points-filling-mark", "owner": "mesh-recovery", "mode": "0600", - "content": "${config:points-filling-mark}" + "template": "files/points-filling-mark" }, { "id": "keep-days", @@ -178,7 +178,7 @@ "path": "${dir:state}/point-keep-days", "owner": "mesh-recovery", "mode": "0600", - "content": "${config:point-keep-days}" + "template": "files/point-keep-days" }, { "id": "dump-wait", @@ -186,7 +186,7 @@ "path": "${dir:state}/dump-wait-minutes", "owner": "mesh-recovery", "mode": "0600", - "content": "${config:dump-wait-minutes}" + "template": "files/dump-wait-minutes" }, { "id": "recovery", @@ -262,6 +262,11 @@ "loads": [ "mesh-recovery-tools" ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } From a81f5ed0e328f15f44aecaeed04e67615f4e831f Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 01:57:34 +0100 Subject: [PATCH 12/17] mesh-recovery A-133: the manifest follows the validator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Its tool check stands on its own runtime beside the process's unit check, its template states are its own by requires alone (not listed again in the process's bus, which keeps its account), and digests takes the state template's fixed cap (A-135). Builds: design 47 G §657, §567, §554, A-133, A-135 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../internal/recovery/manifest_test.go | 32 ++++++++++++++++--- modules/mesh-recovery/module.json | 24 ++++++-------- 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/modules/mesh-recovery/internal/recovery/manifest_test.go b/modules/mesh-recovery/internal/recovery/manifest_test.go index 61da8aa86..b68c808b9 100644 --- a/modules/mesh-recovery/internal/recovery/manifest_test.go +++ b/modules/mesh-recovery/internal/recovery/manifest_test.go @@ -31,18 +31,25 @@ type manifest struct { Type string `json:"type"` Serves []string `json:"serves"` Bus struct { + State []string `json:"state"` Invokes []string `json:"invokes"` } `json:"bus"` Health struct { - Tool string `json:"tool"` + Kind string `json:"kind"` } `json:"health"` } `json:"resources"` + RuntimeHealth struct { + Kind string `json:"kind"` + Tool string `json:"tool"` + } `json:"runtime-health"` + Requires []json.RawMessage `json:"requires"` } // The manifest says what the module is (design 47 G §554, §656, §657, §633, §564, §567): it claims the seat once for the // mesh serving every verb of the seat's definition, its process answers each of them itself (A-115: `serves` beside its -// `bus`), its health check names a tool it serves, its data item is `points`, its settings carry the design's defaults, -// and what the process invokes the module declares. +// `bus`), its health check is its tool beside the process's own check and its template states are its own by `requires` +// alone, the digests state taking the template's fixed cap (A-133, A-135), its data item is `points`, its settings carry +// the design's defaults, and what the process invokes the module declares. func TestTheManifestSaysWhatTheModuleIs(t *testing.T) { raw, err := os.ReadFile("../../module.json") if err != nil { @@ -79,8 +86,11 @@ func TestTheManifestSaysWhatTheModuleIs(t *testing.T) { if !slices.Equal(r.Serves, want) { t.Errorf("the process serves %v, want %v", r.Serves, want) } - if !slices.Contains(m.Tools, r.Health.Tool) || r.Health.Tool != "recovery_kept_points" { - t.Errorf("the health check names %q, a tool of %v", r.Health.Tool, m.Tools) + if r.Health.Kind != "unit" { + t.Errorf("the process's own check is %q, want unit", r.Health.Kind) + } + if len(r.Bus.State) != 0 { + t.Errorf("the process's bus lists the states %v again, which are the module's own by its requires", r.Bus.State) } for _, inv := range r.Bus.Invokes { if !slices.Contains(m.Invokes, inv) { @@ -88,6 +98,18 @@ func TestTheManifestSaysWhatTheModuleIs(t *testing.T) { } } } + if m.RuntimeHealth.Kind != "tool" || m.RuntimeHealth.Tool != "recovery_kept_points" || !slices.Contains(m.Tools, m.RuntimeHealth.Tool) { + t.Errorf("the tool check is %+v, want recovery_kept_points, a tool of %v", m.RuntimeHealth, m.Tools) + } + for _, raw := range m.Requires { + var q struct { + Name string `json:"name"` + Options map[string]any `json:"options"` + } + if json.Unmarshal(raw, &q) == nil && q.Name == "digests" && len(q.Options) != 0 { + t.Errorf("digests gives the options %v beside the template state's fixed cap", q.Options) + } + } for _, c := range []string{"node-backup.item-started", "node-backup.item-ended", "mesh-broker.snapshot-restored"} { if !slices.Contains(m.Consumes, c) { t.Errorf("does not consume %s", c) diff --git a/modules/mesh-recovery/module.json b/modules/mesh-recovery/module.json index fb68370b5..4eef18760 100644 --- a/modules/mesh-recovery/module.json +++ b/modules/mesh-recovery/module.json @@ -60,10 +60,7 @@ { "name": "digests", "storage": "key-value", - "template": "state", - "options": { - "max-mib": 4 - } + "template": "state" }, { "name": "findings", @@ -219,10 +216,6 @@ "seat:mesh-recovery.expire-points" ], "bus": { - "state": [ - "digests", - "findings" - ], "invokes": [ "seat:mesh-catalog.entry", "seat:mesh-module-assigner.assignments", @@ -233,15 +226,18 @@ ] }, "health": { - "kind": "tool", - "tool": "recovery_kept_points", - "interval": "60s", - "timeout": "10s", - "looks": 2, - "grace": "180s" + "kind": "unit" } } ], + "runtime-health": { + "kind": "tool", + "tool": "recovery_kept_points", + "interval": "60s", + "timeout": "10s", + "looks": 2, + "grace": "180s" + }, "build": { "artifacts": [ { From 810254193c3f71b8190cafab2a8b3e86a1fcb986 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 02:12:32 +0100 Subject: [PATCH 13/17] mesh-recovery: the hand verb restore takes a keeper's snapshot by request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An unprotected snapshot is received through snapshot-get into incoming/, refused before the first chunk past the cap, checked against its digest before any aside, and restored by the module; a protected one is verified, the running data set aside and restored in place by the keeper, waited on by snapshot-restored within snapshot-restore-wait-minutes, cancelled at the bound. The bus module sends its chunks with mesh-sdk's form. Builds: design 47 G §681, §682, A-137, A-138, A-139, A-140 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../mesh-recovery/cmd/mesh-recovery/keeper.go | 151 +++++++++++++ .../mesh-recovery/cmd/mesh-recovery/main.go | 13 +- .../cmd/mesh-recovery/main_test.go | 23 ++ .../files/snapshot-restore-wait-minutes | 1 + .../internal/recovery/control.go | 2 + .../mesh-recovery/internal/recovery/keeper.go | 191 ++++++++++++++++ .../internal/recovery/keeper_test.go | 206 ++++++++++++++++++ .../internal/recovery/manifest_test.go | 8 +- .../internal/recovery/recovery.go | 53 +++-- .../internal/recovery/restore.go | 5 +- .../mesh-recovery/internal/recovery/verbs.go | 6 +- modules/mesh-recovery/module.json | 21 +- modules/nats/cmd/nats-tools/snapshots.go | 33 ++- 13 files changed, 666 insertions(+), 47 deletions(-) create mode 100644 modules/mesh-recovery/cmd/mesh-recovery/keeper.go create mode 100644 modules/mesh-recovery/files/snapshot-restore-wait-minutes create mode 100644 modules/mesh-recovery/internal/recovery/keeper.go create mode 100644 modules/mesh-recovery/internal/recovery/keeper_test.go diff --git a/modules/mesh-recovery/cmd/mesh-recovery/keeper.go b/modules/mesh-recovery/cmd/mesh-recovery/keeper.go new file mode 100644 index 000000000..1d60219c4 --- /dev/null +++ b/modules/mesh-recovery/cmd/mesh-recovery/keeper.go @@ -0,0 +1,151 @@ +package main + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "sync" + "time" + + "git.novox.be/novox/mesh-contracts/go/seats/meshbroker" + "git.novox.be/novox/mesh-sdk/go/buscalls" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "mesh-recovery/internal/recovery" +) + +// liveKeeper is the bus module's keeper reached by the seat mesh-broker's verbs (design 47 G §681): snapshot-get's chunks +// received in mesh-sdk's form (A-139), snapshot-restore and its cancel called, and snapshot-restored heard. +type liveKeeper struct { + nc *nats.Conn + call Call + restored *restoredView +} + +func (k liveKeeper) verb(verb string, in, out any) error { + raw, err := k.call("seat:"+meshbroker.Seat+"."+verb, in) + if err != nil { + return fmt.Errorf("%s's %s: %w", meshbroker.Seat, verb, err) + } + if err := json.Unmarshal(raw, out); err != nil { + return fmt.Errorf("%s's %s answered what does not read: %w", meshbroker.Seat, verb, err) + } + return nil +} + +// Get subscribes to the chunks of a call of its own before it asks snapshot-get, so none is missed, then receives them in +// order, acknowledging each (§681, A-139). +func (k liveKeeper) Get(ctx context.Context, request string, fits func(meshbroker.SnapshotGetAnswer) error, w io.Writer) (meshbroker.SnapshotGetAnswer, error) { + var bits [16]byte + if _, err := rand.Read(bits[:]); err != nil { + return meshbroker.SnapshotGetAnswer{}, err + } + send := buscalls.SeatChunks{Seat: meshbroker.Seat, Verb: meshbroker.VerbSnapshotGet, Call: hex.EncodeToString(bits[:])} + chunks := make(chan *nats.Msg, 4) + sub, err := k.nc.ChanSubscribe(send.Chunks(), chunks) + if err != nil { + return meshbroker.SnapshotGetAnswer{}, fmt.Errorf("the chunks of snapshot-get cannot be heard: %w", err) + } + defer func() { _ = sub.Unsubscribe() }() + var ans meshbroker.SnapshotGetAnswer + if err := k.verb(meshbroker.VerbSnapshotGet, meshbroker.SnapshotGetInput{Call: send.Call, Request: request}, &ans); err != nil { + return ans, err + } + if ans.Error != "" { + return ans, nil + } + if err := fits(ans); err != nil { + return ans, err + } + if ans.Size > 0 && ans.Chunks == 0 { + return ans, fmt.Errorf("the keeper answered the request %s's %d bytes with no chunk: this account is granted no chunk subjects of snapshot-get", request, ans.Size) + } + next := func() (string, []byte, error) { + bound := time.NewTimer(buscalls.SeatAckWithin) + defer bound.Stop() + select { + case m := <-chunks: + return m.Subject, m.Data, nil + case <-bound.C: + return "", nil, buscalls.ErrSeatChunkMissed + case <-ctx.Done(): + return "", nil, ctx.Err() + } + } + ack := func(subject string) error { + _, err := k.nc.Request(subject, []byte("{}"), buscalls.SeatAckWithin) + return err + } + if _, err := send.Receive(ans.Chunks, next, ack, w); err != nil { + return ans, fmt.Errorf("the snapshot of the request %s: %w", request, err) + } + return ans, nil +} + +func (k liveKeeper) Restore(_ context.Context, request string, verify bool, why string) (meshbroker.SnapshotRestoreAnswer, error) { + var ans meshbroker.SnapshotRestoreAnswer + err := k.verb(meshbroker.VerbSnapshotRestore, meshbroker.SnapshotRestoreInput{Request: request, Verify: verify, Why: why}, &ans) + return ans, err +} + +func (k liveKeeper) CancelRestore(_ context.Context, act, why string) (meshbroker.SnapshotRestoreCancelAnswer, error) { + var ans meshbroker.SnapshotRestoreCancelAnswer + err := k.verb(meshbroker.VerbSnapshotRestoreCancel, meshbroker.SnapshotRestoreCancelInput{Act: act, Why: why}, &ans) + return ans, err +} + +func (k liveKeeper) Restored(ctx context.Context, request, act string) (meshbroker.SnapshotRestoredEvent, bool) { + return k.restored.wait(ctx, request, act) +} + +var _ recovery.Keeper = liveKeeper{} + +// restoredView is the keeper's snapshot-restored as heard, each request and act kept (§681): read as §556 has it, the last +// of each subject first, so an outcome said before a wait began wakes it too. +type restoredView struct { + mu sync.Mutex + by map[string]meshbroker.SnapshotRestoredEvent + changed chan struct{} +} + +func newRestoredView() *restoredView { + return &restoredView{by: map[string]meshbroker.SnapshotRestoredEvent{}, changed: make(chan struct{})} +} + +func (v *restoredView) hear(m jetstream.Msg) { + var ev meshbroker.SnapshotRestoredEvent + if err := json.Unmarshal(m.Data(), &ev); err == nil && ev.Request != "" && ev.Act != "" { + v.take(ev) + } + _ = m.Ack() +} + +func (v *restoredView) take(ev meshbroker.SnapshotRestoredEvent) { + v.mu.Lock() + v.by[ev.Request+"\x00"+ev.Act] = ev + close(v.changed) + v.changed = make(chan struct{}) + v.mu.Unlock() +} + +func (v *restoredView) wait(ctx context.Context, request, act string) (meshbroker.SnapshotRestoredEvent, bool) { + for { + v.mu.Lock() + ev, ok := v.by[request+"\x00"+act] + changed := v.changed + v.mu.Unlock() + if ok { + return ev, true + } + select { + case <-changed: + case <-ctx.Done(): + return meshbroker.SnapshotRestoredEvent{}, false + } + } +} diff --git a/modules/mesh-recovery/cmd/mesh-recovery/main.go b/modules/mesh-recovery/cmd/mesh-recovery/main.go index 8f48f11a5..3e2f748de 100644 --- a/modules/mesh-recovery/cmd/mesh-recovery/main.go +++ b/modules/mesh-recovery/cmd/mesh-recovery/main.go @@ -35,6 +35,7 @@ const ( envKeepDays = "MESH_RECOVERY_KEEP_DAYS_FILE" envMark = "MESH_RECOVERY_FILLING_MARK_FILE" envDumpWait = "MESH_RECOVERY_DUMP_WAIT_FILE" + envRestore = "MESH_RECOVERY_RESTORE_WAIT_FILE" ) // The settings' defaults: the design's values (§551, §564). @@ -96,6 +97,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } + restoreWait, err := readMinutes(getenv(envRestore), recovery.DefaultRestoreWait) + if err != nil { + return err + } account, err := process.ReadBrokerAccount(getenv) if err != nil { return err @@ -123,6 +128,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } } + // The keeper's snapshot-restored, each request and act, for a restore in place (§681). + restored := newRestoredView() + if _, err := account.HearSeatEvents(ctx, js, meshbroker.StreamSnapshotRestored, restored.hear, logger.Printf); err != nil { + return err + } // The module's findings go through mesh-sdk's held writer over its own state findings (§561), started before the // engine's start, so what the start writes is held and written. writer, err := views.NewHeldWriter(views.WriterConfig[meshconditions.Finding, meshconditions.FindingCleared, meshconditions.FindingsStated, meshconditions.FindingsAskedEvent]{ @@ -143,7 +153,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro node: account.Node, points: dir, running: running}, Digests: digests{conn.State("digests")}, Dump: dump, - Settings: recovery.Settings{Cap: cap, KeepDays: int(keepDays), FillingMark: int(mark), DumpWait: dumpWait}, + Keeper: liveKeeper{nc: nc, call: conn.Call, restored: restored}, + Settings: recovery.Settings{Cap: cap, KeepDays: int(keepDays), FillingMark: int(mark), DumpWait: dumpWait, RestoreWait: restoreWait}, Findings: writer, Node: account.Node, Logf: logger.Printf, diff --git a/modules/mesh-recovery/cmd/mesh-recovery/main_test.go b/modules/mesh-recovery/cmd/mesh-recovery/main_test.go index 7f1d18dce..a16eede55 100644 --- a/modules/mesh-recovery/cmd/mesh-recovery/main_test.go +++ b/modules/mesh-recovery/cmd/mesh-recovery/main_test.go @@ -1,10 +1,12 @@ package main import ( + "context" "strings" "testing" "time" + "git.novox.be/novox/mesh-contracts/go/seats/meshbroker" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/process" @@ -48,6 +50,27 @@ func TestWhatTheProcessReads(t *testing.T) { t.Errorf("two fingerprints answered %v", err) } }) + t.Run("the keeper's snapshot-restored of a request and act (§681)", func(t *testing.T) { + v := newRestoredView() + v.take(meshbroker.SnapshotRestoredEvent{Request: "req-1", Act: "act-0", Outcome: "failed"}) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, ok := v.wait(ctx, "req-1", "act-1"); ok { + t.Error("another act's outcome woke the wait") + } + woken := make(chan meshbroker.SnapshotRestoredEvent, 1) + go func() { + ev, _ := v.wait(context.Background(), "req-1", "act-1") + woken <- ev + }() + v.take(meshbroker.SnapshotRestoredEvent{Request: "req-1", Act: "act-1", Outcome: "done"}) + if ev := <-woken; ev.Outcome != "done" { + t.Errorf("woken by %+v", ev) + } + if ev, ok := v.wait(ctx, "req-1", "act-1"); !ok || ev.Outcome != "done" { + t.Errorf("an outcome said before the wait is not taken: %+v %v", ev, ok) + } + }) t.Run("a dump of the bus module runs (§563)", func(t *testing.T) { d := newDumpView(nil, nil) start := time.Date(2026, 11, 12, 2, 0, 0, 0, time.UTC) diff --git a/modules/mesh-recovery/files/snapshot-restore-wait-minutes b/modules/mesh-recovery/files/snapshot-restore-wait-minutes new file mode 100644 index 000000000..89cd4bdf5 --- /dev/null +++ b/modules/mesh-recovery/files/snapshot-restore-wait-minutes @@ -0,0 +1 @@ +${config:snapshot-restore-wait-minutes} diff --git a/modules/mesh-recovery/internal/recovery/control.go b/modules/mesh-recovery/internal/recovery/control.go index 941b22ccf..ef9ad13ea 100644 --- a/modules/mesh-recovery/internal/recovery/control.go +++ b/modules/mesh-recovery/internal/recovery/control.go @@ -134,6 +134,8 @@ func (e *Engine) Start(ctx context.Context) []UnrestoredAside { } return nil }) + // A keeper's snapshot a cut-off act left half or whole received is deleted (§681). + _ = os.RemoveAll(e.incoming()) _, recs, _ := e.Asides() for i := range recs { r := &recs[i] diff --git a/modules/mesh-recovery/internal/recovery/keeper.go b/modules/mesh-recovery/internal/recovery/keeper.go new file mode 100644 index 000000000..5c8167caa --- /dev/null +++ b/modules/mesh-recovery/internal/recovery/keeper.go @@ -0,0 +1,191 @@ +package recovery + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "git.novox.be/novox/mesh-contracts/go/seats/meshbroker" +) + +// Keeper is the bus module's keeper as the seat mesh-broker answers it (design 47 G §681): its snapshot taken before a +// `drop` or `recreate`, sent by snapshot-get, or restored in place by snapshot-restore when it holds a protected stream. +type Keeper interface { + // Get asks snapshot-get of the request; when its answer names a file and fits accepts it, it receives the chunks into + // w. A refusal of the keeper is the answer's Error, no error of Get. + Get(ctx context.Context, request string, fits func(meshbroker.SnapshotGetAnswer) error, w io.Writer) (meshbroker.SnapshotGetAnswer, error) + // Restore asks snapshot-restore; with verify the keeper checks the file and restores nothing. + Restore(ctx context.Context, request string, verify bool, why string) (meshbroker.SnapshotRestoreAnswer, error) + // CancelRestore asks snapshot-restore-cancel of a running restore. + CancelRestore(ctx context.Context, act, why string) (meshbroker.SnapshotRestoreCancelAnswer, error) + // Restored answers the keeper's snapshot-restored of that request and act once heard (read as §556 has it, so one said + // before the wait counts), or ok false when ctx ends first. + Restored(ctx context.Context, request, act string) (ev meshbroker.SnapshotRestoredEvent, ok bool) +} + +// The keeper's word for a snapshot it never hands out (§681). +const snapshotProtected = "snapshot protected" + +// ErrUnanswered is the end of a restore in place the keeper answered neither by its outcome nor by the cancel's (§681). +const ErrUnanswered = "snapshot-restore unanswered" + +// DefaultRestoreWait is how long a restore in place is waited on: past the keeper's own 60 minutes (§681, A-138). +const DefaultRestoreWait = 90 * time.Minute + +func (e *Engine) restoreWait() time.Duration { + if e.Settings.RestoreWait > 0 { + return e.Settings.RestoreWait + } + return DefaultRestoreWait +} + +func (e *Engine) incoming() string { return filepath.Join(e.Dir, "incoming") } + +// restoreRequest is the hand verb `restore` with `request`, a keeper's snapshot (§681 Its use, §682): an unprotected one is +// read through snapshot-get into incoming/, refused before the first chunk when it would pass the cap, checked +// against its digest before any aside, and restored by the module itself as a point is (§516), the file deleted when the +// act ends; a protected one is restored in place by the keeper (restoreInPlace). +func (e *Engine) restoreRequest(ctx context.Context, a *Act) error { + if e.Keeper == nil { + return e.endedRestored(a, errors.New("no keeper is reachable, so no snapshot of it is restored"), nil, false) + } + e.setStep(a, "receiving the keeper's snapshot", "") + name, err := token(a.Request) + if err != nil { + return e.endedRestored(a, err, nil, false) + } + if err := os.MkdirAll(e.incoming(), 0o700); err != nil { + return e.endedRestored(a, err, nil, false) + } + file := filepath.Join(e.incoming(), name) + defer os.Remove(file) + f, err := os.Create(file) + if err != nil { + return e.endedRestored(a, err, nil, false) + } + ans, err := e.Keeper.Get(ctx, a.Request, func(ans meshbroker.SnapshotGetAnswer) error { + if fill := e.filled(); fill+ans.Size > e.Settings.Cap { + return fmt.Errorf("%s: the cap is %d bytes, the snapshot of the request %s needs %d and the directory holds %d", + ErrCap, e.Settings.Cap, a.Request, ans.Size, fill) + } + return nil + }, f) + if cerr := f.Close(); err == nil { + err = cerr + } + if err != nil { + return e.endedRestored(a, err, nil, false) + } + if ans.Error == snapshotProtected { + _ = os.Remove(file) + return e.restoreInPlace(ctx, a) + } + if ans.Error != "" { + return e.endedRestored(a, errors.New(ans.Error), nil, false) + } + // The bytes are checked against the keeper's digest before any aside is taken (§567). + if got, n, err := digestOf(file); err != nil || got != ans.Digest || n != ans.Size { + return e.endedRestored(a, fmt.Errorf("snapshot damaged: the request %s's bytes received do not match its digest", a.Request), nil, false) + } + rec, err := e.setAside(ctx, a, []string{ans.Stream}, true) + if err != nil { + return e.endedRestored(a, err, nil, false) + } + e.setStep(a, "restoring", "") + if err := e.Bus.Restore(ctx, file); err != nil { + err = fmt.Errorf("stream %s: %w", ans.Stream, err) + if back := e.restoreFromAside(ctx, rec, true); back != nil { + err = fmt.Errorf("%w; and the aside is not put back: %v", err, back) + } + return e.endedRestored(a, err, asidePtr(rec), false) + } + rec.Step = stepRestored + _ = writeJSON(filepath.Join(e.asideDirOf(rec), asideFile), rec) + return e.endedRestored(a, nil, asidePtr(rec), false) +} + +// restoreInPlace is a protected snapshot's restore (§681, O-7): snapshot-restore with verify first, then the live data of +// the restore's module and kept data set aside, its names from the catalog entry of the fingerprint the node runs +// (A-137), then snapshot-restore, waited on by snapshot-restored of that request and the act it was answered for at most +// snapshot-restore-wait-minutes (A-138). Its `failed` puts the aside back. At the bound it asks snapshot-restore-cancel and +// waits one more bound for that act's outcome (A-140); when none comes it puts nothing back and ends failed with +// `snapshot-restore unanswered`, the aside left at `removed` for aside-unrestored (§561). A keeper answering busy ends the +// act failed before any aside, naming the keeper's running act (A-140). +func (e *Engine) restoreInPlace(ctx context.Context, a *Act) error { + e.setStep(a, "verifying the keeper's snapshot", "") + v, err := e.Keeper.Restore(ctx, a.Request, true, a.Why) + if err == nil { + err = keeperAnswer(v) + } + if err != nil { + return e.endedRestored(a, err, nil, false) + } + fp, err := e.Seats.RunningFingerprint(ctx, a.Module) + if err != nil { + return e.endedRestored(a, err, nil, false) + } + a.Fingerprint = fp + names, err := e.Seats.Streams(ctx, a.Module, a.Kept, fp) + if err != nil { + return e.endedRestored(a, err, nil, false) + } + rec, err := e.setAside(ctx, a, names, true) + if err != nil { + return e.endedRestored(a, err, nil, false) + } + putBack := func(why error) error { + if back := e.restoreFromAside(ctx, rec, true); back != nil { + why = fmt.Errorf("%w; and the aside is not put back: %v", why, back) + } + return e.endedRestored(a, why, asidePtr(rec), false) + } + e.setStep(a, "restoring in place", "the keeper's snapshot-restored") + r, err := e.Keeper.Restore(ctx, a.Request, false, a.Why) + if err == nil { + err = keeperAnswer(r) + } + if err != nil { + return putBack(err) + } + act := r.Accepted + ev, ok := e.restoredWithin(ctx, a.Request, act) + if !ok { + e.setStep(a, "cancelling the keeper's restore", "the keeper's snapshot-restored after the cancel") + why := fmt.Sprintf("the recovery module's wait of %s on the restore in place passed (act %s)", e.restoreWait(), a.ID) + if c, err := e.Keeper.CancelRestore(context.Background(), act, why); err != nil || c.Error != "" { + e.logf("the cancel of the keeper's restore %s is not accepted (%v%s); its outcome is waited on once more", act, err, c.Error) + } + if ev, ok = e.restoredWithin(context.Background(), a.Request, act); !ok { + return e.endedRestored(a, errors.New(ErrUnanswered), asidePtr(rec), false) + } + } + if ev.Outcome != meshbroker.SnapshotRestoredEventOutcomeDone { + return putBack(fmt.Errorf("the keeper's restore in place failed: %s", ev.Error)) + } + rec.Step = stepRestored + _ = writeJSON(filepath.Join(e.asideDirOf(rec), asideFile), rec) + return e.endedRestored(a, nil, asidePtr(rec), false) +} + +func (e *Engine) restoredWithin(ctx context.Context, request, act string) (meshbroker.SnapshotRestoredEvent, bool) { + wctx, cancel := context.WithTimeout(ctx, e.restoreWait()) + defer cancel() + return e.Keeper.Restored(wctx, request, act) +} + +// keeperAnswer reads snapshot-restore's one form (§691): accepted, or an error naming the keeper's refusal or its running act. +func keeperAnswer(r meshbroker.SnapshotRestoreAnswer) error { + switch { + case r.Error != "": + return errors.New(r.Error) + case r.Busy != "": + return fmt.Errorf("busy: the keeper restores its act %s", r.Busy) + case r.Accepted == "": + return errors.New("the keeper answered snapshot-restore with nothing") + } + return nil +} diff --git a/modules/mesh-recovery/internal/recovery/keeper_test.go b/modules/mesh-recovery/internal/recovery/keeper_test.go new file mode 100644 index 000000000..5084e2b76 --- /dev/null +++ b/modules/mesh-recovery/internal/recovery/keeper_test.go @@ -0,0 +1,206 @@ +package recovery_test + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "io" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-contracts/go/seats/meshbroker" + + "mesh-recovery/internal/recovery" +) + +// fakeKeeper is the bus module's keeper holding one snapshot of app_notes (the fake bus's file form). in place: how its +// restore in place ends: "done", "failed", "silent" (it says nothing until cancelled, then failed) or "deaf" (never). +type fakeKeeper struct { + mu sync.Mutex + b *fakeBus + file string + protected bool + busy string + damaged bool + inPlace string + asked []string + wrote int + cancelled chan struct{} +} + +func newKeeper(b *fakeBus) *fakeKeeper { + return &fakeKeeper{b: b, file: "snap:app_notes:as dropped", inPlace: "done", cancelled: make(chan struct{})} +} + +func (k *fakeKeeper) note(s string) { + k.mu.Lock() + k.asked = append(k.asked, s) + k.mu.Unlock() +} + +func (k *fakeKeeper) Get(_ context.Context, request string, fits func(meshbroker.SnapshotGetAnswer) error, w io.Writer) (meshbroker.SnapshotGetAnswer, error) { + k.note("get " + request) + if k.protected { + return meshbroker.SnapshotGetAnswer{Error: "snapshot protected"}, nil + } + sum := sha256.Sum256([]byte(k.file)) + ans := meshbroker.SnapshotGetAnswer{Request: request, Module: "app", Local: "notes", Stream: "app_notes", Size: int64(len(k.file)), + Digest: "sha256:" + hex.EncodeToString(sum[:]), Chunks: 1} + if err := fits(ans); err != nil { + return ans, err + } + body := k.file + if k.damaged { + body = "snap:app_notes:altered on its way" + } + n, err := io.WriteString(w, body) + k.wrote += n + return ans, err +} + +func (k *fakeKeeper) Restore(_ context.Context, request string, verify bool, _ string) (meshbroker.SnapshotRestoreAnswer, error) { + if verify { + k.note("verify " + request) + return meshbroker.SnapshotRestoreAnswer{Accepted: request}, nil + } + k.note("restore " + request) + if k.busy != "" { + return meshbroker.SnapshotRestoreAnswer{Busy: k.busy}, nil + } + return meshbroker.SnapshotRestoreAnswer{Accepted: "keeper-act-1"}, nil +} + +func (k *fakeKeeper) CancelRestore(_ context.Context, act, _ string) (meshbroker.SnapshotRestoreCancelAnswer, error) { + k.note("cancel " + act) + close(k.cancelled) + return meshbroker.SnapshotRestoreCancelAnswer{Accepted: act}, nil +} + +func (k *fakeKeeper) Restored(ctx context.Context, request, act string) (meshbroker.SnapshotRestoredEvent, bool) { + ev := meshbroker.SnapshotRestoredEvent{Request: request, Act: act, Outcome: "failed", Error: "the server refused the archive"} + switch k.inPlace { + case "done": + k.b.mu.Lock() + if _, live := k.b.streams["app_notes"]; live { + k.b.mu.Unlock() + return meshbroker.SnapshotRestoredEvent{Request: request, Act: act, Outcome: "failed", Error: "names live"}, true + } + k.b.streams["app_notes"] = "as dropped" + k.b.mu.Unlock() + ev.Outcome, ev.Error = "done", "" + return ev, true + case "failed": + return ev, true + case "silent": + select { + case <-k.cancelled: + ev.Error = "cancelled" + return ev, true + case <-ctx.Done(): + return ev, false + } + } + <-ctx.Done() + return ev, false +} + +func requestRig(t *testing.T) (*rig, *fakeKeeper) { + r := newRig(t) + k := newKeeper(r.b) + r.e.Keeper = k + r.e.Settings.RestoreWait = 20 * time.Millisecond + return r, k +} + +func requestAct() recovery.Act { + return recovery.Act{Kind: recovery.KindRestore, Module: "app", Kept: "notes", Request: "req-1", Why: "test the drop's snapshot", Caller: "tool-runner", Asker: "op@node"} +} + +// A keeper's snapshot by request is read through snapshot-get into incoming/, refused before the first chunk past +// the cap, checked against its digest before any aside, restored by the module itself, and its file deleted at the end +// (design 47 G §681 Its use, §682). +func TestARestoreByRequestReadsTheKeepersSnapshot(t *testing.T) { + for _, tc := range []struct { + name string + cap int64 + damaged bool + err string + }{ + {name: "received and restored", cap: 1 << 20}, + {name: "altered on its way", cap: 1 << 20, damaged: true, err: "snapshot damaged"}, + {name: "past the cap", cap: 10, err: "cap:"}, + } { + t.Run(tc.name, func(t *testing.T) { + r, k := requestRig(t) + r.e.Settings.Cap, k.damaged = tc.cap, tc.damaged + ans := r.do(requestAct()) + ev := lastRestored(t, r) + if ev.Act != ans.Accepted || !strings.HasPrefix(ev.Error, tc.err) || (tc.err == "") != (ev.Outcome == "done") { + t.Fatalf("restored %+v, want error %q", ev, tc.err) + } + if left, _ := os.ReadDir(filepath.Join(r.e.Dir, "incoming")); len(left) != 0 { + t.Errorf("incoming holds %v after the act", left) + } + kp, _ := r.e.KeptPoints("app", "notes") + if tc.err != "" { + if r.b.streams["app_notes"] != "hello" || len(kp.Asides) != 0 { + t.Errorf("a refused restore touched the live data: %v, asides %+v", r.b.streams, kp.Asides) + } + if tc.name == "past the cap" && k.wrote != 0 { + t.Errorf("%d bytes received past the cap", k.wrote) + } + return + } + if r.b.streams["app_notes"] != "as dropped" || r.b.streams["KV_app_seen"] != "world" || len(kp.Asides) != 1 || + ev.Aside == nil || len(ev.Aside.Names) != 1 || ev.Aside.Names[0] != "app_notes" { + t.Errorf("streams %v, asides %+v, event aside %+v", r.b.streams, kp.Asides, ev.Aside) + } + }) + } +} + +// A protected snapshot is verified, the running data of the module and kept data set aside (A-137), and restored in place +// by the keeper; its failure puts the aside back; at the bound the module cancels first and puts the aside back on the +// cancel's outcome, and with no outcome at all ends `snapshot-restore unanswered`, the aside left unrestored; a busy keeper +// ends the act before any aside (§681, A-138, A-140). +func TestAProtectedSnapshotIsRestoredInPlaceByTheKeeper(t *testing.T) { + for _, tc := range []struct { + inPlace, busy string + err string + asked string + live string + unrestored int + }{ + {inPlace: "done", asked: "get req-1|verify req-1|restore req-1", live: "as dropped"}, + {inPlace: "failed", asked: "get req-1|verify req-1|restore req-1", live: "hello", err: "the keeper's restore in place failed"}, + {inPlace: "silent", asked: "get req-1|verify req-1|restore req-1|cancel keeper-act-1", live: "hello", err: "the keeper's restore in place failed: cancelled"}, + {inPlace: "deaf", asked: "get req-1|verify req-1|restore req-1|cancel keeper-act-1", err: recovery.ErrUnanswered, unrestored: 1}, + {inPlace: "done", busy: "keeper-act-0", asked: "get req-1|verify req-1|restore req-1", live: "hello", err: "busy: the keeper restores its act keeper-act-0"}, + } { + t.Run(tc.inPlace+tc.busy, func(t *testing.T) { + r, k := requestRig(t) + k.protected, k.inPlace, k.busy = true, tc.inPlace, tc.busy + r.do(requestAct()) + ev := lastRestored(t, r) + if !strings.HasPrefix(ev.Error, tc.err) || (tc.err == "") != (ev.Outcome == "done") || ev.Fingerprint != "fp-run" { + t.Fatalf("restored %+v, want error %q", ev, tc.err) + } + if got := strings.Join(k.asked, "|"); got != tc.asked { + t.Errorf("asked %s, want %s", got, tc.asked) + } + if r.b.streams["app_notes"] != tc.live { + t.Errorf("app_notes is %q, want %q", r.b.streams["app_notes"], tc.live) + } + if tc.live == "hello" && r.b.streams["KV_app_seen"] != "world" { + t.Errorf("the second name of the running data is not put back: %v", r.b.streams) + } + if n := len(r.e.Unrestored()); n != tc.unrestored { + t.Errorf("%d unrestored asides, want %d", n, tc.unrestored) + } + }) + } +} diff --git a/modules/mesh-recovery/internal/recovery/manifest_test.go b/modules/mesh-recovery/internal/recovery/manifest_test.go index b68c808b9..7e904fa1b 100644 --- a/modules/mesh-recovery/internal/recovery/manifest_test.go +++ b/modules/mesh-recovery/internal/recovery/manifest_test.go @@ -70,11 +70,17 @@ func TestTheManifestSaysWhatTheModuleIs(t *testing.T) { t.Errorf("no data item %q", id) } } - for k, want := range map[string]float64{"points-cap": 10 << 30, "points-filling-mark": 80, "point-keep-days": 90, "dump-wait-minutes": 60} { + for k, want := range map[string]float64{"points-cap": 10 << 30, "points-filling-mark": 80, "point-keep-days": 90, "dump-wait-minutes": 60, "snapshot-restore-wait-minutes": 90} { if m.Config[k].Default != want { t.Errorf("setting %s defaults to %v, want %v", k, m.Config[k].Default, want) } } + // The keeper's snapshot verbs the recovery module asks, the cancel at its bound among them (§681). + for _, inv := range []string{"seat:mesh-broker.snapshot-get", "seat:mesh-broker.snapshot-restore", "seat:mesh-broker.snapshot-restore-cancel"} { + if !slices.Contains(m.Invokes, inv) { + t.Errorf("does not invoke %s", inv) + } + } for _, r := range m.Resources { if r.Type != "process" { continue diff --git a/modules/mesh-recovery/internal/recovery/recovery.go b/modules/mesh-recovery/internal/recovery/recovery.go index f37c7b704..2e1f09076 100644 --- a/modules/mesh-recovery/internal/recovery/recovery.go +++ b/modules/mesh-recovery/internal/recovery/recovery.go @@ -104,15 +104,20 @@ type Settings struct { FillingMark int // DumpWait is how long an act waits on the dump before recovery-waits is written (dump-wait-minutes, 60 unless set). DumpWait time.Duration + // RestoreWait is how long a restore in place is waited on, and once more after its cancel (snapshot-restore-wait-minutes, + // 90 unless set; §681, A-138, A-140). + RestoreWait time.Duration } // Engine is the recovery module's one engine. type Engine struct { - Dir string - Bus Bus - Seats Seats - Digests Digests - Dump Dump + Dir string + Bus Bus + Seats Seats + Digests Digests + Dump Dump + // Keeper is the bus module's keeper, whose snapshots the hand verb restore takes by request (§681). + Keeper Keeper Settings Settings // Findings is the module's finding writer (§561); Node the machine it runs on, the id of points-filling. Findings Findings @@ -130,24 +135,26 @@ type Engine struct { // Act is one accepted act, written in the directory before its caller is answered (§554). type Act struct { - ID string `json:"act"` - Kind string `json:"kind"` - Module string `json:"module,omitempty"` - Kept string `json:"kept,omitempty"` - Form int64 `json:"form,omitempty"` - Fingerprint string `json:"fingerprint,omitempty"` - Snapshot string `json:"snapshot,omitempty"` - AsideID string `json:"aside-id,omitempty"` - FromBackup bool `json:"from-backup,omitempty"` - CatchUp *seat.CatchUp `json:"catch-up,omitempty"` - Why string `json:"why"` - Caller string `json:"caller"` - Asker string `json:"asker,omitempty"` - Since time.Time `json:"since"` - Step string `json:"step,omitempty"` - WaitsOn string `json:"waits-on,omitempty"` - Ended bool `json:"ended,omitempty"` - Error string `json:"error,omitempty"` + ID string `json:"act"` + Kind string `json:"kind"` + Module string `json:"module,omitempty"` + Kept string `json:"kept,omitempty"` + Form int64 `json:"form,omitempty"` + Fingerprint string `json:"fingerprint,omitempty"` + Snapshot string `json:"snapshot,omitempty"` + // Request is a keeper's snapshot by its request, the hand verb restore's other source (§681, §682). + Request string `json:"request,omitempty"` + AsideID string `json:"aside-id,omitempty"` + FromBackup bool `json:"from-backup,omitempty"` + CatchUp *seat.CatchUp `json:"catch-up,omitempty"` + Why string `json:"why"` + Caller string `json:"caller"` + Asker string `json:"asker,omitempty"` + Since time.Time `json:"since"` + Step string `json:"step,omitempty"` + WaitsOn string `json:"waits-on,omitempty"` + Ended bool `json:"ended,omitempty"` + Error string `json:"error,omitempty"` } // Answer is the one form every acting verb answers (§691): exactly one field set. diff --git a/modules/mesh-recovery/internal/recovery/restore.go b/modules/mesh-recovery/internal/recovery/restore.go index 4fdbdfd20..c6936f758 100644 --- a/modules/mesh-recovery/internal/recovery/restore.go +++ b/modules/mesh-recovery/internal/recovery/restore.go @@ -70,7 +70,7 @@ func (e *Engine) cancelledWhy(err error) string { return "" } -// restore is `restore-kept` and the hand verb `restore` with a point (§516, §554, §563, §567): it says restore-started, +// restore is `restore-kept` and the hand verb `restore` with a point (or, past the dump, a keeper's snapshot by request) (§516, §554, §563, §567): it says restore-started, // waits for the dump of the bus's item, finds the point (or has the backup holder bring it back), checks its digests // against the bus, sets the live data aside whole and checked, removes it, restores the point under its names and // sets each consumer to its recorded sequence; a step after the removal that fails puts the aside back. @@ -91,6 +91,9 @@ func (e *Engine) restore(ctx context.Context, a *Act) error { return e.endedRestored(a, err, nil, false) } } + if a.Request != "" { + return e.restoreRequest(ctx, a) + } e.setStep(a, "finding the point", "") pointPath := filepath.Join(e.Dir, "points", filepath.FromSlash(a.Snapshot)) if _, err := os.Stat(filepath.Join(pointPath, pointFile)); err != nil { diff --git a/modules/mesh-recovery/internal/recovery/verbs.go b/modules/mesh-recovery/internal/recovery/verbs.go index 105868eb7..34cbdf7ec 100644 --- a/modules/mesh-recovery/internal/recovery/verbs.go +++ b/modules/mesh-recovery/internal/recovery/verbs.go @@ -46,10 +46,10 @@ func (e *Engine) Handlers() map[string]process.SeatHandler { if (rs.Snapshot == "") == (rs.Request == "") { return Act{}, errors.New("restore takes exactly one of snapshot and request") } - if rs.Request != "" { - return Act{}, errors.New("the restore of a keeper's snapshot by request is not built yet (design 47 G §681)") + if rs.Request != "" && rs.FromBackup { + return Act{}, errors.New("from-backup is taken only with snapshot, a restore point (design 47 G §682)") } - return Act{Kind: KindRestore, Module: rs.Module, Kept: rs.Kept, Snapshot: rs.Snapshot, FromBackup: rs.FromBackup, Why: rs.Why, Asker: rs.Asker}, nil + return Act{Kind: KindRestore, Module: rs.Module, Kept: rs.Kept, Snapshot: rs.Snapshot, Request: rs.Request, FromBackup: rs.FromBackup, Why: rs.Why, Asker: rs.Asker}, nil }) h[seat.VerbAside] = actOf(e, func(ad seat.AsideInput) (Act, error) { return Act{Kind: KindAside, Module: ad.Module, Kept: ad.Kept, Why: ad.Why, Asker: ad.Asker}, nil diff --git a/modules/mesh-recovery/module.json b/modules/mesh-recovery/module.json index 4eef18760..3f1e6e024 100644 --- a/modules/mesh-recovery/module.json +++ b/modules/mesh-recovery/module.json @@ -45,6 +45,7 @@ "seat:mesh-hand-acts.hand-act", "seat:mesh-broker.snapshot-get", "seat:mesh-broker.snapshot-restore", + "seat:mesh-broker.snapshot-restore-cancel", "seat:node-backup.restore", "seat:mesh-recovery.kept-points", "seat:mesh-recovery.restore-aside", @@ -113,6 +114,14 @@ "maximum": 10080, "trusted": false, "default": 60 + }, + "snapshot-restore-wait-minutes": { + "type": "number", + "description": "minutes a restore in place of a protected keeper's snapshot is waited on, and once more after its cancel (design 47 G section 681, A-138, A-140).", + "minimum": 1, + "maximum": 10080, + "trusted": false, + "default": 90 } }, "data": { @@ -185,6 +194,14 @@ "mode": "0600", "template": "files/dump-wait-minutes" }, + { + "id": "restore-wait", + "type": "file", + "path": "${dir:state}/snapshot-restore-wait-minutes", + "owner": "mesh-recovery", + "mode": "0600", + "template": "files/snapshot-restore-wait-minutes" + }, { "id": "recovery", "type": "process", @@ -199,7 +216,8 @@ "MESH_RECOVERY_POINTS_CAP_FILE": "${dir:state}/points-cap", "MESH_RECOVERY_FILLING_MARK_FILE": "${dir:state}/points-filling-mark", "MESH_RECOVERY_KEEP_DAYS_FILE": "${dir:state}/point-keep-days", - "MESH_RECOVERY_DUMP_WAIT_FILE": "${dir:state}/dump-wait-minutes" + "MESH_RECOVERY_DUMP_WAIT_FILE": "${dir:state}/dump-wait-minutes", + "MESH_RECOVERY_RESTORE_WAIT_FILE": "${dir:state}/snapshot-restore-wait-minutes" }, "user": "mesh-recovery", "serves": [ @@ -222,6 +240,7 @@ "seat:mesh-hand-acts.hand-act", "seat:mesh-broker.snapshot-get", "seat:mesh-broker.snapshot-restore", + "seat:mesh-broker.snapshot-restore-cancel", "seat:node-backup.restore" ] }, diff --git a/modules/nats/cmd/nats-tools/snapshots.go b/modules/nats/cmd/nats-tools/snapshots.go index 01f2e03f8..a1d455588 100644 --- a/modules/nats/cmd/nats-tools/snapshots.go +++ b/modules/nats/cmd/nats-tools/snapshots.go @@ -19,6 +19,7 @@ import ( "git.novox.be/novox/mesh-contracts/go/seats/meshbroker" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscalls" "git.novox.be/novox/mesh-sdk/go/process" "nats-tools/internal/keeper" @@ -30,34 +31,32 @@ import ( // snapshot-restore, snapshot-restore-cancel and snapshot-remove are acted on by the keeper at once, with no hand-over through the // state requests. The tools bundle keeps nats_snapshots, which lists them. -// ChunkSize is at most what one chunk carries: 512 KiB. -const ChunkSize = 512 << 10 - -// AckWithin is how long the next chunk waits for its acknowledgement. -const AckWithin = 30 * time.Second +// The form of the chunks is mesh-sdk's (buscalls.SeatChunks, A-139): the caller receives with the same one. +const ( + // ChunkSize is at most what one chunk carries: 512 KiB. + ChunkSize = buscalls.SeatChunkBytes + // AckWithin is how long the next chunk waits for its acknowledgement. + AckWithin = buscalls.SeatAckWithin +) // callToken is the shape of the subject token a caller chooses for a send. var callToken = regexp.MustCompile(`^[A-Za-z0-9_-]{16,64}$`) -// ChunkSubject and AckSubject are the subjects of the nth chunk of a send and of its acknowledgement. -func ChunkSubject(call string, n int64) string { - return "mesh.seat." + meshbroker.Seat + ".chunk." + meshbroker.VerbSnapshotGet + "." + call + "." + strconv.FormatInt(n, 10) +func sendOf(call string) buscalls.SeatChunks { + return buscalls.SeatChunks{Seat: meshbroker.Seat, Verb: meshbroker.VerbSnapshotGet, Call: call} } +// ChunkSubject and AckSubject are the subjects of the nth chunk of a send and of its acknowledgement. +func ChunkSubject(call string, n int64) string { return sendOf(call).Chunk(n) } + // AckSubject is the subject the caller acknowledges the nth chunk on. -func AckSubject(call string, n int64) string { - return "mesh.seat." + meshbroker.Seat + ".ack." + meshbroker.VerbSnapshotGet + "." + call + "." + strconv.FormatInt(n, 10) -} +func AckSubject(call string, n int64) string { return sendOf(call).Ack(n) } // AckPattern is every acknowledgement of snapshot-get. -const AckPattern = "mesh.seat.mesh-broker.ack.snapshot-get.*.*" +var AckPattern = buscalls.SeatAcks(meshbroker.Seat, meshbroker.VerbSnapshotGet) // Chunk is the body of one chunk: its number, how many there are, and its bytes. -type Chunk struct { - N int64 `json:"n"` - Of int64 `json:"of"` - Data []byte `json:"data"` -} +type Chunk = buscalls.SeatChunk // snapshotWorld is what the keeper's snapshot verbs reach: the file, the process's own publish and subscription, and the bus's users. type snapshotWorld struct { From 1ed0dca7df7657172fd30bf19aaecdeb87306d88 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 02:15:38 +0100 Subject: [PATCH 14/17] mesh-recovery: an act cut off by the process's stop ends stopped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A move or a restart stops the process: from its stop signal every call is answered an error naming the stop, the running act ends through its own failure path (an aside put back whole, whatever its context), and every accepted act not ended ends failed with stopped and its event, so its caller asks again. Builds: design 47 G §562, A-141 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../mesh-recovery/cmd/mesh-recovery/main.go | 4 +- .../internal/recovery/control.go | 12 ++++- .../mesh-recovery/internal/recovery/points.go | 1 + .../internal/recovery/recovery.go | 34 +++++++++++++ .../internal/recovery/restore.go | 5 +- .../internal/recovery/restore_test.go | 49 +++++++++++++++++++ .../mesh-recovery/internal/recovery/verbs.go | 9 ++++ 7 files changed, 110 insertions(+), 4 deletions(-) diff --git a/modules/mesh-recovery/cmd/mesh-recovery/main.go b/modules/mesh-recovery/cmd/mesh-recovery/main.go index 3e2f748de..78987366b 100644 --- a/modules/mesh-recovery/cmd/mesh-recovery/main.go +++ b/modules/mesh-recovery/cmd/mesh-recovery/main.go @@ -179,8 +179,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro for { select { case <-ctx.Done(): + // A move or a restart stops the process: it takes no new call, the running act ends through its own failure + // path, and every accepted act ends stopped, so its caller asks again (§562, A-141). + e.Stop("its stop signal") stopAnswers() - e.Wait() return nil case <-tick.C: expire("the daily removal of points past their keep days (design 47 G section 551)") diff --git a/modules/mesh-recovery/internal/recovery/control.go b/modules/mesh-recovery/internal/recovery/control.go index ef9ad13ea..d4841a96c 100644 --- a/modules/mesh-recovery/internal/recovery/control.go +++ b/modules/mesh-recovery/internal/recovery/control.go @@ -17,6 +17,9 @@ import ( // refuses an act that has begun, naming its step, since a restore cut off after its removal is put back only by its own // failure path (§516). It is written in the hand-act log first and answers in §691's one form. func (e *Engine) Cancel(act, why, caller, asker string) Answer { + if stop := e.Stopping(); stop != "" { + return Answer{Error: stop} + } if err := e.Seats.HandAct(caller, asker, "cancel "+act, why); err != nil { return Answer{Error: "the hand-act log was not written, so nothing was done: " + err.Error()} } @@ -150,6 +153,13 @@ func (e *Engine) Start(ctx context.Context) []UnrestoredAside { } } } + e.endUnended() + e.judge() + return e.Unrestored() +} + +// endUnended ends every accepted act that never ended failed with `stopped` and its ending event (§554, §562), oldest first. +func (e *Engine) endUnended() { acts := e.readActs() sort.Slice(acts, func(i, j int) bool { return acts[i].Since.Before(acts[j].Since) }) for i := range acts { @@ -167,8 +177,6 @@ func (e *Engine) Start(ctx context.Context) []UnrestoredAside { a.Error = ErrStopped e.markEnded(&a) } - e.judge() - return e.Unrestored() } func anyExists(os []Object) bool { diff --git a/modules/mesh-recovery/internal/recovery/points.go b/modules/mesh-recovery/internal/recovery/points.go index 18e5ec14c..b3d9f1bed 100644 --- a/modules/mesh-recovery/internal/recovery/points.go +++ b/modules/mesh-recovery/internal/recovery/points.go @@ -52,6 +52,7 @@ func (e *Engine) say(event string, body any, parts ...string) { } func (e *Engine) endedFailed(a *Act, why error) error { + why = e.stoppedOr(why) e.say(seat.EventEnded, seat.EndedEvent{Act: a.ID, At: e.now(), Module: a.Module, Kept: a.Kept, Form: a.Form, Fingerprint: a.Fingerprint, Snapshot: a.Snapshot, Outcome: seat.EndedEventOutcomeFailed, Error: why.Error(), Why: cancelWhy(a, why)}, a.Module, a.Kept) diff --git a/modules/mesh-recovery/internal/recovery/recovery.go b/modules/mesh-recovery/internal/recovery/recovery.go index 2e1f09076..efd0faddf 100644 --- a/modules/mesh-recovery/internal/recovery/recovery.go +++ b/modules/mesh-recovery/internal/recovery/recovery.go @@ -131,6 +131,7 @@ type Engine struct { wg sync.WaitGroup seq int why string // the hand act words of a cancel (A-117) + stop string // why the process stops, once it does (§562, A-141) } // Act is one accepted act, written in the directory before its caller is answered (§554). @@ -242,6 +243,9 @@ var errBusy = errors.New("busy") func (e *Engine) Accept(a Act) Answer { e.mu.Lock() defer e.mu.Unlock() + if e.stop != "" { + return Answer{Error: e.stop} + } if e.running != nil { return Answer{Busy: e.running.ID} } @@ -275,6 +279,36 @@ func (e *Engine) Accept(a Act) Answer { // Wait returns once the running act, if any, has ended (a whole test's and a stop's join). func (e *Engine) Wait() { e.wg.Wait() } +// Stop is the process's stop, a move's or a restart's (§562, A-141): from now on every call is answered an error naming the +// stop, the running act's context is cancelled so it ends through its own failure path (a restore past its removal puts +// the aside back and sets the consumers back), and once it has ended every accepted act not ended is ended failed with +// `stopped` and its ending event, so its caller asks again. +func (e *Engine) Stop(why string) { + e.mu.Lock() + e.stop = "the recovery module is stopping (" + why + "), so it takes no call; ask again once it runs" + if e.cancel != nil { + e.cancel() + } + e.mu.Unlock() + e.wg.Wait() + e.endUnended() +} + +// Stopping is the error a call is answered while the process stops, empty while it runs. +func (e *Engine) Stopping() string { + e.mu.Lock() + defer e.mu.Unlock() + return e.stop +} + +// stoppedOr is an act's failure as its ending event says it: `stopped` once the process stops (§562). +func (e *Engine) stoppedOr(err error) error { + if err != nil && e.Stopping() != "" { + return errors.New(ErrStopped) + } + return err +} + func (e *Engine) setStep(a *Act, step, waitsOn string) { e.mu.Lock() a.Step, a.WaitsOn = step, waitsOn diff --git a/modules/mesh-recovery/internal/recovery/restore.go b/modules/mesh-recovery/internal/recovery/restore.go index c6936f758..6fd4fd60e 100644 --- a/modules/mesh-recovery/internal/recovery/restore.go +++ b/modules/mesh-recovery/internal/recovery/restore.go @@ -54,6 +54,7 @@ func (e *Engine) asideRoot(module, kept string) (string, error) { func (e *Engine) endedRestored(a *Act, outcomeErr error, aside *seat.Aside, missing bool) error { ev := seat.RestoredEvent{Act: a.ID, At: e.now(), Module: a.Module, Kept: a.Kept, Form: a.Form, Fingerprint: a.Fingerprint, Snapshot: a.Snapshot, Outcome: seat.RestoredEventOutcomeDone, Aside: aside, Missing: missing} + outcomeErr = e.stoppedOr(outcomeErr) if outcomeErr != nil { ev.Outcome, ev.Error, ev.Why = seat.RestoredEventOutcomeFailed, outcomeErr.Error(), e.cancelledWhy(outcomeErr) } @@ -257,8 +258,10 @@ func (e *Engine) putBack(ctx context.Context, dir string, ds []seat.Digest, cs [ return e.Bus.SetConsumers(ctx, cs) } -// restoreFromAside puts an aside back under the live names: any partly restored stream is removed first. +// restoreFromAside puts an aside back under the live names: any partly restored stream is removed first. It is a failure +// path, so it runs whole also when the act's context ended, as at the process's stop (§562). func (e *Engine) restoreFromAside(ctx context.Context, rec *asideRecord, clear bool) error { + ctx = context.WithoutCancel(ctx) if clear { for _, s := range rec.Streams { if err := e.Bus.Delete(ctx, s); err != nil { diff --git a/modules/mesh-recovery/internal/recovery/restore_test.go b/modules/mesh-recovery/internal/recovery/restore_test.go index 88031c462..cc168ee6c 100644 --- a/modules/mesh-recovery/internal/recovery/restore_test.go +++ b/modules/mesh-recovery/internal/recovery/restore_test.go @@ -294,3 +294,52 @@ func TestARestoreWhoseStartIsNotAcknowledgedTouchesNothing(t *testing.T) { t.Errorf("an aside was taken: %+v", kp.Asides) } } + +// stallBus blocks its first Restore until the act's context ends, as a restore under way when the process stops. +type stallBus struct { + *fakeBus + in chan struct{} + stalled bool +} + +func (s *stallBus) Restore(ctx context.Context, file string) error { + if !s.stalled { + s.stalled = true + close(s.in) + <-ctx.Done() + return ctx.Err() + } + return s.fakeBus.Restore(ctx, file) +} + +// A stop of the process (a move's or a restart's) answers every new call an error naming it, ends the running restore +// through its failure path, putting the aside back, and ends every accepted act with `stopped`, so its caller asks again +// (design 47 G §562, A-141). +func TestAStopEndsTheRunningActThroughItsFailurePathAndStopped(t *testing.T) { + r := takenRig(t) + r.b.streams["app_notes"] = "changed since" + stall := &stallBus{fakeBus: r.b, in: make(chan struct{})} + r.e.Bus = stall + ans := r.e.Accept(restoreAct("app/notes/2-fp1")) + <-stall.in + // An accepted act a crash left unended is ended by the stop too. + _ = os.WriteFile(filepath.Join(r.e.Dir, "acts", "act-0.json"), []byte(`{"act":"act-0","kind":"take","module":"app","kept":"seen","why":"w","caller":"c","since":"2026-11-11T00:00:00Z"}`), 0o600) + r.e.Stop("a move") + if r.b.streams["app_notes"] != "changed since" || r.b.streams["KV_app_seen"] != "world" { + t.Errorf("the aside is not put back: %v", r.b.streams) + } + if ev := lastRestored(t, r); ev.Act != ans.Accepted || ev.Outcome != "failed" || ev.Error != recovery.ErrStopped { + t.Errorf("restored %+v", ev) + } + ended := r.s.named(seat.EventEnded) + if len(ended) != 1 || ended[0].Body.(seat.EndedEvent).Act != "act-0" || ended[0].Body.(seat.EndedEvent).Error != recovery.ErrStopped { + t.Errorf("ended %+v", ended) + } + if got := r.e.Accept(takeAct()); !strings.Contains(got.Error, "stopping (a move)") { + t.Errorf("a call during the stop answered %+v", got) + } + verb := r.e.Handlers()[seat.VerbKeptPoints] + if got, _ := verb("c", []byte(`{}`)); !strings.Contains(got.(recovery.Answer).Error, "stopping") { + t.Errorf("kept-points during the stop answered %+v", got) + } +} diff --git a/modules/mesh-recovery/internal/recovery/verbs.go b/modules/mesh-recovery/internal/recovery/verbs.go index 34cbdf7ec..bd053a9ca 100644 --- a/modules/mesh-recovery/internal/recovery/verbs.go +++ b/modules/mesh-recovery/internal/recovery/verbs.go @@ -77,6 +77,15 @@ func (e *Engine) Handlers() map[string]process.SeatHandler { } return e.Cancel(in.Act, in.Why, caller, in.Asker), nil } + // A stopping process takes no call, each answered the error naming the stop (§562, A-141). + for verb, handle := range h { + h[verb] = func(caller string, body json.RawMessage) (any, error) { + if stop := e.Stopping(); stop != "" { + return Answer{Error: stop}, nil + } + return handle(caller, body) + } + } return h } From 80ade3cacb2d27a2a384440d9ca01804fdab9f49 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 02:18:22 +0100 Subject: [PATCH 15/17] mesh-recovery: every bound a setting, the expiry on a bound key's expiry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit point-keep-days, dump-wait-minutes and snapshot-restore-wait-minutes may be fractions, and expire-every-hours (24 unless set) times the removal of old points by a key of the state bounds whose own time to live the bus ends, heard as its change: no timer in the process. Builds: design 47 G §551, §557, A-77, A-130, A-138 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../mesh-recovery/cmd/mesh-recovery/expiry.go | 44 ++++++++++++++ .../mesh-recovery/cmd/mesh-recovery/main.go | 37 ++++++------ .../cmd/mesh-recovery/main_test.go | 59 +++++++++++++++++++ .../mesh-recovery/cmd/mesh-recovery/seats.go | 14 +++-- .../mesh-recovery/files/expire-every-hours | 1 + .../internal/recovery/manifest_test.go | 8 ++- .../mesh-recovery/internal/recovery/points.go | 3 +- .../internal/recovery/recovery.go | 2 +- modules/mesh-recovery/module.json | 33 +++++++++-- 9 files changed, 173 insertions(+), 28 deletions(-) create mode 100644 modules/mesh-recovery/cmd/mesh-recovery/expiry.go create mode 100644 modules/mesh-recovery/files/expire-every-hours diff --git a/modules/mesh-recovery/cmd/mesh-recovery/expiry.go b/modules/mesh-recovery/cmd/mesh-recovery/expiry.go new file mode 100644 index 000000000..14b4ad65d --- /dev/null +++ b/modules/mesh-recovery/cmd/mesh-recovery/expiry.go @@ -0,0 +1,44 @@ +package main + +import ( + "errors" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// boundState is the module's own state bounds, whose keys live a time of their own (key-ttl): a bound key's expiry is +// heard as the bus's own change, so nothing in the process waits on a timer (design 47 G A-77, A-138). +type boundState interface { + CreateTTL(key string, value any, ttl time.Duration) (uint64, error) + Watch(key string, handle func(stdio.StateChange) error) error +} + +// expiryKey is the bound key of §551's daily removal. +const expiryKey = "expiry" + +// expireOnBound runs due each time the bound key `expiry`, written with the time to live `every` (expire-every-hours), is +// removed by the bus, and writes it again (§551, A-138). A key a run before this start wrote is kept, so a restart does not +// put the next removal off; its removal at the start is the start's own run. +func expireOnBound(s boundState, every time.Duration, due func(), logf func(string, ...any)) error { + arm := func() error { + _, err := s.CreateTTL(expiryKey, map[string]any{"every": every.String()}, every) + if errors.Is(err, stdio.ErrChanged) { + return nil + } + return err + } + if err := s.Watch(expiryKey, func(c stdio.StateChange) error { + if c.Op == "put" || c.Current { + return nil + } + due() + if err := arm(); err != nil { + logf("the bound of the next removal of points past their keep days is not written: %v", err) + } + return nil + }); err != nil { + return err + } + return arm() +} diff --git a/modules/mesh-recovery/cmd/mesh-recovery/main.go b/modules/mesh-recovery/cmd/mesh-recovery/main.go index 78987366b..6d374b667 100644 --- a/modules/mesh-recovery/cmd/mesh-recovery/main.go +++ b/modules/mesh-recovery/cmd/mesh-recovery/main.go @@ -36,6 +36,7 @@ const ( envMark = "MESH_RECOVERY_FILLING_MARK_FILE" envDumpWait = "MESH_RECOVERY_DUMP_WAIT_FILE" envRestore = "MESH_RECOVERY_RESTORE_WAIT_FILE" + envExpire = "MESH_RECOVERY_EXPIRE_EVERY_FILE" ) // The settings' defaults: the design's values (§551, §564). @@ -44,8 +45,8 @@ const ( defaultKeepDays = 90 ) -// ExpireEvery is how often §551's removal runs beside its start and each snapshot-kept: once a day. -var ExpireEvery = 24 * time.Hour +// defaultExpireEvery is how often §551's removal runs beside its start and each snapshot-kept: once a day (expire-every-hours). +const defaultExpireEvery = 24 * time.Hour func main() { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) @@ -85,7 +86,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - keepDays, err := readNumber(getenv(envKeepDays), defaultKeepDays) + keepDays, err := readFraction(getenv(envKeepDays), defaultKeepDays) if err != nil { return err } @@ -101,6 +102,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } + expireEvery, err := readFraction(getenv(envExpire), defaultExpireEvery.Hours()) + if err != nil { + return err + } account, err := process.ReadBrokerAccount(getenv) if err != nil { return err @@ -154,7 +159,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Digests: digests{conn.State("digests")}, Dump: dump, Keeper: liveKeeper{nc: nc, call: conn.Call, restored: restored}, - Settings: recovery.Settings{Cap: cap, KeepDays: int(keepDays), FillingMark: int(mark), DumpWait: dumpWait, RestoreWait: restoreWait}, + Settings: recovery.Settings{Cap: cap, KeepDays: keepDays, FillingMark: int(mark), DumpWait: dumpWait, RestoreWait: restoreWait}, Findings: writer, Node: account.Node, Logf: logger.Printf, @@ -174,18 +179,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } } expire("the removal of points past their keep days at the start (design 47 G section 551)") - tick := time.NewTicker(ExpireEvery) - defer tick.Stop() - for { - select { - case <-ctx.Done(): - // A move or a restart stops the process: it takes no new call, the running act ends through its own failure - // path, and every accepted act ends stopped, so its caller asks again (§562, A-141). - e.Stop("its stop signal") - stopAnswers() - return nil - case <-tick.C: - expire("the daily removal of points past their keep days (design 47 G section 551)") - } + every := time.Duration(expireEvery * float64(time.Hour)) + if err := expireOnBound(conn.State("bounds"), every, func() { + expire("the removal of points past their keep days on its bound of " + every.String() + " (design 47 G section 551)") + }, logger.Printf); err != nil { + return fmt.Errorf("the bound of the removal of points past their keep days: %w", err) } + <-ctx.Done() + // A move or a restart stops the process: it takes no new call, the running act ends through its own failure path, and + // every accepted act ends stopped, so its caller asks again (§562, A-141). + e.Stop("its stop signal") + stopAnswers() + return nil } diff --git a/modules/mesh-recovery/cmd/mesh-recovery/main_test.go b/modules/mesh-recovery/cmd/mesh-recovery/main_test.go index a16eede55..700a3514a 100644 --- a/modules/mesh-recovery/cmd/mesh-recovery/main_test.go +++ b/modules/mesh-recovery/cmd/mesh-recovery/main_test.go @@ -2,12 +2,15 @@ package main import ( "context" + "os" + "path/filepath" "strings" "testing" "time" "git.novox.be/novox/mesh-contracts/go/seats/meshbroker" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/process" ) @@ -71,6 +74,42 @@ func TestWhatTheProcessReads(t *testing.T) { t.Errorf("an outcome said before the wait is not taken: %+v %v", ev, ok) } }) + t.Run("every bound is a setting, a fraction allowed (§557, A-138)", func(t *testing.T) { + dir := t.TempDir() + write := func(name, v string) string { + p := filepath.Join(dir, name) + _ = os.WriteFile(p, []byte(v+"\n"), 0o600) + return p + } + if d, err := readMinutes(write("wait", "0.05"), time.Hour); err != nil || d != 3*time.Second { + t.Errorf("0.05 minutes read %s, %v", d, err) + } + if n, err := readFraction(write("days", "0.00002"), 90); err != nil || n != 0.00002 { + t.Errorf("a fraction of a day read %v, %v", n, err) + } + if n, err := readFraction(filepath.Join(dir, "none"), 24); err != nil || n != 24 { + t.Errorf("a missing setting read %v, %v", n, err) + } + if _, err := readFraction(write("zero", "0"), 24); err == nil { + t.Error("a bound of 0 was taken") + } + }) + t.Run("the expiry runs on its bound key's expiry (§551, A-77, A-138)", func(t *testing.T) { + s := &fakeBounds{} + runs := 0 + if err := expireOnBound(s, 2*time.Second, func() { runs++ }, t.Logf); err != nil { + t.Fatal(err) + } + _ = s.watch(stdio.StateChange{Key: expiryKey, Op: "put"}) + _ = s.watch(stdio.StateChange{Key: expiryKey, Op: "delete"}) + if runs != 1 || len(s.created) != 2 || s.created[1] != 2*time.Second { + t.Errorf("%d runs, keys written %v", runs, s.created) + } + s.exists = true // a key a run before this start wrote stands + if err := expireOnBound(s, time.Hour, func() {}, t.Logf); err != nil { + t.Errorf("a standing bound key is refused: %v", err) + } + }) t.Run("a dump of the bus module runs (§563)", func(t *testing.T) { d := newDumpView(nil, nil) start := time.Date(2026, 11, 12, 2, 0, 0, 0, time.UTC) @@ -89,3 +128,23 @@ func TestWhatTheProcessReads(t *testing.T) { } }) } + +// fakeBounds is the state bounds: the keys written with their time to live, and the watch's handler. +type fakeBounds struct { + created []time.Duration + exists bool + watch func(stdio.StateChange) error +} + +func (f *fakeBounds) CreateTTL(_ string, _ any, ttl time.Duration) (uint64, error) { + if f.exists { + return 0, stdio.ErrChanged + } + f.created = append(f.created, ttl) + return 1, nil +} + +func (f *fakeBounds) Watch(_ string, handle func(stdio.StateChange) error) error { + f.watch = handle + return nil +} diff --git a/modules/mesh-recovery/cmd/mesh-recovery/seats.go b/modules/mesh-recovery/cmd/mesh-recovery/seats.go index 26f0e06c1..688c596fb 100644 --- a/modules/mesh-recovery/cmd/mesh-recovery/seats.go +++ b/modules/mesh-recovery/cmd/mesh-recovery/seats.go @@ -206,9 +206,15 @@ func (v *runningView) Fingerprint(module string) (string, error) { return "", fmt.Errorf("%s runs at more than one fingerprint (%s)", module, strings.Join(said, "; ")) } -// readMinutes reads a bound in minutes from its setting's file, a fraction allowed so a whole test sets it in seconds -// (§557, A-130); the default when missing. +// readMinutes reads a bound in minutes from its setting's file (§557, A-130, A-138); the default when missing. func readMinutes(path string, fallback time.Duration) (time.Duration, error) { + n, err := readFraction(path, fallback.Minutes()) + return time.Duration(n * float64(time.Minute)), err +} + +// readFraction reads a bound's setting (minutes, hours, days), a fraction allowed so a whole test sets it in seconds +// (§557, A-138); the default when missing. +func readFraction(path string, fallback float64) (float64, error) { if path == "" { return fallback, nil } @@ -221,9 +227,9 @@ func readMinutes(path string, fallback time.Duration) (time.Duration, error) { } var n float64 if err := json.Unmarshal([]byte(strings.TrimSpace(string(raw))), &n); err != nil || n <= 0 { - return 0, fmt.Errorf("the setting in %s is %q, not a positive number of minutes", path, strings.TrimSpace(string(raw))) + return 0, fmt.Errorf("the setting in %s is %q, not a positive number", path, strings.TrimSpace(string(raw))) } - return time.Duration(n * float64(time.Minute)), nil + return n, nil } // readNumber reads a setting's file (design 47 G A-112): its whole content a positive number, the default when missing. diff --git a/modules/mesh-recovery/files/expire-every-hours b/modules/mesh-recovery/files/expire-every-hours new file mode 100644 index 000000000..68cafde19 --- /dev/null +++ b/modules/mesh-recovery/files/expire-every-hours @@ -0,0 +1 @@ +${config:expire-every-hours} diff --git a/modules/mesh-recovery/internal/recovery/manifest_test.go b/modules/mesh-recovery/internal/recovery/manifest_test.go index 7e904fa1b..185840f3d 100644 --- a/modules/mesh-recovery/internal/recovery/manifest_test.go +++ b/modules/mesh-recovery/internal/recovery/manifest_test.go @@ -70,7 +70,7 @@ func TestTheManifestSaysWhatTheModuleIs(t *testing.T) { t.Errorf("no data item %q", id) } } - for k, want := range map[string]float64{"points-cap": 10 << 30, "points-filling-mark": 80, "point-keep-days": 90, "dump-wait-minutes": 60, "snapshot-restore-wait-minutes": 90} { + for k, want := range map[string]float64{"points-cap": 10 << 30, "points-filling-mark": 80, "point-keep-days": 90, "dump-wait-minutes": 60, "snapshot-restore-wait-minutes": 90, "expire-every-hours": 24} { if m.Config[k].Default != want { t.Errorf("setting %s defaults to %v, want %v", k, m.Config[k].Default, want) } @@ -107,6 +107,7 @@ func TestTheManifestSaysWhatTheModuleIs(t *testing.T) { if m.RuntimeHealth.Kind != "tool" || m.RuntimeHealth.Tool != "recovery_kept_points" || !slices.Contains(m.Tools, m.RuntimeHealth.Tool) { t.Errorf("the tool check is %+v, want recovery_kept_points, a tool of %v", m.RuntimeHealth, m.Tools) } + bounds := false for _, raw := range m.Requires { var q struct { Name string `json:"name"` @@ -115,6 +116,11 @@ func TestTheManifestSaysWhatTheModuleIs(t *testing.T) { if json.Unmarshal(raw, &q) == nil && q.Name == "digests" && len(q.Options) != 0 { t.Errorf("digests gives the options %v beside the template state's fixed cap", q.Options) } + // The expiry's bound is a bound key of its own state, its keys living a time of their own (A-77, A-138). + bounds = bounds || q.Name == "bounds" && q.Options["key-ttl"] == true + } + if !bounds { + t.Error("no state bounds with key-ttl for the expiry's bound key") } for _, c := range []string{"node-backup.item-started", "node-backup.item-ended", "mesh-broker.snapshot-restored"} { if !slices.Contains(m.Consumes, c) { diff --git a/modules/mesh-recovery/internal/recovery/points.go b/modules/mesh-recovery/internal/recovery/points.go index b3d9f1bed..7f904c2f3 100644 --- a/modules/mesh-recovery/internal/recovery/points.go +++ b/modules/mesh-recovery/internal/recovery/points.go @@ -11,6 +11,7 @@ import ( "path/filepath" "sort" "strings" + "time" seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" "git.novox.be/novox/mesh-sdk/go/keys" @@ -209,7 +210,7 @@ func (e *Engine) expire(act string) { newest[k] = p.Snapshot } } - cutoff := e.now().AddDate(0, 0, -e.Settings.KeepDays) + cutoff := e.now().Add(-time.Duration(e.Settings.KeepDays * float64(24*time.Hour))) removed := int64(0) for _, p := range pts { if !p.At.Before(cutoff) || newest[p.Module+"\x00"+p.Kept] == p.Snapshot { diff --git a/modules/mesh-recovery/internal/recovery/recovery.go b/modules/mesh-recovery/internal/recovery/recovery.go index efd0faddf..e5ee57af2 100644 --- a/modules/mesh-recovery/internal/recovery/recovery.go +++ b/modules/mesh-recovery/internal/recovery/recovery.go @@ -99,7 +99,7 @@ type Dump interface { // Settings are the bounds a whole test sets in seconds (§557, §564, §633). type Settings struct { Cap int64 - KeepDays int + KeepDays float64 // FillingMark is the percent of the cap past which points-filling is written (points-filling-mark, 80 unless set). FillingMark int // DumpWait is how long an act waits on the dump before recovery-waits is written (dump-wait-minutes, 60 unless set). diff --git a/modules/mesh-recovery/module.json b/modules/mesh-recovery/module.json index 3f1e6e024..bbfe35474 100644 --- a/modules/mesh-recovery/module.json +++ b/modules/mesh-recovery/module.json @@ -68,6 +68,14 @@ "storage": "key-value", "template": "state" }, + { + "name": "bounds", + "storage": "key-value", + "template": "state", + "options": { + "key-ttl": true + } + }, { "name": "finding-events", "storage": "stream", @@ -102,7 +110,7 @@ "point-keep-days": { "type": "number", "description": "days after which a restore point is removed, the newest of its kept data never (design 47 G sections 551, 557).", - "minimum": 1, + "minimum": 0, "maximum": 3650, "trusted": false, "default": 90 @@ -110,7 +118,7 @@ "dump-wait-minutes": { "type": "number", "description": "minutes an act waits on the dump of the bus's item before recovery-waits is written (design 47 G sections 561, 557).", - "minimum": 1, + "minimum": 0, "maximum": 10080, "trusted": false, "default": 60 @@ -118,10 +126,18 @@ "snapshot-restore-wait-minutes": { "type": "number", "description": "minutes a restore in place of a protected keeper's snapshot is waited on, and once more after its cancel (design 47 G section 681, A-138, A-140).", - "minimum": 1, + "minimum": 0, "maximum": 10080, "trusted": false, "default": 90 + }, + "expire-every-hours": { + "type": "number", + "description": "hours between two removals of the points past their keep days, beside the one at the start and at each snapshot-kept; timed by a bound key of the state bounds (design 47 G sections 551, 557, A-138).", + "minimum": 0, + "maximum": 8760, + "trusted": false, + "default": 24 } }, "data": { @@ -202,6 +218,14 @@ "mode": "0600", "template": "files/snapshot-restore-wait-minutes" }, + { + "id": "expire-every", + "type": "file", + "path": "${dir:state}/expire-every-hours", + "owner": "mesh-recovery", + "mode": "0600", + "template": "files/expire-every-hours" + }, { "id": "recovery", "type": "process", @@ -217,7 +241,8 @@ "MESH_RECOVERY_FILLING_MARK_FILE": "${dir:state}/points-filling-mark", "MESH_RECOVERY_KEEP_DAYS_FILE": "${dir:state}/point-keep-days", "MESH_RECOVERY_DUMP_WAIT_FILE": "${dir:state}/dump-wait-minutes", - "MESH_RECOVERY_RESTORE_WAIT_FILE": "${dir:state}/snapshot-restore-wait-minutes" + "MESH_RECOVERY_RESTORE_WAIT_FILE": "${dir:state}/snapshot-restore-wait-minutes", + "MESH_RECOVERY_EXPIRE_EVERY_FILE": "${dir:state}/expire-every-hours" }, "user": "mesh-recovery", "serves": [ From bf7d040489acff68f8198cc8ca34d92ff17d9eb3 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 02:24:44 +0100 Subject: [PATCH 16/17] mesh-recovery A-143: the dump wait and the snapshot-restored wait are bound keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each wait's bound is a key of the module's state bounds with its own time to live, dump-wait. and restore-wait., its expiry heard on the key's watch, so no timer stays in the process. Builds: design 47 G A-143, A-77, A-138, §681, §561 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../mesh-recovery/cmd/mesh-recovery/expiry.go | 38 ++++++++++++++++ .../mesh-recovery/cmd/mesh-recovery/main.go | 1 + .../cmd/mesh-recovery/main_test.go | 31 +++++++++++++ .../internal/recovery/fakes_test.go | 38 +++++++++++++++- .../internal/recovery/findings.go | 25 +++++++---- .../mesh-recovery/internal/recovery/keeper.go | 43 ++++++++++++++++--- .../internal/recovery/keeper_test.go | 20 ++++++--- .../internal/recovery/recovery.go | 18 +++++++- .../internal/recovery/restore_test.go | 4 ++ 9 files changed, 194 insertions(+), 24 deletions(-) diff --git a/modules/mesh-recovery/cmd/mesh-recovery/expiry.go b/modules/mesh-recovery/cmd/mesh-recovery/expiry.go index 14b4ad65d..9dfb5b96a 100644 --- a/modules/mesh-recovery/cmd/mesh-recovery/expiry.go +++ b/modules/mesh-recovery/cmd/mesh-recovery/expiry.go @@ -2,9 +2,12 @@ package main import ( "errors" + "sync/atomic" "time" stdio "git.novox.be/novox/mesh-sdk/go" + + "mesh-recovery/internal/recovery" ) // boundState is the module's own state bounds, whose keys live a time of their own (key-ttl): a bound key's expiry is @@ -12,8 +15,43 @@ import ( type boundState interface { CreateTTL(key string, value any, ttl time.Duration) (uint64, error) Watch(key string, handle func(stdio.StateChange) error) error + Delete(key string) error } +// errBoundEnded ends a bound key's watch once the key is gone. +var errBoundEnded = errors.New("the bound key is removed") + +// stateBounds times an act's waits by bound keys of the state bounds (A-77, A-143): the key is watched before it is +// written, so its expiry is never missed, and the watch ends with the key's removal. +type stateBounds struct{ s boundState } + +func (b stateBounds) Bound(key string, ttl time.Duration, expired func()) (func(), error) { + var ended atomic.Bool + if err := b.s.Watch(key, func(c stdio.StateChange) error { + if c.Op == "put" || c.Current { + return nil + } + if !ended.Swap(true) { + expired() + } + return errBoundEnded + }); err != nil { + return nil, err + } + if _, err := b.s.CreateTTL(key, map[string]any{"ttl": ttl.String()}, ttl); err != nil { + ended.Store(true) + _ = b.s.Delete(key) + return nil, err + } + return func() { + if !ended.Swap(true) { + _ = b.s.Delete(key) + } + }, nil +} + +var _ recovery.Bounds = stateBounds{} + // expiryKey is the bound key of §551's daily removal. const expiryKey = "expiry" diff --git a/modules/mesh-recovery/cmd/mesh-recovery/main.go b/modules/mesh-recovery/cmd/mesh-recovery/main.go index 6d374b667..3c50f938c 100644 --- a/modules/mesh-recovery/cmd/mesh-recovery/main.go +++ b/modules/mesh-recovery/cmd/mesh-recovery/main.go @@ -159,6 +159,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Digests: digests{conn.State("digests")}, Dump: dump, Keeper: liveKeeper{nc: nc, call: conn.Call, restored: restored}, + Bounds: stateBounds{conn.State("bounds")}, Settings: recovery.Settings{Cap: cap, KeepDays: keepDays, FillingMark: int(mark), DumpWait: dumpWait, RestoreWait: restoreWait}, Findings: writer, Node: account.Node, diff --git a/modules/mesh-recovery/cmd/mesh-recovery/main_test.go b/modules/mesh-recovery/cmd/mesh-recovery/main_test.go index 700a3514a..e0e4482d8 100644 --- a/modules/mesh-recovery/cmd/mesh-recovery/main_test.go +++ b/modules/mesh-recovery/cmd/mesh-recovery/main_test.go @@ -110,6 +110,31 @@ func TestWhatTheProcessReads(t *testing.T) { t.Errorf("a standing bound key is refused: %v", err) } }) + t.Run("an act's wait is a bound key, its expiry heard on its watch (A-143)", func(t *testing.T) { + s := &fakeBounds{} + expired := 0 + end, err := stateBounds{s}.Bound("dump-wait.act-1", 3*time.Second, func() { expired++ }) + if err != nil || len(s.created) != 1 || s.created[0] != 3*time.Second { + t.Fatalf("%v, keys written %v", err, s.created) + } + if s.watch(stdio.StateChange{Key: "dump-wait.act-1", Op: "put"}) != nil || expired != 0 { + t.Error("the key's own write ended the wait") + } + if s.watch(stdio.StateChange{Key: "dump-wait.act-1", Op: "delete"}) == nil || expired != 1 { + t.Errorf("the key's expiry: %d calls, its watch not ended", expired) + } + end() + if expired != 1 || len(s.deleted) != 0 { + t.Errorf("an expired key's end: %d calls, deleted %v", expired, s.deleted) + } + s = &fakeBounds{} + end, _ = stateBounds{s}.Bound("restore-wait.act-2", time.Minute, func() { expired++ }) + end() + if s.watch(stdio.StateChange{Key: "restore-wait.act-2", Op: "delete"}) == nil || expired != 1 || + strings.Join(s.deleted, " ") != "restore-wait.act-2" { + t.Errorf("an ended key's removal: %d calls, deleted %v", expired, s.deleted) + } + }) t.Run("a dump of the bus module runs (§563)", func(t *testing.T) { d := newDumpView(nil, nil) start := time.Date(2026, 11, 12, 2, 0, 0, 0, time.UTC) @@ -134,6 +159,7 @@ type fakeBounds struct { created []time.Duration exists bool watch func(stdio.StateChange) error + deleted []string } func (f *fakeBounds) CreateTTL(_ string, _ any, ttl time.Duration) (uint64, error) { @@ -148,3 +174,8 @@ func (f *fakeBounds) Watch(_ string, handle func(stdio.StateChange) error) error f.watch = handle return nil } + +func (f *fakeBounds) Delete(key string) error { + f.deleted = append(f.deleted, key) + return nil +} diff --git a/modules/mesh-recovery/internal/recovery/fakes_test.go b/modules/mesh-recovery/internal/recovery/fakes_test.go index 38fca691b..254956aa7 100644 --- a/modules/mesh-recovery/internal/recovery/fakes_test.go +++ b/modules/mesh-recovery/internal/recovery/fakes_test.go @@ -6,6 +6,7 @@ import ( "io" "os" "path/filepath" + "strings" "sync" "testing" "time" @@ -192,14 +193,16 @@ type rig struct { s *fakeSeats d *memDigests t time.Time + // bounds is the state bounds the act's waits are timed by (A-143). + bounds *fakeBounds } func newRig(t *testing.T) *rig { t.Helper() - r := &rig{b: &fakeBus{streams: map[string]string{"app_notes": "hello", "KV_app_seen": "world"}, consumers: map[string][]seat.Consumer{}}, + r := &rig{bounds: &fakeBounds{}, b: &fakeBus{streams: map[string]string{"app_notes": "hello", "KV_app_seen": "world"}, consumers: map[string][]seat.Consumer{}}, s: &fakeSeats{streams: []string{"app_notes", "KV_app_seen"}}, d: &memDigests{m: map[string][]byte{}}, t: time.Date(2026, 11, 12, 1, 0, 0, 0, time.UTC)} r.b.consumers["app_notes"] = []seat.Consumer{{Name: "c1", Stream: "app_notes", Sequence: 7}} - r.e = &recovery.Engine{Dir: filepath.Join(t.TempDir(), "points-dir"), Bus: r.b, Seats: r.s, Digests: r.d, Dump: noDump{}, + r.e = &recovery.Engine{Dir: filepath.Join(t.TempDir(), "points-dir"), Bus: r.b, Seats: r.s, Digests: r.d, Dump: noDump{}, Bounds: r.bounds, Settings: recovery.Settings{Cap: 1 << 20, KeepDays: 90}, Now: func() time.Time { return r.t }} return r } @@ -266,3 +269,34 @@ func (h *heldFindings) has(kind string) bool { } return false } + +// fakeBounds is the state bounds: each key removed by the bus when its time to live passes, as the bus does, or by its +// end; written and removed say which keys, with each key's time to live (A-143). +type fakeBounds struct { + mu sync.Mutex + written []string + removed []string +} + +func (f *fakeBounds) Bound(key string, ttl time.Duration, expired func()) (func(), error) { + f.mu.Lock() + f.written = append(f.written, key+" "+ttl.String()) + f.mu.Unlock() + var once sync.Once + gone := func(by string, then func()) { + once.Do(func() { + f.mu.Lock() + f.removed = append(f.removed, key+" "+by) + f.mu.Unlock() + then() + }) + } + t := time.AfterFunc(ttl, func() { gone("expired", expired) }) + return func() { t.Stop(); gone("ended", func() {}) }, nil +} + +func (f *fakeBounds) keys() (written, removed string) { + f.mu.Lock() + defer f.mu.Unlock() + return strings.Join(f.written, "|"), strings.Join(f.removed, "|") +} diff --git a/modules/mesh-recovery/internal/recovery/findings.go b/modules/mesh-recovery/internal/recovery/findings.go index 09fa7310a..ca4be0c11 100644 --- a/modules/mesh-recovery/internal/recovery/findings.go +++ b/modules/mesh-recovery/internal/recovery/findings.go @@ -124,25 +124,32 @@ func (e *Engine) largest(n int) []string { return out } -// waitsOn is the waiting of one act on the dump: past the bound it writes recovery-waits, naming the item and since when; -// done clears it once the act starts its work or ends (§561, §563). +// waitsOn is the waiting of one act on the dump: past the bound, the key `dump-wait.` of the state bounds expiring +// (A-143), it writes recovery-waits, naming the item and since when; done removes the key and clears the finding once the +// act starts its work or ends (§561, §563). func (e *Engine) waitsOn(a *Act) (waiting func(on string), done func()) { - var timer *time.Timer + var end func() written := make(chan bool, 1) written <- false waiting = func(on string) { e.setStep(a, "waiting", on) since := e.now() - timer = time.AfterFunc(e.dumpWait(), func() { - <-written - e.write(e.finding(FindingRecoveryWaits, a.Module, a.ID, "A restore waits on the dump of the bus", - fmt.Sprintf("the act %s waits on %s since %s", a.ID, on, since.Format(time.RFC3339)), "the dump to end, or recovery_cancel")) + key := BoundDumpWait + "." + a.ID + var err error + end, err = e.bound(key, e.dumpWait(), func() { + if !<-written { + e.write(e.finding(FindingRecoveryWaits, a.Module, a.ID, "A restore waits on the dump of the bus", + fmt.Sprintf("the act %s waits on %s since %s", a.ID, on, since.Format(time.RFC3339)), "the dump to end, or recovery_cancel")) + } written <- true }) + if err != nil { + e.logf("the bound %s of the act's wait on the dump is not written, so recovery-waits is never written for it: %v", key, err) + } } done = func() { - if timer != nil { - timer.Stop() + if end != nil { + end() } if <-written { e.clear(a.Module, a.ID, FindingRecoveryWaits, "the act no longer waits on the dump") diff --git a/modules/mesh-recovery/internal/recovery/keeper.go b/modules/mesh-recovery/internal/recovery/keeper.go index 5c8167caa..38dde2e82 100644 --- a/modules/mesh-recovery/internal/recovery/keeper.go +++ b/modules/mesh-recovery/internal/recovery/keeper.go @@ -144,24 +144,39 @@ func (e *Engine) restoreInPlace(ctx context.Context, a *Act) error { return e.endedRestored(a, why, asidePtr(rec), false) } e.setStep(a, "restoring in place", "the keeper's snapshot-restored") + // The wait's bound is the key `restore-wait.` of the state bounds, written before the keeper is asked, so a bound + // that cannot be written asks nothing of the keeper (A-143). + wctx, passed, err := e.restoreBound(ctx, a.ID) + if err != nil { + return putBack(err) + } r, err := e.Keeper.Restore(ctx, a.Request, false, a.Why) if err == nil { err = keeperAnswer(r) } if err != nil { + passed() return putBack(err) } act := r.Accepted - ev, ok := e.restoredWithin(ctx, a.Request, act) + ev, ok := e.Keeper.Restored(wctx, a.Request, act) + passed() if !ok { e.setStep(a, "cancelling the keeper's restore", "the keeper's snapshot-restored after the cancel") why := fmt.Sprintf("the recovery module's wait of %s on the restore in place passed (act %s)", e.restoreWait(), a.ID) if c, err := e.Keeper.CancelRestore(context.Background(), act, why); err != nil || c.Error != "" { e.logf("the cancel of the keeper's restore %s is not accepted (%v%s); its outcome is waited on once more", act, err, c.Error) } - if ev, ok = e.restoredWithin(context.Background(), a.Request, act); !ok { - return e.endedRestored(a, errors.New(ErrUnanswered), asidePtr(rec), false) + wctx, passed, err := e.restoreBound(context.Background(), a.ID) + if err != nil { + e.logf("the bound of the wait after the cancel of %s is not written: %v", act, err) + return e.endedRestored(a, fmt.Errorf("%s: %v", ErrUnanswered, err), asidePtr(rec), false) } + ev, ok = e.Keeper.Restored(wctx, a.Request, act) + passed() + } + if !ok { + return e.endedRestored(a, errors.New(ErrUnanswered), asidePtr(rec), false) } if ev.Outcome != meshbroker.SnapshotRestoredEventOutcomeDone { return putBack(fmt.Errorf("the keeper's restore in place failed: %s", ev.Error)) @@ -171,10 +186,24 @@ func (e *Engine) restoreInPlace(ctx context.Context, a *Act) error { return e.endedRestored(a, nil, asidePtr(rec), false) } -func (e *Engine) restoredWithin(ctx context.Context, request, act string) (meshbroker.SnapshotRestoredEvent, bool) { - wctx, cancel := context.WithTimeout(ctx, e.restoreWait()) - defer cancel() - return e.Keeper.Restored(wctx, request, act) +// restoreBound is the bound of one wait on snapshot-restored (§681, A-138, A-143): the key `restore-wait.` with the time +// to live snapshot-restore-wait-minutes; its expiry ends the context it answers. end removes the key. +func (e *Engine) restoreBound(ctx context.Context, act string) (context.Context, func(), error) { + wctx, cancel := context.WithCancel(ctx) + end, err := e.bound(BoundRestoreWait+"."+act, e.restoreWait(), cancel) + if err != nil { + cancel() + return nil, nil, err + } + return wctx, func() { end(); cancel() }, nil +} + +// bound writes a bound key in the state bounds (A-77, A-143). +func (e *Engine) bound(key string, ttl time.Duration, expired func()) (func(), error) { + if e.Bounds == nil { + return nil, fmt.Errorf("the bound %s is not written: the engine has no state bounds", key) + } + return e.Bounds.Bound(key, ttl, expired) } // keeperAnswer reads snapshot-restore's one form (§691): accepted, or an error naming the keeper's refusal or its running act. diff --git a/modules/mesh-recovery/internal/recovery/keeper_test.go b/modules/mesh-recovery/internal/recovery/keeper_test.go index 5084e2b76..f816966ab 100644 --- a/modules/mesh-recovery/internal/recovery/keeper_test.go +++ b/modules/mesh-recovery/internal/recovery/keeper_test.go @@ -174,12 +174,13 @@ func TestAProtectedSnapshotIsRestoredInPlaceByTheKeeper(t *testing.T) { asked string live string unrestored int + bounds string // how each bound key restore-wait. was removed (A-143) }{ - {inPlace: "done", asked: "get req-1|verify req-1|restore req-1", live: "as dropped"}, - {inPlace: "failed", asked: "get req-1|verify req-1|restore req-1", live: "hello", err: "the keeper's restore in place failed"}, - {inPlace: "silent", asked: "get req-1|verify req-1|restore req-1|cancel keeper-act-1", live: "hello", err: "the keeper's restore in place failed: cancelled"}, - {inPlace: "deaf", asked: "get req-1|verify req-1|restore req-1|cancel keeper-act-1", err: recovery.ErrUnanswered, unrestored: 1}, - {inPlace: "done", busy: "keeper-act-0", asked: "get req-1|verify req-1|restore req-1", live: "hello", err: "busy: the keeper restores its act keeper-act-0"}, + {inPlace: "done", asked: "get req-1|verify req-1|restore req-1", live: "as dropped", bounds: "ended"}, + {inPlace: "failed", asked: "get req-1|verify req-1|restore req-1", live: "hello", err: "the keeper's restore in place failed", bounds: "ended"}, + {inPlace: "silent", asked: "get req-1|verify req-1|restore req-1|cancel keeper-act-1", live: "hello", err: "the keeper's restore in place failed: cancelled", bounds: "expired|ended"}, + {inPlace: "deaf", asked: "get req-1|verify req-1|restore req-1|cancel keeper-act-1", err: recovery.ErrUnanswered, unrestored: 1, bounds: "expired|expired"}, + {inPlace: "done", busy: "keeper-act-0", asked: "get req-1|verify req-1|restore req-1", live: "hello", err: "busy: the keeper restores its act keeper-act-0", bounds: "ended"}, } { t.Run(tc.inPlace+tc.busy, func(t *testing.T) { r, k := requestRig(t) @@ -201,6 +202,15 @@ func TestAProtectedSnapshotIsRestoredInPlaceByTheKeeper(t *testing.T) { if n := len(r.e.Unrestored()); n != tc.unrestored { t.Errorf("%d unrestored asides, want %d", n, tc.unrestored) } + w, rm := r.bounds.keys() + var wantW, wantR []string + for _, by := range strings.Split(tc.bounds, "|") { + wantW = append(wantW, "restore-wait."+ev.Act+" 20ms") + wantR = append(wantR, "restore-wait."+ev.Act+" "+by) + } + if w != strings.Join(wantW, "|") || rm != strings.Join(wantR, "|") { + t.Errorf("bound keys written %q, removed %q; want %q, %q", w, rm, strings.Join(wantW, "|"), strings.Join(wantR, "|")) + } }) } } diff --git a/modules/mesh-recovery/internal/recovery/recovery.go b/modules/mesh-recovery/internal/recovery/recovery.go index e5ee57af2..eeae6ea63 100644 --- a/modules/mesh-recovery/internal/recovery/recovery.go +++ b/modules/mesh-recovery/internal/recovery/recovery.go @@ -96,6 +96,20 @@ type Dump interface { Wait(ctx context.Context, waiting func(on string)) error } +// Bounds is the module's state bounds (design 47 G A-77, A-143): a wait's bound is a key with its own time to live, its +// expiry heard on the key's watch as the bus's own change, so no timer runs in the process. +type Bounds interface { + // Bound writes key with the time to live ttl and calls expired once when the bus removes it; end removes it first, and + // expired is then never called. + Bound(key string, ttl time.Duration, expired func()) (end func(), err error) +} + +// The bound keys of an act's waits (A-143): `.`. +const ( + BoundDumpWait = "dump-wait" + BoundRestoreWait = "restore-wait" +) + // Settings are the bounds a whole test sets in seconds (§557, §564, §633). type Settings struct { Cap int64 @@ -117,7 +131,9 @@ type Engine struct { Digests Digests Dump Dump // Keeper is the bus module's keeper, whose snapshots the hand verb restore takes by request (§681). - Keeper Keeper + Keeper Keeper + // Bounds times the dump wait and the snapshot-restored wait (A-143). + Bounds Bounds Settings Settings // Findings is the module's finding writer (§561); Node the machine it runs on, the id of points-filling. Findings Findings diff --git a/modules/mesh-recovery/internal/recovery/restore_test.go b/modules/mesh-recovery/internal/recovery/restore_test.go index cc168ee6c..b63c44fe7 100644 --- a/modules/mesh-recovery/internal/recovery/restore_test.go +++ b/modules/mesh-recovery/internal/recovery/restore_test.go @@ -164,6 +164,10 @@ func TestAWaitingActIsCancelledAndABegunOneRefused(t *testing.T) { if fs.has(recovery.FindingRecoveryWaits) { t.Errorf("recovery-waits stands after the act ended: %+v", fs.Held()) } + // The wait's bound is a key of the state bounds, its expiry what wrote recovery-waits (A-143). + if w, rm := r.bounds.keys(); w != "dump-wait."+ans.Accepted+" 1ms" || rm != "dump-wait."+ans.Accepted+" expired" { + t.Errorf("bound keys written %q, removed %q", w, rm) + } } // retry asks again the last failed act with its fields under an act of its own, and answers `nothing to retry` when its From 6c2d6605a55469bfc8fcb5c03588b310deb2605e Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 02:36:19 +0100 Subject: [PATCH 17/17] mesh-recovery: its one whole test against the bus, and an act's end said once it is free MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The whole test runs the process on its own account against a bus of its own, every bound a setting of seconds, with a stand-in bus module for snapshot-get, snapshot-restore and snapshot-restored: a hand take's log, a restore waiting on the dump past its bound key, busy and the ask again, the expiry on its bound key, a keeper's snapshot in three chunks, a protected restore cancelled at its bound and its aside put back, one the keeper answers neither, and a stop putting the aside back. It found that an act said its ending event while it was still the running one, so a caller told busy that asked again on hearing it was told busy with the same act and heard nothing more: the act is now released and written ended before its ending event, and the removal at a snapshot-kept runs before its ended. Builds: design 47 G §557, §554, §681, §562, §551, A-138, A-140, A-141, A-143 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../mesh-recovery/whole_integration_test.go | 473 ++++++++++++++++++ .../mesh-recovery/internal/recovery/points.go | 33 +- .../internal/recovery/recovery.go | 4 +- .../internal/recovery/restore.go | 6 +- 4 files changed, 505 insertions(+), 11 deletions(-) create mode 100644 modules/mesh-recovery/cmd/mesh-recovery/whole_integration_test.go diff --git a/modules/mesh-recovery/cmd/mesh-recovery/whole_integration_test.go b/modules/mesh-recovery/cmd/mesh-recovery/whole_integration_test.go new file mode 100644 index 000000000..5cbaf0e31 --- /dev/null +++ b/modules/mesh-recovery/cmd/mesh-recovery/whole_integration_test.go @@ -0,0 +1,473 @@ +//go:build integration + +package main + +// The recovery module's one whole test (design 47 G §557, §554, §681, §562, A-138, A-143): its run() as the node starts it, +// on its own account on a bus of this test alone, every bound a setting of a few seconds. A stand-in bus module answers +// the seat mesh-broker's snapshot-get (in chunks), snapshot-restore and snapshot-restore-cancel and says snapshot-restored; +// stand-ins answer the hand-act log, the assigner's assignments and the catalog's entry; the node-engine seat's running and +// the backup holder's item events are said on their streams. The live data is a real bucket, set aside and restored by +// the bus's own snapshots. + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-contracts/go/seats/meshbroker" + seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + "git.novox.be/novox/mesh-sdk/go/buscalls" + "git.novox.be/novox/mesh-sdk/go/bussnap" + "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/testbus" + + "mesh-recovery/internal/recovery" +) + +// standKeeper is the bus module's keeper as the test plays it: a snapshot by request, protected or not, and what it says +// after a restore in place and after its cancel ("" says nothing). +type standKeeper struct { + mu sync.Mutex + file []byte // the snapshot an unprotected request sends + protected bool + onRestore string + onCancel string + asked []string +} + +func (k *standKeeper) set(protected bool, onRestore, onCancel string) { + k.mu.Lock() + k.protected, k.onRestore, k.onCancel, k.asked = protected, onRestore, onCancel, nil + k.mu.Unlock() +} + +func (k *standKeeper) said() string { + k.mu.Lock() + defer k.mu.Unlock() + return strings.Join(k.asked, "|") +} + +func TestRecoveryWhole(t *testing.T) { + url := testbus.URL(t) + nc, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + js, _ := jetstream.New(nc) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + acct := process.Account{URL: url, Node: "srv", Module: "mesh-recovery", User: "srv.mesh-recovery", Password: "unused"} + + // The module's own states (bounds takes keys with their own time to live), the app's live kept data, and the streams the + // grants part and the bus composer make: the seat events it hears with its durables, its own events and its findings. + for _, b := range []jetstream.KeyValueConfig{{Bucket: "mesh-recovery_bounds", LimitMarkerTTL: time.Second}, + {Bucket: "mesh-recovery_digests"}, {Bucket: "mesh-recovery_findings"}, {Bucket: "app_notes"}} { + if _, err := js.CreateKeyValue(ctx, b); err != nil { + t.Fatal(err) + } + } + for _, s := range []jetstream.StreamConfig{ + {Name: "RECOVERY_EVENTS", Subjects: []string{"mesh.seat.mesh-recovery.event.>"}}, + {Name: "RECOVERY_FINDINGS", Subjects: []string{"mesh.stream.mesh-recovery.event.>"}}, + {Name: "FINDINGS_ASKED", Subjects: []string{meshconditions.SubjectFindingsAsked}}, + } { + if _, err := js.CreateStream(ctx, s); err != nil { + t.Fatal(err) + } + } + for stream, subject := range map[string]string{ + nodeengine.StreamRunning: "mesh.seat.node-engine.event.running.>", + itemStartedSteam: "mesh.seat.node-backup.event.item-started.>", + itemEndedStream: "mesh.seat.node-backup.event.item-ended.>", + meshbroker.StreamSnapshotRestored: "mesh.seat.mesh-broker.event.snapshot-restored.>", + } { + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{subject}, MaxMsgsPerSubject: 1}); err != nil { + t.Fatal(err) + } + if _, err := js.CreateOrUpdateConsumer(ctx, stream, acct.SeatEventConsumerConfig([]string{subject}, true)); err != nil { + t.Fatal(err) + } + } + notes, _ := js.KeyValue(ctx, "app_notes") + live := func() string { + e, err := notes.Get(ctx, "n") + if err != nil { + return "none: " + err.Error() + } + return string(e.Value()) + } + if _, err := notes.Put(ctx, "n", []byte("v1")); err != nil { + t.Fatal(err) + } + running, _ := json.Marshal(nodeengine.Running{V: 1, Node: "srv", Module: "app", Fingerprint: "src2:app1", At: time.Now()}) + if _, err := js.Publish(ctx, nodeengine.SubjectRunning("srv", "app"), running); err != nil { + t.Fatal(err) + } + + // The stand-ins of the seats the module calls. + answer := func(m *nats.Msg, result any) { + b, _ := json.Marshal(map[string]any{"result": result, "node": "srv"}) + _ = m.Respond(b) + } + var logMu sync.Mutex + var handActs []string + sub := func(subject string, h func(*nats.Msg)) { + if _, err := nc.Subscribe(subject, h); err != nil { + t.Fatal(err) + } + } + sub("mesh.seat.mesh-hand-acts.call.>", func(m *nats.Msg) { + logMu.Lock() + handActs = append(handActs, string(m.Data)) + logMu.Unlock() + answer(m, map[string]any{"recorded": true}) + }) + sub("mesh.seat.mesh-module-assigner.call.assignments.*", func(m *nats.Msg) { + answer(m, map[string]any{"seats": []any{map[string]any{"seat": meshbroker.Seat, "holders": []any{map[string]any{"module": "nats"}}}}}) + }) + sub("mesh.seat.mesh-catalog.call.entry.*", func(m *nats.Msg) { + answer(m, map[string]any{"entry": map[string]any{"module": "app", "fingerprint": "src2:app1", + "manifest": map[string]any{"module": "app", "version": "1", "state": []any{map[string]any{"name": "notes"}}}}}) + }) + keeper := &standKeeper{} + restored := func(request, act, outcome string) { + b, _ := json.Marshal(meshbroker.SnapshotRestoredEvent{Request: request, Act: act, Outcome: outcome, Error: "cancelled"}) + if _, err := js.Publish(ctx, meshbroker.SubjectSnapshotRestored(request, act), b); err != nil { + t.Error(err) + } + } + sub("mesh.seat.mesh-broker.call.snapshot-get.*", func(m *nats.Msg) { + var in meshbroker.SnapshotGetInput + _ = json.Unmarshal(m.Data, &in) + keeper.mu.Lock() + keeper.asked = append(keeper.asked, "get "+in.Request) + file, protected := keeper.file, keeper.protected + keeper.mu.Unlock() + if protected { + answer(m, meshbroker.SnapshotGetAnswer{Request: in.Request, Error: "snapshot protected"}) + return + } + sum := sha256.Sum256(file) + answer(m, meshbroker.SnapshotGetAnswer{Request: in.Request, Module: "app", Local: "notes", Stream: "KV_app_notes", + Size: int64(len(file)), Digest: "sha256:" + hex.EncodeToString(sum[:]), Chunks: 3}) + // Three chunks, each sent once the one before was acknowledged (§681). + go func() { + send := buscalls.SeatChunks{Seat: meshbroker.Seat, Verb: meshbroker.VerbSnapshotGet, Call: in.Call} + third := (len(file) + 2) / 3 + for n := int64(1); n <= 3; n++ { + acks, _ := nc.SubscribeSync(send.Ack(n)) + part := file[min(int(n-1)*third, len(file)):min(int(n)*third, len(file))] + body, _ := json.Marshal(buscalls.SeatChunk{N: n, Of: 3, Data: part}) + _ = nc.Publish(send.Chunk(n), body) + ack, err := acks.NextMsg(10 * time.Second) + _ = acks.Unsubscribe() + if err != nil { + t.Errorf("chunk %d not acknowledged: %v", n, err) + return + } + _ = ack.Respond([]byte("{}")) + } + }() + }) + sub("mesh.seat.mesh-broker.call.snapshot-restore.*", func(m *nats.Msg) { + var in meshbroker.SnapshotRestoreInput + _ = json.Unmarshal(m.Data, &in) + keeper.mu.Lock() + if in.Verify { + keeper.asked = append(keeper.asked, "verify "+in.Request) + keeper.mu.Unlock() + answer(m, meshbroker.SnapshotRestoreAnswer{Accepted: "keeper-verify"}) + return + } + keeper.asked = append(keeper.asked, "restore "+in.Request) + then := keeper.onRestore + keeper.mu.Unlock() + answer(m, meshbroker.SnapshotRestoreAnswer{Accepted: "keeper-" + in.Request}) + if then != "" { + restored(in.Request, "keeper-"+in.Request, then) + } + }) + sub("mesh.seat.mesh-broker.call.snapshot-restore-cancel.*", func(m *nats.Msg) { + var in meshbroker.SnapshotRestoreCancelInput + _ = json.Unmarshal(m.Data, &in) + keeper.mu.Lock() + keeper.asked = append(keeper.asked, "cancel "+in.Act) + then := keeper.onCancel + keeper.mu.Unlock() + answer(m, meshbroker.SnapshotRestoreCancelAnswer{Accepted: in.Act}) + if then != "" { + restored(strings.TrimPrefix(in.Act, "keeper-"), in.Act, then) + } + }) + + // Every event the module says, heard as it is said. + type heard struct { + subject string + body map[string]any + } + events := make(chan heard, 256) + sub("mesh.seat.mesh-recovery.event.>", func(m *nats.Msg) { + var b map[string]any + _ = json.Unmarshal(m.Data, &b) + events <- heard{m.Subject, b} + }) + var kept []heard + waitEvent := func(t *testing.T, what string, ok func(heard) bool) map[string]any { + t.Helper() + for i, h := range kept { + if ok(h) { + kept = append(kept[:i:i], kept[i+1:]...) + return h.body + } + } + deadline := time.After(30 * time.Second) + for { + select { + case h := <-events: + if ok(h) { + return h.body + } + kept = append(kept, h) + case <-deadline: + t.Fatalf("not said within 30s: %s", what) + } + } + } + endOf := func(t *testing.T, event, act string) map[string]any { + t.Helper() + return waitEvent(t, event+" of "+act, func(h heard) bool { + return strings.HasPrefix(h.subject, "mesh.seat.mesh-recovery.event."+event+".") && h.body["act"] == act + }) + } + ask := func(t *testing.T, verb, caller string, body any) seat.SnapshotKeptAnswer { + t.Helper() + data, _ := json.Marshal(body) + m, err := nc.Request("mesh.seat.mesh-recovery.call."+verb+"."+strings.ReplaceAll(caller, ".", "~"), data, 10*time.Second) + if err != nil { + t.Fatalf("%s: %v", verb, err) + } + var r struct { + Result seat.SnapshotKeptAnswer `json:"result"` + Error string `json:"error"` + } + _ = json.Unmarshal(m.Data, &r) + if r.Error != "" { + r.Result.Error = r.Error + } + return r.Result + } + // A caller told busy asks again when it hears the ending event of the act it was named (§554, §560): the expiry runs + // on its own bound beside the test's acts. + act := func(t *testing.T, verb, caller string, body any) string { + t.Helper() + for { + a := ask(t, verb, caller, body) + if a.Busy == "" { + if a.Accepted == "" { + t.Fatalf("%s answered %+v", verb, a) + } + return a.Accepted + } + waitEvent(t, "the ending of "+a.Busy, func(h heard) bool { + return h.body["act"] == a.Busy && !strings.Contains(h.subject, ".restore-started.") + }) + } + } + const engine, runner = "srv.node-engine", "srv.tool-runner" + + // The process, every bound a setting in seconds (§557, A-138): the dump wait and the restore wait 1.8 s, the expiry + // every 1.8 s, a point kept about 4 s. + dir := t.TempDir() + setting := func(name, v string) string { + p := filepath.Join(dir, name) + _ = os.WriteFile(p, []byte(v+"\n"), 0o600) + return p + } + acctFile, _ := json.Marshal(map[string]any{"url": url, "node": acct.Node, "module": acct.Module, "user": acct.User, "password": acct.Password}) + env := map[string]string{ + process.BrokerFileEnv: setting("broker", string(acctFile)), + envPoints: filepath.Join(dir, "points"), + envDumpWait: setting("dump-wait-minutes", "0.03"), + envRestore: setting("snapshot-restore-wait-minutes", "0.03"), + envExpire: setting("expire-every-hours", "0.0005"), + envKeepDays: setting("point-keep-days", "0.00005"), + } + runCtx, stop := context.WithCancel(ctx) + done := make(chan error, 1) + go func() { done <- run(runCtx, func(k string) string { return env[k] }, testLog{t}) }() + defer func() { stop(); <-done }() + waitEvent(t, "the start's removal of points past their keep days", func(h heard) bool { + return h.subject == "mesh.seat.mesh-recovery.event."+seat.EventExpiryEnded + }) + + var first string + t.Run("a hand take's log names the account the bus proved and the asker (§554)", func(t *testing.T) { + ev := endOf(t, seat.EventEnded, act(t, seat.VerbTake, runner, seat.TakeInput{Module: "app", Kept: "notes", Fingerprint: "src2:app1", Why: "before a test", Asker: "op@srv"})) + if ev["outcome"] != "done" { + t.Fatalf("ended %v", ev) + } + first, _ = ev["snapshot"].(string) + logMu.Lock() + defer logMu.Unlock() + if len(handActs) == 0 || !strings.Contains(handActs[len(handActs)-1], runner) || !strings.Contains(handActs[len(handActs)-1], "op@srv") { + t.Errorf("the hand-act log holds %v", handActs) + } + }) + + t.Run("a restore waits on the dump past its bound key, a second ask is busy and asks again on its ended (§554, §563, A-143)", func(t *testing.T) { + if a := ask(t, seat.VerbRestoreKept, engine, map[string]any{"module": "app", "kept": "notes", "snapshot": first, "why": "w", "aside-only": true}); !strings.Contains(a.Error, "aside-only") { + t.Errorf("a node-engine's restore-kept with aside-only answered %+v", a) + } + item := func(event string) { + b, _ := json.Marshal(map[string]any{"at": time.Now()}) + if _, err := js.Publish(ctx, itemSubject(event, "srv", "nats", "data"), b); err != nil { + t.Fatal(err) + } + } + item(itemStarted) + _, _ = notes.Put(ctx, "n", []byte("changed")) + r := seat.SnapshotKeptAnswer{Accepted: act(t, seat.VerbRestoreKept, engine, seat.RestoreKeptInput{Module: "app", Kept: "notes", Form: 2, Fingerprint: "src2:app1", Snapshot: first, Why: "a form change"})} + if b := ask(t, seat.VerbSnapshotKept, engine, seat.SnapshotKeptInput{Module: "app", Kept: "notes", Form: 2, Fingerprint: "src2:app1", Why: "x"}); b.Busy != r.Accepted { + t.Errorf("a second ask answered %+v, want busy with %s", b, r.Accepted) + } + findings, _ := js.KeyValue(ctx, "mesh-recovery_findings") + w, _ := findings.Watch(ctx, "app."+r.Accepted+"."+recovery.FindingRecoveryWaits) + for e := range w.Updates() { + if e != nil && e.Operation() == jetstream.KeyValuePut { + break + } + } + _ = w.Stop() + item(itemEnded) + if ev := endOf(t, seat.EventRestored, r.Accepted); ev["outcome"] != "done" || live() != "v1" { + t.Fatalf("restored %v, live %q", ev, live()) + } + if e, err := findings.Get(ctx, "app."+r.Accepted+"."+recovery.FindingRecoveryWaits); err == nil { + t.Errorf("recovery-waits stands after the act ended: %s", e.Value()) + } + again := act(t, seat.VerbSnapshotKept, engine, seat.SnapshotKeptInput{Module: "app", Kept: "notes", Form: 2, Fingerprint: "src2:app1", Why: "asked again"}) + if ev := endOf(t, seat.EventEnded, again); ev["outcome"] != "done" { + t.Errorf("the ask again ended %v", ev) + } + }) + + t.Run("a point past its keep days is removed on the expiry's bound key, never the newest (§551, A-138)", func(t *testing.T) { + ev := waitEvent(t, "point-removed of "+first, func(h heard) bool { + return strings.HasPrefix(h.subject, "mesh.seat.mesh-recovery.event."+seat.EventPointRemoved+".") && h.body["snapshot"] == first + }) + if ev["module"] != "app" { + t.Errorf("point-removed %v", ev) + } + var kp seat.KeptPointsAnswer + m, _ := nc.Request("mesh.seat.mesh-recovery.call.kept-points."+strings.ReplaceAll(runner, ".", "~"), []byte(`{"module":"app","kept":"notes"}`), 10*time.Second) + var r struct { + Result seat.KeptPointsAnswer `json:"result"` + } + _ = json.Unmarshal(m.Data, &r) + kp = r.Result + if len(kp.Points) != 1 { + t.Errorf("the newest point is not kept alone: %+v", kp.Points) + } + }) + + t.Run("a keeper's snapshot of three chunks is received and restored whole (§681)", func(t *testing.T) { + _, _ = notes.Put(ctx, "n", []byte("from the keeper")) + var snap bytes.Buffer + if _, err := bussnap.Snapshot(ctx, nc, &snap, bussnap.SnapshotOptions{Streams: []string{"KV_app_notes"}, TempDir: t.TempDir()}); err != nil { + t.Fatal(err) + } + keeper.mu.Lock() + keeper.file = snap.Bytes() + keeper.mu.Unlock() + keeper.set(false, "", "") + _, _ = notes.Put(ctx, "n", []byte("since the drop")) + a := act(t, seat.VerbRestore, runner, seat.RestoreInput{Module: "app", Kept: "notes", Request: "req-1", Why: "test the drop's snapshot", Asker: "op@srv"}) + if ev := endOf(t, seat.EventRestored, a); ev["outcome"] != "done" || live() != "from the keeper" { + t.Errorf("restored %v, live %q", ev, live()) + } + if left, _ := os.ReadDir(filepath.Join(dir, "points", "incoming")); len(left) != 0 { + t.Errorf("incoming holds %v", left) + } + }) + + inPlace := func(t *testing.T, request string) string { + t.Helper() + _, _ = notes.Put(ctx, "n", []byte("live before "+request)) + return act(t, seat.VerbRestore, runner, seat.RestoreInput{Module: "app", Kept: "notes", Request: request, Why: "restore in place", Asker: "op@srv"}) + } + t.Run("a protected snapshot's bound asks the cancel first, then puts the aside back on its failed (§681, A-140, A-143)", func(t *testing.T) { + keeper.set(true, "", meshbroker.SnapshotRestoredEventOutcomeFailed) + id := inPlace(t, "req-2") + if ev := endOf(t, seat.EventRestored, id); ev["outcome"] != "failed" || live() != "live before req-2" { + t.Errorf("restored %v, live %q", ev, live()) + } + if got := keeper.said(); got != "get req-2|verify req-2|restore req-2|cancel keeper-req-2" { + t.Errorf("the keeper was asked %s", got) + } + }) + + t.Run("a keeper that answers neither leaves the aside unrestored, which restore-aside puts back (§681, §554)", func(t *testing.T) { + keeper.set(true, "", "") + id := inPlace(t, "req-3") + ev := endOf(t, seat.EventRestored, id) + if ev["outcome"] != "failed" || ev["error"] != "snapshot-restore unanswered" { + t.Fatalf("restored %v", ev) + } + var r struct { + Result seat.KeptPointsAnswer `json:"result"` + } + m, _ := nc.Request("mesh.seat.mesh-recovery.call.kept-points."+strings.ReplaceAll(runner, ".", "~"), []byte(`{}`), 10*time.Second) + _ = json.Unmarshal(m.Data, &r) + if len(r.Result.Unrestored) != 1 { + t.Fatalf("unrestored %+v", r.Result.Unrestored) + } + back := act(t, seat.VerbRestoreAside, runner, seat.RestoreAsideInput{Module: "app", Kept: "notes", ID: r.Result.Unrestored[0].ID, Why: "put it back", Asker: "op@srv"}) + if ev := endOf(t, seat.EventRestored, back); ev["outcome"] != "done" || live() != "live before req-3" { + t.Errorf("restore-aside %v, live %q", ev, live()) + } + }) + + t.Run("a stop during a restore past its removal puts the aside back and ends it stopped (§562, A-141)", func(t *testing.T) { + keeper.set(true, "", meshbroker.SnapshotRestoredEventOutcomeFailed) + id := inPlace(t, "req-4") + // The wait on snapshot-restored is a bound key: once it stands, the live data is removed and the keeper asked. + bounds, _ := js.KeyValue(ctx, "mesh-recovery_bounds") + w, _ := bounds.Watch(ctx, "restore-wait."+id) + for e := range w.Updates() { + if e != nil && e.Operation() == jetstream.KeyValuePut { + break + } + } + _ = w.Stop() + stop() + ev := endOf(t, seat.EventRestored, id) + if err := <-done; err != nil { + t.Errorf("the process ended %v", err) + } + done <- nil + if ev["outcome"] != "failed" || ev["error"] != "stopped" || live() != "live before req-4" { + t.Errorf("restored %v, live %q", ev, live()) + } + }) +} + +type testLog struct{ t *testing.T } + +func (l testLog) Write(p []byte) (int, error) { + l.t.Log(strings.TrimRight(string(p), "\n")) + return len(p), nil +} diff --git a/modules/mesh-recovery/internal/recovery/points.go b/modules/mesh-recovery/internal/recovery/points.go index 7f904c2f3..aa7207074 100644 --- a/modules/mesh-recovery/internal/recovery/points.go +++ b/modules/mesh-recovery/internal/recovery/points.go @@ -26,7 +26,7 @@ func (e *Engine) run(ctx context.Context, a *Act) { case KindTake: err = e.take(ctx, a) case KindExpire: - e.expire(a.ID) + e.expire(a.ID, true) case KindRestore: err = e.restore(ctx, a) case KindAside: @@ -45,6 +45,18 @@ func (e *Engine) run(ctx context.Context, a *Act) { e.markEnded(a) } +// sayEnd says the ending event of the act id (§554, §560): the act is first no longer the running one and written ended, so +// a caller told busy that asks again on hearing it is never told busy with the same act again. +func (e *Engine) sayEnd(id, event string, body any, parts ...string) { + e.mu.Lock() + if e.running != nil && e.running.ID == id { + e.markEnded(e.running) + e.running, e.cancel = nil, nil + } + e.mu.Unlock() + e.say(event, body, parts...) +} + // say publishes an event and logs the failure; an event lost is never silent. func (e *Engine) say(event string, body any, parts ...string) { if err := e.Seats.Say(event, body, parts...); err != nil { @@ -54,7 +66,7 @@ func (e *Engine) say(event string, body any, parts ...string) { func (e *Engine) endedFailed(a *Act, why error) error { why = e.stoppedOr(why) - e.say(seat.EventEnded, seat.EndedEvent{Act: a.ID, At: e.now(), Module: a.Module, Kept: a.Kept, Form: a.Form, + e.sayEnd(a.ID, seat.EventEnded, seat.EndedEvent{Act: a.ID, At: e.now(), Module: a.Module, Kept: a.Kept, Form: a.Form, Fingerprint: a.Fingerprint, Snapshot: a.Snapshot, Outcome: seat.EndedEventOutcomeFailed, Error: why.Error(), Why: cancelWhy(a, why)}, a.Module, a.Kept) return why @@ -136,10 +148,11 @@ func (e *Engine) take(ctx context.Context, a *Act) error { _ = os.RemoveAll(dir) return e.endedFailed(a, fmt.Errorf("the digests are not on the bus: %w", err)) } - e.say(seat.EventEnded, seat.EndedEvent{Act: a.ID, At: e.now(), Module: a.Module, Kept: a.Kept, Form: a.Form, + // The removal at each snapshot-kept (§551) runs within the take, before its ended, so one act runs at a time. + e.expire(a.ID, false) + e.sayEnd(a.ID, seat.EventEnded, seat.EndedEvent{Act: a.ID, At: e.now(), Module: a.Module, Kept: a.Kept, Form: a.Form, Fingerprint: a.Fingerprint, Snapshot: p.Snapshot, Outcome: seat.EndedEventOutcomeDone, Consumers: p.Consumers, Digests: p.Digests, Size: p.Size, CatchUp: p.CatchUp}, a.Module, a.Kept) - e.expire(a.ID) return nil } @@ -197,8 +210,9 @@ func (e *Engine) Points() ([]seat.Point, error) { } // expire is §551's removal: a point 90 days after its `at` (the setting), never the newest of its kept data, each removal -// said as point-removed and the run as expiry-ended, also when it removes nothing (§560). It runs inside an act. -func (e *Engine) expire(act string) { +// said as point-removed and the run as expiry-ended, also when it removes nothing (§560). It runs inside an act: own when +// it is the act expire-points, whose ending event its expiry-ended is. +func (e *Engine) expire(act string, own bool) { pts, err := e.Points() if err != nil { e.logf("expiry: %v", err) @@ -228,5 +242,10 @@ func (e *Engine) expire(act string) { Fingerprint: p.Fingerprint, Snapshot: p.Snapshot}, p.Module, p.Kept) removed++ } - e.say(seat.EventExpiryEnded, seat.ExpiryEndedEvent{Act: act, At: e.now(), Removed: removed}) + ev := seat.ExpiryEndedEvent{Act: act, At: e.now(), Removed: removed} + if own { + e.sayEnd(act, seat.EventExpiryEnded, ev) + return + } + e.say(seat.EventExpiryEnded, ev) } diff --git a/modules/mesh-recovery/internal/recovery/recovery.go b/modules/mesh-recovery/internal/recovery/recovery.go index eeae6ea63..aa3568b3a 100644 --- a/modules/mesh-recovery/internal/recovery/recovery.go +++ b/modules/mesh-recovery/internal/recovery/recovery.go @@ -284,7 +284,9 @@ func (e *Engine) Accept(a Act) Answer { defer e.wg.Done() e.run(ctx, &a) e.mu.Lock() - e.running, e.cancel = nil, nil + if e.running == &a { // an act that said its ending event is no longer the running one (sayEnd) + e.running, e.cancel = nil, nil + } e.mu.Unlock() cancel() e.judge() diff --git a/modules/mesh-recovery/internal/recovery/restore.go b/modules/mesh-recovery/internal/recovery/restore.go index 6fd4fd60e..73f067e45 100644 --- a/modules/mesh-recovery/internal/recovery/restore.go +++ b/modules/mesh-recovery/internal/recovery/restore.go @@ -58,7 +58,7 @@ func (e *Engine) endedRestored(a *Act, outcomeErr error, aside *seat.Aside, miss if outcomeErr != nil { ev.Outcome, ev.Error, ev.Why = seat.RestoredEventOutcomeFailed, outcomeErr.Error(), e.cancelledWhy(outcomeErr) } - e.say(seat.EventRestored, ev, a.Module, a.Kept) + e.sayEnd(a.ID, seat.EventRestored, ev, a.Module, a.Kept) return outcomeErr } @@ -310,7 +310,7 @@ func (e *Engine) aside(ctx context.Context, a *Act) error { return err } x := rec.aside() - e.say(seat.EventSetAside, seat.SetAsideEvent{Act: a.ID, At: e.now(), ID: x.ID, Names: x.Names, Size: x.Size}, a.Module, a.Kept) + e.sayEnd(a.ID, seat.EventSetAside, seat.SetAsideEvent{Act: a.ID, At: e.now(), ID: x.ID, Names: x.Names, Size: x.Size}, a.Module, a.Kept) return nil } @@ -347,7 +347,7 @@ func (e *Engine) discardAside(a *Act) error { return err } x := rec.aside() - e.say(seat.EventAsideDiscarded, seat.AsideDiscardedEvent{Act: a.ID, At: e.now(), ID: x.ID, Names: x.Names, Size: x.Size}, a.Module, a.Kept) + e.sayEnd(a.ID, seat.EventAsideDiscarded, seat.AsideDiscardedEvent{Act: a.ID, At: e.now(), ID: x.ID, Names: x.Names, Size: x.Size}, a.Module, a.Kept) return nil }