diff --git a/modules/mesh-recovery/.gitignore b/modules/mesh-recovery/.gitignore new file mode 100644 index 000000000..1b8d4339c --- /dev/null +++ b/modules/mesh-recovery/.gitignore @@ -0,0 +1,3 @@ +/mesh-recovery +/cmd/mesh-recovery/mesh-recovery +/cmd/mesh-recovery-tools/mesh-recovery-tools diff --git a/modules/mesh-recovery/README.md b/modules/mesh-recovery/README.md new file mode 100644 index 000000000..a1156d343 --- /dev/null +++ b/modules/mesh-recovery/README.md @@ -0,0 +1,9 @@ +# mesh-recovery + +Holds the seat `mesh-recovery` (design 47 G §554): the recovery module takes, sets aside and restores a module's kept +data on the live bus, one act at a time, each act in the hand-act log, its result said as an event under the act's id. +Go: one process (`cmd/mesh-recovery`) that answers the seat's verbs itself (A-107, A-115) and a tools bundle +(`cmd/mesh-recovery-tools`) whose `recovery_*` tools call those verbs and hold no grant of their own (§656). + +Its data item `points` holds the restore points and the asides; the digests of the points are on the bus in its state +`digests` (§567). Its settings are `points-cap`, `points-filling-mark`, `point-keep-days` and `dump-wait-minutes`. diff --git a/modules/mesh-recovery/go.mod b/modules/mesh-recovery/go.mod new file mode 100644 index 000000000..ee3bd0cc0 --- /dev/null +++ b/modules/mesh-recovery/go.mod @@ -0,0 +1,24 @@ +module mesh-recovery + +go 1.26.0 + +require ( + git.novox.be/novox/mesh-contracts/go/seats/meshbroker v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/meshrecovery v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 + github.com/nats-io/nats.go v1.54.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.1 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.2 // indirect + github.com/nats-io/nats-server/v2 v2.11.17 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.16.0 // indirect +) diff --git a/modules/mesh-recovery/go.sum b/modules/mesh-recovery/go.sum new file mode 100644 index 000000000..8ca93dfd0 --- /dev/null +++ b/modules/mesh-recovery/go.sum @@ -0,0 +1,25 @@ +github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E= +github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ= +github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc= +github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE= +golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s= diff --git a/modules/mesh-recovery/internal/recovery/manifest_test.go b/modules/mesh-recovery/internal/recovery/manifest_test.go new file mode 100644 index 000000000..61da8aa86 --- /dev/null +++ b/modules/mesh-recovery/internal/recovery/manifest_test.go @@ -0,0 +1,101 @@ +package recovery_test + +import ( + "encoding/json" + "os" + "slices" + "testing" + + seat "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" +) + +type manifest struct { + Module string `json:"module"` + RunsAs string `json:"runs-as"` + Claims []struct { + Name string `json:"name"` + Scope string `json:"scope"` + Serves []string `json:"serves"` + } `json:"claims"` + Tools []string `json:"tools"` + Consumes []string `json:"consumes"` + Invokes []string `json:"invokes"` + Config map[string]struct { + Default float64 `json:"default"` + } `json:"config"` + Data struct { + Own []struct{ ID string } `json:"own"` + } `json:"data"` + Resources []struct { + ID string `json:"id"` + Type string `json:"type"` + Serves []string `json:"serves"` + Bus struct { + Invokes []string `json:"invokes"` + } `json:"bus"` + Health struct { + Tool string `json:"tool"` + } `json:"health"` + } `json:"resources"` +} + +// The manifest says what the module is (design 47 G §554, §656, §657, §633, §564, §567): it claims the seat once for the +// mesh serving every verb of the seat's definition, its process answers each of them itself (A-115: `serves` beside its +// `bus`), its health check names a tool it serves, its data item is `points`, its settings carry the design's defaults, +// and what the process invokes the module declares. +func TestTheManifestSaysWhatTheModuleIs(t *testing.T) { + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + var m manifest + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + if m.Module != "mesh-recovery" || m.RunsAs != "mesh-recovery" { + t.Errorf("module %q runs as %q", m.Module, m.RunsAs) + } + if len(m.Claims) != 1 || m.Claims[0].Name != seat.Seat || m.Claims[0].Scope != "mesh" || !slices.Equal(m.Claims[0].Serves, seat.Verbs()) { + t.Errorf("claims %+v, want %s at mesh scope serving %v", m.Claims, seat.Seat, seat.Verbs()) + } + for _, id := range []string{"points"} { + if !slices.ContainsFunc(m.Data.Own, func(o struct{ ID string }) bool { return o.ID == id }) { + t.Errorf("no data item %q", id) + } + } + for k, want := range map[string]float64{"points-cap": 10 << 30, "points-filling-mark": 80, "point-keep-days": 90, "dump-wait-minutes": 60} { + if m.Config[k].Default != want { + t.Errorf("setting %s defaults to %v, want %v", k, m.Config[k].Default, want) + } + } + for _, r := range m.Resources { + if r.Type != "process" { + continue + } + var want []string + for _, v := range seat.Verbs() { + want = append(want, "seat:"+seat.Seat+"."+v) + } + if !slices.Equal(r.Serves, want) { + t.Errorf("the process serves %v, want %v", r.Serves, want) + } + if !slices.Contains(m.Tools, r.Health.Tool) || r.Health.Tool != "recovery_kept_points" { + t.Errorf("the health check names %q, a tool of %v", r.Health.Tool, m.Tools) + } + for _, inv := range r.Bus.Invokes { + if !slices.Contains(m.Invokes, inv) { + t.Errorf("the process invokes %s, which the module does not declare", inv) + } + } + } + for _, c := range []string{"node-backup.item-started", "node-backup.item-ended", "mesh-broker.snapshot-restored"} { + if !slices.Contains(m.Consumes, c) { + t.Errorf("does not consume %s", c) + } + } + for _, tool := range m.Tools { + if len(tool) < 9 || tool[:9] != "recovery_" { + t.Errorf("the tool %q is not named recovery_", tool) + } + } +} diff --git a/modules/mesh-recovery/module.json b/modules/mesh-recovery/module.json new file mode 100644 index 000000000..ac477062c --- /dev/null +++ b/modules/mesh-recovery/module.json @@ -0,0 +1,265 @@ +{ + "module": "mesh-recovery", + "version": "0.1", + "runs-as": "mesh-recovery", + "claims": [ + { + "name": "mesh-recovery", + "scope": "mesh", + "serves": [ + "snapshot-kept", + "restore-kept", + "kept-points", + "restore-aside", + "cancel", + "take", + "restore", + "retry", + "aside", + "discard-aside", + "expire-points" + ] + } + ], + "tools": [ + "recovery_kept_points", + "recovery_take", + "recovery_restore", + "recovery_retry", + "recovery_aside", + "recovery_discard_aside", + "recovery_restore_aside", + "recovery_cancel", + "recovery_expire_points" + ], + "consumes": [ + "node-backup.item-started", + "node-backup.item-ended", + "mesh-broker.snapshot-restored", + "mesh-conditions.findings-asked" + ], + "invokes": [ + "seat:mesh-catalog.entry", + "seat:mesh-hand-acts.hand-act", + "seat:mesh-broker.snapshot-get", + "seat:mesh-broker.snapshot-restore", + "seat:node-backup.restore", + "seat:mesh-recovery.kept-points", + "seat:mesh-recovery.restore-aside", + "seat:mesh-recovery.cancel", + "seat:mesh-recovery.take", + "seat:mesh-recovery.restore", + "seat:mesh-recovery.retry", + "seat:mesh-recovery.aside", + "seat:mesh-recovery.discard-aside", + "seat:mesh-recovery.expire-points" + ], + "requires": [ + { + "name": "digests", + "storage": "key-value", + "template": "state", + "options": { + "max-mib": 4 + } + }, + { + "name": "findings", + "storage": "key-value", + "template": "state" + }, + { + "name": "finding-events", + "storage": "stream", + "template": "events", + "options": { + "events": [ + "finding-written.*", + "finding-cleared.*", + "findings-stated.*" + ], + "max-age-days": 7, + "max-mib": 16 + } + } + ], + "config": { + "points-cap": { + "type": "number", + "description": "bytes the points directory may hold; an act that would pass it ends failed, naming it, before it removes anything (design 47 G section 564).", + "minimum": 1, + "trusted": false, + "default": 10737418240 + }, + "points-filling-mark": { + "type": "number", + "description": "percent of the cap past which the finding points-filling is written (design 47 G section 564).", + "minimum": 1, + "maximum": 100, + "trusted": false, + "default": 80 + }, + "point-keep-days": { + "type": "number", + "description": "days after which a restore point is removed, the newest of its kept data never (design 47 G sections 551, 557).", + "minimum": 1, + "maximum": 3650, + "trusted": false, + "default": 90 + }, + "dump-wait-minutes": { + "type": "number", + "description": "minutes an act waits on the dump of the bus's item before recovery-waits is written (design 47 G sections 561, 557).", + "minimum": 1, + "maximum": 10080, + "trusted": false, + "default": 60 + } + }, + "data": { + "own": [ + { + "id": "points", + "path": "${dir:points}", + "class": "valuable", + "why": "the restore points taken of modules' kept data before a form change, and the asides of live data set aside before a restore (design 47 G sections 516, 554)" + } + ] + }, + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-recovery", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-recovery", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-recovery" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-recovery", + "place": ".", + "unknown": "program" + }, + { + "id": "points", + "type": "directory", + "path": "/var/lib/mesh-recovery/points", + "mode": "0700", + "owner": "mesh-recovery", + "unknown": "program" + }, + { + "id": "cap", + "type": "file", + "path": "${dir:state}/points-cap", + "owner": "mesh-recovery", + "mode": "0600", + "content": "${config:points-cap}" + }, + { + "id": "mark", + "type": "file", + "path": "${dir:state}/points-filling-mark", + "owner": "mesh-recovery", + "mode": "0600", + "content": "${config:points-filling-mark}" + }, + { + "id": "keep-days", + "type": "file", + "path": "${dir:state}/point-keep-days", + "owner": "mesh-recovery", + "mode": "0600", + "content": "${config:point-keep-days}" + }, + { + "id": "dump-wait", + "type": "file", + "path": "${dir:state}/dump-wait-minutes", + "owner": "mesh-recovery", + "mode": "0600", + "content": "${config:dump-wait-minutes}" + }, + { + "id": "recovery", + "type": "process", + "name": "mesh-recovery", + "artifact": "program", + "run": [ + "./mesh-recovery" + ], + "env": { + "MESH_BROKER_FILE": "${dir:state}/broker", + "MESH_RECOVERY_POINTS": "${dir:points}", + "MESH_RECOVERY_POINTS_CAP_FILE": "${dir:state}/points-cap", + "MESH_RECOVERY_FILLING_MARK_FILE": "${dir:state}/points-filling-mark", + "MESH_RECOVERY_KEEP_DAYS_FILE": "${dir:state}/point-keep-days", + "MESH_RECOVERY_DUMP_WAIT_FILE": "${dir:state}/dump-wait-minutes" + }, + "user": "mesh-recovery", + "serves": [ + "seat:mesh-recovery.snapshot-kept", + "seat:mesh-recovery.restore-kept", + "seat:mesh-recovery.kept-points", + "seat:mesh-recovery.restore-aside", + "seat:mesh-recovery.cancel", + "seat:mesh-recovery.take", + "seat:mesh-recovery.restore", + "seat:mesh-recovery.retry", + "seat:mesh-recovery.aside", + "seat:mesh-recovery.discard-aside", + "seat:mesh-recovery.expire-points" + ], + "bus": { + "state": [ + "digests", + "findings" + ], + "invokes": [ + "seat:mesh-catalog.entry", + "seat:mesh-hand-acts.hand-act", + "seat:mesh-broker.snapshot-get", + "seat:mesh-broker.snapshot-restore", + "seat:node-backup.restore" + ] + }, + "health": { + "kind": "tool", + "tool": "recovery_kept_points", + "interval": "60s", + "timeout": "10s", + "looks": 2, + "grace": "180s" + } + } + ], + "build": { + "artifacts": [ + { + "name": "program", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-recovery", + "binary": "mesh-recovery" + }, + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-recovery-tools", + "binary": "mesh-recovery-tools", + "loads": [ + "mesh-recovery-tools" + ] + } + ] + } +}