restic: write each item's last good night when a night's files change, with no timer, on the sdk's layout; the night is judged by its unit and the awake process by the health tool (issue 1955)
This commit is contained in:
@@ -33,6 +33,7 @@ import (
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
@@ -192,6 +193,12 @@ type Backups struct {
|
||||
// Readings is every reading disk-load keeps for this machine (its state `readings`, ADR 0571 clause 3); nil where the
|
||||
// process has no runtime to read it.
|
||||
Readings func() ([]health.Reading, error)
|
||||
// PutGood writes one row of the state `good` (health.GoodRow): the night's process through its own bus account, the
|
||||
// tools through the runtime; nil where there is no bus to write to.
|
||||
PutGood func(key string, row health.GoodRow) error
|
||||
|
||||
goodMu sync.Mutex
|
||||
goodWrote map[string][]byte
|
||||
}
|
||||
|
||||
// The locks, files in this module's state. The night runs in a process of its own and `now`, `restore` and a
|
||||
|
||||
@@ -102,6 +102,7 @@ func (b *Backups) recordGood(module string, at time.Time) {
|
||||
if err := os.WriteFile(b.goodFile(), append(raw, '\n'), 0o600); err != nil {
|
||||
b.Say("recording %s's good night failed: %v", module, err)
|
||||
}
|
||||
b.keepGood()
|
||||
}
|
||||
|
||||
// itemReports is every item of one module, with its newest reading and its newest good backup: the
|
||||
|
||||
@@ -80,11 +80,14 @@ func TestEveryItemIsSaidWithItsReadingAndItsLastGoodBackup(t *testing.T) {
|
||||
}
|
||||
// The same last good night is kept on the bus beside good.json, one row per item a backup path covers.
|
||||
b.Node = "ace"
|
||||
rows, err := b.GoodRows()
|
||||
must(t, err)
|
||||
row, ok := rows[GoodKey("ace", "postgres", "store")]
|
||||
if len(rows) != 1 || !ok || !row.LastGood.Equal(night) || row.CoveredBy != "/var/lib/mesh-store/dumps" || row.V != GoodVersion {
|
||||
t.Fatalf("the good rows are %+v", rows)
|
||||
wrote := map[string]health.GoodRow{}
|
||||
b.PutGood = func(k string, r health.GoodRow) error { wrote[k] = r; return nil }
|
||||
b.recordGood("postgres", night) // a good night writes the rows, once; the same night again writes nothing
|
||||
n := len(wrote)
|
||||
b.recordGood("postgres", night)
|
||||
row, ok := wrote[health.GoodKey("ace", "postgres", "store")]
|
||||
if len(wrote) != 1 || n != 1 || !ok || !row.LastGood.Equal(night) || row.CoveredBy != "/var/lib/mesh-store/dumps" || row.V != health.GoodVersion {
|
||||
t.Fatalf("the good rows are %+v", wrote)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,39 +1,23 @@
|
||||
package main
|
||||
|
||||
// **Each item's last good night, on the bus** (novox/hq ADR 0571 clause 3, ADR 0564 §6): beside good.json, the module's
|
||||
// own state `good`, one key `<node>.<module>.<item>` for every item a module declares that a backup path covers, written
|
||||
// when it changes, so the self-check's probe of a stale backup reads it there and no part reads the controller's records.
|
||||
// own state `good` (health.GoodRow), one key `<node>.<module>.<item>`, written when a night's files change, so the
|
||||
// self-check's probe of a stale backup reads it there.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
"unsafe"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// GoodState is the state, GoodVersion the layout's version.
|
||||
const (
|
||||
GoodState = "good"
|
||||
GoodVersion = 1
|
||||
)
|
||||
|
||||
// GoodRow is one key: the item, the path whose snapshot keeps it, and the newest good night of its module (zero before
|
||||
// the first). Moves to mesh-sdk's health package with the readers that share it.
|
||||
type GoodRow struct {
|
||||
V int `json:"v"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Item string `json:"item"`
|
||||
CoveredBy string `json:"covered_by"`
|
||||
LastGood time.Time `json:"last_good,omitempty"`
|
||||
}
|
||||
|
||||
// GoodKey is a row's key.
|
||||
func GoodKey(node, module, item string) string { return node + "." + module + "." + item }
|
||||
|
||||
// GoodRows is the rows of this machine: every item a module declares that a path of the module's backup covers, with
|
||||
// the module's newest good night.
|
||||
func (b *Backups) GoodRows() (map[string]GoodRow, error) {
|
||||
// GoodRows is the rows of this machine (health.GoodRow): every item a module declares that a path of the module's backup
|
||||
// covers, with the module's newest good night.
|
||||
func (b *Backups) GoodRows() (map[string]health.GoodRow, error) {
|
||||
items, err := b.Items()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -47,7 +31,7 @@ func (b *Backups) GoodRows() (map[string]GoodRow, error) {
|
||||
paths[d.Module] = d.Paths
|
||||
}
|
||||
good := b.Good()
|
||||
out := map[string]GoodRow{}
|
||||
out := map[string]health.GoodRow{}
|
||||
for module, its := range items {
|
||||
for _, it := range its {
|
||||
if it.Class == "cache" || it.CoveredBy == "" {
|
||||
@@ -60,39 +44,85 @@ func (b *Backups) GoodRows() (map[string]GoodRow, error) {
|
||||
if !covered {
|
||||
continue
|
||||
}
|
||||
out[GoodKey(b.Node, module, it.ID)] = GoodRow{V: GoodVersion, Node: b.Node, Module: module, Item: it.ID,
|
||||
CoveredBy: it.CoveredBy, LastGood: good[module].UTC()}
|
||||
out[health.GoodKey(b.Node, module, it.ID)] = health.GoodRow{V: health.GoodVersion, Node: b.Node, Module: module,
|
||||
Item: it.ID, CoveredBy: it.CoveredBy, LastGood: good[module].UTC()}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// syncGood writes the rows that differ from what it last wrote, once a minute while it runs (good.json is a file of
|
||||
// the night's process, so a minute is how fast it can hear of a night); put is the state's write.
|
||||
func syncGood(b *Backups, put func(key string, row GoodRow) error) {
|
||||
time.Sleep(20 * time.Second) // the runtime serves first
|
||||
wrote := map[string][]byte{}
|
||||
for {
|
||||
rows, err := b.GoodRows()
|
||||
if err != nil {
|
||||
b.Say("the last good nights could not be read: %v", err)
|
||||
// keepGood writes the rows that differ from what was last written, when a night succeeded (recordGood). A bus that
|
||||
// cannot be reached loses nothing: good.json is the local copy, and the next good night writes the rows again.
|
||||
func (b *Backups) keepGood() {
|
||||
if b.PutGood == nil {
|
||||
return
|
||||
}
|
||||
rows, err := b.GoodRows()
|
||||
if err != nil {
|
||||
b.Say("the last good nights could not be read: %v", err)
|
||||
return
|
||||
}
|
||||
b.goodMu.Lock()
|
||||
defer b.goodMu.Unlock()
|
||||
if b.goodWrote == nil {
|
||||
b.goodWrote = map[string][]byte{}
|
||||
}
|
||||
for k, r := range rows {
|
||||
raw, _ := json.Marshal(r)
|
||||
if string(b.goodWrote[k]) == string(raw) {
|
||||
continue
|
||||
}
|
||||
for k, r := range rows {
|
||||
raw, _ := json.Marshal(r)
|
||||
if string(wrote[k]) == string(raw) {
|
||||
continue
|
||||
}
|
||||
if err := put(k, r); err != nil {
|
||||
b.Say("the last good night of %s was not put: %v", k, err)
|
||||
continue
|
||||
}
|
||||
wrote[k] = raw
|
||||
if err := b.PutGood(k, r); err != nil {
|
||||
b.Say("the last good night of %s was not put on the bus: %v", k, err)
|
||||
continue
|
||||
}
|
||||
time.Sleep(time.Minute)
|
||||
b.goodWrote[k] = raw
|
||||
}
|
||||
}
|
||||
|
||||
func busGood(key string, row GoodRow) error {
|
||||
_, err := stdio.State(GoodState).Put(key, row)
|
||||
return err
|
||||
// watchGood writes the rows once at start and then each time the night's process changes what it keeps in the state
|
||||
// directory (good.json, the nights' file), as the file system says it: the night runs as the operator account, which a
|
||||
// process with a bus account of its own may not (ADR 0436 §2), so the tools' runtime, which has the module's account,
|
||||
// writes at the night's event instead of the night itself. No timer.
|
||||
func watchGood(b *Backups) {
|
||||
time.Sleep(20 * time.Second) // the runtime serves first
|
||||
b.keepGood()
|
||||
for {
|
||||
err := whenWritten(b.Where.State, func(name string) bool { return name == "good.json" || name == filepath.Base(b.nightsFile()) }, b.keepGood)
|
||||
b.Say("watching the state directory for a night failed (%v); trying again in a minute", err)
|
||||
time.Sleep(time.Minute)
|
||||
b.keepGood()
|
||||
}
|
||||
}
|
||||
|
||||
// whenWritten calls then each time a file of dir that match names is written or put in place, until the watch fails.
|
||||
func whenWritten(dir string, match func(name string) bool, then func()) error {
|
||||
fd, err := syscall.InotifyInit1(syscall.IN_CLOEXEC)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer syscall.Close(fd)
|
||||
if _, err := syscall.InotifyAddWatch(fd, dir, syscall.IN_CLOSE_WRITE|syscall.IN_MOVED_TO); err != nil {
|
||||
return err
|
||||
}
|
||||
buf := make([]byte, 64*syscall.SizeofInotifyEvent+4096)
|
||||
for {
|
||||
n, err := syscall.Read(fd, buf)
|
||||
if err != nil {
|
||||
if err == syscall.EINTR {
|
||||
continue
|
||||
}
|
||||
return err
|
||||
}
|
||||
heard := false
|
||||
for off := 0; off+syscall.SizeofInotifyEvent <= n; {
|
||||
ev := (*syscall.InotifyEvent)(unsafe.Pointer(&buf[off]))
|
||||
name := strings.TrimRight(string(buf[off+syscall.SizeofInotifyEvent:off+syscall.SizeofInotifyEvent+int(ev.Len)]), "\x00")
|
||||
heard = heard || match(name)
|
||||
off += syscall.SizeofInotifyEvent + int(ev.Len)
|
||||
}
|
||||
if heard {
|
||||
then()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -263,8 +263,8 @@ func TestTheManifestRunsTheNightJudgedByItsHealthTool(t *testing.T) {
|
||||
}
|
||||
env := m.Build.Artifacts[0].Env
|
||||
want := map[string]struct{ run, kind, tool string }{
|
||||
"night": {"night", "tool", "backup_health"},
|
||||
"awake": {"awake", "unit", ""},
|
||||
"night": {"night", "unit", ""},
|
||||
"awake": {"awake", "tool", "backup_health"},
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
w, ok := want[r["id"].(string)]
|
||||
|
||||
@@ -56,11 +56,15 @@ func main() {
|
||||
case "awake":
|
||||
counting(b)
|
||||
case "":
|
||||
if err := stdio.Keeps(announce.State{Name: GoodState}); err != nil {
|
||||
if err := stdio.Keeps(announce.State{Name: health.GoodState}); err != nil {
|
||||
say("%v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
go syncGood(b, busGood)
|
||||
b.PutGood = func(key string, row health.GoodRow) error {
|
||||
_, err := stdio.State(health.GoodState).Put(key, row)
|
||||
return err
|
||||
}
|
||||
go watchGood(b)
|
||||
if err := stdio.Serve("", tools(b)); err != nil {
|
||||
say("%v", err)
|
||||
os.Exit(1)
|
||||
|
||||
+10
-1
@@ -2,4 +2,13 @@ module restic
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029221656-196c5b6af277
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029223737-9f0bb1cfa4ba
|
||||
|
||||
require (
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/nats-io/nats.go v1.54.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
)
|
||||
|
||||
+26
-2
@@ -1,2 +1,26 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029221656-196c5b6af277 h1:ajjsH4MWcx45MGYASjUSRQKhLzEringuk3TA0oxZRCU=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029221656-196c5b6af277/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029223737-9f0bb1cfa4ba h1:4By9fDPDDiJXX32ySd6+WjAMAIX9vkVTT+Zs9+vhcLk=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029223737-9f0bb1cfa4ba/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
|
||||
@@ -139,12 +139,7 @@
|
||||
"MESH_BACKUP_DATA": "${dir:state}/data.conf"
|
||||
},
|
||||
"health": {
|
||||
"kind": "tool",
|
||||
"tool": "backup_health",
|
||||
"interval": "60s",
|
||||
"timeout": "20s",
|
||||
"looks": 2,
|
||||
"grace": "60s"
|
||||
"kind": "unit"
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -165,7 +160,12 @@
|
||||
"MESH_BACKUP_DATA": "${dir:state}/data.conf"
|
||||
},
|
||||
"health": {
|
||||
"kind": "unit"
|
||||
"kind": "tool",
|
||||
"tool": "backup_health",
|
||||
"interval": "60s",
|
||||
"timeout": "20s",
|
||||
"looks": 2,
|
||||
"grace": "60s"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
||||
Reference in New Issue
Block a user