From 86e1082fa0bd94738c426b75554f798eb6884f88 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:40:32 +0100 Subject: [PATCH] restic: write each item's last good night when a night's files change, with no timer, on the sdk's layout; the night is judged by its unit and the awake process by the health tool (issue 1955) --- checks/health/go.mod | 2 +- checks/health/go.sum | 4 +- modules/restic/cmd/restic-backups/backups.go | 7 + modules/restic/cmd/restic-backups/data.go | 1 + .../restic/cmd/restic-backups/data_test.go | 13 +- modules/restic/cmd/restic-backups/good.go | 134 +++++++++++------- .../restic/cmd/restic-backups/health_test.go | 4 +- modules/restic/cmd/restic-backups/main.go | 8 +- modules/restic/go.mod | 11 +- modules/restic/go.sum | 28 +++- modules/restic/module.json | 14 +- 11 files changed, 152 insertions(+), 74 deletions(-) diff --git a/checks/health/go.mod b/checks/health/go.mod index fd5c93e8..f9e75ef6 100644 --- a/checks/health/go.mod +++ b/checks/health/go.mod @@ -2,4 +2,4 @@ module checks/health go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029223134-08fc70783a7c +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029223737-9f0bb1cfa4ba diff --git a/checks/health/go.sum b/checks/health/go.sum index fbfd7431..8195008f 100644 --- a/checks/health/go.sum +++ b/checks/health/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029223134-08fc70783a7c h1:inVTVNIZI4/v7/aVhL+EkKzHCDlN73hJrWKz3jI59mY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029223134-08fc70783a7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029223737-9f0bb1cfa4ba h1:4By9fDPDDiJXX32ySd6+WjAMAIX9vkVTT+Zs9+vhcLk= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029223737-9f0bb1cfa4ba/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/modules/restic/cmd/restic-backups/backups.go b/modules/restic/cmd/restic-backups/backups.go index 8f428db4..f0954581 100644 --- a/modules/restic/cmd/restic-backups/backups.go +++ b/modules/restic/cmd/restic-backups/backups.go @@ -33,6 +33,7 @@ import ( "slices" "strconv" "strings" + "sync" "syscall" "time" ) @@ -192,6 +193,12 @@ type Backups struct { // Readings is every reading disk-load keeps for this machine (its state `readings`, ADR 0571 clause 3); nil where the // process has no runtime to read it. Readings func() ([]health.Reading, error) + // PutGood writes one row of the state `good` (health.GoodRow): the night's process through its own bus account, the + // tools through the runtime; nil where there is no bus to write to. + PutGood func(key string, row health.GoodRow) error + + goodMu sync.Mutex + goodWrote map[string][]byte } // The locks, files in this module's state. The night runs in a process of its own and `now`, `restore` and a diff --git a/modules/restic/cmd/restic-backups/data.go b/modules/restic/cmd/restic-backups/data.go index 8c597684..67fb4d46 100644 --- a/modules/restic/cmd/restic-backups/data.go +++ b/modules/restic/cmd/restic-backups/data.go @@ -102,6 +102,7 @@ func (b *Backups) recordGood(module string, at time.Time) { if err := os.WriteFile(b.goodFile(), append(raw, '\n'), 0o600); err != nil { b.Say("recording %s's good night failed: %v", module, err) } + b.keepGood() } // itemReports is every item of one module, with its newest reading and its newest good backup: the diff --git a/modules/restic/cmd/restic-backups/data_test.go b/modules/restic/cmd/restic-backups/data_test.go index 53f6d0ff..9e41411d 100644 --- a/modules/restic/cmd/restic-backups/data_test.go +++ b/modules/restic/cmd/restic-backups/data_test.go @@ -80,11 +80,14 @@ func TestEveryItemIsSaidWithItsReadingAndItsLastGoodBackup(t *testing.T) { } // The same last good night is kept on the bus beside good.json, one row per item a backup path covers. b.Node = "ace" - rows, err := b.GoodRows() - must(t, err) - row, ok := rows[GoodKey("ace", "postgres", "store")] - if len(rows) != 1 || !ok || !row.LastGood.Equal(night) || row.CoveredBy != "/var/lib/mesh-store/dumps" || row.V != GoodVersion { - t.Fatalf("the good rows are %+v", rows) + wrote := map[string]health.GoodRow{} + b.PutGood = func(k string, r health.GoodRow) error { wrote[k] = r; return nil } + b.recordGood("postgres", night) // a good night writes the rows, once; the same night again writes nothing + n := len(wrote) + b.recordGood("postgres", night) + row, ok := wrote[health.GoodKey("ace", "postgres", "store")] + if len(wrote) != 1 || n != 1 || !ok || !row.LastGood.Equal(night) || row.CoveredBy != "/var/lib/mesh-store/dumps" || row.V != health.GoodVersion { + t.Fatalf("the good rows are %+v", wrote) } } diff --git a/modules/restic/cmd/restic-backups/good.go b/modules/restic/cmd/restic-backups/good.go index e6aa9e0f..5ea3c180 100644 --- a/modules/restic/cmd/restic-backups/good.go +++ b/modules/restic/cmd/restic-backups/good.go @@ -1,39 +1,23 @@ package main // **Each item's last good night, on the bus** (novox/hq ADR 0571 clause 3, ADR 0564 §6): beside good.json, the module's -// own state `good`, one key `..` for every item a module declares that a backup path covers, written -// when it changes, so the self-check's probe of a stale backup reads it there and no part reads the controller's records. +// own state `good` (health.GoodRow), one key `..`, written when a night's files change, so the +// self-check's probe of a stale backup reads it there. import ( "encoding/json" + "path/filepath" + "strings" + "syscall" "time" + "unsafe" - stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/health" ) -// GoodState is the state, GoodVersion the layout's version. -const ( - GoodState = "good" - GoodVersion = 1 -) - -// GoodRow is one key: the item, the path whose snapshot keeps it, and the newest good night of its module (zero before -// the first). Moves to mesh-sdk's health package with the readers that share it. -type GoodRow struct { - V int `json:"v"` - Node string `json:"node"` - Module string `json:"module"` - Item string `json:"item"` - CoveredBy string `json:"covered_by"` - LastGood time.Time `json:"last_good,omitempty"` -} - -// GoodKey is a row's key. -func GoodKey(node, module, item string) string { return node + "." + module + "." + item } - -// GoodRows is the rows of this machine: every item a module declares that a path of the module's backup covers, with -// the module's newest good night. -func (b *Backups) GoodRows() (map[string]GoodRow, error) { +// GoodRows is the rows of this machine (health.GoodRow): every item a module declares that a path of the module's backup +// covers, with the module's newest good night. +func (b *Backups) GoodRows() (map[string]health.GoodRow, error) { items, err := b.Items() if err != nil { return nil, err @@ -47,7 +31,7 @@ func (b *Backups) GoodRows() (map[string]GoodRow, error) { paths[d.Module] = d.Paths } good := b.Good() - out := map[string]GoodRow{} + out := map[string]health.GoodRow{} for module, its := range items { for _, it := range its { if it.Class == "cache" || it.CoveredBy == "" { @@ -60,39 +44,85 @@ func (b *Backups) GoodRows() (map[string]GoodRow, error) { if !covered { continue } - out[GoodKey(b.Node, module, it.ID)] = GoodRow{V: GoodVersion, Node: b.Node, Module: module, Item: it.ID, - CoveredBy: it.CoveredBy, LastGood: good[module].UTC()} + out[health.GoodKey(b.Node, module, it.ID)] = health.GoodRow{V: health.GoodVersion, Node: b.Node, Module: module, + Item: it.ID, CoveredBy: it.CoveredBy, LastGood: good[module].UTC()} } } return out, nil } -// syncGood writes the rows that differ from what it last wrote, once a minute while it runs (good.json is a file of -// the night's process, so a minute is how fast it can hear of a night); put is the state's write. -func syncGood(b *Backups, put func(key string, row GoodRow) error) { - time.Sleep(20 * time.Second) // the runtime serves first - wrote := map[string][]byte{} - for { - rows, err := b.GoodRows() - if err != nil { - b.Say("the last good nights could not be read: %v", err) +// keepGood writes the rows that differ from what was last written, when a night succeeded (recordGood). A bus that +// cannot be reached loses nothing: good.json is the local copy, and the next good night writes the rows again. +func (b *Backups) keepGood() { + if b.PutGood == nil { + return + } + rows, err := b.GoodRows() + if err != nil { + b.Say("the last good nights could not be read: %v", err) + return + } + b.goodMu.Lock() + defer b.goodMu.Unlock() + if b.goodWrote == nil { + b.goodWrote = map[string][]byte{} + } + for k, r := range rows { + raw, _ := json.Marshal(r) + if string(b.goodWrote[k]) == string(raw) { + continue } - for k, r := range rows { - raw, _ := json.Marshal(r) - if string(wrote[k]) == string(raw) { - continue - } - if err := put(k, r); err != nil { - b.Say("the last good night of %s was not put: %v", k, err) - continue - } - wrote[k] = raw + if err := b.PutGood(k, r); err != nil { + b.Say("the last good night of %s was not put on the bus: %v", k, err) + continue } - time.Sleep(time.Minute) + b.goodWrote[k] = raw } } -func busGood(key string, row GoodRow) error { - _, err := stdio.State(GoodState).Put(key, row) - return err +// watchGood writes the rows once at start and then each time the night's process changes what it keeps in the state +// directory (good.json, the nights' file), as the file system says it: the night runs as the operator account, which a +// process with a bus account of its own may not (ADR 0436 §2), so the tools' runtime, which has the module's account, +// writes at the night's event instead of the night itself. No timer. +func watchGood(b *Backups) { + time.Sleep(20 * time.Second) // the runtime serves first + b.keepGood() + for { + err := whenWritten(b.Where.State, func(name string) bool { return name == "good.json" || name == filepath.Base(b.nightsFile()) }, b.keepGood) + b.Say("watching the state directory for a night failed (%v); trying again in a minute", err) + time.Sleep(time.Minute) + b.keepGood() + } +} + +// whenWritten calls then each time a file of dir that match names is written or put in place, until the watch fails. +func whenWritten(dir string, match func(name string) bool, then func()) error { + fd, err := syscall.InotifyInit1(syscall.IN_CLOEXEC) + if err != nil { + return err + } + defer syscall.Close(fd) + if _, err := syscall.InotifyAddWatch(fd, dir, syscall.IN_CLOSE_WRITE|syscall.IN_MOVED_TO); err != nil { + return err + } + buf := make([]byte, 64*syscall.SizeofInotifyEvent+4096) + for { + n, err := syscall.Read(fd, buf) + if err != nil { + if err == syscall.EINTR { + continue + } + return err + } + heard := false + for off := 0; off+syscall.SizeofInotifyEvent <= n; { + ev := (*syscall.InotifyEvent)(unsafe.Pointer(&buf[off])) + name := strings.TrimRight(string(buf[off+syscall.SizeofInotifyEvent:off+syscall.SizeofInotifyEvent+int(ev.Len)]), "\x00") + heard = heard || match(name) + off += syscall.SizeofInotifyEvent + int(ev.Len) + } + if heard { + then() + } + } } diff --git a/modules/restic/cmd/restic-backups/health_test.go b/modules/restic/cmd/restic-backups/health_test.go index 24bd7353..76ce1550 100644 --- a/modules/restic/cmd/restic-backups/health_test.go +++ b/modules/restic/cmd/restic-backups/health_test.go @@ -263,8 +263,8 @@ func TestTheManifestRunsTheNightJudgedByItsHealthTool(t *testing.T) { } env := m.Build.Artifacts[0].Env want := map[string]struct{ run, kind, tool string }{ - "night": {"night", "tool", "backup_health"}, - "awake": {"awake", "unit", ""}, + "night": {"night", "unit", ""}, + "awake": {"awake", "tool", "backup_health"}, } for _, r := range m.Resources { w, ok := want[r["id"].(string)] diff --git a/modules/restic/cmd/restic-backups/main.go b/modules/restic/cmd/restic-backups/main.go index cdbb802a..20c0a76b 100644 --- a/modules/restic/cmd/restic-backups/main.go +++ b/modules/restic/cmd/restic-backups/main.go @@ -56,11 +56,15 @@ func main() { case "awake": counting(b) case "": - if err := stdio.Keeps(announce.State{Name: GoodState}); err != nil { + if err := stdio.Keeps(announce.State{Name: health.GoodState}); err != nil { say("%v", err) os.Exit(1) } - go syncGood(b, busGood) + b.PutGood = func(key string, row health.GoodRow) error { + _, err := stdio.State(health.GoodState).Put(key, row) + return err + } + go watchGood(b) if err := stdio.Serve("", tools(b)); err != nil { say("%v", err) os.Exit(1) diff --git a/modules/restic/go.mod b/modules/restic/go.mod index 27304285..d387e111 100644 --- a/modules/restic/go.mod +++ b/modules/restic/go.mod @@ -2,4 +2,13 @@ module restic go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029221656-196c5b6af277 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029223737-9f0bb1cfa4ba + +require ( + github.com/klauspost/compress v1.20.0 // indirect + github.com/nats-io/nats.go v1.54.0 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect +) diff --git a/modules/restic/go.sum b/modules/restic/go.sum index fc894705..360dc422 100644 --- a/modules/restic/go.sum +++ b/modules/restic/go.sum @@ -1,2 +1,26 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029221656-196c5b6af277 h1:ajjsH4MWcx45MGYASjUSRQKhLzEringuk3TA0oxZRCU= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029221656-196c5b6af277/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029223737-9f0bb1cfa4ba h1:4By9fDPDDiJXX32ySd6+WjAMAIX9vkVTT+Zs9+vhcLk= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029223737-9f0bb1cfa4ba/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/restic/module.json b/modules/restic/module.json index df58a37b..05927fe2 100644 --- a/modules/restic/module.json +++ b/modules/restic/module.json @@ -139,12 +139,7 @@ "MESH_BACKUP_DATA": "${dir:state}/data.conf" }, "health": { - "kind": "tool", - "tool": "backup_health", - "interval": "60s", - "timeout": "20s", - "looks": 2, - "grace": "60s" + "kind": "unit" } }, { @@ -165,7 +160,12 @@ "MESH_BACKUP_DATA": "${dir:state}/data.conf" }, "health": { - "kind": "unit" + "kind": "tool", + "tool": "backup_health", + "interval": "60s", + "timeout": "20s", + "looks": 2, + "grace": "60s" } } ],