diff --git a/modules/restic/cmd/restic-backups/data_test.go b/modules/restic/cmd/restic-backups/data_test.go index aee683b3..53f6d0ff 100644 --- a/modules/restic/cmd/restic-backups/data_test.go +++ b/modules/restic/cmd/restic-backups/data_test.go @@ -78,6 +78,14 @@ func TestEveryItemIsSaidWithItsReadingAndItsLastGoodBackup(t *testing.T) { if dumps == nil || dumps.LastBackup != nil || dumps.SizeBytes != nil { t.Fatalf("an item with no reading or no backup is said with one: %+v", dumps) } + // The same last good night is kept on the bus beside good.json, one row per item a backup path covers. + b.Node = "ace" + rows, err := b.GoodRows() + must(t, err) + row, ok := rows[GoodKey("ace", "postgres", "store")] + if len(rows) != 1 || !ok || !row.LastGood.Equal(night) || row.CoveredBy != "/var/lib/mesh-store/dumps" || row.V != GoodVersion { + t.Fatalf("the good rows are %+v", rows) + } } // A retired item is deleted only after a last restore point of it, tagged as retired; a restore point diff --git a/modules/restic/cmd/restic-backups/good.go b/modules/restic/cmd/restic-backups/good.go new file mode 100644 index 00000000..e6aa9e0f --- /dev/null +++ b/modules/restic/cmd/restic-backups/good.go @@ -0,0 +1,98 @@ +package main + +// **Each item's last good night, on the bus** (novox/hq ADR 0571 clause 3, ADR 0564 ยง6): beside good.json, the module's +// own state `good`, one key `..` for every item a module declares that a backup path covers, written +// when it changes, so the self-check's probe of a stale backup reads it there and no part reads the controller's records. + +import ( + "encoding/json" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// GoodState is the state, GoodVersion the layout's version. +const ( + GoodState = "good" + GoodVersion = 1 +) + +// GoodRow is one key: the item, the path whose snapshot keeps it, and the newest good night of its module (zero before +// the first). Moves to mesh-sdk's health package with the readers that share it. +type GoodRow struct { + V int `json:"v"` + Node string `json:"node"` + Module string `json:"module"` + Item string `json:"item"` + CoveredBy string `json:"covered_by"` + LastGood time.Time `json:"last_good,omitempty"` +} + +// GoodKey is a row's key. +func GoodKey(node, module, item string) string { return node + "." + module + "." + item } + +// GoodRows is the rows of this machine: every item a module declares that a path of the module's backup covers, with +// the module's newest good night. +func (b *Backups) GoodRows() (map[string]GoodRow, error) { + items, err := b.Items() + if err != nil { + return nil, err + } + declared, err := b.Declared() + if err != nil { + return nil, err + } + paths := map[string][]string{} + for _, d := range declared { + paths[d.Module] = d.Paths + } + good := b.Good() + out := map[string]GoodRow{} + for module, its := range items { + for _, it := range its { + if it.Class == "cache" || it.CoveredBy == "" { + continue + } + covered := false + for _, p := range paths[module] { + covered = covered || p == it.CoveredBy + } + if !covered { + continue + } + out[GoodKey(b.Node, module, it.ID)] = GoodRow{V: GoodVersion, Node: b.Node, Module: module, Item: it.ID, + CoveredBy: it.CoveredBy, LastGood: good[module].UTC()} + } + } + return out, nil +} + +// syncGood writes the rows that differ from what it last wrote, once a minute while it runs (good.json is a file of +// the night's process, so a minute is how fast it can hear of a night); put is the state's write. +func syncGood(b *Backups, put func(key string, row GoodRow) error) { + time.Sleep(20 * time.Second) // the runtime serves first + wrote := map[string][]byte{} + for { + rows, err := b.GoodRows() + if err != nil { + b.Say("the last good nights could not be read: %v", err) + } + for k, r := range rows { + raw, _ := json.Marshal(r) + if string(wrote[k]) == string(raw) { + continue + } + if err := put(k, r); err != nil { + b.Say("the last good night of %s was not put: %v", k, err) + continue + } + wrote[k] = raw + } + time.Sleep(time.Minute) + } +} + +func busGood(key string, row GoodRow) error { + _, err := stdio.State(GoodState).Put(key, row) + return err +} diff --git a/modules/restic/cmd/restic-backups/health_test.go b/modules/restic/cmd/restic-backups/health_test.go index 8034fd7f..24bd7353 100644 --- a/modules/restic/cmd/restic-backups/health_test.go +++ b/modules/restic/cmd/restic-backups/health_test.go @@ -263,7 +263,8 @@ func TestTheManifestRunsTheNightJudgedByItsHealthTool(t *testing.T) { } env := m.Build.Artifacts[0].Env want := map[string]struct{ run, kind, tool string }{ - "night": {"night", "unit", ""}, + "night": {"night", "tool", "backup_health"}, + "awake": {"awake", "unit", ""}, } for _, r := range m.Resources { w, ok := want[r["id"].(string)] diff --git a/modules/restic/cmd/restic-backups/main.go b/modules/restic/cmd/restic-backups/main.go index ce5b7525..cdbb802a 100644 --- a/modules/restic/cmd/restic-backups/main.go +++ b/modules/restic/cmd/restic-backups/main.go @@ -24,6 +24,7 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/announce" "git.novox.be/novox/mesh-sdk/go/health" ) @@ -52,13 +53,20 @@ func main() { switch way { case "night": nights(b) + case "awake": + counting(b) case "": + if err := stdio.Keeps(announce.State{Name: GoodState}); err != nil { + say("%v", err) + os.Exit(1) + } + go syncGood(b, busGood) if err := stdio.Serve("", tools(b)); err != nil { say("%v", err) os.Exit(1) } default: - say("%q is not a way to run; with no argument it serves its tools, with night it runs the nights", way) + say("%q is not a way to run; with no argument it serves its tools, with night it runs the nights, with awake it counts how long the machine is up since a night", way) os.Exit(2) } } @@ -71,7 +79,6 @@ func nights(b *Backups) { if h, err := strconv.Atoi(os.Getenv("MESH_BACKUP_HOUR")); err == nil && h >= 0 && h < 24 { hour = h } - go counting(b) if missedANight(b.Nights(), time.Now()) { time.Sleep(10 * time.Minute) night(b) diff --git a/modules/restic/module.json b/modules/restic/module.json index 91a0eb4e..df58a37b 100644 --- a/modules/restic/module.json +++ b/modules/restic/module.json @@ -138,6 +138,32 @@ "MESH_BACKUP_STATE": "${dir:state}", "MESH_BACKUP_DATA": "${dir:state}/data.conf" }, + "health": { + "kind": "tool", + "tool": "backup_health", + "interval": "60s", + "timeout": "20s", + "looks": 2, + "grace": "60s" + } + }, + { + "id": "awake", + "type": "process", + "name": "restic-awake", + "artifact": "tools", + "run": [ + "./restic-backups", + "awake" + ], + "user": "${machine:account}", + "env": { + "MESH_BACKUP_DECLARED": "${dir:state}/backups.conf", + "MESH_BACKUP_REPOSITORY": "${dir:repository}", + "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret", + "MESH_BACKUP_STATE": "${dir:state}", + "MESH_BACKUP_DATA": "${dir:state}/data.conf" + }, "health": { "kind": "unit" } @@ -172,5 +198,8 @@ }, "reads": [ "disk-load.readings" + ], + "state": [ + "good" ] }