restic: judge the night by its health tool again beside an awake process, and keep each item's last good night as its own state good (issue 1955, ADR 0571)

modules/restic/cmd/restic-backups/data_test.go

@@ -78,6 +78,14 @@ func TestEveryItemIsSaidWithItsReadingAndItsLastGoodBackup(t *testing.T) {
 	if dumps == nil || dumps.LastBackup != nil || dumps.SizeBytes != nil {
 		t.Fatalf("an item with no reading or no backup is said with one: %+v", dumps)
 	}
+	// The same last good night is kept on the bus beside good.json, one row per item a backup path covers.
+	b.Node = "ace"
+	rows, err := b.GoodRows()
+	must(t, err)
+	row, ok := rows[GoodKey("ace", "postgres", "store")]
+	if len(rows) != 1 || !ok || !row.LastGood.Equal(night) || row.CoveredBy != "/var/lib/mesh-store/dumps" || row.V != GoodVersion {
+		t.Fatalf("the good rows are %+v", rows)
+	}
 }
 
 // A retired item is deleted only after a last restore point of it, tagged as retired; a restore point

modules/restic/cmd/restic-backups/good.go

@@ -0,0 +1,98 @@
+package main
+
+// **Each item's last good night, on the bus** (novox/hq ADR 0571 clause 3, ADR 0564 §6): beside good.json, the module's
+// own state `good`, one key `<node>.<module>.<item>` for every item a module declares that a backup path covers, written
+// when it changes, so the self-check's probe of a stale backup reads it there and no part reads the controller's records.
+
+import (
+	"encoding/json"
+	"time"
+
+	stdio "git.novox.be/novox/mesh-sdk/go"
+)
+
+// GoodState is the state, GoodVersion the layout's version.
+const (
+	GoodState   = "good"
+	GoodVersion = 1
+)
+
+// GoodRow is one key: the item, the path whose snapshot keeps it, and the newest good night of its module (zero before
+// the first). Moves to mesh-sdk's health package with the readers that share it.
+type GoodRow struct {
+	V         int       `json:"v"`
+	Node      string    `json:"node"`
+	Module    string    `json:"module"`
+	Item      string    `json:"item"`
+	CoveredBy string    `json:"covered_by"`
+	LastGood  time.Time `json:"last_good,omitempty"`
+}
+
+// GoodKey is a row's key.
+func GoodKey(node, module, item string) string { return node + "." + module + "." + item }
+
+// GoodRows is the rows of this machine: every item a module declares that a path of the module's backup covers, with
+// the module's newest good night.
+func (b *Backups) GoodRows() (map[string]GoodRow, error) {
+	items, err := b.Items()
+	if err != nil {
+		return nil, err
+	}
+	declared, err := b.Declared()
+	if err != nil {
+		return nil, err
+	}
+	paths := map[string][]string{}
+	for _, d := range declared {
+		paths[d.Module] = d.Paths
+	}
+	good := b.Good()
+	out := map[string]GoodRow{}
+	for module, its := range items {
+		for _, it := range its {
+			if it.Class == "cache" || it.CoveredBy == "" {
+				continue
+			}
+			covered := false
+			for _, p := range paths[module] {
+				covered = covered || p == it.CoveredBy
+			}
+			if !covered {
+				continue
+			}
+			out[GoodKey(b.Node, module, it.ID)] = GoodRow{V: GoodVersion, Node: b.Node, Module: module, Item: it.ID,
+				CoveredBy: it.CoveredBy, LastGood: good[module].UTC()}
+		}
+	}
+	return out, nil
+}
+
+// syncGood writes the rows that differ from what it last wrote, once a minute while it runs (good.json is a file of
+// the night's process, so a minute is how fast it can hear of a night); put is the state's write.
+func syncGood(b *Backups, put func(key string, row GoodRow) error) {
+	time.Sleep(20 * time.Second) // the runtime serves first
+	wrote := map[string][]byte{}
+	for {
+		rows, err := b.GoodRows()
+		if err != nil {
+			b.Say("the last good nights could not be read: %v", err)
+		}
+		for k, r := range rows {
+			raw, _ := json.Marshal(r)
+			if string(wrote[k]) == string(raw) {
+				continue
+			}
+			if err := put(k, r); err != nil {
+				b.Say("the last good night of %s was not put: %v", k, err)
+				continue
+			}
+			wrote[k] = raw
+		}
+		time.Sleep(time.Minute)
+	}
+}
+
+func busGood(key string, row GoodRow) error {
+	_, err := stdio.State(GoodState).Put(key, row)
+	return err
+}

modules/restic/cmd/restic-backups/health_test.go

@@ -263,7 +263,8 @@ func TestTheManifestRunsTheNightJudgedByItsHealthTool(t *testing.T) {
 	}
 	env := m.Build.Artifacts[0].Env
 	want := map[string]struct{ run, kind, tool string }{
-		"night": {"night", "unit", ""},
+		"night": {"night", "tool", "backup_health"},
+		"awake": {"awake", "unit", ""},
 	}
 	for _, r := range m.Resources {
 		w, ok := want[r["id"].(string)]

modules/restic/cmd/restic-backups/main.go

@@ -24,6 +24,7 @@ import (
 	"time"
 
 	stdio "git.novox.be/novox/mesh-sdk/go"
+	"git.novox.be/novox/mesh-sdk/go/announce"
 	"git.novox.be/novox/mesh-sdk/go/health"
 )
 
@@ -52,13 +53,20 @@ func main() {
 	switch way {
 	case "night":
 		nights(b)
+	case "awake":
+		counting(b)
 	case "":
+		if err := stdio.Keeps(announce.State{Name: GoodState}); err != nil {
+			say("%v", err)
+			os.Exit(1)
+		}
+		go syncGood(b, busGood)
 		if err := stdio.Serve("", tools(b)); err != nil {
 			say("%v", err)
 			os.Exit(1)
 		}
 	default:
-		say("%q is not a way to run; with no argument it serves its tools, with night it runs the nights", way)
+		say("%q is not a way to run; with no argument it serves its tools, with night it runs the nights, with awake it counts how long the machine is up since a night", way)
 		os.Exit(2)
 	}
 }
@@ -71,7 +79,6 @@ func nights(b *Backups) {
 	if h, err := strconv.Atoi(os.Getenv("MESH_BACKUP_HOUR")); err == nil && h >= 0 && h < 24 {
 		hour = h
 	}
-	go counting(b)
 	if missedANight(b.Nights(), time.Now()) {
 		time.Sleep(10 * time.Minute)
 		night(b)

modules/restic/module.json

@@ -138,6 +138,32 @@
         "MESH_BACKUP_STATE": "${dir:state}",
         "MESH_BACKUP_DATA": "${dir:state}/data.conf"
       },
+      "health": {
+        "kind": "tool",
+        "tool": "backup_health",
+        "interval": "60s",
+        "timeout": "20s",
+        "looks": 2,
+        "grace": "60s"
+      }
+    },
+    {
+      "id": "awake",
+      "type": "process",
+      "name": "restic-awake",
+      "artifact": "tools",
+      "run": [
+        "./restic-backups",
+        "awake"
+      ],
+      "user": "${machine:account}",
+      "env": {
+        "MESH_BACKUP_DECLARED": "${dir:state}/backups.conf",
+        "MESH_BACKUP_REPOSITORY": "${dir:repository}",
+        "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret",
+        "MESH_BACKUP_STATE": "${dir:state}",
+        "MESH_BACKUP_DATA": "${dir:state}/data.conf"
+      },
       "health": {
         "kind": "unit"
       }
@@ -172,5 +198,8 @@
   },
   "reads": [
     "disk-load.readings"
+  ],
+  "state": [
+    "good"
   ]
 }