The node-backup seat: a module contributes its backup, the mesh fills its directories

ADR 0214 / to-be 43: a node seat whose holder keeps nightly restore points of what every module on
the machine declares. A contribution of kind backup may name its module's own directories, filled
per module when placed. The catalogue check refuses a store provider that contributes no backup;
parsing does not, so the providers already running stay readable.

internal/catalogue/backup_test.go

@@ -0,0 +1,72 @@
+package catalogue
+
+import (
+	"strings"
+	"testing"
+)
+
+// A store provider says how its data is backed up (novox/hq ADR 0214); a provider of something that
+// holds nothing does not have to.
+func TestAStoreProviderWithoutABackupIsRefusedByTheCheck(t *testing.T) {
+	bare, err := ParseManifest([]byte(`{"module":"pg","version":"1",
+		"provides":[{"name":"postgres-database","scope":"mesh"}]}`))
+	if err != nil {
+		t.Fatalf("parsing refused it, and the rule is the check's: %v", err)
+	}
+	if problems := CheckBackup(bare); len(problems) != 1 || !strings.Contains(problems[0], "node-backup") {
+		t.Fatalf("a store with no backup passed the check: %v", problems)
+	}
+	backed, err := ParseManifest([]byte(`{"module":"pg","version":"1",
+		"provides":[{"name":"postgres-database","scope":"mesh"}],
+		"resources":[{"id":"dumps","type":"directory","mode":"0700"}],
+		"contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:dumps}"}]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if problems := CheckBackup(backed); len(problems) != 0 {
+		t.Fatalf("a store that contributes a backup was refused: %v", problems)
+	}
+	route, _ := ParseManifest([]byte(`{"module":"r","version":"1","provides":[{"name":"route","scope":"mesh"}]}`))
+	if problems := CheckBackup(route); len(problems) != 0 {
+		t.Fatalf("a route was asked for a backup: %v", problems)
+	}
+}
+
+// A backup contribution names its module's own directories; one it does not declare is refused
+// where it is written rather than reaching the holder as the literal text.
+func TestABackupNamingAnUndeclaredDirectoryIsRefused(t *testing.T) {
+	_, err := ParseManifest([]byte(`{"module":"pg","version":"1",
+		"contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:nowhere}"}]}`))
+	if err == nil || !strings.Contains(err.Error(), "nowhere") {
+		t.Fatalf("a backup of an undeclared directory was accepted: %v", err)
+	}
+}
+
+// The holder receives every module's backup lines with each module's own directories filled, each
+// module's under a comment naming it — and a kind written in a shell's grammar is left untouched.
+func TestBackupContributionsArePlacedWithTheirModulesDirectories(t *testing.T) {
+	pg, err := ParseManifest([]byte(`{"module":"pg","version":"1",
+		"resources":[{"id":"dumps","type":"directory","path":"/srv/pg/dumps","mode":"0700"}],
+		"contributions":[{"seat":"node-backup","kind":"backup","content":"run pg-dump-all\npath ${dir:dumps}"}]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	mail, err := ParseManifest([]byte(`{"module":"mail","version":"1",
+		"resources":[{"id":"spool","type":"directory","mode":"0700"}],
+		"contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:spool}"}]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	placed, err := seatContributions([]Manifest{pg, mail}, BackupSeat, "backup", Rendering{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, want := range []string{"# pg\nrun pg-dump-all\npath /srv/pg/dumps\n", "# mail\npath /var/lib/mail/spool\n"} {
+		if !strings.Contains(placed, want) {
+			t.Errorf("placed contributions lack %q:\n%s", want, placed)
+		}
+	}
+	if strings.Contains(placed, "${dir:") {
+		t.Errorf("a directory reached the holder unfilled:\n%s", placed)
+	}
+}

internal/catalogue/catalogue_check_test.go

@@ -86,3 +86,36 @@ func TestNoCatalogueManifestNamesAnInstallation(t *testing.T) {
 		t.Fatalf("%d value(s) name an installation:\n  %s", len(named), strings.Join(named, "\n  "))
 	}
 }
+
+// TestEveryCatalogueStoreSaysHowItIsBackedUp is ADR 0214's check over the real catalogue: a module
+// providing a store contributes a backup to node-backup, so a store added is a store backed up.
+func TestEveryCatalogueStoreSaysHowItIsBackedUp(t *testing.T) {
+	root := catalogueRoot(t)
+	found, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
+	if err != nil || len(found) == 0 {
+		t.Fatalf("no manifests under %s: %v", root, err)
+	}
+	stores := 0
+	for _, p := range found {
+		raw, err := os.ReadFile(p)
+		if err != nil {
+			t.Fatalf("%s: %v", p, err)
+		}
+		m, err := ParseManifest(raw)
+		if err != nil {
+			continue // TestEveryCatalogueManifestParses says why
+		}
+		for _, o := range m.Provides {
+			if storeProvisions[o.Name] {
+				stores++
+				break
+			}
+		}
+		for _, problem := range CheckBackup(m) {
+			t.Error(problem)
+		}
+	}
+	if stores == 0 {
+		t.Fatal("no module in the catalogue provides a store, so this proved nothing")
+	}
+}

internal/catalogue/declaration.go

@@ -914,7 +914,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
 			// seat that places them (novox/hq ADR 0203, ADR 0204). Gathered from every module on
 			// the node, as the jails are, and **last of every placeholder pass**: shell code is a
 			// shell's own syntax, full of `${…}` no pass above should ever be shown.
-			if err := contributionsInto(copied, m, r.Modules, thisMachine); err != nil {
+			if err := contributionsInto(copied, m, r.Modules, thisMachine, with); err != nil {
 				return nil, err
 			}
 			copied["id"] = m.Module + "." + fmt.Sprint(resource["id"])

internal/catalogue/environment_into.go

@@ -547,7 +547,7 @@ func shellCode(modules []Manifest, shell, slot string) string {
 // `${machine:…}` some module wrote for its shell to see. So nothing runs after them, the environment
 // is filled before the shell's code is, and each is replaced in a single pass over what the holder
 // wrote, so a contributed piece is never scanned again.
-func contributionsInto(resource map[string]any, m Manifest, modules []Manifest, facts map[string]string) error {
+func contributionsInto(resource map[string]any, m Manifest, modules []Manifest, facts map[string]string, with Rendering) error {
 	if problems := append(placeholderProblems(m, resource), seatPlaceholderProblems(m, resource)...); len(problems) > 0 {
 		return fmt.Errorf("%s", problems[0])
 	}
@@ -571,14 +571,22 @@ func contributionsInto(resource map[string]any, m Manifest, modules []Manifest,
 	// the controller does not read, so neither may be scanned after the other is in place — a
 	// contributed line that happened to spell the other's placeholder would be filled.
 	if ofContributed.MatchString(content) {
+		var failed error
 		content = ofContributed.ReplaceAllStringFunc(content, func(placeholder string) string {
 			found := ofContributed.FindStringSubmatch(placeholder)
 			first, second, _ := strings.Cut(found[2], ":")
 			if found[1] == "contribution" {
-				return seatContributions(modules, first, second)
+				placed, err := seatContributions(modules, first, second, with)
+				if err != nil && failed == nil {
+					failed = err
+				}
+				return placed
 			}
 			return shellCode(modules, first, second)
 		})
+		if failed != nil {
+			return failed
+		}
 	}
 	resource["content"] = content
 	return nil

internal/catalogue/seat_contributions.go

@@ -20,6 +20,49 @@ const HotkeysSeat = "node-hotkeys"
 // MessageBusSeat is the machine's D-Bus (novox/hq ADR 0215).
 const MessageBusSeat = "node-message-bus"
 
+// BackupSeat is the machine's backups (novox/hq ADR 0214, to-be 43).
+const BackupSeat = "node-backup"
+
+// storeProvisions are the provisions whose provider keeps its consumers' data (novox/hq ADR 0214):
+// a module providing one must say how to back it up, or the data the mesh hands out is the data it
+// cannot restore — issue 241's seven databases. A provision that holds nothing worth keeping (a
+// route, a cache, a name) is not here; adding one is adding a store.
+var storeProvisions = map[string]bool{
+	"postgres-database": true,
+	"mssql-database":    true,
+	"mongodb-database":  true,
+	"s3-bucket":         true,
+	"influxdb-api":      true,
+	"secret":            true,
+}
+
+// CheckBackup is a store provider that contributes no backup (novox/hq ADR 0214, "How it is
+// checked").
+//
+// **The catalogue check's, not parsing's.** A manifest already registered and running was written
+// before the rule; refusing it on read would make the controller refuse the very providers whose
+// data the rule protects. New definitions meet it in the catalogue check, where they are written.
+func CheckBackup(m Manifest) []string {
+	backs := false
+	for _, c := range m.Contributions {
+		if s, known := SeatNamed(c.Seat); known && s.Name == BackupSeat && c.Kind == "backup" {
+			backs = true
+		}
+	}
+	if backs {
+		return nil
+	}
+	var problems []string
+	for _, o := range m.Provides {
+		if storeProvisions[o.Name] {
+			problems = append(problems, fmt.Sprintf(
+				"%s provides %s, which keeps its consumers' data, and contributes no backup to %s; a "+
+					"store says how its data is copied (novox/hq ADR 0214)", m.Module, o.Name, BackupSeat))
+		}
+	}
+	return problems
+}
+
 // SeatContribution is one piece of configuration a module gives a seat's holder to place.
 type SeatContribution struct {
 	// Seat is the seat whose holder places it.
@@ -66,7 +109,20 @@ func kindsOf(s Seat) string {
 func (m Manifest) seatContributionProblems() []string {
 	var problems []string
 	for i, c := range m.Contributions {
-		s, _, ok := receivable(c.Seat, c.Kind)
+		s, r, ok := receivable(c.Seat, c.Kind)
+		// A directory a contribution names must be one of this module's own, here rather than on the
+		// machine — where a `${dir:x}` nobody declared would reach the holder as the literal text.
+		if ok && r.Dirs {
+			declared := map[string]string{}
+			for _, res := range m.Resources {
+				if fmt.Sprint(res["type"]) == "directory" {
+					declared[fmt.Sprint(res["id"])] = ""
+				}
+			}
+			if _, err := dirFill(c.Content, declared, m.Module); err != nil {
+				problems = append(problems, fmt.Sprintf("%s's contribution %d: %v", m.Module, i+1, err))
+			}
+		}
 		switch {
 		case s.Name == "":
 			problems = append(problems, fmt.Sprintf(
@@ -129,12 +185,15 @@ func seatPlaceholderProblems(m Manifest, r map[string]any) []string {
 
 // seatContributions is every module's contribution of one kind to one seat (novox/hq ADR 0212 §3):
 // in module order, each module's pieces in the order it declared them, each module's preceded by a
-// comment line naming it in the tool's grammar, and empty when nothing is contributed.
-func seatContributions(modules []Manifest, seat, kind string) string {
+// comment line naming it in the tool's grammar, and empty when nothing is contributed. A kind that
+// takes directories has each contributor's `${dir:<id>}` filled with where that module's directory
+// is on this machine (novox/hq to-be 43).
+func seatContributions(modules []Manifest, seat, kind string, with Rendering) (string, error) {
 	s, r, ok := receivable(seat, kind)
 	if !ok {
-		return ""
+		return "", nil
 	}
+	var failed error
 	var b strings.Builder
 	for _, m := range inModuleOrder(modules) {
 		named := false
@@ -149,11 +208,19 @@ func seatContributions(modules []Manifest, seat, kind string) string {
 				fmt.Fprintf(&b, "%s %s\n", r.Comment, m.Module)
 				named = true
 			}
-			b.WriteString(c.Content)
-			if !strings.HasSuffix(c.Content, "\n") {
+			content := c.Content
+			if r.Dirs {
+				filled, err := dirFill(content, dirsFor(m, with), m.Module)
+				if err != nil && failed == nil {
+					failed = err
+				}
+				content = filled
+			}
+			b.WriteString(content)
+			if !strings.HasSuffix(content, "\n") {
 				b.WriteString("\n")
 			}
 		}
 	}
-	return b.String()
+	return b.String(), failed
 }

internal/catalogue/seats.go

@@ -54,6 +54,12 @@ type Seat struct {
 type Receivable struct {
 	Kind    string
 	Comment string
+	// Dirs says a contribution of this kind may name its contributor's own directories as
+	// `${dir:<id>}`, filled with where they are on the machine before the holder places it (novox/hq
+	// to-be 43): a module saying which of its data to back up names a directory the mesh placed, and
+	// only the mesh knows where. Off for every kind written in a tool's grammar that has its own
+	// `${…}` — a shell's — where the mesh filling one would change what the tool reads.
+	Dirs bool
 }
 
 // defaultSeats is the set the mesh ships with — the seed for the control plane's seat table and the
@@ -221,6 +227,13 @@ var defaultSeats = append([]Seat{
 	// system service, never restarts it live, and publishes curated events about it, never its traffic.
 	// It receives nothing yet: packages ship their own policies.
 	{Name: MessageBusSeat, Scope: ScopeNode, Decision: "novox/hq ADR 0215"},
+	// The machine's backups (novox/hq ADR 0214, to-be 43): its holder keeps nightly restore points of
+	// the data every module on the machine declares, on the machine, against mistakes rather than
+	// disasters. A module contributes `backup` lines — what to run to take a consistent copy, and
+	// which of its directories to keep.
+	{Name: BackupSeat, Scope: ScopeNode, Decision: "novox/hq ADR 0214",
+		Serves:   backupVerbs(),
+		Receives: []Receivable{{Kind: "backup", Comment: "#", Dirs: true}}},
 	// Deferred (novox/hq ADR 0121): renaming to mesh-private-network is a scope + server/client
 	// model change, not a rename, so it stays until that is built.
 	{Name: "the-private-network", Scope: ScopeNode, Decision: "novox/hq ADR 0110"},
@@ -537,6 +550,27 @@ func serviceManagerVerbs() []Verb {
 // loginShellVerbs is the contract every holder of node-login-shell serves (novox/hq ADR 0176, ADR
 // 0204): one command, run the way the operator's own terminal would run it, bounded below the
 // runtime's thirty-second call limit so a hung command answers rather than times the caller out.
+// backupVerbs is the node-backup seat's protocol (novox/hq to-be 43): what is kept, take one now,
+// and restore beside the live data — never over it.
+func backupVerbs() []Verb {
+	return []Verb{
+		{Name: "backed-up", Description: "What this machine backs up: each module, what it declared, " +
+			"its last good night, how many restore points are kept and the repository's size.",
+			Input: schema(map[string]string{"module": "one module (optional)"}, nil)},
+		{Name: "now", Description: "Take a backup now, of one module or of every module on this " +
+			"machine — before a migration, a retirement or anything else that could go wrong.",
+			Input: schema(map[string]string{"module": "one module (optional)"}, nil)},
+		{Name: "restore", Description: "Restore one module's data from a restore point BESIDE the live " +
+			"data, never over it: each directory as <path>.restored-<date>. Swapping it in is a " +
+			"person's act. Lists the restore points when none is named.",
+			Input: schema(map[string]string{
+				"module":   "the module",
+				"snapshot": "the restore point (from `backed-up`; the newest when omitted)",
+				"path":     "one of the module's directories (all of them when omitted)",
+			}, []string{"module"})},
+	}
+}
+
 func loginShellVerbs() []Verb {
 	return []Verb{
 		{Name: "execute", Description: "Run one command on this machine as the operator account, in a " +

internal/catalogue/seats_test.go

@@ -46,14 +46,14 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
 			delivered[s.Delivers] = s.Name
 		}
 	}
-	// Thirty-seven with node-message-bus (novox/hq ADR 0215); thirty-six with mesh-dns-resolver (novox/hq ADR 0194) and node-hosts-file (ADR 0199); thirty-four
+	// Thirty-eight with node-backup (novox/hq ADR 0214); thirty-seven with node-message-bus (novox/hq ADR 0215); thirty-six with mesh-dns-resolver (novox/hq ADR 0194) and node-hosts-file (ADR 0199); thirty-four
 	// with node-hotkeys (ADR 0212); thirty-three with node-power (ADR 0211); thirty-two since the
 	// graphical session's eleven (ADR 0208); twenty-one with node-package-manager and
 	// node-container-runtime (ADR 0207); nineteen with node-environment and node-login-shell (ADR 0203,
 	// ADR 0204); seventeen with node-build-agent (ADR 0190). Two fewer once the retired
 	// mesh-build-machine and node-dns-resolver rows go, when no registered manifest claims either.
-	if len(Seats()) != 37 {
-		t.Errorf("the mesh defines %d seats rather than 37; the set is closed, so a change here is "+
+	if len(Seats()) != 38 {
+		t.Errorf("the mesh defines %d seats rather than 38; the set is closed, so a change here is "+
 			"a decision (novox/hq ADR 0110): %s", len(Seats()), seatNames())
 	}
 }