From d25b69178b8c6d6419cb576e6a9a6d2edd351a7b Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 5 Oct 2026 11:42:40 +0200 Subject: [PATCH] The node-backup seat: a module contributes its backup, the mesh fills its directories ADR 0214 / to-be 43: a node seat whose holder keeps nightly restore points of what every module on the machine declares. A contribution of kind backup may name its module's own directories, filled per module when placed. The catalogue check refuses a store provider that contributes no backup; parsing does not, so the providers already running stay readable. --- internal/catalogue/backup_test.go | 72 +++++++++++++++++++ internal/catalogue/catalogue_check_test.go | 33 +++++++++ internal/catalogue/declaration.go | 2 +- internal/catalogue/environment_into.go | 12 +++- internal/catalogue/seat_contributions.go | 81 ++++++++++++++++++++-- internal/catalogue/seats.go | 34 +++++++++ internal/catalogue/seats_test.go | 6 +- 7 files changed, 227 insertions(+), 13 deletions(-) create mode 100644 internal/catalogue/backup_test.go diff --git a/internal/catalogue/backup_test.go b/internal/catalogue/backup_test.go new file mode 100644 index 0000000..0b46efc --- /dev/null +++ b/internal/catalogue/backup_test.go @@ -0,0 +1,72 @@ +package catalogue + +import ( + "strings" + "testing" +) + +// A store provider says how its data is backed up (novox/hq ADR 0214); a provider of something that +// holds nothing does not have to. +func TestAStoreProviderWithoutABackupIsRefusedByTheCheck(t *testing.T) { + bare, err := ParseManifest([]byte(`{"module":"pg","version":"1", + "provides":[{"name":"postgres-database","scope":"mesh"}]}`)) + if err != nil { + t.Fatalf("parsing refused it, and the rule is the check's: %v", err) + } + if problems := CheckBackup(bare); len(problems) != 1 || !strings.Contains(problems[0], "node-backup") { + t.Fatalf("a store with no backup passed the check: %v", problems) + } + backed, err := ParseManifest([]byte(`{"module":"pg","version":"1", + "provides":[{"name":"postgres-database","scope":"mesh"}], + "resources":[{"id":"dumps","type":"directory","mode":"0700"}], + "contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:dumps}"}]}`)) + if err != nil { + t.Fatal(err) + } + if problems := CheckBackup(backed); len(problems) != 0 { + t.Fatalf("a store that contributes a backup was refused: %v", problems) + } + route, _ := ParseManifest([]byte(`{"module":"r","version":"1","provides":[{"name":"route","scope":"mesh"}]}`)) + if problems := CheckBackup(route); len(problems) != 0 { + t.Fatalf("a route was asked for a backup: %v", problems) + } +} + +// A backup contribution names its module's own directories; one it does not declare is refused +// where it is written rather than reaching the holder as the literal text. +func TestABackupNamingAnUndeclaredDirectoryIsRefused(t *testing.T) { + _, err := ParseManifest([]byte(`{"module":"pg","version":"1", + "contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:nowhere}"}]}`)) + if err == nil || !strings.Contains(err.Error(), "nowhere") { + t.Fatalf("a backup of an undeclared directory was accepted: %v", err) + } +} + +// The holder receives every module's backup lines with each module's own directories filled, each +// module's under a comment naming it — and a kind written in a shell's grammar is left untouched. +func TestBackupContributionsArePlacedWithTheirModulesDirectories(t *testing.T) { + pg, err := ParseManifest([]byte(`{"module":"pg","version":"1", + "resources":[{"id":"dumps","type":"directory","path":"/srv/pg/dumps","mode":"0700"}], + "contributions":[{"seat":"node-backup","kind":"backup","content":"run pg-dump-all\npath ${dir:dumps}"}]}`)) + if err != nil { + t.Fatal(err) + } + mail, err := ParseManifest([]byte(`{"module":"mail","version":"1", + "resources":[{"id":"spool","type":"directory","mode":"0700"}], + "contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:spool}"}]}`)) + if err != nil { + t.Fatal(err) + } + placed, err := seatContributions([]Manifest{pg, mail}, BackupSeat, "backup", Rendering{}) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"# pg\nrun pg-dump-all\npath /srv/pg/dumps\n", "# mail\npath /var/lib/mail/spool\n"} { + if !strings.Contains(placed, want) { + t.Errorf("placed contributions lack %q:\n%s", want, placed) + } + } + if strings.Contains(placed, "${dir:") { + t.Errorf("a directory reached the holder unfilled:\n%s", placed) + } +} diff --git a/internal/catalogue/catalogue_check_test.go b/internal/catalogue/catalogue_check_test.go index 4b2bed4..06f578d 100644 --- a/internal/catalogue/catalogue_check_test.go +++ b/internal/catalogue/catalogue_check_test.go @@ -86,3 +86,36 @@ func TestNoCatalogueManifestNamesAnInstallation(t *testing.T) { t.Fatalf("%d value(s) name an installation:\n %s", len(named), strings.Join(named, "\n ")) } } + +// TestEveryCatalogueStoreSaysHowItIsBackedUp is ADR 0214's check over the real catalogue: a module +// providing a store contributes a backup to node-backup, so a store added is a store backed up. +func TestEveryCatalogueStoreSaysHowItIsBackedUp(t *testing.T) { + root := catalogueRoot(t) + found, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json")) + if err != nil || len(found) == 0 { + t.Fatalf("no manifests under %s: %v", root, err) + } + stores := 0 + for _, p := range found { + raw, err := os.ReadFile(p) + if err != nil { + t.Fatalf("%s: %v", p, err) + } + m, err := ParseManifest(raw) + if err != nil { + continue // TestEveryCatalogueManifestParses says why + } + for _, o := range m.Provides { + if storeProvisions[o.Name] { + stores++ + break + } + } + for _, problem := range CheckBackup(m) { + t.Error(problem) + } + } + if stores == 0 { + t.Fatal("no module in the catalogue provides a store, so this proved nothing") + } +} diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 096d439..fb64418 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -914,7 +914,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string, // seat that places them (novox/hq ADR 0203, ADR 0204). Gathered from every module on // the node, as the jails are, and **last of every placeholder pass**: shell code is a // shell's own syntax, full of `${…}` no pass above should ever be shown. - if err := contributionsInto(copied, m, r.Modules, thisMachine); err != nil { + if err := contributionsInto(copied, m, r.Modules, thisMachine, with); err != nil { return nil, err } copied["id"] = m.Module + "." + fmt.Sprint(resource["id"]) diff --git a/internal/catalogue/environment_into.go b/internal/catalogue/environment_into.go index 44d9711..52f662e 100644 --- a/internal/catalogue/environment_into.go +++ b/internal/catalogue/environment_into.go @@ -547,7 +547,7 @@ func shellCode(modules []Manifest, shell, slot string) string { // `${machine:…}` some module wrote for its shell to see. So nothing runs after them, the environment // is filled before the shell's code is, and each is replaced in a single pass over what the holder // wrote, so a contributed piece is never scanned again. -func contributionsInto(resource map[string]any, m Manifest, modules []Manifest, facts map[string]string) error { +func contributionsInto(resource map[string]any, m Manifest, modules []Manifest, facts map[string]string, with Rendering) error { if problems := append(placeholderProblems(m, resource), seatPlaceholderProblems(m, resource)...); len(problems) > 0 { return fmt.Errorf("%s", problems[0]) } @@ -571,14 +571,22 @@ func contributionsInto(resource map[string]any, m Manifest, modules []Manifest, // the controller does not read, so neither may be scanned after the other is in place — a // contributed line that happened to spell the other's placeholder would be filled. if ofContributed.MatchString(content) { + var failed error content = ofContributed.ReplaceAllStringFunc(content, func(placeholder string) string { found := ofContributed.FindStringSubmatch(placeholder) first, second, _ := strings.Cut(found[2], ":") if found[1] == "contribution" { - return seatContributions(modules, first, second) + placed, err := seatContributions(modules, first, second, with) + if err != nil && failed == nil { + failed = err + } + return placed } return shellCode(modules, first, second) }) + if failed != nil { + return failed + } } resource["content"] = content return nil diff --git a/internal/catalogue/seat_contributions.go b/internal/catalogue/seat_contributions.go index 247be91..fbd51b1 100644 --- a/internal/catalogue/seat_contributions.go +++ b/internal/catalogue/seat_contributions.go @@ -20,6 +20,49 @@ const HotkeysSeat = "node-hotkeys" // MessageBusSeat is the machine's D-Bus (novox/hq ADR 0215). const MessageBusSeat = "node-message-bus" +// BackupSeat is the machine's backups (novox/hq ADR 0214, to-be 43). +const BackupSeat = "node-backup" + +// storeProvisions are the provisions whose provider keeps its consumers' data (novox/hq ADR 0214): +// a module providing one must say how to back it up, or the data the mesh hands out is the data it +// cannot restore — issue 241's seven databases. A provision that holds nothing worth keeping (a +// route, a cache, a name) is not here; adding one is adding a store. +var storeProvisions = map[string]bool{ + "postgres-database": true, + "mssql-database": true, + "mongodb-database": true, + "s3-bucket": true, + "influxdb-api": true, + "secret": true, +} + +// CheckBackup is a store provider that contributes no backup (novox/hq ADR 0214, "How it is +// checked"). +// +// **The catalogue check's, not parsing's.** A manifest already registered and running was written +// before the rule; refusing it on read would make the controller refuse the very providers whose +// data the rule protects. New definitions meet it in the catalogue check, where they are written. +func CheckBackup(m Manifest) []string { + backs := false + for _, c := range m.Contributions { + if s, known := SeatNamed(c.Seat); known && s.Name == BackupSeat && c.Kind == "backup" { + backs = true + } + } + if backs { + return nil + } + var problems []string + for _, o := range m.Provides { + if storeProvisions[o.Name] { + problems = append(problems, fmt.Sprintf( + "%s provides %s, which keeps its consumers' data, and contributes no backup to %s; a "+ + "store says how its data is copied (novox/hq ADR 0214)", m.Module, o.Name, BackupSeat)) + } + } + return problems +} + // SeatContribution is one piece of configuration a module gives a seat's holder to place. type SeatContribution struct { // Seat is the seat whose holder places it. @@ -66,7 +109,20 @@ func kindsOf(s Seat) string { func (m Manifest) seatContributionProblems() []string { var problems []string for i, c := range m.Contributions { - s, _, ok := receivable(c.Seat, c.Kind) + s, r, ok := receivable(c.Seat, c.Kind) + // A directory a contribution names must be one of this module's own, here rather than on the + // machine — where a `${dir:x}` nobody declared would reach the holder as the literal text. + if ok && r.Dirs { + declared := map[string]string{} + for _, res := range m.Resources { + if fmt.Sprint(res["type"]) == "directory" { + declared[fmt.Sprint(res["id"])] = "" + } + } + if _, err := dirFill(c.Content, declared, m.Module); err != nil { + problems = append(problems, fmt.Sprintf("%s's contribution %d: %v", m.Module, i+1, err)) + } + } switch { case s.Name == "": problems = append(problems, fmt.Sprintf( @@ -129,12 +185,15 @@ func seatPlaceholderProblems(m Manifest, r map[string]any) []string { // seatContributions is every module's contribution of one kind to one seat (novox/hq ADR 0212 §3): // in module order, each module's pieces in the order it declared them, each module's preceded by a -// comment line naming it in the tool's grammar, and empty when nothing is contributed. -func seatContributions(modules []Manifest, seat, kind string) string { +// comment line naming it in the tool's grammar, and empty when nothing is contributed. A kind that +// takes directories has each contributor's `${dir:}` filled with where that module's directory +// is on this machine (novox/hq to-be 43). +func seatContributions(modules []Manifest, seat, kind string, with Rendering) (string, error) { s, r, ok := receivable(seat, kind) if !ok { - return "" + return "", nil } + var failed error var b strings.Builder for _, m := range inModuleOrder(modules) { named := false @@ -149,11 +208,19 @@ func seatContributions(modules []Manifest, seat, kind string) string { fmt.Fprintf(&b, "%s %s\n", r.Comment, m.Module) named = true } - b.WriteString(c.Content) - if !strings.HasSuffix(c.Content, "\n") { + content := c.Content + if r.Dirs { + filled, err := dirFill(content, dirsFor(m, with), m.Module) + if err != nil && failed == nil { + failed = err + } + content = filled + } + b.WriteString(content) + if !strings.HasSuffix(content, "\n") { b.WriteString("\n") } } } - return b.String() + return b.String(), failed } diff --git a/internal/catalogue/seats.go b/internal/catalogue/seats.go index a7ee871..39a35ef 100644 --- a/internal/catalogue/seats.go +++ b/internal/catalogue/seats.go @@ -54,6 +54,12 @@ type Seat struct { type Receivable struct { Kind string Comment string + // Dirs says a contribution of this kind may name its contributor's own directories as + // `${dir:}`, filled with where they are on the machine before the holder places it (novox/hq + // to-be 43): a module saying which of its data to back up names a directory the mesh placed, and + // only the mesh knows where. Off for every kind written in a tool's grammar that has its own + // `${…}` — a shell's — where the mesh filling one would change what the tool reads. + Dirs bool } // defaultSeats is the set the mesh ships with — the seed for the control plane's seat table and the @@ -221,6 +227,13 @@ var defaultSeats = append([]Seat{ // system service, never restarts it live, and publishes curated events about it, never its traffic. // It receives nothing yet: packages ship their own policies. {Name: MessageBusSeat, Scope: ScopeNode, Decision: "novox/hq ADR 0215"}, + // The machine's backups (novox/hq ADR 0214, to-be 43): its holder keeps nightly restore points of + // the data every module on the machine declares, on the machine, against mistakes rather than + // disasters. A module contributes `backup` lines — what to run to take a consistent copy, and + // which of its directories to keep. + {Name: BackupSeat, Scope: ScopeNode, Decision: "novox/hq ADR 0214", + Serves: backupVerbs(), + Receives: []Receivable{{Kind: "backup", Comment: "#", Dirs: true}}}, // Deferred (novox/hq ADR 0121): renaming to mesh-private-network is a scope + server/client // model change, not a rename, so it stays until that is built. {Name: "the-private-network", Scope: ScopeNode, Decision: "novox/hq ADR 0110"}, @@ -537,6 +550,27 @@ func serviceManagerVerbs() []Verb { // loginShellVerbs is the contract every holder of node-login-shell serves (novox/hq ADR 0176, ADR // 0204): one command, run the way the operator's own terminal would run it, bounded below the // runtime's thirty-second call limit so a hung command answers rather than times the caller out. +// backupVerbs is the node-backup seat's protocol (novox/hq to-be 43): what is kept, take one now, +// and restore beside the live data — never over it. +func backupVerbs() []Verb { + return []Verb{ + {Name: "backed-up", Description: "What this machine backs up: each module, what it declared, " + + "its last good night, how many restore points are kept and the repository's size.", + Input: schema(map[string]string{"module": "one module (optional)"}, nil)}, + {Name: "now", Description: "Take a backup now, of one module or of every module on this " + + "machine — before a migration, a retirement or anything else that could go wrong.", + Input: schema(map[string]string{"module": "one module (optional)"}, nil)}, + {Name: "restore", Description: "Restore one module's data from a restore point BESIDE the live " + + "data, never over it: each directory as .restored-. Swapping it in is a " + + "person's act. Lists the restore points when none is named.", + Input: schema(map[string]string{ + "module": "the module", + "snapshot": "the restore point (from `backed-up`; the newest when omitted)", + "path": "one of the module's directories (all of them when omitted)", + }, []string{"module"})}, + } +} + func loginShellVerbs() []Verb { return []Verb{ {Name: "execute", Description: "Run one command on this machine as the operator account, in a " + diff --git a/internal/catalogue/seats_test.go b/internal/catalogue/seats_test.go index 5adca77..ffca17b 100644 --- a/internal/catalogue/seats_test.go +++ b/internal/catalogue/seats_test.go @@ -46,14 +46,14 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) { delivered[s.Delivers] = s.Name } } - // Thirty-seven with node-message-bus (novox/hq ADR 0215); thirty-six with mesh-dns-resolver (novox/hq ADR 0194) and node-hosts-file (ADR 0199); thirty-four + // Thirty-eight with node-backup (novox/hq ADR 0214); thirty-seven with node-message-bus (novox/hq ADR 0215); thirty-six with mesh-dns-resolver (novox/hq ADR 0194) and node-hosts-file (ADR 0199); thirty-four // with node-hotkeys (ADR 0212); thirty-three with node-power (ADR 0211); thirty-two since the // graphical session's eleven (ADR 0208); twenty-one with node-package-manager and // node-container-runtime (ADR 0207); nineteen with node-environment and node-login-shell (ADR 0203, // ADR 0204); seventeen with node-build-agent (ADR 0190). Two fewer once the retired // mesh-build-machine and node-dns-resolver rows go, when no registered manifest claims either. - if len(Seats()) != 37 { - t.Errorf("the mesh defines %d seats rather than 37; the set is closed, so a change here is "+ + if len(Seats()) != 38 { + t.Errorf("the mesh defines %d seats rather than 38; the set is closed, so a change here is "+ "a decision (novox/hq ADR 0110): %s", len(Seats()), seatNames()) } }