From dedd988fc929a7c9e92874721651e1287eb2657a Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:44:49 +0100 Subject: [PATCH] Export a bus in memory and a mesh whose plan hands a module with a provision, so mesh-provisioner's whole test composes a real plan (issue 1954 H5) --- go/compose/frombus/frombustest/frombustest.go | 183 ++++++++++++++++ .../frombus/frombustest/frombustest_test.go | 26 +++ ...5aa5e6-mesh-provisioner-client.module.json | 60 ++++++ .../backups.conf | 2 + .../catalogue-c05aa5e6-restic.files/data.conf | 2 + .../retention.conf | 6 + .../catalogue-c05aa5e6-restic.module.json | 199 ++++++++++++++++++ 7 files changed, 478 insertions(+) create mode 100644 go/compose/frombus/frombustest/frombustest.go create mode 100644 go/compose/frombus/frombustest/frombustest_test.go create mode 100644 go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-mesh-provisioner-client.module.json create mode 100644 go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.files/backups.conf create mode 100644 go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.files/data.conf create mode 100644 go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.files/retention.conf create mode 100644 go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.module.json diff --git a/go/compose/frombus/frombustest/frombustest.go b/go/compose/frombus/frombustest/frombustest.go new file mode 100644 index 0000000..3bfc971 --- /dev/null +++ b/go/compose/frombus/frombustest/frombustest.go @@ -0,0 +1,183 @@ +// Package frombustest is a bus in memory for frombus's Reader, and a mesh on it whose composition hands a module to the +// provisioner with a provision to make, so a module that composes plans from the bus (mesh-provisioner, novox/hq ADR +// 0570 clause 2) tests against a real plan rather than one written by hand. The manifests are the catalogue's (mesh-catalog +// c05aa5e6: restic and the provisioner's client) as frombus's own tests read them, restic given one requirement more. +package frombustest + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "embed" + "encoding/json" + "errors" + "strings" + + "git.novox.be/novox/mesh-sdk/go/compose" + "git.novox.be/novox/mesh-sdk/go/compose/frombus" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +//go:embed testdata +var testdata embed.FS + +// Bus is a frombus.Reader over values held in memory. A credential asked is answered held when Held is set. +type Bus struct { + Placement map[string]delivery.Placement + Record map[string]delivery.NodeRecord + Module map[string]delivery.Module + Manifests map[string][]byte + Value map[string]map[string]any // by node.module + Own []compose.OwnSecretRow + Builds map[string]*delivery.BuilderBuild // by " " + Archives map[string][]byte // by digest + Serves map[string]map[string]any // another node's values, by node.module + Held bool +} + +var _ frombus.Reader = (*Bus)(nil) + +func (b *Bus) Placements(context.Context) (map[string]delivery.Placement, error) { + return b.Placement, nil +} +func (b *Bus) Records(context.Context) (map[string]delivery.NodeRecord, error) { return b.Record, nil } +func (b *Bus) Modules(context.Context) (map[string]delivery.Module, error) { return b.Module, nil } +func (b *Bus) VaultAuthority(context.Context) string { return "" } + +func (b *Bus) Manifest(_ context.Context, digest string) ([]byte, error) { + raw, ok := b.Manifests[digest] + if !ok { + return nil, errors.New("no such manifest") + } + return raw, nil +} + +func (b *Bus) Values(_ context.Context, node, module string) (map[string]any, error) { + return b.Value[node+"."+module], nil +} + +func (b *Bus) OwnSecrets(context.Context, string) ([]compose.OwnSecretRow, error) { return b.Own, nil } + +func (b *Bus) Credentials(_ context.Context, _ string, asConsumer, asProvider []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) { + var out []compose.Credential + var issued []compose.Issued + for _, k := range append(append([]compose.CredentialKey{}, asConsumer...), asProvider...) { + if b.Held { + out = append(out, compose.Credential{Provision: k.Provision, Consumer: k.Consumer, ConsumerModule: k.ConsumerModule, + Provider: k.Provider, Local: k.Local, State: compose.CredentialHeld, Sealed: `"box"`}) + issued = append(issued, compose.Issued{Provision: k.Provision, Consumer: k.Consumer, ConsumerModule: k.ConsumerModule, + Provider: k.Provider, Local: k.Local, ForProvider: `"box"`}) + } + } + return out, issued, nil +} + +func (b *Bus) Build(_ context.Context, module, fp string) (*delivery.BuilderBuild, error) { + return b.Builds[module+" "+fp], nil +} + +func (b *Bus) Archive(_ context.Context, _, digest string) ([]byte, error) { + body, ok := b.Archives[digest] + if !ok { + return nil, errors.New("the store holds no such blob") + } + return body, nil +} + +func (b *Bus) Served(_ context.Context, node, module string, keys []string) (map[string]any, error) { + out := map[string]any{} + for _, k := range keys { + if v, ok := b.Serves[node+"."+module][k]; ok { + out[k] = v + } + } + return out, nil +} + +// Node is the one node of the mesh Provisioned composes. +const Node = "n1" + +// Provisioned is one node, n1, that runs the artifact store, a postgres providing postgres-database, the provisioner's +// client and restic: restic built and handed to the provisioner, and requiring postgres-database. Its plans (frombus.Plans +// with Scope{Node: Node}) hand restic, whose plan names that provision with postgres on n1 as its provider. +func Provisioned() (*Bus, error) { + read := func(name string) ([]byte, error) { return testdata.ReadFile("testdata/" + name) } + resticRaw, err := read("catalogue-c05aa5e6-restic.module.json") + if err != nil { + return nil, err + } + var restic map[string]any + if err := json.Unmarshal(resticRaw, &restic); err != nil { + return nil, err + } + restic["requires"] = []string{"postgres-database"} + if resticRaw, err = json.Marshal(restic); err != nil { + return nil, err + } + clientRaw, err := read("catalogue-c05aa5e6-mesh-provisioner-client.module.json") + if err != nil { + return nil, err + } + store := []byte(`{"module":"store","version":"1","provides":[{"name":"artifact-store","scope":"mesh"}],` + + `"serves":{"artifact-store":{"port":5000}}}`) + postgres := []byte(`{"module":"postgres","version":"1","provides":[{"name":"postgres-database","scope":"mesh"}],` + + `"serves":{"postgres-database":{"port":5432}}}`) + + var packed bytes.Buffer + zw := gzip.NewWriter(&packed) + tw := tar.NewWriter(zw) + for _, f := range []string{"backups.conf", "data.conf", "retention.conf"} { + text, err := read("catalogue-c05aa5e6-restic.files/" + f) + if err != nil { + return nil, err + } + if err := tw.WriteHeader(&tar.Header{Name: "./" + f, Mode: 0o644, Size: int64(len(text)), Typeflag: tar.TypeReg}); err != nil { + return nil, err + } + if _, err := tw.Write(text); err != nil { + return nil, err + } + } + if err := tw.Close(); err != nil { + return nil, err + } + if err := zw.Close(); err != nil { + return nil, err + } + archive := delivery.ManifestDigest(packed.Bytes()) + digestOf := func(c string) string { return "sha256:" + strings.Repeat(c, 64) } + fps := FPs() + b := &Bus{ + Placement: map[string]delivery.Placement{Node: {Node: Node, + Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1", Account: "operator", AccountHome: "/home/operator"}, + Modules: []delivery.PlacedModule{{Module: "store"}, {Module: "postgres"}, {Module: "mesh-provisioner-client"}, {Module: "restic"}}}}, + Record: map[string]delivery.NodeRecord{Node: {Node: Node, TunnelKey: "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdGtleXRlc3Q=", + SealingKey: "sk", Contract: compose.HandedContract, Capabilities: []string{"systemd", "package-manager"}}}, + Module: map[string]delivery.Module{}, Manifests: map[string][]byte{}, + Own: []compose.OwnSecretRow{{Node: Node, Module: "mesh-provisioner-client", Name: "provisioner-broker", Origin: "made", SealedTo: "sk", Sealed: `"box"`}}, + Builds: map[string]*delivery.BuilderBuild{ + "restic " + fps["restic"]: {State: delivery.BuildDone, Manifest: delivery.ManifestDigest(resticRaw), Artifacts: []delivery.BuiltArtifact{ + {Name: "tools", Kind: "bundle", Reference: "artifact-store://restic/tools/blobs/" + digestOf("6"), Digest: digestOf("6")}, + {Name: "files", Kind: "archive", Reference: "artifact-store://restic/files/blobs/" + archive, Digest: archive}}}, + "mesh-provisioner-client " + fps["mesh-provisioner-client"]: {State: delivery.BuildDone, Manifest: delivery.ManifestDigest(clientRaw), + Artifacts: []delivery.BuiltArtifact{{Name: "client", Kind: "bundle", + Reference: "artifact-store://mesh-provisioner-client/client/blobs/" + digestOf("7"), Digest: digestOf("7")}}}, + "store " + fps["store"]: {State: delivery.BuildDone, Manifest: delivery.ManifestDigest(store)}, + "postgres " + fps["postgres"]: {State: delivery.BuildDone, Manifest: delivery.ManifestDigest(postgres)}, + }, + Archives: map[string][]byte{archive: packed.Bytes()}, + Held: true, + } + for module, raw := range map[string][]byte{"restic": resticRaw, "mesh-provisioner-client": clientRaw, "store": store, "postgres": postgres} { + b.Module[module] = delivery.Module{FP2: fps[module], Manifest: delivery.ManifestDigest(raw)} + b.Manifests[delivery.ManifestDigest(raw)] = raw + } + return b, nil +} + +// FPs are the fingerprints Provisioned's modules are built at. +func FPs() map[string]string { + return map[string]string{"restic": "src2:" + strings.Repeat("3", 64), "mesh-provisioner-client": "src2:" + strings.Repeat("4", 64), + "store": "src2:" + strings.Repeat("5", 64), "postgres": "src2:" + strings.Repeat("8", 64)} +} diff --git a/go/compose/frombus/frombustest/frombustest_test.go b/go/compose/frombus/frombustest/frombustest_test.go new file mode 100644 index 0000000..edb8388 --- /dev/null +++ b/go/compose/frombus/frombustest/frombustest_test.go @@ -0,0 +1,26 @@ +package frombustest + +import ( + "context" + "testing" + + "git.novox.be/novox/mesh-sdk/go/compose/frombus" +) + +// ADR 0570 clause 2 (issue 1954; a3's H5): the in-memory mesh composes a plan from the bus that hands restic to the +// provisioner and names its postgres-database, provided by postgres on n1. +func TestTheProvisionedMeshHandsAPlanWithItsProvision(t *testing.T) { + b, err := Provisioned() + if err != nil { + t.Fatal(err) + } + plans, err := frombus.Plans(context.Background(), b, frombus.Mesh{}, frombus.Scope{Node: Node}) + if err != nil { + t.Fatal(err) + } + restic, handed := plans["restic"] + if !handed || len(restic.Provisions) != 1 || restic.Provisions[0].Provision != "postgres-database" || + restic.Provisions[0].Provider != Node || restic.Provisions[0].ProviderModule != "postgres" { + t.Fatalf("the plans are %+v", plans) + } +} diff --git a/go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-mesh-provisioner-client.module.json b/go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-mesh-provisioner-client.module.json new file mode 100644 index 0000000..095128a --- /dev/null +++ b/go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-mesh-provisioner-client.module.json @@ -0,0 +1,60 @@ +{ + "module": "mesh-provisioner-client", + "version": "1", + "claims": [ + { + "name": "node-provisioner", + "scope": "node" + } + ], + "own-secrets": { + "provisioner-broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-provisioner-client", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-provisioner-client", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-provisioner-client" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-provisioner-client", + "place": ".", + "unknown": "program" + }, + { + "id": "client", + "type": "process", + "name": "mesh-provisioner-client", + "artifact": "client", + "run": [ + "./mesh-provisioner-client" + ], + "env": { + "MESH_PROVISIONER_CREDENTIAL": "${dir:state}/broker", + "MESH_PROVISIONER_STATE": "${dir:state}" + }, + "user": "mesh-provisioner-client", + "health": { + "kind": "unit" + } + } + ], + "build": { + "artifacts": [ + { + "name": "client", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-provisioner-client", + "binary": "mesh-provisioner-client" + } + ] + } +} diff --git a/go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.files/backups.conf b/go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.files/backups.conf new file mode 100644 index 0000000..2f23f90 --- /dev/null +++ b/go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.files/backups.conf @@ -0,0 +1,2 @@ +# What the modules on this machine back up, composed by the mesh (novox/hq to-be 43). Do not edit. +${backup:lines} \ No newline at end of file diff --git a/go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.files/data.conf b/go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.files/data.conf new file mode 100644 index 0000000..f19e849 --- /dev/null +++ b/go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.files/data.conf @@ -0,0 +1,2 @@ +# The data the modules on this machine declare, composed by the mesh (novox/hq ADR 0233). Do not edit. +${backup:data} \ No newline at end of file diff --git a/go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.files/retention.conf b/go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.files/retention.conf new file mode 100644 index 0000000..cd7954c --- /dev/null +++ b/go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.files/retention.conf @@ -0,0 +1,6 @@ +# How many restore points the night keeps, written by the mesh from this machine's settings keep-daily, +# keep-weekly, keep-monthly (novox/hq issue 416) and keep-hand-taken-days (novox/hq issue 453). Replaced on every push; change the settings, never this file. +daily=${config:keep-daily} +weekly=${config:keep-weekly} +monthly=${config:keep-monthly} +hand=${config:keep-hand-taken-days} diff --git a/go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.module.json b/go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.module.json new file mode 100644 index 0000000..ea5ec8b --- /dev/null +++ b/go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.module.json @@ -0,0 +1,199 @@ +{ + "module": "restic", + "version": "1", + "moved": true, + "claims": [ + { + "name": "node-backup", + "scope": "node", + "serves": [ + "backed-up", + "now", + "restore" + ] + } + ], + "emits": [ + "backed-up" + ], + "tools": [ + "backup_health", + "backup_delete_retired", + "backup_deleted" + ], + "own-secrets": { + "repository": "${dir:state}/repository.secret" + }, + "data": { + "own": [ + { + "id": "repository", + "path": "${dir:repository}", + "class": "rebuildable", + "backup": "none", + "why": "the restore points themselves: a copy of data kept elsewhere on this machine, never the only copy of anything; lost, the history goes and nothing live does" + }, + { + "id": "state", + "path": "${dir:state}", + "class": "cache", + "why": "what the last nights and measurements were; the next night writes it again" + } + ] + }, + "config": { + "keep-daily": { + "type": "number", + "description": "How many nightly restore points are kept, a count from 1 to 9999: two weeks undo a mistake noticed within a fortnight. Read by the night, which thins nothing while it does not read (novox/hq issue 416); lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 14 + }, + "keep-weekly": { + "type": "number", + "description": "How many weekly restore points are kept behind the dailies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 8 + }, + "keep-monthly": { + "type": "number", + "description": "How many monthly restore points are kept behind the weeklies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 6 + }, + "keep-hand-taken-days": { + "type": "number", + "description": "How many days a restore point taken by hand (node-backup.now, or the mesh's before a module moves) is kept at least, a count from 1 to 9999; it is kept until a later good night of its module exists, and this many days in any case, so the next thinning never takes the one taken against a risky change (novox/hq issue 453). Lowering it lets go of the older ones taken by hand at the next thinning, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 7 + } + }, + "resources": [ + { + "id": "state", + "type": "directory", + "mode": "0700", + "place": ".", + "unknown": "program" + }, + { + "id": "repository", + "type": "directory", + "mode": "0700", + "unknown": "program" + }, + { + "id": "declared", + "type": "file", + "path": "${dir:state}/backups.conf", + "mode": "0600", + "template": "files/backups.conf" + }, + { + "id": "data-declared", + "type": "file", + "path": "${dir:state}/data.conf", + "mode": "0600", + "template": "files/data.conf" + }, + { + "id": "retention", + "type": "file", + "path": "${dir:state}/retention.conf", + "mode": "0644", + "trusted": true, + "template": "files/retention.conf" + }, + { + "id": "tool", + "type": "package", + "package": "restic" + }, + { + "id": "sqlite", + "type": "package", + "package": "sqlite" + }, + { + "id": "night", + "type": "process", + "name": "restic-night", + "artifact": "tools", + "run": [ + "./restic-backups", + "night" + ], + "user": "${machine:account}", + "env": { + "MESH_BACKUP_DECLARED": "${dir:state}/backups.conf", + "MESH_BACKUP_REPOSITORY": "${dir:repository}", + "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret", + "MESH_BACKUP_STATE": "${dir:state}", + "MESH_BACKUP_DATA": "${dir:state}/data.conf" + }, + "health": { + "kind": "tool", + "tool": "backup_health", + "interval": "60s", + "timeout": "20s", + "looks": 2, + "grace": "60s" + } + }, + { + "id": "measure", + "type": "process", + "name": "restic-measure", + "artifact": "tools", + "run": [ + "./restic-backups", + "measure" + ], + "user": "${machine:account}", + "env": { + "MESH_BACKUP_DECLARED": "${dir:state}/backups.conf", + "MESH_BACKUP_REPOSITORY": "${dir:repository}", + "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret", + "MESH_BACKUP_STATE": "${dir:state}", + "MESH_BACKUP_DATA": "${dir:state}/data.conf" + }, + "health": { + "kind": "unit" + } + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/restic-backups", + "binary": "restic-backups", + "loads": [ + "restic-backups" + ], + "env": { + "MESH_BACKUP_DECLARED": "${dir:state}/backups.conf", + "MESH_BACKUP_REPOSITORY": "${dir:repository}", + "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret", + "MESH_BACKUP_STATE": "${dir:state}", + "MESH_BACKUP_DATA": "${dir:state}/data.conf" + } + }, + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + } +}