Export a bus in memory and a mesh whose plan hands a module with a provision, so mesh-provisioner's whole test composes a real plan (issue 1954 H5)
This commit is contained in:
@@ -0,0 +1,183 @@
|
||||
// Package frombustest is a bus in memory for frombus's Reader, and a mesh on it whose composition hands a module to the
|
||||
// provisioner with a provision to make, so a module that composes plans from the bus (mesh-provisioner, novox/hq ADR
|
||||
// 0570 clause 2) tests against a real plan rather than one written by hand. The manifests are the catalogue's (mesh-catalog
|
||||
// c05aa5e6: restic and the provisioner's client) as frombus's own tests read them, restic given one requirement more.
|
||||
package frombustest
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"embed"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
//go:embed testdata
|
||||
var testdata embed.FS
|
||||
|
||||
// Bus is a frombus.Reader over values held in memory. A credential asked is answered held when Held is set.
|
||||
type Bus struct {
|
||||
Placement map[string]delivery.Placement
|
||||
Record map[string]delivery.NodeRecord
|
||||
Module map[string]delivery.Module
|
||||
Manifests map[string][]byte
|
||||
Value map[string]map[string]any // by node.module
|
||||
Own []compose.OwnSecretRow
|
||||
Builds map[string]*delivery.BuilderBuild // by "<module> <fp>"
|
||||
Archives map[string][]byte // by digest
|
||||
Serves map[string]map[string]any // another node's values, by node.module
|
||||
Held bool
|
||||
}
|
||||
|
||||
var _ frombus.Reader = (*Bus)(nil)
|
||||
|
||||
func (b *Bus) Placements(context.Context) (map[string]delivery.Placement, error) {
|
||||
return b.Placement, nil
|
||||
}
|
||||
func (b *Bus) Records(context.Context) (map[string]delivery.NodeRecord, error) { return b.Record, nil }
|
||||
func (b *Bus) Modules(context.Context) (map[string]delivery.Module, error) { return b.Module, nil }
|
||||
func (b *Bus) VaultAuthority(context.Context) string { return "" }
|
||||
|
||||
func (b *Bus) Manifest(_ context.Context, digest string) ([]byte, error) {
|
||||
raw, ok := b.Manifests[digest]
|
||||
if !ok {
|
||||
return nil, errors.New("no such manifest")
|
||||
}
|
||||
return raw, nil
|
||||
}
|
||||
|
||||
func (b *Bus) Values(_ context.Context, node, module string) (map[string]any, error) {
|
||||
return b.Value[node+"."+module], nil
|
||||
}
|
||||
|
||||
func (b *Bus) OwnSecrets(context.Context, string) ([]compose.OwnSecretRow, error) { return b.Own, nil }
|
||||
|
||||
func (b *Bus) Credentials(_ context.Context, _ string, asConsumer, asProvider []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) {
|
||||
var out []compose.Credential
|
||||
var issued []compose.Issued
|
||||
for _, k := range append(append([]compose.CredentialKey{}, asConsumer...), asProvider...) {
|
||||
if b.Held {
|
||||
out = append(out, compose.Credential{Provision: k.Provision, Consumer: k.Consumer, ConsumerModule: k.ConsumerModule,
|
||||
Provider: k.Provider, Local: k.Local, State: compose.CredentialHeld, Sealed: `"box"`})
|
||||
issued = append(issued, compose.Issued{Provision: k.Provision, Consumer: k.Consumer, ConsumerModule: k.ConsumerModule,
|
||||
Provider: k.Provider, Local: k.Local, ForProvider: `"box"`})
|
||||
}
|
||||
}
|
||||
return out, issued, nil
|
||||
}
|
||||
|
||||
func (b *Bus) Build(_ context.Context, module, fp string) (*delivery.BuilderBuild, error) {
|
||||
return b.Builds[module+" "+fp], nil
|
||||
}
|
||||
|
||||
func (b *Bus) Archive(_ context.Context, _, digest string) ([]byte, error) {
|
||||
body, ok := b.Archives[digest]
|
||||
if !ok {
|
||||
return nil, errors.New("the store holds no such blob")
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func (b *Bus) Served(_ context.Context, node, module string, keys []string) (map[string]any, error) {
|
||||
out := map[string]any{}
|
||||
for _, k := range keys {
|
||||
if v, ok := b.Serves[node+"."+module][k]; ok {
|
||||
out[k] = v
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Node is the one node of the mesh Provisioned composes.
|
||||
const Node = "n1"
|
||||
|
||||
// Provisioned is one node, n1, that runs the artifact store, a postgres providing postgres-database, the provisioner's
|
||||
// client and restic: restic built and handed to the provisioner, and requiring postgres-database. Its plans (frombus.Plans
|
||||
// with Scope{Node: Node}) hand restic, whose plan names that provision with postgres on n1 as its provider.
|
||||
func Provisioned() (*Bus, error) {
|
||||
read := func(name string) ([]byte, error) { return testdata.ReadFile("testdata/" + name) }
|
||||
resticRaw, err := read("catalogue-c05aa5e6-restic.module.json")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var restic map[string]any
|
||||
if err := json.Unmarshal(resticRaw, &restic); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
restic["requires"] = []string{"postgres-database"}
|
||||
if resticRaw, err = json.Marshal(restic); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clientRaw, err := read("catalogue-c05aa5e6-mesh-provisioner-client.module.json")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
store := []byte(`{"module":"store","version":"1","provides":[{"name":"artifact-store","scope":"mesh"}],` +
|
||||
`"serves":{"artifact-store":{"port":5000}}}`)
|
||||
postgres := []byte(`{"module":"postgres","version":"1","provides":[{"name":"postgres-database","scope":"mesh"}],` +
|
||||
`"serves":{"postgres-database":{"port":5432}}}`)
|
||||
|
||||
var packed bytes.Buffer
|
||||
zw := gzip.NewWriter(&packed)
|
||||
tw := tar.NewWriter(zw)
|
||||
for _, f := range []string{"backups.conf", "data.conf", "retention.conf"} {
|
||||
text, err := read("catalogue-c05aa5e6-restic.files/" + f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := tw.WriteHeader(&tar.Header{Name: "./" + f, Mode: 0o644, Size: int64(len(text)), Typeflag: tar.TypeReg}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := tw.Write(text); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
archive := delivery.ManifestDigest(packed.Bytes())
|
||||
digestOf := func(c string) string { return "sha256:" + strings.Repeat(c, 64) }
|
||||
fps := FPs()
|
||||
b := &Bus{
|
||||
Placement: map[string]delivery.Placement{Node: {Node: Node,
|
||||
Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1", Account: "operator", AccountHome: "/home/operator"},
|
||||
Modules: []delivery.PlacedModule{{Module: "store"}, {Module: "postgres"}, {Module: "mesh-provisioner-client"}, {Module: "restic"}}}},
|
||||
Record: map[string]delivery.NodeRecord{Node: {Node: Node, TunnelKey: "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdGtleXRlc3Q=",
|
||||
SealingKey: "sk", Contract: compose.HandedContract, Capabilities: []string{"systemd", "package-manager"}}},
|
||||
Module: map[string]delivery.Module{}, Manifests: map[string][]byte{},
|
||||
Own: []compose.OwnSecretRow{{Node: Node, Module: "mesh-provisioner-client", Name: "provisioner-broker", Origin: "made", SealedTo: "sk", Sealed: `"box"`}},
|
||||
Builds: map[string]*delivery.BuilderBuild{
|
||||
"restic " + fps["restic"]: {State: delivery.BuildDone, Manifest: delivery.ManifestDigest(resticRaw), Artifacts: []delivery.BuiltArtifact{
|
||||
{Name: "tools", Kind: "bundle", Reference: "artifact-store://restic/tools/blobs/" + digestOf("6"), Digest: digestOf("6")},
|
||||
{Name: "files", Kind: "archive", Reference: "artifact-store://restic/files/blobs/" + archive, Digest: archive}}},
|
||||
"mesh-provisioner-client " + fps["mesh-provisioner-client"]: {State: delivery.BuildDone, Manifest: delivery.ManifestDigest(clientRaw),
|
||||
Artifacts: []delivery.BuiltArtifact{{Name: "client", Kind: "bundle",
|
||||
Reference: "artifact-store://mesh-provisioner-client/client/blobs/" + digestOf("7"), Digest: digestOf("7")}}},
|
||||
"store " + fps["store"]: {State: delivery.BuildDone, Manifest: delivery.ManifestDigest(store)},
|
||||
"postgres " + fps["postgres"]: {State: delivery.BuildDone, Manifest: delivery.ManifestDigest(postgres)},
|
||||
},
|
||||
Archives: map[string][]byte{archive: packed.Bytes()},
|
||||
Held: true,
|
||||
}
|
||||
for module, raw := range map[string][]byte{"restic": resticRaw, "mesh-provisioner-client": clientRaw, "store": store, "postgres": postgres} {
|
||||
b.Module[module] = delivery.Module{FP2: fps[module], Manifest: delivery.ManifestDigest(raw)}
|
||||
b.Manifests[delivery.ManifestDigest(raw)] = raw
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// FPs are the fingerprints Provisioned's modules are built at.
|
||||
func FPs() map[string]string {
|
||||
return map[string]string{"restic": "src2:" + strings.Repeat("3", 64), "mesh-provisioner-client": "src2:" + strings.Repeat("4", 64),
|
||||
"store": "src2:" + strings.Repeat("5", 64), "postgres": "src2:" + strings.Repeat("8", 64)}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package frombustest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||
)
|
||||
|
||||
// ADR 0570 clause 2 (issue 1954; a3's H5): the in-memory mesh composes a plan from the bus that hands restic to the
|
||||
// provisioner and names its postgres-database, provided by postgres on n1.
|
||||
func TestTheProvisionedMeshHandsAPlanWithItsProvision(t *testing.T) {
|
||||
b, err := Provisioned()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plans, err := frombus.Plans(context.Background(), b, frombus.Mesh{}, frombus.Scope{Node: Node})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
restic, handed := plans["restic"]
|
||||
if !handed || len(restic.Provisions) != 1 || restic.Provisions[0].Provision != "postgres-database" ||
|
||||
restic.Provisions[0].Provider != Node || restic.Provisions[0].ProviderModule != "postgres" {
|
||||
t.Fatalf("the plans are %+v", plans)
|
||||
}
|
||||
}
|
||||
Vendored
+60
@@ -0,0 +1,60 @@
|
||||
{
|
||||
"module": "mesh-provisioner-client",
|
||||
"version": "1",
|
||||
"claims": [
|
||||
{
|
||||
"name": "node-provisioner",
|
||||
"scope": "node"
|
||||
}
|
||||
],
|
||||
"own-secrets": {
|
||||
"provisioner-broker": "${dir:state}/broker"
|
||||
},
|
||||
"secrets-owner": "mesh-provisioner-client",
|
||||
"resources": [
|
||||
{
|
||||
"id": "account",
|
||||
"type": "user",
|
||||
"name": "mesh-provisioner-client",
|
||||
"shell": "/usr/bin/nologin",
|
||||
"home": "/var/lib/mesh-provisioner-client"
|
||||
},
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"owner": "mesh-provisioner-client",
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "client",
|
||||
"type": "process",
|
||||
"name": "mesh-provisioner-client",
|
||||
"artifact": "client",
|
||||
"run": [
|
||||
"./mesh-provisioner-client"
|
||||
],
|
||||
"env": {
|
||||
"MESH_PROVISIONER_CREDENTIAL": "${dir:state}/broker",
|
||||
"MESH_PROVISIONER_STATE": "${dir:state}"
|
||||
},
|
||||
"user": "mesh-provisioner-client",
|
||||
"health": {
|
||||
"kind": "unit"
|
||||
}
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "client",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-provisioner-client",
|
||||
"binary": "mesh-provisioner-client"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
# What the modules on this machine back up, composed by the mesh (novox/hq to-be 43). Do not edit.
|
||||
${backup:lines}
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
# The data the modules on this machine declare, composed by the mesh (novox/hq ADR 0233). Do not edit.
|
||||
${backup:data}
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
# How many restore points the night keeps, written by the mesh from this machine's settings keep-daily,
|
||||
# keep-weekly, keep-monthly (novox/hq issue 416) and keep-hand-taken-days (novox/hq issue 453). Replaced on every push; change the settings, never this file.
|
||||
daily=${config:keep-daily}
|
||||
weekly=${config:keep-weekly}
|
||||
monthly=${config:keep-monthly}
|
||||
hand=${config:keep-hand-taken-days}
|
||||
+199
@@ -0,0 +1,199 @@
|
||||
{
|
||||
"module": "restic",
|
||||
"version": "1",
|
||||
"moved": true,
|
||||
"claims": [
|
||||
{
|
||||
"name": "node-backup",
|
||||
"scope": "node",
|
||||
"serves": [
|
||||
"backed-up",
|
||||
"now",
|
||||
"restore"
|
||||
]
|
||||
}
|
||||
],
|
||||
"emits": [
|
||||
"backed-up"
|
||||
],
|
||||
"tools": [
|
||||
"backup_health",
|
||||
"backup_delete_retired",
|
||||
"backup_deleted"
|
||||
],
|
||||
"own-secrets": {
|
||||
"repository": "${dir:state}/repository.secret"
|
||||
},
|
||||
"data": {
|
||||
"own": [
|
||||
{
|
||||
"id": "repository",
|
||||
"path": "${dir:repository}",
|
||||
"class": "rebuildable",
|
||||
"backup": "none",
|
||||
"why": "the restore points themselves: a copy of data kept elsewhere on this machine, never the only copy of anything; lost, the history goes and nothing live does"
|
||||
},
|
||||
{
|
||||
"id": "state",
|
||||
"path": "${dir:state}",
|
||||
"class": "cache",
|
||||
"why": "what the last nights and measurements were; the next night writes it again"
|
||||
}
|
||||
]
|
||||
},
|
||||
"config": {
|
||||
"keep-daily": {
|
||||
"type": "number",
|
||||
"description": "How many nightly restore points are kept, a count from 1 to 9999: two weeks undo a mistake noticed within a fortnight. Read by the night, which thins nothing while it does not read (novox/hq issue 416); lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.",
|
||||
"minimum": 1,
|
||||
"maximum": 9999,
|
||||
"trusted": true,
|
||||
"default": 14
|
||||
},
|
||||
"keep-weekly": {
|
||||
"type": "number",
|
||||
"description": "How many weekly restore points are kept behind the dailies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.",
|
||||
"minimum": 1,
|
||||
"maximum": 9999,
|
||||
"trusted": true,
|
||||
"default": 8
|
||||
},
|
||||
"keep-monthly": {
|
||||
"type": "number",
|
||||
"description": "How many monthly restore points are kept behind the weeklies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.",
|
||||
"minimum": 1,
|
||||
"maximum": 9999,
|
||||
"trusted": true,
|
||||
"default": 6
|
||||
},
|
||||
"keep-hand-taken-days": {
|
||||
"type": "number",
|
||||
"description": "How many days a restore point taken by hand (node-backup.now, or the mesh's before a module moves) is kept at least, a count from 1 to 9999; it is kept until a later good night of its module exists, and this many days in any case, so the next thinning never takes the one taken against a risky change (novox/hq issue 453). Lowering it lets go of the older ones taken by hand at the next thinning, and that cannot be undone.",
|
||||
"minimum": 1,
|
||||
"maximum": 9999,
|
||||
"trusted": true,
|
||||
"default": 7
|
||||
}
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "repository",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "declared",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/backups.conf",
|
||||
"mode": "0600",
|
||||
"template": "files/backups.conf"
|
||||
},
|
||||
{
|
||||
"id": "data-declared",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/data.conf",
|
||||
"mode": "0600",
|
||||
"template": "files/data.conf"
|
||||
},
|
||||
{
|
||||
"id": "retention",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/retention.conf",
|
||||
"mode": "0644",
|
||||
"trusted": true,
|
||||
"template": "files/retention.conf"
|
||||
},
|
||||
{
|
||||
"id": "tool",
|
||||
"type": "package",
|
||||
"package": "restic"
|
||||
},
|
||||
{
|
||||
"id": "sqlite",
|
||||
"type": "package",
|
||||
"package": "sqlite"
|
||||
},
|
||||
{
|
||||
"id": "night",
|
||||
"type": "process",
|
||||
"name": "restic-night",
|
||||
"artifact": "tools",
|
||||
"run": [
|
||||
"./restic-backups",
|
||||
"night"
|
||||
],
|
||||
"user": "${machine:account}",
|
||||
"env": {
|
||||
"MESH_BACKUP_DECLARED": "${dir:state}/backups.conf",
|
||||
"MESH_BACKUP_REPOSITORY": "${dir:repository}",
|
||||
"MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret",
|
||||
"MESH_BACKUP_STATE": "${dir:state}",
|
||||
"MESH_BACKUP_DATA": "${dir:state}/data.conf"
|
||||
},
|
||||
"health": {
|
||||
"kind": "tool",
|
||||
"tool": "backup_health",
|
||||
"interval": "60s",
|
||||
"timeout": "20s",
|
||||
"looks": 2,
|
||||
"grace": "60s"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "measure",
|
||||
"type": "process",
|
||||
"name": "restic-measure",
|
||||
"artifact": "tools",
|
||||
"run": [
|
||||
"./restic-backups",
|
||||
"measure"
|
||||
],
|
||||
"user": "${machine:account}",
|
||||
"env": {
|
||||
"MESH_BACKUP_DECLARED": "${dir:state}/backups.conf",
|
||||
"MESH_BACKUP_REPOSITORY": "${dir:repository}",
|
||||
"MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret",
|
||||
"MESH_BACKUP_STATE": "${dir:state}",
|
||||
"MESH_BACKUP_DATA": "${dir:state}/data.conf"
|
||||
},
|
||||
"health": {
|
||||
"kind": "unit"
|
||||
}
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/restic-backups",
|
||||
"binary": "restic-backups",
|
||||
"loads": [
|
||||
"restic-backups"
|
||||
],
|
||||
"env": {
|
||||
"MESH_BACKUP_DECLARED": "${dir:state}/backups.conf",
|
||||
"MESH_BACKUP_REPOSITORY": "${dir:repository}",
|
||||
"MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret",
|
||||
"MESH_BACKUP_STATE": "${dir:state}",
|
||||
"MESH_BACKUP_DATA": "${dir:state}/data.conf"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "files",
|
||||
"kind": "archive",
|
||||
"from": "files"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user