novox/mesh-host
8
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request 'Raise genesis on the bus the mesh runs on, from one bundle (issue 146, ADR 0331)' (#96) from fix/146-genesis-on-the-bus into main

Browse Source
This commit was merged in pull request #96.
This commit is contained in:
mesh-admin
2026-10-12 21:11:52 +00:00
parent 8e95c1d9ed cb55a46400
commit 6fe82c61bd
22 changed files with 801 additions and 570 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/bootstrap/adopted.go
+8 -8
View File
@@ -106,11 +106,11 @@ type AdoptedRewrite struct {
}
// RewriteAdopted makes the produced bundle one for an adopted machine: the foundation's own filter
// taken out, and the mesh's guard put in its place, guarding on this node the store's port, the
// broker's management port and the broker's plaintext port. The last is published on every
// interface and the foundation's filter admits it from the private network only, so a found
// firewall that filters only incoming traffic would leave it reachable from anywhere (novox/hq ADR
// 0103). Every one is a port of a module genesis takes. The nftables package stays: the guard is loaded with it, and
// taken out, and the mesh's guard put in its place, guarding on this node the store's port. It is
// published on every interface and the foundation's filter admits it from the private network only,
// so a found firewall that filters only incoming traffic would leave it reachable from anywhere
// (novox/hq ADR 0103). The bus is reached from anywhere on purpose — a node enrols over it — and its
// monitoring port is published on loopback only, so neither is guarded. The nftables package stays: the guard is loaded with it, and
// installing a package loads no table. Openings are not the bundle's — the first push declares
// them, once there is a controller to derive them.
func RewriteAdopted(r *Rewritten, p FoundationPorts) (AdoptedRewrite, error) {
@@ -128,11 +128,11 @@ func RewriteAdopted(r *Rewritten, p FoundationPorts) (AdoptedRewrite, error) {
out.Removed = append(out.Removed, id)
}
out.Guarded = []int{p.Store, p.Management, p.AMQP}
out.Guarded = []int{p.Store}
var text bytes.Buffer
text.WriteString(",\n // The mesh's guard (novox/hq ADR 0100): this machine is adopted, so its own firewall\n" +
" // stays in force and the foundation's filter is not loaded. The guard only refuses: the\n" +
" // store's port and the broker's management and plaintext ports, except from the machine and\n" +
" // store's port, except from the machine and\n" +
" // the private network.")
for _, res := range guardResources(out.Guarded) {
var one bytes.Buffer
@@ -185,7 +185,7 @@ func (r Rewritten) declares(id string) bool {
// container runtime's configuration whole — and neither is anything the operator installs later.
var genesisTakes = map[string]bool{
RegistryModule: true, ControlPlaneModule: true, BuilderModule: true,
"postgres": true, "lavinmq": true, "mesh-vault": true, "mesh-catalog": true,
"postgres": true, BusModule: true, "mesh-vault": true, "mesh-catalog": true,
}
// takeIfAdopted takes one of genesis's own modules on an adopted node, once it is assigned and
internal/bootstrap/adopted_test.go
+4 -4
View File
@@ -70,7 +70,7 @@ func TestTheGuardRefusesOnlyWhatIsAddressedToThisMachine(t *testing.T) {
func TestAnAdoptedBundleLoadsNoDroppingTableAndExactlyTheGuard(t *testing.T) {
r := producedBundle(t)
p := FoundationPorts{Store: 5433, Management: 15673, AMQP: 5773}
p := FoundationPorts{Store: 5433}
if _, err := RewritePorts(&r, p, ""); err != nil {
t.Fatal(err)
}
@@ -101,9 +101,9 @@ func TestAnAdoptedBundleLoadsNoDroppingTableAndExactlyTheGuard(t *testing.T) {
if len(guards) != 1 {
t.Fatalf("%d guard table(s)", len(guards))
}
// The store's, the broker's plaintext and its management port: each one the filter admits
// from the private network only (novox/hq ADR 0103).
if !strings.Contains(guards[0].Content, "tcp dport { 5433, 5773, 15673 } drop") {
// The store's: the one foundation port the filter admits from the private network only
// (novox/hq ADR 0103). The bus is reached from anywhere, its monitoring only on loopback.
if !strings.Contains(guards[0].Content, "tcp dport 5433 drop") && !strings.Contains(guards[0].Content, "tcp dport { 5433 } drop") {
t.Errorf("the guard does not refuse this node's ports: %s", guards[0].Content)
}
if strings.Count(guards[0].Content, "accept") != 1 || !strings.Contains(guards[0].Content, "policy accept") {
internal/bootstrap/bootstrap.go
+5 -5
View File
@@ -503,7 +503,7 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
for _, c := range []struct {
what, path string
made bool
}{{"store superuser", StoreSuperuserFile, creds.StoreMade}, {"broker admin", BrokerAdminFile, creds.BrokerMade}} {
}{{"store superuser", StoreSuperuserFile, creds.StoreMade}, {"bus controller", BusControllerFile, creds.BusMade}} {
switch {
case c.made && o.DryRun:
say(fmt.Sprintf(" %-17s would be made and kept at %s (0600)", c.what, c.path))
@@ -513,9 +513,9 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
say(fmt.Sprintf(" %-17s already at %s — kept", c.what, c.path))
}
}
say(fmt.Sprintf(" credentials the template's bootstrap and guest are gone: %d store and %d broker "+
"connection(s) rewritten, the store reads its password from a file, the broker's admin is changed once it answers",
root.StoreURLs, root.BrokerURLs))
say(fmt.Sprintf(" credentials the template's bootstrap is gone: %d store and %d bus "+
"connection(s) rewritten, the store reads its password from a file, the bus is raised with the controller's account",
root.StoreURLs, root.BusURLs))
if rewritten.Renamed {
say(fmt.Sprintf(" control plane %s, renamed from %s",
rewritten.TempName, rewritten.WasCalled))
@@ -733,7 +733,7 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
}
// ---- broker ---------------------------------------------------------------------------
say("broker — the foundation's broker, adopted as the lavinmq module: one bus, not two")
say("broker — the foundation's bus, adopted as the nats module: one bus, not two")
if err := InstallBroker(ctx, o, permanentControl, rewritten.Declaration, say); err != nil {
return result, failed(StepBroker, err)
}
internal/bootstrap/control_test.go
+18 -16
View File
@@ -30,14 +30,13 @@ const theControlPlaneModule = `{
"inventory": "/var/lib/mesh/mesh-controller/inventory",
"identity": "/var/lib/mesh/mesh-controller/identity",
"licences": "/var/lib/mesh/mesh-controller/licences",
"broker": "/var/lib/mesh/mesh-controller/broker",
"broker-management": "/var/lib/mesh/mesh-controller/broker-management"
"bus": "/var/lib/mesh/mesh-controller/bus"
},
"resources": [
{"id": "mesh-state", "type": "directory", "path": "/var/lib/mesh/mesh-controller", "mode": "0700"},
{"id": "broker-env", "type": "file", "path": "/var/lib/mesh/mesh-controller/broker.env",
"mode": "0600",
"content": "MESH_BROKER_AMQP=${secret:broker}\nMESH_BROKER_MANAGEMENT=${secret:broker-management}\nMESH_BROKER_ADDRESS=${machine:at}:5671\n"},
"content": "MESH_BUS_NATS=${secret:bus}\nMESH_BROKER_ADDRESS=${machine:at}:5671\n"},
{"id": "server", "type": "container", "name": "mesh-controller",
"image": "` + builtImage + `",
"network": "host", "args": ["serve"],
@@ -133,11 +132,10 @@ func TestTheStoreConnectionsComeFromTheBundleThatMadeThem(t *testing.T) {
// them at /run/secrets. Matching on the path alone would find nothing and refuse a correct
// manifest, which is exactly the ordinary case in the catalogue.
for variable, secret := range map[string]string{
"MESH_STORE_INVENTORY": "inventory",
"MESH_STORE_IDENTITY": "identity",
"MESH_STORE_LICENCES": "licences",
"MESH_BROKER_AMQP": "broker",
"MESH_BROKER_MANAGEMENT": "broker-management",
"MESH_STORE_INVENTORY": "inventory",
"MESH_STORE_IDENTITY": "identity",
"MESH_STORE_LICENCES": "licences",
"MESH_BUS_NATS": "bus",
} {
if wanted[variable] != secret {
t.Errorf("%s would be accepted as %q, want %q", variable, wanted[variable], secret)
@@ -161,9 +159,10 @@ func TestTheStoreConnectionsComeFromTheBundleThatMadeThem(t *testing.T) {
if err != nil {
t.Fatal(err)
}
// Three stores and both halves of the broker: everything the foundation made and nothing else.
if len(delivered) != 5 {
t.Fatalf("%d values were delivered, and the foundation names five: %v",
// Three stores and the controller's account on the bus: everything the foundation made and
// nothing else.
if len(delivered) != 4 {
t.Fatalf("%d values were delivered, and the foundation names four: %v",
len(delivered), delivered)
}
for _, secret := range delivered {
@@ -177,12 +176,15 @@ func TestTheStoreConnectionsComeFromTheBundleThatMadeThem(t *testing.T) {
// secret in a mesh is one the mesh made; `secret accept` is only for what predates the mesh.
func TestASecretTheFoundationNeverMadeIsLeftToTheMesh(t *testing.T) {
extra := strings.Replace(theControlPlaneModule,
`"broker": "/var/lib/mesh/mesh-controller/broker",`,
`"broker": "/var/lib/mesh/mesh-controller/broker",
"something-new": "/var/lib/mesh/mesh-controller/something-new",`, 1)
`"bus": "/var/lib/mesh/mesh-controller/bus"`,
`"bus": "/var/lib/mesh/mesh-controller/bus",
"something-new": "/var/lib/mesh/mesh-controller/something-new"`, 1)
extra = strings.Replace(extra,
`"content": "MESH_BROKER_AMQP=${secret:broker}\n`,
`"content": "MESH_SOMETHING_NEW=${secret:something-new}\nMESH_BROKER_AMQP=${secret:broker}\n`, 1)
`"content": "MESH_BUS_NATS=${secret:bus}\n`,
`"content": "MESH_SOMETHING_NEW=${secret:something-new}\nMESH_BUS_NATS=${secret:bus}\n`, 1)
if extra == theControlPlaneModule {
t.Fatal("the fixture no longer says what this test adds to")
}
rewritten, err := Rewrite(theRealBundle(t), held)
if err != nil {
internal/bootstrap/enrol.go
+30 -1
View File
@@ -94,6 +94,13 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla
switch mine, err := identity.Load(where); {
case err == nil && mine.Node == o.Node:
say(" already enrolled " + o.Node + " — " + where)
// **Enrolled is not adopted** (novox/hq ADR 0331 §5). This run's apply wrote the bundle's
// user list again — the controller's entry and nobody else's — over the list placed when
// the machine enrolled. Until the bus is the nats module, that list is still the
// installer's to place, so it is placed again, before anything dials the bus as a node.
if err := placeUntilAdopted(ctx, control, say); err != nil {
return out, err
}
case err == nil:
return out, fmt.Errorf(
"this machine is already node %q and was asked to become %q.\n"+
@@ -159,7 +166,8 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla
// busAccounts and busContainer are where the installer's bundle raises the bus: the file its
// configuration includes, and the container that reads it. The installer raised them, so it is the
// one that knows them (examples/foundation-first-node-nats.lock).
// one that knows them (examples/foundation-first-node.lock). A bridge until the bus is adopted as the
// nats module, which receives the list in its declaration from then on (novox/hq ADR 0331 §5).
const (
busAccounts = "/var/lib/mesh-bus-conf/accounts.conf"
busContainer = "mesh-broker"
@@ -191,6 +199,27 @@ func placeTheBusUsers(ctx context.Context, control controlPlane, say func(string
return nil
}
// placeUntilAdopted places the bus's users unless the nats module has them now (novox/hq ADR 0331
// §5): from the bus's adoption on, the module receives the list in its declaration, and the
// installer's bridge is not run again.
//
// **Placed again rather than left alone.** The other way out — writing the bundle's list only
// where no list exists yet — needs a file the applier creates and then never reconciles, which no
// resource of a declaration is; and it would keep whatever list was there, while the controller's
// composition is the one that is right. So a re-run converges the file on what the controller says.
// The module is asked of the controller by name (`module list`), which is where adoption registers it.
func placeUntilAdopted(ctx context.Context, control controlPlane, say func(string)) error {
known, err := control.tell(ctx, "module", "list")
if err != nil {
return err
}
if mentions(known, BusModule) {
say(" bus users the " + BusModule + " module's — not placed by the installer")
return nil
}
return placeTheBusUsers(ctx, control, say)
}
// runTheHost makes sure something on this machine is listening to the mesh, and proves it.
//
// **The installer does not install the service, and says so.** A unit file is a packaging decision
internal/bootstrap/enrol_test.go
+60
View File
@@ -59,6 +59,9 @@ func TestAMachineThatHasAlreadyEnrolledIsNotEnrolledAgain(t *testing.T) {
switch {
case strings.Contains(joined, "node list"):
return "anchor here 01J0\n", nil
// The bus is the nats module already, so its users are the module's.
case strings.Contains(joined, "module list"):
return "nats 1\npostgres 1\n", nil
// An agent is running here too. Both halves are needed: enrolling makes the mesh hear from
// a machine once, so the first answer alone also describes a machine with no agent at all.
case name == "pgrep":
@@ -82,6 +85,11 @@ func TestAMachineThatHasAlreadyEnrolledIsNotEnrolledAgain(t *testing.T) {
t.Errorf("a token was issued for a machine that already holds an identity: %v",
runtime.commands)
}
// Nor are the bus's users placed: the bus is adopted, so they are the nats module's, and the
// installer's bridge is not run again (novox/hq ADR 0331 §5).
if runtime.ran("broker accounts") || runtime.ran("accounts.conf") {
t.Errorf("the bus's users were placed on a re-run: %v", runtime.commands)
}
if out.Agent != "already running" {
t.Errorf("the host agent is reported as %q", out.Agent)
}
@@ -95,6 +103,8 @@ func TestAMeshThatHasHeardFromAMachineWithNoAgentStartsOne(t *testing.T) {
runtime := &asked{answer: func(name string, args []string) (string, error) {
joined := strings.Join(args, " ")
switch {
case strings.Contains(joined, "module list"):
return "nats 1\n", nil // the bus is adopted: no users to place
case strings.Contains(joined, "node list"):
// Exactly what enrolling leaves behind, with nothing running.
return "anchor here 01J0\n", nil
@@ -159,6 +169,8 @@ func TestAHostThatIsRunningAndUnheardOfIsNotAnInstall(t *testing.T) {
runtime := &asked{answer: func(_ string, args []string) (string, error) {
joined := strings.Join(args, " ")
switch {
case strings.Contains(joined, "module list"):
return "nats 1\n", nil // the bus is adopted: no users to place
case strings.Contains(joined, "node list"):
// Enrolled, and never spoken.
return "anchor never spoken 01J0\n", nil
@@ -190,6 +202,8 @@ func TestAMachineWithNoHostServiceIsRefusedRatherThanGivenOne(t *testing.T) {
runtime := &asked{answer: func(_ string, args []string) (string, error) {
joined := strings.Join(args, " ")
switch {
case strings.Contains(joined, "module list"):
return "nats 1\n", nil // the bus is adopted: no users to place
case strings.Contains(joined, "node list"):
return "anchor never spoken 01J0\n", nil
case args[0] == "show":
@@ -314,3 +328,49 @@ func TestAFirstNodeIsLetOntoTheBusItRaised(t *testing.T) {
"(placed at %v, enrolled at %d):\n%s", placed, enrol, strings.Join(runtime.commands, "\n"))
}
}
// A re-run on a machine that enrolled and whose bus is not yet the nats module places the
// controller's list again (novox/hq ADR 0331 §5, review of mesh-host #96). The apply just wrote the
// bundle's list — the controller alone — over the one placed at enrolment; left so, the next time the
// bus re-reads it every node and the enrolment account are refused.
func TestARerunOnAnEnrolledMachineWhoseBusIsNotAdoptedPlacesTheUsersAgain(t *testing.T) {
runtime := &asked{answer: func(name string, args []string) (string, error) {
joined := strings.Join(args, " ")
switch {
case strings.Contains(joined, "node list"):
return "anchor here 01J0\n", nil
case strings.Contains(joined, "module list"):
return "mesh-controller 1\npostgres 1\n", nil
case strings.Contains(joined, "broker accounts"):
return "accounts {\n MESH { users = [ { user: \"node.anchor\" } ] }\n}\n", nil
case name == "pgrep":
return "4242\n", nil
case name == "sh":
return "", nil
}
return "", fmt.Errorf("unexpected: %s %v", name, args)
}}
out, err := Enrol(context.Background(), Options{
Node: "anchor", State: alreadyEnrolled(t, "anchor"), Timeout: time.Second,
Host: "/usr/local/bin/mesh-host", HostInBackground: true,
}, arch(t), controlPlane{container: "temp-mesh-controller", run: runtime.run, timeout: time.Second},
func(string) {})
if err != nil {
t.Fatal(err)
}
if out.Joined || runtime.ran("token issue") {
t.Errorf("a re-run enrolled again: %v", runtime.commands)
}
placed := false
for _, c := range runtime.commands {
if strings.HasPrefix(c, "sh -c") && strings.Contains(c, "node.anchor") &&
strings.Contains(c, "/var/lib/mesh-bus-conf/accounts.conf") && strings.Contains(c, "kill -s HUP mesh-broker") {
placed = true
}
}
if !placed {
t.Fatalf("the controller's list was not placed again on a re-run before adoption:\n%s",
strings.Join(runtime.commands, "\n"))
}
}
internal/bootstrap/phase3.go
+64 -18
View File
@@ -14,8 +14,8 @@ import (
// Genesis raises a store and a broker before any module system exists, because the control plane
// cannot ask provisioning for the store it keeps its own records in or the broker it is reached over
// (novox/hq ADR 0006). That leaves two servers behind: the foundation's, and a second one the
// `postgres`/`lavinmq` modules used to raise for other modules to use. This turns the foundation's
// own servers into those modules, so a mesh runs ONE postgres and ONE lavinmq — the control plane's
// `postgres` and bus modules used to raise for other modules to use. This turns the foundation's
// own servers into those modules, so a mesh runs ONE postgres and ONE bus — the control plane's
// contexts and the database of each module that asks for one, in the same server (WBS 3.1/3.2).
//
// **Adopted in place, not replaced.** The control plane is stateless and is swapped for a fresh
@@ -155,37 +155,83 @@ func deliverCredential(ctx context.Context, o Options, control controlPlane, mod
return nil
}
// InstallBroker makes the foundation's broker the `lavinmq` module, adopted in place — the same
// shape as InstallStore, for the same reasons. The administrator's password is the one genesis
// gave the image's default account (rootsecrets.go), carried in so the module's provisioner can
// reach the management API as it.
// BusModule is the module the foundation's bus is adopted as (novox/hq ADR 0331 §1). The previous
// broker's module, lavinmq, left the catalogue with the move to one bus (ADR 0131).
const BusModule = "nats"
// InstallBroker makes the foundation's bus the `nats` module, adopted in place — the same shape as
// InstallStore, for the same reasons: the bus holds the mesh's streams, so it is kept, and from the
// module's first push the bus's user list reaches it in the module's declaration like any module's
// (ADR 0331 §1, §5). The installer's own placing of that list (enrol.go placeTheBusUsers) was the
// bridge to here and is not run again.
//
// **What is compared is the container's name, not its image.** The store's module pins the upstream
// postgres the foundation runs; the bus's module builds its server image (it carries the mesh's
// snapshot tool), so no pinned reference can be equal to the foundation's. A module whose server is
// not the foundation's container by name would raise a second bus beside the first, and is refused
// by name before anything is registered.
func InstallBroker(ctx context.Context, o Options, control controlPlane,
foundation *declaration.Declaration, say func(string)) error {
const module = "lavinmq"
const module = BusModule
manifest, err := readManifest(o.Catalogue, module)
if err != nil {
return err
}
broker, err := brokerIn(foundation)
bus, err := brokerIn(foundation)
if err != nil {
return err
}
if err := serverMatchesFoundation(manifest, broker, module); err != nil {
if err := busServerIsTheFoundations(manifest, bus, module); err != nil {
return err
}
say(" adopting " + broker.Name + " — the broker the foundation raised, unchanged")
if err := installProvider(ctx, o, control, module, manifest, "(the provisioner; the server is adopted, not built)",
func() error {
return deliverCredential(ctx, o, control, module, "admin", BrokerAdminFile, "the broker's administrator", say)
},
say); err != nil {
say(" adopting " + bus.Name + " — the bus the foundation raised, its streams kept")
if err := installProvider(ctx, o, control, module, manifest, "(the server, adopting the foundation's bus)",
nil, say); err != nil {
return err
}
say(" adopted " + broker.Name + " — the foundation's broker is now the " + module + " module")
say(" adopted " + bus.Name + " — the foundation's bus is now the " + module +
" module, and its user list is the module's from here on")
return nil
}
// busServerIsTheFoundations checks that the bus module's server container is the one the
// foundation raised, by name, so the applier takes it over rather than raising a second bus.
func busServerIsTheFoundations(manifest []byte, bus *declaration.Container, module string) error {
var m struct {
Resources []struct {
Type string `json:"type"`
Name string `json:"name"`
} `json:"resources"`
}
if err := json.Unmarshal(manifest, &m); err != nil {
return fmt.Errorf("the %s module's manifest is not readable: %w", module, err)
}
var names []string
for _, r := range m.Resources {
if r.Type != "container" {
continue
}
if r.Name == bus.Name {
return nil
}
names = append(names, r.Name)
}
return fmt.Errorf(
"the %s module's server is %s, and the foundation's bus is the container %q.\n"+
"The module adopts the foundation's bus in place, so its server has to carry the name the "+
"foundation raised; under another name it would raise a second bus beside the first, and "+
"the mesh would run two (novox/hq ADR 0331 §1). Nothing has been registered",
module, describeNames(names), bus.Name)
}
func describeNames(names []string) string {
if len(names) == 0 {
return "no container at all"
}
return "the container " + strings.Join(names, ", ")
}
// InstallVault installs the vault as a foundation module (novox/hq ADR 0085, amended). Nothing to
// adopt and nothing to carry in: it is its own runtime, built from the catalogue like any provider,
// and from its first push it keeps the export of every operator-sealed secret on its own disk.
@@ -207,9 +253,9 @@ func storeIn(d *declaration.Declaration) (*declaration.Container, error) {
return foundationContainer(d, StoreID, "store")
}
// brokerIn finds the broker container in the bundle this installer produced.
// brokerIn finds the bus's container in the bundle this installer produced.
func brokerIn(d *declaration.Declaration) (*declaration.Container, error) {
return foundationContainer(d, BrokerID, "broker")
return foundationContainer(d, BrokerID, "bus")
}
func foundationContainer(d *declaration.Declaration, id, what string) (*declaration.Container, error) {
internal/bootstrap/phase3_test.go
+93
View File
@@ -0,0 +1,93 @@
package bootstrap
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
)
// aCatalogue is a catalogue checkout holding these modules' manifests.
func aCatalogue(t *testing.T, manifests map[string]string) string {
t.Helper()
dir := t.TempDir()
for module, manifest := range manifests {
at := filepath.Join(dir, catalogueDir, module)
if err := os.MkdirAll(at, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(at, "module.json"), []byte(manifest), 0o644); err != nil {
t.Fatal(err)
}
}
return dir
}
// aBusModule is the nats module's manifest as far as adoption reads it: a server container by name.
func aBusModule(server string) string {
return `{"module": "nats", "version": "1",
"claims": [{"name": "mesh-broker", "scope": "mesh"}],
"bus-users": "/var/lib/nats-module/conf/accounts.conf",
"resources": [{"id": "server", "type": "container", "name": "` + server + `", "ports": ["4222:4222"], "artifact": "server"}],
"build": {"artifacts": [{"name": "server", "kind": "image", "from": "Dockerfile"}]}}`
}
func busStep(t *testing.T, catalogue string) (*asked, error) {
t.Helper()
rewritten, err := Rewrite(theRealBundle(t), held)
if err != nil {
t.Fatal(err)
}
runtime := &asked{answer: aMeshThatAgrees(nil)}
control := controlPlane{container: "mesh-controller", run: runtime.run, timeout: 0}
err = InstallBroker(context.Background(), installing(t, catalogue), control, rewritten.Declaration, func(string) {})
return runtime, err
}
// The broker step adopts the foundation's bus as the nats module (novox/hq ADR 0331 §1): the module
// the mesh goes on managing, registered from the catalogue — never lavinmq, which left it.
func TestTheBrokerStepAdoptsTheBusAsTheNatsModule(t *testing.T) {
catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule("mesh-broker")})
runtime, _ := busStep(t, catalogue)
if !runtime.ran("module add /nats-module.json") {
t.Fatalf("the nats module was not registered: %v", runtime.commands)
}
if runtime.ran("lavinmq") {
t.Errorf("the broker step still speaks of lavinmq: %v", runtime.commands)
}
}
// A catalogue that holds no nats module fails the step by name, before anything is registered —
// which is what today's step did for lavinmq, silently adopting a module that no longer exists.
func TestTheBrokerStepFailsForAModuleTheCatalogueDoesNotHold(t *testing.T) {
catalogue := aCatalogue(t, map[string]string{"lavinmq": `{"module": "lavinmq", "version": "1"}`})
runtime, err := busStep(t, catalogue)
if err == nil {
t.Fatal("a catalogue without the nats module was adopted from")
}
if !strings.Contains(err.Error(), "nats module") {
t.Errorf("the refusal does not name the module it looked for: %v", err)
}
if runtime.ran("module add") {
t.Errorf("something was registered: %v", runtime.commands)
}
}
// A bus module whose server is not the foundation's container by name would raise a second bus
// beside the first; it is refused naming both, before anything is registered.
func TestABusModuleWhoseServerIsNotTheFoundationsIsRefused(t *testing.T) {
catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule("mesh-broker-nats")})
runtime, err := busStep(t, catalogue)
if err == nil {
t.Fatal("a bus module with another server was adopted")
}
for _, named := range []string{"mesh-broker-nats", `"mesh-broker"`} {
if !strings.Contains(err.Error(), named) {
t.Errorf("the refusal does not name %s: %v", named, err)
}
}
if runtime.ran("module add") {
t.Errorf("something was registered: %v", runtime.commands)
}
}
internal/bootstrap/phase_packages.go
+2 -2
View File
@@ -144,8 +144,8 @@ func RaisePackageRegistry(ctx context.Context, o Options, d Deps, control contro
// recordTheForgesPort makes the port given for the package registry a setting of the module that
// serves it, on this node (novox/hq 04-ISSUES/085, ADR 0100).
//
// **The forge is the one foundation port that was not a setting.** The store's, the bus's, the
// broker's management port and the registry's each become a `ports` setting of the module that
// **The forge is the one foundation port that was not a setting.** The store's, the bus's and the
// registry's each become a `ports` setting of the module that
// binds them, set where that module is registered and assigned; the forge is raised by hand here,
// long before its module can be built, so there was no registration to hang it on and the number
// lived in rewritten manifest text instead. So the manifest is registered here — not assigned, and
internal/bootstrap/ports.go
+32 -29
View File
@@ -20,24 +20,21 @@ import (
// FoundationPorts are the machine's ports the foundation binds (novox/hq ADR 0100).
//
// **The node's, not the catalogue's.** A machine in use may already hold one — a predecessor's
// registry on 5000, its broker's management port — and a port fixed in the bundle and the manifests
// registry on 5000, its bus on 5671 — and a port fixed in the bundle and the manifests
// surfaces as a container that fails to bind, and one changed at genesis would be changed back when
// the foundation is adopted as modules. So each is an input here, checked free, rewritten into the
// bundle, and handed to the controller as that node's setting for the module that binds it.
type FoundationPorts struct {
Store int `json:"store"`
Bus int `json:"bus"`
AMQP int `json:"amqp"`
Management int `json:"management"`
Registry int `json:"registry"`
Packages int `json:"packages"`
Hub int `json:"hub"`
Store int `json:"store"`
Bus int `json:"bus"`
Registry int `json:"registry"`
Packages int `json:"packages"`
Hub int `json:"hub"`
}
// DefaultPorts are the catalogue's numbers.
func DefaultPorts() FoundationPorts {
return FoundationPorts{Store: 5432, Bus: 5671, AMQP: 5672, Management: 15672, Registry: 5000,
Packages: 3000, Hub: 51820}
return FoundationPorts{Store: 5432, Bus: 5671, Registry: 5000, Packages: 3000, Hub: 51820}
}
// DefaultOverlayRange is the controller's default private-network range.
@@ -47,7 +44,7 @@ const DefaultOverlayRange = "10.42.0.0/16"
func (p FoundationPorts) orDefaults() FoundationPorts {
d := DefaultPorts()
for _, f := range []struct{ got, def *int }{
{&p.Store, &d.Store}, {&p.Bus, &d.Bus}, {&p.AMQP, &d.AMQP}, {&p.Management, &d.Management},
{&p.Store, &d.Store}, {&p.Bus, &d.Bus},
{&p.Registry, &d.Registry}, {&p.Packages, &d.Packages}, {&p.Hub, &d.Hub},
} {
if *f.got == 0 {
@@ -60,8 +57,7 @@ func (p FoundationPorts) orDefaults() FoundationPorts {
// named is each port with what it is and its protocol, in a fixed order.
func (p FoundationPorts) named() []namedPort {
return []namedPort{
{"the store", "tcp", p.Store}, {"the bus", "tcp", p.Bus}, {"the broker's AMQP", "tcp", p.AMQP},
{"the broker's management", "tcp", p.Management}, {"the registry", "tcp", p.Registry},
{"the store", "tcp", p.Store}, {"the bus", "tcp", p.Bus}, {"the registry", "tcp", p.Registry},
{"the package registry", "tcp", p.Packages}, {"the private network's hub", "udp", p.Hub},
}
}
@@ -88,8 +84,11 @@ func (p FoundationPorts) Check() error {
}
// moduleSettings is what each foundation module is told about its ports on this node: the port it
// declares, to the machine's port it is given. Only what differs from the catalogue — a converged
// genesis on the defaults sets nothing, and so changes nothing it did before.
// declares, to the machine's port it is given. Only what differs from the catalogue's manifest is
// set. For the store, the registry and the forge the manifest declares the foundation's default, so
// a genesis on the defaults sets nothing for them. The bus is the exception: the nats module
// declares 4222 and the foundation publishes the bus on 5671, the port every enrolment token
// names, so the bus always carries a setting.
func (p FoundationPorts) moduleSettings() map[string]map[string]int {
d := DefaultPorts()
out := map[string]map[string]int{}
@@ -103,14 +102,18 @@ func (p FoundationPorts) moduleSettings() map[string]map[string]int {
out[module][strconv.Itoa(declared)] = given
}
add("postgres", d.Store, p.Store)
add("lavinmq", d.Bus, p.Bus)
add("lavinmq", d.AMQP, p.AMQP)
add("lavinmq", d.Management, p.Management)
// The bus's module declares the server's own port, and the foundation published the bus on the
// node's: adopted in place, it keeps the port every token already names (ADR 0331 §1).
add(BusModule, busModulePort, p.Bus)
add(RegistryModule, d.Registry, p.Registry)
add(ForgeModule, d.Packages, p.Packages)
return out
}
// busModulePort is the machine's port the nats module declares for the bus (mesh-catalog
// modules/nats, "4222:4222").
const busModulePort = 4222
// PortsSetting is the controller's settings key for a module's given ports.
const PortsSetting = "ports"
@@ -200,10 +203,10 @@ func RewritePorts(r *Rewritten, p FoundationPorts, overlayRange string) (PortsRe
if p.Store != d.Store {
replace(`"ports": ["5432:5432"]`, fmt.Sprintf(`"ports": ["%d:5432"]`, p.Store), "the store's published port")
}
if p.Bus != d.Bus || p.AMQP != d.AMQP || p.Management != d.Management {
replace(`"ports": ["5671:5671", "5672:5672", "127.0.0.1:15672:15672"]`,
fmt.Sprintf(`"ports": ["%d:5671", "%d:5672", "127.0.0.1:%d:15672"]`, p.Bus, p.AMQP, p.Management),
"the broker's published ports")
if p.Bus != d.Bus {
replace(`"ports": ["5671:4222", "127.0.0.1:8222:8222"]`,
fmt.Sprintf(`"ports": ["%d:4222", "127.0.0.1:8222:8222"]`, p.Bus),
"the bus's published ports")
}
if err != nil {
return out, err
@@ -221,9 +224,9 @@ func RewritePorts(r *Rewritten, p FoundationPorts, overlayRange string) (PortsRe
value := control.Env[key]
now := value
now = strings.ReplaceAll(now, "@127.0.0.1:5432/", fmt.Sprintf("@127.0.0.1:%d/", p.Store))
now = strings.ReplaceAll(now, "@127.0.0.1:5672/", fmt.Sprintf("@127.0.0.1:%d/", p.AMQP))
if strings.HasSuffix(now, "@127.0.0.1:15672") {
now = strings.TrimSuffix(now, "15672") + strconv.Itoa(p.Management)
// The bus, as the control plane dials it (MESH_BUS_NATS) on the machine's loopback.
if strings.HasSuffix(now, "@127.0.0.1:5671") {
now = strings.TrimSuffix(now, "5671") + strconv.Itoa(p.Bus)
}
if key == brokerAddressVar {
if host, port, splitErr := net.SplitHostPort(value); splitErr == nil && port == "5671" {
@@ -316,8 +319,8 @@ func PortsFree(ctx context.Context, run Runner, p FoundationPorts, ours func(rea
}
if len(problems) > 0 {
return fmt.Errorf("the foundation's ports must be free before anything is raised:\n - %s\n"+
"Give it another with the matching flag (--store-port, --bus-port, --amqp-port, "+
"--management-port, --registry-port, --packages-port, --hub-port); nothing was changed",
"Give it another with the matching flag (--store-port, --bus-port, "+
"--registry-port, --packages-port, --hub-port); nothing was changed",
strings.Join(problems, "\n - "))
}
return nil
@@ -504,8 +507,8 @@ func CheckTheMachine(ctx context.Context, o Options, run Runner, bundle *declara
if err := PortsFree(ctx, run, p, ours); err != nil {
return err
}
say(fmt.Sprintf(" ports free store %d, bus %d, amqp %d, management %d, registry %d, packages %d, hub %d/udp",
p.Store, p.Bus, p.AMQP, p.Management, p.Registry, p.Packages, p.Hub))
say(fmt.Sprintf(" ports free store %d, bus %d, registry %d, packages %d, hub %d/udp",
p.Store, p.Bus, p.Registry, p.Packages, p.Hub))
if o.Tunnel != "" {
// One tunnel and one range: the mesh's range IS the found tunnel's, so the rule that the
// two must not overlap applies only where a found tunnel is left running beside the mesh's
internal/bootstrap/ports_test.go
+10 -11
View File
@@ -31,7 +31,7 @@ func producedBundle(t *testing.T) Rewritten {
if err != nil {
t.Fatal(err)
}
if _, err := RewriteRoot(&r, RootCredentials{Store: "s", Broker: "b"}); err != nil {
if _, err := RewriteRoot(&r, testCredentials(t, "s", "b")); err != nil {
t.Fatal(err)
}
return r
@@ -60,7 +60,7 @@ func TestTheDefaultPortsLeaveTheBundleAsItWas(t *testing.T) {
func TestGivenPortsMoveOnlyTheMachinesSide(t *testing.T) {
r := producedBundle(t)
p := FoundationPorts{Store: 5433, Bus: 5771, AMQP: 5772, Management: 15673, Registry: 5100}
p := FoundationPorts{Store: 5433, Bus: 5771, Registry: 5100}
got, err := RewritePorts(&r, p, "10.77.0.0/16")
if err != nil {
t.Fatal(err)
@@ -72,9 +72,9 @@ func TestGivenPortsMoveOnlyTheMachinesSide(t *testing.T) {
if len(storeC.Ports) != 1 || storeC.Ports[0] != "5433:5432" {
t.Errorf("the store publishes %v", storeC.Ports)
}
broker := containerNamed(r.Declaration, "mesh-broker")
if strings.Join(broker.Ports, " ") != "5771:5671 5772:5672 127.0.0.1:15673:15672" {
t.Errorf("the broker publishes %v", broker.Ports)
bus := containerNamed(r.Declaration, "mesh-broker")
if strings.Join(bus.Ports, " ") != "5771:4222 127.0.0.1:8222:8222" {
t.Errorf("the bus publishes %v", bus.Ports)
}
control, err := controlPlaneIn(r.Declaration)
if err != nil {
@@ -85,9 +85,8 @@ func TestGivenPortsMoveOnlyTheMachinesSide(t *testing.T) {
t.Errorf("%s still dials %s", key, value)
}
}
if !strings.Contains(control.Env["MESH_BROKER_AMQP"], "@127.0.0.1:5772/") ||
!strings.HasSuffix(control.Env["MESH_BROKER_MANAGEMENT"], "@127.0.0.1:15673") {
t.Errorf("the broker is dialled at %s and %s", control.Env["MESH_BROKER_AMQP"], control.Env["MESH_BROKER_MANAGEMENT"])
if !strings.HasSuffix(control.Env[BusVar], "@127.0.0.1:5771") {
t.Errorf("the control plane dials the bus at %s", control.Env[BusVar])
}
if control.Env["MESH_BROKER_ADDRESS"] != "192.0.2.10:5771" || r.BrokerAddress != "192.0.2.10:5771" {
t.Errorf("nodes are told to dial %s (%s)", control.Env["MESH_BROKER_ADDRESS"], r.BrokerAddress)
@@ -169,20 +168,20 @@ func noneOurs(reachable.Reach) bool { return false }
func TestABusyPortIsRefusedNamingItsHolder(t *testing.T) {
m := machineRunner{
ss: "tcp LISTEN 0 4096 0.0.0.0:5000 0.0.0.0:* users:((\"docker-proxy\",pid=1,fd=7))\n" +
"tcp LISTEN 0 4096 127.0.0.1:15672 0.0.0.0:* users:((\"beam.smp\",pid=2,fd=7))\n",
"tcp LISTEN 0 4096 0.0.0.0:5671 0.0.0.0:* users:((\"beam.smp\",pid=2,fd=7))\n",
ps: "predecessor-registry\t0.0.0.0:5000->5000/tcp\n",
}
err := PortsFree(context.Background(), m.run, DefaultPorts(), noneOurs)
if err == nil {
t.Fatal("held ports were not refused")
}
for _, want := range []string{"predecessor-registry", "beam.smp", "tcp/5000", "tcp/15672", "--registry-port"} {
for _, want := range []string{"predecessor-registry", "beam.smp", "tcp/5000", "tcp/5671", "--registry-port"} {
if !strings.Contains(err.Error(), want) {
t.Errorf("the refusal does not say %q: %v", want, err)
}
}
p := DefaultPorts()
p.Registry, p.Management = 5100, 15673
p.Registry, p.Bus = 5100, 5771
if err := PortsFree(context.Background(), m.run, p, noneOurs); err != nil {
t.Errorf("other ports given and still refused: %v", err)
}
internal/bootstrap/preflight.go
+23 -6
View File
@@ -12,6 +12,28 @@ import (
"github.com/novox/mesh-host/internal/profile"
)
// CheckTemplate is what preflight asks of the bundle template before anything is changed: that it
// is a declaration, that it raises a control plane, and that it names the bus the way this
// installer fills it (novox/hq ADR 0331 §3, §4).
//
// Parsed here as well as at the rewrite, because a template that is not a declaration should cost
// a second rather than an image load and a written file. And a control plane given no bus never
// answers: the verify step would wait its whole timeout to say so, and a template like that is a
// fault in the bundle, said at once.
func CheckTemplate(template []byte) (*declaration.Declaration, error) {
parsed, err := declaration.ParseFileTrusted(template)
if err != nil {
return nil, fmt.Errorf("the bundle template is not a declaration: %w", err)
}
if _, err := controlPlaneIn(parsed); err != nil {
return nil, err
}
if err := BusPlaceholders(template, parsed); err != nil {
return nil, err
}
return parsed, nil
}
// DefaultRegistry is where an image reference that names no host comes from.
const DefaultRegistry = "registry-1.docker.io:443"
@@ -76,13 +98,8 @@ func Preflight(ctx context.Context, o Options, d Deps, say func(string)) ([]byte
"it. Point --bundle at one; mesh-host's examples/foundation-first-node.lock is "+
"the shape", err)
}
// Parsed here as well as at the rewrite, because a template that is not a declaration should
// cost a second rather than an image load and a written file.
parsed, err := declaration.ParseFileTrusted(template)
parsed, err := CheckTemplate(template)
if err != nil {
return nil, fmt.Errorf("the bundle template is not a declaration: %w", err)
}
if _, err := controlPlaneIn(parsed); err != nil {
return nil, err
}
say(fmt.Sprintf(" bundle template %s (%d resources)", o.Template, len(parsed.Resources)))
internal/bootstrap/preflight_test.go
+18
View File
@@ -149,3 +149,21 @@ func TestWhereAnImageWouldBeFetchedFrom(t *testing.T) {
}
}
}
// A template whose temporary control plane is given no bus is refused at preflight, before anything
// is changed, rather than raised and waited on at the verify step until its timeout (novox/hq issue
// 146, ADR 0331 §4): that control plane says it has no MESH_BUS_NATS every few seconds, for ever.
func TestATemplateWhoseControlPlaneHasNoBusIsRefusedBeforeAnythingIsRaised(t *testing.T) {
template := string(theRealBundle(t))
if _, err := CheckTemplate([]byte(template)); err != nil {
t.Fatalf("the shipped bundle is refused at preflight: %v", err)
}
const bus = `"MESH_BUS_NATS": "nats://controller:bootstrap@127.0.0.1:5671",`
if !strings.Contains(template, bus) {
t.Fatalf("the shipped bundle does not give its control plane %s", bus)
}
_, err := CheckTemplate([]byte(strings.Replace(template, bus, "", 1)))
if err == nil || !strings.Contains(err.Error(), BusVar) {
t.Fatalf("a control plane with no bus was let through preflight: %v", err)
}
}
internal/bootstrap/rewrite.go
+1 -1
View File
@@ -99,7 +99,7 @@ type Rewritten struct {
// migration pointing at an image no registry serves, and the apply would fail in the middle —
// after the store is up, before the broker. They are one image and they move together.
//
// **Third-party images are not touched.** postgres and lavinmq keep the `name@sha256:` references
// **Third-party images are not touched.** postgres and nats keep the `name@sha256:` references
// the template carries and are pulled from wherever those name (novox/hq ADR 0006). This is
// checked afterwards rather than merely intended: the produced bundle is re-parsed and every other
// container's image is compared against what it was.
internal/bootstrap/rewrite_test.go
+1 -1
View File
@@ -183,7 +183,7 @@ func TestTheProducedBundleKeepsTheTemplatesComments(t *testing.T) {
if err != nil {
t.Fatal(err)
}
const remembered = "NAMED VOLUME"
const remembered = "A filter before anything listens"
if !strings.Contains(string(out.Bundle), remembered) {
t.Errorf("the produced bundle lost the template's comments; %q is gone", remembered)
}
internal/bootstrap/rootsecrets.go
+185 -102
View File
@@ -4,49 +4,58 @@ import (
"bytes"
"context"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"fmt"
"os"
"path/filepath"
"strings"
"golang.org/x/crypto/bcrypt"
"github.com/novox/mesh-host/internal/declaration"
)
// The mesh's root credentials, made at genesis rather than copied from the template.
//
// **The template carries `bootstrap` and `guest`, and a mesh raised from it kept them** (novox/hq
// issue 071). The store's superuser and the broker's administrator are the two credentials every
// other one rests on, and they were the two that were not secret: constants in a file anybody can
// read, carried into the mesh by `secret accept` and marked as something the mesh must never
// replace — which is correct for a credential that already created the databases, and made the
// well-known value permanent.
// **The template carries `bootstrap`, and a mesh raised from it kept it** (novox/hq issue 071). The
// store's superuser and the controller's account on the bus are the two credentials every other one
// rests on, and they were the two that were not secret: constants in a file anybody can read,
// carried into the mesh by `secret accept` and marked as something the mesh must never replace —
// which is correct for a credential that already created the databases, and made the well-known
// value permanent.
//
// So the installer makes them. Two random values, **made once and kept on this machine** at the
// paths the postgres and lavinmq modules declare as their own secrets — so that when phase three
// adopts the store and the broker, `secret accept` carries in exactly the value the servers were
// raised with, and the host's later write of the sealed secret lands the same bytes in the same
// file. A second run finds the files and changes nothing, which is what lets the installer say
// "already done" about a store it must not restart.
// So the installer makes them. Two random values, **made once and kept on this machine** — the
// store's at the path the postgres module declares as its own secret, so that when phase three
// adopts the store `secret accept` carries in exactly the value the server was raised with; the
// bus's beside the other things only genesis holds. A second run finds the files and changes
// nothing, which is what lets the installer say "already done" about a store it must not restart.
//
// **The store reads its password from a file, not its environment.** `POSTGRES_PASSWORD` in a
// container's environment is in `docker inspect` for ever; the module that adopts the store
// declares the same file mount, so the two specs are one and the applier reconciles rather than
// recreates (phase3.go). The broker has no such file: its image's default administrator is changed
// in place by an action once the broker answers, and the produced bundle carries that action.
// recreates (phase3.go).
//
// **The bus has no administrator to change; it has a user list** (novox/hq ADR 0331 §3). The
// bundle carries the first list — the controller's own account at `bootstrap` — and the
// controller's address names the same account. Both are the bundle's placeholders: the address's
// `nats://controller:bootstrap@` and the controller's entry in the carried list. The installer
// puts the made password in the first and its bcrypt hash in the second, so the bus is raised with
// the account the controller dials with. The controller records a hash of what it actually uses
// (mesh-controller internal/broker, CredentialIn), so its own compositions carry the same account
// onward, and the permanent controller is given it by `secret accept` (control.go deliverStores).
const (
// StoreSuperuserFile is where the store's superuser password lives on the machine — the
// postgres module's own-secret path, so genesis and adoption write the same file.
StoreSuperuserFile = "/var/lib/postgres/superuser.secret"
// BrokerAdminFile is the same for the broker's administrator — the lavinmq module's.
BrokerAdminFile = "/var/lib/lavinmq-module/admin.secret"
// BrokerAdminUser is the broker's administrator. The image's default account, kept by name
// and given a password that is not the image's default; a renamed account would have to be
// created before anything can authenticate, and the thing that creates accounts is the thing
// that has to authenticate first.
BrokerAdminUser = "guest"
// BusControllerFile is where the controller's password on the bus lives on the machine. Not a
// module's path: no module reads it — the permanent controller is handed the address the
// temporary one was given — and genesis keeps it only so a second run puts the same account
// in the same two places.
BusControllerFile = "/var/lib/mesh-genesis/bus-controller.secret"
// BusControllerHashFile is the bcrypt hash of it, kept with it. bcrypt salts every hash, so a
// hash made afresh on every run would rewrite the bus's user list on every run; kept, the
// produced bundle is the same bundle twice.
BusControllerHashFile = BusControllerFile + ".bcrypt"
storeSuperuserMount = "/run/secrets/superuser"
@@ -55,15 +64,81 @@ const (
templateStorePassword = `"POSTGRES_PASSWORD": "bootstrap"`
templateStoreVolumes = `"volumes": ["mesh-store-data:/var/lib/postgresql/data"]`
templateStoreURL = "postgres:bootstrap@"
templateBrokerURL = "guest:guest@"
templateBrokerReady = "\"verify\": [\"lavinmqctl\", \"status\"]\n },"
brokerAdminMarker = "/var/lib/lavinmq/.mesh-admin"
// BusURLPlaceholder is the controller's account in its bus address, as the template names it.
BusURLPlaceholder = "nats://" + busControllerUser + ":bootstrap@"
// BusUserPlaceholder is the controller's entry in the user list the template carries, up to
// its password's hash — written as it stands inside the bundle's JSON string, quotes escaped.
BusUserPlaceholder = `{ user: \"` + busControllerUser + `\", password: \"`
busControllerUser = "controller"
)
// RootCredentials are the two values, and whether this run made them.
// BusVar is where the control plane finds the mesh's bus, with its own account (mesh-controller
// internal/broker NATSVar), and CertificateVar the certificate it pins the bus to (ADR 0004).
const (
BusVar = "MESH_BUS_NATS"
CertificateVar = "MESH_BROKER_CERTIFICATE"
)
// BusPlaceholders refuses a template whose bus connection this installer does not recognise,
// naming the placeholder it looked for (novox/hq ADR 0331 §3, §4).
//
// Asked at preflight, before anything is changed, and again where the bundle is rewritten. A
// template with no `MESH_BUS_NATS` raises a control plane that cannot reach the mesh's bus and
// says so every few seconds for ever (issue 146): the verify step would wait out its whole timeout
// for an answer that cannot come. That is a fault in the bundle, and it is found here instead.
func BusPlaceholders(template []byte, d *declaration.Declaration) error {
control, err := controlPlaneIn(d)
if err != nil {
return err
}
address := strings.TrimSpace(control.Env[BusVar])
if address == "" {
return fmt.Errorf("the template's control plane (%q) is given no %s, so it could not reach the mesh's "+
"bus and would never answer. A genesis template names the bus as %s<address>, which this "+
"installer fills with the account it makes; mesh-host's examples/foundation-first-node.lock "+
"is the shape", control.Name, BusVar, BusURLPlaceholder)
}
if !strings.HasPrefix(address, BusURLPlaceholder) {
return fmt.Errorf("the template's control plane reaches the bus at %s, and this installer looked for "+
"%s<address> there: the account it makes goes where that placeholder is, and it will not "+
"guess where else a bus connection could be", redactURL(address), BusURLPlaceholder)
}
if strings.TrimSpace(control.Env[CertificateVar]) == "" {
return fmt.Errorf("the template's control plane is given no %s, so it could not pin the bus's "+
"certificate (novox/hq ADR 0004) and would refuse to dial it", CertificateVar)
}
if n := bytes.Count(template, []byte(BusUserPlaceholder)); n != 1 {
return fmt.Errorf("the template's user list for the bus names the controller's account %d times, and "+
"this installer looked for it once, as %s<hash>: the bus has to be raised with the account the "+
"controller dials with, and that is where it goes", n, BusUserPlaceholder)
}
return nil
}
// redactURL is a bus address with any password in it replaced, for an error that quotes it.
func redactURL(address string) string {
scheme, rest, has := strings.Cut(address, "://")
if !has {
rest, scheme = address, ""
}
at := strings.LastIndex(rest, "@")
user, _, hasPassword := strings.Cut(rest[:max(at, 0)], ":")
if at < 0 || !hasPassword {
return address
}
if scheme != "" {
scheme += "://"
}
return scheme + user + ":…" + rest[at:]
}
// RootCredentials are the two values, and whether this run made them. BusHash is the bcrypt hash of
// Bus, as the bus's user list carries it.
type RootCredentials struct {
Store, Broker string
StoreMade, BrokerMade bool
Store, Bus string
BusHash string
StoreMade, BusMade bool
}
// RootSecrets reads the credentials this machine already holds, or makes them.
@@ -71,17 +146,44 @@ type RootCredentials struct {
// A dry run makes them in memory and writes nothing — so the bundle it reports is the shape of the
// real one, and a machine that was only asked is not left holding half a genesis.
func RootSecrets(dryRun bool) (RootCredentials, error) {
return rootSecretsAt(StoreSuperuserFile, BusControllerFile, BusControllerHashFile, dryRun)
}
func rootSecretsAt(storeFile, busFile, hashFile string, dryRun bool) (RootCredentials, error) {
var out RootCredentials
var err error
if out.Store, out.StoreMade, err = keptOrMade(StoreSuperuserFile, dryRun); err != nil {
if out.Store, out.StoreMade, err = keptOrMade(storeFile, dryRun); err != nil {
return out, err
}
if out.Broker, out.BrokerMade, err = keptOrMade(BrokerAdminFile, dryRun); err != nil {
if out.Bus, out.BusMade, err = keptOrMade(busFile, dryRun); err != nil {
return out, err
}
if out.BusHash, err = keptHash(hashFile, out.Bus, dryRun); err != nil {
return out, err
}
return out, nil
}
// keptHash is the bcrypt hash of password kept at path, made again only when the kept one is
// missing or no longer the hash of this password.
func keptHash(path, password string, dryRun bool) (string, error) {
if kept, err := readCredentialFile(path); err == nil {
if bcrypt.CompareHashAndPassword([]byte(kept), []byte(password)) == nil {
return kept, nil
}
} else if !os.IsNotExist(err) {
return "", err
}
hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
if err != nil {
return "", err
}
if dryRun {
return string(hash), nil
}
return string(hash), writeCredentialFile(path, string(hash))
}
func keptOrMade(path string, dryRun bool) (value string, made bool, err error) {
value, err = readCredentialFile(path)
if err == nil {
@@ -97,8 +199,13 @@ func keptOrMade(path string, dryRun bool) (value string, made bool, err error) {
if dryRun {
return value, true, nil
}
return value, true, writeCredentialFile(path, value)
}
// writeCredentialFile writes a credential as genesis keeps it.
func writeCredentialFile(path, value string) error {
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
return "", false, err
return err
}
// Written whole and renamed into place, at 0600, owned by whoever runs the installer — root,
// which is also who the host runs as when it later writes the sealed copy here.
@@ -112,12 +219,9 @@ func keptOrMade(path string, dryRun bool) (value string, made bool, err error) {
// tolerates either ending, so a file an earlier genesis wrote still reads.
tmp := path + ".genesis"
if err := os.WriteFile(tmp, []byte(value), 0o600); err != nil {
return "", false, err
return err
}
if err := os.Rename(tmp, path); err != nil {
return "", false, err
}
return value, true, nil
return os.Rename(tmp, path)
}
// readCredentialFile is a credential as genesis keeps it: the value alone, its line ending gone.
@@ -134,13 +238,6 @@ func readCredentialFile(path string) (string, error) {
return value, nil
}
// credentialFingerprint names a credential without being one — what the broker-admin action
// leaves on the broker's volume, so its verify holds for this value and not for any value.
func credentialFingerprint(value string) string {
sum := sha256.Sum256([]byte(value))
return hex.EncodeToString(sum[:8])
}
// freshSecret is the same shape the controller mints: 30 random bytes as unpadded base64url, 40
// characters, URL-safe — it lands inside connection strings.
func freshSecret() (string, error) {
@@ -153,55 +250,48 @@ func freshSecret() (string, error) {
// Masking makes a reporter that never says the credentials this run made.
//
// The applier reports each action with its command line, and two of them now carry a real
// password — the context schemas' connection strings and the broker's change_password. Those
// lines go to a terminal and to whatever keeps the transcript, which for the lab is a file. The
// exact values are known here, so they are replaced wherever they appear, in every line said.
// The applier reports each action with its command line, and the context schemas' connection
// strings carry a real password. Those lines go to a terminal and to whatever keeps the
// transcript, which for the lab is a file. The exact values are known here, so they are replaced
// wherever they appear, in every line said.
func Masking(say func(string), c RootCredentials) func(string) {
replacer := strings.NewReplacer(c.Store, "…", c.Broker, "…")
if c.Store == "" || c.Broker == "" {
if c.Store == "" || c.Bus == "" {
return say
}
replacer := strings.NewReplacer(c.Store, "…", c.Bus, "…")
return func(line string) { say(replacer.Replace(line)) }
}
// RefuseExistingServers stops a run that would put a made credential in front of a server raised
// RefuseExistingServers stops a run that would put a made credential in front of a store raised
// by an earlier installer with the template's.
//
// The store's password is set by initdb, once, on an empty volume; the broker's by the action
// above, once. A machine that already holds `mesh-store-data` or `mesh-broker-data` and has no
// credential file was raised with `bootstrap` and `guest`, and minting new values here would make a
// bundle that dials with passwords the servers do not have — failing three steps later, in the
// schemas, with nothing pointing back here. Refused by name instead, with the way forward.
// The store's password is set by initdb, once, on an empty volume. A machine that already holds
// `mesh-store-data` and has no credential file was raised with `bootstrap`, and minting a new
// value here would make a bundle that dials with a password the server does not have — failing
// three steps later, in the schemas, with nothing pointing back here. Refused by name instead,
// with the way forward.
//
// **The bus is not asked.** Its accounts live in the user list beside its configuration, which
// the bundle writes and the bus re-reads, not in its data volume — so a bus raised earlier takes
// the made account as readily as a new one.
func RefuseExistingServers(ctx context.Context, run Runner, c RootCredentials) error {
for _, check := range []struct {
made bool
volume string
what string
file string
}{
{c.StoreMade, "mesh-store-data", "store", StoreSuperuserFile},
{c.BrokerMade, "mesh-broker-data", "broker", BrokerAdminFile},
} {
if !check.made {
continue
}
if _, err := run(ctx, "docker", "volume", "inspect", check.volume); err != nil {
continue // no such volume: a fresh machine, which is the case this installer makes
}
return fmt.Errorf(
"this machine already holds the %s's data (volume %s) and no credential at %s, so it was raised "+
"by an earlier installer with the template's password. A new one made here would not open it. "+
"Put the password the %s has into %s (0600, the value alone) and run again; then change it "+
"on the server and accept the new value — this installer does not rotate a running %s",
check.what, check.volume, check.file, check.what, check.file, check.what)
if !c.StoreMade {
return nil
}
return nil
if _, err := run(ctx, "docker", "volume", "inspect", "mesh-store-data"); err != nil {
return nil // no such volume: a fresh machine, which is the case this installer makes
}
return fmt.Errorf(
"this machine already holds the store's data (volume mesh-store-data) and no credential at %s, so it was raised "+
"by an earlier installer with the template's password. A new one made here would not open it. "+
"Put the password the store has into %s (0600, the value alone) and run again; then change it "+
"on the server and accept the new value — this installer does not rotate a running store",
StoreSuperuserFile, StoreSuperuserFile)
}
// RootRewrite says what RewriteRoot did to the bundle.
type RootRewrite struct {
StoreURLs, BrokerURLs int
StoreURLs, BusURLs int
}
// RewriteRoot puts the made credentials into the produced bundle, in place of the template's.
@@ -225,37 +315,30 @@ func RewriteRoot(r *Rewritten, c RootCredentials) (RootRewrite, error) {
"the store's volumes"); err != nil {
return out, err
}
// Everything that dials the store or the broker with the template's credentials.
// Everything that dials the store with the template's credentials.
out.StoreURLs = bytes.Count(bundle, []byte(templateStoreURL))
if out.StoreURLs == 0 {
return out, fmt.Errorf("the template names no %q connection, so this installer cannot tell what it would be leaving well-known", templateStoreURL)
}
bundle = bytes.ReplaceAll(bundle, []byte(templateStoreURL), []byte("postgres:"+c.Store+"@"))
out.BrokerURLs = bytes.Count(bundle, []byte(templateBrokerURL))
if out.BrokerURLs == 0 {
return out, fmt.Errorf("the template names no %q connection, so this installer cannot tell what it would be leaving well-known", templateBrokerURL)
}
bundle = bytes.ReplaceAll(bundle, []byte(templateBrokerURL), []byte(BrokerAdminUser+":"+c.Broker+"@"))
// The broker's administrator, changed once the broker answers and before anything dials it.
// Verified by a marker on the broker's own data volume holding this password's fingerprint —
// the image carries nothing that can try a password from inside, and a marker that merely
// existed would let a regenerated password go unapplied for ever. What proves the password
// works is the control plane answering over it, a few resources later. The marker ends in a
// newline because the verify reads it with the shell's `read`, which fails at end of file
// without one — an action that ran and a verify that said no, once, in the lab.
fp := credentialFingerprint(c.Broker)
action := templateBrokerReady + "\n" +
" {\n" +
" \"id\": \"broker-admin\",\n" +
" \"type\": \"action\",\n" +
" \"in\": \"mesh-broker\",\n" +
" \"command\": [\"sh\", \"-c\", \"lavinmqctl change_password " + BrokerAdminUser + " '" + c.Broker + "' && echo " + fp + " > " + brokerAdminMarker + "\"],\n" +
" \"verify\": [\"sh\", \"-c\", \"read m < " + brokerAdminMarker + " && [ \\\"$m\\\" = " + fp + " ]\"]\n" +
" },"
if bundle, err = replaceOnce(bundle, templateBrokerReady, action, "the broker's readiness check"); err != nil {
// The bus: the controller's account, in its address and in the user list the bus is raised
// with, the same account in both (novox/hq ADR 0331 §3).
if err := BusPlaceholders(bundle, r.Declaration); err != nil {
return out, err
}
out.BusURLs = bytes.Count(bundle, []byte(BusURLPlaceholder))
bundle = bytes.ReplaceAll(bundle, []byte(BusURLPlaceholder),
[]byte("nats://"+busControllerUser+":"+c.Bus+"@"))
at := bytes.Index(bundle, []byte(BusUserPlaceholder)) + len(BusUserPlaceholder)
end := bytes.Index(bundle[at:], []byte(`\"`))
if end < 0 {
return out, fmt.Errorf("the controller's entry in the template's user list for the bus has no end to its password")
}
if !strings.HasPrefix(c.BusHash, "$2") {
return out, fmt.Errorf("the controller's bus password was given no bcrypt hash, so the bus could not be raised with it; this is the installer's fault")
}
bundle = append(append(append([]byte{}, bundle[:at]...), c.BusHash...), bundle[at+end:]...)
parsed, err := declaration.ParseFileTrusted(bundle)
if err != nil {
internal/bootstrap/rootsecrets_test.go
+128 -91
View File
@@ -5,13 +5,24 @@ import (
"strings"
"testing"
"golang.org/x/crypto/bcrypt"
"github.com/novox/mesh-host/internal/declaration"
)
// The produced bundle carries no well-known credential: the store reads its password from the
// file genesis made, every connection string names the made values, and the broker's default
// administrator is changed by an action before anything dials it (novox/hq issue 071).
func TestTheProducedBundleCarriesNoWellKnownCredential(t *testing.T) {
// testCredentials are made values with a real bcrypt hash of the bus password, as RootSecrets makes.
func testCredentials(t *testing.T, store, bus string) RootCredentials {
t.Helper()
hash, err := bcrypt.GenerateFromPassword([]byte(bus), bcrypt.MinCost)
if err != nil {
t.Fatal(err)
}
return RootCredentials{Store: store, Bus: bus, BusHash: string(hash)}
}
// theExample is the one genesis bundle, rewritten for an image as the installer does.
func theExample(t *testing.T) Rewritten {
t.Helper()
template, err := os.ReadFile("../../examples/foundation-first-node.lock")
if err != nil {
t.Skip("no example bundle beside this checkout")
@@ -20,81 +31,142 @@ func TestTheProducedBundleCarriesNoWellKnownCredential(t *testing.T) {
if err != nil {
t.Fatal(err)
}
creds := RootCredentials{Store: "STORE-PW-40-characters-of-random-base64u", Broker: "BROKER-PW-40-characters-of-random-base64"}
return r
}
// The produced bundle carries no well-known credential: the store reads its password from the
// file genesis made, and the controller's account on the bus is the made one in both places the
// bundle names it — the address the control plane dials and the user list the bus is raised with
// (novox/hq issue 071, ADR 0331 §3).
func TestTheProducedBundleCarriesNoWellKnownCredential(t *testing.T) {
r := theExample(t)
creds := testCredentials(t, "STORE-PW-40-characters-of-random-base64u", "BUS-PW-40-characters-of-random-base64uuu")
got, err := RewriteRoot(&r, creds)
if err != nil {
t.Fatal(err)
}
text := string(r.Bundle)
for _, gone := range []string{`"bootstrap"`, "postgres:bootstrap@", "guest:guest@"} {
for _, gone := range []string{`"bootstrap"`, "postgres:bootstrap@", "controller:bootstrap@"} {
if strings.Contains(text, gone) {
t.Errorf("the produced bundle still says %s", gone)
}
}
if got.StoreURLs < 3 || got.BrokerURLs < 2 {
t.Errorf("rewrote %d store and %d broker connections; the template has three and two", got.StoreURLs, got.BrokerURLs)
if got.StoreURLs < 3 || got.BusURLs != 1 {
t.Errorf("rewrote %d store and %d bus connections; the template has three and one", got.StoreURLs, got.BusURLs)
}
var store, action bool
var store, accounts, control bool
for _, res := range r.Declaration.Resources {
switch x := res.(type) {
case *declaration.Container:
if x.Name != "mesh-store" {
switch x.Name {
case "mesh-store":
store = true
if _, has := x.Env["POSTGRES_PASSWORD"]; has {
t.Error("the store still takes its password from its environment")
}
if x.Env["POSTGRES_PASSWORD_FILE"] != storeSuperuserMount {
t.Errorf("the store reads its password from %q", x.Env["POSTGRES_PASSWORD_FILE"])
}
if !strings.Contains(strings.Join(x.Volumes, " "), StoreSuperuserFile+":"+storeSuperuserMount) {
t.Errorf("the store does not mount %s: %v", StoreSuperuserFile, x.Volumes)
}
case r.TempName:
control = true
if want := "nats://controller:" + creds.Bus + "@127.0.0.1:5671"; x.Env[BusVar] != want {
t.Errorf("the temporary control plane dials the bus at %q, not with the made account", x.Env[BusVar])
}
}
case *declaration.File:
if x.ID != "bus-accounts" {
continue
}
store = true
if _, has := x.Env["POSTGRES_PASSWORD"]; has {
t.Error("the store still takes its password from its environment")
accounts = true
hash := hashOf(t, x.Content, "controller")
if bcrypt.CompareHashAndPassword([]byte(hash), []byte(creds.Bus)) != nil {
t.Errorf("the bus is raised with a controller account whose hash %q is not the made password's", hash)
}
if x.Env["POSTGRES_PASSWORD_FILE"] != storeSuperuserMount {
t.Errorf("the store reads its password from %q", x.Env["POSTGRES_PASSWORD_FILE"])
}
if !strings.Contains(strings.Join(x.Volumes, " "), StoreSuperuserFile+":"+storeSuperuserMount) {
t.Errorf("the store does not mount %s: %v", StoreSuperuserFile, x.Volumes)
}
case *declaration.Action:
if x.ID != "broker-admin" {
continue
}
action = true
if x.In != "mesh-broker" || !strings.Contains(strings.Join(x.Command, " "), "change_password "+BrokerAdminUser+" '"+creds.Broker+"'") {
t.Errorf("the broker-admin action is %v in %q", x.Command, x.In)
if bcrypt.CompareHashAndPassword([]byte(hash), []byte("bootstrap")) == nil {
t.Error("the bus is still raised with the controller at bootstrap")
}
}
}
if !store || !action {
t.Fatalf("store=%v action=%v", store, action)
}
// The order matters: the broker's password changes after it answers and before the control
// plane, which dials it with the new one, is raised.
var readyAt, adminAt, controlAt int
for i, res := range r.Declaration.Resources {
switch res.Identity() {
case "broker-ready":
readyAt = i
case "broker-admin":
adminAt = i
case "control-plane":
controlAt = i
}
}
if !(readyAt < adminAt && adminAt < controlAt) {
t.Errorf("order ready=%d admin=%d control=%d", readyAt, adminAt, controlAt)
if !store || !accounts || !control {
t.Fatalf("store=%v accounts=%v control=%v", store, accounts, control)
}
}
// A template that no longer says what this expects is refused, not half-rewritten.
func TestATemplateWithoutTheKnownCredentialsIsRefused(t *testing.T) {
// hashOf is the password hash a NATS user list gives one user.
func hashOf(t *testing.T, list, user string) string {
t.Helper()
marker := `{ user: "` + user + `", password: "`
at := strings.Index(list, marker)
if at < 0 {
t.Fatalf("the user list names no %s:\n%s", user, list)
}
rest := list[at+len(marker):]
return rest[:strings.Index(rest, `"`)]
}
// The bus's placeholders are what the installer looks for, and a template naming no bus connection
// it knows is refused at preflight, naming the placeholder (novox/hq ADR 0331 §3, §4).
func TestABundleNamingNoBusConnectionItKnowsIsRefusedByItsPlaceholder(t *testing.T) {
template, err := os.ReadFile("../../examples/foundation-first-node.lock")
if err != nil {
t.Skip("no example bundle beside this checkout")
}
changed := strings.Replace(string(template), `"POSTGRES_PASSWORD": "bootstrap"`, `"POSTGRES_PASSWORD": "other"`, 1)
r, err := Rewrite([]byte(changed), "sha256:"+strings.Repeat("ab", 32))
parse := func(b string) *declaration.Declaration {
d, err := declaration.ParseFileTrusted([]byte(b))
if err != nil {
t.Fatal(err)
}
return d
}
if err := BusPlaceholders(template, parse(string(template))); err != nil {
t.Fatalf("the shipped bundle is refused: %v", err)
}
for _, c := range []struct{ what, from, to, names string }{
{"no bus address", `"MESH_BUS_NATS": "nats://controller:bootstrap@127.0.0.1:5671",`, "", BusVar},
{"the previous broker's", `"MESH_BUS_NATS": "nats://controller:bootstrap@127.0.0.1:5671"`,
`"MESH_BUS_NATS": "amqp://guest:guest@127.0.0.1:5671"`, BusURLPlaceholder},
{"no pinned certificate", `"MESH_BROKER_CERTIFICATE": "/broker-tls/tls.crt"`, `"MESH_OTHER": "x"`, CertificateVar},
{"no controller in the user list", `{ user: \"controller\"`, `{ user: \"someone\"`, BusUserPlaceholder},
} {
if !strings.Contains(string(template), c.from) {
t.Fatalf("%s: the bundle does not say %s", c.what, c.from)
}
changed := strings.Replace(string(template), c.from, c.to, 1)
err := BusPlaceholders([]byte(changed), parse(changed))
if err == nil {
t.Errorf("%s: accepted", c.what)
continue
}
if !strings.Contains(err.Error(), c.names) {
t.Errorf("%s: refused without naming %s: %v", c.what, c.names, err)
}
if strings.Contains(err.Error(), "guest:guest") {
t.Errorf("%s: the refusal quotes a password: %v", c.what, err)
}
}
}
// The account is made once, with its hash, and a second run puts the same bytes in the bundle — a
// hash salted afresh every run would rewrite the bus's user list every run.
func TestTheBusAccountIsKeptWithItsHash(t *testing.T) {
dir := t.TempDir()
first, err := rootSecretsAt(dir+"/store", dir+"/bus", dir+"/bus.bcrypt", false)
if err != nil {
t.Fatal(err)
}
if _, err := RewriteRoot(&r, RootCredentials{Store: "x", Broker: "y"}); err == nil {
t.Fatal("a template with an unknown store password was rewritten")
if !first.BusMade || len(first.Bus) != 40 || bcrypt.CompareHashAndPassword([]byte(first.BusHash), []byte(first.Bus)) != nil {
t.Fatalf("first run: made=%v hash ok=%v", first.BusMade, bcrypt.CompareHashAndPassword([]byte(first.BusHash), []byte(first.Bus)) == nil)
}
second, err := rootSecretsAt(dir+"/store", dir+"/bus", dir+"/bus.bcrypt", false)
if err != nil {
t.Fatal(err)
}
if second.BusMade || second.Bus != first.Bus || second.BusHash != first.BusHash {
t.Fatalf("a second run changed the bus account: made=%v same=%v same hash=%v",
second.BusMade, second.Bus == first.Bus, second.BusHash == first.BusHash)
}
}
@@ -125,60 +197,25 @@ func TestRootSecretsAreKeptAcrossRuns(t *testing.T) {
// Nothing the installer says after making the credentials contains them.
func TestTheTranscriptNeverSaysTheCredentials(t *testing.T) {
var said []string
say := Masking(func(l string) { said = append(said, l) }, RootCredentials{Store: "STORE-PW", Broker: "BROKER-PW"})
say := Masking(func(l string) { said = append(said, l) }, RootCredentials{Store: "STORE-PW", Bus: "BUS-PW"})
say("created context-schemas (docker run -e MESH_STORE_INVENTORY=postgres://postgres:STORE-PW@127.0.0.1:5432/inventory)")
say("failed broker-admin (sh -c lavinmqctl change_password guest 'BROKER-PW' && echo x)")
say("failed control-plane (MESH_BUS_NATS=nats://controller:BUS-PW@127.0.0.1:5671)")
for _, l := range said {
if strings.Contains(l, "STORE-PW") || strings.Contains(l, "BROKER-PW") {
if strings.Contains(l, "STORE-PW") || strings.Contains(l, "BUS-PW") {
t.Errorf("said a credential: %s", l)
}
}
if !strings.Contains(said[0], "postgres:…@") || !strings.Contains(said[1], "guest '…'") {
if !strings.Contains(said[0], "postgres:…@") || !strings.Contains(said[1], "controller:…@") {
t.Errorf("the lines were not the same lines with the values masked: %v", said)
}
}
// The broker-admin marker is written with a line ending, because the verify reads it with `read`.
func TestTheBrokerAdminMarkerHasALineEnding(t *testing.T) {
template, err := os.ReadFile("../../examples/foundation-first-node.lock")
if err != nil {
t.Skip("no example bundle beside this checkout")
}
r, err := Rewrite(template, "sha256:"+strings.Repeat("ab", 32))
if err != nil {
t.Fatal(err)
}
if _, err := RewriteRoot(&r, RootCredentials{Store: "s", Broker: "b"}); err != nil {
t.Fatal(err)
}
for _, res := range r.Declaration.Resources {
a, ok := res.(*declaration.Action)
if !ok || a.ID != "broker-admin" {
continue
}
cmd := strings.Join(a.Command, " ")
if !strings.Contains(cmd, "&& echo ") || strings.Contains(cmd, "printf %s") {
t.Errorf("the marker is written without a line ending: %s", cmd)
}
if !strings.Contains(strings.Join(a.Verify, " "), "read m <") {
t.Errorf("the verify does not read the marker: %v", a.Verify)
}
}
}
// The foundation filters before anything listens: nftables is in place before the store, and its
// rules refuse the store's and broker's client ports from outside while keeping ssh, the bus a node
// enrols over and the registry a node pulls from (novox/hq issue 054).
func TestTheFoundationFiltersBeforeAnythingListens(t *testing.T) {
template, err := os.ReadFile("../../examples/foundation-first-node.lock")
if err != nil {
t.Skip("no example bundle beside this checkout")
}
r, err := Rewrite(template, "sha256:"+strings.Repeat("ab", 32))
if err != nil {
t.Fatal(err)
}
if _, err := RewriteRoot(&r, RootCredentials{Store: "s", Broker: "b"}); err != nil {
r := theExample(t)
if _, err := RewriteRoot(&r, testCredentials(t, "s", "b")); err != nil {
t.Fatal(err)
}
var filterAt, loadedAt, storeAt, brokerAt = -1, -1, -1, -1