Raise genesis on the bus the mesh runs on, from one bundle (hq issue 146, ADR 0331)
mesh/merge-gate head 37947ca4eed1 merged into trunk d8e785bb4b4b — pass: the change touches no module of the mesh's graph
mesh/repo-check head 37947ca4eed1 merged into trunk d8e785bb4b4b — pass: its merge-check.sh passed
mesh/delivery superseded: a newer head of the same pull request

The installer could not raise a first node: the old bundle's control plane had no
MESH_BUS_NATS and crash-looped, and the NATS bundle was refused because rootsecrets
only knew the previous broker's guest:guest@ template.

One bundle now, the NATS one. rootsecrets makes the controller's bus account and puts
it in both placeholders (the address and the bcrypt hash in the carried user list),
kept with its hash so a re-run writes the same bundle. Preflight refuses a template
naming no bus connection it knows, naming the placeholder, instead of waiting out the
verify step. The broker step adopts the bus as the nats module and refuses, by name,
a catalogue without it or a server that is not the foundation's container.
This commit is contained in:
jochen
2026-10-12 22:49:49 +02:00
parent d8e785bb4b
commit 37947ca4ee
19 changed files with 681 additions and 530 deletions
+1 -1
View File
@@ -185,7 +185,7 @@ func (r Rewritten) declares(id string) bool {
// container runtime's configuration whole — and neither is anything the operator installs later.
var genesisTakes = map[string]bool{
RegistryModule: true, ControlPlaneModule: true, BuilderModule: true,
"postgres": true, "lavinmq": true, "mesh-vault": true, "mesh-catalog": true,
"postgres": true, BusModule: true, "mesh-vault": true, "mesh-catalog": true,
}
// takeIfAdopted takes one of genesis's own modules on an adopted node, once it is assigned and
+5 -5
View File
@@ -503,7 +503,7 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
for _, c := range []struct {
what, path string
made bool
}{{"store superuser", StoreSuperuserFile, creds.StoreMade}, {"broker admin", BrokerAdminFile, creds.BrokerMade}} {
}{{"store superuser", StoreSuperuserFile, creds.StoreMade}, {"bus controller", BusControllerFile, creds.BusMade}} {
switch {
case c.made && o.DryRun:
say(fmt.Sprintf(" %-17s would be made and kept at %s (0600)", c.what, c.path))
@@ -513,9 +513,9 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
say(fmt.Sprintf(" %-17s already at %s — kept", c.what, c.path))
}
}
say(fmt.Sprintf(" credentials the template's bootstrap and guest are gone: %d store and %d broker "+
"connection(s) rewritten, the store reads its password from a file, the broker's admin is changed once it answers",
root.StoreURLs, root.BrokerURLs))
say(fmt.Sprintf(" credentials the template's bootstrap is gone: %d store and %d bus "+
"connection(s) rewritten, the store reads its password from a file, the bus is raised with the controller's account",
root.StoreURLs, root.BusURLs))
if rewritten.Renamed {
say(fmt.Sprintf(" control plane %s, renamed from %s",
rewritten.TempName, rewritten.WasCalled))
@@ -733,7 +733,7 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
}
// ---- broker ---------------------------------------------------------------------------
say("broker — the foundation's broker, adopted as the lavinmq module: one bus, not two")
say("broker — the foundation's bus, adopted as the nats module: one bus, not two")
if err := InstallBroker(ctx, o, permanentControl, rewritten.Declaration, say); err != nil {
return result, failed(StepBroker, err)
}
+18 -16
View File
@@ -30,14 +30,13 @@ const theControlPlaneModule = `{
"inventory": "/var/lib/mesh/mesh-controller/inventory",
"identity": "/var/lib/mesh/mesh-controller/identity",
"licences": "/var/lib/mesh/mesh-controller/licences",
"broker": "/var/lib/mesh/mesh-controller/broker",
"broker-management": "/var/lib/mesh/mesh-controller/broker-management"
"bus": "/var/lib/mesh/mesh-controller/bus"
},
"resources": [
{"id": "mesh-state", "type": "directory", "path": "/var/lib/mesh/mesh-controller", "mode": "0700"},
{"id": "broker-env", "type": "file", "path": "/var/lib/mesh/mesh-controller/broker.env",
"mode": "0600",
"content": "MESH_BROKER_AMQP=${secret:broker}\nMESH_BROKER_MANAGEMENT=${secret:broker-management}\nMESH_BROKER_ADDRESS=${machine:at}:5671\n"},
"content": "MESH_BUS_NATS=${secret:bus}\nMESH_BROKER_ADDRESS=${machine:at}:5671\n"},
{"id": "server", "type": "container", "name": "mesh-controller",
"image": "` + builtImage + `",
"network": "host", "args": ["serve"],
@@ -133,11 +132,10 @@ func TestTheStoreConnectionsComeFromTheBundleThatMadeThem(t *testing.T) {
// them at /run/secrets. Matching on the path alone would find nothing and refuse a correct
// manifest, which is exactly the ordinary case in the catalogue.
for variable, secret := range map[string]string{
"MESH_STORE_INVENTORY": "inventory",
"MESH_STORE_IDENTITY": "identity",
"MESH_STORE_LICENCES": "licences",
"MESH_BROKER_AMQP": "broker",
"MESH_BROKER_MANAGEMENT": "broker-management",
"MESH_STORE_INVENTORY": "inventory",
"MESH_STORE_IDENTITY": "identity",
"MESH_STORE_LICENCES": "licences",
"MESH_BUS_NATS": "bus",
} {
if wanted[variable] != secret {
t.Errorf("%s would be accepted as %q, want %q", variable, wanted[variable], secret)
@@ -161,9 +159,10 @@ func TestTheStoreConnectionsComeFromTheBundleThatMadeThem(t *testing.T) {
if err != nil {
t.Fatal(err)
}
// Three stores and both halves of the broker: everything the foundation made and nothing else.
if len(delivered) != 5 {
t.Fatalf("%d values were delivered, and the foundation names five: %v",
// Three stores and the controller's account on the bus: everything the foundation made and
// nothing else.
if len(delivered) != 4 {
t.Fatalf("%d values were delivered, and the foundation names four: %v",
len(delivered), delivered)
}
for _, secret := range delivered {
@@ -177,12 +176,15 @@ func TestTheStoreConnectionsComeFromTheBundleThatMadeThem(t *testing.T) {
// secret in a mesh is one the mesh made; `secret accept` is only for what predates the mesh.
func TestASecretTheFoundationNeverMadeIsLeftToTheMesh(t *testing.T) {
extra := strings.Replace(theControlPlaneModule,
`"broker": "/var/lib/mesh/mesh-controller/broker",`,
`"broker": "/var/lib/mesh/mesh-controller/broker",
"something-new": "/var/lib/mesh/mesh-controller/something-new",`, 1)
`"bus": "/var/lib/mesh/mesh-controller/bus"`,
`"bus": "/var/lib/mesh/mesh-controller/bus",
"something-new": "/var/lib/mesh/mesh-controller/something-new"`, 1)
extra = strings.Replace(extra,
`"content": "MESH_BROKER_AMQP=${secret:broker}\n`,
`"content": "MESH_SOMETHING_NEW=${secret:something-new}\nMESH_BROKER_AMQP=${secret:broker}\n`, 1)
`"content": "MESH_BUS_NATS=${secret:bus}\n`,
`"content": "MESH_SOMETHING_NEW=${secret:something-new}\nMESH_BUS_NATS=${secret:bus}\n`, 1)
if extra == theControlPlaneModule {
t.Fatal("the fixture no longer says what this test adds to")
}
rewritten, err := Rewrite(theRealBundle(t), held)
if err != nil {
+2 -1
View File
@@ -159,7 +159,8 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla
// busAccounts and busContainer are where the installer's bundle raises the bus: the file its
// configuration includes, and the container that reads it. The installer raised them, so it is the
// one that knows them (examples/foundation-first-node-nats.lock).
// one that knows them (examples/foundation-first-node.lock). A bridge until the bus is adopted as the
// nats module, which receives the list in its declaration from then on (novox/hq ADR 0331 §5).
const (
busAccounts = "/var/lib/mesh-bus-conf/accounts.conf"
busContainer = "mesh-broker"
+5
View File
@@ -82,6 +82,11 @@ func TestAMachineThatHasAlreadyEnrolledIsNotEnrolledAgain(t *testing.T) {
t.Errorf("a token was issued for a machine that already holds an identity: %v",
runtime.commands)
}
// Nor are the bus's users placed again: from the adoption on they are the nats module's, and
// the installer's bridge is not run a second time (novox/hq ADR 0331 §5).
if runtime.ran("broker accounts") || runtime.ran("accounts.conf") {
t.Errorf("the bus's users were placed on a re-run: %v", runtime.commands)
}
if out.Agent != "already running" {
t.Errorf("the host agent is reported as %q", out.Agent)
}
+64 -18
View File
@@ -14,8 +14,8 @@ import (
// Genesis raises a store and a broker before any module system exists, because the control plane
// cannot ask provisioning for the store it keeps its own records in or the broker it is reached over
// (novox/hq ADR 0006). That leaves two servers behind: the foundation's, and a second one the
// `postgres`/`lavinmq` modules used to raise for other modules to use. This turns the foundation's
// own servers into those modules, so a mesh runs ONE postgres and ONE lavinmq — the control plane's
// `postgres` and bus modules used to raise for other modules to use. This turns the foundation's
// own servers into those modules, so a mesh runs ONE postgres and ONE bus — the control plane's
// contexts and the database of each module that asks for one, in the same server (WBS 3.1/3.2).
//
// **Adopted in place, not replaced.** The control plane is stateless and is swapped for a fresh
@@ -155,37 +155,83 @@ func deliverCredential(ctx context.Context, o Options, control controlPlane, mod
return nil
}
// InstallBroker makes the foundation's broker the `lavinmq` module, adopted in place — the same
// shape as InstallStore, for the same reasons. The administrator's password is the one genesis
// gave the image's default account (rootsecrets.go), carried in so the module's provisioner can
// reach the management API as it.
// BusModule is the module the foundation's bus is adopted as (novox/hq ADR 0331 §1). The previous
// broker's module, lavinmq, left the catalogue with the move to one bus (ADR 0131).
const BusModule = "nats"
// InstallBroker makes the foundation's bus the `nats` module, adopted in place — the same shape as
// InstallStore, for the same reasons: the bus holds the mesh's streams, so it is kept, and from the
// module's first push the bus's user list reaches it in the module's declaration like any module's
// (ADR 0331 §1, §5). The installer's own placing of that list (enrol.go placeTheBusUsers) was the
// bridge to here and is not run again.
//
// **What is compared is the container's name, not its image.** The store's module pins the upstream
// postgres the foundation runs; the bus's module builds its server image (it carries the mesh's
// snapshot tool), so no pinned reference can be equal to the foundation's. A module whose server is
// not the foundation's container by name would raise a second bus beside the first, and is refused
// by name before anything is registered.
func InstallBroker(ctx context.Context, o Options, control controlPlane,
foundation *declaration.Declaration, say func(string)) error {
const module = "lavinmq"
const module = BusModule
manifest, err := readManifest(o.Catalogue, module)
if err != nil {
return err
}
broker, err := brokerIn(foundation)
bus, err := brokerIn(foundation)
if err != nil {
return err
}
if err := serverMatchesFoundation(manifest, broker, module); err != nil {
if err := busServerIsTheFoundations(manifest, bus, module); err != nil {
return err
}
say(" adopting " + broker.Name + " — the broker the foundation raised, unchanged")
if err := installProvider(ctx, o, control, module, manifest, "(the provisioner; the server is adopted, not built)",
func() error {
return deliverCredential(ctx, o, control, module, "admin", BrokerAdminFile, "the broker's administrator", say)
},
say); err != nil {
say(" adopting " + bus.Name + " — the bus the foundation raised, its streams kept")
if err := installProvider(ctx, o, control, module, manifest, "(the server, adopting the foundation's bus)",
nil, say); err != nil {
return err
}
say(" adopted " + broker.Name + " — the foundation's broker is now the " + module + " module")
say(" adopted " + bus.Name + " — the foundation's bus is now the " + module +
" module, and its user list is the module's from here on")
return nil
}
// busServerIsTheFoundations checks that the bus module's server container is the one the
// foundation raised, by name, so the applier takes it over rather than raising a second bus.
func busServerIsTheFoundations(manifest []byte, bus *declaration.Container, module string) error {
var m struct {
Resources []struct {
Type string `json:"type"`
Name string `json:"name"`
} `json:"resources"`
}
if err := json.Unmarshal(manifest, &m); err != nil {
return fmt.Errorf("the %s module's manifest is not readable: %w", module, err)
}
var names []string
for _, r := range m.Resources {
if r.Type != "container" {
continue
}
if r.Name == bus.Name {
return nil
}
names = append(names, r.Name)
}
return fmt.Errorf(
"the %s module's server is %s, and the foundation's bus is the container %q.\n"+
"The module adopts the foundation's bus in place, so its server has to carry the name the "+
"foundation raised; under another name it would raise a second bus beside the first, and "+
"the mesh would run two (novox/hq ADR 0331 §1). Nothing has been registered",
module, describeNames(names), bus.Name)
}
func describeNames(names []string) string {
if len(names) == 0 {
return "no container at all"
}
return "the container " + strings.Join(names, ", ")
}
// InstallVault installs the vault as a foundation module (novox/hq ADR 0085, amended). Nothing to
// adopt and nothing to carry in: it is its own runtime, built from the catalogue like any provider,
// and from its first push it keeps the export of every operator-sealed secret on its own disk.
@@ -207,9 +253,9 @@ func storeIn(d *declaration.Declaration) (*declaration.Container, error) {
return foundationContainer(d, StoreID, "store")
}
// brokerIn finds the broker container in the bundle this installer produced.
// brokerIn finds the bus's container in the bundle this installer produced.
func brokerIn(d *declaration.Declaration) (*declaration.Container, error) {
return foundationContainer(d, BrokerID, "broker")
return foundationContainer(d, BrokerID, "bus")
}
func foundationContainer(d *declaration.Declaration, id, what string) (*declaration.Container, error) {
+93
View File
@@ -0,0 +1,93 @@
package bootstrap
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
)
// aCatalogue is a catalogue checkout holding these modules' manifests.
func aCatalogue(t *testing.T, manifests map[string]string) string {
t.Helper()
dir := t.TempDir()
for module, manifest := range manifests {
at := filepath.Join(dir, catalogueDir, module)
if err := os.MkdirAll(at, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(at, "module.json"), []byte(manifest), 0o644); err != nil {
t.Fatal(err)
}
}
return dir
}
// aBusModule is the nats module's manifest as far as adoption reads it: a server container by name.
func aBusModule(server string) string {
return `{"module": "nats", "version": "1",
"claims": [{"name": "mesh-broker", "scope": "mesh"}],
"bus-users": "/var/lib/nats-module/conf/accounts.conf",
"resources": [{"id": "server", "type": "container", "name": "` + server + `", "ports": ["4222:4222"], "artifact": "server"}],
"build": {"artifacts": [{"name": "server", "kind": "image", "from": "Dockerfile"}]}}`
}
func busStep(t *testing.T, catalogue string) (*asked, error) {
t.Helper()
rewritten, err := Rewrite(theRealBundle(t), held)
if err != nil {
t.Fatal(err)
}
runtime := &asked{answer: aMeshThatAgrees(nil)}
control := controlPlane{container: "mesh-controller", run: runtime.run, timeout: 0}
err = InstallBroker(context.Background(), installing(t, catalogue), control, rewritten.Declaration, func(string) {})
return runtime, err
}
// The broker step adopts the foundation's bus as the nats module (novox/hq ADR 0331 §1): the module
// the mesh goes on managing, registered from the catalogue — never lavinmq, which left it.
func TestTheBrokerStepAdoptsTheBusAsTheNatsModule(t *testing.T) {
catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule("mesh-broker")})
runtime, _ := busStep(t, catalogue)
if !runtime.ran("module add /nats-module.json") {
t.Fatalf("the nats module was not registered: %v", runtime.commands)
}
if runtime.ran("lavinmq") {
t.Errorf("the broker step still speaks of lavinmq: %v", runtime.commands)
}
}
// A catalogue that holds no nats module fails the step by name, before anything is registered —
// which is what today's step did for lavinmq, silently adopting a module that no longer exists.
func TestTheBrokerStepFailsForAModuleTheCatalogueDoesNotHold(t *testing.T) {
catalogue := aCatalogue(t, map[string]string{"lavinmq": `{"module": "lavinmq", "version": "1"}`})
runtime, err := busStep(t, catalogue)
if err == nil {
t.Fatal("a catalogue without the nats module was adopted from")
}
if !strings.Contains(err.Error(), "nats module") {
t.Errorf("the refusal does not name the module it looked for: %v", err)
}
if runtime.ran("module add") {
t.Errorf("something was registered: %v", runtime.commands)
}
}
// A bus module whose server is not the foundation's container by name would raise a second bus
// beside the first; it is refused naming both, before anything is registered.
func TestABusModuleWhoseServerIsNotTheFoundationsIsRefused(t *testing.T) {
catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule("mesh-broker-nats")})
runtime, err := busStep(t, catalogue)
if err == nil {
t.Fatal("a bus module with another server was adopted")
}
for _, named := range []string{"mesh-broker-nats", `"mesh-broker"`} {
if !strings.Contains(err.Error(), named) {
t.Errorf("the refusal does not name %s: %v", named, err)
}
}
if runtime.ran("module add") {
t.Errorf("something was registered: %v", runtime.commands)
}
}
+14 -10
View File
@@ -103,14 +103,18 @@ func (p FoundationPorts) moduleSettings() map[string]map[string]int {
out[module][strconv.Itoa(declared)] = given
}
add("postgres", d.Store, p.Store)
add("lavinmq", d.Bus, p.Bus)
add("lavinmq", d.AMQP, p.AMQP)
add("lavinmq", d.Management, p.Management)
// The bus's module declares the server's own port, and the foundation published the bus on the
// node's: adopted in place, it keeps the port every token already names (ADR 0331 §1).
add(BusModule, busModulePort, p.Bus)
add(RegistryModule, d.Registry, p.Registry)
add(ForgeModule, d.Packages, p.Packages)
return out
}
// busModulePort is the machine's port the nats module declares for the bus (mesh-catalog
// modules/nats, "4222:4222").
const busModulePort = 4222
// PortsSetting is the controller's settings key for a module's given ports.
const PortsSetting = "ports"
@@ -200,10 +204,10 @@ func RewritePorts(r *Rewritten, p FoundationPorts, overlayRange string) (PortsRe
if p.Store != d.Store {
replace(`"ports": ["5432:5432"]`, fmt.Sprintf(`"ports": ["%d:5432"]`, p.Store), "the store's published port")
}
if p.Bus != d.Bus || p.AMQP != d.AMQP || p.Management != d.Management {
replace(`"ports": ["5671:5671", "5672:5672", "127.0.0.1:15672:15672"]`,
fmt.Sprintf(`"ports": ["%d:5671", "%d:5672", "127.0.0.1:%d:15672"]`, p.Bus, p.AMQP, p.Management),
"the broker's published ports")
if p.Bus != d.Bus {
replace(`"ports": ["5671:4222", "127.0.0.1:8222:8222"]`,
fmt.Sprintf(`"ports": ["%d:4222", "127.0.0.1:8222:8222"]`, p.Bus),
"the bus's published ports")
}
if err != nil {
return out, err
@@ -221,9 +225,9 @@ func RewritePorts(r *Rewritten, p FoundationPorts, overlayRange string) (PortsRe
value := control.Env[key]
now := value
now = strings.ReplaceAll(now, "@127.0.0.1:5432/", fmt.Sprintf("@127.0.0.1:%d/", p.Store))
now = strings.ReplaceAll(now, "@127.0.0.1:5672/", fmt.Sprintf("@127.0.0.1:%d/", p.AMQP))
if strings.HasSuffix(now, "@127.0.0.1:15672") {
now = strings.TrimSuffix(now, "15672") + strconv.Itoa(p.Management)
// The bus, as the control plane dials it (MESH_BUS_NATS) on the machine's loopback.
if strings.HasSuffix(now, "@127.0.0.1:5671") {
now = strings.TrimSuffix(now, "5671") + strconv.Itoa(p.Bus)
}
if key == brokerAddressVar {
if host, port, splitErr := net.SplitHostPort(value); splitErr == nil && port == "5671" {
+6 -7
View File
@@ -31,7 +31,7 @@ func producedBundle(t *testing.T) Rewritten {
if err != nil {
t.Fatal(err)
}
if _, err := RewriteRoot(&r, RootCredentials{Store: "s", Broker: "b"}); err != nil {
if _, err := RewriteRoot(&r, testCredentials(t, "s", "b")); err != nil {
t.Fatal(err)
}
return r
@@ -72,9 +72,9 @@ func TestGivenPortsMoveOnlyTheMachinesSide(t *testing.T) {
if len(storeC.Ports) != 1 || storeC.Ports[0] != "5433:5432" {
t.Errorf("the store publishes %v", storeC.Ports)
}
broker := containerNamed(r.Declaration, "mesh-broker")
if strings.Join(broker.Ports, " ") != "5771:5671 5772:5672 127.0.0.1:15673:15672" {
t.Errorf("the broker publishes %v", broker.Ports)
bus := containerNamed(r.Declaration, "mesh-broker")
if strings.Join(bus.Ports, " ") != "5771:4222 127.0.0.1:8222:8222" {
t.Errorf("the bus publishes %v", bus.Ports)
}
control, err := controlPlaneIn(r.Declaration)
if err != nil {
@@ -85,9 +85,8 @@ func TestGivenPortsMoveOnlyTheMachinesSide(t *testing.T) {
t.Errorf("%s still dials %s", key, value)
}
}
if !strings.Contains(control.Env["MESH_BROKER_AMQP"], "@127.0.0.1:5772/") ||
!strings.HasSuffix(control.Env["MESH_BROKER_MANAGEMENT"], "@127.0.0.1:15673") {
t.Errorf("the broker is dialled at %s and %s", control.Env["MESH_BROKER_AMQP"], control.Env["MESH_BROKER_MANAGEMENT"])
if !strings.HasSuffix(control.Env[BusVar], "@127.0.0.1:5771") {
t.Errorf("the control plane dials the bus at %s", control.Env[BusVar])
}
if control.Env["MESH_BROKER_ADDRESS"] != "192.0.2.10:5771" || r.BrokerAddress != "192.0.2.10:5771" {
t.Errorf("nodes are told to dial %s (%s)", control.Env["MESH_BROKER_ADDRESS"], r.BrokerAddress)
+23 -6
View File
@@ -12,6 +12,28 @@ import (
"github.com/novox/mesh-host/internal/profile"
)
// CheckTemplate is what preflight asks of the bundle template before anything is changed: that it
// is a declaration, that it raises a control plane, and that it names the bus the way this
// installer fills it (novox/hq ADR 0331 §3, §4).
//
// Parsed here as well as at the rewrite, because a template that is not a declaration should cost
// a second rather than an image load and a written file. And a control plane given no bus never
// answers: the verify step would wait its whole timeout to say so, and a template like that is a
// fault in the bundle, said at once.
func CheckTemplate(template []byte) (*declaration.Declaration, error) {
parsed, err := declaration.ParseFileTrusted(template)
if err != nil {
return nil, fmt.Errorf("the bundle template is not a declaration: %w", err)
}
if _, err := controlPlaneIn(parsed); err != nil {
return nil, err
}
if err := BusPlaceholders(template, parsed); err != nil {
return nil, err
}
return parsed, nil
}
// DefaultRegistry is where an image reference that names no host comes from.
const DefaultRegistry = "registry-1.docker.io:443"
@@ -76,13 +98,8 @@ func Preflight(ctx context.Context, o Options, d Deps, say func(string)) ([]byte
"it. Point --bundle at one; mesh-host's examples/foundation-first-node.lock is "+
"the shape", err)
}
// Parsed here as well as at the rewrite, because a template that is not a declaration should
// cost a second rather than an image load and a written file.
parsed, err := declaration.ParseFileTrusted(template)
parsed, err := CheckTemplate(template)
if err != nil {
return nil, fmt.Errorf("the bundle template is not a declaration: %w", err)
}
if _, err := controlPlaneIn(parsed); err != nil {
return nil, err
}
say(fmt.Sprintf(" bundle template %s (%d resources)", o.Template, len(parsed.Resources)))
+18
View File
@@ -149,3 +149,21 @@ func TestWhereAnImageWouldBeFetchedFrom(t *testing.T) {
}
}
}
// A template whose temporary control plane is given no bus is refused at preflight, before anything
// is changed, rather than raised and waited on at the verify step until its timeout (novox/hq issue
// 146, ADR 0331 §4): that control plane says it has no MESH_BUS_NATS every few seconds, for ever.
func TestATemplateWhoseControlPlaneHasNoBusIsRefusedBeforeAnythingIsRaised(t *testing.T) {
template := string(theRealBundle(t))
if _, err := CheckTemplate([]byte(template)); err != nil {
t.Fatalf("the shipped bundle is refused at preflight: %v", err)
}
const bus = `"MESH_BUS_NATS": "nats://controller:bootstrap@127.0.0.1:5671",`
if !strings.Contains(template, bus) {
t.Fatalf("the shipped bundle does not give its control plane %s", bus)
}
_, err := CheckTemplate([]byte(strings.Replace(template, bus, "", 1)))
if err == nil || !strings.Contains(err.Error(), BusVar) {
t.Fatalf("a control plane with no bus was let through preflight: %v", err)
}
}
+1 -1
View File
@@ -99,7 +99,7 @@ type Rewritten struct {
// migration pointing at an image no registry serves, and the apply would fail in the middle —
// after the store is up, before the broker. They are one image and they move together.
//
// **Third-party images are not touched.** postgres and lavinmq keep the `name@sha256:` references
// **Third-party images are not touched.** postgres and nats keep the `name@sha256:` references
// the template carries and are pulled from wherever those name (novox/hq ADR 0006). This is
// checked afterwards rather than merely intended: the produced bundle is re-parsed and every other
// container's image is compared against what it was.
+1 -1
View File
@@ -183,7 +183,7 @@ func TestTheProducedBundleKeepsTheTemplatesComments(t *testing.T) {
if err != nil {
t.Fatal(err)
}
const remembered = "NAMED VOLUME"
const remembered = "A filter before anything listens"
if !strings.Contains(string(out.Bundle), remembered) {
t.Errorf("the produced bundle lost the template's comments; %q is gone", remembered)
}
+185 -102
View File
@@ -4,49 +4,58 @@ import (
"bytes"
"context"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"fmt"
"os"
"path/filepath"
"strings"
"golang.org/x/crypto/bcrypt"
"github.com/novox/mesh-host/internal/declaration"
)
// The mesh's root credentials, made at genesis rather than copied from the template.
//
// **The template carries `bootstrap` and `guest`, and a mesh raised from it kept them** (novox/hq
// issue 071). The store's superuser and the broker's administrator are the two credentials every
// other one rests on, and they were the two that were not secret: constants in a file anybody can
// read, carried into the mesh by `secret accept` and marked as something the mesh must never
// replace — which is correct for a credential that already created the databases, and made the
// well-known value permanent.
// **The template carries `bootstrap`, and a mesh raised from it kept it** (novox/hq issue 071). The
// store's superuser and the controller's account on the bus are the two credentials every other one
// rests on, and they were the two that were not secret: constants in a file anybody can read,
// carried into the mesh by `secret accept` and marked as something the mesh must never replace —
// which is correct for a credential that already created the databases, and made the well-known
// value permanent.
//
// So the installer makes them. Two random values, **made once and kept on this machine** at the
// paths the postgres and lavinmq modules declare as their own secrets — so that when phase three
// adopts the store and the broker, `secret accept` carries in exactly the value the servers were
// raised with, and the host's later write of the sealed secret lands the same bytes in the same
// file. A second run finds the files and changes nothing, which is what lets the installer say
// "already done" about a store it must not restart.
// So the installer makes them. Two random values, **made once and kept on this machine** — the
// store's at the path the postgres module declares as its own secret, so that when phase three
// adopts the store `secret accept` carries in exactly the value the server was raised with; the
// bus's beside the other things only genesis holds. A second run finds the files and changes
// nothing, which is what lets the installer say "already done" about a store it must not restart.
//
// **The store reads its password from a file, not its environment.** `POSTGRES_PASSWORD` in a
// container's environment is in `docker inspect` for ever; the module that adopts the store
// declares the same file mount, so the two specs are one and the applier reconciles rather than
// recreates (phase3.go). The broker has no such file: its image's default administrator is changed
// in place by an action once the broker answers, and the produced bundle carries that action.
// recreates (phase3.go).
//
// **The bus has no administrator to change; it has a user list** (novox/hq ADR 0331 §3). The
// bundle carries the first list — the controller's own account at `bootstrap` — and the
// controller's address names the same account. Both are the bundle's placeholders: the address's
// `nats://controller:bootstrap@` and the controller's entry in the carried list. The installer
// puts the made password in the first and its bcrypt hash in the second, so the bus is raised with
// the account the controller dials with. The controller records a hash of what it actually uses
// (mesh-controller internal/broker, CredentialIn), so its own compositions carry the same account
// onward, and the permanent controller is given it by `secret accept` (control.go deliverStores).
const (
// StoreSuperuserFile is where the store's superuser password lives on the machine — the
// postgres module's own-secret path, so genesis and adoption write the same file.
StoreSuperuserFile = "/var/lib/postgres/superuser.secret"
// BrokerAdminFile is the same for the broker's administrator — the lavinmq module's.
BrokerAdminFile = "/var/lib/lavinmq-module/admin.secret"
// BrokerAdminUser is the broker's administrator. The image's default account, kept by name
// and given a password that is not the image's default; a renamed account would have to be
// created before anything can authenticate, and the thing that creates accounts is the thing
// that has to authenticate first.
BrokerAdminUser = "guest"
// BusControllerFile is where the controller's password on the bus lives on the machine. Not a
// module's path: no module reads it — the permanent controller is handed the address the
// temporary one was given — and genesis keeps it only so a second run puts the same account
// in the same two places.
BusControllerFile = "/var/lib/mesh-genesis/bus-controller.secret"
// BusControllerHashFile is the bcrypt hash of it, kept with it. bcrypt salts every hash, so a
// hash made afresh on every run would rewrite the bus's user list on every run; kept, the
// produced bundle is the same bundle twice.
BusControllerHashFile = BusControllerFile + ".bcrypt"
storeSuperuserMount = "/run/secrets/superuser"
@@ -55,15 +64,81 @@ const (
templateStorePassword = `"POSTGRES_PASSWORD": "bootstrap"`
templateStoreVolumes = `"volumes": ["mesh-store-data:/var/lib/postgresql/data"]`
templateStoreURL = "postgres:bootstrap@"
templateBrokerURL = "guest:guest@"
templateBrokerReady = "\"verify\": [\"lavinmqctl\", \"status\"]\n },"
brokerAdminMarker = "/var/lib/lavinmq/.mesh-admin"
// BusURLPlaceholder is the controller's account in its bus address, as the template names it.
BusURLPlaceholder = "nats://" + busControllerUser + ":bootstrap@"
// BusUserPlaceholder is the controller's entry in the user list the template carries, up to
// its password's hash — written as it stands inside the bundle's JSON string, quotes escaped.
BusUserPlaceholder = `{ user: \"` + busControllerUser + `\", password: \"`
busControllerUser = "controller"
)
// RootCredentials are the two values, and whether this run made them.
// BusVar is where the control plane finds the mesh's bus, with its own account (mesh-controller
// internal/broker NATSVar), and CertificateVar the certificate it pins the bus to (ADR 0004).
const (
BusVar = "MESH_BUS_NATS"
CertificateVar = "MESH_BROKER_CERTIFICATE"
)
// BusPlaceholders refuses a template whose bus connection this installer does not recognise,
// naming the placeholder it looked for (novox/hq ADR 0331 §3, §4).
//
// Asked at preflight, before anything is changed, and again where the bundle is rewritten. A
// template with no `MESH_BUS_NATS` raises a control plane that cannot reach the mesh's bus and
// says so every few seconds for ever (issue 146): the verify step would wait out its whole timeout
// for an answer that cannot come. That is a fault in the bundle, and it is found here instead.
func BusPlaceholders(template []byte, d *declaration.Declaration) error {
control, err := controlPlaneIn(d)
if err != nil {
return err
}
address := strings.TrimSpace(control.Env[BusVar])
if address == "" {
return fmt.Errorf("the template's control plane (%q) is given no %s, so it could not reach the mesh's "+
"bus and would never answer. A genesis template names the bus as %s<address>, which this "+
"installer fills with the account it makes; mesh-host's examples/foundation-first-node.lock "+
"is the shape", control.Name, BusVar, BusURLPlaceholder)
}
if !strings.HasPrefix(address, BusURLPlaceholder) {
return fmt.Errorf("the template's control plane reaches the bus at %s, and this installer looked for "+
"%s<address> there: the account it makes goes where that placeholder is, and it will not "+
"guess where else a bus connection could be", redactURL(address), BusURLPlaceholder)
}
if strings.TrimSpace(control.Env[CertificateVar]) == "" {
return fmt.Errorf("the template's control plane is given no %s, so it could not pin the bus's "+
"certificate (novox/hq ADR 0004) and would refuse to dial it", CertificateVar)
}
if n := bytes.Count(template, []byte(BusUserPlaceholder)); n != 1 {
return fmt.Errorf("the template's user list for the bus names the controller's account %d times, and "+
"this installer looked for it once, as %s<hash>: the bus has to be raised with the account the "+
"controller dials with, and that is where it goes", n, BusUserPlaceholder)
}
return nil
}
// redactURL is a bus address with any password in it replaced, for an error that quotes it.
func redactURL(address string) string {
scheme, rest, has := strings.Cut(address, "://")
if !has {
rest, scheme = address, ""
}
at := strings.LastIndex(rest, "@")
user, _, hasPassword := strings.Cut(rest[:max(at, 0)], ":")
if at < 0 || !hasPassword {
return address
}
if scheme != "" {
scheme += "://"
}
return scheme + user + ":…" + rest[at:]
}
// RootCredentials are the two values, and whether this run made them. BusHash is the bcrypt hash of
// Bus, as the bus's user list carries it.
type RootCredentials struct {
Store, Broker string
StoreMade, BrokerMade bool
Store, Bus string
BusHash string
StoreMade, BusMade bool
}
// RootSecrets reads the credentials this machine already holds, or makes them.
@@ -71,17 +146,44 @@ type RootCredentials struct {
// A dry run makes them in memory and writes nothing — so the bundle it reports is the shape of the
// real one, and a machine that was only asked is not left holding half a genesis.
func RootSecrets(dryRun bool) (RootCredentials, error) {
return rootSecretsAt(StoreSuperuserFile, BusControllerFile, BusControllerHashFile, dryRun)
}
func rootSecretsAt(storeFile, busFile, hashFile string, dryRun bool) (RootCredentials, error) {
var out RootCredentials
var err error
if out.Store, out.StoreMade, err = keptOrMade(StoreSuperuserFile, dryRun); err != nil {
if out.Store, out.StoreMade, err = keptOrMade(storeFile, dryRun); err != nil {
return out, err
}
if out.Broker, out.BrokerMade, err = keptOrMade(BrokerAdminFile, dryRun); err != nil {
if out.Bus, out.BusMade, err = keptOrMade(busFile, dryRun); err != nil {
return out, err
}
if out.BusHash, err = keptHash(hashFile, out.Bus, dryRun); err != nil {
return out, err
}
return out, nil
}
// keptHash is the bcrypt hash of password kept at path, made again only when the kept one is
// missing or no longer the hash of this password.
func keptHash(path, password string, dryRun bool) (string, error) {
if kept, err := readCredentialFile(path); err == nil {
if bcrypt.CompareHashAndPassword([]byte(kept), []byte(password)) == nil {
return kept, nil
}
} else if !os.IsNotExist(err) {
return "", err
}
hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
if err != nil {
return "", err
}
if dryRun {
return string(hash), nil
}
return string(hash), writeCredentialFile(path, string(hash))
}
func keptOrMade(path string, dryRun bool) (value string, made bool, err error) {
value, err = readCredentialFile(path)
if err == nil {
@@ -97,8 +199,13 @@ func keptOrMade(path string, dryRun bool) (value string, made bool, err error) {
if dryRun {
return value, true, nil
}
return value, true, writeCredentialFile(path, value)
}
// writeCredentialFile writes a credential as genesis keeps it.
func writeCredentialFile(path, value string) error {
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
return "", false, err
return err
}
// Written whole and renamed into place, at 0600, owned by whoever runs the installer — root,
// which is also who the host runs as when it later writes the sealed copy here.
@@ -112,12 +219,9 @@ func keptOrMade(path string, dryRun bool) (value string, made bool, err error) {
// tolerates either ending, so a file an earlier genesis wrote still reads.
tmp := path + ".genesis"
if err := os.WriteFile(tmp, []byte(value), 0o600); err != nil {
return "", false, err
return err
}
if err := os.Rename(tmp, path); err != nil {
return "", false, err
}
return value, true, nil
return os.Rename(tmp, path)
}
// readCredentialFile is a credential as genesis keeps it: the value alone, its line ending gone.
@@ -134,13 +238,6 @@ func readCredentialFile(path string) (string, error) {
return value, nil
}
// credentialFingerprint names a credential without being one — what the broker-admin action
// leaves on the broker's volume, so its verify holds for this value and not for any value.
func credentialFingerprint(value string) string {
sum := sha256.Sum256([]byte(value))
return hex.EncodeToString(sum[:8])
}
// freshSecret is the same shape the controller mints: 30 random bytes as unpadded base64url, 40
// characters, URL-safe — it lands inside connection strings.
func freshSecret() (string, error) {
@@ -153,55 +250,48 @@ func freshSecret() (string, error) {
// Masking makes a reporter that never says the credentials this run made.
//
// The applier reports each action with its command line, and two of them now carry a real
// password — the context schemas' connection strings and the broker's change_password. Those
// lines go to a terminal and to whatever keeps the transcript, which for the lab is a file. The
// exact values are known here, so they are replaced wherever they appear, in every line said.
// The applier reports each action with its command line, and the context schemas' connection
// strings carry a real password. Those lines go to a terminal and to whatever keeps the
// transcript, which for the lab is a file. The exact values are known here, so they are replaced
// wherever they appear, in every line said.
func Masking(say func(string), c RootCredentials) func(string) {
replacer := strings.NewReplacer(c.Store, "…", c.Broker, "…")
if c.Store == "" || c.Broker == "" {
if c.Store == "" || c.Bus == "" {
return say
}
replacer := strings.NewReplacer(c.Store, "…", c.Bus, "…")
return func(line string) { say(replacer.Replace(line)) }
}
// RefuseExistingServers stops a run that would put a made credential in front of a server raised
// RefuseExistingServers stops a run that would put a made credential in front of a store raised
// by an earlier installer with the template's.
//
// The store's password is set by initdb, once, on an empty volume; the broker's by the action
// above, once. A machine that already holds `mesh-store-data` or `mesh-broker-data` and has no
// credential file was raised with `bootstrap` and `guest`, and minting new values here would make a
// bundle that dials with passwords the servers do not have — failing three steps later, in the
// schemas, with nothing pointing back here. Refused by name instead, with the way forward.
// The store's password is set by initdb, once, on an empty volume. A machine that already holds
// `mesh-store-data` and has no credential file was raised with `bootstrap`, and minting a new
// value here would make a bundle that dials with a password the server does not have — failing
// three steps later, in the schemas, with nothing pointing back here. Refused by name instead,
// with the way forward.
//
// **The bus is not asked.** Its accounts live in the user list beside its configuration, which
// the bundle writes and the bus re-reads, not in its data volume — so a bus raised earlier takes
// the made account as readily as a new one.
func RefuseExistingServers(ctx context.Context, run Runner, c RootCredentials) error {
for _, check := range []struct {
made bool
volume string
what string
file string
}{
{c.StoreMade, "mesh-store-data", "store", StoreSuperuserFile},
{c.BrokerMade, "mesh-broker-data", "broker", BrokerAdminFile},
} {
if !check.made {
continue
}
if _, err := run(ctx, "docker", "volume", "inspect", check.volume); err != nil {
continue // no such volume: a fresh machine, which is the case this installer makes
}
return fmt.Errorf(
"this machine already holds the %s's data (volume %s) and no credential at %s, so it was raised "+
"by an earlier installer with the template's password. A new one made here would not open it. "+
"Put the password the %s has into %s (0600, the value alone) and run again; then change it "+
"on the server and accept the new value — this installer does not rotate a running %s",
check.what, check.volume, check.file, check.what, check.file, check.what)
if !c.StoreMade {
return nil
}
return nil
if _, err := run(ctx, "docker", "volume", "inspect", "mesh-store-data"); err != nil {
return nil // no such volume: a fresh machine, which is the case this installer makes
}
return fmt.Errorf(
"this machine already holds the store's data (volume mesh-store-data) and no credential at %s, so it was raised "+
"by an earlier installer with the template's password. A new one made here would not open it. "+
"Put the password the store has into %s (0600, the value alone) and run again; then change it "+
"on the server and accept the new value — this installer does not rotate a running store",
StoreSuperuserFile, StoreSuperuserFile)
}
// RootRewrite says what RewriteRoot did to the bundle.
type RootRewrite struct {
StoreURLs, BrokerURLs int
StoreURLs, BusURLs int
}
// RewriteRoot puts the made credentials into the produced bundle, in place of the template's.
@@ -225,37 +315,30 @@ func RewriteRoot(r *Rewritten, c RootCredentials) (RootRewrite, error) {
"the store's volumes"); err != nil {
return out, err
}
// Everything that dials the store or the broker with the template's credentials.
// Everything that dials the store with the template's credentials.
out.StoreURLs = bytes.Count(bundle, []byte(templateStoreURL))
if out.StoreURLs == 0 {
return out, fmt.Errorf("the template names no %q connection, so this installer cannot tell what it would be leaving well-known", templateStoreURL)
}
bundle = bytes.ReplaceAll(bundle, []byte(templateStoreURL), []byte("postgres:"+c.Store+"@"))
out.BrokerURLs = bytes.Count(bundle, []byte(templateBrokerURL))
if out.BrokerURLs == 0 {
return out, fmt.Errorf("the template names no %q connection, so this installer cannot tell what it would be leaving well-known", templateBrokerURL)
}
bundle = bytes.ReplaceAll(bundle, []byte(templateBrokerURL), []byte(BrokerAdminUser+":"+c.Broker+"@"))
// The broker's administrator, changed once the broker answers and before anything dials it.
// Verified by a marker on the broker's own data volume holding this password's fingerprint —
// the image carries nothing that can try a password from inside, and a marker that merely
// existed would let a regenerated password go unapplied for ever. What proves the password
// works is the control plane answering over it, a few resources later. The marker ends in a
// newline because the verify reads it with the shell's `read`, which fails at end of file
// without one — an action that ran and a verify that said no, once, in the lab.
fp := credentialFingerprint(c.Broker)
action := templateBrokerReady + "\n" +
" {\n" +
" \"id\": \"broker-admin\",\n" +
" \"type\": \"action\",\n" +
" \"in\": \"mesh-broker\",\n" +
" \"command\": [\"sh\", \"-c\", \"lavinmqctl change_password " + BrokerAdminUser + " '" + c.Broker + "' && echo " + fp + " > " + brokerAdminMarker + "\"],\n" +
" \"verify\": [\"sh\", \"-c\", \"read m < " + brokerAdminMarker + " && [ \\\"$m\\\" = " + fp + " ]\"]\n" +
" },"
if bundle, err = replaceOnce(bundle, templateBrokerReady, action, "the broker's readiness check"); err != nil {
// The bus: the controller's account, in its address and in the user list the bus is raised
// with, the same account in both (novox/hq ADR 0331 §3).
if err := BusPlaceholders(bundle, r.Declaration); err != nil {
return out, err
}
out.BusURLs = bytes.Count(bundle, []byte(BusURLPlaceholder))
bundle = bytes.ReplaceAll(bundle, []byte(BusURLPlaceholder),
[]byte("nats://"+busControllerUser+":"+c.Bus+"@"))
at := bytes.Index(bundle, []byte(BusUserPlaceholder)) + len(BusUserPlaceholder)
end := bytes.Index(bundle[at:], []byte(`\"`))
if end < 0 {
return out, fmt.Errorf("the controller's entry in the template's user list for the bus has no end to its password")
}
if !strings.HasPrefix(c.BusHash, "$2") {
return out, fmt.Errorf("the controller's bus password was given no bcrypt hash, so the bus could not be raised with it; this is the installer's fault")
}
bundle = append(append(append([]byte{}, bundle[:at]...), c.BusHash...), bundle[at+end:]...)
parsed, err := declaration.ParseFileTrusted(bundle)
if err != nil {
+128 -91
View File
@@ -5,13 +5,24 @@ import (
"strings"
"testing"
"golang.org/x/crypto/bcrypt"
"github.com/novox/mesh-host/internal/declaration"
)
// The produced bundle carries no well-known credential: the store reads its password from the
// file genesis made, every connection string names the made values, and the broker's default
// administrator is changed by an action before anything dials it (novox/hq issue 071).
func TestTheProducedBundleCarriesNoWellKnownCredential(t *testing.T) {
// testCredentials are made values with a real bcrypt hash of the bus password, as RootSecrets makes.
func testCredentials(t *testing.T, store, bus string) RootCredentials {
t.Helper()
hash, err := bcrypt.GenerateFromPassword([]byte(bus), bcrypt.MinCost)
if err != nil {
t.Fatal(err)
}
return RootCredentials{Store: store, Bus: bus, BusHash: string(hash)}
}
// theExample is the one genesis bundle, rewritten for an image as the installer does.
func theExample(t *testing.T) Rewritten {
t.Helper()
template, err := os.ReadFile("../../examples/foundation-first-node.lock")
if err != nil {
t.Skip("no example bundle beside this checkout")
@@ -20,81 +31,142 @@ func TestTheProducedBundleCarriesNoWellKnownCredential(t *testing.T) {
if err != nil {
t.Fatal(err)
}
creds := RootCredentials{Store: "STORE-PW-40-characters-of-random-base64u", Broker: "BROKER-PW-40-characters-of-random-base64"}
return r
}
// The produced bundle carries no well-known credential: the store reads its password from the
// file genesis made, and the controller's account on the bus is the made one in both places the
// bundle names it — the address the control plane dials and the user list the bus is raised with
// (novox/hq issue 071, ADR 0331 §3).
func TestTheProducedBundleCarriesNoWellKnownCredential(t *testing.T) {
r := theExample(t)
creds := testCredentials(t, "STORE-PW-40-characters-of-random-base64u", "BUS-PW-40-characters-of-random-base64uuu")
got, err := RewriteRoot(&r, creds)
if err != nil {
t.Fatal(err)
}
text := string(r.Bundle)
for _, gone := range []string{`"bootstrap"`, "postgres:bootstrap@", "guest:guest@"} {
for _, gone := range []string{`"bootstrap"`, "postgres:bootstrap@", "controller:bootstrap@"} {
if strings.Contains(text, gone) {
t.Errorf("the produced bundle still says %s", gone)
}
}
if got.StoreURLs < 3 || got.BrokerURLs < 2 {
t.Errorf("rewrote %d store and %d broker connections; the template has three and two", got.StoreURLs, got.BrokerURLs)
if got.StoreURLs < 3 || got.BusURLs != 1 {
t.Errorf("rewrote %d store and %d bus connections; the template has three and one", got.StoreURLs, got.BusURLs)
}
var store, action bool
var store, accounts, control bool
for _, res := range r.Declaration.Resources {
switch x := res.(type) {
case *declaration.Container:
if x.Name != "mesh-store" {
switch x.Name {
case "mesh-store":
store = true
if _, has := x.Env["POSTGRES_PASSWORD"]; has {
t.Error("the store still takes its password from its environment")
}
if x.Env["POSTGRES_PASSWORD_FILE"] != storeSuperuserMount {
t.Errorf("the store reads its password from %q", x.Env["POSTGRES_PASSWORD_FILE"])
}
if !strings.Contains(strings.Join(x.Volumes, " "), StoreSuperuserFile+":"+storeSuperuserMount) {
t.Errorf("the store does not mount %s: %v", StoreSuperuserFile, x.Volumes)
}
case r.TempName:
control = true
if want := "nats://controller:" + creds.Bus + "@127.0.0.1:5671"; x.Env[BusVar] != want {
t.Errorf("the temporary control plane dials the bus at %q, not with the made account", x.Env[BusVar])
}
}
case *declaration.File:
if x.ID != "bus-accounts" {
continue
}
store = true
if _, has := x.Env["POSTGRES_PASSWORD"]; has {
t.Error("the store still takes its password from its environment")
accounts = true
hash := hashOf(t, x.Content, "controller")
if bcrypt.CompareHashAndPassword([]byte(hash), []byte(creds.Bus)) != nil {
t.Errorf("the bus is raised with a controller account whose hash %q is not the made password's", hash)
}
if x.Env["POSTGRES_PASSWORD_FILE"] != storeSuperuserMount {
t.Errorf("the store reads its password from %q", x.Env["POSTGRES_PASSWORD_FILE"])
}
if !strings.Contains(strings.Join(x.Volumes, " "), StoreSuperuserFile+":"+storeSuperuserMount) {
t.Errorf("the store does not mount %s: %v", StoreSuperuserFile, x.Volumes)
}
case *declaration.Action:
if x.ID != "broker-admin" {
continue
}
action = true
if x.In != "mesh-broker" || !strings.Contains(strings.Join(x.Command, " "), "change_password "+BrokerAdminUser+" '"+creds.Broker+"'") {
t.Errorf("the broker-admin action is %v in %q", x.Command, x.In)
if bcrypt.CompareHashAndPassword([]byte(hash), []byte("bootstrap")) == nil {
t.Error("the bus is still raised with the controller at bootstrap")
}
}
}
if !store || !action {
t.Fatalf("store=%v action=%v", store, action)
}
// The order matters: the broker's password changes after it answers and before the control
// plane, which dials it with the new one, is raised.
var readyAt, adminAt, controlAt int
for i, res := range r.Declaration.Resources {
switch res.Identity() {
case "broker-ready":
readyAt = i
case "broker-admin":
adminAt = i
case "control-plane":
controlAt = i
}
}
if !(readyAt < adminAt && adminAt < controlAt) {
t.Errorf("order ready=%d admin=%d control=%d", readyAt, adminAt, controlAt)
if !store || !accounts || !control {
t.Fatalf("store=%v accounts=%v control=%v", store, accounts, control)
}
}
// A template that no longer says what this expects is refused, not half-rewritten.
func TestATemplateWithoutTheKnownCredentialsIsRefused(t *testing.T) {
// hashOf is the password hash a NATS user list gives one user.
func hashOf(t *testing.T, list, user string) string {
t.Helper()
marker := `{ user: "` + user + `", password: "`
at := strings.Index(list, marker)
if at < 0 {
t.Fatalf("the user list names no %s:\n%s", user, list)
}
rest := list[at+len(marker):]
return rest[:strings.Index(rest, `"`)]
}
// The bus's placeholders are what the installer looks for, and a template naming no bus connection
// it knows is refused at preflight, naming the placeholder (novox/hq ADR 0331 §3, §4).
func TestABundleNamingNoBusConnectionItKnowsIsRefusedByItsPlaceholder(t *testing.T) {
template, err := os.ReadFile("../../examples/foundation-first-node.lock")
if err != nil {
t.Skip("no example bundle beside this checkout")
}
changed := strings.Replace(string(template), `"POSTGRES_PASSWORD": "bootstrap"`, `"POSTGRES_PASSWORD": "other"`, 1)
r, err := Rewrite([]byte(changed), "sha256:"+strings.Repeat("ab", 32))
parse := func(b string) *declaration.Declaration {
d, err := declaration.ParseFileTrusted([]byte(b))
if err != nil {
t.Fatal(err)
}
return d
}
if err := BusPlaceholders(template, parse(string(template))); err != nil {
t.Fatalf("the shipped bundle is refused: %v", err)
}
for _, c := range []struct{ what, from, to, names string }{
{"no bus address", `"MESH_BUS_NATS": "nats://controller:bootstrap@127.0.0.1:5671",`, "", BusVar},
{"the previous broker's", `"MESH_BUS_NATS": "nats://controller:bootstrap@127.0.0.1:5671"`,
`"MESH_BUS_NATS": "amqp://guest:guest@127.0.0.1:5671"`, BusURLPlaceholder},
{"no pinned certificate", `"MESH_BROKER_CERTIFICATE": "/broker-tls/tls.crt"`, `"MESH_OTHER": "x"`, CertificateVar},
{"no controller in the user list", `{ user: \"controller\"`, `{ user: \"someone\"`, BusUserPlaceholder},
} {
if !strings.Contains(string(template), c.from) {
t.Fatalf("%s: the bundle does not say %s", c.what, c.from)
}
changed := strings.Replace(string(template), c.from, c.to, 1)
err := BusPlaceholders([]byte(changed), parse(changed))
if err == nil {
t.Errorf("%s: accepted", c.what)
continue
}
if !strings.Contains(err.Error(), c.names) {
t.Errorf("%s: refused without naming %s: %v", c.what, c.names, err)
}
if strings.Contains(err.Error(), "guest:guest") {
t.Errorf("%s: the refusal quotes a password: %v", c.what, err)
}
}
}
// The account is made once, with its hash, and a second run puts the same bytes in the bundle — a
// hash salted afresh every run would rewrite the bus's user list every run.
func TestTheBusAccountIsKeptWithItsHash(t *testing.T) {
dir := t.TempDir()
first, err := rootSecretsAt(dir+"/store", dir+"/bus", dir+"/bus.bcrypt", false)
if err != nil {
t.Fatal(err)
}
if _, err := RewriteRoot(&r, RootCredentials{Store: "x", Broker: "y"}); err == nil {
t.Fatal("a template with an unknown store password was rewritten")
if !first.BusMade || len(first.Bus) != 40 || bcrypt.CompareHashAndPassword([]byte(first.BusHash), []byte(first.Bus)) != nil {
t.Fatalf("first run: made=%v hash ok=%v", first.BusMade, bcrypt.CompareHashAndPassword([]byte(first.BusHash), []byte(first.Bus)) == nil)
}
second, err := rootSecretsAt(dir+"/store", dir+"/bus", dir+"/bus.bcrypt", false)
if err != nil {
t.Fatal(err)
}
if second.BusMade || second.Bus != first.Bus || second.BusHash != first.BusHash {
t.Fatalf("a second run changed the bus account: made=%v same=%v same hash=%v",
second.BusMade, second.Bus == first.Bus, second.BusHash == first.BusHash)
}
}
@@ -125,60 +197,25 @@ func TestRootSecretsAreKeptAcrossRuns(t *testing.T) {
// Nothing the installer says after making the credentials contains them.
func TestTheTranscriptNeverSaysTheCredentials(t *testing.T) {
var said []string
say := Masking(func(l string) { said = append(said, l) }, RootCredentials{Store: "STORE-PW", Broker: "BROKER-PW"})
say := Masking(func(l string) { said = append(said, l) }, RootCredentials{Store: "STORE-PW", Bus: "BUS-PW"})
say("created context-schemas (docker run -e MESH_STORE_INVENTORY=postgres://postgres:STORE-PW@127.0.0.1:5432/inventory)")
say("failed broker-admin (sh -c lavinmqctl change_password guest 'BROKER-PW' && echo x)")
say("failed control-plane (MESH_BUS_NATS=nats://controller:BUS-PW@127.0.0.1:5671)")
for _, l := range said {
if strings.Contains(l, "STORE-PW") || strings.Contains(l, "BROKER-PW") {
if strings.Contains(l, "STORE-PW") || strings.Contains(l, "BUS-PW") {
t.Errorf("said a credential: %s", l)
}
}
if !strings.Contains(said[0], "postgres:…@") || !strings.Contains(said[1], "guest '…'") {
if !strings.Contains(said[0], "postgres:…@") || !strings.Contains(said[1], "controller:…@") {
t.Errorf("the lines were not the same lines with the values masked: %v", said)
}
}
// The broker-admin marker is written with a line ending, because the verify reads it with `read`.
func TestTheBrokerAdminMarkerHasALineEnding(t *testing.T) {
template, err := os.ReadFile("../../examples/foundation-first-node.lock")
if err != nil {
t.Skip("no example bundle beside this checkout")
}
r, err := Rewrite(template, "sha256:"+strings.Repeat("ab", 32))
if err != nil {
t.Fatal(err)
}
if _, err := RewriteRoot(&r, RootCredentials{Store: "s", Broker: "b"}); err != nil {
t.Fatal(err)
}
for _, res := range r.Declaration.Resources {
a, ok := res.(*declaration.Action)
if !ok || a.ID != "broker-admin" {
continue
}
cmd := strings.Join(a.Command, " ")
if !strings.Contains(cmd, "&& echo ") || strings.Contains(cmd, "printf %s") {
t.Errorf("the marker is written without a line ending: %s", cmd)
}
if !strings.Contains(strings.Join(a.Verify, " "), "read m <") {
t.Errorf("the verify does not read the marker: %v", a.Verify)
}
}
}
// The foundation filters before anything listens: nftables is in place before the store, and its
// rules refuse the store's and broker's client ports from outside while keeping ssh, the bus a node
// enrols over and the registry a node pulls from (novox/hq issue 054).
func TestTheFoundationFiltersBeforeAnythingListens(t *testing.T) {
template, err := os.ReadFile("../../examples/foundation-first-node.lock")
if err != nil {
t.Skip("no example bundle beside this checkout")
}
r, err := Rewrite(template, "sha256:"+strings.Repeat("ab", 32))
if err != nil {
t.Fatal(err)
}
if _, err := RewriteRoot(&r, RootCredentials{Store: "s", Broker: "b"}); err != nil {
r := theExample(t)
if _, err := RewriteRoot(&r, testCredentials(t, "s", "b")); err != nil {
t.Fatal(err)
}
var filterAt, loadedAt, storeAt, brokerAt = -1, -1, -1, -1