Compare commits
8
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f0d39b4a4a | ||
|
|
89fd0c06e1 | ||
|
|
17038b2a07 | ||
|
|
4dde21f0ab | ||
|
|
7d1d6bffe3 | ||
|
|
7283209d1a | ||
|
|
64f0ab4051 | ||
|
|
5f147fe6b9 |
@@ -49,7 +49,7 @@ func TestTheBundleCarriesTheFoundationAndTheControlPlaneAndNothingElse(t *testin
|
||||
}
|
||||
sort.Strings(images)
|
||||
|
||||
want := []string{"mesh-controller", "nats", "postgres"}
|
||||
want := []string{"mesh-controller", "nats", "pgvector"}
|
||||
if strings.Join(images, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("the bundle carries %v; expected exactly %v.\n\n"+
|
||||
"Adding one is a change to what every first node becomes, and to ADR 0006's "+
|
||||
|
||||
@@ -28,11 +28,12 @@
|
||||
// image, and this file is applied on a machine with no mesh to ask about anything (ADR 0006). The
|
||||
// registry prefix is the lab's; the digests are upstream's.
|
||||
//
|
||||
// The store's data is a NAMED VOLUME, not a directory on the machine. A directory the host
|
||||
// creates is owned by root, and the database runs as somebody else inside the container — so it
|
||||
// could not write, and the container crash-looped. A named volume lets the image set up its own
|
||||
// ownership, and outlives the container, which is what you want for the thing holding the mesh's
|
||||
// state.
|
||||
// **The store is raised where the `postgres` module keeps it** (novox/hq ADR 0338): the container
|
||||
// `postgres`, on the module's pgvector image, its data in the directory /var/lib/mesh-store owned by
|
||||
// the image's postgres user, its superuser password read from the module's own-secret file — so the
|
||||
// store step adopts it as it runs. Pinned upstream by the module's own digest, not through the lab's
|
||||
// registry prefix, because the two must be the same reference (mesh-host internal/bootstrap holds
|
||||
// them to each other against the module's manifest).
|
||||
|
||||
{
|
||||
"declaration": 1,
|
||||
@@ -83,17 +84,28 @@
|
||||
"boot": "enabled",
|
||||
"restart-on": ["base-filter"]
|
||||
},
|
||||
// The store's data is a directory owned by the image's postgres user (999), made before the
|
||||
// container starts: a directory the runtime made would be root's, and the database could not
|
||||
// write it. Measured on the module's image 2026-10-13: it initialises a new store on a directory
|
||||
// made so (novox/hq ADR 0338 §5).
|
||||
{
|
||||
"id": "store-data",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/mesh-store",
|
||||
"mode": "0700",
|
||||
"owner": "999:70"
|
||||
},
|
||||
{
|
||||
"id": "store",
|
||||
"type": "container",
|
||||
"name": "mesh-store",
|
||||
"image": "192.0.2.250:5000/postgres@sha256:7abf537131b66ed5af448d90653abf1679b0c7e9a1f07efdd4c3108a401b259a",
|
||||
"name": "postgres",
|
||||
"image": "pgvector/pgvector@sha256:cf134a767f474095eeba57e0117be8e568e011a63f33fbf252f14c9b760f8e6f",
|
||||
"env": {
|
||||
"POSTGRES_PASSWORD": "bootstrap",
|
||||
"PGDATA": "/var/lib/postgresql/data/pgdata"
|
||||
},
|
||||
"ports": ["5432:5432"],
|
||||
"volumes": ["mesh-store-data:/var/lib/postgresql/data"]
|
||||
"volumes": ["/var/lib/mesh-store:/var/lib/postgresql/data"]
|
||||
},
|
||||
// Over TCP, not the socket. While the store initialises it runs a temporary server on the
|
||||
// socket ONLY, then stops it and starts the real one — so a socket check passes, the action
|
||||
@@ -102,21 +114,21 @@
|
||||
{
|
||||
"id": "store-ready",
|
||||
"type": "action",
|
||||
"in": "mesh-store",
|
||||
"in": "postgres",
|
||||
"command": ["sh", "-c", "for i in $(seq 1 180); do pg_isready -h 127.0.0.1 -U postgres >/dev/null 2>&1 && exit 0; sleep 1; done; echo 'the store did not answer within 180s; its own last words follow'; pg_isready -h 127.0.0.1 -U postgres; tail -n 20 /var/lib/postgresql/data/log/*.log 2>/dev/null; exit 1"],
|
||||
"verify": ["pg_isready", "-h", "127.0.0.1", "-U", "postgres"]
|
||||
},
|
||||
{
|
||||
"id": "inventory-database",
|
||||
"type": "action",
|
||||
"in": "mesh-store",
|
||||
"in": "postgres",
|
||||
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE inventory'"],
|
||||
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw inventory"]
|
||||
},
|
||||
{
|
||||
"id": "identity-database",
|
||||
"type": "action",
|
||||
"in": "mesh-store",
|
||||
"in": "postgres",
|
||||
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE identity'"],
|
||||
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw identity"]
|
||||
},
|
||||
@@ -126,20 +138,20 @@
|
||||
{
|
||||
"id": "licences-database",
|
||||
"type": "action",
|
||||
"in": "mesh-store",
|
||||
"in": "postgres",
|
||||
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE licences'"],
|
||||
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw licences"]
|
||||
},
|
||||
{
|
||||
"id": "context-schemas",
|
||||
"type": "action",
|
||||
"command": ["docker", "run", "--rm", "--network", "container:mesh-store",
|
||||
"command": ["docker", "run", "--rm", "--network", "container:postgres",
|
||||
"-e", "MESH_STORE_INVENTORY=postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable",
|
||||
"-e", "MESH_STORE_IDENTITY=postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable",
|
||||
"-e", "MESH_STORE_LICENCES=postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable",
|
||||
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
||||
"migrate"],
|
||||
"verify": ["sh", "-c", "docker exec mesh-store psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec mesh-store psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key && docker exec mesh-store psql -U postgres -d licences -tAc \"select to_regclass('public.licence')\" | grep -qx licence"]
|
||||
"verify": ["sh", "-c", "docker exec postgres psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec postgres psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key && docker exec postgres psql -U postgres -d licences -tAc \"select to_regclass('public.licence')\" | grep -qx licence"]
|
||||
},
|
||||
// **The bus is raised where the `nats` module keeps it** (novox/hq ADR 0333 §1): the container
|
||||
// `mesh-broker-nats`, its configuration and user list in /var/lib/nats-module/conf, its streams in
|
||||
|
||||
@@ -36,6 +36,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/firewall"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
"github.com/novox/mesh-host/internal/tunnel"
|
||||
)
|
||||
|
||||
@@ -292,12 +293,14 @@ type Result struct {
|
||||
Agent string `json:"host-agent,omitempty"`
|
||||
|
||||
// Registry is the mesh's own artifact store, once it answers.
|
||||
Registry string `json:"registry,omitempty"`
|
||||
RegistryReplied int `json:"registry-replied,omitempty"`
|
||||
RegistryKnown bool `json:"registry-already-registered,omitempty"`
|
||||
RegistryRunning string `json:"registry-container,omitempty"`
|
||||
PublishedAs string `json:"control-plane-image,omitempty"`
|
||||
PublishedAlready bool `json:"control-plane-image-already-published,omitempty"`
|
||||
Registry string `json:"registry,omitempty"`
|
||||
RegistryReplied int `json:"registry-replied,omitempty"`
|
||||
RegistryKnown bool `json:"registry-already-registered,omitempty"`
|
||||
RegistryRunning string `json:"registry-container,omitempty"`
|
||||
// RegistryTrusted are the names the runtime was told to trust the registry by, before the push.
|
||||
RegistryTrusted []string `json:"registry-trusted,omitempty"`
|
||||
PublishedAs string `json:"control-plane-image,omitempty"`
|
||||
PublishedAlready bool `json:"control-plane-image-already-published,omitempty"`
|
||||
|
||||
// Permanent is the control plane as an ordinary module.
|
||||
Permanent string `json:"permanent-container,omitempty"`
|
||||
@@ -555,7 +558,9 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
|
||||
// Named rather than attempted. Everything from step 6 on is a conversation with a
|
||||
// control plane that a dry run has not raised, so there is nothing to ask and nothing
|
||||
// honest to report about the answers.
|
||||
say(" would then enrol " + o.Node + ", install the registry from " +
|
||||
say(" would then enrol " + o.Node + ", have the runtime trust the registry as " +
|
||||
strings.Join(RegistryNames(o), ", ") + ",")
|
||||
say(" install the registry from " +
|
||||
o.Catalogue + ", push the control plane's image into it,")
|
||||
say(" reinstall the control plane as a module, and drop " +
|
||||
rewritten.TempName)
|
||||
@@ -654,26 +659,10 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
|
||||
return result, failed(StepOperator, err)
|
||||
}
|
||||
|
||||
// ---- 7. registry ----------------------------------------------------------------------
|
||||
say("registry — somewhere for this mesh to keep its own images")
|
||||
registry, err := InstallRegistry(ctx, o, d, temporary, say)
|
||||
result.Registry, result.RegistryRunning = registry.Address, registry.Container
|
||||
result.RegistryKnown, result.RegistryReplied = registry.Known, registry.Answered
|
||||
// ---- 7. registry, 8. publish -----------------------------------------------------------
|
||||
published, err := RegistryAndPublish(ctx, o, d, sys, temporary, controlPlaneImage, &result, say)
|
||||
if err != nil {
|
||||
return result, failed(StepRegistry, err)
|
||||
}
|
||||
|
||||
// ---- 8. publish -----------------------------------------------------------------------
|
||||
say("publish — the control plane's image gets its first manifest digest")
|
||||
// **The image this mesh built, not the one the installer carried.** The carried one is the
|
||||
// builder; publishing it here would put a builder in the registry under the control plane's
|
||||
// name and install it as the control plane — which is exactly what happened the first time
|
||||
// this ran, and presented as a control plane that started, printed a builder's usage, and
|
||||
// exited cleanly over and over.
|
||||
published, err := PublishControlPlane(ctx, o, d, controlPlaneImage, say)
|
||||
result.PublishedAs, result.PublishedAlready = published.Reference, published.Already
|
||||
if err != nil {
|
||||
return result, failed(StepPublish, err)
|
||||
return result, err
|
||||
}
|
||||
|
||||
// ---- 9. control plane -----------------------------------------------------------------
|
||||
@@ -803,3 +792,44 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// RegistryAndPublish is steps 7 and 8: the runtime trusts the mesh's registry, the registry is
|
||||
// installed and answers, and the control plane's image is pushed into it.
|
||||
//
|
||||
// **One function so the order is held by a test** (novox/hq issue 92). The runtime refuses plain
|
||||
// HTTP to a registry it has not been told to trust, and the module that tells it is applied long
|
||||
// after the push — so the trust is written and confirmed here, before anything is tagged or pushed.
|
||||
// A refusal names the step it happened in.
|
||||
func RegistryAndPublish(ctx context.Context, o Options, d Deps, sys system.System, temporary controlPlane,
|
||||
controlPlaneImage string, result *Result, say func(string)) (Published, error) {
|
||||
|
||||
// ---- 7. registry ----------------------------------------------------------------------
|
||||
say("registry — somewhere for this mesh to keep its own images")
|
||||
// Trusted by the runtime before anything is pushed to it, by every name the mesh will use for
|
||||
// it here: the module that writes this trust is applied long after the push (novox/hq issue 92).
|
||||
trusted, err := TrustTheRegistry(ctx, o, sys, d.Run, say)
|
||||
result.RegistryTrusted = trusted
|
||||
if err != nil {
|
||||
return Published{}, failed(StepRegistry, err)
|
||||
}
|
||||
registry, err := InstallRegistry(ctx, o, d, temporary, say)
|
||||
result.Registry, result.RegistryRunning = registry.Address, registry.Container
|
||||
result.RegistryKnown, result.RegistryReplied = registry.Known, registry.Answered
|
||||
if err != nil {
|
||||
return Published{}, failed(StepRegistry, err)
|
||||
}
|
||||
|
||||
// ---- 8. publish -----------------------------------------------------------------------
|
||||
say("publish — the control plane's image gets its first manifest digest")
|
||||
// **The image this mesh built, not the one the installer carried.** The carried one is the
|
||||
// builder; publishing it here would put a builder in the registry under the control plane's
|
||||
// name and install it as the control plane — which is exactly what happened the first time
|
||||
// this ran, and presented as a control plane that started, printed a builder's usage, and
|
||||
// exited cleanly over and over.
|
||||
published, err := PublishControlPlane(ctx, o, d, controlPlaneImage, say)
|
||||
result.PublishedAs, result.PublishedAlready = published.Reference, published.Already
|
||||
if err != nil {
|
||||
return published, failed(StepPublish, err)
|
||||
}
|
||||
return published, nil
|
||||
}
|
||||
|
||||
@@ -38,27 +38,6 @@ func recording(t *testing.T) (*controlRecorder, controlPlane) {
|
||||
return c, controlPlane{container: "temp-mesh-controller", run: c.run, timeout: time.Second}
|
||||
}
|
||||
|
||||
func TestTheBuildersPackageRegistryPortIsASettingAndNotTheManifest(t *testing.T) {
|
||||
c, control := recording(t)
|
||||
o := Options{Node: "anchor", Ports: FoundationPorts{Packages: 3100}, Wait: time.Second}
|
||||
if _, err := installModule(context.Background(), o, control, BuilderModule,
|
||||
[]byte(theBuildersBinding), quietly); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := c.settings[BuilderModule+"-settings.json"]; got != `{"serves":{"port":3100}}` {
|
||||
t.Errorf("the builder was told %q about the package registry's port", got)
|
||||
}
|
||||
// The manifest reaches the mesh as the catalogue wrote it. A port rewritten into it here is a
|
||||
// port the next registration from the catalogue silently takes back.
|
||||
if got := c.settings[BuilderModule+"-module.json"]; got != theBuildersBinding {
|
||||
t.Errorf("the installer changed the builder's manifest:\n%s", got)
|
||||
}
|
||||
set, push := c.index("settings set "+BuilderModule), c.index("push anchor")
|
||||
if set < 0 || push < 0 || set > push {
|
||||
t.Fatalf("the port was not set before the push that raises the builder: %v", c.told)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheBuilderOnTheDefaultPackagesPortIsToldNothing(t *testing.T) {
|
||||
c, control := recording(t)
|
||||
o := Options{Node: "anchor", Wait: time.Second}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Genesis seals the build agent the forge it raised (novox/hq ADR 0336 §3, issue 121): the agent's
|
||||
// npm-package-registry requirement is optional and binds only once the gitea module is assigned, after
|
||||
// the base, so until then the packages step's genesis document is how the sdk and base steps reach the
|
||||
// forge.
|
||||
|
||||
func TestThePackagesStepSealsTheGenesisDocumentToTheBuildAgent(t *testing.T) {
|
||||
if BuilderModule != "build-agent" {
|
||||
t.Fatalf("genesis installs %q; the document is the build agent's", BuilderModule)
|
||||
}
|
||||
c, control := recording(t)
|
||||
o := Options{Node: "anchor", Ports: FoundationPorts{Packages: 3100}, Wait: time.Second}
|
||||
doc := genesisRegistry(o.Ports.orDefaults(), "builder-password")
|
||||
if err := deliverGenesisRegistry(context.Background(), o, control, doc, quietly); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
carried := c.settings["accepting-"+GenesisRegistrySecret]
|
||||
var got GenesisRegistry
|
||||
if err := json.Unmarshal([]byte(carried), &got); err != nil {
|
||||
t.Fatalf("the genesis document carried is not JSON: %q", carried)
|
||||
}
|
||||
want := GenesisRegistry{Scheme: "http", At: "127.0.0.1", Port: 3100, NpmPath: "/api/packages/novox/npm/",
|
||||
As: "mesh-builder", Password: "builder-password"}
|
||||
if got != want {
|
||||
t.Fatalf("the genesis document says %+v, want %+v", got, want)
|
||||
}
|
||||
accept := c.index("secret accept anchor build-agent genesis-package-registry --from /accepting-genesis-package-registry")
|
||||
push := c.index("push anchor")
|
||||
if accept < 0 || push < 0 || accept > push {
|
||||
t.Fatalf("the genesis document was not sealed to the build agent before the push: %v", c.told)
|
||||
}
|
||||
}
|
||||
|
||||
// What the retired builder was given and the build agent declares neither of: an `npm-password`
|
||||
// own-secret and a `serves` setting for its carried binding (ADR 0336 §3).
|
||||
func TestNoStepGivesTheBuildAgentAServesSettingOrAnNpmPassword(t *testing.T) {
|
||||
for _, module := range []string{BuilderModule} {
|
||||
c, control := recording(t)
|
||||
o := Options{Node: "anchor", Ports: FoundationPorts{Packages: 3100}, Wait: time.Second}
|
||||
if _, err := installModule(context.Background(), o, control, module, []byte(`{"module":"`+module+`"}`), quietly); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, set := c.settings[module+"-settings.json"]; set {
|
||||
t.Errorf("%s was given a setting on a node with a non-default packages port: %s", module, got)
|
||||
}
|
||||
if err := deliverGenesisRegistry(context.Background(), o, control,
|
||||
genesisRegistry(o.Ports.orDefaults(), "p"), quietly); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, told := range c.told {
|
||||
if strings.Contains(told, "npm-password") || strings.Contains(told, "serves") {
|
||||
t.Errorf("genesis told the controller %q", told)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -137,7 +137,26 @@ func registerAndAssign(ctx context.Context, o Options, control controlPlane, mod
|
||||
// should hold, seals every secret in them and publishes them, and on a first node that is the
|
||||
// slowest thing the mesh does.
|
||||
func pushNode(ctx context.Context, o Options, control controlPlane, say func(string)) (string, error) {
|
||||
said, err := control.within(o.Wait).tell(ctx, "push", o.Node)
|
||||
return pushing(ctx, o, control, false, say)
|
||||
}
|
||||
|
||||
// pushNodeApplied is pushNode that returns only once the node reports it applied exactly what it was
|
||||
// sent (`push <node> --wait`, novox/hq ADR 0010): for a push whose content the very next step reads
|
||||
// on the machine. A plain push returns as soon as the declaration is sent, and the node applies it a
|
||||
// moment later (issue 615).
|
||||
func pushNodeApplied(ctx context.Context, o Options, control controlPlane, say func(string)) (string, error) {
|
||||
return pushing(ctx, o, control, true, say)
|
||||
}
|
||||
|
||||
func pushing(ctx context.Context, o Options, control controlPlane, applied bool, say func(string)) (string, error) {
|
||||
args := []string{"push", o.Node}
|
||||
patience := o.Wait
|
||||
if applied {
|
||||
args = append(args, "--wait", o.Wait.String())
|
||||
// The controller's own wait, and time to compose and send before it starts.
|
||||
patience = 2 * o.Wait
|
||||
}
|
||||
said, err := control.within(patience).tell(ctx, args...)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf(
|
||||
"%w\n\nWhat was registered and assigned is registered and assigned, and this node has "+
|
||||
@@ -145,7 +164,20 @@ func pushNode(ctx context.Context, o Options, control controlPlane, say func(str
|
||||
"at all. Fix what it named and run this installer again: it will find the module "+
|
||||
"already registered and try the push again", err)
|
||||
}
|
||||
say(" pushed " + o.Node)
|
||||
if applied {
|
||||
say(" pushed " + o.Node + ", and it applied it")
|
||||
} else {
|
||||
say(" pushed " + o.Node)
|
||||
}
|
||||
// **Every push may carry a new bus user, and until the adoption the bus hears of it only from here**
|
||||
// (novox/hq issue 608, ADR 0331 §5, ADR 0333 §4). A module's account is minted into the mesh's
|
||||
// records and sealed to this node with the push; the bus reads its user list from a file, and until
|
||||
// the broker step adopts the bus as the nats module nothing but this installer writes that file. The
|
||||
// list was placed only when the node enrolled, so the build agent's account (step 13) never reached
|
||||
// the bus and the agent was refused ("Authorization Violation") until the sdk step gave up.
|
||||
if err := placeUntilAdopted(ctx, control, say); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return strings.TrimSpace(said), nil
|
||||
}
|
||||
|
||||
|
||||
@@ -84,7 +84,7 @@ func InstallStore(ctx context.Context, o Options, control controlPlane,
|
||||
say); err != nil {
|
||||
return err
|
||||
}
|
||||
say(" adopted mesh-store — the foundation's store is now the " + module + " module")
|
||||
say(" adopted " + store.Name + " — the foundation's store is now the " + module + " module")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -478,11 +478,3 @@ func serverMatchesFoundation(manifest []byte, store *declaration.Container, modu
|
||||
"store with. Its server container has to carry the name the foundation raised",
|
||||
module, store.Name)
|
||||
}
|
||||
|
||||
// orUnknown is s, or a word for nothing: trust.go held it, and this branch takes trust.go out.
|
||||
func orUnknown(s string) string {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
return "(nothing)"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
@@ -24,7 +25,7 @@ import (
|
||||
const (
|
||||
// foundationStore is the foundation's postgres container — the mesh's own memory, raised from the
|
||||
// bundle. gitea's bootstrap database lives here too, so a mesh runs one postgres (issue 051).
|
||||
foundationStore = "mesh-store"
|
||||
foundationStore = "postgres"
|
||||
// giteaBootstrap is the gitea server raised directly at genesis, before gitea is a module —
|
||||
// under the name the gitea MODULE declares for its container, so the module finds it and holds
|
||||
// it rather than raising a second forge beside it (novox/hq ADR 0163, rule 7; issue 090).
|
||||
@@ -49,8 +50,8 @@ const (
|
||||
packagesTeam = "packages"
|
||||
// giteaAdminUser is the admin the bootstrap creates and the provisioner later authenticates as.
|
||||
giteaAdminUser = "mesh-admin"
|
||||
// builderGiteaUser is the gitea account the builder publishes and pulls with at genesis. It is
|
||||
// the `as` the builder's static package binding names.
|
||||
// builderGiteaUser is the gitea account the build agent publishes and pulls with at genesis: the
|
||||
// `as` of the genesis document (novox/hq ADR 0336 §3). Retiring it is issue 602.
|
||||
builderGiteaUser = "mesh-builder"
|
||||
// giteaDBRole/giteaDBName is gitea's own database in the foundation store.
|
||||
giteaDBRole = "mesh_gitea"
|
||||
@@ -133,8 +134,8 @@ func RaisePackageRegistry(ctx context.Context, o Options, d Deps, control contro
|
||||
return err
|
||||
}
|
||||
|
||||
say(" delivering the builder its registry credential")
|
||||
if err := deliverBuilderNpm(ctx, o, control, builderPassword, say); err != nil {
|
||||
say(" sealing the build agent the genesis document for the forge it raised")
|
||||
if err := deliverGenesisRegistry(ctx, o, control, genesisRegistry(ports, builderPassword), say); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -362,27 +363,69 @@ func packagePasswords() (db, admin, builder string, err error) {
|
||||
return db, admin, builder, nil
|
||||
}
|
||||
|
||||
// deliverBuilderNpm seals the builder's registry password to this node as its `npm-password`
|
||||
// own-secret, the same way the control plane's store connections are delivered — carry the value in,
|
||||
// `secret accept`, and the next push writes it sealed where the builder reads it.
|
||||
func deliverBuilderNpm(ctx context.Context, o Options, control controlPlane, password string,
|
||||
say func(string)) error {
|
||||
at := "/accepting-npm-password"
|
||||
if err := control.carryingSecret(ctx, "mesh-accepting-npm-password", []byte(password), at); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := control.tell(ctx, "secret", "accept", o.Node, BuilderModule, "npm-password", "--from", at); err != nil {
|
||||
return err
|
||||
}
|
||||
// Push so the sealed secret reaches the builder, which restarts on it and comes back credentialed.
|
||||
if _, err := control.tell(ctx, "push", o.Node); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
// GenesisRegistrySecret is the build agent's own-secret the genesis document is sealed as (novox/hq
|
||||
// ADR 0336 §3). The build-agent manifest declares it, issued outside the mesh: only this installer
|
||||
// gives one, on genesis's node.
|
||||
const GenesisRegistrySecret = "genesis-package-registry"
|
||||
|
||||
// GenesisRegistry is the genesis document (novox/hq ADR 0336 §3): the forge this step raised, the port
|
||||
// this node gave the package registry, its npm path, and the account this step made with its password.
|
||||
// mesh-controller's cmd/mesh-builder reads it under the same field names while the build agent's
|
||||
// npm-package-registry requirement is unbound, and prefers the binding once the gitea module is
|
||||
// assigned and it binds.
|
||||
type GenesisRegistry struct {
|
||||
Scheme string `json:"scheme"`
|
||||
At string `json:"at"`
|
||||
Port int `json:"port"`
|
||||
NpmPath string `json:"npm-path"`
|
||||
As string `json:"as"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
|
||||
// PublishTheSDK dispatches a build of the SDK to the builder. The builder has its registry
|
||||
// credential by now, so the build's `npm publish` authenticates; the artifact is a `package`, built
|
||||
// genesisRegistry is the document for the forge raised here: on the machine's own loopback, where it
|
||||
// answers on the host network and where its ROOT_URL says it is (raiseGiteaServer), on the port this
|
||||
// node gave the package registry.
|
||||
func genesisRegistry(ports FoundationPorts, password string) GenesisRegistry {
|
||||
return GenesisRegistry{Scheme: "http", At: "127.0.0.1", Port: ports.Packages,
|
||||
NpmPath: "/api/packages/" + packagesOrg + "/npm/", As: builderGiteaUser, Password: password}
|
||||
}
|
||||
|
||||
// deliverGenesisRegistry seals the genesis document to the build agent on this node as its
|
||||
// `genesis-package-registry` own-secret (novox/hq ADR 0336 §3), the same way the control plane's store
|
||||
// connections are delivered — carry the value in, `secret accept`, and the next push writes it sealed
|
||||
// where the build agent reads it.
|
||||
//
|
||||
// **The build agent's one way to the forge until the registry binds.** Its npm-package-registry
|
||||
// requirement is optional (ADR 0336 §1), and nothing provides it until the gitea module is assigned
|
||||
// after the base, so the sdk step (15) publishes and the base step (16) resolves with the account in
|
||||
// this document. It replaces the retired builder's `npm-password` own-secret and the `serves.port`
|
||||
// setting its carried binding took, neither of which the build agent declares.
|
||||
func deliverGenesisRegistry(ctx context.Context, o Options, control controlPlane, doc GenesisRegistry,
|
||||
say func(string)) error {
|
||||
raw, err := json.Marshal(doc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
at := "/accepting-" + GenesisRegistrySecret
|
||||
if err := control.carryingSecret(ctx, "mesh-accepting-"+GenesisRegistrySecret, raw, at); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := control.tell(ctx, "secret", "accept", o.Node, BuilderModule, GenesisRegistrySecret, "--from", at); err != nil {
|
||||
return err
|
||||
}
|
||||
// Push so the sealed document reaches the build agent, which reads it at its next build — and,
|
||||
// like every push before the adoption, the bus hears the users it carries (issue 608).
|
||||
//
|
||||
// **Waited for until the node applied it** (issue 615). The next step asks the build agent to
|
||||
// publish the SDK, and the agent reads the document from the file the node writes when it applies
|
||||
// this push. A push that returned on sending let the ask race the apply: one genesis found the
|
||||
// document in time, the next was refused, "neither its binding nor the genesis document".
|
||||
_, err = pushNodeApplied(ctx, o, control, say)
|
||||
return err
|
||||
}
|
||||
|
||||
// PublishTheSDK dispatches a build of the SDK to the build agent. The agent has the genesis document by
|
||||
// now (novox/hq ADR 0336 §7), so the build's `npm publish` authenticates; the artifact is a `package`, built
|
||||
// on a public base, so this needs no toolchain — which is the whole point of doing it before the base.
|
||||
func PublishTheSDK(ctx context.Context, o Options, control controlPlane, say func(string)) error {
|
||||
if o.SDKSource.Repository == "" {
|
||||
|
||||
@@ -117,24 +117,6 @@ const busModulePort = 4222
|
||||
// PortsSetting is the controller's settings key for a module's given ports.
|
||||
const PortsSetting = "ports"
|
||||
|
||||
// ServesSetting is the settings key that carries what a provider serves, inside the one binding a
|
||||
// module carries rather than resolves — the builder's package binding (novox/hq 04-ISSUES/085).
|
||||
//
|
||||
// **A port given at genesis has to be a setting wherever it is read, or something puts it back.**
|
||||
// The builder reaches the package registry through a binding written in its own manifest, because
|
||||
// at genesis no module yet provides `package-registry` and so there is nothing for the controller
|
||||
// to resolve it from. That binding used to be rewritten, as text, when the installer registered the
|
||||
// builder — which a later registration from the catalogue undid silently. Set as a node's setting
|
||||
// instead, it is held by the controller rather than by the manifest, so re-registering the builder
|
||||
// leaves it where it was.
|
||||
//
|
||||
// **A settings layer is the module's, not one resource's.** The controller lays it over every file
|
||||
// of that module which merges as JSON, so `serves` lands in the package binding only because that
|
||||
// binding is the builder's one mergeable file. A second mergeable file added to the builder would
|
||||
// be given a `serves` key too, meaning nothing to whatever reads it. Worth knowing before adding
|
||||
// one.
|
||||
const ServesSetting = "serves"
|
||||
|
||||
// setFoundationSettings tells the controller the ports this node gave a foundation module — and,
|
||||
// on an adopted node, that the registry is reached from anywhere, as a node pulls from it before it
|
||||
// has a private-network address (novox/hq ADR 0100). Done after the module is registered and before
|
||||
@@ -148,13 +130,8 @@ func setFoundationSettings(ctx context.Context, o Options, control controlPlane,
|
||||
if o.Adopted && module == RegistryModule {
|
||||
values["expose"] = map[string]string{strconv.Itoa(DefaultPorts().Registry): "anywhere"}
|
||||
}
|
||||
if packages := o.Ports.orDefaults().Packages; module == BuilderModule &&
|
||||
packages != DefaultPorts().Packages {
|
||||
// The one binding nothing resolves: the builder dials the forge by a number it carries.
|
||||
// The rest of the binding — the provision, the forge it is from, the account it presents —
|
||||
// is the manifest's and stays there.
|
||||
values[ServesSetting] = map[string]int{"port": packages}
|
||||
}
|
||||
// No `serves` setting for the build agent: the port given for the package registry reaches it in
|
||||
// the genesis document (novox/hq ADR 0336 §3), and from the binding once the registry binds.
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -68,7 +68,7 @@ func TestGivenPortsMoveOnlyTheMachinesSide(t *testing.T) {
|
||||
if got.Places == 0 {
|
||||
t.Fatal("nothing was rewritten")
|
||||
}
|
||||
storeC := containerNamed(r.Declaration, "mesh-store")
|
||||
storeC := containerNamed(r.Declaration, "postgres")
|
||||
if len(storeC.Ports) != 1 || storeC.Ports[0] != "5433:5432" {
|
||||
t.Errorf("the store publishes %v", storeC.Ports)
|
||||
}
|
||||
@@ -239,6 +239,12 @@ func (c *controlRecorder) run(_ context.Context, name string, args ...string) (s
|
||||
}
|
||||
if name == "docker" && args[0] == "exec" {
|
||||
c.told = append(c.told, strings.Join(args[3:], " "))
|
||||
if strings.Join(args[3:], " ") == "broker accounts" {
|
||||
return "accounts {}\n", nil // the composed user list, placed after every push (issue 608)
|
||||
}
|
||||
}
|
||||
if name == "sh" {
|
||||
c.told = append(c.told, "placed the bus's users")
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
@@ -127,6 +127,13 @@ func Preflight(ctx context.Context, o Options, d Deps, say func(string)) ([]byte
|
||||
if err := RefuseAMachineInUse(ctx, o, d.Run, say); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The registry's trust is written at step 7, before the push (novox/hq issue 92). What can be
|
||||
// known about whether it will take, without writing anything, is asked now.
|
||||
if o.pivots() {
|
||||
if err := RuntimeWillTakeTheTrust(ctx, o, d.Run, say); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Can this machine reach what the bundle's images come from?
|
||||
//
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Defends novox/hq issue 608 (ADR 0331 §5, ADR 0333 §4): until the bus is adopted as the nats module,
|
||||
// every push genesis makes is followed by the installer placing the composed user list where the bus
|
||||
// reads it, and making the bus re-read it. The build agent's account was minted at step 13 and sealed
|
||||
// with that push, and the list was placed only at enrolment, so the bus refused the agent
|
||||
// ("Authorization Violation") and genesis stopped at step 15.
|
||||
|
||||
// placing answers the controller as a mesh whose bus is or is not the nats module yet, and records
|
||||
// what was run, in order.
|
||||
func placing(adopted bool) (*asked, controlPlane) {
|
||||
listed := "build-agent 1\n"
|
||||
if adopted {
|
||||
listed += "nats 1\n"
|
||||
}
|
||||
runtime := &asked{}
|
||||
runtime.answer = func(name string, args []string) (string, error) {
|
||||
joined := strings.Join(args, " ")
|
||||
switch {
|
||||
case name == "sh":
|
||||
return "", nil
|
||||
case strings.HasSuffix(joined, "module list"):
|
||||
return listed, nil
|
||||
case strings.HasSuffix(joined, "broker accounts"):
|
||||
return "accounts {\n MESH { users = [ { user: \"anchor.build-agent\" } ] }\n}\n", nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
return runtime, controlPlane{container: "mesh-controller", run: runtime.run, timeout: time.Second}
|
||||
}
|
||||
|
||||
// placedAfterThePush is whether the bus's users were placed — written at the module's bus-users path,
|
||||
// the bus told to re-read them — after the push, and how often.
|
||||
func placedAfterThePush(t *testing.T, commands []string) int {
|
||||
t.Helper()
|
||||
pushed, placed := -1, 0
|
||||
for i, c := range commands {
|
||||
if strings.Contains(c, "/mesh-controller push anchor") && pushed < 0 {
|
||||
pushed = i
|
||||
}
|
||||
if strings.HasPrefix(c, "sh -c") && strings.Contains(c, busAccounts) &&
|
||||
strings.Contains(c, "kill -s HUP "+busContainer) {
|
||||
if pushed < 0 || i < pushed {
|
||||
t.Fatalf("the bus's users were placed before the push that carries the account: %v", commands)
|
||||
}
|
||||
placed++
|
||||
}
|
||||
}
|
||||
if pushed < 0 {
|
||||
t.Fatalf("nothing was pushed: %v", commands)
|
||||
}
|
||||
return placed
|
||||
}
|
||||
|
||||
func TestTheBusHearsTheBuildAgentsAccountAfterItsPush(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
runtime, control := placing(false)
|
||||
o := Options{Node: "anchor", Wait: time.Second}
|
||||
if _, err := installModule(context.Background(), o, control, BuilderModule, []byte(`{}`), quietly); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := placedAfterThePush(t, runtime.commands); n != 1 {
|
||||
t.Fatalf("the bus's users were placed %d time(s) after the build agent's push; once: %v", n, runtime.commands)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheGenesisDocumentsPushIsHeardByTheBusToo(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
runtime, control := placing(false)
|
||||
o := Options{Node: "anchor", Wait: time.Second}
|
||||
if err := deliverGenesisRegistry(context.Background(), o, control, genesisRegistry(DefaultPorts(), "pw"), quietly); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := placedAfterThePush(t, runtime.commands); n != 1 {
|
||||
t.Fatalf("the bus's users were placed %d time(s) after the genesis document's push; once: %v", n, runtime.commands)
|
||||
}
|
||||
}
|
||||
|
||||
// From the adoption on the nats module writes the list, and the installer does not (ADR 0333 §4).
|
||||
func TestOnceTheBusIsTheNatsModulesThePushPlacesNothing(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
runtime, control := placing(true)
|
||||
o := Options{Node: "anchor", Wait: time.Second}
|
||||
if _, err := installModule(context.Background(), o, control, BuilderModule, []byte(`{}`), quietly); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := placedAfterThePush(t, runtime.commands); n != 0 {
|
||||
t.Fatalf("the installer placed the bus's users over the nats module's: %v", runtime.commands)
|
||||
}
|
||||
}
|
||||
|
||||
// Defends novox/hq issue 615: the packages step's push is waited for until the node applied it, because
|
||||
// the sdk step right after asks the build agent for a build that reads the genesis document from the
|
||||
// file that apply writes. A push that returned on sending let the ask race the apply.
|
||||
func TestThePackagesStepWaitsUntilTheNodeAppliedTheGenesisDocument(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
runtime, control := placing(false)
|
||||
o := Options{Node: "anchor", Wait: 3 * time.Minute}
|
||||
if err := deliverGenesisRegistry(context.Background(), o, control, genesisRegistry(DefaultPorts(), "pw"), quietly); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range runtime.commands {
|
||||
if strings.Contains(c, "/mesh-controller push anchor") {
|
||||
if !strings.HasSuffix(c, "push anchor --wait 3m0s") {
|
||||
t.Fatalf("the genesis document's push does not wait for the node to apply it: %s", c)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("the genesis document was never pushed: %v", runtime.commands)
|
||||
}
|
||||
|
||||
// Every other push of genesis still returns on sending: only a push whose content the next step reads on
|
||||
// the machine waits.
|
||||
func TestAnOrdinaryPushDoesNotWait(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
runtime, control := placing(false)
|
||||
o := Options{Node: "anchor", Wait: time.Second}
|
||||
if _, err := installModule(context.Background(), o, control, BuilderModule, []byte(`{}`), quietly); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range runtime.commands {
|
||||
if strings.Contains(c, "push anchor --wait") {
|
||||
t.Fatalf("an ordinary push waits: %s", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -58,8 +58,12 @@ func aMeshThatAgrees(answers map[string]string) func(string, []string) (string,
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case name == "sh" && len(args) == 2 && strings.Contains(args[1], "accounts.conf"):
|
||||
return "", nil // the bus's users placed after a push (issue 608)
|
||||
case name != "docker":
|
||||
return "", fmt.Errorf("unexpected program %q", name)
|
||||
case strings.HasSuffix(joined, "broker accounts"):
|
||||
return "accounts {}\n", nil
|
||||
case args[0] == "cp":
|
||||
return "", nil
|
||||
case args[0] == "container":
|
||||
|
||||
@@ -47,6 +47,9 @@ const (
|
||||
// StoreSuperuserFile is where the store's superuser password lives on the machine — the
|
||||
// postgres module's own-secret path, so genesis and adoption write the same file.
|
||||
StoreSuperuserFile = "/var/lib/postgres/superuser.secret"
|
||||
// StoreDataDir is where the store keeps its data — the postgres module's data directory, so
|
||||
// genesis raises the store where the module adopts it (novox/hq ADR 0338 §1).
|
||||
StoreDataDir = "/var/lib/mesh-store"
|
||||
// BusControllerFile is where the controller's password on the bus lives on the machine. Not a
|
||||
// module's path: no module reads it — the permanent controller is handed the address the
|
||||
// temporary one was given — and genesis keeps it only so a second run puts the same account
|
||||
@@ -62,7 +65,7 @@ const (
|
||||
// What the template says, matched exactly. A template that says something else is a template
|
||||
// this installer does not know how to make safe, and it says so rather than guessing.
|
||||
templateStorePassword = `"POSTGRES_PASSWORD": "bootstrap"`
|
||||
templateStoreVolumes = `"volumes": ["mesh-store-data:/var/lib/postgresql/data"]`
|
||||
templateStoreVolumes = `"volumes": ["` + StoreDataDir + `:/var/lib/postgresql/data"]`
|
||||
templateStoreURL = "postgres:bootstrap@"
|
||||
|
||||
// BusURLPlaceholder is the controller's account in its bus address, as the template names it.
|
||||
@@ -265,8 +268,8 @@ func Masking(say func(string), c RootCredentials) func(string) {
|
||||
// RefuseExistingServers stops a run that would put a made credential in front of a store raised
|
||||
// by an earlier installer with the template's.
|
||||
//
|
||||
// The store's password is set by initdb, once, on an empty volume. A machine that already holds
|
||||
// `mesh-store-data` and has no credential file was raised with `bootstrap`, and minting a new
|
||||
// The store's password is set by initdb, once, on an empty data directory. A machine that already holds
|
||||
// a cluster in the store's data directory and has no credential file was raised with `bootstrap`, and minting a new
|
||||
// value here would make a bundle that dials with a password the server does not have — failing
|
||||
// three steps later, in the schemas, with nothing pointing back here. Refused by name instead,
|
||||
// with the way forward.
|
||||
@@ -278,11 +281,14 @@ func RefuseExistingServers(ctx context.Context, run Runner, c RootCredentials) e
|
||||
if !c.StoreMade {
|
||||
return nil
|
||||
}
|
||||
if _, err := run(ctx, "docker", "volume", "inspect", "mesh-store-data"); err != nil {
|
||||
return nil // no such volume: a fresh machine, which is the case this installer makes
|
||||
// Where the store keeps its data, as the postgres module does (novox/hq ADR 0338 §2): a cluster
|
||||
// there is one initdb already set a password for. The docker volume genesis used before is not
|
||||
// the store's data any more, and a machine holding only that is not refused.
|
||||
if _, err := run(ctx, "test", "-e", StoreDataDir+"/pgdata/PG_VERSION"); err != nil {
|
||||
return nil // no cluster there: a fresh machine, which is the case this installer makes
|
||||
}
|
||||
return fmt.Errorf(
|
||||
"this machine already holds the store's data (volume mesh-store-data) and no credential at %s, so it was raised "+
|
||||
"this machine already holds the store's data (in "+StoreDataDir+") and no credential at %s, so it was raised "+
|
||||
"by an earlier installer with the template's password. A new one made here would not open it. "+
|
||||
"Put the password the store has into %s (0600, the value alone) and run again; then change it "+
|
||||
"on the server and accept the new value — this installer does not rotate a running store",
|
||||
@@ -311,7 +317,7 @@ func RewriteRoot(r *Rewritten, c RootCredentials) (RootRewrite, error) {
|
||||
return out, err
|
||||
}
|
||||
if bundle, err = replaceOnce(bundle, templateStoreVolumes,
|
||||
`"volumes": ["mesh-store-data:/var/lib/postgresql/data", "`+StoreSuperuserFile+":"+storeSuperuserMount+`:ro"]`,
|
||||
`"volumes": ["`+StoreDataDir+`:/var/lib/postgresql/data", "`+StoreSuperuserFile+":"+storeSuperuserMount+`:ro"]`,
|
||||
"the store's volumes"); err != nil {
|
||||
return out, err
|
||||
}
|
||||
|
||||
@@ -59,7 +59,7 @@ func TestTheProducedBundleCarriesNoWellKnownCredential(t *testing.T) {
|
||||
switch x := res.(type) {
|
||||
case *declaration.Container:
|
||||
switch x.Name {
|
||||
case "mesh-store":
|
||||
case "postgres":
|
||||
store = true
|
||||
if _, has := x.Env["POSTGRES_PASSWORD"]; has {
|
||||
t.Error("the store still takes its password from its environment")
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
)
|
||||
|
||||
// Defends novox/hq ADR 0338 (issue 609): genesis raises its store as the postgres module's server, so
|
||||
// the store step adopts it as it runs. Until then the bundle raised `mesh-store`, on another image, in a
|
||||
// docker volume, and genesis stopped at step 17 of 24.
|
||||
|
||||
// theStoreGenesisRaises is the store container of the bundle as the installer applies it: rewritten for
|
||||
// this machine's credentials, so its password is the file the module reads too.
|
||||
func theStoreGenesisRaises(t *testing.T) *declaration.Container {
|
||||
t.Helper()
|
||||
r := producedBundle(t)
|
||||
store, err := storeIn(r.Declaration)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return store
|
||||
}
|
||||
|
||||
// The bundle's store is exactly the catalogue's postgres module's server, read from the module's own
|
||||
// manifest (ADR 0338 §1): its name, its image (so a drift of either pin fails here), where its data is,
|
||||
// and the file its superuser password is read from, with the environment that says so.
|
||||
func TestTheBundleRaisesTheStoreAsTheCataloguesPostgresModuleDeclaresIt(t *testing.T) {
|
||||
manifest := theCataloguesModule(t, "postgres")
|
||||
var m struct {
|
||||
Resources []struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Name string `json:"name"`
|
||||
Image string `json:"image"`
|
||||
Env map[string]string `json:"env"`
|
||||
Ports []string `json:"ports"`
|
||||
Volumes []string `json:"volumes"`
|
||||
} `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(manifest, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ours := theStoreGenesisRaises(t)
|
||||
found := false
|
||||
for _, r := range m.Resources {
|
||||
if r.Type != "container" || r.ID != "server" {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
if ours.Name != r.Name {
|
||||
t.Errorf("the bundle raises the store as %q; the module's server is %q", ours.Name, r.Name)
|
||||
}
|
||||
if ours.Image != r.Image {
|
||||
t.Errorf("the bundle raises the store on %s; the module pins %s — the two pins move together", ours.Image, r.Image)
|
||||
}
|
||||
for _, key := range []string{"POSTGRES_PASSWORD_FILE", "PGDATA"} {
|
||||
if ours.Env[key] != r.Env[key] {
|
||||
t.Errorf("the bundle's store has %s=%q; the module's %q", key, ours.Env[key], r.Env[key])
|
||||
}
|
||||
}
|
||||
if _, has := ours.Env["POSTGRES_PASSWORD"]; has {
|
||||
t.Error("the bundle's store takes its password from its environment; the module reads a file")
|
||||
}
|
||||
// The module's own directory for its state is <data root>/<module>, where genesis keeps the
|
||||
// superuser's file (StoreSuperuserFile).
|
||||
var theirs []string
|
||||
for _, v := range r.Volumes {
|
||||
theirs = append(theirs, strings.ReplaceAll(v, "${dir:state}", "/var/lib/postgres"))
|
||||
}
|
||||
if strings.Join(ours.Volumes, " ") != strings.Join(theirs, " ") {
|
||||
t.Errorf("the bundle's store mounts %v; the module's %v", ours.Volumes, theirs)
|
||||
}
|
||||
if strings.Join(ours.Ports, " ") != strings.Join(r.Ports, " ") {
|
||||
t.Errorf("the bundle publishes the store on %v; the module on %v", ours.Ports, r.Ports)
|
||||
}
|
||||
// And the store step's own check agrees: by name and image, before anything is registered.
|
||||
if err := serverMatchesFoundation(manifest, ours, "postgres"); err != nil {
|
||||
t.Errorf("the store step would refuse the store the bundle raises: %v", err)
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("the postgres module declares no server container")
|
||||
}
|
||||
}
|
||||
|
||||
// The store's data directory is made by the bundle before the store starts, owned by the image's
|
||||
// postgres user, as the module declares it (ADR 0338 §1, and the measurement under its §5).
|
||||
func TestTheBundleMakesTheStoresDataDirectoryAsTheModuleDoes(t *testing.T) {
|
||||
r := producedBundle(t)
|
||||
at, store := -1, -1
|
||||
for i, res := range r.Declaration.Resources {
|
||||
if d, ok := res.(*declaration.Directory); ok && d.Path == StoreDataDir {
|
||||
at = i
|
||||
if d.Mode != "0700" || d.Owner != "999:70" {
|
||||
t.Errorf("the store's data directory is made %s %s; the module makes it 0700 999:70", d.Mode, d.Owner)
|
||||
}
|
||||
}
|
||||
if res.Identity() == StoreID {
|
||||
store = i
|
||||
}
|
||||
}
|
||||
if at < 0 || store < 0 || at > store {
|
||||
t.Fatalf("the store's data directory %s is not made before the store (directory at %d, store at %d)", StoreDataDir, at, store)
|
||||
}
|
||||
}
|
||||
|
||||
// No step of the bundle and no step of the installer names `mesh-store`, which is the seat's name only
|
||||
// (ADR 0338 §2): every step that reaches the store reaches the container the bundle raised.
|
||||
func TestNoStepReachesTheStoreByTheSeatsName(t *testing.T) {
|
||||
text := string(theRealBundle(t))
|
||||
for _, by := range []string{`"in": "mesh-store"`, `container:mesh-store`, `exec mesh-store`, `"name": "mesh-store"`, `mesh-store-data`} {
|
||||
if strings.Contains(text, by) {
|
||||
t.Errorf("the bundle still reaches the store as %s", by)
|
||||
}
|
||||
}
|
||||
store := theStoreGenesisRaises(t)
|
||||
if foundationStore != store.Name {
|
||||
t.Errorf("the packages step reaches the store as %q; the bundle raises it as %q", foundationStore, store.Name)
|
||||
}
|
||||
r := producedBundle(t)
|
||||
for _, res := range r.Declaration.Resources {
|
||||
if a, ok := res.(*declaration.Action); ok && a.In != "" && a.In != store.Name && strings.Contains(a.ID, "database") {
|
||||
t.Errorf("the step %s reaches the store in %q", a.ID, a.In)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The installer's refusal to raise a store over data it cannot open looks where the store keeps its data
|
||||
// (ADR 0338 §2): a cluster there and no credential is refused; the old docker volume alone is not.
|
||||
func TestTheRefusalOverExistingDataLooksInTheStoresDirectory(t *testing.T) {
|
||||
made := RootCredentials{Store: "s", Bus: "b", StoreMade: true}
|
||||
machine := func(cluster, volume bool) Runner {
|
||||
return func(_ context.Context, name string, args ...string) (string, error) {
|
||||
joined := name + " " + strings.Join(args, " ")
|
||||
switch {
|
||||
case joined == "test -e "+StoreDataDir+"/pgdata/PG_VERSION":
|
||||
if cluster {
|
||||
return "", nil
|
||||
}
|
||||
case strings.HasPrefix(joined, "docker volume inspect"):
|
||||
if volume {
|
||||
return "[]", nil
|
||||
}
|
||||
}
|
||||
return "", errors.New("exit status 1")
|
||||
}
|
||||
}
|
||||
err := RefuseExistingServers(context.Background(), machine(true, false), made)
|
||||
if err == nil || !strings.Contains(err.Error(), StoreDataDir) {
|
||||
t.Fatalf("a cluster in %s and no credential was not refused naming it: %v", StoreDataDir, err)
|
||||
}
|
||||
if err := RefuseExistingServers(context.Background(), machine(false, true), made); err != nil {
|
||||
t.Fatalf("a machine holding only the old volume was refused: %v", err)
|
||||
}
|
||||
if err := RefuseExistingServers(context.Background(), machine(true, false), RootCredentials{Store: "s"}); err != nil {
|
||||
t.Fatalf("a kept credential was refused: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,317 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
)
|
||||
|
||||
// The registry's trust, written by genesis before anything is pushed to it (novox/hq issue 92).
|
||||
//
|
||||
// The mesh's registry speaks plain HTTP on purpose: every path to it is inside the private
|
||||
// network's encryption (novox/hq ADR 0082). The container runtime refuses plain HTTP to a registry
|
||||
// it has not been told to trust, and the module that tells it — `docker`, which writes the
|
||||
// artifact store's reach into the runtime's configuration — is applied long after genesis pushes
|
||||
// the control plane's image. So genesis writes that trust itself, first, with the same writer the
|
||||
// module uses: into the runtime's file beside what is there (ADR 0102), the runtime reloaded and
|
||||
// never restarted. It then asks the runtime whether it will speak to every name, and refuses with
|
||||
// the runtime's answer before anything is pushed.
|
||||
//
|
||||
// **The handover.** What genesis writes is kept in no record of its own: no record of the foundation
|
||||
// could hold it, because the retire step re-applies the bundle and takes away a carried record the
|
||||
// bundle does not name. The `docker` module, applied later, declares the artifact store's reach —
|
||||
// the mesh name on the node's port — finds it already in the file it has no record of, and takes it
|
||||
// as its own (ADR 0102: a declared member found in an unrecorded file is the mesh's). That member
|
||||
// does not change at the handover. The module's first apply usually still changes the file and
|
||||
// reloads the runtime once, for the keys it writes beside it (`live-restore`, `log-driver`,
|
||||
// `log-opts`); a reload stops nothing. The address the operator gave, when it is not the mesh name,
|
||||
// stays as part of the machine's own configuration, as the hand-written trust it replaces would have.
|
||||
//
|
||||
// **This rests on an assumption about order.** The module takes the mesh name's member over only
|
||||
// because its FIRST apply on this node already declares it: genesis assigns no `docker` before the
|
||||
// network step (16), and once the node is on the network the controller composes the reach as
|
||||
// `<node>.<suffix>:<port>`.
|
||||
// Were `docker` applied earlier — while the reach is empty or `127.0.0.1:<port>`, as it is before the
|
||||
// network — its record would exist without the mesh name, and the member would stay the machine's
|
||||
// for good: undeclaring the module would never take it out. Installing the node's seat holders
|
||||
// earlier in genesis must revisit this.
|
||||
//
|
||||
// **The suffix** is read from this process's environment, and the controller reads its own: a mesh
|
||||
// with a custom `MESH_INTERNAL_SUFFIX` sets it for both.
|
||||
|
||||
// runtimeConfig is the container runtime's configuration, which the trust is written into. A
|
||||
// variable so a test writes into a file of its own.
|
||||
var runtimeConfig = "/etc/docker/daemon.json"
|
||||
|
||||
// runtimeUnit is the runtime's service, reloaded when the file changed.
|
||||
const runtimeUnit = "docker.service"
|
||||
|
||||
// internalSuffixVar and defaultInternalSuffix are what a node's mesh name ends in: the same variable
|
||||
// and default the controller composes `<node>.<suffix>` from (mesh-controller internal/overlay),
|
||||
// which is the name the artifact store's reach — and so every push by the mesh's builder — uses.
|
||||
const (
|
||||
internalSuffixVar = "MESH_INTERNAL_SUFFIX"
|
||||
defaultInternalSuffix = "internal"
|
||||
)
|
||||
|
||||
// RegistryNames are the names the mesh will use for its registry on this node, each as host:port.
|
||||
//
|
||||
// The address the operator gave, and the node's mesh name: the controller's reach for the artifact
|
||||
// store, which the builder pushes to without being told. Both on the port the registry module
|
||||
// serves on this node — the node's setting, never the module's default (novox/hq issue 088). A
|
||||
// loopback name is left out: the runtime trusts loopback without being told, and the mesh never
|
||||
// names it to another node.
|
||||
func RegistryNames(o Options) []string {
|
||||
port := strconv.Itoa(o.Ports.orDefaults().Registry)
|
||||
var names []string
|
||||
add := func(hostport string) {
|
||||
host, _, err := net.SplitHostPort(hostport)
|
||||
if err != nil || host == "" || isLoopback(host) {
|
||||
return
|
||||
}
|
||||
for _, n := range names {
|
||||
if n == hostport {
|
||||
return
|
||||
}
|
||||
}
|
||||
names = append(names, hostport)
|
||||
}
|
||||
if host, _, err := net.SplitHostPort(o.Registry); err == nil {
|
||||
add(net.JoinHostPort(strings.Trim(host, "[]"), port))
|
||||
}
|
||||
if node := strings.TrimSpace(o.Node); node != "" {
|
||||
add(net.JoinHostPort(node+"."+internalSuffix(), port))
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func internalSuffix() string {
|
||||
if v := strings.TrimSpace(os.Getenv(internalSuffixVar)); v != "" {
|
||||
return strings.TrimPrefix(v, ".")
|
||||
}
|
||||
return defaultInternalSuffix
|
||||
}
|
||||
|
||||
func isLoopback(host string) bool {
|
||||
if strings.EqualFold(host, "localhost") {
|
||||
return true
|
||||
}
|
||||
ip := net.ParseIP(strings.Trim(host, "[]"))
|
||||
return ip != nil && ip.IsLoopback()
|
||||
}
|
||||
|
||||
// trustDeclaration is the declaration genesis applies to write the trust: the same two resources
|
||||
// the `docker` module declares for it — the runtime's file written into, the runtime reloaded on it.
|
||||
func trustDeclaration(names []string) (*declaration.Declaration, error) {
|
||||
keys, err := json.Marshal(map[string][]string{"insecure-registries": names})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := json.Marshal(map[string]any{
|
||||
"declaration": 1,
|
||||
"resources": []map[string]any{
|
||||
{"id": "genesis.registry-trust", "type": "file", "path": runtimeConfig, "mode": "0644",
|
||||
"into": "json", "content": string(keys) + "\n"},
|
||||
{"id": "genesis.registry-trust-reload", "type": "service", "unit": runtimeUnit,
|
||||
"state": "running", "reload-on": []string{"genesis.registry-trust"}},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return declaration.ParseTrusted(raw)
|
||||
}
|
||||
|
||||
// TrustTheRegistry writes the runtime's trust of every name the mesh serves its registry on here,
|
||||
// then refuses unless the runtime says it will speak plain HTTP to each.
|
||||
func TrustTheRegistry(ctx context.Context, o Options, sys system.System, run Runner,
|
||||
say func(string)) ([]string, error) {
|
||||
|
||||
names := RegistryNames(o)
|
||||
if len(names) == 0 {
|
||||
say(" trusted only loopback names this registry, which the runtime trusts already")
|
||||
return nil, nil
|
||||
}
|
||||
d, err := trustDeclaration(names)
|
||||
if err != nil {
|
||||
return names, err
|
||||
}
|
||||
|
||||
// One apply at a time on this machine, as ApplyBundle does: a host already running here may be
|
||||
// writing the same file.
|
||||
unlock, err := store.Lock(o.State, func() { say(" waiting another apply holds this node's state") })
|
||||
if err != nil {
|
||||
return names, err
|
||||
}
|
||||
report, _, err := apply.Apply(ctx, sys, d, store.State{}, store.OriginCarried, run,
|
||||
func(line string) { say(" " + strings.TrimPrefix(line, " ")) }, refuseSealed)
|
||||
unlock()
|
||||
if err != nil {
|
||||
return names, fmt.Errorf("the registry's trust could not be written into %s: %w\n"+
|
||||
"The registry is plain HTTP over the private network (novox/hq ADR 0082), and the container "+
|
||||
"runtime refuses to push to it until it trusts it. Nothing has been pushed", runtimeConfig, err)
|
||||
}
|
||||
if report.Changed() {
|
||||
say(" trusted " + strings.Join(names, ", ") + " — written into " + runtimeConfig +
|
||||
", the runtime reloaded")
|
||||
} else {
|
||||
say(" trusted " + strings.Join(names, ", ") + " — " + runtimeConfig + " already says so")
|
||||
}
|
||||
return names, runtimeWillSpeakTo(ctx, o, run, names)
|
||||
}
|
||||
|
||||
// runtimeWillSpeakTo asks the runtime, until it says it trusts every name or the wait runs out.
|
||||
//
|
||||
// The file being right is not the runtime trusting it: a reload the runtime refused, or a flag on
|
||||
// its command line that contradicts the file, leaves the push to fail at step 8 with an HTTPS error.
|
||||
func runtimeWillSpeakTo(ctx context.Context, o Options, run Runner, names []string) error {
|
||||
deadline := time.Now().Add(o.Wait)
|
||||
var missing []string
|
||||
var last error
|
||||
for {
|
||||
asking, cancel := context.WithTimeout(ctx, o.Timeout)
|
||||
out, err := run(asking, "docker", "info", "--format", "{{json .RegistryConfig}}")
|
||||
cancel()
|
||||
last = err
|
||||
if err == nil {
|
||||
missing, last = untrusted(out, names)
|
||||
if last == nil && len(missing) == 0 {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(answerEvery):
|
||||
}
|
||||
}
|
||||
if last != nil {
|
||||
return fmt.Errorf("the container runtime could not be asked whether it trusts the registry: %w", last)
|
||||
}
|
||||
return fmt.Errorf(
|
||||
"the container runtime does not trust %s after %s says it should, so it would refuse to push to "+
|
||||
"it.\nThe file was written and the runtime reloaded; a runtime that still refuses was started "+
|
||||
"with its own registry options, or did not take the reload — `journalctl -u %s` says which. "+
|
||||
"Nothing has been pushed", strings.Join(missing, ", "), runtimeConfig, runtimeUnit)
|
||||
}
|
||||
|
||||
// untrusted reads the runtime's registry configuration and gives back the names it would speak
|
||||
// HTTPS to: a name is trusted when the runtime lists it as not secure, or when its address falls in
|
||||
// a range the runtime trusts.
|
||||
func untrusted(registryConfig string, names []string) ([]string, error) {
|
||||
var c struct {
|
||||
InsecureRegistryCIDRs []string `json:"InsecureRegistryCIDRs"`
|
||||
IndexConfigs map[string]struct {
|
||||
Secure bool `json:"Secure"`
|
||||
} `json:"IndexConfigs"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(strings.TrimSpace(registryConfig)), &c); err != nil {
|
||||
return nil, fmt.Errorf("the runtime's registry configuration is not readable: %w", err)
|
||||
}
|
||||
var missing []string
|
||||
for _, name := range names {
|
||||
if index, ok := c.IndexConfigs[name]; ok && !index.Secure {
|
||||
continue
|
||||
}
|
||||
host, _, _ := net.SplitHostPort(name)
|
||||
if ip := net.ParseIP(strings.Trim(host, "[]")); ip != nil && inAny(ip, c.InsecureRegistryCIDRs) {
|
||||
continue
|
||||
}
|
||||
missing = append(missing, name)
|
||||
}
|
||||
return missing, nil
|
||||
}
|
||||
|
||||
func inAny(ip net.IP, cidrs []string) bool {
|
||||
for _, cidr := range cidrs {
|
||||
if _, n, err := net.ParseCIDR(cidr); err == nil && n.Contains(ip) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// procRoot is where a process's command line is read from. A variable so a test gives its own.
|
||||
var procRoot = "/proc"
|
||||
|
||||
// RuntimeWillTakeTheTrust is preflight's half of the registry's trust: everything that can be known
|
||||
// about it without writing anything, asked at step 1 so a machine that cannot take the trust is
|
||||
// refused before the foundation is raised, rather than at step 7 after it (novox/hq issue 92).
|
||||
//
|
||||
// - The runtime's file is absent or a JSON object: the mesh writes into an object, and refuses to
|
||||
// write over anything else (ADR 0102).
|
||||
// - The runtime's service is known to systemd and active: the trust is taken by reloading it.
|
||||
// - The runtime was not started with `--insecure-registry`: dockerd refuses a directive given both
|
||||
// as a flag and in its file, so it would refuse the reload and keep distrusting the registry.
|
||||
func RuntimeWillTakeTheTrust(ctx context.Context, o Options, run Runner, say func(string)) error {
|
||||
names := RegistryNames(o)
|
||||
if len(names) == 0 {
|
||||
return nil
|
||||
}
|
||||
if raw, err := os.ReadFile(runtimeConfig); err == nil {
|
||||
var object map[string]json.RawMessage
|
||||
if len(strings.TrimSpace(string(raw))) > 0 {
|
||||
if err := json.Unmarshal(raw, &object); err != nil || object == nil {
|
||||
return fmt.Errorf("%s is not a JSON object, and the mesh writes the trust of its registry "+
|
||||
"(%s) into it beside what is there, never over it (novox/hq ADR 0102). Make it an object "+
|
||||
"or remove it; nothing has been changed", runtimeConfig, strings.Join(names, ", "))
|
||||
}
|
||||
}
|
||||
} else if !os.IsNotExist(err) {
|
||||
return fmt.Errorf("the runtime's configuration %s cannot be read: %w", runtimeConfig, err)
|
||||
}
|
||||
|
||||
asking, cancel := context.WithTimeout(ctx, o.Timeout)
|
||||
out, err := run(asking, "systemctl", "show", runtimeUnit, "--property=LoadState,ActiveState,MainPID")
|
||||
cancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("systemd could not be asked about %s: %w\nThe registry's trust is taken by "+
|
||||
"reloading it; nothing has been changed", runtimeUnit, err)
|
||||
}
|
||||
props := map[string]string{}
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
if k, v, ok := strings.Cut(strings.TrimSpace(line), "="); ok {
|
||||
props[k] = v
|
||||
}
|
||||
}
|
||||
if props["LoadState"] != "loaded" || props["ActiveState"] != "active" {
|
||||
return fmt.Errorf("%s is %s and %s, and the registry's trust (%s) is taken by reloading it. "+
|
||||
"Genesis needs the runtime run by systemd as %s; nothing has been changed",
|
||||
runtimeUnit, orUnknown(props["LoadState"]), orUnknown(props["ActiveState"]),
|
||||
strings.Join(names, ", "), runtimeUnit)
|
||||
}
|
||||
if pid := props["MainPID"]; pid != "" && pid != "0" {
|
||||
if cmdline, err := os.ReadFile(procRoot + "/" + pid + "/cmdline"); err == nil {
|
||||
for _, arg := range strings.Split(string(cmdline), "\x00") {
|
||||
if strings.HasPrefix(arg, "--insecure-registry") {
|
||||
return fmt.Errorf("the runtime was started with %s, and dockerd refuses insecure registries "+
|
||||
"given both as a flag and in %s — so it would refuse the reload that makes it trust "+
|
||||
"%s. Move the flag's registries into the file and restart the runtime once; nothing "+
|
||||
"has been changed", arg, runtimeConfig, strings.Join(names, ", "))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
say(" registry trust " + runtimeConfig + " can be written into and " + runtimeUnit + " reloaded")
|
||||
return nil
|
||||
}
|
||||
|
||||
func orUnknown(s string) string {
|
||||
if s == "" {
|
||||
return "unknown"
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,347 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
)
|
||||
|
||||
// novox/hq issue 92: genesis pushed the control plane's image to its registry before anything had
|
||||
// told the container runtime to trust that registry, so the push failed with "server gave HTTP
|
||||
// response to HTTPS client" — first by the address given, then by the mesh name the builder used,
|
||||
// and the trust the mesh wrote later named the registry's default port rather than the node's.
|
||||
|
||||
// aRuntime is a container runtime on systemd whose registry configuration is what its file said at
|
||||
// its last reload: what a real dockerd does, and why a written file is not yet a trusted registry.
|
||||
type aRuntime struct {
|
||||
path string
|
||||
trusted []string
|
||||
commands []string
|
||||
// ignoresReload is a runtime started with options of its own, which a reload does not change.
|
||||
ignoresReload bool
|
||||
}
|
||||
|
||||
func (r *aRuntime) run(_ context.Context, name string, args ...string) (string, error) {
|
||||
line := name + " " + strings.Join(args, " ")
|
||||
r.commands = append(r.commands, line)
|
||||
switch {
|
||||
case name == "docker" && len(args) > 0 && args[0] == "info":
|
||||
index := map[string]map[string]any{"docker.io": {"Secure": true}}
|
||||
for _, n := range r.trusted {
|
||||
index[n] = map[string]any{"Name": n, "Secure": false}
|
||||
}
|
||||
out, _ := json.Marshal(map[string]any{"InsecureRegistryCIDRs": []string{"127.0.0.0/8"}, "IndexConfigs": index})
|
||||
return string(out), nil
|
||||
case name == "systemctl" && strings.Contains(line, "reload"):
|
||||
if !r.ignoresReload {
|
||||
r.trusted = r.trusted[:0]
|
||||
raw, _ := os.ReadFile(r.path)
|
||||
var c struct {
|
||||
Insecure []string `json:"insecure-registries"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &c)
|
||||
r.trusted = append(r.trusted, c.Insecure...)
|
||||
}
|
||||
case strings.Contains(line, "is-enabled"):
|
||||
return "enabled", nil
|
||||
case strings.Contains(line, "show") && strings.Contains(line, "ActiveState"):
|
||||
return "LoadState=loaded\nActiveState=active\nSubState=running", nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (r *aRuntime) did(word string) int {
|
||||
n := 0
|
||||
for _, c := range r.commands {
|
||||
if strings.Contains(c, "systemctl") && strings.Contains(c, " "+word+" ") || strings.HasSuffix(c, " "+word) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func trustSetup(t *testing.T, existing string) (Options, *aRuntime, system.System) {
|
||||
t.Helper()
|
||||
restore := apply.ForTests(t.TempDir(), t.TempDir())
|
||||
t.Cleanup(restore)
|
||||
dir := t.TempDir()
|
||||
was := runtimeConfig
|
||||
runtimeConfig = filepath.Join(dir, "daemon.json")
|
||||
t.Cleanup(func() { runtimeConfig = was })
|
||||
if existing != "" {
|
||||
if err := os.WriteFile(runtimeConfig, []byte(existing), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
t.Setenv(internalSuffixVar, "")
|
||||
sys, err := system.For("arch")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
o := Options{Node: "anchor", Registry: "192.0.2.10:5100", Ports: FoundationPorts{Registry: 5100},
|
||||
State: filepath.Join(dir, "state", "state.json"), Timeout: time.Second, Wait: 0}
|
||||
return o, &aRuntime{path: runtimeConfig}, sys
|
||||
}
|
||||
|
||||
func insecureIn(t *testing.T) (string, map[string]json.RawMessage) {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(runtimeConfig)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var o map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &o); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var list []string
|
||||
_ = json.Unmarshal(o["insecure-registries"], &list)
|
||||
return fmt.Sprint(list), o
|
||||
}
|
||||
|
||||
func TestGenesisTrustsTheRegistryByEveryNameBeforeItPushes(t *testing.T) {
|
||||
o, runtime, sys := trustSetup(t, `{"data-root":"/srv/docker","insecure-registries":["198.51.100.7:5000"]}`)
|
||||
|
||||
names, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The address given and the mesh name the builder pushes by, both on the node's port.
|
||||
if fmt.Sprint(names) != "[192.0.2.10:5100 anchor.internal:5100]" {
|
||||
t.Errorf("trusted by %v", names)
|
||||
}
|
||||
list, object := insecureIn(t)
|
||||
if list != "[198.51.100.7:5000 192.0.2.10:5100 anchor.internal:5100]" {
|
||||
t.Errorf("the runtime's list is %s", list)
|
||||
}
|
||||
if string(object["data-root"]) != `"/srv/docker"` {
|
||||
t.Errorf("the machine's own key was not kept: %s", object["data-root"])
|
||||
}
|
||||
if runtime.did("reload") != 1 || runtime.did("restart") != 0 || runtime.did("stop") != 0 {
|
||||
t.Errorf("the runtime was not reloaded exactly once and never restarted: %v", runtime.commands)
|
||||
}
|
||||
|
||||
// Re-run: nothing changes, nothing is reloaded.
|
||||
runtime.commands = nil
|
||||
if _, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if runtime.did("reload") != 0 {
|
||||
t.Errorf("a re-run reloaded the runtime: %v", runtime.commands)
|
||||
}
|
||||
}
|
||||
|
||||
// The port is the node's setting for the registry, not the module's default, whatever the operator
|
||||
// typed the address with — and loopback, which the runtime trusts already, is not written.
|
||||
func TestTheRegistrysTrustNamesThePortTheNodeServesItOn(t *testing.T) {
|
||||
o := Options{Node: "anchor", Registry: "127.0.0.1:5100", Ports: FoundationPorts{Registry: 5100}}
|
||||
t.Setenv(internalSuffixVar, "")
|
||||
if got := fmt.Sprint(RegistryNames(o)); got != "[anchor.internal:5100]" {
|
||||
t.Errorf("names %s", got)
|
||||
}
|
||||
o.Ports = FoundationPorts{}
|
||||
o.Registry = "192.0.2.10:5000"
|
||||
t.Setenv(internalSuffixVar, ".mesh.example")
|
||||
if got := fmt.Sprint(RegistryNames(o)); got != "[192.0.2.10:5000 anchor.mesh.example:5000]" {
|
||||
t.Errorf("names %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A runtime that still refuses after the write is refused here, with the names, before any push.
|
||||
func TestGenesisRefusesARuntimeThatStillDoesNotTrustTheRegistry(t *testing.T) {
|
||||
o, runtime, sys := trustSetup(t, "")
|
||||
runtime.ignoresReload = true
|
||||
_, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {})
|
||||
if err == nil {
|
||||
t.Fatal("a runtime that does not trust the registry was let through to the push")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "192.0.2.10:5100, anchor.internal:5100") ||
|
||||
!strings.Contains(err.Error(), "Nothing has been pushed") {
|
||||
t.Errorf("the refusal does not say what: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The handover: the `docker` module, applied later, writes the reach the controller composes for
|
||||
// the artifact store — the mesh name and the node's port. It finds it there: the file does not
|
||||
// change, the runtime is not reloaded, and the member it declares is recorded as the module's from
|
||||
// then on (a member found in a file with no record of it is the mesh's, ADR 0102), so the module
|
||||
// owns the trust it writes and taking the module away takes that member out.
|
||||
func TestTheDockerModuleTakesTheTrustOverWithoutAChange(t *testing.T) {
|
||||
o, runtime, sys := trustSetup(t, `{"live-restore":true}`)
|
||||
if _, err := TrustTheRegistry(context.Background(), o, sys, runtime.run, func(string) {}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before, _ := os.ReadFile(runtimeConfig)
|
||||
|
||||
module, err := declaration.ParseTrusted([]byte(fmt.Sprintf(`{"declaration":1,"resources":[
|
||||
{"id":"docker.daemon","type":"file","path":%q,"mode":"0644","into":"json",
|
||||
"content":"{\"live-restore\": true, \"insecure-registries\": [\"anchor.internal:5100\"]}\n"},
|
||||
{"id":"docker.runtime","type":"service","unit":"docker.service","state":"running","boot":"enabled",
|
||||
"reload-on":["docker.daemon"]}]}`, runtimeConfig)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runtime.commands = nil
|
||||
_, state, err := apply.Apply(context.Background(), sys, module, store.State{}, store.OriginDeclared,
|
||||
runtime.run, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after, _ := os.ReadFile(runtimeConfig)
|
||||
if string(before) != string(after) {
|
||||
t.Errorf("the handover changed the file:\n%s\n%s", before, after)
|
||||
}
|
||||
if runtime.did("reload") != 0 || runtime.did("restart") != 0 {
|
||||
t.Errorf("the handover touched the runtime: %v", runtime.commands)
|
||||
}
|
||||
if rec, _ := state.Find("docker.daemon"); rec.Into == nil || len(rec.Into.Added["insecure-registries"]) != 1 {
|
||||
t.Errorf("the module did not take the mesh name's member over: %+v", rec.Into)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARangeTheRuntimeTrustsCoversAnAddressInIt(t *testing.T) {
|
||||
missing, err := untrusted(`{"InsecureRegistryCIDRs":["10.42.0.0/16"],"IndexConfigs":{}}`,
|
||||
[]string{"10.42.0.1:5000", "anchor.internal:5000"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fmt.Sprint(missing) != "[anchor.internal:5000]" {
|
||||
t.Errorf("missing %v", missing)
|
||||
}
|
||||
}
|
||||
|
||||
// **The order, which is the fault issue 92 is about.** Steps 7 and 8 against a runtime that behaves
|
||||
// as dockerd does: a push to a registry it has not been told to trust is refused with the error the
|
||||
// first genesis met. The trust must be written and confirmed before anything is tagged or pushed;
|
||||
// moving the trust after the push, or dropping it, fails this test.
|
||||
func TestNothingIsPushedToTheRegistryBeforeTheRuntimeTrustsIt(t *testing.T) {
|
||||
o, runtime, sys := trustSetup(t, `{"data-root":"/srv/docker"}`)
|
||||
o.Catalogue = catalogueWith(t, RegistryModule, upstreamRegistryManifest)
|
||||
o.Registry = "192.0.2.10:5100"
|
||||
|
||||
pushed := false
|
||||
mesh := aMeshThatAgrees(nil)
|
||||
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
||||
if name == "systemctl" || (name == "docker" && args[0] == "info") {
|
||||
return runtime.run(ctx, name, args...)
|
||||
}
|
||||
runtime.commands = append(runtime.commands, name+" "+strings.Join(args, " "))
|
||||
if name == "docker" {
|
||||
switch args[0] {
|
||||
case "tag":
|
||||
return "", nil
|
||||
case "push":
|
||||
trusted := false
|
||||
for _, n := range runtime.trusted {
|
||||
trusted = trusted || strings.HasPrefix(args[1], n+"/")
|
||||
}
|
||||
if !trusted {
|
||||
return "", fmt.Errorf(`Get "https://%s/v2/": http: server gave HTTP response to HTTPS client`, o.Registry)
|
||||
}
|
||||
pushed = true
|
||||
return "", nil
|
||||
case "image":
|
||||
return `["` + o.Registry + `/mesh-controller@sha256:` + strings.Repeat("a", 64) + `"]`, nil
|
||||
}
|
||||
}
|
||||
return mesh(name, args)
|
||||
}
|
||||
d := Deps{Run: run, Fetch: func(_ context.Context, url string) (int, string, error) {
|
||||
if strings.HasSuffix(url, "/v2/") || pushed {
|
||||
return http.StatusOK, `{"name":"mesh-controller","tags":["genesis"]}`, nil
|
||||
}
|
||||
return http.StatusNotFound, "", nil
|
||||
}}
|
||||
var result Result
|
||||
published, err := RegistryAndPublish(context.Background(), o, d, sys,
|
||||
controlPlane{container: "temp-mesh-controller", run: run, timeout: time.Second}, held, &result, func(string) {})
|
||||
if err != nil {
|
||||
t.Fatalf("genesis did not get past the push: %v\ncommands: %v", err, runtime.commands)
|
||||
}
|
||||
if !strings.HasPrefix(published.Reference, o.Registry+"/mesh-controller@sha256:") {
|
||||
t.Errorf("published as %q", published.Reference)
|
||||
}
|
||||
|
||||
firstPush, reloaded, confirmed := -1, -1, -1
|
||||
for i, c := range runtime.commands {
|
||||
switch {
|
||||
case firstPush < 0 && (strings.HasPrefix(c, "docker tag") || strings.HasPrefix(c, "docker push")):
|
||||
firstPush = i
|
||||
case strings.HasPrefix(c, "systemctl") && strings.Contains(c, "reload"):
|
||||
reloaded = i
|
||||
case strings.HasPrefix(c, "docker info"):
|
||||
confirmed = i
|
||||
}
|
||||
}
|
||||
if firstPush < 0 || reloaded < 0 || confirmed < 0 || !(reloaded < confirmed && confirmed < firstPush) {
|
||||
t.Errorf("the trust was not written (%d) and confirmed (%d) before the first tag or push (%d): %v",
|
||||
reloaded, confirmed, firstPush, runtime.commands)
|
||||
}
|
||||
if fmt.Sprint(result.RegistryTrusted) != "[192.0.2.10:5100 anchor.internal:5100]" {
|
||||
t.Errorf("the result says it trusted %v", result.RegistryTrusted)
|
||||
}
|
||||
}
|
||||
|
||||
// Preflight asks, without writing, what would stop the trust from taking at step 7.
|
||||
func TestPreflightRefusesARuntimeThatCouldNotTakeTheTrust(t *testing.T) {
|
||||
proc := t.TempDir()
|
||||
was := procRoot
|
||||
procRoot = proc
|
||||
t.Cleanup(func() { procRoot = was })
|
||||
if err := os.MkdirAll(filepath.Join(proc, "42"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
unit := func(load, active string) Runner {
|
||||
return func(_ context.Context, name string, args ...string) (string, error) {
|
||||
return "LoadState=" + load + "\nActiveState=" + active + "\nMainPID=42\n", nil
|
||||
}
|
||||
}
|
||||
cmdline := func(args ...string) {
|
||||
if err := os.WriteFile(filepath.Join(proc, "42", "cmdline"), []byte(strings.Join(args, "\x00")+"\x00"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
cmdline("/usr/bin/dockerd", "-H", "fd://")
|
||||
|
||||
for _, c := range []struct {
|
||||
name, file string
|
||||
run Runner
|
||||
flag bool
|
||||
refused string
|
||||
}{
|
||||
{"a runtime that can take it", `{"data-root":"/srv/docker"}`, unit("loaded", "active"), false, ""},
|
||||
{"no file yet", "", unit("loaded", "active"), false, ""},
|
||||
{"a file that is not an object", `["not", "an", "object"]`, unit("loaded", "active"), false, "not a JSON object"},
|
||||
{"a runtime systemd does not run", `{}`, unit("not-found", "inactive"), false, "not-found and inactive"},
|
||||
{"a runtime with the flag", `{}`, unit("loaded", "active"), true, "--insecure-registry=10.0.0.1:5000"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
o, _, _ := trustSetup(t, c.file)
|
||||
if c.flag {
|
||||
cmdline("/usr/bin/dockerd", "--insecure-registry=10.0.0.1:5000")
|
||||
defer cmdline("/usr/bin/dockerd", "-H", "fd://")
|
||||
}
|
||||
before, _ := os.ReadFile(runtimeConfig)
|
||||
err := RuntimeWillTakeTheTrust(context.Background(), o, c.run, func(string) {})
|
||||
switch {
|
||||
case c.refused == "" && err != nil:
|
||||
t.Errorf("refused: %v", err)
|
||||
case c.refused != "" && (err == nil || !strings.Contains(err.Error(), c.refused)):
|
||||
t.Errorf("wanted a refusal naming %q, got %v", c.refused, err)
|
||||
}
|
||||
if after, _ := os.ReadFile(runtimeConfig); string(after) != string(before) {
|
||||
t.Error("preflight wrote the runtime's file")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -89,7 +89,7 @@ func TestAFoundationThatIsUpAndAnsweringIsAccepted(t *testing.T) {
|
||||
// run-once and scheduled shapes are excluded on purpose — a step that has exited is not a
|
||||
// fault. The name is `temp-mesh-controller` because the permanent one is a module and takes the
|
||||
// plain name (novox/hq ADR 0067), which is what makes the two of them coexist at all.
|
||||
want := []string{"mesh-store", "mesh-broker-nats", "temp-mesh-controller"}
|
||||
want := []string{"postgres", "mesh-broker-nats", "temp-mesh-controller"}
|
||||
if len(verified.Running) != len(want) {
|
||||
t.Fatalf("confirmed %v running, want %v", verified.Running, want)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user