diff --git a/examples/bundle_test.go b/examples/bundle_test.go index 85fec2d..ca3a346 100644 --- a/examples/bundle_test.go +++ b/examples/bundle_test.go @@ -49,7 +49,7 @@ func TestTheBundleCarriesTheFoundationAndTheControlPlaneAndNothingElse(t *testin } sort.Strings(images) - want := []string{"mesh-controller", "nats", "postgres"} + want := []string{"mesh-controller", "nats", "pgvector"} if strings.Join(images, ",") != strings.Join(want, ",") { t.Fatalf("the bundle carries %v; expected exactly %v.\n\n"+ "Adding one is a change to what every first node becomes, and to ADR 0006's "+ diff --git a/examples/foundation-first-node.lock b/examples/foundation-first-node.lock index 2e32f68..d58c038 100644 --- a/examples/foundation-first-node.lock +++ b/examples/foundation-first-node.lock @@ -28,11 +28,12 @@ // image, and this file is applied on a machine with no mesh to ask about anything (ADR 0006). The // registry prefix is the lab's; the digests are upstream's. // -// The store's data is a NAMED VOLUME, not a directory on the machine. A directory the host -// creates is owned by root, and the database runs as somebody else inside the container — so it -// could not write, and the container crash-looped. A named volume lets the image set up its own -// ownership, and outlives the container, which is what you want for the thing holding the mesh's -// state. +// **The store is raised where the `postgres` module keeps it** (novox/hq ADR 0338): the container +// `postgres`, on the module's pgvector image, its data in the directory /var/lib/mesh-store owned by +// the image's postgres user, its superuser password read from the module's own-secret file — so the +// store step adopts it as it runs. Pinned upstream by the module's own digest, not through the lab's +// registry prefix, because the two must be the same reference (mesh-host internal/bootstrap holds +// them to each other against the module's manifest). { "declaration": 1, @@ -83,17 +84,28 @@ "boot": "enabled", "restart-on": ["base-filter"] }, + // The store's data is a directory owned by the image's postgres user (999), made before the + // container starts: a directory the runtime made would be root's, and the database could not + // write it. Measured on the module's image 2026-10-13: it initialises a new store on a directory + // made so (novox/hq ADR 0338 §5). + { + "id": "store-data", + "type": "directory", + "path": "/var/lib/mesh-store", + "mode": "0700", + "owner": "999:70" + }, { "id": "store", "type": "container", - "name": "mesh-store", - "image": "192.0.2.250:5000/postgres@sha256:7abf537131b66ed5af448d90653abf1679b0c7e9a1f07efdd4c3108a401b259a", + "name": "postgres", + "image": "pgvector/pgvector@sha256:cf134a767f474095eeba57e0117be8e568e011a63f33fbf252f14c9b760f8e6f", "env": { "POSTGRES_PASSWORD": "bootstrap", "PGDATA": "/var/lib/postgresql/data/pgdata" }, "ports": ["5432:5432"], - "volumes": ["mesh-store-data:/var/lib/postgresql/data"] + "volumes": ["/var/lib/mesh-store:/var/lib/postgresql/data"] }, // Over TCP, not the socket. While the store initialises it runs a temporary server on the // socket ONLY, then stops it and starts the real one — so a socket check passes, the action @@ -102,21 +114,21 @@ { "id": "store-ready", "type": "action", - "in": "mesh-store", + "in": "postgres", "command": ["sh", "-c", "for i in $(seq 1 180); do pg_isready -h 127.0.0.1 -U postgres >/dev/null 2>&1 && exit 0; sleep 1; done; echo 'the store did not answer within 180s; its own last words follow'; pg_isready -h 127.0.0.1 -U postgres; tail -n 20 /var/lib/postgresql/data/log/*.log 2>/dev/null; exit 1"], "verify": ["pg_isready", "-h", "127.0.0.1", "-U", "postgres"] }, { "id": "inventory-database", "type": "action", - "in": "mesh-store", + "in": "postgres", "command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE inventory'"], "verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw inventory"] }, { "id": "identity-database", "type": "action", - "in": "mesh-store", + "in": "postgres", "command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE identity'"], "verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw identity"] }, @@ -126,20 +138,20 @@ { "id": "licences-database", "type": "action", - "in": "mesh-store", + "in": "postgres", "command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE licences'"], "verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw licences"] }, { "id": "context-schemas", "type": "action", - "command": ["docker", "run", "--rm", "--network", "container:mesh-store", + "command": ["docker", "run", "--rm", "--network", "container:postgres", "-e", "MESH_STORE_INVENTORY=postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable", "-e", "MESH_STORE_IDENTITY=postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable", "-e", "MESH_STORE_LICENCES=postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable", "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace", "migrate"], - "verify": ["sh", "-c", "docker exec mesh-store psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec mesh-store psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key && docker exec mesh-store psql -U postgres -d licences -tAc \"select to_regclass('public.licence')\" | grep -qx licence"] + "verify": ["sh", "-c", "docker exec postgres psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec postgres psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key && docker exec postgres psql -U postgres -d licences -tAc \"select to_regclass('public.licence')\" | grep -qx licence"] }, // **The bus is raised where the `nats` module keeps it** (novox/hq ADR 0333 §1): the container // `mesh-broker-nats`, its configuration and user list in /var/lib/nats-module/conf, its streams in diff --git a/internal/bootstrap/phase3.go b/internal/bootstrap/phase3.go index 216bc88..12217fb 100644 --- a/internal/bootstrap/phase3.go +++ b/internal/bootstrap/phase3.go @@ -84,7 +84,7 @@ func InstallStore(ctx context.Context, o Options, control controlPlane, say); err != nil { return err } - say(" adopted mesh-store — the foundation's store is now the " + module + " module") + say(" adopted " + store.Name + " — the foundation's store is now the " + module + " module") return nil } diff --git a/internal/bootstrap/phase_packages.go b/internal/bootstrap/phase_packages.go index 02bc574..6a5db0a 100644 --- a/internal/bootstrap/phase_packages.go +++ b/internal/bootstrap/phase_packages.go @@ -25,7 +25,7 @@ import ( const ( // foundationStore is the foundation's postgres container — the mesh's own memory, raised from the // bundle. gitea's bootstrap database lives here too, so a mesh runs one postgres (issue 051). - foundationStore = "mesh-store" + foundationStore = "postgres" // giteaBootstrap is the gitea server raised directly at genesis, before gitea is a module — // under the name the gitea MODULE declares for its container, so the module finds it and holds // it rather than raising a second forge beside it (novox/hq ADR 0163, rule 7; issue 090). diff --git a/internal/bootstrap/ports_test.go b/internal/bootstrap/ports_test.go index 0e2c008..661fdce 100644 --- a/internal/bootstrap/ports_test.go +++ b/internal/bootstrap/ports_test.go @@ -68,7 +68,7 @@ func TestGivenPortsMoveOnlyTheMachinesSide(t *testing.T) { if got.Places == 0 { t.Fatal("nothing was rewritten") } - storeC := containerNamed(r.Declaration, "mesh-store") + storeC := containerNamed(r.Declaration, "postgres") if len(storeC.Ports) != 1 || storeC.Ports[0] != "5433:5432" { t.Errorf("the store publishes %v", storeC.Ports) } diff --git a/internal/bootstrap/rootsecrets.go b/internal/bootstrap/rootsecrets.go index 9e63874..9f1fa1e 100644 --- a/internal/bootstrap/rootsecrets.go +++ b/internal/bootstrap/rootsecrets.go @@ -47,6 +47,9 @@ const ( // StoreSuperuserFile is where the store's superuser password lives on the machine — the // postgres module's own-secret path, so genesis and adoption write the same file. StoreSuperuserFile = "/var/lib/postgres/superuser.secret" + // StoreDataDir is where the store keeps its data — the postgres module's data directory, so + // genesis raises the store where the module adopts it (novox/hq ADR 0338 §1). + StoreDataDir = "/var/lib/mesh-store" // BusControllerFile is where the controller's password on the bus lives on the machine. Not a // module's path: no module reads it — the permanent controller is handed the address the // temporary one was given — and genesis keeps it only so a second run puts the same account @@ -62,7 +65,7 @@ const ( // What the template says, matched exactly. A template that says something else is a template // this installer does not know how to make safe, and it says so rather than guessing. templateStorePassword = `"POSTGRES_PASSWORD": "bootstrap"` - templateStoreVolumes = `"volumes": ["mesh-store-data:/var/lib/postgresql/data"]` + templateStoreVolumes = `"volumes": ["` + StoreDataDir + `:/var/lib/postgresql/data"]` templateStoreURL = "postgres:bootstrap@" // BusURLPlaceholder is the controller's account in its bus address, as the template names it. @@ -265,8 +268,8 @@ func Masking(say func(string), c RootCredentials) func(string) { // RefuseExistingServers stops a run that would put a made credential in front of a store raised // by an earlier installer with the template's. // -// The store's password is set by initdb, once, on an empty volume. A machine that already holds -// `mesh-store-data` and has no credential file was raised with `bootstrap`, and minting a new +// The store's password is set by initdb, once, on an empty data directory. A machine that already holds +// a cluster in the store's data directory and has no credential file was raised with `bootstrap`, and minting a new // value here would make a bundle that dials with a password the server does not have — failing // three steps later, in the schemas, with nothing pointing back here. Refused by name instead, // with the way forward. @@ -278,11 +281,14 @@ func RefuseExistingServers(ctx context.Context, run Runner, c RootCredentials) e if !c.StoreMade { return nil } - if _, err := run(ctx, "docker", "volume", "inspect", "mesh-store-data"); err != nil { - return nil // no such volume: a fresh machine, which is the case this installer makes + // Where the store keeps its data, as the postgres module does (novox/hq ADR 0338 §2): a cluster + // there is one initdb already set a password for. The docker volume genesis used before is not + // the store's data any more, and a machine holding only that is not refused. + if _, err := run(ctx, "test", "-e", StoreDataDir+"/pgdata/PG_VERSION"); err != nil { + return nil // no cluster there: a fresh machine, which is the case this installer makes } return fmt.Errorf( - "this machine already holds the store's data (volume mesh-store-data) and no credential at %s, so it was raised "+ + "this machine already holds the store's data (in "+StoreDataDir+") and no credential at %s, so it was raised "+ "by an earlier installer with the template's password. A new one made here would not open it. "+ "Put the password the store has into %s (0600, the value alone) and run again; then change it "+ "on the server and accept the new value — this installer does not rotate a running store", @@ -311,7 +317,7 @@ func RewriteRoot(r *Rewritten, c RootCredentials) (RootRewrite, error) { return out, err } if bundle, err = replaceOnce(bundle, templateStoreVolumes, - `"volumes": ["mesh-store-data:/var/lib/postgresql/data", "`+StoreSuperuserFile+":"+storeSuperuserMount+`:ro"]`, + `"volumes": ["`+StoreDataDir+`:/var/lib/postgresql/data", "`+StoreSuperuserFile+":"+storeSuperuserMount+`:ro"]`, "the store's volumes"); err != nil { return out, err } diff --git a/internal/bootstrap/rootsecrets_test.go b/internal/bootstrap/rootsecrets_test.go index c0db52a..29c61e1 100644 --- a/internal/bootstrap/rootsecrets_test.go +++ b/internal/bootstrap/rootsecrets_test.go @@ -59,7 +59,7 @@ func TestTheProducedBundleCarriesNoWellKnownCredential(t *testing.T) { switch x := res.(type) { case *declaration.Container: switch x.Name { - case "mesh-store": + case "postgres": store = true if _, has := x.Env["POSTGRES_PASSWORD"]; has { t.Error("the store still takes its password from its environment") diff --git a/internal/bootstrap/store_layout_test.go b/internal/bootstrap/store_layout_test.go new file mode 100644 index 0000000..809ded7 --- /dev/null +++ b/internal/bootstrap/store_layout_test.go @@ -0,0 +1,163 @@ +package bootstrap + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" +) + +// Defends novox/hq ADR 0338 (issue 609): genesis raises its store as the postgres module's server, so +// the store step adopts it as it runs. Until then the bundle raised `mesh-store`, on another image, in a +// docker volume, and genesis stopped at step 17 of 24. + +// theStoreGenesisRaises is the store container of the bundle as the installer applies it: rewritten for +// this machine's credentials, so its password is the file the module reads too. +func theStoreGenesisRaises(t *testing.T) *declaration.Container { + t.Helper() + r := producedBundle(t) + store, err := storeIn(r.Declaration) + if err != nil { + t.Fatal(err) + } + return store +} + +// The bundle's store is exactly the catalogue's postgres module's server, read from the module's own +// manifest (ADR 0338 §1): its name, its image (so a drift of either pin fails here), where its data is, +// and the file its superuser password is read from, with the environment that says so. +func TestTheBundleRaisesTheStoreAsTheCataloguesPostgresModuleDeclaresIt(t *testing.T) { + manifest := theCataloguesModule(t, "postgres") + var m struct { + Resources []struct { + ID string `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + Image string `json:"image"` + Env map[string]string `json:"env"` + Ports []string `json:"ports"` + Volumes []string `json:"volumes"` + } `json:"resources"` + } + if err := json.Unmarshal(manifest, &m); err != nil { + t.Fatal(err) + } + ours := theStoreGenesisRaises(t) + found := false + for _, r := range m.Resources { + if r.Type != "container" || r.ID != "server" { + continue + } + found = true + if ours.Name != r.Name { + t.Errorf("the bundle raises the store as %q; the module's server is %q", ours.Name, r.Name) + } + if ours.Image != r.Image { + t.Errorf("the bundle raises the store on %s; the module pins %s — the two pins move together", ours.Image, r.Image) + } + for _, key := range []string{"POSTGRES_PASSWORD_FILE", "PGDATA"} { + if ours.Env[key] != r.Env[key] { + t.Errorf("the bundle's store has %s=%q; the module's %q", key, ours.Env[key], r.Env[key]) + } + } + if _, has := ours.Env["POSTGRES_PASSWORD"]; has { + t.Error("the bundle's store takes its password from its environment; the module reads a file") + } + // The module's own directory for its state is /, where genesis keeps the + // superuser's file (StoreSuperuserFile). + var theirs []string + for _, v := range r.Volumes { + theirs = append(theirs, strings.ReplaceAll(v, "${dir:state}", "/var/lib/postgres")) + } + if strings.Join(ours.Volumes, " ") != strings.Join(theirs, " ") { + t.Errorf("the bundle's store mounts %v; the module's %v", ours.Volumes, theirs) + } + if strings.Join(ours.Ports, " ") != strings.Join(r.Ports, " ") { + t.Errorf("the bundle publishes the store on %v; the module on %v", ours.Ports, r.Ports) + } + // And the store step's own check agrees: by name and image, before anything is registered. + if err := serverMatchesFoundation(manifest, ours, "postgres"); err != nil { + t.Errorf("the store step would refuse the store the bundle raises: %v", err) + } + } + if !found { + t.Fatal("the postgres module declares no server container") + } +} + +// The store's data directory is made by the bundle before the store starts, owned by the image's +// postgres user, as the module declares it (ADR 0338 §1, and the measurement under its §5). +func TestTheBundleMakesTheStoresDataDirectoryAsTheModuleDoes(t *testing.T) { + r := producedBundle(t) + at, store := -1, -1 + for i, res := range r.Declaration.Resources { + if d, ok := res.(*declaration.Directory); ok && d.Path == StoreDataDir { + at = i + if d.Mode != "0700" || d.Owner != "999:70" { + t.Errorf("the store's data directory is made %s %s; the module makes it 0700 999:70", d.Mode, d.Owner) + } + } + if res.Identity() == StoreID { + store = i + } + } + if at < 0 || store < 0 || at > store { + t.Fatalf("the store's data directory %s is not made before the store (directory at %d, store at %d)", StoreDataDir, at, store) + } +} + +// No step of the bundle and no step of the installer names `mesh-store`, which is the seat's name only +// (ADR 0338 §2): every step that reaches the store reaches the container the bundle raised. +func TestNoStepReachesTheStoreByTheSeatsName(t *testing.T) { + text := string(theRealBundle(t)) + for _, by := range []string{`"in": "mesh-store"`, `container:mesh-store`, `exec mesh-store`, `"name": "mesh-store"`, `mesh-store-data`} { + if strings.Contains(text, by) { + t.Errorf("the bundle still reaches the store as %s", by) + } + } + store := theStoreGenesisRaises(t) + if foundationStore != store.Name { + t.Errorf("the packages step reaches the store as %q; the bundle raises it as %q", foundationStore, store.Name) + } + r := producedBundle(t) + for _, res := range r.Declaration.Resources { + if a, ok := res.(*declaration.Action); ok && a.In != "" && a.In != store.Name && strings.Contains(a.ID, "database") { + t.Errorf("the step %s reaches the store in %q", a.ID, a.In) + } + } +} + +// The installer's refusal to raise a store over data it cannot open looks where the store keeps its data +// (ADR 0338 §2): a cluster there and no credential is refused; the old docker volume alone is not. +func TestTheRefusalOverExistingDataLooksInTheStoresDirectory(t *testing.T) { + made := RootCredentials{Store: "s", Bus: "b", StoreMade: true} + machine := func(cluster, volume bool) Runner { + return func(_ context.Context, name string, args ...string) (string, error) { + joined := name + " " + strings.Join(args, " ") + switch { + case joined == "test -e "+StoreDataDir+"/pgdata/PG_VERSION": + if cluster { + return "", nil + } + case strings.HasPrefix(joined, "docker volume inspect"): + if volume { + return "[]", nil + } + } + return "", errors.New("exit status 1") + } + } + err := RefuseExistingServers(context.Background(), machine(true, false), made) + if err == nil || !strings.Contains(err.Error(), StoreDataDir) { + t.Fatalf("a cluster in %s and no credential was not refused naming it: %v", StoreDataDir, err) + } + if err := RefuseExistingServers(context.Background(), machine(false, true), made); err != nil { + t.Fatalf("a machine holding only the old volume was refused: %v", err) + } + if err := RefuseExistingServers(context.Background(), machine(true, false), RootCredentials{Store: "s"}); err != nil { + t.Fatalf("a kept credential was refused: %v", err) + } +} diff --git a/internal/bootstrap/verify_test.go b/internal/bootstrap/verify_test.go index b743177..ddd37ab 100644 --- a/internal/bootstrap/verify_test.go +++ b/internal/bootstrap/verify_test.go @@ -89,7 +89,7 @@ func TestAFoundationThatIsUpAndAnsweringIsAccepted(t *testing.T) { // run-once and scheduled shapes are excluded on purpose — a step that has exited is not a // fault. The name is `temp-mesh-controller` because the permanent one is a module and takes the // plain name (novox/hq ADR 0067), which is what makes the two of them coexist at all. - want := []string{"mesh-store", "mesh-broker-nats", "temp-mesh-controller"} + want := []string{"postgres", "mesh-broker-nats", "temp-mesh-controller"} if len(verified.Running) != len(want) { t.Fatalf("confirmed %v running, want %v", verified.Running, want) }