Merge pull request 'Write the store's superuser secret where the postgres module now keeps it, and read the bus's release from its upstream image (issues 1873, 1875)' (#130) from fix/1873-system-state-dirs into main

The nats module takes its server as an upstream artifact, pinned by digest
(issue 1641), and builds no image on a NATS_BASE argument any more. The test of
the bundle's bus read that argument, found none, and failed every mesh-host check
against the catalogue's main. Bootstrap's own code never read it: the bundle's
bus image comes from the bundle, so genesis was not affected. The test now
compares the bundle's image with the module's server by repository and digest.

internal/apply/apply.go

@@ -2833,6 +2833,14 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner,
 				return out, nil
 			}
 		}
+		// **The replacement's image is on the machine before the running container is removed**
+		// (novox/hq issue 1742). `docker run` pulls on its own, but only after the old container is
+		// gone: an image the registry does not hold then left the service down and every later apply
+		// failing. Pulled (or found present) first, an unobtainable image fails this resource loudly,
+		// naming the image, and the old container keeps running.
+		if err := ensureImage(ctx, cri, r.Image, run); err != nil {
+			return out, fmt.Errorf("not replacing container %s, which keeps running: %w", r.Name, err)
+		}
 		if _, err := run(ctx, cri, "rm", "-f", r.Name); err != nil {
 			return out, fmt.Errorf("replacing container %s: %w", r.Name, err)
 		}

internal/apply/hold_test.go

@@ -185,6 +185,13 @@ func (m *machine) run(_ context.Context, name string, args ...string) (string, e
 			return "", errors.New("no such image")
 		}
 		return "", nil
+	case "pull":
+		// A real runtime leaves the image present after a pull.
+		if m.images == nil {
+			m.images = map[string]string{}
+		}
+		m.images[args[len(args)-1]] = "2026-01-01T00:00:00Z"
+		return "", nil
 	case "network":
 		if len(args) > 1 && args[1] == "inspect" {
 			return strings.Join(m.members[args[len(args)-1]], ",") + ",\n", nil

internal/apply/replace_image_test.go

@@ -0,0 +1,93 @@
+package apply
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// Issue 1742: the engine removed a running container and only then found the replacement image
+// unobtainable, leaving the service down. The replacement image comes first.
+func TestARunningContainerIsKeptWhenItsReplacementImageCannotBeObtained(t *testing.T) {
+	d := parseTrusted(t, `{"declaration":1,"resources":[
+	  {"id":"store","type":"container","name":"store","image":"`+pinned+`","env":{"PGDATA":"/data"}}
+	]}`)
+	var removed, created bool
+	run := func(ctx context.Context, name string, args ...string) (string, error) {
+		args = oldShape(name, args)
+		switch args[0] {
+		case "info":
+			return "27.0\n", nil
+		case "container":
+			return "true\tsome-older-spec", nil
+		case "image":
+			return "", errors.New("no such image")
+		case "pull":
+			return "", errors.New("manifest unknown")
+		case "rm":
+			removed = true
+			return "", nil
+		case "run":
+			created = true
+			return "deadbeef\n", nil
+		}
+		return "", nil
+	}
+	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
+	if removed || created {
+		t.Fatalf("the running container was touched although its image is unobtainable (removed=%v created=%v)", removed, created)
+	}
+	if err == nil {
+		t.Fatal("an unobtainable image failed nothing")
+	}
+	if !strings.Contains(err.Error(), pinned) {
+		t.Errorf("the failure does not name the image %s: %v", pinned, err)
+	}
+}
+
+func TestAReplacementImageIsPresentBeforeTheRunningContainerIsRemoved(t *testing.T) {
+	d := parseTrusted(t, `{"declaration":1,"resources":[
+	  {"id":"store","type":"container","name":"store","image":"`+pinned+`","env":{"PGDATA":"/data"}}
+	]}`)
+	want := containerSpec(d.Resources[0].(*declaration.Container), inputs{})
+	var order []string
+	created := false
+	run := func(ctx context.Context, name string, args ...string) (string, error) {
+		args = oldShape(name, args)
+		switch args[0] {
+		case "info":
+			return "27.0\n", nil
+		case "container":
+			if created {
+				return "true\t" + want, nil
+			}
+			return "true\tsome-older-spec", nil
+		case "image":
+			order = append(order, "inspect")
+			if len(order) == 1 {
+				return "", errors.New("no such image")
+			}
+			return "", nil
+		case "pull":
+			order = append(order, "pull")
+		case "rm":
+			order = append(order, "rm")
+		case "run":
+			order = append(order, "run")
+			created = true
+			return "deadbeef\n", nil
+		}
+		return "", nil
+	}
+	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil); err != nil {
+		t.Fatalf("apply failed: %v", err)
+	}
+	got := strings.Join(order, ",")
+	if !strings.HasPrefix(got, "inspect,pull,inspect,rm,run") {
+		t.Errorf("order of runtime calls = %s, want the image pulled before rm", got)
+	}
+}

internal/bootstrap/bus_adoption_test.go

@@ -108,28 +108,35 @@ func TestABusStepThatEndsFailedIsRefused(t *testing.T) {
 	}
 }
 
-// The bundle's bus runs the release the module's image is built on, pinned by the same digest, so the
-// adoption changes no server release and is reversible as the step says (ADR 0333 §3).
+// The bundle's bus runs the release the module's server is, pinned by the same digest, so the adoption changes no
+// server release and is reversible as the step says (ADR 0333 §3). The module takes the server as an upstream
+// artifact, its image reference pinned by digest (issue 1641); it no longer builds an image on a NATS_BASE argument,
+// which this test read until issue 1875.
 func TestTheBundlesBusRunsTheReleaseTheModulesImageIsBuiltOn(t *testing.T) {
 	var m struct {
 		Build struct {
-			On []struct{ Arg, Image string } `json:"on"`
+			Artifacts []struct{ Name, Kind, From string } `json:"artifacts"`
 		} `json:"build"`
 	}
 	if err := json.Unmarshal(theCataloguesModule(t, BusModule), &m); err != nil {
 		t.Fatal(err)
 	}
-	base := ""
-	for _, on := range m.Build.On {
-		if on.Arg == "NATS_BASE" {
-			base = on.Image
+	from := ""
+	for _, a := range m.Build.Artifacts {
+		if a.Name == "server" && a.Kind == "upstream" {
+			from = a.From
 		}
 	}
+	name, digest, pinned := strings.Cut(from, "@")
+	if !pinned || !strings.HasPrefix(digest, "sha256:") {
+		t.Fatalf("the nats module's server is no upstream image pinned by digest: %q", from)
+	}
+	repository, _, _ := strings.Cut(name, ":")
 	rewritten, err := Rewrite(theRealBundle(t), held)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if got := containerNamed(rewritten.Declaration, "mesh-broker-nats").Image; got != base {
-		t.Errorf("the bundle raises the bus on %s; the nats module's image is built on %s", got, base)
+	if got, want := containerNamed(rewritten.Declaration, "mesh-broker-nats").Image, repository+"@"+digest; got != want {
+		t.Errorf("the bundle raises the bus on %s; the nats module's server is %s (%s)", got, want, from)
 	}
 }

internal/bootstrap/rootsecrets.go

@@ -45,8 +45,10 @@ import (
 // onward, and the permanent controller is given it by `secret accept` (control.go deliverStores).
 const (
 	// StoreSuperuserFile is where the store's superuser password lives on the machine — the
-	// postgres module's own-secret path, so genesis and adoption write the same file.
-	StoreSuperuserFile = "/var/lib/postgres/superuser.secret"
+	// postgres module's own-secret path, so genesis and adoption write the same file: in the module's
+	// directory under <data root>/mesh, not in /var/lib/postgres, the postgresql package's own (novox/hq
+	// issue 1873).
+	StoreSuperuserFile = "/var/lib/mesh/postgres/superuser.secret"
 	// StoreDataDir is where the store keeps its data — the postgres module's data directory, so
 	// genesis raises the store where the module adopts it (novox/hq ADR 0338 §1).
 	StoreDataDir = "/var/lib/mesh-store"
@@ -207,7 +209,7 @@ func keptOrMade(path string, dryRun bool) (value string, made bool, err error) {
 
 // writeCredentialFile writes a credential as genesis keeps it.
 func writeCredentialFile(path, value string) error {
-	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
+	if err := makeCredentialDir(filepath.Dir(path)); err != nil {
 		return err
 	}
 	// Written whole and renamed into place, at 0600, owned by whoever runs the installer — root,
@@ -227,6 +229,32 @@ func writeCredentialFile(path, value string) error {
 	return os.Rename(tmp, path)
 }
 
+// makeCredentialDir makes a credential's directory 0700 and every missing one above it 0755, the mode the mesh's
+// directories have (/var/lib/mesh above a module's own: novox/hq issue 1873), so a module's account still reaches its
+// own directory below one genesis made. A directory already there is left as it is.
+func makeCredentialDir(dir string) error {
+	dir = filepath.Clean(dir)
+	if _, err := os.Stat(dir); err == nil {
+		return nil
+	} else if !os.IsNotExist(err) {
+		return err
+	}
+	if parent := filepath.Dir(dir); parent != dir {
+		if _, err := os.Stat(parent); os.IsNotExist(err) {
+			if err := makeCredentialDir(parent); err != nil {
+				return err
+			}
+			if err := os.Chmod(parent, 0o755); err != nil {
+				return err
+			}
+		}
+	}
+	if err := os.Mkdir(dir, 0o700); err != nil && !os.IsExist(err) {
+		return err
+	}
+	return os.Chmod(dir, 0o700)
+}
+
 // readCredentialFile is a credential as genesis keeps it: the value alone, its line ending gone.
 // Missing is reported as os.IsNotExist so a caller can tell "not made yet" from "unreadable".
 func readCredentialFile(path string) (string, error) {

internal/bootstrap/rootsecrets_test.go

@@ -249,3 +249,27 @@ func TestTheFoundationFiltersBeforeAnythingListens(t *testing.T) {
 		}
 	}
 }
+
+// novox/hq issue 1873: the store's superuser file is the postgres module's own secret in its directory under
+// <data root>/mesh (/var/lib/mesh/postgres), no longer in /var/lib/postgres, the postgresql package's own. A
+// directory genesis makes on the way keeps the mode the mesh's directories have (0755, so every module's account
+// reaches its own below it); only the credential's own directory is closed, 0700.
+func TestTheStoreSuperuserFileLiesInTheModulesOwnDirectory(t *testing.T) {
+	if StoreSuperuserFile != "/var/lib/mesh/postgres/superuser.secret" {
+		t.Errorf("the store's superuser file is %s, not the postgres module's /var/lib/mesh/postgres/superuser.secret", StoreSuperuserFile)
+	}
+	dir := t.TempDir()
+	store := dir + "/mesh/postgres/superuser.secret"
+	if _, err := rootSecretsAt(store, dir+"/genesis/bus", dir+"/genesis/bus.bcrypt", false); err != nil {
+		t.Fatal(err)
+	}
+	for path, want := range map[string]os.FileMode{dir + "/mesh": 0o755, dir + "/mesh/postgres": 0o700} {
+		st, err := os.Stat(path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if st.Mode().Perm() != want {
+			t.Errorf("%s is made %04o, not %04o", path, st.Mode().Perm(), want)
+		}
+	}
+}

internal/bootstrap/store_layout_test.go

@@ -66,11 +66,11 @@ func TestTheBundleRaisesTheStoreAsTheCataloguesPostgresModuleDeclaresIt(t *testi
 		if _, has := ours.Env["POSTGRES_PASSWORD"]; has {
 			t.Error("the bundle's store takes its password from its environment; the module reads a file")
 		}
-		// The module's own directory for its state is <data root>/<module>, where genesis keeps the
-		// superuser's file (StoreSuperuserFile).
+		// The module's own directory is <data root>/mesh/<module>, where genesis keeps the superuser's
+		// file (StoreSuperuserFile; novox/hq issue 1873).
 		var theirs []string
 		for _, v := range r.Volumes {
-			theirs = append(theirs, strings.ReplaceAll(v, "${dir:state}", "/var/lib/postgres"))
+			theirs = append(theirs, strings.ReplaceAll(v, "${dir:data}", "/var/lib/mesh/postgres"))
 		}
 		if strings.Join(ours.Volumes, " ") != strings.Join(theirs, " ") {
 			t.Errorf("the bundle's store mounts %v; the module's %v", ours.Volumes, theirs)