From 0d12fb11a48be930b5b7b7660425db109f66b9a9 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 22:29:39 +0200 Subject: [PATCH] Make no container before a file it mounts is there, and clear the empty directory a runtime left at a file's path (issue 1873) On ace the postgres container was replaced at 22:11:51 mounting superuser.secret, a file the provisioner's client fills only after the apply. The runtime made the missing source a directory, and every fill after it failed with "is a directory". A container whose mount source is a file the declaration declares, not yet on the machine, now waits and is neither created nor replaced. The write of that file (WriteFilled) applies what reads it, as before, so the container is made once the file is there. What the runtime already left is cured by the mesh: a declared file whose path is an empty directory in the mesh's own tree (by its path and by where it leads), owned by the engine's account, has that directory removed before the file is written, and the outcome says so. A directory with anything in it, or outside the tree, is never removed. --- internal/apply/apply.go | 27 +++++++ internal/apply/filled.go | 1 + internal/apply/mount_waits.go | 58 ++++++++++++++ internal/apply/mount_waits_test.go | 121 +++++++++++++++++++++++++++++ 4 files changed, 207 insertions(+) create mode 100644 internal/apply/mount_waits.go create mode 100644 internal/apply/mount_waits_test.go diff --git a/internal/apply/apply.go b/internal/apply/apply.go index a6dac27..a22e345 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -584,6 +584,7 @@ func ApplyMindingWindows( in.refused = refusedPlaces(d.Resources) in.agentHomes = rootNeverHomes(d) in.dirsBefore = map[string]bool{} + in.files = declaredFiles(d.Resources) for _, resource := range d.Resources { in.declares[resource.Identity()] = declaredDigest(resource) if dir, ok := resource.(*declaration.Directory); ok { @@ -1333,6 +1334,16 @@ func applyOne(ctx context.Context, sys system.System, r declaration.Resource, ru "created once it is (novox/hq ADR 0328 §9)" return out, nil } + // Not created or replaced before a file it mounts is there: the runtime would make the missing source a + // directory, and the file could never be written there (novox/hq issue 1873). The write of the file applies + // what reads it (WriteFilled, readsChange), this container among them. + if src := mountsAFileNotThere(res, in.files); src != "" { + out := begin(r) + out.Action = "waiting" + out.Detail = "not created: it mounts " + src + ", a file the declaration declares that is not there " + + "yet; created once it is (novox/hq issue 1873)" + return out, nil + } } // Nothing below a home is touched through a link an account put there (novox/hq ADR 0266). // And nothing is unpacked or written into below an agent's home (agent_home_guard.go). @@ -1633,6 +1644,19 @@ func applyHandedOver(r *declaration.Directory, out Outcome, mode os.FileMode) (O // keepFound, when not nil, is where the original of a file this host has no record of is kept // before it is written over (novox/hq ADR 0100): once, never overwritten, and named in the outcome. func applyFile(r *declaration.File, previous store.Applied, unseal Unseal, keepFound Keep) (Outcome, error) { + cleared, err := clearRuntimesEmptyDirectory(r.Path) + if err != nil { + return begin(r), err + } + out, err := applyFileAt(r, previous, unseal, keepFound) + if cleared != "" { + out.Detail = joinDetail(cleared, out.Detail) + } + return out, err +} + +// applyFileAt writes a declared file: applyFile's work once its path is cleared of what a runtime left there. +func applyFileAt(r *declaration.File, previous store.Applied, unseal Unseal, keepFound Keep) (Outcome, error) { if r.Into == declaration.IntoBlock { return applyBlock(r, previous) } @@ -2465,6 +2489,9 @@ type inputs struct { // held is the root of every library whose disk or mount is not there, with its name: no container that mounts // it is created (novox/hq ADR 0328 §9). held map[string]string + // files is the path of every file the declaration declares: a container mounting one that is not there yet waits + // for it (novox/hq issue 1873). + files map[string]bool // handed is every group a record that went this apply handed to a resource still asking for it. handed Handed } diff --git a/internal/apply/filled.go b/internal/apply/filled.go index e010323..5d4d2ca 100644 --- a/internal/apply/filled.go +++ b/internal/apply/filled.go @@ -76,6 +76,7 @@ func WriteFilled( in.refused = refusedPlaces(d.Resources) in.agentHomes = rootNeverHomes(d) in.dirsBefore = map[string]bool{} + in.files = declaredFiles(d.Resources) // A container that reads the file and mounts a library whose disk or mount is not there is held, as a full apply // holds it: not created over the missing disk (novox/hq ADR 0328 §9). in.held = heldLibraries(ctx, d.Libraries) diff --git a/internal/apply/mount_waits.go b/internal/apply/mount_waits.go new file mode 100644 index 0000000..e6d8661 --- /dev/null +++ b/internal/apply/mount_waits.go @@ -0,0 +1,58 @@ +package apply + +import ( + "errors" + "os" + "path/filepath" + + "git.novox.be/novox/mesh-sdk/go/declaration" +) + +// declaredFiles is the path of every file a declaration declares. +func declaredFiles(resources []declaration.Resource) map[string]bool { + out := map[string]bool{} + for _, r := range resources { + if f, ok := r.(*declaration.File); ok { + out[filepath.Clean(f.Path)] = true + } + } + return out +} + +// mountsAFileNotThere is the first mount source of a container that is a file the declaration declares and is not +// on the machine yet, or "" (novox/hq issue 1873): the runtime makes a missing source a directory. +func mountsAFileNotThere(c *declaration.Container, files map[string]bool) string { + for _, v := range c.Volumes { + src := mountSource(v) + if !filepath.IsAbs(src) || !files[filepath.Clean(src)] { + continue + } + if _, err := os.Lstat(src); errors.Is(err, os.ErrNotExist) { + return src + } + } + return "" +} + +// clearRuntimesEmptyDirectory removes what a container runtime leaves at a file's path when it mounted the file before +// it was there (novox/hq issue 1873): an empty directory, in the mesh's own tree by its path and by where its path +// leads, owned by the engine's own account. It says what it removed, or "" when nothing was. Anything else at the path +// is left for the write to meet as before: a directory with anything in it, or one outside the tree, is never removed. +func clearRuntimesEmptyDirectory(path string) (string, error) { + info, err := os.Lstat(path) + if err != nil || !info.IsDir() || !inTheMeshsTree(path) { + return "", nil + } + if uid, _, ok := ownerOf(info); !ok || uid != os.Geteuid() { + return "", nil + } + entries, err := os.ReadDir(path) + if err != nil || len(entries) > 0 { + return "", nil + } + if err := os.Remove(path); err != nil { + return "", err + } + return "removed the empty directory a container runtime left at " + path + ", where this file belongs " + + "(novox/hq issue 1873)", nil +} diff --git a/internal/apply/mount_waits_test.go b/internal/apply/mount_waits_test.go new file mode 100644 index 0000000..5e2c33d --- /dev/null +++ b/internal/apply/mount_waits_test.go @@ -0,0 +1,121 @@ +//go:build linux + +package apply + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/store" +) + +// novox/hq issue 1873, its second fault: on ace the postgres container was replaced at 22:11:51 mounting +// superuser.secret, a file the provisioner's client fills only after the apply; the runtime made the missing source +// a directory, and every fill after it failed with "is a directory". A container whose mount source is a file the +// declaration declares, not yet there, is not created or replaced: it waits, and the write of that file (WriteFilled) +// applies what reads it, the container among them (readsChange), once the file is there. +func TestAContainerWaitsForAFileItMountsAndIsMadeWhenTheFillLands(t *testing.T) { + secret := filepath.Join(t.TempDir(), "postgres", "superuser.secret") + d := filledDecl(t, secret, `"content":"","mode":"0600","owner-at-runtime":"owned"`, + `{"id":"postgres.server","type":"container","name":"postgres","image":"`+pinned+`","volumes":["`+ + secret+`:/run/secrets/superuser:ro"]}`) + m := &machine{containers: map[string]*fakeContainer{}} + report, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil) + if err != nil { + t.Fatal(err) + } + if o := outcomeOf(report, "postgres.server"); o.Action != "waiting" || !strings.Contains(o.Detail, secret) { + t.Errorf("the container mounting a file not yet filled is %+v, not waiting for it", o) + } + if m.did("docker run") { + t.Fatalf("the container was made before the file it mounts was there: %v", m.asked) + } + if _, err := os.Lstat(secret); err == nil { + t.Fatal("something was made at the file's path before the fill") + } + + report, _, err = WriteFilled(context.Background(), archHost(t), d, state, secret, []byte("s3cret"), m.run, nil, + nil, nil) + if err != nil { + t.Fatal(err) + } + if !m.did("docker run") { + t.Fatalf("the container was not made when the file it mounts was filled: %v", m.asked) + } + if info, err := os.Lstat(secret); err != nil || !info.Mode().IsRegular() { + t.Fatalf("the filled file is not a file: %v %v", info, err) + } +} + +// novox/hq issue 1873: what the runtime left on ace is cured by the mesh. A declared file whose path is an empty +// directory in the mesh's own tree, the engine's account's, is the runtime's missing mount source: the engine +// removes that empty directory and writes the file, and says so. A directory with anything in it, or one outside the +// tree, is never removed: the write fails as before. +func TestAnEmptyDirectoryTheRuntimeLeftAtAFilesPathIsReplacedByTheFile(t *testing.T) { + rootAlone(t, "") + root := t.TempDir() + before := meshDataRoot + meshDataRoot = filepath.Join(root, "mesh") + t.Cleanup(func() { meshDataRoot = before }) + + secret := filepath.Join(meshDataRoot, "postgres", "superuser.secret") + if err := os.MkdirAll(secret, 0o755); err != nil { + t.Fatal(err) + } + d := filledDecl(t, secret, `"content":"","mode":"0600","owner-at-runtime":"owned"`, "") + m := &machine{containers: map[string]*fakeContainer{}} + report, _, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, secret, []byte("s3cret"), m.run, + nil, nil, nil) + if err != nil { + t.Fatalf("the file was not written over the empty directory the runtime left: %v", err) + } + if info, err := os.Lstat(secret); err != nil || !info.Mode().IsRegular() { + t.Fatalf("%s is not a file: %v %v", secret, info, err) + } + if o := outcomeOf(report, "m.conf"); !strings.Contains(o.Detail, "empty directory") { + t.Errorf("the write did not say it removed the empty directory: %+v", o) + } + + // Not empty: kept, and the write fails. + full := filepath.Join(meshDataRoot, "postgres", "reader.secret") + if err := os.MkdirAll(filepath.Join(full, "something"), 0o755); err != nil { + t.Fatal(err) + } + d = filledDecl(t, full, `"content":"","mode":"0600","owner-at-runtime":"owned"`, "") + if _, _, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, full, []byte("x"), m.run, nil, + nil, nil); err == nil { + t.Error("a directory with something in it was replaced by a file") + } + if _, err := os.Stat(filepath.Join(full, "something")); err != nil { + t.Errorf("what was in the directory is gone: %v", err) + } + + // Outside the tree: kept, and the write fails. + outside := filepath.Join(root, "var-lib", "postgres", "superuser.secret") + if err := os.MkdirAll(outside, 0o755); err != nil { + t.Fatal(err) + } + d = filledDecl(t, outside, `"content":"","mode":"0600","owner-at-runtime":"owned"`, "") + if _, _, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, outside, []byte("x"), m.run, nil, + nil, nil); err == nil { + t.Error("an empty directory outside the mesh's tree was replaced by a file") + } + + // Reached through a link out of the tree: kept, and the write fails. + elsewhere := filepath.Join(root, "elsewhere") + if err := os.MkdirAll(filepath.Join(elsewhere, "x.secret"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(elsewhere, filepath.Join(meshDataRoot, "linked")); err != nil { + t.Fatal(err) + } + through := filepath.Join(meshDataRoot, "linked", "x.secret") + d = filledDecl(t, through, `"content":"","mode":"0600","owner-at-runtime":"owned"`, "") + _, _, _ = WriteFilled(context.Background(), archHost(t), d, store.State{}, through, []byte("x"), m.run, nil, nil, nil) + if info, err := os.Lstat(filepath.Join(elsewhere, "x.secret")); err != nil || !info.IsDir() { + t.Errorf("a directory outside the tree, reached through a link in it, was replaced: %v %v", info, err) + } +}