diff --git a/internal/apply/apply.go b/internal/apply/apply.go index a6dac27..a22e345 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -584,6 +584,7 @@ func ApplyMindingWindows( in.refused = refusedPlaces(d.Resources) in.agentHomes = rootNeverHomes(d) in.dirsBefore = map[string]bool{} + in.files = declaredFiles(d.Resources) for _, resource := range d.Resources { in.declares[resource.Identity()] = declaredDigest(resource) if dir, ok := resource.(*declaration.Directory); ok { @@ -1333,6 +1334,16 @@ func applyOne(ctx context.Context, sys system.System, r declaration.Resource, ru "created once it is (novox/hq ADR 0328 §9)" return out, nil } + // Not created or replaced before a file it mounts is there: the runtime would make the missing source a + // directory, and the file could never be written there (novox/hq issue 1873). The write of the file applies + // what reads it (WriteFilled, readsChange), this container among them. + if src := mountsAFileNotThere(res, in.files); src != "" { + out := begin(r) + out.Action = "waiting" + out.Detail = "not created: it mounts " + src + ", a file the declaration declares that is not there " + + "yet; created once it is (novox/hq issue 1873)" + return out, nil + } } // Nothing below a home is touched through a link an account put there (novox/hq ADR 0266). // And nothing is unpacked or written into below an agent's home (agent_home_guard.go). @@ -1633,6 +1644,19 @@ func applyHandedOver(r *declaration.Directory, out Outcome, mode os.FileMode) (O // keepFound, when not nil, is where the original of a file this host has no record of is kept // before it is written over (novox/hq ADR 0100): once, never overwritten, and named in the outcome. func applyFile(r *declaration.File, previous store.Applied, unseal Unseal, keepFound Keep) (Outcome, error) { + cleared, err := clearRuntimesEmptyDirectory(r.Path) + if err != nil { + return begin(r), err + } + out, err := applyFileAt(r, previous, unseal, keepFound) + if cleared != "" { + out.Detail = joinDetail(cleared, out.Detail) + } + return out, err +} + +// applyFileAt writes a declared file: applyFile's work once its path is cleared of what a runtime left there. +func applyFileAt(r *declaration.File, previous store.Applied, unseal Unseal, keepFound Keep) (Outcome, error) { if r.Into == declaration.IntoBlock { return applyBlock(r, previous) } @@ -2465,6 +2489,9 @@ type inputs struct { // held is the root of every library whose disk or mount is not there, with its name: no container that mounts // it is created (novox/hq ADR 0328 §9). held map[string]string + // files is the path of every file the declaration declares: a container mounting one that is not there yet waits + // for it (novox/hq issue 1873). + files map[string]bool // handed is every group a record that went this apply handed to a resource still asking for it. handed Handed } diff --git a/internal/apply/filled.go b/internal/apply/filled.go index e010323..5d4d2ca 100644 --- a/internal/apply/filled.go +++ b/internal/apply/filled.go @@ -76,6 +76,7 @@ func WriteFilled( in.refused = refusedPlaces(d.Resources) in.agentHomes = rootNeverHomes(d) in.dirsBefore = map[string]bool{} + in.files = declaredFiles(d.Resources) // A container that reads the file and mounts a library whose disk or mount is not there is held, as a full apply // holds it: not created over the missing disk (novox/hq ADR 0328 §9). in.held = heldLibraries(ctx, d.Libraries) diff --git a/internal/apply/mount_waits.go b/internal/apply/mount_waits.go new file mode 100644 index 0000000..e6d8661 --- /dev/null +++ b/internal/apply/mount_waits.go @@ -0,0 +1,58 @@ +package apply + +import ( + "errors" + "os" + "path/filepath" + + "git.novox.be/novox/mesh-sdk/go/declaration" +) + +// declaredFiles is the path of every file a declaration declares. +func declaredFiles(resources []declaration.Resource) map[string]bool { + out := map[string]bool{} + for _, r := range resources { + if f, ok := r.(*declaration.File); ok { + out[filepath.Clean(f.Path)] = true + } + } + return out +} + +// mountsAFileNotThere is the first mount source of a container that is a file the declaration declares and is not +// on the machine yet, or "" (novox/hq issue 1873): the runtime makes a missing source a directory. +func mountsAFileNotThere(c *declaration.Container, files map[string]bool) string { + for _, v := range c.Volumes { + src := mountSource(v) + if !filepath.IsAbs(src) || !files[filepath.Clean(src)] { + continue + } + if _, err := os.Lstat(src); errors.Is(err, os.ErrNotExist) { + return src + } + } + return "" +} + +// clearRuntimesEmptyDirectory removes what a container runtime leaves at a file's path when it mounted the file before +// it was there (novox/hq issue 1873): an empty directory, in the mesh's own tree by its path and by where its path +// leads, owned by the engine's own account. It says what it removed, or "" when nothing was. Anything else at the path +// is left for the write to meet as before: a directory with anything in it, or one outside the tree, is never removed. +func clearRuntimesEmptyDirectory(path string) (string, error) { + info, err := os.Lstat(path) + if err != nil || !info.IsDir() || !inTheMeshsTree(path) { + return "", nil + } + if uid, _, ok := ownerOf(info); !ok || uid != os.Geteuid() { + return "", nil + } + entries, err := os.ReadDir(path) + if err != nil || len(entries) > 0 { + return "", nil + } + if err := os.Remove(path); err != nil { + return "", err + } + return "removed the empty directory a container runtime left at " + path + ", where this file belongs " + + "(novox/hq issue 1873)", nil +} diff --git a/internal/apply/mount_waits_test.go b/internal/apply/mount_waits_test.go new file mode 100644 index 0000000..5e2c33d --- /dev/null +++ b/internal/apply/mount_waits_test.go @@ -0,0 +1,121 @@ +//go:build linux + +package apply + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/store" +) + +// novox/hq issue 1873, its second fault: on ace the postgres container was replaced at 22:11:51 mounting +// superuser.secret, a file the provisioner's client fills only after the apply; the runtime made the missing source +// a directory, and every fill after it failed with "is a directory". A container whose mount source is a file the +// declaration declares, not yet there, is not created or replaced: it waits, and the write of that file (WriteFilled) +// applies what reads it, the container among them (readsChange), once the file is there. +func TestAContainerWaitsForAFileItMountsAndIsMadeWhenTheFillLands(t *testing.T) { + secret := filepath.Join(t.TempDir(), "postgres", "superuser.secret") + d := filledDecl(t, secret, `"content":"","mode":"0600","owner-at-runtime":"owned"`, + `{"id":"postgres.server","type":"container","name":"postgres","image":"`+pinned+`","volumes":["`+ + secret+`:/run/secrets/superuser:ro"]}`) + m := &machine{containers: map[string]*fakeContainer{}} + report, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil) + if err != nil { + t.Fatal(err) + } + if o := outcomeOf(report, "postgres.server"); o.Action != "waiting" || !strings.Contains(o.Detail, secret) { + t.Errorf("the container mounting a file not yet filled is %+v, not waiting for it", o) + } + if m.did("docker run") { + t.Fatalf("the container was made before the file it mounts was there: %v", m.asked) + } + if _, err := os.Lstat(secret); err == nil { + t.Fatal("something was made at the file's path before the fill") + } + + report, _, err = WriteFilled(context.Background(), archHost(t), d, state, secret, []byte("s3cret"), m.run, nil, + nil, nil) + if err != nil { + t.Fatal(err) + } + if !m.did("docker run") { + t.Fatalf("the container was not made when the file it mounts was filled: %v", m.asked) + } + if info, err := os.Lstat(secret); err != nil || !info.Mode().IsRegular() { + t.Fatalf("the filled file is not a file: %v %v", info, err) + } +} + +// novox/hq issue 1873: what the runtime left on ace is cured by the mesh. A declared file whose path is an empty +// directory in the mesh's own tree, the engine's account's, is the runtime's missing mount source: the engine +// removes that empty directory and writes the file, and says so. A directory with anything in it, or one outside the +// tree, is never removed: the write fails as before. +func TestAnEmptyDirectoryTheRuntimeLeftAtAFilesPathIsReplacedByTheFile(t *testing.T) { + rootAlone(t, "") + root := t.TempDir() + before := meshDataRoot + meshDataRoot = filepath.Join(root, "mesh") + t.Cleanup(func() { meshDataRoot = before }) + + secret := filepath.Join(meshDataRoot, "postgres", "superuser.secret") + if err := os.MkdirAll(secret, 0o755); err != nil { + t.Fatal(err) + } + d := filledDecl(t, secret, `"content":"","mode":"0600","owner-at-runtime":"owned"`, "") + m := &machine{containers: map[string]*fakeContainer{}} + report, _, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, secret, []byte("s3cret"), m.run, + nil, nil, nil) + if err != nil { + t.Fatalf("the file was not written over the empty directory the runtime left: %v", err) + } + if info, err := os.Lstat(secret); err != nil || !info.Mode().IsRegular() { + t.Fatalf("%s is not a file: %v %v", secret, info, err) + } + if o := outcomeOf(report, "m.conf"); !strings.Contains(o.Detail, "empty directory") { + t.Errorf("the write did not say it removed the empty directory: %+v", o) + } + + // Not empty: kept, and the write fails. + full := filepath.Join(meshDataRoot, "postgres", "reader.secret") + if err := os.MkdirAll(filepath.Join(full, "something"), 0o755); err != nil { + t.Fatal(err) + } + d = filledDecl(t, full, `"content":"","mode":"0600","owner-at-runtime":"owned"`, "") + if _, _, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, full, []byte("x"), m.run, nil, + nil, nil); err == nil { + t.Error("a directory with something in it was replaced by a file") + } + if _, err := os.Stat(filepath.Join(full, "something")); err != nil { + t.Errorf("what was in the directory is gone: %v", err) + } + + // Outside the tree: kept, and the write fails. + outside := filepath.Join(root, "var-lib", "postgres", "superuser.secret") + if err := os.MkdirAll(outside, 0o755); err != nil { + t.Fatal(err) + } + d = filledDecl(t, outside, `"content":"","mode":"0600","owner-at-runtime":"owned"`, "") + if _, _, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, outside, []byte("x"), m.run, nil, + nil, nil); err == nil { + t.Error("an empty directory outside the mesh's tree was replaced by a file") + } + + // Reached through a link out of the tree: kept, and the write fails. + elsewhere := filepath.Join(root, "elsewhere") + if err := os.MkdirAll(filepath.Join(elsewhere, "x.secret"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(elsewhere, filepath.Join(meshDataRoot, "linked")); err != nil { + t.Fatal(err) + } + through := filepath.Join(meshDataRoot, "linked", "x.secret") + d = filledDecl(t, through, `"content":"","mode":"0600","owner-at-runtime":"owned"`, "") + _, _, _ = WriteFilled(context.Background(), archHost(t), d, store.State{}, through, []byte("x"), m.run, nil, nil, nil) + if info, err := os.Lstat(filepath.Join(elsewhere, "x.secret")); err != nil || !info.IsDir() { + t.Errorf("a directory outside the tree, reached through a link in it, was replaced: %v %v", info, err) + } +}