A directory holding anything the mesh did not put there is never removed

Found by asking what the conversion needs, and it is the one failure in
this system that cannot be undone.

Unassigning a module made its directory an orphan, and an orphan
directory was deleted with everything under it — os.RemoveAll — while
the report said "removed". A database's files, a mail spool, somebody's
uploads. Reproduced before fixing: assign a module, let a service write
into its directory, unassign the module, and the file is gone.

Now a directory that still holds something is kept and said so, naming
how many items are in it.

What makes that safe rather than merely cautious is the removal order,
which was already right. Everything the mesh puts in a directory is
itself a declared resource, and orphans are removed in reverse
declaration order — so what the mesh wrote is already gone by the time
the directory is reached. Anything still there was put there by
something else, which is the definition of data.

It is the host's own line applied to the one shape where getting it
wrong does not recover: it removes what it made and leaves what it
merely configured. An empty directory is what it made; a full one is
not, and an empty one is still removed so nothing accumulates.

Files are unchanged. A declared file is the mesh's own, and losing a
config file is not the failure this is about.

internal/apply/apply.go

@@ -601,7 +601,37 @@ func applyService(ctx context.Context, sys system.System, r *declaration.Service
 // installed would be describing an effect it declined to have.
 func remove(ctx context.Context, sys system.System, a store.Applied, run Runner) (string, string, error) {
 	switch declaration.Type(a.Type) {
-	case declaration.TypeFile, declaration.TypeDirectory:
+	case declaration.TypeDirectory:
+		// **A directory with anything left in it is kept, and that is the rule that protects
+		// data.** Everything the mesh put inside is itself a declared resource, and orphans are
+		// removed in reverse declaration order — so by the time a directory comes to be removed,
+		// what the mesh wrote there is already gone. Anything still present is something nobody
+		// declared: a database's files, a mail spool, somebody's uploads.
+		//
+		// Before this, unassigning a module deleted its data directory and everything under it,
+		// and the report said "removed". Nothing anywhere said what had been in there.
+		//
+		// This is the host's own line, applied to the one shape where getting it wrong is not
+		// recoverable: it removes what it made and leaves what it merely configured. An empty
+		// directory is what it made. A full one is not.
+		entries, err := os.ReadDir(a.Target)
+		if errors.Is(err, os.ErrNotExist) {
+			return "forgotten", "no longer there", nil
+		}
+		if err != nil {
+			return "", "", err
+		}
+		if len(entries) > 0 {
+			return "kept", fmt.Sprintf(
+				"no longer declared, and %d item(s) inside that the mesh did not put there — "+
+					"remove it by hand once you know what it is", len(entries)), nil
+		}
+		if err := os.Remove(a.Target); err != nil {
+			return "", "", err
+		}
+		return "removed", "no longer declared, and empty", nil
+
+	case declaration.TypeFile:
 		if err := os.RemoveAll(a.Target); err != nil {
 			return "", "", err
 		}

internal/apply/apply_test.go

@@ -1360,3 +1360,74 @@ func TestResourcesAreAppliedInTheOrderTheyWereDeclared(t *testing.T) {
 			"another has no way to say so", ran)
 	}
 }
+
+// **A data directory is never removed by the mesh.** The one failure in this system that cannot
+// be undone.
+//
+// Unassigning a module made its directory an orphan, and an orphan directory was deleted with
+// everything under it — a database's files, a mail spool, somebody's uploads — and the report
+// said "removed". Reproduced before it was fixed: a module was assigned, a service wrote into
+// its directory, the module was unassigned, and the file was gone.
+//
+// What makes the rule safe rather than merely cautious is the removal order. Everything the mesh
+// puts in a directory is itself a declared resource, and orphans are removed in reverse
+// declaration order — so what the mesh wrote is already gone by the time the directory is
+// reached. Anything still there was put there by something else.
+func TestADirectoryHoldingAnythingTheMeshDidNotPutThereIsKept(t *testing.T) {
+	root := t.TempDir()
+	data := filepath.Join(root, "keycloak")
+
+	d := declare(t, `{"id":"data","type":"directory","path":"`+data+`","mode":"0700"}`)
+	_, state, err := Apply(context.Background(), archHost(t), d, store.State{},
+		store.OriginDeclared, nil, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	live := filepath.Join(data, "realm.db")
+	if err := os.WriteFile(live, []byte("everybody's logins"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	// The module is unassigned: the mesh declares something else entirely.
+	after := declare(t, `{"id":"other","type":"directory","path":"`+filepath.Join(root, "other")+`","mode":"0700"}`)
+	report, _, err := Apply(context.Background(), archHost(t), after, state,
+		store.OriginDeclared, nil, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := os.Stat(live); err != nil {
+		t.Fatalf("the data was deleted by unassigning a module: %v", err)
+	}
+	// And it is said, rather than left for somebody to notice. A directory quietly left behind is
+	// how a machine accumulates things nobody can account for.
+	var said bool
+	for _, o := range report.Outcomes {
+		if o.Action == "kept" && strings.Contains(o.Detail, "did not put there") {
+			said = true
+		}
+	}
+	if !said {
+		t.Errorf("the directory was kept and nothing reported it: %+v", report.Outcomes)
+	}
+}
+
+// An empty one is the mesh's own, and goes.
+func TestAnEmptyDirectoryIsStillRemoved(t *testing.T) {
+	root := t.TempDir()
+	mine := filepath.Join(root, "config")
+	d := declare(t, `{"id":"c","type":"directory","path":"`+mine+`","mode":"0700"}`)
+	_, state, err := Apply(context.Background(), archHost(t), d, store.State{},
+		store.OriginDeclared, nil, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	after := declare(t, `{"id":"other","type":"directory","path":"`+filepath.Join(root, "other")+`","mode":"0700"}`)
+	if _, _, err := Apply(context.Background(), archHost(t), after, state,
+		store.OriginDeclared, nil, nil, nil); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(mine); !errors.Is(err, os.ErrNotExist) {
+		t.Fatal("an empty directory the mesh made was left behind, so nothing is ever cleaned up")
+	}
+}