Hold what mounts a missing disk, and say entries the mesh could not take out (hq ADR 0325 re-review)
A container was still created over an unmounted mount point, so its writes landed on the filesystem below; a folder moved to another disk was kept on the old warrant; and entries left by a drop or a move that could not run lived only in a log line. Every container mounting an access whose disk is away now waits, the warrant holds only for the filesystem it was carried out on (the wait names the new one, path-free), entries left on a filesystem are recorded per mount and taken out when it is back, and every leftover is said as unhealthy until it is. A hung stat is shared per path, and a widened mask takes from other entries only the bits it adds.
This commit is contained in:
@@ -9,6 +9,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -861,3 +862,191 @@ func TestMountUnitsAndMountsSeenSayWhereADiskIsMeantToBe(t *testing.T) {
|
||||
t.Fatal("a folder below a mount point seen mounted and gone is not a missing disk")
|
||||
}
|
||||
}
|
||||
|
||||
// keptOnDataset is a folder kept on a dataset the test says is mounted, with the users given, and the dataset.
|
||||
func keptOnDataset(t *testing.T, users ...string) (dataset, lib string) {
|
||||
t.Helper()
|
||||
dir := aclFolder(t)
|
||||
dataset = filepath.Join(dir, "tank")
|
||||
if err := os.Mkdir(dataset, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
keeping(t, []string{dataset}, nil)
|
||||
lib = filepath.Join(dataset, "movies")
|
||||
if _, _, err := applyAccesses(t, store.State{}, accessJSON("radarr.movies", lib, "read-write", false,
|
||||
users...)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return dataset, lib
|
||||
}
|
||||
|
||||
// Review M1: a container that mounts an access whose disk is not there is not created, though its empty mount
|
||||
// point is at the path: its writes would land on the filesystem below (§9).
|
||||
func TestAContainerWaitsWhileItsAccessesDiskIsNotThere(t *testing.T) {
|
||||
_, lib := keptOnDataset(t, "mesh-radarr:61001:read-write:container")
|
||||
MountPoints = func() []string { return []string{"/"} }
|
||||
var ran []string
|
||||
run := func(_ context.Context, name string, args ...string) (string, error) {
|
||||
ran = append(ran, name+" "+strings.Join(args, " "))
|
||||
if len(args) > 0 && args[0] == "container" {
|
||||
return "", errors.New("no such container")
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
d := declare(t, accessJSON("radarr.movies", lib, "read-write", false, "mesh-radarr:61001:read-write:container")+
|
||||
`,{"id":"radarr.app","type":"container","name":"radarr","image":"`+pinned+`","volumes":["`+lib+`:/movies"]}`)
|
||||
report, _, _ := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, run, nil, nil)
|
||||
if o := outcomeOf(report, "radarr.app"); o.Action != "waiting" {
|
||||
t.Fatalf("a container mounting a folder whose disk is not there was not held: %+v", o)
|
||||
}
|
||||
for _, r := range ran {
|
||||
if strings.Contains(r, " run ") || strings.HasSuffix(strings.Fields(r)[0], "docker") && strings.Contains(r, "run") {
|
||||
t.Fatalf("the container was started: %s", r)
|
||||
}
|
||||
}
|
||||
if !WaitingOnAccess(d)["radarr.app"] {
|
||||
t.Error("liveness is not told the container waits on its access")
|
||||
}
|
||||
}
|
||||
|
||||
// Review L2: a folder no access names any more, whose disk is away, keeps its entries' record and is said until
|
||||
// the drop walk can run; then it is forgotten.
|
||||
func TestEntriesThatCouldNotBeDroppedAreSaidUntilTheyAre(t *testing.T) {
|
||||
dataset, lib := keptOnDataset(t, "mesh-radarr:61001:read-write:container")
|
||||
other := `{"id":"x.dir","type":"directory","path":"` + filepath.Join(filepath.Dir(dataset), "x") + `"}`
|
||||
if _, _, err := applyAccesses(t, store.State{}, other); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
MountPoints = func() []string { return []string{"/"} }
|
||||
ReconcileKept(context.Background())
|
||||
left := Leftovers()
|
||||
if len(left) != 1 || left[0].Access != "radarr.movies" || !strings.Contains(left[0].Why, "could not be removed") ||
|
||||
strings.Contains(left[0].Why, dataset) {
|
||||
t.Fatalf("entries that could not be dropped are not said, path-free: %+v", left)
|
||||
}
|
||||
MountPoints = func() []string { return []string{"/", dataset} }
|
||||
ReconcileKept(context.Background())
|
||||
if left := Leftovers(); len(left) != 0 {
|
||||
t.Fatalf("said after the drop ran: %+v", left)
|
||||
}
|
||||
if _, had := keptRecord(lib); had {
|
||||
t.Error("a folder nobody uses was not forgotten once its entries were out")
|
||||
}
|
||||
if _, still := func() (uint16, bool) { a, _, _ := readACLs(t, lib); return a.perm(aclUser, radarr) }(); still {
|
||||
t.Error("the entries were not taken out")
|
||||
}
|
||||
}
|
||||
|
||||
// Review L3: a disk mounted over a folder kept below it leaves the mesh's entries there recorded and said; once that
|
||||
// filesystem holds the folder again, they are taken out.
|
||||
func TestEntriesLeftOnAFilesystemNoLongerAtTheFolderAreDroppedWhenItIsBack(t *testing.T) {
|
||||
dir := aclFolder(t)
|
||||
keeping(t, nil, nil)
|
||||
pool := filepath.Join(dir, "pool")
|
||||
lib := filepath.Join(pool, "media")
|
||||
if err := os.MkdirAll(lib, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := accessJSON("radarr.media", lib, "read-write", false, "mesh-radarr:61001:read-write:container")
|
||||
if _, _, err := applyAccesses(t, store.State{}, a); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
film := filepath.Join(lib, "film")
|
||||
if err := os.WriteFile(film, []byte("data"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = unix.Removexattr(film, aclAccessXattr)
|
||||
MountPoints = func() []string { return []string{"/", pool} }
|
||||
if _, _, err := applyAccesses(t, store.State{}, a); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if left := Leftovers(); len(left) != 1 || !strings.Contains(left[0].Why, "no longer at its folder") {
|
||||
t.Fatalf("entries left on the filesystem below are not said: %+v", left)
|
||||
}
|
||||
// The pool goes: the filesystem below holds the folder again, and the entries left there are taken out.
|
||||
MountPoints = func() []string { return []string{"/"} }
|
||||
ReconcileKept(context.Background())
|
||||
if _, still := func() (uint16, bool) { a, _, _ := readACLs(t, lib); return a.perm(aclUser, radarr) }(); still {
|
||||
t.Fatal("the entries left on the filesystem that came back were not taken out")
|
||||
}
|
||||
if left := Leftovers(); len(left) != 0 {
|
||||
t.Fatalf("still said after they were taken out: %+v", left)
|
||||
}
|
||||
}
|
||||
|
||||
// Review L4: the controller's word that a folder was taken holds for the filesystem the warrant was carried out on;
|
||||
// moved to another, it is not taken, the wait names the new filesystem, and nothing is walked.
|
||||
func TestAMovedFolderIsNotTakenOnTheOldWarrant(t *testing.T) {
|
||||
dir := aclFolder(t)
|
||||
keeping(t, nil, nil)
|
||||
pool := filepath.Join(dir, "pool")
|
||||
lib := filepath.Join(pool, "media")
|
||||
if err := os.MkdirAll(filepath.Join(lib, "old"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
notTaken := accessJSON("radarr.media", lib, "read-write", false, "mesh-radarr:61001:read-write:container")
|
||||
if _, _, err := applyAccesses(t, store.State{}, notTaken); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
done, _, err := TakeFolder(context.Background(), declare(t, notTaken), "radarr.media", lib, "w-1", "op", noServices)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
<-done
|
||||
before := AccessVerdict(context.Background(), lib, true, true)
|
||||
film := filepath.Join(lib, "film")
|
||||
if err := os.WriteFile(film, []byte("data"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = unix.Removexattr(film, aclAccessXattr)
|
||||
MountPoints = func() []string { return []string{"/", pool} }
|
||||
taken := accessJSON("radarr.media", lib, "read-write", true, "mesh-radarr:61001:read-write:container")
|
||||
report, _, err := applyAccesses(t, store.State{}, taken)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" {
|
||||
t.Fatalf("a folder moved to another filesystem was kept on the old warrant: %+v", o)
|
||||
}
|
||||
// And on the next apply too, its record now on the new filesystem.
|
||||
report, _, _ = applyAccesses(t, store.State{}, taken)
|
||||
if o := outcomeOf(report, "radarr.media"); o.Action != "waiting" {
|
||||
t.Fatalf("the second apply kept it on the old warrant: %+v", o)
|
||||
}
|
||||
v := AccessVerdict(context.Background(), lib, true, true)
|
||||
if v.Kind != VerdictWaitKeep || v.Placed == "" || v.Placed == before.Placed || strings.Contains(v.Placed, "/") {
|
||||
t.Fatalf("the look does not ask again naming the new filesystem: %+v (before %+v)", v, before)
|
||||
}
|
||||
ReconcileKept(context.Background())
|
||||
if got := effectiveOn(t, film, radarr); got != 0 {
|
||||
t.Fatal("the data on the new filesystem was walked")
|
||||
}
|
||||
}
|
||||
|
||||
// Review L5: a read that hangs holds one goroutine per path, whoever asks meanwhile.
|
||||
func TestAHungReadIsSharedByEveryoneWhoAsks(t *testing.T) {
|
||||
was := statBound
|
||||
statBound = 20 * time.Millisecond
|
||||
defer func() { statBound = was }()
|
||||
release := make(chan struct{})
|
||||
defer close(release)
|
||||
var started int32
|
||||
var mu sync.Mutex
|
||||
hang := func(string) (os.FileInfo, error) {
|
||||
mu.Lock()
|
||||
started++
|
||||
mu.Unlock()
|
||||
<-release
|
||||
return nil, nil
|
||||
}
|
||||
for i := 0; i < 5; i++ {
|
||||
if _, err := within("test", hang, "/a/dead/mount"); !errors.Is(err, errNoAnswer) {
|
||||
t.Fatalf("a hung read answered: %v", err)
|
||||
}
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if started != 1 {
|
||||
t.Fatalf("%d reads of one path were started while the first hung", started)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user