mesh-catalog/modules/nfs-server/cmd/nfs-server/reconcile_test.go

package main

import (
	"context"
	"strings"
	"testing"
	"time"
)

func countExportfs(ran []string) int {
	n := 0
	for _, r := range ran {
		if strings.Contains(r, "exportfs -ra") {
			n++
		}
	}
	return n
}

// The review of #142: pass 1 exports /srv/pool; pass 2 no longer exports it and exportfs fails; pass 3 ran nothing,
// and the kernel still exported the refused folder while the module said it was healthy. A failed reload
// is tried again on the next pass, and while the kernel exports anything of the module's that the file no
// longer carries, the module is unhealthy and says what.
func TestARefusalWhoseReloadFailedIsReloadedAgainAndIsUnhealthyUntilItIs(t *testing.T) {
	f := newFake(t)
	f.mountinfo("/", "/srv/pool")
	f.dir("/srv/pool")
	f.file(ConfigFile, grantsConfig("pool=/srv/pool", "pool=ws:rw"))
	f.roster("ws 10.0.0.3")
	f.wants(map[string][]Want{"ws": {{Share: "pool", Access: "rw"}}})
	f.kernelReads()
	m := f.machine()
	s := m.Pass(context.Background(), nil)
	if !strings.Contains(f.read(Etab), "/srv/pool") {
		t.Fatalf("pass 1 did not export the pool: %+v", s)
	}

	f.file(ConfigFile, grantsConfig("pool=/srv/pool", "none")) // pass 2: the grant is taken away
	f.onExportfs = nil
	f.answers["exportfs -ra"] = Ran{Code: 1, Err: "exportfs: stuck"}
	f.answers["sudo -n exportfs -ra"] = Ran{Code: 1, Err: "exportfs: stuck"}
	s = m.Pass(context.Background(), &s)
	if s.ReloadError == "" {
		t.Fatalf("pass 2's reload did not fail as arranged: %+v", s)
	}

	f.ran = nil
	s = m.Pass(context.Background(), &s)
	if countExportfs(f.ran) == 0 {
		t.Errorf("pass 3 did not try the failed reload again: %v", f.ran)
	}
	h := m.Health()
	if h.Healthy || !strings.Contains(h.Why, "/srv/pool") {
		t.Errorf("the kernel still exports the refused /srv/pool and the module says: %+v", h)
	}

	delete(f.answers, "exportfs -ra")
	delete(f.answers, "sudo -n exportfs -ra")
	f.kernelReads()
	s = m.Pass(context.Background(), &s)
	if strings.Contains(f.read(Etab), "/srv/pool") {
		t.Errorf("a reload that works again did not let the refused pool go: %q", f.read(Etab))
	}
	for _, p := range s.Problems {
		if strings.Contains(p, "/srv/pool") {
			t.Errorf("a problem stays after the kernel let the pool go: %q", p)
		}
	}
}

// A grant taken away while the kernel still holds it — exported before the process started, or a reload
// that never happened — is reconciled: the first pass always reloads, a later pass reloads when the kernel
// exports one of the module's folders to a client the file no longer has, and the module is unhealthy
// while it cannot.
func TestAWithdrawnGrantIsTakenFromTheKernel(t *testing.T) {
	f := newFake(t)
	f.mountinfo("/", "/srv/library")
	f.dir("/srv/library")
	f.file(ConfigFile, grantsConfig("library=/srv/library", "library=ws:rw"))
	f.roster("ws 10.0.0.3", "laptop 10.0.0.4")
	f.wants(map[string][]Want{"ws": {{Share: "library", Access: "rw"}}, "laptop": {{Share: "library", Access: "ro"}}})
	f.kernelReads()
	m := f.machine()
	s := m.Pass(context.Background(), nil)
	// The file and the kernel agree on ws only; the laptop's old export is left in the kernel's table, as
	// a grant withdrawn while the process was down would be.
	f.file(Etab, f.read(Etab)+"/srv/library\t10.0.0.4(ro,sync)\n")

	f.ran = nil
	m2 := f.machine() // a restarted process: its first pass
	m2.Pass(context.Background(), nil)
	if countExportfs(f.ran) == 0 {
		t.Errorf("the first pass of a process did not reload: %v", f.ran)
	}
	if strings.Contains(f.read(Etab), "10.0.0.4") {
		t.Errorf("the withdrawn grant stays in the kernel after the first pass: %q", f.read(Etab))
	}

	f.file(Etab, f.read(Etab)+"/srv/library\t10.0.0.4(ro,sync)\n")
	f.ran = nil
	s = m.Pass(context.Background(), &s)
	if countExportfs(f.ran) == 0 {
		t.Errorf("a later pass did not reload while the kernel exported a folder of the module's to a client the file no longer has: %v", f.ran)
	}
	if strings.Contains(f.read(Etab), "10.0.0.4") {
		t.Errorf("the withdrawn grant stays in the kernel: %q", f.read(Etab))
	}

	// A client the module never exported to — the home network's range, from ZFS's own file — is not
	// the module's, and makes no reload and no finding.
	f.file("/etc/exports.d/zfs.exports", "/srv/library 192.0.2.0/24(rw)\n")
	f.file(Etab, f.read(Etab)+"/srv/library\t192.0.2.0/24(rw)\n")
	f.ran = nil
	s = m.Pass(context.Background(), &s)
	if n := countExportfs(f.ran); n != 0 {
		t.Errorf("an export that is not the module's made a reload: %v", f.ran)
	}
	if h := m.Health(); !h.Healthy {
		t.Errorf("an export that is not the module's made it unhealthy: %+v", h)
	}
}

// The review of #142: SIGTERM in the middle of a pass on a plain restart (the config is still there) ends
// the pass, and the exports stay: the export file is unchanged and the kernel's table keeps the share.
func TestSIGTERMDuringAPassOnARestartLeavesTheExports(t *testing.T) {
	f := newFake(t)
	f.mountinfo("/", "/srv/library")
	f.dir("/srv/library")
	f.file(ConfigFile, grantsConfig("library=/srv/library", "library=ws:rw"))
	f.roster("ws 10.0.0.3")
	f.wants(map[string][]Want{"ws": {{Share: "library", Access: "rw"}}})
	f.kernelReads()
	m := f.machine()
	m.Pass(context.Background(), nil)
	file, table := f.read(ExportFile), f.read(Etab)
	if !strings.Contains(table, "/srv/library") {
		t.Fatalf("the library was not exported: %q", table)
	}

	inner := m.Run
	hung := make(chan struct{}, 8)
	m.Run = func(ctx context.Context, name string, args ...string) Ran {
		if strings.Contains(strings.Join(append([]string{name}, args...), " "), "exportfs") {
			hung <- struct{}{}
			<-ctx.Done()
			return Ran{Code: -1, Err: ctx.Err().Error()}
		}
		return inner(ctx, name, args...)
	}
	ctx, cancel := context.WithCancel(context.Background())
	done := make(chan error, 1)
	go func() { done <- m.RunExports(ctx) }() // its first pass reloads, and hangs there
	select {
	case <-hung:
	case <-time.After(5 * time.Second):
		t.Fatal("the pass never reached exportfs")
	}
	cancel()
	select {
	case <-done:
	case <-time.After(3 * time.Second):
		t.Fatal("the process did not stop after SIGTERM")
	}
	if got := f.read(ExportFile); got != file {
		t.Errorf("a restart changed the export file:\nwas %q\nnow %q", file, got)
	}
	if got := f.read(Etab); got != table {
		t.Errorf("a restart changed what the kernel exports:\nwas %q\nnow %q", table, got)
	}
}

// mesh-host #62's caveat: when the process's first apply failed, it can be recorded after its config, and
// unassignment then stops it with the config still there. The node-engine disables a process's unit before
// it stops it (`systemctl disable --now`), and a restart or a shutdown leaves it enabled, so a process
// whose own unit is disabled when it is stopped is being taken away, whatever the config says.
func TestAProcessWhoseUnitIsDisabledAtStopCleansUpWithTheConfigStillThere(t *testing.T) {
	f := newFake(t)
	f.mountinfo("/", "/srv/library")
	f.dir("/srv/library")
	f.file(ConfigFile, grantsConfig("library=/srv/library", "library=ws:rw"))
	f.roster("ws 10.0.0.3")
	f.wants(map[string][]Want{"ws": {{Share: "library", Access: "rw"}}})
	f.kernelReads()
	m := f.machine()
	m.Pass(context.Background(), nil)

	f.answers["systemctl is-enabled "+ProcessUnit] = Ran{Out: "enabled\n"}
	m.Stopped()
	if !strings.Contains(f.read(ExportFile), "/srv/library") {
		t.Fatalf("a stop with the unit enabled (a restart) removed the exports")
	}

	f.answers["systemctl is-enabled "+ProcessUnit] = Ran{Code: 1, Out: "disabled\n"}
	m.Stopped()
	if strings.Contains(f.read(ExportFile), "/srv/library") {
		t.Errorf("a stop with the unit disabled (unassigned, config still there) left the export file")
	}
	if strings.Contains(f.read(Etab), "/srv/library") {
		t.Errorf("the kernel still exports the library after an unassignment with the config in place: %q", f.read(Etab))
	}
}