mesh-catalog/modules/zfs/cmd/zfs-tools/manifest_test.go

package main

import (
	"encoding/json"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"testing"
	"time"
)

type manifest struct {
	Tools     []string            `json:"tools"`
	Claims    []any               `json:"claims"`
	Emits     []string            `json:"emits"`
	Replaces  map[string][]string `json:"replaces"`
	Resources []map[string]any    `json:"resources"`
}

func readManifest(t *testing.T) (manifest, string) {
	t.Helper()
	raw, err := os.ReadFile("../../module.json")
	if err != nil {
		t.Fatal(err)
	}
	var m manifest
	if err := json.Unmarshal(raw, &m); err != nil {
		t.Fatal(err)
	}
	return m, string(raw)
}

// The manifest lists exactly the tools the bundle serves; every one but zfs_snapshot says it only reads,
// and zfs_snapshot says it acts. Each says what it replaces (novox/hq ADR 0245), but the health check,
// which replaces nothing. No seat: a new mesh or node seat is the controller's to define (ADR 0121).
// handTools are the tools that start or steer the scheduled run, which replace no command.
var handTools = map[string]bool{"zfs_snapshots_run_now": true, "zfs_snapshots_prune_now": true, "zfs_config_apply": true,
	"zfs_snapshots_pause": true, "zfs_snapshots_resume": true, "zfs_snapshots_record": true,
	// The module's own read tools (§28) replace no command.
	"zfs_config": true, "zfs_snapshots_schedule": true, "zfs_module_events": true, "zfs_snapshots_check": true, "zfs_findings": true}

var ownReads = map[string]bool{"zfs_config": true, "zfs_snapshots_schedule": true, "zfs_module_events": true, "zfs_snapshots_check": true, "zfs_findings": true}

func TestTheManifestNamesTheToolsTheBundleServesAndWhatEachReplaces(t *testing.T) {
	m, raw := readManifest(t)
	var served []string
	for _, tl := range Tools(newFake(t).machine()) {
		served = append(served, tl.Name)
		want := "(r)"
		if tl.Name == "zfs_snapshot" || strings.HasPrefix(tl.Name, "zfs_scrub_") || (handTools[tl.Name] && !ownReads[tl.Name]) {
			want = "(a)"
		}
		if !strings.HasSuffix(tl.Description, want) {
			t.Errorf("%s does not end with %s", tl.Name, want)
		}
		if tl.Name != "zfs_health" && !handTools[tl.Name] && len(m.Replaces[tl.Name]) == 0 {
			t.Errorf("%s says nothing of what it replaces", tl.Name)
		}
		for _, c := range m.Replaces[tl.Name] {
			if !strings.Contains(tl.Description, c) {
				t.Errorf("%s replaces %q in the manifest and not in its description", tl.Name, c)
			}
		}
	}
	sort.Strings(served)
	listed := append([]string(nil), m.Tools...)
	sort.Strings(listed)
	if strings.Join(served, ",") != strings.Join(listed, ",") {
		t.Fatalf("served %v, listed %v", served, listed)
	}
	if _, ok := m.Replaces["zfs_health"]; ok {
		t.Errorf("the health check replaces no command")
	}
	if len(m.Claims) != 0 {
		t.Errorf("the module claims no seat: %v", m.Claims)
	}
	if strings.Join(m.Emits, ",") != "snapshot.taken,"+ScrubEvent {
		t.Errorf("emits %v", m.Emits)
	}
	if hit := regexp.MustCompile(`\b(g14|novox|ace|shanks|jochen)\b|/home/`).FindString(strings.ReplaceAll(raw, "novox/hq", "")); hit != "" {
		t.Errorf("the manifest says %q", hit)
	}
}

// ADR 0240: zfs_health on the watch, which runs as the operator account; the follower judged by its
// unit. The scheduled run is not long-running and declares none. The module installs nothing, so it never
// puts ZFS on a machine.
func TestTheHealthChecksAndWhoRunsWhat(t *testing.T) {
	m, raw := readManifest(t)
	byID := map[string]map[string]any{}
	for _, r := range m.Resources {
		byID[r["id"].(string)] = r
	}
	w := byID["watch"]
	if h, _ := w["health"].(map[string]any); h["kind"] != "tool" || h["tool"] != "zfs_health" || w["user"] != "${machine:account}" {
		t.Errorf("the watch runs as the operator account and carries zfs_health: %v", w)
	}
	e := byID["events"]
	if h, _ := e["health"].(map[string]any); h["kind"] != "unit" || e["user"] != nil {
		t.Errorf("the follower runs as root, judged by its unit: %v", e)
	}
	if d := byID["state-dir"]; d["path"] != "/var/lib/zfs-tools" || d["owner"] != "${machine:account}" ||
		filepath.Dir(ErrorsFile) != d["path"] {
		t.Errorf("the state the account keeps: %v", d)
	}
	// The scheduled run's state is root's alone, apart from the account's (third review).
	if d := byID["scrub-state-dir"]; d["path"] != filepath.Dir(ServedFile) || d["mode"] != "0700" || d["owner"] != nil {
		t.Errorf("the state the run keeps: %v", d)
	}
	if strings.Contains(raw, `"package"`) {
		t.Errorf("the module installs no package: ZFS is the machine's, and its kernel module is built against its kernel")
	}
}

// One schedule per pool, and no setting in any unit's name but the distribution's timers', which the
// controller fills (mesh-controller #171) and checks. The module's own schedule is the node-engine's timer
// for the scheduled run, a fixed name; the day and time reach only the configuration, which the run checks.
// The run comes before the distribution's timers are stopped, so a run that cannot be installed stops the
// apply before the timer that scrubs the pool today is touched.
func TestExactlyOneScrubScheduleAndNoSettingInAUnitFile(t *testing.T) {
	m, raw := readManifest(t)
	byID := map[string]map[string]any{}
	order := map[string]int{}
	for i, r := range m.Resources {
		byID[r["id"].(string)] = r
		order[r["id"].(string)] = i
	}
	run := byID["run"]
	args, _ := run["run"].([]any)
	if run["type"] != "process" || run["name"] != "zfs-mesh-run" || run["schedule"] != "0,15,30,45 * * * *" || run["user"] != nil ||
		len(args) != 2 || args[1] != "run" {
		t.Errorf("the scheduled run: %v", run)
	}
	// The node-engine writes a cron step ("*/15") as a calendar systemd refuses ("*:*/15"), and the timer never
	// starts (novox/hq issue 1979, on the home server 2026-11-12). A list of minutes is written as a calendar
	// systemd takes ("*-*-* *:0,15,30,45:00"), so the schedule names its minutes and never a step.
	if s, _ := run["schedule"].(string); strings.Contains(s, "/") {
		t.Errorf("the scheduled run's schedule %q uses a step, which the node-engine writes as a calendar systemd refuses", s)
	}
	if RunTimer != run["name"].(string)+".timer" || RunService != run["name"].(string)+".service" {
		t.Errorf("the tools read %s and %s; the node-engine names them after %v", RunTimer, RunService, run["name"])
	}
	for _, c := range []string{"weekly", "monthly"} {
		d := byID["distribution-"+c]
		if d["unit"] != "zfs-scrub-"+c+"@${config:scrub-pool}.timer" || d["state"] != "stopped" || d["boot"] != "disabled" {
			t.Errorf("the distribution's %s timer: %v", c, d)
		}
		if order["run"] > order["distribution-"+c] {
			t.Errorf("the distribution's %s timer is stopped before the run is installed", c)
		}
	}
	for _, r := range m.Resources {
		p, _ := r["path"].(string)
		if strings.HasPrefix(p, "/etc/systemd/") {
			t.Errorf("the module writes a unit file of its own: %s", p)
		}
	}
	// The configuration's text is its template's (novox/hq ADR 0349 §2): the manifest itself names none of them.
	conf := templateText(t, byID["scrub-config"])
	if strings.Contains(raw, "scrub-day}") || !strings.Contains(conf, "${config:scrub-day}") {
		t.Errorf("the day reaches something other than the configuration")
	}
	if n := strings.Count(conf, "${config:scrub-day}") + strings.Count(conf, "${config:scrub-time}") + strings.Count(conf, "${config:scrub-cadence}"); n != 3 {
		t.Errorf("the day, time and cadence are written %d times; only into the configuration", n)
	}
	var declared struct {
		Config map[string]map[string]any `json:"config"`
	}
	json.Unmarshal([]byte(raw), &declared)
	if declared.Config["scrub-cadence"]["default"] != "weekly" || declared.Config["scrub-day"]["default"] != "Mon" ||
		declared.Config["scrub-time"]["default"] != "00:00" {
		t.Errorf("defaults: %v", declared.Config)
	}
	if p := declared.Config["scrub-pool"]; p["required"] != true || p["default"] != nil {
		t.Errorf("the pool is the operator's value: required, with no default (ADR 0112, ADR 0348 §4): %v", p)
	}
}

// The home server, composed as the controller fills its settings (pool storage, cadence monthly, the
// operator's choice): the units the node-engine is sent and the configuration the run reads.
func TestTheHomeServersComposedUnitsAndConfiguration(t *testing.T) {
	m, _ := readManifest(t)
	fill := strings.NewReplacer("${config:scrub-pool}", "storage", "${config:scrub-cadence}", "monthly",
		"${config:scrub-day}", "Mon", "${config:scrub-time}", "00:00")
	var units []string
	var conf string
	for _, r := range m.Resources {
		switch {
		case r["type"] == "service":
			units = append(units, fill.Replace(r["unit"].(string))+" "+r["state"].(string)+"/"+r["boot"].(string))
		case r["type"] == "process" && r["schedule"] != nil:
			units = append(units, r["name"].(string)+".timer running/enabled (the node-engine's, every quarter of an hour)")
		case r["id"] == "scrub-config":
			conf = fill.Replace(templateText(t, r))
		}
	}
	want := []string{"zfs-mesh-run.timer running/enabled (the node-engine's, every quarter of an hour)",
		"zfs-scrub-weekly@storage.timer stopped/disabled", "zfs-scrub-monthly@storage.timer stopped/disabled"}
	if strings.Join(units, "\n") != strings.Join(want, "\n") {
		t.Errorf("units:\n%s", strings.Join(units, "\n"))
	}
	for _, u := range units {
		t.Log(u)
	}
	_ = conf
	cfg, refused := ParseConfig([]byte(scrubConf("storage", "monthly", "Mon", "00:00")), 16000)
	s := ScheduleOf(cfg)
	if len(refused) != 0 || s == nil || s.Pool != "storage" || s.Cadence != "monthly" {
		t.Fatalf("configuration: %v", refused)
	}
	t.Logf("next scrub of storage after 2026-10-09 12:00: %s", s.NextSlot(time.Date(2026, 10, 9, 12, 0, 0, 0, time.Local)).Format("2006-01-02 15:04"))
	if next := s.NextSlot(time.Date(2026, 10, 9, 12, 0, 0, 0, time.Local)); !next.Equal(time.Date(2026, 11, 2, 0, 0, 0, 0, time.Local)) {
		t.Errorf("next monthly slot %s, want the first Monday of November", next)
	}
}

// Nothing but reading, a snapshot and a scrub's start, pause or stop. Every command the bundle runs goes
// through ask, AsRoot, Escalated, Run, exec.Command or exec.CommandContext with an argument list from one
// of the functions below, each held to its program and verb; a call site that builds its own list fails
// this test. A new verb needs a decision first, and then a line here.
func TestTheBundleRunsOnlyReadingVerbsSnapshotAndScrub(t *testing.T) {
	allowed := map[string][2]string{
		"DatasetsArgv": {"zfs", "list"}, "SnapshotsArgv": {"zfs", "list"}, "PropertiesArgv": {"zfs", "get"},
		"SnapshotArgv": {"zfs", "snapshot"},
		"PoolsArgv":    {"zpool", "list"}, "StatusArgv": {"zpool", "status"}, "IOArgv": {"zpool", "iostat"},
		"EventsArgv": {"zpool", "events"}, "HistoryArgv": {"zpool", "history"}, "FollowArgv": {"zpool", "events"},
		"ScrubArgv": {"zpool", "scrub"}, "TrimArgv": {"zpool", "trim"},
		"TimerShowArgv": {"systemctl", "show"}, "ServiceShowArgv": {"systemctl", "show"},
		"PoolGetArgv": {"zpool", "get"}, "PoolSetArgv": {"zpool", "set"}, "DatasetGetArgv": {"zfs", "get"},
		"DatasetSetArgv": {"zfs", "set"}, "DatasetInheritArgv": {"zfs", "inherit"},
		"DeviceStatusArgv": {"zpool", "status"}, "AllRealStatusArgv": {"zpool", "status"}, "RemoveArgv": {"zpool", "remove"},
		"AddArgv": {"zpool", "add"}, "WipefsArgv": {"wipefs", "--no-act"}, "ZdbLabelArgv": {"zdb", "-l"},
		"UdevArgv": {"udevadm", "monitor"}, "SnapshotSetArgv": {"zfs", "set"}, "StatusPathsArgv": {"zpool", "status"}, "HoldsArgv": {"zfs", "holds"}, "SnapshotPropertyArgv": {"zfs", "get"}, "DestroyArgv": {"zfs", "destroy"}, "ZdbConfigArgv": {"zdb", "-C"}, "ZgenhostidArgv": {"zgenhostid", "-f"},
	}
	verbs := map[string][]string{"DatasetsArgv": DatasetsArgv(), "SnapshotsArgv": SnapshotsArgv(), "PoolsArgv": PoolsArgv(),
		"StatusArgv": StatusArgv(), "IOArgv": IOArgv(3), "EventsArgv": EventsArgv(), "HistoryArgv": HistoryArgv("tank"),
		"PropertiesArgv": PropertiesArgv("tank"), "SnapshotArgv": SnapshotArgv("tank", "x"), "FollowArgv": FollowArgv(),
		"TimerShowArgv": TimerShowArgv("x.timer"), "ServiceShowArgv": ServiceShowArgv("x.service"), "TrimArgv": TrimArgv("tank"),
		"PoolGetArgv": PoolGetArgv("tank", "autotrim"), "PoolSetArgv": PoolSetArgv("tank", "autotrim", "on"),
		"DatasetGetArgv": DatasetGetArgv("tank", "atime"), "DatasetSetArgv": DatasetSetArgv("tank", "atime", "on"),
		"DatasetInheritArgv": DatasetInheritArgv("tank", "atime"),
		"DeviceStatusArgv":   DeviceStatusArgv("tank", true), "AllRealStatusArgv": AllRealStatusArgv(),
		"RemoveArgv": RemoveArgv("tank", "x"), "AddArgv": AddArgv("tank", "cache", false, "/dev/x"),
		"WipefsArgv": WipefsArgv("/dev/x"), "ZdbLabelArgv": ZdbLabelArgv("/dev/x"),
		"HoldsArgv": HoldsArgv("tank@x"), "SnapshotPropertyArgv": SnapshotPropertyArgv("tank@x", "clones"), "DestroyArgv": DestroyArgv("tank", "x"),
		"StatusPathsArgv": StatusPathsArgv(), "UdevArgv": UdevArgv(),
		"SnapshotSetArgv": SnapshotSetArgv("tank@mesh-hourly-x", "hourly"), "ZdbConfigArgv": ZdbConfigArgv("tank"), "ZgenhostidArgv": ZgenhostidArgv("00bab10c")}
	for fn, argv := range verbs {
		for _, a := range argv {
			if fn != "FollowArgv" && fn != "ZgenhostidArgv" && (a == "-f" || a == "-r" || a == "-R" || a == "-d") {
				t.Errorf("%s runs %q", fn, a)
			}
		}
		if argv[0] != allowed[fn][1] {
			t.Errorf("%s runs %q, not %q", fn, argv[0], allowed[fn][1])
		}
	}
	for _, act := range []string{"start", "pause", "stop", "anything"} {
		argv := ScrubArgv(act, "tank")
		if argv[0] != "scrub" || argv[len(argv)-2] != "--" || argv[len(argv)-1] != "tank" || len(argv) > 4 {
			t.Errorf("%s: %v", act, argv)
		}
		if len(argv) == 4 && argv[1] != "-p" && argv[1] != "-s" {
			t.Errorf("%s: %v", act, argv)
		}
	}
	call := regexp.MustCompile(`(\.(ask|AsRoot|Escalated|Run|Start|Output|CombinedOutput)\(|exec\.Command(Context)?\(|os\.StartProcess\(|syscall\.(Exec|ForkExec)\()`)
	site := regexp.MustCompile(`(?:\.(?:ask|AsRoot|Escalated)|exec\.Command)\((?:\w+, )?"(zfs|zpool|systemctl|wipefs|zdb|zgenhostid|udevadm)", (\w+)\(`)
	// The bodies that run what they are given, each once.
	bodies := map[string]bool{
		"machine.go:	r := m.Run(ctx, name, args...)":              true,
		"machine.go:	r := m.Run(ctx, program, argv...)":           true,
		"machine.go:	program, argv := m.Escalated(name, args...)": true,
		"machine.go:	cmd := exec.Command(name, args...)":          true,
		"machine.go:	if err := cmd.Start(); err != nil {":         true,
		"follow.go:	if err := cmd.Start(); err != nil {":          true,
		"follow_wake.go:	if err := cmd.Start(); err != nil {":     true,
	}
	files, _ := filepath.Glob("*.go")
	sites := 0
	for _, file := range files {
		if strings.HasSuffix(file, "_test.go") {
			continue
		}
		b, err := os.ReadFile(file)
		if err != nil {
			t.Fatal(err)
		}
		for i, l := range strings.Split(string(b), "\n") {
			trimmed := strings.TrimSpace(l)
			if !call.MatchString(l) || strings.HasPrefix(trimmed, "//") || bodies[file+":"+l] {
				continue
			}
			m := site.FindStringSubmatch(l)
			if m == nil {
				t.Errorf("%s:%d runs a command not built by an argument function: %s", file, i+1, trimmed)
				continue
			}
			want, ok := allowed[m[2]]
			if !ok {
				t.Errorf("%s:%d runs %s's list, which is not held here", file, i+1, m[2])
			} else if want[0] != m[1] {
				t.Errorf("%s:%d runs %s with %s's list", file, i+1, m[1], m[2])
			}
			sites++
		}
	}
	if sites < 14 {
		t.Errorf("found %d call sites; the test no longer sees them", sites)
	}
}