mesh-host/internal/witness/draining_test.go

package witness

import (
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"slices"
	"strconv"
	"testing"
	"time"
)

// A real process running from a build, as a controller runs from its own: a copy of sleep, started
// from the directory the build is unpacked in.
func runFrom(t *testing.T, dir string) *exec.Cmd {
	t.Helper()
	if runtime.GOOS != "linux" {
		t.Skip("which process runs from a build is read from /proc")
	}
	sleep, err := exec.LookPath("sleep")
	if err != nil {
		t.Skip("no sleep to run")
	}
	if sleep, err = filepath.EvalSymlinks(sleep); err != nil {
		t.Fatal(err)
	}
	in, err := os.Open(sleep)
	if err != nil {
		t.Fatal(err)
	}
	defer in.Close()
	if err := os.MkdirAll(dir, 0o755); err != nil {
		t.Fatal(err)
	}
	image := filepath.Join(dir, "sleep")
	out, err := os.OpenFile(image, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o755)
	if err != nil {
		t.Fatal(err)
	}
	if _, err := io.Copy(out, in); err != nil {
		t.Fatal(err)
	}
	if err := out.Close(); err != nil {
		t.Fatal(err)
	}
	cmd := exec.Command(image, "60")
	if err := cmd.Start(); err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait() })
	// The process is known to run from the image only once /proc says so: waited for, never slept on (a check that
	// read /proc before the kernel showed the new image found nothing, issue 1945's try of 2026-10-25 at 12:47).
	exe := filepath.Join("/proc", strconv.Itoa(cmd.Process.Pid), "exe")
	deadline := time.Now().Add(5 * time.Second)
	for {
		if got, err := os.Readlink(exe); err == nil && got == image {
			break
		}
		if time.Now().After(deadline) {
			got, err := os.Readlink(exe)
			t.Fatalf("PID %d does not run from %s within 5s: %s reads %q (%v)", cmd.Process.Pid, image, exe, got, err)
		}
		runtime.Gosched()
	}
	return cmd
}

func retiredBuilds(t *testing.T, root, name string) []string {
	t.Helper()
	entries, err := os.ReadDir(retiredDir(root, name))
	if err != nil && !os.IsNotExist(err) {
		t.Fatal(err)
	}
	var out []string
	for _, e := range entries {
		out = append(out, e.Name())
	}
	return out
}

// **A build moved aside while its process runs stays reachable by that process's user, and is deleted
// only once no process runs from it** (novox/hq issue 289). On 2026-10-07 the controller still serving
// after its build was moved into a 0700 directory, and then deleted, could not run its own verbs.
func TestABuildIsKeptReachableAndUndeletedWhileAProcessRunsFromIt(t *testing.T) {
	root := t.TempDir()
	name := "mesh-controller"
	start := time.Date(2026, 10, 7, 2, 0, 0, 0, time.UTC)

	// Build A placed and running.
	p, err := Place(root, name, ByLease, buildA, "", "", start)
	if err != nil {
		t.Fatal(err)
	}
	old := runFrom(t, filepath.Join(root, name))
	if err := p.Commit(); err != nil {
		t.Fatal(err)
	}

	// Build B placed while A still runs: A is moved aside, and reachable at its new path.
	place(t, root, name, ByLease, buildB, "", "", start.Add(time.Minute))
	for _, dir := range []string{filepath.Join(root, ".witness"), Dir(root, name)} {
		info, err := os.Stat(dir)
		if err != nil {
			t.Fatal(err)
		}
		if info.Mode().Perm() != reachable {
			t.Fatalf("%s is %v: the user the moved build runs as cannot reach it", dir, info.Mode().Perm())
		}
	}
	if pids := RunningFrom(previousDir(root, name)); !slices.Contains(pids, old.Process.Pid) {
		t.Fatalf("the process running from the moved build (PID %d) is not found there: %v", old.Process.Pid, pids)
	}

	// B proved while A has not stopped yet: A is retired, not deleted.
	if err := Proved(root, name); err != nil {
		t.Fatal(err)
	}
	if kept(root, name) {
		t.Fatal("the build before the proved one is still kept to go back to")
	}
	if got := retiredBuilds(t, root, name); len(got) != 1 {
		t.Fatalf("the build a process still runs from was deleted: retired %v", got)
	}
	draining, err := Sweep(root, name)
	if err != nil || len(draining) != 1 {
		t.Fatalf("a sweep while it runs: %v %v", draining, err)
	}

	// A stops: the next sweep deletes it.
	_ = old.Process.Kill()
	_ = old.Wait()
	if draining, err := Sweep(root, name); err != nil || len(draining) != 0 {
		t.Fatalf("a sweep after it stopped: %v %v", draining, err)
	}
	if got := retiredBuilds(t, root, name); len(got) != 0 {
		t.Fatalf("a build nothing runs from was kept: %v", got)
	}
}

// A process whose image was deleted from under it still counts as running from where it was.
func TestAProcessRunsFromABuildDeletedUnderIt(t *testing.T) {
	dir := filepath.Join(t.TempDir(), "build")
	proc := runFrom(t, dir)
	if err := os.Remove(filepath.Join(dir, "sleep")); err != nil {
		t.Fatal(err)
	}
	if pids := RunningFrom(dir); !slices.Contains(pids, proc.Process.Pid) {
		t.Fatalf("PID %d runs a deleted image from %s and was not found: %v", proc.Process.Pid, dir, pids)
	}
	if pids := RunningFrom(filepath.Join(t.TempDir(), "elsewhere")); slices.Contains(pids, proc.Process.Pid) {
		t.Fatal("found running from a directory it never ran from")
	}
}

// A build on trial replaced by another is retired too: it runs until the restart stops it.
func TestABuildOnTrialReplacedIsRetiredNotDeletedUnderItsProcess(t *testing.T) {
	root := t.TempDir()
	name := "node-tools"
	start := time.Date(2026, 10, 7, 2, 0, 0, 0, time.UTC)
	place(t, root, name, ByPing, buildA, "", "", start)
	p, err := Place(root, name, ByPing, buildB, "", "", start.Add(time.Minute))
	if err != nil {
		t.Fatal(err)
	}
	trial := runFrom(t, filepath.Join(root, name))
	if err := p.Commit(); err != nil {
		t.Fatal(err)
	}
	place(t, root, name, ByPing, buildC, "", "", start.Add(2*time.Minute))
	if got := retiredBuilds(t, root, name); len(got) != 1 {
		t.Fatalf("the build on trial was deleted while it ran: %v", got)
	}
	_ = trial.Process.Kill()
	_ = trial.Wait()
	SweepAll(root)
	if got := retiredBuilds(t, root, name); len(got) != 0 {
		t.Fatalf("kept after its process stopped: %v", got)
	}
	if running(t, root, name) != buildC || !kept(root, name) {
		t.Fatal("C should run with A kept to go back to")
	}
}