mesh-host/internal/witness/draining.go

package witness

import (
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"
)

// **A build is never deleted while a process runs from it** (novox/hq issue 289).
//
// The order of a witnessed update is: the running build moved aside as the previous one, the new one
// unpacked in its place, the unit restarted — the supervisor stops the old process, which drains, then
// starts the new — and the build before it retired once the new one is proved. Between the move and the
// stop the old process still serves, and a process may start another from its own image (the
// controller runs every verb as a command of its own binary). So, while that can happen:
//
//   - the old build stays **reachable by the user it runs as**: the directories the engine keeps beside
//     a process are 0711 — enterable by name, listable by root alone — never 0700, so a build moved
//     aside is still at a path its process may open;
//   - a build that has no reader any more is **retired**, not deleted: renamed under
//     `.witness/<name>/retired/`, which leaves every process running from it untouched, and deleted
//     only when no process on the machine runs from it. Which processes run from it is asked of the
//     kernel — every process's executable — so the one it ran as is found by its PID, and so are the
//     commands it started, wherever its unit is in stopping.
//
// The controller does its half too: it runs its verbs from its own running image rather than the path
// it started from, and refuses as a handover what it cannot run (mesh-controller, issue 289).

// reachable is the mode of every directory the engine keeps beside a process: enterable by the
// process's own user, listable by nobody but root.
const reachable = 0o711

func retiredDir(root, name string) string { return filepath.Join(Dir(root, name), "retired") }

// keep makes the directories the engine keeps about a process — made 0700 before issue 289 — reachable.
func keep(root, name string) error {
	for _, dir := range []string{filepath.Join(root, ".witness"), Dir(root, name)} {
		if err := os.MkdirAll(dir, reachable); err != nil {
			return err
		}
		if err := os.Chmod(dir, reachable); err != nil {
			return err
		}
	}
	return nil
}

// Retire takes a process's build out of the way before another is unpacked where it was: renamed under
// retired/, and deleted at once if nothing runs from it, else at a later sweep. A process nobody judges
// is replaced through it as a judged one is (novox/hq issue 734): deleting the running build before the
// restart deleted the working directory of the process still running from it, and every command it
// started until the restart failed with "getwd: no such file or directory".
func Retire(root, name, dir string, now time.Time) error { return retire(root, name, dir, now) }

// retire takes a build that has no reader any more out of the way: renamed under retired/, and deleted
// at once if nothing runs from it, else at a later sweep. A path that is not there is nothing to do.
func retire(root, name, dir string, now time.Time) error {
	if _, err := os.Lstat(dir); os.IsNotExist(err) {
		return nil
	}
	if err := keep(root, name); err != nil {
		return err
	}
	into := retiredDir(root, name)
	if err := os.MkdirAll(into, reachable); err != nil {
		return err
	}
	if err := os.Chmod(into, reachable); err != nil {
		return err
	}
	at := filepath.Join(into, fmt.Sprintf("%s-%d", filepath.Base(dir), now.UnixNano()))
	if err := os.Rename(dir, at); err != nil {
		return fmt.Errorf("cannot retire %s's build at %s: %w", name, dir, err)
	}
	_, err := Sweep(root, name)
	return err
}

// Sweep deletes every retired build of a process that no process runs from any more, and says which
// are kept, with the processes still running from each.
func Sweep(root, name string) ([]string, error) {
	into := retiredDir(root, name)
	entries, err := os.ReadDir(into)
	if os.IsNotExist(err) {
		return nil, nil
	}
	if err != nil {
		return nil, err
	}
	var draining []string
	for _, e := range entries {
		dir := filepath.Join(into, e.Name())
		if pids := RunningFrom(dir); len(pids) > 0 {
			draining = append(draining, fmt.Sprintf("%s (still run by PID %s)", dir, joinPIDs(pids)))
			continue
		}
		if err := os.RemoveAll(dir); err != nil {
			return draining, err
		}
	}
	return draining, nil
}

// RunningFrom is every process whose executable is a file under dir, or whose working directory is in
// it — renamed there, or deleted from there since it started. A working directory counts as much as an
// image: a process started by the one that runs from a build (a `go version` a builder asks) works in
// the build's directory and runs nothing from it (novox/hq issue 734). A variable so a test can name the
// machine's processes.
var RunningFrom = func(dir string) []int {
	return runningFromProc("/proc", dir)
}

func runningFromProc(proc, dir string) []int {
	entries, err := os.ReadDir(proc)
	if err != nil {
		return nil
	}
	dir = filepath.Clean(dir) + string(filepath.Separator)
	var pids []int
	for _, e := range entries {
		pid, err := strconv.Atoi(e.Name())
		if err != nil {
			continue
		}
		for _, link := range []string{"exe", "cwd"} {
			at, err := os.Readlink(filepath.Join(proc, e.Name(), link))
			if err != nil {
				// Gone between the listing and the read, a kernel thread, or another user's process this
				// engine may not read — it runs as root, so in practice the first two.
				continue
			}
			if strings.HasPrefix(strings.TrimSuffix(at, " (deleted)")+string(filepath.Separator), dir) {
				pids = append(pids, pid)
				break
			}
		}
	}
	sort.Ints(pids)
	return pids
}

func joinPIDs(pids []int) string {
	words := make([]string, len(pids))
	for i, p := range pids {
		words[i] = strconv.Itoa(p)
	}
	return strings.Join(words, ", ")
}

// SweepAll is Sweep for every process the engine keeps anything about: what the witness does on every
// look, so a build retired while its process drained goes once the process has.
//
// Every process the engine keeps a directory for, not only those it judges: a process nobody judges keeps
// nothing but the builds it retired, and those drain the same way (novox/hq issue 734).
func SweepAll(root string) []string {
	entries, err := os.ReadDir(filepath.Join(root, ".witness"))
	if err != nil {
		return nil
	}
	var draining []string
	for _, e := range entries {
		if !e.IsDir() {
			continue
		}
		name := e.Name()
		kept, err := Sweep(root, name)
		if err != nil {
			draining = append(draining, fmt.Sprintf("%s: %v", name, err))
		}
		draining = append(draining, kept...)
		// A process nobody judges whose last retired build has drained leaves nothing to keep.
		if s, err := Load(root, name); err == nil && s.Process == "" && len(kept) == 0 {
			_ = os.Remove(retiredDir(root, name))
			_ = os.Remove(Dir(root, name))
		}
	}
	return draining
}