mesh-host/internal/witness/draining.go

package witness

import (
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"
)

// **A build is never deleted while a process runs from it** (novox/hq issue 289).
//
// The order of a witnessed update is: the running build moved aside as the previous one, the new one
// unpacked in its place, the unit restarted — the supervisor stops the old process, which drains, then
// starts the new — and the build before it retired once the new one is proved. Between the move and the
// stop the old process still serves, and a process may start another from its own image (the
// controller runs every verb as a command of its own binary). So, while that can happen:
//
//   - the old build stays **reachable by the user it runs as**: the directories the engine keeps beside
//     a process are 0711 — enterable by name, listable by root alone — never 0700, so a build moved
//     aside is still at a path its process may open;
//   - a build that has no reader any more is **retired**, not deleted: renamed under
//     `.witness/<name>/retired/`, which leaves every process running from it untouched, and deleted
//     only when no process on the machine runs from it. Which processes run from it is asked of the
//     kernel — every process's executable — so the one it ran as is found by its PID, and so are the
//     commands it started, wherever its unit is in stopping.
//
// The controller does its half too: it runs its verbs from its own running image rather than the path
// it started from, and refuses as a handover what it cannot run (mesh-controller, issue 289).

// reachable is the mode of every directory the engine keeps beside a process: enterable by the
// process's own user, listable by nobody but root.
const reachable = 0o711

func retiredDir(root, name string) string { return filepath.Join(Dir(root, name), "retired") }

// keep makes the directories the engine keeps about a process — made 0700 before issue 289 — reachable.
func keep(root, name string) error {
	for _, dir := range []string{filepath.Join(root, ".witness"), Dir(root, name)} {
		if err := os.MkdirAll(dir, reachable); err != nil {
			return err
		}
		if err := os.Chmod(dir, reachable); err != nil {
			return err
		}
	}
	return nil
}

// retire takes a build that has no reader any more out of the way: renamed under retired/, and deleted
// at once if nothing runs from it, else at a later sweep. A path that is not there is nothing to do.
func retire(root, name, dir string, now time.Time) error {
	if _, err := os.Lstat(dir); os.IsNotExist(err) {
		return nil
	}
	if err := keep(root, name); err != nil {
		return err
	}
	into := retiredDir(root, name)
	if err := os.MkdirAll(into, reachable); err != nil {
		return err
	}
	if err := os.Chmod(into, reachable); err != nil {
		return err
	}
	at := filepath.Join(into, fmt.Sprintf("%s-%d", filepath.Base(dir), now.UnixNano()))
	if err := os.Rename(dir, at); err != nil {
		return fmt.Errorf("cannot retire %s's build at %s: %w", name, dir, err)
	}
	_, err := Sweep(root, name)
	return err
}

// Sweep deletes every retired build of a process that no process runs from any more, and says which
// are kept, with the processes still running from each.
func Sweep(root, name string) ([]string, error) {
	into := retiredDir(root, name)
	entries, err := os.ReadDir(into)
	if os.IsNotExist(err) {
		return nil, nil
	}
	if err != nil {
		return nil, err
	}
	var draining []string
	for _, e := range entries {
		dir := filepath.Join(into, e.Name())
		if pids := RunningFrom(dir); len(pids) > 0 {
			draining = append(draining, fmt.Sprintf("%s (still run by PID %s)", dir, joinPIDs(pids)))
			continue
		}
		if err := os.RemoveAll(dir); err != nil {
			return draining, err
		}
	}
	return draining, nil
}

// RunningFrom is every process whose executable is a file under dir — renamed there, or deleted from
// there since it started. A variable so a test can name the machine's processes.
var RunningFrom = func(dir string) []int {
	return runningFromProc("/proc", dir)
}

func runningFromProc(proc, dir string) []int {
	entries, err := os.ReadDir(proc)
	if err != nil {
		return nil
	}
	dir = filepath.Clean(dir) + string(filepath.Separator)
	var pids []int
	for _, e := range entries {
		pid, err := strconv.Atoi(e.Name())
		if err != nil {
			continue
		}
		exe, err := os.Readlink(filepath.Join(proc, e.Name(), "exe"))
		if err != nil {
			// Gone between the listing and the read, a kernel thread, or another user's process this
			// engine may not read — it runs as root, so in practice the first two.
			continue
		}
		if strings.HasPrefix(strings.TrimSuffix(exe, " (deleted)"), dir) {
			pids = append(pids, pid)
		}
	}
	sort.Ints(pids)
	return pids
}

func joinPIDs(pids []int) string {
	words := make([]string, len(pids))
	for i, p := range pids {
		words[i] = strconv.Itoa(p)
	}
	return strings.Join(words, ", ")
}

// SweepAll is Sweep for every process the engine keeps anything about: what the witness does on every
// look, so a build retired while its process drained goes once the process has.
func SweepAll(root string) []string {
	var draining []string
	for _, s := range all(root) {
		kept, err := Sweep(root, s.Process)
		if err != nil {
			draining = append(draining, fmt.Sprintf("%s: %v", s.Process, err))
		}
		draining = append(draining, kept...)
	}
	return draining
}