package witness import ( "fmt" "os" "path/filepath" "sort" "strconv" "strings" "time" ) // **A build is never deleted while a process runs from it** (novox/hq issue 289). // // The order of a witnessed update is: the running build moved aside as the previous one, the new one // unpacked in its place, the unit restarted — the supervisor stops the old process, which drains, then // starts the new — and the build before it retired once the new one is proved. Between the move and the // stop the old process still serves, and a process may start another from its own image (the // controller runs every verb as a command of its own binary). So, while that can happen: // // - the old build stays **reachable by the user it runs as**: the directories the engine keeps beside // a process are 0711 — enterable by name, listable by root alone — never 0700, so a build moved // aside is still at a path its process may open; // - a build that has no reader any more is **retired**, not deleted: renamed under // `.witness//retired/`, which leaves every process running from it untouched, and deleted // only when no process on the machine runs from it. Which processes run from it is asked of the // kernel — every process's executable — so the one it ran as is found by its PID, and so are the // commands it started, wherever its unit is in stopping. // // The controller does its half too: it runs its verbs from its own running image rather than the path // it started from, and refuses as a handover what it cannot run (mesh-controller, issue 289). // reachable is the mode of every directory the engine keeps beside a process: enterable by the // process's own user, listable by nobody but root. const reachable = 0o711 func retiredDir(root, name string) string { return filepath.Join(Dir(root, name), "retired") } // keep makes the directories the engine keeps about a process — made 0700 before issue 289 — reachable. func keep(root, name string) error { for _, dir := range []string{filepath.Join(root, ".witness"), Dir(root, name)} { if err := os.MkdirAll(dir, reachable); err != nil { return err } if err := os.Chmod(dir, reachable); err != nil { return err } } return nil } // retire takes a build that has no reader any more out of the way: renamed under retired/, and deleted // at once if nothing runs from it, else at a later sweep. A path that is not there is nothing to do. func retire(root, name, dir string, now time.Time) error { if _, err := os.Lstat(dir); os.IsNotExist(err) { return nil } if err := keep(root, name); err != nil { return err } into := retiredDir(root, name) if err := os.MkdirAll(into, reachable); err != nil { return err } if err := os.Chmod(into, reachable); err != nil { return err } at := filepath.Join(into, fmt.Sprintf("%s-%d", filepath.Base(dir), now.UnixNano())) if err := os.Rename(dir, at); err != nil { return fmt.Errorf("cannot retire %s's build at %s: %w", name, dir, err) } _, err := Sweep(root, name) return err } // Sweep deletes every retired build of a process that no process runs from any more, and says which // are kept, with the processes still running from each. func Sweep(root, name string) ([]string, error) { into := retiredDir(root, name) entries, err := os.ReadDir(into) if os.IsNotExist(err) { return nil, nil } if err != nil { return nil, err } var draining []string for _, e := range entries { dir := filepath.Join(into, e.Name()) if pids := RunningFrom(dir); len(pids) > 0 { draining = append(draining, fmt.Sprintf("%s (still run by PID %s)", dir, joinPIDs(pids))) continue } if err := os.RemoveAll(dir); err != nil { return draining, err } } return draining, nil } // RunningFrom is every process whose executable is a file under dir — renamed there, or deleted from // there since it started. A variable so a test can name the machine's processes. var RunningFrom = func(dir string) []int { return runningFromProc("/proc", dir) } func runningFromProc(proc, dir string) []int { entries, err := os.ReadDir(proc) if err != nil { return nil } dir = filepath.Clean(dir) + string(filepath.Separator) var pids []int for _, e := range entries { pid, err := strconv.Atoi(e.Name()) if err != nil { continue } exe, err := os.Readlink(filepath.Join(proc, e.Name(), "exe")) if err != nil { // Gone between the listing and the read, a kernel thread, or another user's process this // engine may not read — it runs as root, so in practice the first two. continue } if strings.HasPrefix(strings.TrimSuffix(exe, " (deleted)"), dir) { pids = append(pids, pid) } } sort.Ints(pids) return pids } func joinPIDs(pids []int) string { words := make([]string, len(pids)) for i, p := range pids { words[i] = strconv.Itoa(p) } return strings.Join(words, ", ") } // SweepAll is Sweep for every process the engine keeps anything about: what the witness does on every // look, so a build retired while its process drained goes once the process has. func SweepAll(root string) []string { var draining []string for _, s := range all(root) { kept, err := Sweep(root, s.Process) if err != nil { draining = append(draining, fmt.Sprintf("%s: %v", s.Process, err)) } draining = append(draining, kept...) } return draining }