mesh-sdk/go/buildstep/isolate.go

package buildstep

import (
	"context"
	"fmt"
	"io/fs"
	"os"
	"path"
	"path/filepath"
	"slices"
	"strconv"
	"strings"
	"sync"

	"git.novox.be/novox/mesh-sdk/go/forgekey"
)

// Bubblewrap is the program a run step runs in (novox/hq ADR 0448): a build module declares its package, which needs
// unprivileged user namespaces (issue 1308).
const Bubblewrap = "bwrap"

// Isolation is how a run step's command is shut off (novox/hq ADR 0448; issue 1309, the lead's decision L2 of R4e: one
// runner in the SDK for every build module). The fetch step holds the forge key and the network and runs none of the
// change's code; the run step runs the change's build and tests in bubblewrap with:
//
//   - no network: its own network namespace, with only a loopback, so a test may start a bus or a store of its own but
//     resolves no name and reaches no socket of the machine;
//   - none of the account: not its home, not the forge key, not a bus login, not the runtime directory (/run), since
//     none of them is bound; HOME is a directory in the step's own /tmp unless Env names one;
//   - an environment of exactly Env and the runner's env: nothing of the build module's own passes;
//   - the system's programs and libraries read-only, and of /etc only what a program needs to run (users, libraries,
//     certificates, time zone), with a hosts file naming only the loopback;
//   - the Workspace and Writable directories read-write, the ReadOnly ones (the toolchain, the modules the fetch step
//     downloaded) read-only;
//   - the shared warm Caches read-only under a throwaway writable layer (S18), or read-write for a trunk build alone
//     (WriteCaches), so a candidate's code never changes what a later build reads;
//   - with a CacheLayer, what one command writes into the LayerCaches kept for the later commands of the same ask (issue
//     1921, the try speed-up): the vet, the unit tests and the integration compile of one ask compile each package once.
type Isolation struct {
	Workspace   string
	Writable    []string
	ReadOnly    []string
	Caches      []string
	WriteCaches bool
	// CacheLayer is a directory of the ask's own, gone with its workspace (Workspaces.CacheLayer): each command writes
	// the LayerCaches into a layer of its own beneath it (overlayfs refuses an upper layer two mounts share, and an
	// ask's commands run side by side), which Isolated moves into the ask's kept layer when the command ends; every
	// later command reads the kept layer above the shared cache. The shared cache stays unchanged. It must lie outside
	// the Workspace and every Writable directory, so no command reaches it (a command that could would make the move,
	// which runs outside the sandbox, write where it points). Empty: what a command writes is thrown away.
	CacheLayer string
	// LayerCaches are the Caches kept in the CacheLayer: a build cache, whose entries are whole files written by
	// rename. Every other cache (a module cache, with read-only directories and half-written modules) keeps a
	// throwaway layer.
	LayerCaches []string
	Env         []string
	// Hosts is the file bound as /etc/hosts; Isolated writes one naming only the loopback.
	Hosts string
	// run is the directory of this command's own writable layers under CacheLayer; Isolated makes one per command.
	run string
}

// etcNeeded is what of /etc a program in the step may read: who the users are, where the libraries are, the
// certificates, the time zone. Not resolv.conf, not hosts: the step resolves no name.
var etcNeeded = []string{"/etc/passwd", "/etc/group", "/etc/nsswitch.conf", "/etc/ld.so.cache", "/etc/ld.so.conf",
	"/etc/ld.so.conf.d", "/etc/ssl", "/etc/ca-certificates", "/etc/pki", "/etc/localtime", "/etc/os-release", "/etc/alternatives"}

// systemArgs binds the system's programs and libraries read-only: /usr, and /bin, /sbin, /lib, /lib64 as the symlinks
// they are on a merged-usr system, or bound when they are directories. A variable so a unit test fixes it.
var systemArgs = func() []string {
	args := []string{"--ro-bind", "/usr", "/usr"}
	for _, d := range []string{"/bin", "/sbin", "/lib", "/lib64"} {
		info, err := os.Lstat(d)
		switch {
		case err != nil:
		case info.Mode()&os.ModeSymlink != 0:
			if target, err := os.Readlink(d); err == nil {
				args = append(args, "--symlink", target, d)
			}
		case info.IsDir():
			args = append(args, "--ro-bind", d, d)
		}
	}
	for _, e := range etcNeeded {
		args = append(args, "--ro-bind-try", e, e)
	}
	return args
}

// stepHome is the HOME of a step that is given none: inside its own /tmp, gone with it.
const stepHome = "/tmp/home"

func checkDir(what, dir string) error {
	if dir == "" || !strings.HasPrefix(dir, "/") || path.Clean(dir) != dir || dir == "/" {
		return fmt.Errorf("buildstep: the %s %q is not an absolute, clean directory other than the root", what, dir)
	}
	return nil
}

// Command is the bubblewrap command line that runs name with args shut off as iso says, env added to iso.Env. It
// refuses a directory that is not absolute and clean, or is the root, and an environment word without "=".
func (iso Isolation) Command(name string, args ...string) ([]string, error) {
	return iso.command(nil, name, args...)
}

func (iso Isolation) command(env []string, name string, args ...string) ([]string, error) {
	if err := checkDir("workspace", iso.Workspace); err != nil {
		return nil, err
	}
	for _, group := range []struct {
		what string
		dirs []string
	}{{"writable directory", iso.Writable}, {"read-only directory", iso.ReadOnly}, {"cache", iso.Caches}, {"cache layer", layerDirs(iso.CacheLayer)}} {
		for _, d := range group.dirs {
			if err := checkDir(group.what, d); err != nil {
				return nil, err
			}
		}
	}
	if err := iso.checkLayer(); err != nil {
		return nil, err
	}
	words := append(append([]string{}, iso.Env...), env...)
	home := false
	for _, w := range words {
		k, _, ok := strings.Cut(w, "=")
		if !ok || k == "" {
			return nil, fmt.Errorf("buildstep: the environment word %q is not NAME=value", w)
		}
		home = home || k == "HOME"
	}
	argv := []string{"--unshare-all", "--die-with-parent", "--new-session", "--clearenv",
		"--proc", "/proc", "--dev", "/dev", "--tmpfs", "/tmp", "--dir", stepHome}
	argv = append(argv, systemArgs()...)
	for _, d := range iso.ReadOnly {
		argv = append(argv, "--ro-bind", d, d)
	}
	argv = append(argv, iso.cacheArgs()...)
	argv = append(argv, "--bind", iso.Workspace, iso.Workspace)
	for _, d := range iso.Writable {
		argv = append(argv, "--bind", d, d)
	}
	if iso.Hosts != "" {
		argv = append(argv, "--ro-bind", iso.Hosts, "/etc/hosts")
	}
	for _, w := range words {
		k, v, _ := strings.Cut(w, "=")
		argv = append(argv, "--setenv", k, v)
	}
	if !home {
		argv = append(argv, "--setenv", "HOME", stepHome)
	}
	argv = append(argv, "--chdir", iso.Workspace, "--", name)
	return append(argv, args...), nil
}

// cacheArgs binds the shared warm caches (S18): for a trunk build read-write, so the next build is warm; for any other
// read-only beneath an overlay whose writes go to a tmpfs of the step's own, so the step writes as a build does and
// nothing it wrote outlives it (bubblewrap's --tmp-overlay, an overlay mount in the step's user namespace). A cache of
// LayerCaches, with a CacheLayer, has the ask's kept layer above the shared cache and the command's writes go to its own
// layer on disk (--overlay), which Isolated keeps for the ask when the command ends.
func (iso Isolation) cacheArgs() []string {
	var argv []string
	for i, c := range iso.Caches {
		switch {
		case iso.WriteCaches:
			argv = append(argv, "--bind", c, c)
		case iso.layered(c):
			upper, work := iso.runLayer(i)
			argv = append(argv, "--overlay-src", c, "--overlay-src", keptLayer(iso.CacheLayer, i), "--overlay", upper, work, c)
		default:
			argv = append(argv, "--overlay-src", c, "--tmp-overlay", c)
		}
	}
	return argv
}

// layered says whether the cache c is kept in the ask's cache layer.
func (iso Isolation) layered(c string) bool {
	return iso.CacheLayer != "" && !iso.WriteCaches && slices.Contains(iso.LayerCaches, c)
}

// checkLayer refuses a cache layer a command could reach (in or above the workspace or a writable directory, or a
// cache), and a layered cache that is not one of the caches.
func (iso Isolation) checkLayer() error {
	if iso.CacheLayer == "" {
		return nil
	}
	for _, d := range append(append([]string{iso.Workspace}, iso.Writable...), iso.Caches...) {
		if within(iso.CacheLayer, d) || within(d, iso.CacheLayer) {
			return fmt.Errorf("buildstep: the cache layer %s is within reach of the step's %s", iso.CacheLayer, d)
		}
	}
	for _, c := range iso.LayerCaches {
		if !slices.Contains(iso.Caches, c) {
			return fmt.Errorf("buildstep: the layered cache %s is not one of the caches", c)
		}
	}
	return nil
}

// within says whether p is dir or lies beneath it.
func within(p, dir string) bool { return p == dir || strings.HasPrefix(p, dir+"/") }

func layerDirs(layer string) []string {
	if layer == "" {
		return nil
	}
	return []string{layer}
}

// keptRel is where, in the cache layer, the ask keeps what its commands wrote into the cache i; keptLayer is its path.
func keptRel(i int) string                 { return filepath.Join("kept", strconv.Itoa(i)) }
func keptLayer(layer string, i int) string { return filepath.Join(layer, keptRel(i)) }

// runLayer is the upper and work directory of this command's writes into the cache i. Command, which makes no
// directory, names the layer "run"; Isolated gives each command one of its own.
func (iso Isolation) runLayer(i int) (upper, work string) {
	run := iso.run
	if run == "" {
		run = filepath.Join(iso.CacheLayer, "run")
	}
	return filepath.Join(run, strconv.Itoa(i), "upper"), filepath.Join(run, strconv.Itoa(i), "work")
}

// layerLocks hold one mutex per cache layer, so two commands ending at once keep their writes one after the other.
var layerLocks sync.Map

// prepareLayer makes the kept layers and a run directory of the command's own under the cache layer.
func (iso *Isolation) prepareLayer() error {
	if iso.CacheLayer == "" || iso.WriteCaches || len(iso.LayerCaches) == 0 {
		return nil
	}
	if err := iso.checkLayer(); err != nil {
		return err
	}
	if err := os.MkdirAll(filepath.Join(iso.CacheLayer, "runs"), 0o700); err != nil {
		return err
	}
	run, err := os.MkdirTemp(filepath.Join(iso.CacheLayer, "runs"), "run-")
	if err != nil {
		return err
	}
	iso.run = run
	for i, c := range iso.Caches {
		if !iso.layered(c) {
			continue
		}
		upper, work := iso.runLayer(i)
		for _, d := range []string{keptLayer(iso.CacheLayer, i), upper, work} {
			if err := os.MkdirAll(d, 0o700); err != nil {
				return err
			}
		}
	}
	return nil
}

// keepLayer moves what the command wrote into the layered caches into the ask's kept layers, and removes its run
// directory. A file the command wrote replaces the kept one (a cache's newest write wins); a whiteout (a file the
// command deleted), a symlink or any other entry that is not a directory or a regular file is not kept. Every move goes
// through an os.Root on the cache layer, so no symlink leads it outside the layer. A write that cannot be kept only
// means a later command builds it again.
func (iso Isolation) keepLayer() {
	if iso.run == "" {
		return
	}
	defer os.RemoveAll(iso.run)
	mu, _ := layerLocks.LoadOrStore(iso.CacheLayer, &sync.Mutex{})
	mu.(*sync.Mutex).Lock()
	defer mu.(*sync.Mutex).Unlock()
	root, err := os.OpenRoot(iso.CacheLayer)
	if err != nil {
		return
	}
	defer root.Close()
	runRel, err := filepath.Rel(iso.CacheLayer, iso.run)
	if err != nil {
		return
	}
	for i, c := range iso.Caches {
		if !iso.layered(c) {
			continue
		}
		upper := filepath.Join(runRel, strconv.Itoa(i), "upper")
		fs.WalkDir(root.FS(), upper, func(p string, d fs.DirEntry, err error) error {
			if err != nil || p == upper {
				return nil
			}
			rel, _ := filepath.Rel(upper, p)
			to := filepath.Join(keptRel(i), rel)
			switch {
			case d.IsDir():
				if fi, err := root.Lstat(to); err == nil && !fi.IsDir() {
					root.Remove(to)
				}
				root.MkdirAll(to, 0o700)
			case d.Type().IsRegular():
				if fi, err := root.Lstat(to); err == nil && !fi.Mode().IsRegular() {
					return nil // never replace what is not a file the layer kept
				}
				root.Rename(p, to)
			}
			return nil
		})
	}
}

// loopbackHosts is the hosts file of a step: the loopback and nothing else.
const loopbackHosts = "127.0.0.1 localhost\n::1 localhost\n"

// Isolated is run with every command shut off as iso says (Command), run's env going inside the step: the build modules'
// run steps call their test, build and digest commands through it, never run itself. Each command gets a hosts file of
// its own, naming only the loopback, removed when it ends.
func Isolated(run forgekey.Runner, iso Isolation) forgekey.Runner {
	return func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
		dir, err := os.MkdirTemp("", "step-hosts-")
		if err != nil {
			return forgekey.Ran{Status: 1, Err: "the step's hosts file: " + err.Error()}
		}
		defer os.RemoveAll(dir)
		hosts := filepath.Join(dir, "hosts")
		if err := os.WriteFile(hosts, []byte(loopbackHosts), 0o644); err != nil {
			return forgekey.Ran{Status: 1, Err: "the step's hosts file: " + err.Error()}
		}
		iso := iso
		iso.Hosts = hosts
		if err := iso.prepareLayer(); err != nil {
			return forgekey.Ran{Status: 1, Err: "the step's cache layer: " + err.Error()}
		}
		defer iso.keepLayer()
		argv, err := iso.command(env, name, args...)
		if err != nil {
			return forgekey.Ran{Status: 1, Err: err.Error()}
		}
		return run(ctx, nil, Bubblewrap, argv...)
	}
}