package buildstep import ( "context" "fmt" "io/fs" "os" "path" "path/filepath" "slices" "strconv" "strings" "sync" "git.novox.be/novox/mesh-sdk/go/forgekey" ) // Bubblewrap is the program a run step runs in (novox/hq ADR 0448): a build module declares its package, which needs // unprivileged user namespaces (issue 1308). const Bubblewrap = "bwrap" // Isolation is how a run step's command is shut off (novox/hq ADR 0448; issue 1309, the lead's decision L2 of R4e: one // runner in the SDK for every build module). The fetch step holds the forge key and the network and runs none of the // change's code; the run step runs the change's build and tests in bubblewrap with: // // - no network: its own network namespace, with only a loopback, so a test may start a bus or a store of its own but // resolves no name and reaches no socket of the machine; // - none of the account: not its home, not the forge key, not a bus login, not the runtime directory (/run), since // none of them is bound; HOME is a directory in the step's own /tmp unless Env names one; // - an environment of exactly Env and the runner's env: nothing of the build module's own passes; // - the system's programs and libraries read-only, and of /etc only what a program needs to run (users, libraries, // certificates, time zone), with a hosts file naming only the loopback; // - the Workspace and Writable directories read-write, the ReadOnly ones (the toolchain, the modules the fetch step // downloaded) read-only; // - the shared warm Caches read-only under a throwaway writable layer (S18), or read-write for a trunk build alone // (WriteCaches), so a candidate's code never changes what a later build reads; // - with a CacheLayer, what one command writes into the LayerCaches kept for the later commands of the same ask (issue // 1921, the try speed-up): the vet, the unit tests and the integration compile of one ask compile each package once. type Isolation struct { Workspace string Writable []string ReadOnly []string Caches []string WriteCaches bool // CacheLayer is a directory of the ask's own, gone with its workspace (Workspaces.CacheLayer): each command writes // the LayerCaches into a layer of its own beneath it (overlayfs refuses an upper layer two mounts share, and an // ask's commands run side by side), which Isolated moves into the ask's kept layer when the command ends; every // later command reads the kept layer above the shared cache. The shared cache stays unchanged. It must lie outside // the Workspace and every Writable directory, so no command reaches it (a command that could would make the move, // which runs outside the sandbox, write where it points). Empty: what a command writes is thrown away. CacheLayer string // LayerCaches are the Caches kept in the CacheLayer: a build cache, whose entries are whole files written by // rename. Every other cache (a module cache, with read-only directories and half-written modules) keeps a // throwaway layer. LayerCaches []string Env []string // Hosts is the file bound as /etc/hosts; Isolated writes one naming only the loopback. Hosts string // run is the directory of this command's own writable layers under CacheLayer; Isolated makes one per command. run string } // etcNeeded is what of /etc a program in the step may read: who the users are, where the libraries are, the // certificates, the time zone. Not resolv.conf, not hosts: the step resolves no name. var etcNeeded = []string{"/etc/passwd", "/etc/group", "/etc/nsswitch.conf", "/etc/ld.so.cache", "/etc/ld.so.conf", "/etc/ld.so.conf.d", "/etc/ssl", "/etc/ca-certificates", "/etc/pki", "/etc/localtime", "/etc/os-release", "/etc/alternatives"} // systemArgs binds the system's programs and libraries read-only: /usr, and /bin, /sbin, /lib, /lib64 as the symlinks // they are on a merged-usr system, or bound when they are directories. A variable so a unit test fixes it. var systemArgs = func() []string { args := []string{"--ro-bind", "/usr", "/usr"} for _, d := range []string{"/bin", "/sbin", "/lib", "/lib64"} { info, err := os.Lstat(d) switch { case err != nil: case info.Mode()&os.ModeSymlink != 0: if target, err := os.Readlink(d); err == nil { args = append(args, "--symlink", target, d) } case info.IsDir(): args = append(args, "--ro-bind", d, d) } } for _, e := range etcNeeded { args = append(args, "--ro-bind-try", e, e) } return args } // stepHome is the HOME of a step that is given none: inside its own /tmp, gone with it. const stepHome = "/tmp/home" func checkDir(what, dir string) error { if dir == "" || !strings.HasPrefix(dir, "/") || path.Clean(dir) != dir || dir == "/" { return fmt.Errorf("buildstep: the %s %q is not an absolute, clean directory other than the root", what, dir) } return nil } // Command is the bubblewrap command line that runs name with args shut off as iso says, env added to iso.Env. It // refuses a directory that is not absolute and clean, or is the root, and an environment word without "=". func (iso Isolation) Command(name string, args ...string) ([]string, error) { return iso.command(nil, name, args...) } func (iso Isolation) command(env []string, name string, args ...string) ([]string, error) { if err := checkDir("workspace", iso.Workspace); err != nil { return nil, err } for _, group := range []struct { what string dirs []string }{{"writable directory", iso.Writable}, {"read-only directory", iso.ReadOnly}, {"cache", iso.Caches}, {"cache layer", layerDirs(iso.CacheLayer)}} { for _, d := range group.dirs { if err := checkDir(group.what, d); err != nil { return nil, err } } } if err := iso.checkLayer(); err != nil { return nil, err } words := append(append([]string{}, iso.Env...), env...) home := false for _, w := range words { k, _, ok := strings.Cut(w, "=") if !ok || k == "" { return nil, fmt.Errorf("buildstep: the environment word %q is not NAME=value", w) } home = home || k == "HOME" } argv := []string{"--unshare-all", "--die-with-parent", "--new-session", "--clearenv", "--proc", "/proc", "--dev", "/dev", "--tmpfs", "/tmp", "--dir", stepHome} argv = append(argv, systemArgs()...) for _, d := range iso.ReadOnly { argv = append(argv, "--ro-bind", d, d) } argv = append(argv, iso.cacheArgs()...) argv = append(argv, "--bind", iso.Workspace, iso.Workspace) for _, d := range iso.Writable { argv = append(argv, "--bind", d, d) } if iso.Hosts != "" { argv = append(argv, "--ro-bind", iso.Hosts, "/etc/hosts") } for _, w := range words { k, v, _ := strings.Cut(w, "=") argv = append(argv, "--setenv", k, v) } if !home { argv = append(argv, "--setenv", "HOME", stepHome) } argv = append(argv, "--chdir", iso.Workspace, "--", name) return append(argv, args...), nil } // cacheArgs binds the shared warm caches (S18): for a trunk build read-write, so the next build is warm; for any other // read-only beneath an overlay whose writes go to a tmpfs of the step's own, so the step writes as a build does and // nothing it wrote outlives it (bubblewrap's --tmp-overlay, an overlay mount in the step's user namespace). A cache of // LayerCaches, with a CacheLayer, has the ask's kept layer above the shared cache and the command's writes go to its own // layer on disk (--overlay), which Isolated keeps for the ask when the command ends. func (iso Isolation) cacheArgs() []string { var argv []string for i, c := range iso.Caches { switch { case iso.WriteCaches: argv = append(argv, "--bind", c, c) case iso.layered(c): upper, work := iso.runLayer(i) argv = append(argv, "--overlay-src", c, "--overlay-src", keptLayer(iso.CacheLayer, i), "--overlay", upper, work, c) default: argv = append(argv, "--overlay-src", c, "--tmp-overlay", c) } } return argv } // layered says whether the cache c is kept in the ask's cache layer. func (iso Isolation) layered(c string) bool { return iso.CacheLayer != "" && !iso.WriteCaches && slices.Contains(iso.LayerCaches, c) } // checkLayer refuses a cache layer a command could reach (in or above the workspace or a writable directory, or a // cache), and a layered cache that is not one of the caches. func (iso Isolation) checkLayer() error { if iso.CacheLayer == "" { return nil } for _, d := range append(append([]string{iso.Workspace}, iso.Writable...), iso.Caches...) { if within(iso.CacheLayer, d) || within(d, iso.CacheLayer) { return fmt.Errorf("buildstep: the cache layer %s is within reach of the step's %s", iso.CacheLayer, d) } } for _, c := range iso.LayerCaches { if !slices.Contains(iso.Caches, c) { return fmt.Errorf("buildstep: the layered cache %s is not one of the caches", c) } } return nil } // within says whether p is dir or lies beneath it. func within(p, dir string) bool { return p == dir || strings.HasPrefix(p, dir+"/") } func layerDirs(layer string) []string { if layer == "" { return nil } return []string{layer} } // keptRel is where, in the cache layer, the ask keeps what its commands wrote into the cache i; keptLayer is its path. func keptRel(i int) string { return filepath.Join("kept", strconv.Itoa(i)) } func keptLayer(layer string, i int) string { return filepath.Join(layer, keptRel(i)) } // runLayer is the upper and work directory of this command's writes into the cache i. Command, which makes no // directory, names the layer "run"; Isolated gives each command one of its own. func (iso Isolation) runLayer(i int) (upper, work string) { run := iso.run if run == "" { run = filepath.Join(iso.CacheLayer, "run") } return filepath.Join(run, strconv.Itoa(i), "upper"), filepath.Join(run, strconv.Itoa(i), "work") } // layerLocks hold one mutex per cache layer, so two commands ending at once keep their writes one after the other. var layerLocks sync.Map // prepareLayer makes the kept layers and a run directory of the command's own under the cache layer. func (iso *Isolation) prepareLayer() error { if iso.CacheLayer == "" || iso.WriteCaches || len(iso.LayerCaches) == 0 { return nil } if err := iso.checkLayer(); err != nil { return err } if err := os.MkdirAll(filepath.Join(iso.CacheLayer, "runs"), 0o700); err != nil { return err } run, err := os.MkdirTemp(filepath.Join(iso.CacheLayer, "runs"), "run-") if err != nil { return err } iso.run = run for i, c := range iso.Caches { if !iso.layered(c) { continue } upper, work := iso.runLayer(i) for _, d := range []string{keptLayer(iso.CacheLayer, i), upper, work} { if err := os.MkdirAll(d, 0o700); err != nil { return err } } } return nil } // keepLayer moves what the command wrote into the layered caches into the ask's kept layers, and removes its run // directory. A file the command wrote replaces the kept one (a cache's newest write wins); a whiteout (a file the // command deleted), a symlink or any other entry that is not a directory or a regular file is not kept. Every move goes // through an os.Root on the cache layer, so no symlink leads it outside the layer. A write that cannot be kept only // means a later command builds it again. func (iso Isolation) keepLayer() { if iso.run == "" { return } defer os.RemoveAll(iso.run) mu, _ := layerLocks.LoadOrStore(iso.CacheLayer, &sync.Mutex{}) mu.(*sync.Mutex).Lock() defer mu.(*sync.Mutex).Unlock() root, err := os.OpenRoot(iso.CacheLayer) if err != nil { return } defer root.Close() runRel, err := filepath.Rel(iso.CacheLayer, iso.run) if err != nil { return } for i, c := range iso.Caches { if !iso.layered(c) { continue } upper := filepath.Join(runRel, strconv.Itoa(i), "upper") fs.WalkDir(root.FS(), upper, func(p string, d fs.DirEntry, err error) error { if err != nil || p == upper { return nil } rel, _ := filepath.Rel(upper, p) to := filepath.Join(keptRel(i), rel) switch { case d.IsDir(): if fi, err := root.Lstat(to); err == nil && !fi.IsDir() { root.Remove(to) } root.MkdirAll(to, 0o700) case d.Type().IsRegular(): if fi, err := root.Lstat(to); err == nil && !fi.Mode().IsRegular() { return nil // never replace what is not a file the layer kept } root.Rename(p, to) } return nil }) } } // loopbackHosts is the hosts file of a step: the loopback and nothing else. const loopbackHosts = "127.0.0.1 localhost\n::1 localhost\n" // Isolated is run with every command shut off as iso says (Command), run's env going inside the step: the build modules' // run steps call their test, build and digest commands through it, never run itself. Each command gets a hosts file of // its own, naming only the loopback, removed when it ends. func Isolated(run forgekey.Runner, iso Isolation) forgekey.Runner { return func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { dir, err := os.MkdirTemp("", "step-hosts-") if err != nil { return forgekey.Ran{Status: 1, Err: "the step's hosts file: " + err.Error()} } defer os.RemoveAll(dir) hosts := filepath.Join(dir, "hosts") if err := os.WriteFile(hosts, []byte(loopbackHosts), 0o644); err != nil { return forgekey.Ran{Status: 1, Err: "the step's hosts file: " + err.Error()} } iso := iso iso.Hosts = hosts if err := iso.prepareLayer(); err != nil { return forgekey.Ran{Status: 1, Err: "the step's cache layer: " + err.Error()} } defer iso.keepLayer() argv, err := iso.command(env, name, args...) if err != nil { return forgekey.Ran{Status: 1, Err: err.Error()} } return run(ctx, nil, Bubblewrap, argv...) } }