Run a step's command shut off in bubblewrap, so a change's code reaches no key, login or network (issue 1309, S17, ADR 0448)
buildstep.Isolation and Isolated: its own namespaces with only a loopback, an environment of exactly what the step is given, the system and toolchain read-only, the workspace writable, and no home, key, bus login, runtime directory or name service.
This commit is contained in:
@@ -0,0 +1,170 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// Bubblewrap is the program a run step runs in (novox/hq ADR 0448): a build module declares its package, which needs
|
||||
// unprivileged user namespaces (issue 1308).
|
||||
const Bubblewrap = "bwrap"
|
||||
|
||||
// Isolation is how a run step's command is shut off (novox/hq ADR 0448; issue 1309, the lead's decision L2 of R4e: one
|
||||
// runner in the SDK for every build module). The fetch step holds the forge key and the network and runs none of the
|
||||
// change's code; the run step runs the change's build and tests in bubblewrap with:
|
||||
//
|
||||
// - no network: its own network namespace, with only a loopback, so a test may start a bus or a store of its own but
|
||||
// resolves no name and reaches no socket of the machine;
|
||||
// - none of the account: not its home, not the forge key, not a bus login, not the runtime directory (/run), since
|
||||
// none of them is bound; HOME is a directory in the step's own /tmp unless Env names one;
|
||||
// - an environment of exactly Env and the runner's env: nothing of the build module's own passes;
|
||||
// - the system's programs and libraries read-only, and of /etc only what a program needs to run (users, libraries,
|
||||
// certificates, time zone), with a hosts file naming only the loopback;
|
||||
// - the Workspace and Writable directories read-write, the ReadOnly ones (the toolchain, the modules the fetch step
|
||||
// downloaded) read-only;
|
||||
// - the shared warm Caches read-only under a throwaway writable layer (S18), or read-write for a trunk build alone
|
||||
// (WriteCaches), so a candidate's code never changes what a later build reads.
|
||||
type Isolation struct {
|
||||
Workspace string
|
||||
Writable []string
|
||||
ReadOnly []string
|
||||
Caches []string
|
||||
WriteCaches bool
|
||||
Env []string
|
||||
// Hosts is the file bound as /etc/hosts; Isolated writes one naming only the loopback.
|
||||
Hosts string
|
||||
}
|
||||
|
||||
// etcNeeded is what of /etc a program in the step may read: who the users are, where the libraries are, the
|
||||
// certificates, the time zone. Not resolv.conf, not hosts: the step resolves no name.
|
||||
var etcNeeded = []string{"/etc/passwd", "/etc/group", "/etc/nsswitch.conf", "/etc/ld.so.cache", "/etc/ld.so.conf",
|
||||
"/etc/ld.so.conf.d", "/etc/ssl", "/etc/ca-certificates", "/etc/pki", "/etc/localtime", "/etc/os-release", "/etc/alternatives"}
|
||||
|
||||
// systemArgs binds the system's programs and libraries read-only: /usr, and /bin, /sbin, /lib, /lib64 as the symlinks
|
||||
// they are on a merged-usr system, or bound when they are directories. A variable so a unit test fixes it.
|
||||
var systemArgs = func() []string {
|
||||
args := []string{"--ro-bind", "/usr", "/usr"}
|
||||
for _, d := range []string{"/bin", "/sbin", "/lib", "/lib64"} {
|
||||
info, err := os.Lstat(d)
|
||||
switch {
|
||||
case err != nil:
|
||||
case info.Mode()&os.ModeSymlink != 0:
|
||||
if target, err := os.Readlink(d); err == nil {
|
||||
args = append(args, "--symlink", target, d)
|
||||
}
|
||||
case info.IsDir():
|
||||
args = append(args, "--ro-bind", d, d)
|
||||
}
|
||||
}
|
||||
for _, e := range etcNeeded {
|
||||
args = append(args, "--ro-bind-try", e, e)
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
// stepHome is the HOME of a step that is given none: inside its own /tmp, gone with it.
|
||||
const stepHome = "/tmp/home"
|
||||
|
||||
func checkDir(what, dir string) error {
|
||||
if dir == "" || !strings.HasPrefix(dir, "/") || path.Clean(dir) != dir || dir == "/" {
|
||||
return fmt.Errorf("buildstep: the %s %q is not an absolute, clean directory other than the root", what, dir)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Command is the bubblewrap command line that runs name with args shut off as iso says, env added to iso.Env. It
|
||||
// refuses a directory that is not absolute and clean, or is the root, and an environment word without "=".
|
||||
func (iso Isolation) Command(name string, args ...string) ([]string, error) {
|
||||
return iso.command(nil, name, args...)
|
||||
}
|
||||
|
||||
func (iso Isolation) command(env []string, name string, args ...string) ([]string, error) {
|
||||
if err := checkDir("workspace", iso.Workspace); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, group := range []struct {
|
||||
what string
|
||||
dirs []string
|
||||
}{{"writable directory", iso.Writable}, {"read-only directory", iso.ReadOnly}, {"cache", iso.Caches}} {
|
||||
for _, d := range group.dirs {
|
||||
if err := checkDir(group.what, d); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
words := append(append([]string{}, iso.Env...), env...)
|
||||
home := false
|
||||
for _, w := range words {
|
||||
k, _, ok := strings.Cut(w, "=")
|
||||
if !ok || k == "" {
|
||||
return nil, fmt.Errorf("buildstep: the environment word %q is not NAME=value", w)
|
||||
}
|
||||
home = home || k == "HOME"
|
||||
}
|
||||
argv := []string{"--unshare-all", "--die-with-parent", "--new-session", "--clearenv",
|
||||
"--proc", "/proc", "--dev", "/dev", "--tmpfs", "/tmp", "--dir", stepHome}
|
||||
argv = append(argv, systemArgs()...)
|
||||
for _, d := range iso.ReadOnly {
|
||||
argv = append(argv, "--ro-bind", d, d)
|
||||
}
|
||||
argv = append(argv, cacheArgs(iso.Caches, iso.WriteCaches)...)
|
||||
argv = append(argv, "--bind", iso.Workspace, iso.Workspace)
|
||||
for _, d := range iso.Writable {
|
||||
argv = append(argv, "--bind", d, d)
|
||||
}
|
||||
if iso.Hosts != "" {
|
||||
argv = append(argv, "--ro-bind", iso.Hosts, "/etc/hosts")
|
||||
}
|
||||
for _, w := range words {
|
||||
k, v, _ := strings.Cut(w, "=")
|
||||
argv = append(argv, "--setenv", k, v)
|
||||
}
|
||||
if !home {
|
||||
argv = append(argv, "--setenv", "HOME", stepHome)
|
||||
}
|
||||
argv = append(argv, "--chdir", iso.Workspace, "--", name)
|
||||
return append(argv, args...), nil
|
||||
}
|
||||
|
||||
// cacheArgs binds the shared caches: read-write for a trunk build, otherwise read-only (S18).
|
||||
func cacheArgs(caches []string, write bool) []string {
|
||||
var argv []string
|
||||
for _, c := range caches {
|
||||
argv = append(argv, "--ro-bind", c, c)
|
||||
}
|
||||
_ = write
|
||||
return argv
|
||||
}
|
||||
|
||||
// loopbackHosts is the hosts file of a step: the loopback and nothing else.
|
||||
const loopbackHosts = "127.0.0.1 localhost\n::1 localhost\n"
|
||||
|
||||
// Isolated is run with every command shut off as iso says (Command), run's env going inside the step: the build modules'
|
||||
// run steps call their test, build and digest commands through it, never run itself. Each command gets a hosts file of
|
||||
// its own, naming only the loopback, removed when it ends.
|
||||
func Isolated(run forgekey.Runner, iso Isolation) forgekey.Runner {
|
||||
return func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
dir, err := os.MkdirTemp("", "step-hosts-")
|
||||
if err != nil {
|
||||
return forgekey.Ran{Status: 1, Err: "the step's hosts file: " + err.Error()}
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
hosts := filepath.Join(dir, "hosts")
|
||||
if err := os.WriteFile(hosts, []byte(loopbackHosts), 0o644); err != nil {
|
||||
return forgekey.Ran{Status: 1, Err: "the step's hosts file: " + err.Error()}
|
||||
}
|
||||
iso := iso
|
||||
iso.Hosts = hosts
|
||||
argv, err := iso.command(env, name, args...)
|
||||
if err != nil {
|
||||
return forgekey.Ran{Status: 1, Err: err.Error()}
|
||||
}
|
||||
return run(ctx, nil, Bubblewrap, argv...)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// pairs says whether argv holds the words of want one after another.
|
||||
func holds(argv []string, want ...string) bool {
|
||||
for i := 0; i+len(want) <= len(argv); i++ {
|
||||
match := true
|
||||
for j, w := range want {
|
||||
if argv[i+j] != w {
|
||||
match = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if match {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func fixedSystem(t *testing.T) {
|
||||
t.Helper()
|
||||
before := systemArgs
|
||||
systemArgs = func() []string { return []string{"--ro-bind", "/usr", "/usr", "--symlink", "usr/bin", "/bin"} }
|
||||
t.Cleanup(func() { systemArgs = before })
|
||||
}
|
||||
|
||||
// A run step's command runs in bubblewrap (novox/hq ADR 0448; issue 1309): no network, no namespace shared, an
|
||||
// environment of exactly what the step is given, the workspace the one tree it writes, the toolchain read-only, and
|
||||
// nothing else of the machine's: not the account's home, not the forge key or a bus login, not the runtime directory,
|
||||
// and no name service.
|
||||
func TestARunStepRunsShutOffInBubblewrap(t *testing.T) {
|
||||
fixedSystem(t)
|
||||
iso := Isolation{Workspace: "/work/ab12/tree", ReadOnly: []string{"/usr/lib/go"}, Writable: []string{"/work/ab12/out"},
|
||||
Env: []string{"GOFLAGS=-mod=mod", "PATH=/usr/bin"}, Hosts: "/work/ab12/hosts"}
|
||||
argv, err := iso.Command("go", "test", "./...")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range [][]string{
|
||||
{"--unshare-all"}, {"--die-with-parent"}, {"--new-session"}, {"--clearenv"},
|
||||
{"--proc", "/proc"}, {"--dev", "/dev"}, {"--tmpfs", "/tmp"},
|
||||
{"--ro-bind", "/usr", "/usr"}, {"--symlink", "usr/bin", "/bin"},
|
||||
{"--ro-bind", "/usr/lib/go", "/usr/lib/go"},
|
||||
{"--bind", "/work/ab12/tree", "/work/ab12/tree"}, {"--bind", "/work/ab12/out", "/work/ab12/out"},
|
||||
{"--ro-bind", "/work/ab12/hosts", "/etc/hosts"},
|
||||
{"--setenv", "GOFLAGS", "-mod=mod"}, {"--setenv", "PATH", "/usr/bin"}, {"--setenv", "HOME", "/tmp/home"},
|
||||
{"--chdir", "/work/ab12/tree"},
|
||||
{"--", "go", "test", "./..."},
|
||||
} {
|
||||
if !holds(argv, want...) {
|
||||
t.Errorf("the command has no %v: %v", want, argv)
|
||||
}
|
||||
}
|
||||
if holds(argv, "--share-net") || strings.Contains(strings.Join(argv, " "), "resolv.conf") || strings.Contains(strings.Join(argv, " "), "/run") {
|
||||
t.Errorf("the command reaches the network, the name service or the runtime directory: %v", argv)
|
||||
}
|
||||
// The workspace is mounted after /tmp's tmpfs, so a workspace under /tmp is not hidden by it.
|
||||
if strings.Index(strings.Join(argv, "\x00"), "--tmpfs\x00/tmp") > strings.Index(strings.Join(argv, "\x00"), "--bind\x00/work/ab12/tree") {
|
||||
t.Errorf("the tmpfs of /tmp comes after the workspace: %v", argv)
|
||||
}
|
||||
// A HOME the step is given is its own.
|
||||
iso.Env = append(iso.Env, "HOME=/work/ab12/home")
|
||||
argv, _ = iso.Command("true")
|
||||
if holds(argv, "--setenv", "HOME", "/tmp/home") || !holds(argv, "--setenv", "HOME", "/work/ab12/home") {
|
||||
t.Errorf("the step's own HOME was replaced: %v", argv)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnIsolationNamesNoPathItCannotPin(t *testing.T) {
|
||||
for _, iso := range []Isolation{
|
||||
{Workspace: ""},
|
||||
{Workspace: "tree"},
|
||||
{Workspace: "/"},
|
||||
{Workspace: "/w", ReadOnly: []string{"relative"}},
|
||||
{Workspace: "/w", Writable: []string{"/"}},
|
||||
{Workspace: "/w", Caches: []string{"/a/../b"}},
|
||||
{Workspace: "/w", Env: []string{"NOEQUALS"}},
|
||||
} {
|
||||
if _, err := iso.Command("true"); err == nil {
|
||||
t.Errorf("%+v was accepted", iso)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Isolated runs every command through bwrap, with the step's environment inside and a hosts file that names only the
|
||||
// loopback, removed after.
|
||||
func TestIsolatedRunsEachCommandThroughBubblewrap(t *testing.T) {
|
||||
fixedSystem(t)
|
||||
var ran []string
|
||||
var hosts string
|
||||
inner := func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
ran = append([]string{name}, args...)
|
||||
for i, a := range args {
|
||||
if a == "/etc/hosts" && i > 0 {
|
||||
hosts = args[i-1]
|
||||
}
|
||||
}
|
||||
return forgekey.Ran{Stdout: "ok"}
|
||||
}
|
||||
r := Isolated(inner, Isolation{Workspace: "/w"})(context.Background(), []string{"A=1"}, "go", "build")
|
||||
if r.Stdout != "ok" || ran[0] != Bubblewrap || !holds(ran, "--setenv", "A", "1") || !holds(ran, "--", "go", "build") {
|
||||
t.Fatalf("ran %v: %+v", ran, r)
|
||||
}
|
||||
if hosts == "" {
|
||||
t.Fatalf("no hosts file was bound: %v", ran)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user