From 39a2144cd60d371aeb7391ea3e5132f8ae786549 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 17:45:21 +0200 Subject: [PATCH] Run a step's command shut off in bubblewrap, so a change's code reaches no key, login or network (issue 1309, S17, ADR 0448) buildstep.Isolation and Isolated: its own namespaces with only a loopback, an environment of exactly what the step is given, the system and toolchain read-only, the workspace writable, and no home, key, bus login, runtime directory or name service. --- go/buildstep/isolate.go | 170 +++++++++++++++++++++++++++++++++++ go/buildstep/isolate_test.go | 115 ++++++++++++++++++++++++ 2 files changed, 285 insertions(+) create mode 100644 go/buildstep/isolate.go create mode 100644 go/buildstep/isolate_test.go diff --git a/go/buildstep/isolate.go b/go/buildstep/isolate.go new file mode 100644 index 0000000..e431dfa --- /dev/null +++ b/go/buildstep/isolate.go @@ -0,0 +1,170 @@ +package buildstep + +import ( + "context" + "fmt" + "os" + "path" + "path/filepath" + "strings" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// Bubblewrap is the program a run step runs in (novox/hq ADR 0448): a build module declares its package, which needs +// unprivileged user namespaces (issue 1308). +const Bubblewrap = "bwrap" + +// Isolation is how a run step's command is shut off (novox/hq ADR 0448; issue 1309, the lead's decision L2 of R4e: one +// runner in the SDK for every build module). The fetch step holds the forge key and the network and runs none of the +// change's code; the run step runs the change's build and tests in bubblewrap with: +// +// - no network: its own network namespace, with only a loopback, so a test may start a bus or a store of its own but +// resolves no name and reaches no socket of the machine; +// - none of the account: not its home, not the forge key, not a bus login, not the runtime directory (/run), since +// none of them is bound; HOME is a directory in the step's own /tmp unless Env names one; +// - an environment of exactly Env and the runner's env: nothing of the build module's own passes; +// - the system's programs and libraries read-only, and of /etc only what a program needs to run (users, libraries, +// certificates, time zone), with a hosts file naming only the loopback; +// - the Workspace and Writable directories read-write, the ReadOnly ones (the toolchain, the modules the fetch step +// downloaded) read-only; +// - the shared warm Caches read-only under a throwaway writable layer (S18), or read-write for a trunk build alone +// (WriteCaches), so a candidate's code never changes what a later build reads. +type Isolation struct { + Workspace string + Writable []string + ReadOnly []string + Caches []string + WriteCaches bool + Env []string + // Hosts is the file bound as /etc/hosts; Isolated writes one naming only the loopback. + Hosts string +} + +// etcNeeded is what of /etc a program in the step may read: who the users are, where the libraries are, the +// certificates, the time zone. Not resolv.conf, not hosts: the step resolves no name. +var etcNeeded = []string{"/etc/passwd", "/etc/group", "/etc/nsswitch.conf", "/etc/ld.so.cache", "/etc/ld.so.conf", + "/etc/ld.so.conf.d", "/etc/ssl", "/etc/ca-certificates", "/etc/pki", "/etc/localtime", "/etc/os-release", "/etc/alternatives"} + +// systemArgs binds the system's programs and libraries read-only: /usr, and /bin, /sbin, /lib, /lib64 as the symlinks +// they are on a merged-usr system, or bound when they are directories. A variable so a unit test fixes it. +var systemArgs = func() []string { + args := []string{"--ro-bind", "/usr", "/usr"} + for _, d := range []string{"/bin", "/sbin", "/lib", "/lib64"} { + info, err := os.Lstat(d) + switch { + case err != nil: + case info.Mode()&os.ModeSymlink != 0: + if target, err := os.Readlink(d); err == nil { + args = append(args, "--symlink", target, d) + } + case info.IsDir(): + args = append(args, "--ro-bind", d, d) + } + } + for _, e := range etcNeeded { + args = append(args, "--ro-bind-try", e, e) + } + return args +} + +// stepHome is the HOME of a step that is given none: inside its own /tmp, gone with it. +const stepHome = "/tmp/home" + +func checkDir(what, dir string) error { + if dir == "" || !strings.HasPrefix(dir, "/") || path.Clean(dir) != dir || dir == "/" { + return fmt.Errorf("buildstep: the %s %q is not an absolute, clean directory other than the root", what, dir) + } + return nil +} + +// Command is the bubblewrap command line that runs name with args shut off as iso says, env added to iso.Env. It +// refuses a directory that is not absolute and clean, or is the root, and an environment word without "=". +func (iso Isolation) Command(name string, args ...string) ([]string, error) { + return iso.command(nil, name, args...) +} + +func (iso Isolation) command(env []string, name string, args ...string) ([]string, error) { + if err := checkDir("workspace", iso.Workspace); err != nil { + return nil, err + } + for _, group := range []struct { + what string + dirs []string + }{{"writable directory", iso.Writable}, {"read-only directory", iso.ReadOnly}, {"cache", iso.Caches}} { + for _, d := range group.dirs { + if err := checkDir(group.what, d); err != nil { + return nil, err + } + } + } + words := append(append([]string{}, iso.Env...), env...) + home := false + for _, w := range words { + k, _, ok := strings.Cut(w, "=") + if !ok || k == "" { + return nil, fmt.Errorf("buildstep: the environment word %q is not NAME=value", w) + } + home = home || k == "HOME" + } + argv := []string{"--unshare-all", "--die-with-parent", "--new-session", "--clearenv", + "--proc", "/proc", "--dev", "/dev", "--tmpfs", "/tmp", "--dir", stepHome} + argv = append(argv, systemArgs()...) + for _, d := range iso.ReadOnly { + argv = append(argv, "--ro-bind", d, d) + } + argv = append(argv, cacheArgs(iso.Caches, iso.WriteCaches)...) + argv = append(argv, "--bind", iso.Workspace, iso.Workspace) + for _, d := range iso.Writable { + argv = append(argv, "--bind", d, d) + } + if iso.Hosts != "" { + argv = append(argv, "--ro-bind", iso.Hosts, "/etc/hosts") + } + for _, w := range words { + k, v, _ := strings.Cut(w, "=") + argv = append(argv, "--setenv", k, v) + } + if !home { + argv = append(argv, "--setenv", "HOME", stepHome) + } + argv = append(argv, "--chdir", iso.Workspace, "--", name) + return append(argv, args...), nil +} + +// cacheArgs binds the shared caches: read-write for a trunk build, otherwise read-only (S18). +func cacheArgs(caches []string, write bool) []string { + var argv []string + for _, c := range caches { + argv = append(argv, "--ro-bind", c, c) + } + _ = write + return argv +} + +// loopbackHosts is the hosts file of a step: the loopback and nothing else. +const loopbackHosts = "127.0.0.1 localhost\n::1 localhost\n" + +// Isolated is run with every command shut off as iso says (Command), run's env going inside the step: the build modules' +// run steps call their test, build and digest commands through it, never run itself. Each command gets a hosts file of +// its own, naming only the loopback, removed when it ends. +func Isolated(run forgekey.Runner, iso Isolation) forgekey.Runner { + return func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { + dir, err := os.MkdirTemp("", "step-hosts-") + if err != nil { + return forgekey.Ran{Status: 1, Err: "the step's hosts file: " + err.Error()} + } + defer os.RemoveAll(dir) + hosts := filepath.Join(dir, "hosts") + if err := os.WriteFile(hosts, []byte(loopbackHosts), 0o644); err != nil { + return forgekey.Ran{Status: 1, Err: "the step's hosts file: " + err.Error()} + } + iso := iso + iso.Hosts = hosts + argv, err := iso.command(env, name, args...) + if err != nil { + return forgekey.Ran{Status: 1, Err: err.Error()} + } + return run(ctx, nil, Bubblewrap, argv...) + } +} diff --git a/go/buildstep/isolate_test.go b/go/buildstep/isolate_test.go new file mode 100644 index 0000000..861aafc --- /dev/null +++ b/go/buildstep/isolate_test.go @@ -0,0 +1,115 @@ +package buildstep + +import ( + "context" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// pairs says whether argv holds the words of want one after another. +func holds(argv []string, want ...string) bool { + for i := 0; i+len(want) <= len(argv); i++ { + match := true + for j, w := range want { + if argv[i+j] != w { + match = false + break + } + } + if match { + return true + } + } + return false +} + +func fixedSystem(t *testing.T) { + t.Helper() + before := systemArgs + systemArgs = func() []string { return []string{"--ro-bind", "/usr", "/usr", "--symlink", "usr/bin", "/bin"} } + t.Cleanup(func() { systemArgs = before }) +} + +// A run step's command runs in bubblewrap (novox/hq ADR 0448; issue 1309): no network, no namespace shared, an +// environment of exactly what the step is given, the workspace the one tree it writes, the toolchain read-only, and +// nothing else of the machine's: not the account's home, not the forge key or a bus login, not the runtime directory, +// and no name service. +func TestARunStepRunsShutOffInBubblewrap(t *testing.T) { + fixedSystem(t) + iso := Isolation{Workspace: "/work/ab12/tree", ReadOnly: []string{"/usr/lib/go"}, Writable: []string{"/work/ab12/out"}, + Env: []string{"GOFLAGS=-mod=mod", "PATH=/usr/bin"}, Hosts: "/work/ab12/hosts"} + argv, err := iso.Command("go", "test", "./...") + if err != nil { + t.Fatal(err) + } + for _, want := range [][]string{ + {"--unshare-all"}, {"--die-with-parent"}, {"--new-session"}, {"--clearenv"}, + {"--proc", "/proc"}, {"--dev", "/dev"}, {"--tmpfs", "/tmp"}, + {"--ro-bind", "/usr", "/usr"}, {"--symlink", "usr/bin", "/bin"}, + {"--ro-bind", "/usr/lib/go", "/usr/lib/go"}, + {"--bind", "/work/ab12/tree", "/work/ab12/tree"}, {"--bind", "/work/ab12/out", "/work/ab12/out"}, + {"--ro-bind", "/work/ab12/hosts", "/etc/hosts"}, + {"--setenv", "GOFLAGS", "-mod=mod"}, {"--setenv", "PATH", "/usr/bin"}, {"--setenv", "HOME", "/tmp/home"}, + {"--chdir", "/work/ab12/tree"}, + {"--", "go", "test", "./..."}, + } { + if !holds(argv, want...) { + t.Errorf("the command has no %v: %v", want, argv) + } + } + if holds(argv, "--share-net") || strings.Contains(strings.Join(argv, " "), "resolv.conf") || strings.Contains(strings.Join(argv, " "), "/run") { + t.Errorf("the command reaches the network, the name service or the runtime directory: %v", argv) + } + // The workspace is mounted after /tmp's tmpfs, so a workspace under /tmp is not hidden by it. + if strings.Index(strings.Join(argv, "\x00"), "--tmpfs\x00/tmp") > strings.Index(strings.Join(argv, "\x00"), "--bind\x00/work/ab12/tree") { + t.Errorf("the tmpfs of /tmp comes after the workspace: %v", argv) + } + // A HOME the step is given is its own. + iso.Env = append(iso.Env, "HOME=/work/ab12/home") + argv, _ = iso.Command("true") + if holds(argv, "--setenv", "HOME", "/tmp/home") || !holds(argv, "--setenv", "HOME", "/work/ab12/home") { + t.Errorf("the step's own HOME was replaced: %v", argv) + } +} + +func TestAnIsolationNamesNoPathItCannotPin(t *testing.T) { + for _, iso := range []Isolation{ + {Workspace: ""}, + {Workspace: "tree"}, + {Workspace: "/"}, + {Workspace: "/w", ReadOnly: []string{"relative"}}, + {Workspace: "/w", Writable: []string{"/"}}, + {Workspace: "/w", Caches: []string{"/a/../b"}}, + {Workspace: "/w", Env: []string{"NOEQUALS"}}, + } { + if _, err := iso.Command("true"); err == nil { + t.Errorf("%+v was accepted", iso) + } + } +} + +// Isolated runs every command through bwrap, with the step's environment inside and a hosts file that names only the +// loopback, removed after. +func TestIsolatedRunsEachCommandThroughBubblewrap(t *testing.T) { + fixedSystem(t) + var ran []string + var hosts string + inner := func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { + ran = append([]string{name}, args...) + for i, a := range args { + if a == "/etc/hosts" && i > 0 { + hosts = args[i-1] + } + } + return forgekey.Ran{Stdout: "ok"} + } + r := Isolated(inner, Isolation{Workspace: "/w"})(context.Background(), []string{"A=1"}, "go", "build") + if r.Stdout != "ok" || ran[0] != Bubblewrap || !holds(ran, "--setenv", "A", "1") || !holds(ran, "--", "go", "build") { + t.Fatalf("ran %v: %+v", ran, r) + } + if hosts == "" { + t.Fatalf("no hosts file was bound: %v", ran) + } +}