novox/mesh-sdk
18
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

buildstep: keep what one command of an ask writes into the caches for its later commands

Browse Source 
Every command of a check ran over the shared Go build cache beneath a throwaway
tmpfs layer of its own, so the digests, the vet, the unit tests and the integration
compile of one ask each compiled every package again. With Isolation.CacheLayer a
command writes into a layer on disk of its own and Isolated moves it into the ask's
kept layer when it ends; later commands read the kept layer above the shared cache,
which stays unchanged (ADR 0541 §2, ADR 0448 §3). Empty CacheLayer keeps today's
throwaway layer. Ported from feat/try-speed S2 (issue 1921).
This commit is contained in:
jschoubben
2026-10-29 00:51:26 +01:00
parent d4c61ee8ff
commit 2a86701891
4 changed files with 222 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go/buildstep/isolate.go
+114 -10
View File
@@ -3,10 +3,13 @@ package buildstep
import (
"context"
"fmt"
"io/fs"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"sync"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
@@ -29,16 +32,25 @@ const Bubblewrap = "bwrap"
// - the Workspace and Writable directories read-write, the ReadOnly ones (the toolchain, the modules the fetch step
// downloaded) read-only;
// - the shared warm Caches read-only under a throwaway writable layer (S18), or read-write for a trunk build alone
// (WriteCaches), so a candidate's code never changes what a later build reads.
// (WriteCaches), so a candidate's code never changes what a later build reads;
// - with a CacheLayer, what one command writes into the caches kept for the later commands of the same ask (issue 1921,
// the try speed-up): the vet, the unit tests and the integration compile of one ask compile each package once.
type Isolation struct {
Workspace string
Writable []string
ReadOnly []string
Caches []string
WriteCaches bool
Env []string
// CacheLayer is a directory of the ask's own, in its workspace and gone with it: each command writes the caches into
// a layer of its own beneath it (overlayfs refuses an upper layer two mounts share, and an ask's commands run side by
// side), which Isolated moves into the ask's kept layer when the command ends; every later command reads the kept
// layer above the shared cache. The shared cache stays unchanged. Empty: what a command writes is thrown away.
CacheLayer string
Env []string
// Hosts is the file bound as /etc/hosts; Isolated writes one naming only the loopback.
Hosts string
// run is the directory of this command's own writable layers under CacheLayer; Isolated makes one per command.
run string
}
// etcNeeded is what of /etc a program in the step may read: who the users are, where the libraries are, the
@@ -91,7 +103,7 @@ func (iso Isolation) command(env []string, name string, args ...string) ([]strin
for _, group := range []struct {
what string
dirs []string
}{{"writable directory", iso.Writable}, {"read-only directory", iso.ReadOnly}, {"cache", iso.Caches}} {
}{{"writable directory", iso.Writable}, {"read-only directory", iso.ReadOnly}, {"cache", iso.Caches}, {"cache layer", layerDirs(iso.CacheLayer)}} {
for _, d := range group.dirs {
if err := checkDir(group.what, d); err != nil {
return nil, err
@@ -113,7 +125,7 @@ func (iso Isolation) command(env []string, name string, args ...string) ([]strin
for _, d := range iso.ReadOnly {
argv = append(argv, "--ro-bind", d, d)
}
argv = append(argv, cacheArgs(iso.Caches, iso.WriteCaches)...)
argv = append(argv, iso.cacheArgs()...)
argv = append(argv, "--bind", iso.Workspace, iso.Workspace)
for _, d := range iso.Writable {
argv = append(argv, "--bind", d, d)
@@ -134,19 +146,107 @@ func (iso Isolation) command(env []string, name string, args ...string) ([]strin
// cacheArgs binds the shared warm caches (S18): for a trunk build read-write, so the next build is warm; for any other
// read-only beneath an overlay whose writes go to a tmpfs of the step's own, so the step writes as a build does and
// nothing it wrote outlives it (bubblewrap's --tmp-overlay, an overlay mount in the step's user namespace).
func cacheArgs(caches []string, write bool) []string {
// nothing it wrote outlives it (bubblewrap's --tmp-overlay, an overlay mount in the step's user namespace). With a
// CacheLayer the ask's kept layer lies above the shared cache and the command's writes go to its own layer on disk
// (--overlay), which Isolated keeps for the ask when the command ends.
func (iso Isolation) cacheArgs() []string {
var argv []string
for _, c := range caches {
if write {
for i, c := range iso.Caches {
switch {
case iso.WriteCaches:
argv = append(argv, "--bind", c, c)
continue
case iso.CacheLayer != "":
upper, work := iso.runLayer(i)
argv = append(argv, "--overlay-src", c, "--overlay-src", keptLayer(iso.CacheLayer, i), "--overlay", upper, work, c)
default:
argv = append(argv, "--overlay-src", c, "--tmp-overlay", c)
}
argv = append(argv, "--overlay-src", c, "--tmp-overlay", c)
}
return argv
}
func layerDirs(layer string) []string {
if layer == "" {
return nil
}
return []string{layer}
}
// keptLayer is where the ask keeps what its commands wrote into the cache i.
func keptLayer(layer string, i int) string { return filepath.Join(layer, "kept", strconv.Itoa(i)) }
// runLayer is the upper and work directory of this command's writes into the cache i. Command, which makes no
// directory, names the layer "run"; Isolated gives each command one of its own.
func (iso Isolation) runLayer(i int) (upper, work string) {
run := iso.run
if run == "" {
run = filepath.Join(iso.CacheLayer, "run")
}
return filepath.Join(run, strconv.Itoa(i), "upper"), filepath.Join(run, strconv.Itoa(i), "work")
}
// layerLocks hold one mutex per cache layer, so two commands ending at once keep their writes one after the other.
var layerLocks sync.Map
// prepareLayer makes the kept layers and a run directory of the command's own under the cache layer.
func (iso *Isolation) prepareLayer() error {
if iso.CacheLayer == "" || iso.WriteCaches || len(iso.Caches) == 0 {
return nil
}
if err := os.MkdirAll(filepath.Join(iso.CacheLayer, "runs"), 0o700); err != nil {
return err
}
run, err := os.MkdirTemp(filepath.Join(iso.CacheLayer, "runs"), "run-")
if err != nil {
return err
}
iso.run = run
for i := range iso.Caches {
upper, work := iso.runLayer(i)
for _, d := range []string{keptLayer(iso.CacheLayer, i), upper, work} {
if err := os.MkdirAll(d, 0o700); err != nil {
return err
}
}
}
return nil
}
// keepLayer moves what the command wrote into the caches into the ask's kept layers, and removes its run directory.
// A file the command wrote replaces the kept one (a cache's newest write wins); a whiteout (a file the command
// deleted) is not kept, since a cache entry deleted by one command is no fault of a later one. A write that cannot be
// kept only means a later command builds it again.
func (iso Isolation) keepLayer() {
if iso.run == "" {
return
}
defer os.RemoveAll(iso.run)
mu, _ := layerLocks.LoadOrStore(iso.CacheLayer, &sync.Mutex{})
mu.(*sync.Mutex).Lock()
defer mu.(*sync.Mutex).Unlock()
for i := range iso.Caches {
upper, _ := iso.runLayer(i)
kept := keptLayer(iso.CacheLayer, i)
filepath.WalkDir(upper, func(p string, d fs.DirEntry, err error) error {
if err != nil || p == upper {
return nil
}
rel, _ := filepath.Rel(upper, p)
to := filepath.Join(kept, rel)
switch {
case d.IsDir():
if fi, err := os.Lstat(to); err == nil && !fi.IsDir() {
os.Remove(to)
}
os.MkdirAll(to, 0o700)
case d.Type().IsRegular():
os.Rename(p, to)
}
return nil
})
}
}
// loopbackHosts is the hosts file of a step: the loopback and nothing else.
const loopbackHosts = "127.0.0.1 localhost\n::1 localhost\n"
@@ -166,6 +266,10 @@ func Isolated(run forgekey.Runner, iso Isolation) forgekey.Runner {
}
iso := iso
iso.Hosts = hosts
if err := iso.prepareLayer(); err != nil {
return forgekey.Ran{Status: 1, Err: "the step's cache layer: " + err.Error()}
}
defer iso.keepLayer()
argv, err := iso.command(env, name, args...)
if err != nil {
return forgekey.Ran{Status: 1, Err: err.Error()}