Read an xz release in Go, so the build seat needs no xz tool (hq issue 345)
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery superseded: a newer head of the same pull request

Betterbird's pinned .tar.xz failed on the build seat with "this build seat has
no xz to read it": the seat's image is the catalogue's build-agent recipe,
which installs git and docker-cli alone, and xz had been added only to
cmd/mesh-builder/Dockerfile, which the seat does not build from. The stream
is now read with github.com/ulikunitz/xz (vendored), its dictionary capped
at xz's own largest preset, and what it unpacks to bounded as before; the
xz line in the unused Dockerfile is taken back out.
This commit is contained in:
jochen
2026-10-12 15:10:52 +02:00
parent 2843834813
commit 10f76cafc4
55 changed files with 7707 additions and 44 deletions
+14 -41
View File
@@ -14,13 +14,13 @@ import (
"io"
"net/http"
"os"
"os/exec"
"path"
"path/filepath"
"strings"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/ulikunitz/xz"
)
// An archive whose source is somebody else's published file, pinned by URL and sha256 (novox/hq ADR 0316
@@ -183,7 +183,6 @@ func unpackUpstream(ctx context.Context, file, into, within string) error {
buffered := bufio.NewReader(f)
magic, _ := buffered.Peek(6)
var stream io.Reader
var wait func() error
switch {
case bytes.HasPrefix(magic, []byte{0x1f, 0x8b}):
zipped, err := gzip.NewReader(buffered)
@@ -195,53 +194,27 @@ func unpackUpstream(ctx context.Context, file, into, within string) error {
case bytes.HasPrefix(magic, []byte("BZh")):
stream = bzip2.NewReader(buffered)
case bytes.HasPrefix(magic, []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}):
// Go's standard library reads no xz; the build seat's image carries xz (cmd/mesh-builder/Dockerfile).
if _, err := exec.LookPath("xz"); err != nil {
return errors.New("it is compressed with xz, and this build seat has no xz to read it")
}
cmd := exec.CommandContext(ctx, "xz", "-dc")
cmd.Stdin = buffered
var stderr bytes.Buffer
cmd.Stderr = &stderr
out, err := cmd.StdoutPipe()
// Read in Go (github.com/ulikunitz/xz), never by a tool the build seat's image may not carry: the seat's
// image is the catalogue's build-agent recipe, which installs git and docker-cli alone, and a release
// pinned as .tar.xz failed there on 2026-10-12 (hq issue 345). The dictionary is bounded, and what it
// unpacks to is bounded by untar (maxUpstream, maxEntries).
config := xz.ReaderConfig{DictCap: xzDictCap}
zipped, err := config.NewReader(buffered)
if err != nil {
return err
return fmt.Errorf("it is not an xz stream this build seat reads: %v", err)
}
if err := cmd.Start(); err != nil {
return err
}
stream = out
wait = func() error {
if err := cmd.Wait(); err != nil {
return fmt.Errorf("xz could not read it: %v %s", err, strings.TrimSpace(stderr.String()))
}
return nil
}
defer func() {
if wait != nil {
_ = cmd.Process.Kill()
_ = cmd.Wait()
}
}()
stream = zipped
default:
return errors.New("it is not a tar archive compressed with gzip, bzip2 or xz; a single file is kept " +
"whole by naming it in `file`")
}
if err := untar(stream, into, within); err != nil {
return err
}
if wait != nil {
// Whatever follows the tar's end is read before xz is waited for: xz blocks on a full pipe otherwise.
if _, err := io.Copy(io.Discard, stream); err != nil {
return err
}
w := wait
wait = nil
return w()
}
return nil
return untar(stream, into, within)
}
// xzDictCap is the largest xz dictionary read: xz's own presets use at most 64 MiB (-9); a stream asking for
// more is refused rather than given the memory.
const xzDictCap = 64 << 20
// untar writes a tar stream's files and directories within `within` into a directory.
func untar(stream io.Reader, into, within string) error {
reader := tar.NewReader(stream)