Read an xz release in Go, so the build seat needs no xz tool (hq issue 345)
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery superseded: a newer head of the same pull request

Betterbird's pinned .tar.xz failed on the build seat with "this build seat has
no xz to read it": the seat's image is the catalogue's build-agent recipe,
which installs git and docker-cli alone, and xz had been added only to
cmd/mesh-builder/Dockerfile, which the seat does not build from. The stream
is now read with github.com/ulikunitz/xz (vendored), its dictionary capped
at xz's own largest preset, and what it unpacks to bounded as before; the
xz line in the unused Dockerfile is taken back out.
This commit is contained in:
jochen
2026-10-12 15:10:52 +02:00
parent 2843834813
commit 10f76cafc4
55 changed files with 7707 additions and 44 deletions
Binary file not shown.
+14 -41
View File
@@ -14,13 +14,13 @@ import (
"io"
"net/http"
"os"
"os/exec"
"path"
"path/filepath"
"strings"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/ulikunitz/xz"
)
// An archive whose source is somebody else's published file, pinned by URL and sha256 (novox/hq ADR 0316
@@ -183,7 +183,6 @@ func unpackUpstream(ctx context.Context, file, into, within string) error {
buffered := bufio.NewReader(f)
magic, _ := buffered.Peek(6)
var stream io.Reader
var wait func() error
switch {
case bytes.HasPrefix(magic, []byte{0x1f, 0x8b}):
zipped, err := gzip.NewReader(buffered)
@@ -195,53 +194,27 @@ func unpackUpstream(ctx context.Context, file, into, within string) error {
case bytes.HasPrefix(magic, []byte("BZh")):
stream = bzip2.NewReader(buffered)
case bytes.HasPrefix(magic, []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}):
// Go's standard library reads no xz; the build seat's image carries xz (cmd/mesh-builder/Dockerfile).
if _, err := exec.LookPath("xz"); err != nil {
return errors.New("it is compressed with xz, and this build seat has no xz to read it")
}
cmd := exec.CommandContext(ctx, "xz", "-dc")
cmd.Stdin = buffered
var stderr bytes.Buffer
cmd.Stderr = &stderr
out, err := cmd.StdoutPipe()
// Read in Go (github.com/ulikunitz/xz), never by a tool the build seat's image may not carry: the seat's
// image is the catalogue's build-agent recipe, which installs git and docker-cli alone, and a release
// pinned as .tar.xz failed there on 2026-10-12 (hq issue 345). The dictionary is bounded, and what it
// unpacks to is bounded by untar (maxUpstream, maxEntries).
config := xz.ReaderConfig{DictCap: xzDictCap}
zipped, err := config.NewReader(buffered)
if err != nil {
return err
return fmt.Errorf("it is not an xz stream this build seat reads: %v", err)
}
if err := cmd.Start(); err != nil {
return err
}
stream = out
wait = func() error {
if err := cmd.Wait(); err != nil {
return fmt.Errorf("xz could not read it: %v %s", err, strings.TrimSpace(stderr.String()))
}
return nil
}
defer func() {
if wait != nil {
_ = cmd.Process.Kill()
_ = cmd.Wait()
}
}()
stream = zipped
default:
return errors.New("it is not a tar archive compressed with gzip, bzip2 or xz; a single file is kept " +
"whole by naming it in `file`")
}
if err := untar(stream, into, within); err != nil {
return err
}
if wait != nil {
// Whatever follows the tar's end is read before xz is waited for: xz blocks on a full pipe otherwise.
if _, err := io.Copy(io.Discard, stream); err != nil {
return err
}
w := wait
wait = nil
return w()
}
return nil
return untar(stream, into, within)
}
// xzDictCap is the largest xz dictionary read: xz's own presets use at most 64 MiB (-9); a stream asking for
// more is refused rather than given the memory.
const xzDictCap = 64 << 20
// untar writes a tar stream's files and directories within `within` into a directory.
func untar(stream io.Reader, into, within string) error {
reader := tar.NewReader(stream)
+22
View File
@@ -383,3 +383,25 @@ func TestARedirectOffHttpsIsNotFollowed(t *testing.T) {
t.Fatalf("%v", err)
}
}
func TestAnXzReleaseIsReadWithoutAnyXzToolOnTheBuildSeat(t *testing.T) {
// The build seat's image (the catalogue's build-agent recipe) carries git and docker-cli alone: Betterbird's
// .tar.xz failed there on 2026-10-12 with "this build seat has no xz to read it" (hq issue 345). It is
// read in Go, with nothing on PATH.
t.Setenv("PATH", "")
body, err := os.ReadFile(filepath.Join("testdata", "release.tar.xz"))
if err != nil {
t.Fatal(err)
}
v := &vendor{files: map[string][]byte{"/r.tar.xz": body}}
base := serveVendor(t, v)
packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive",
URL: base + "/r.tar.xz", SHA256: sumHex(body), Version: "1", Within: "betterbird"}, func(string, string, ...any) {})
if err != nil {
t.Fatal(err)
}
e := entriesOf(t, packed)
if len(e) != 2 || e["betterbird"] != "#!/bin/sh\n -rwxr-xr-x" || e["defaults/p.js"] != "pref(\"a\", 1);\n -rw-r--r--" {
t.Fatalf("%v", e)
}
}