From d369adf2459c9b8cd2d7c588814c2072e368d4fa Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Wed, 21 Oct 2026 09:11:29 +0000 Subject: [PATCH] =?UTF-8?q?Unpack=20and=20pack=20archive=20artifacts=20in?= =?UTF-8?q?=20mesh-build-go:=20a=20module's=20directory=20or=20a=20pinned?= =?UTF-8?q?=20file=20(issue=201589,=20ADR=200478=20=C2=A71)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cmd/mesh-build-go/archive.go | 277 ++++++++++++++++++ 1 file changed, 277 insertions(+) create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/archive.go diff --git a/modules/mesh-build-go/cmd/mesh-build-go/archive.go b/modules/mesh-build-go/cmd/mesh-build-go/archive.go new file mode 100644 index 00000000..ae75363c --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/archive.go @@ -0,0 +1,277 @@ +package main + +// The archive artifact (novox/hq ADR 0478 ยง1, issue 1589): a directory of a module's tree, or a file pinned by url and +// sha256, packed as the build agent packs it (mesh-controller internal/builder: the archive case of `one`, `pack`, and +// upstream_file.go), with mesh-sdk's Pack. Any module's archive is packed here, whatever language the module is in. + +import ( + "archive/tar" + "bufio" + "bytes" + "compress/bzip2" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "os" + "path" + "path/filepath" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// maxUpstream is the largest pinned file fetched and the largest it may unpack to, as the build agent's and the +// node-engine's (a machine unpacks no archive larger). +const maxUpstream = 512 << 20 + +// maxEntries is the most entries a pinned archive may hold. +const maxEntries = 100_000 + +func (m *Module) httpClient() *http.Client { + if m.HTTP != nil { + return m.HTTP + } + return &http.Client{Timeout: 15 * time.Minute} +} + +// buildArchive packs one archive artifact of the module in dir: a pinned file when it names a url, else the directory +// `from` of the module's tree. +func (m *Module) buildArchive(ctx context.Context, workspace, module, dir string, art catalogue.Artifact) (any, error) { + if art.URL != "" { + return m.pinnedArchive(ctx, workspace, module, art) + } + from := strings.Trim(path.Clean("/"+filepath.ToSlash(art.From)), "/") + if strings.TrimSpace(art.From) == "" { + return nil, fmt.Errorf("the artifact %s says nothing it is built from", art.Name) + } + root := filepath.Join(dir, filepath.FromSlash(from)) + if rel, err := filepath.Rel(dir, root); err != nil || strings.HasPrefix(rel, "..") { + return nil, fmt.Errorf("the archive %s is taken from %q, which lies outside the module", art.Name, art.From) + } + if strings.Contains(filepath.ToSlash(art.From), "..") { + return nil, fmt.Errorf("the archive %s is taken from %q, which lies outside the module", art.Name, art.From) + } + return m.packInto(workspace, module, art.Name, root) +} + +// pinnedArchive fetches the file an artifact is pinned to, checks it against the pin, lays it out as the artifact says +// (kept whole as `file`, or unpacked from `within`) and packs it. +func (m *Module) pinnedArchive(ctx context.Context, workspace, module string, art catalogue.Artifact) (any, error) { + file, err := m.pinnedFile(ctx, workspace, art) + if err != nil { + return nil, fmt.Errorf("%s: %w", art.Name, err) + } + laid := filepath.Join(m.Work, workspace, "upstream-"+art.Name) + if err := os.RemoveAll(laid); err != nil { + return nil, err + } + defer os.RemoveAll(laid) + if err := os.MkdirAll(laid, 0o755); err != nil { + return nil, err + } + if art.File != "" { + body, err := os.ReadFile(file) + if err != nil { + return nil, err + } + if err := os.WriteFile(filepath.Join(laid, art.File), body, 0o644); err != nil { + return nil, err + } + } else if err := unpackUpstream(file, laid, art.Within); err != nil { + return nil, fmt.Errorf("%s: %s (%s): %w", art.Name, art.URL, art.Version, err) + } + return m.packInto(workspace, module, art.Name, laid) +} + +// pinnedFile is the path of the pinned file in the workspace: kept from an earlier build when its bytes are still the +// pin's, else fetched now and kept only when they are. +func (m *Module) pinnedFile(ctx context.Context, workspace string, art catalogue.Artifact) (string, error) { + want := strings.ToLower(art.SHA256) + dir := filepath.Join(m.Work, workspace, "upstream") + kept := filepath.Join(dir, want) + if got, err := sumOf(kept); err == nil { + if got == want { + return kept, nil + } + os.Remove(kept) + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return "", err + } + request, err := http.NewRequestWithContext(ctx, http.MethodGet, art.URL, nil) + if err != nil { + return "", err + } + client := *m.httpClient() + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + if req.URL.Scheme != "https" { + return fmt.Errorf("%s redirected to %s, which is not https", via[0].URL, req.URL) + } + if len(via) >= 10 { + return errors.New("more than 10 redirects") + } + return nil + } + response, err := client.Do(request) + if err != nil { + return "", fmt.Errorf("cannot fetch %s: %w", art.URL, err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return "", fmt.Errorf("%s answered %s", art.URL, response.Status) + } + part, err := os.CreateTemp(dir, want+".*.part") + if err != nil { + return "", err + } + defer os.Remove(part.Name()) + hash := sha256.New() + n, err := io.Copy(io.MultiWriter(part, hash), io.LimitReader(response.Body, maxUpstream+1)) + if cerr := part.Close(); err == nil { + err = cerr + } + if err != nil { + return "", fmt.Errorf("fetching %s: %w", art.URL, err) + } + if n > maxUpstream { + return "", fmt.Errorf("%s is larger than %d bytes, more than a machine unpacks; nothing was kept", art.URL, maxUpstream) + } + if got := hex.EncodeToString(hash.Sum(nil)); got != want { + return "", fmt.Errorf("%s was pinned as sha256:%s and what it served is sha256:%s (%d bytes); nothing was kept. "+ + "The address serves another file than the one reviewed: compare it, then move the pin in a reviewed change "+ + "(novox/hq ADR 0316)", art.URL, want, got, n) + } + if err := os.Rename(part.Name(), kept); err != nil { + return "", err + } + return kept, nil +} + +func sumOf(file string) (string, error) { + f, err := os.Open(file) + if err != nil { + return "", err + } + defer f.Close() + hash := sha256.New() + if _, err := io.Copy(hash, f); err != nil { + return "", err + } + return hex.EncodeToString(hash.Sum(nil)), nil +} + +// unpackUpstream unpacks a tar archive compressed with gzip or bzip2 into a directory: of its entries only those within +// `within` (all, when empty), with that prefix taken off. Only files and directories, and nothing outside: the +// node-engine unpacks nothing else. An xz archive is refused by name; the build agent reads it, this module does not +// yet (it is a later issue, not a quiet difference in the archive). +func unpackUpstream(file, into, within string) error { + f, err := os.Open(file) + if err != nil { + return err + } + defer f.Close() + buffered := bufio.NewReader(f) + magic, _ := buffered.Peek(6) + var stream io.Reader + switch { + case bytes.HasPrefix(magic, []byte{0x1f, 0x8b}): + zipped, err := gzip.NewReader(buffered) + if err != nil { + return err + } + defer zipped.Close() + stream = zipped + case bytes.HasPrefix(magic, []byte("BZh")): + stream = bzip2.NewReader(buffered) + case bytes.HasPrefix(magic, []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}): + return errors.New("it is an xz archive, which this build module does not read yet; the build agent does") + default: + return errors.New("it is not a tar archive compressed with gzip or bzip2; a single file is kept whole by naming it in `file`") + } + return untar(stream, into, within) +} + +func untar(stream io.Reader, into, within string) error { + reader := tar.NewReader(stream) + prefix := "" + if within != "" { + prefix = path.Clean(within) + "/" + } + var total int64 + files, entries := 0, 0 + for { + header, err := reader.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + if entries++; entries > maxEntries { + return fmt.Errorf("it holds more than %d entries", maxEntries) + } + if header.Typeflag == tar.TypeXGlobalHeader { + continue + } + name := path.Clean(strings.TrimPrefix(header.Name, "./")) + if path.IsAbs(name) || name == ".." || strings.HasPrefix(name, "../") { + return fmt.Errorf("%s names a path outside the archive", header.Name) + } + if name == "." || name+"/" == prefix || + (header.Typeflag == tar.TypeDir && strings.HasPrefix(prefix, name+"/")) { + continue + } + if prefix != "" { + if !strings.HasPrefix(name, prefix) { + return fmt.Errorf("%s is outside %s, the directory the artifact is taken from", header.Name, within) + } + name = strings.TrimPrefix(name, prefix) + } + target := filepath.Join(into, filepath.FromSlash(name)) + switch header.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(target, 0o755); err != nil { + return err + } + case tar.TypeReg: + total += header.Size + if total > maxUpstream { + return fmt.Errorf("it unpacks to more than %d bytes, more than a machine unpacks", maxUpstream) + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + mode := os.FileMode(0o644) + if header.Mode&0o111 != 0 { + mode = 0o755 + } + out, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode) + if err != nil { + return err + } + if _, err := io.Copy(out, io.LimitReader(reader, header.Size)); err != nil { + out.Close() + return err + } + if err := out.Close(); err != nil { + return err + } + files++ + default: + return fmt.Errorf("%s is a %c, and an archive carries only files and directories", header.Name, header.Typeflag) + } + } + if files == 0 { + if within != "" { + return fmt.Errorf("it holds no file within %s", within) + } + return errors.New("it holds no file") + } + return nil +}