Unpack and pack archive artifacts in mesh-build-go: a module's directory or a pinned file (issue 1589, ADR 0478 §1)
This commit is contained in:
@@ -0,0 +1,277 @@
|
||||
package main
|
||||
|
||||
// The archive artifact (novox/hq ADR 0478 §1, issue 1589): a directory of a module's tree, or a file pinned by url and
|
||||
// sha256, packed as the build agent packs it (mesh-controller internal/builder: the archive case of `one`, `pack`, and
|
||||
// upstream_file.go), with mesh-sdk's Pack. Any module's archive is packed here, whatever language the module is in.
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"compress/bzip2"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// maxUpstream is the largest pinned file fetched and the largest it may unpack to, as the build agent's and the
|
||||
// node-engine's (a machine unpacks no archive larger).
|
||||
const maxUpstream = 512 << 20
|
||||
|
||||
// maxEntries is the most entries a pinned archive may hold.
|
||||
const maxEntries = 100_000
|
||||
|
||||
func (m *Module) httpClient() *http.Client {
|
||||
if m.HTTP != nil {
|
||||
return m.HTTP
|
||||
}
|
||||
return &http.Client{Timeout: 15 * time.Minute}
|
||||
}
|
||||
|
||||
// buildArchive packs one archive artifact of the module in dir: a pinned file when it names a url, else the directory
|
||||
// `from` of the module's tree.
|
||||
func (m *Module) buildArchive(ctx context.Context, workspace, module, dir string, art catalogue.Artifact) (any, error) {
|
||||
if art.URL != "" {
|
||||
return m.pinnedArchive(ctx, workspace, module, art)
|
||||
}
|
||||
from := strings.Trim(path.Clean("/"+filepath.ToSlash(art.From)), "/")
|
||||
if strings.TrimSpace(art.From) == "" {
|
||||
return nil, fmt.Errorf("the artifact %s says nothing it is built from", art.Name)
|
||||
}
|
||||
root := filepath.Join(dir, filepath.FromSlash(from))
|
||||
if rel, err := filepath.Rel(dir, root); err != nil || strings.HasPrefix(rel, "..") {
|
||||
return nil, fmt.Errorf("the archive %s is taken from %q, which lies outside the module", art.Name, art.From)
|
||||
}
|
||||
if strings.Contains(filepath.ToSlash(art.From), "..") {
|
||||
return nil, fmt.Errorf("the archive %s is taken from %q, which lies outside the module", art.Name, art.From)
|
||||
}
|
||||
return m.packInto(workspace, module, art.Name, root)
|
||||
}
|
||||
|
||||
// pinnedArchive fetches the file an artifact is pinned to, checks it against the pin, lays it out as the artifact says
|
||||
// (kept whole as `file`, or unpacked from `within`) and packs it.
|
||||
func (m *Module) pinnedArchive(ctx context.Context, workspace, module string, art catalogue.Artifact) (any, error) {
|
||||
file, err := m.pinnedFile(ctx, workspace, art)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", art.Name, err)
|
||||
}
|
||||
laid := filepath.Join(m.Work, workspace, "upstream-"+art.Name)
|
||||
if err := os.RemoveAll(laid); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer os.RemoveAll(laid)
|
||||
if err := os.MkdirAll(laid, 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if art.File != "" {
|
||||
body, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(laid, art.File), body, 0o644); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else if err := unpackUpstream(file, laid, art.Within); err != nil {
|
||||
return nil, fmt.Errorf("%s: %s (%s): %w", art.Name, art.URL, art.Version, err)
|
||||
}
|
||||
return m.packInto(workspace, module, art.Name, laid)
|
||||
}
|
||||
|
||||
// pinnedFile is the path of the pinned file in the workspace: kept from an earlier build when its bytes are still the
|
||||
// pin's, else fetched now and kept only when they are.
|
||||
func (m *Module) pinnedFile(ctx context.Context, workspace string, art catalogue.Artifact) (string, error) {
|
||||
want := strings.ToLower(art.SHA256)
|
||||
dir := filepath.Join(m.Work, workspace, "upstream")
|
||||
kept := filepath.Join(dir, want)
|
||||
if got, err := sumOf(kept); err == nil {
|
||||
if got == want {
|
||||
return kept, nil
|
||||
}
|
||||
os.Remove(kept)
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, art.URL, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
client := *m.httpClient()
|
||||
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
|
||||
if req.URL.Scheme != "https" {
|
||||
return fmt.Errorf("%s redirected to %s, which is not https", via[0].URL, req.URL)
|
||||
}
|
||||
if len(via) >= 10 {
|
||||
return errors.New("more than 10 redirects")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("cannot fetch %s: %w", art.URL, err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("%s answered %s", art.URL, response.Status)
|
||||
}
|
||||
part, err := os.CreateTemp(dir, want+".*.part")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer os.Remove(part.Name())
|
||||
hash := sha256.New()
|
||||
n, err := io.Copy(io.MultiWriter(part, hash), io.LimitReader(response.Body, maxUpstream+1))
|
||||
if cerr := part.Close(); err == nil {
|
||||
err = cerr
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("fetching %s: %w", art.URL, err)
|
||||
}
|
||||
if n > maxUpstream {
|
||||
return "", fmt.Errorf("%s is larger than %d bytes, more than a machine unpacks; nothing was kept", art.URL, maxUpstream)
|
||||
}
|
||||
if got := hex.EncodeToString(hash.Sum(nil)); got != want {
|
||||
return "", fmt.Errorf("%s was pinned as sha256:%s and what it served is sha256:%s (%d bytes); nothing was kept. "+
|
||||
"The address serves another file than the one reviewed: compare it, then move the pin in a reviewed change "+
|
||||
"(novox/hq ADR 0316)", art.URL, want, got, n)
|
||||
}
|
||||
if err := os.Rename(part.Name(), kept); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return kept, nil
|
||||
}
|
||||
|
||||
func sumOf(file string) (string, error) {
|
||||
f, err := os.Open(file)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer f.Close()
|
||||
hash := sha256.New()
|
||||
if _, err := io.Copy(hash, f); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(hash.Sum(nil)), nil
|
||||
}
|
||||
|
||||
// unpackUpstream unpacks a tar archive compressed with gzip or bzip2 into a directory: of its entries only those within
|
||||
// `within` (all, when empty), with that prefix taken off. Only files and directories, and nothing outside: the
|
||||
// node-engine unpacks nothing else. An xz archive is refused by name; the build agent reads it, this module does not
|
||||
// yet (it is a later issue, not a quiet difference in the archive).
|
||||
func unpackUpstream(file, into, within string) error {
|
||||
f, err := os.Open(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
buffered := bufio.NewReader(f)
|
||||
magic, _ := buffered.Peek(6)
|
||||
var stream io.Reader
|
||||
switch {
|
||||
case bytes.HasPrefix(magic, []byte{0x1f, 0x8b}):
|
||||
zipped, err := gzip.NewReader(buffered)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer zipped.Close()
|
||||
stream = zipped
|
||||
case bytes.HasPrefix(magic, []byte("BZh")):
|
||||
stream = bzip2.NewReader(buffered)
|
||||
case bytes.HasPrefix(magic, []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}):
|
||||
return errors.New("it is an xz archive, which this build module does not read yet; the build agent does")
|
||||
default:
|
||||
return errors.New("it is not a tar archive compressed with gzip or bzip2; a single file is kept whole by naming it in `file`")
|
||||
}
|
||||
return untar(stream, into, within)
|
||||
}
|
||||
|
||||
func untar(stream io.Reader, into, within string) error {
|
||||
reader := tar.NewReader(stream)
|
||||
prefix := ""
|
||||
if within != "" {
|
||||
prefix = path.Clean(within) + "/"
|
||||
}
|
||||
var total int64
|
||||
files, entries := 0, 0
|
||||
for {
|
||||
header, err := reader.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entries++; entries > maxEntries {
|
||||
return fmt.Errorf("it holds more than %d entries", maxEntries)
|
||||
}
|
||||
if header.Typeflag == tar.TypeXGlobalHeader {
|
||||
continue
|
||||
}
|
||||
name := path.Clean(strings.TrimPrefix(header.Name, "./"))
|
||||
if path.IsAbs(name) || name == ".." || strings.HasPrefix(name, "../") {
|
||||
return fmt.Errorf("%s names a path outside the archive", header.Name)
|
||||
}
|
||||
if name == "." || name+"/" == prefix ||
|
||||
(header.Typeflag == tar.TypeDir && strings.HasPrefix(prefix, name+"/")) {
|
||||
continue
|
||||
}
|
||||
if prefix != "" {
|
||||
if !strings.HasPrefix(name, prefix) {
|
||||
return fmt.Errorf("%s is outside %s, the directory the artifact is taken from", header.Name, within)
|
||||
}
|
||||
name = strings.TrimPrefix(name, prefix)
|
||||
}
|
||||
target := filepath.Join(into, filepath.FromSlash(name))
|
||||
switch header.Typeflag {
|
||||
case tar.TypeDir:
|
||||
if err := os.MkdirAll(target, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
case tar.TypeReg:
|
||||
total += header.Size
|
||||
if total > maxUpstream {
|
||||
return fmt.Errorf("it unpacks to more than %d bytes, more than a machine unpacks", maxUpstream)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
mode := os.FileMode(0o644)
|
||||
if header.Mode&0o111 != 0 {
|
||||
mode = 0o755
|
||||
}
|
||||
out, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(out, io.LimitReader(reader, header.Size)); err != nil {
|
||||
out.Close()
|
||||
return err
|
||||
}
|
||||
if err := out.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
files++
|
||||
default:
|
||||
return fmt.Errorf("%s is a %c, and an archive carries only files and directories", header.Name, header.Typeflag)
|
||||
}
|
||||
}
|
||||
if files == 0 {
|
||||
if within != "" {
|
||||
return fmt.Errorf("it holds no file within %s", within)
|
||||
}
|
||||
return errors.New("it holds no file")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user