jschoubben
b6247f837d
After the build ended the module pushes the packed bundle or archive with mesh-sdk's PushBlob under <module>/<artifact> by its digest, a candidate's as a trunk build's, and the step answers artifact-store://<module>/<artifact>/blobs/<digest> instead of a path. A push that fails fails the step not stored: <artifact>: <the store's words>. The packed bytes are never written to the node. The manifest requires artifact-store and places its address.
372 lines
12 KiB
Go
372 lines
12 KiB
Go
package main
|
|
|
|
// The archive artifact (novox/hq ADR 0478 §1, issue 1589): a directory of a module's tree, or a file pinned by url and
|
|
// sha256, packed as the build agent packs it (mesh-controller internal/builder: the archive case of `one`, `pack`, and
|
|
// upstream_file.go), with mesh-sdk's Pack. Any module's archive is packed here, whatever language the module is in.
|
|
|
|
import (
|
|
"archive/tar"
|
|
"bufio"
|
|
"bytes"
|
|
"compress/bzip2"
|
|
"compress/gzip"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
"github.com/therootcompany/xz"
|
|
)
|
|
|
|
// maxUpstream is the largest pinned file fetched and the largest it may unpack to, as the build agent's and the
|
|
// node-engine's (a machine unpacks no archive larger).
|
|
const maxUpstream = 512 << 20
|
|
|
|
// maxEntries is the most entries a pinned archive may hold.
|
|
const maxEntries = 100_000
|
|
|
|
func (m *Module) httpClient() *http.Client {
|
|
if m.HTTP != nil {
|
|
return m.HTTP
|
|
}
|
|
return &http.Client{Timeout: 15 * time.Minute}
|
|
}
|
|
|
|
// buildArchive packs one archive artifact of the module in dir: a pinned file when it names a url, else the directory
|
|
// `from` of the module's tree.
|
|
func (m *Module) buildArchive(ctx context.Context, workspace, module, name, dir string, art catalogue.Artifact) (any, error) {
|
|
if art.URL != "" {
|
|
return m.pinnedArchive(ctx, workspace, module, name, art)
|
|
}
|
|
from := strings.Trim(path.Clean("/"+filepath.ToSlash(art.From)), "/")
|
|
if strings.TrimSpace(art.From) == "" {
|
|
return nil, fmt.Errorf("the artifact %s says nothing it is built from", art.Name)
|
|
}
|
|
root := filepath.Join(dir, filepath.FromSlash(from))
|
|
if rel, err := filepath.Rel(dir, root); err != nil || strings.HasPrefix(rel, "..") {
|
|
return nil, fmt.Errorf("the archive %s is taken from %q, which lies outside the module", art.Name, art.From)
|
|
}
|
|
if strings.Contains(filepath.ToSlash(art.From), "..") {
|
|
return nil, fmt.Errorf("the archive %s is taken from %q, which lies outside the module", art.Name, art.From)
|
|
}
|
|
// A link in the fetched tree must not lead out of the module: compare where both really are.
|
|
realDir, err := filepath.EvalSymlinks(dir)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
realTree, err := filepath.EvalSymlinks(filepath.Join(m.Work, workspace, "tree"))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !within(realTree, realDir) {
|
|
return nil, fmt.Errorf("the module directory of %s resolves to %s, which lies outside the workspace's tree %s", art.Name, realDir, realTree)
|
|
}
|
|
realRoot, err := filepath.EvalSymlinks(root)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !within(realDir, realRoot) {
|
|
return nil, fmt.Errorf("the archive %s is taken from %q, which lies outside the module: it resolves to %s, and the module is %s", art.Name, art.From, realRoot, realDir)
|
|
}
|
|
return m.packInto(ctx, workspace, module, name, art.Name, root)
|
|
}
|
|
|
|
// pinnedArchive fetches the file an artifact is pinned to, checks it against the pin, lays it out as the artifact says
|
|
// (kept whole as `file`, or unpacked from `within`) and packs it.
|
|
func (m *Module) pinnedArchive(ctx context.Context, workspace, module, name string, art catalogue.Artifact) (any, error) {
|
|
file, err := m.pinnedFile(ctx, workspace, art)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%s: %w", art.Name, err)
|
|
}
|
|
laid := filepath.Join(m.Work, workspace, "upstream-"+art.Name)
|
|
if err := os.RemoveAll(laid); err != nil {
|
|
return nil, err
|
|
}
|
|
defer os.RemoveAll(laid)
|
|
if err := os.MkdirAll(laid, 0o755); err != nil {
|
|
return nil, err
|
|
}
|
|
if art.File != "" {
|
|
body, err := os.ReadFile(file)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
kept := filepath.Join(laid, art.File)
|
|
if err := os.WriteFile(kept, body, 0o644); err != nil {
|
|
return nil, err
|
|
}
|
|
// The mode is the archive's, never the account's umask: the digest is the same on every build node.
|
|
if err := os.Chmod(kept, 0o644); err != nil {
|
|
return nil, err
|
|
}
|
|
} else if err := unpackUpstream(file, laid, art.Within); err != nil {
|
|
return nil, fmt.Errorf("%s: %s (%s): %w", art.Name, art.URL, art.Version, err)
|
|
}
|
|
return m.packInto(ctx, workspace, module, name, art.Name, laid)
|
|
}
|
|
|
|
// pinnedFile is the path of the pinned file in the workspace: kept from an earlier build when its bytes are still the
|
|
// pin's, else fetched now and kept only when they are.
|
|
func (m *Module) pinnedFile(ctx context.Context, workspace string, art catalogue.Artifact) (string, error) {
|
|
want := strings.ToLower(art.SHA256)
|
|
dir := filepath.Join(m.Work, workspace, "upstream")
|
|
kept := filepath.Join(dir, want)
|
|
if got, err := sumOf(kept); err == nil {
|
|
if got == want {
|
|
return kept, nil
|
|
}
|
|
os.Remove(kept)
|
|
}
|
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
|
return "", err
|
|
}
|
|
request, err := http.NewRequestWithContext(ctx, http.MethodGet, art.URL, nil)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
client := *m.httpClient()
|
|
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
|
|
if req.URL.Scheme != "https" {
|
|
return fmt.Errorf("%s redirected to %s, which is not https", via[0].URL, req.URL)
|
|
}
|
|
if len(via) >= 10 {
|
|
return errors.New("more than 10 redirects")
|
|
}
|
|
return nil
|
|
}
|
|
response, err := client.Do(request)
|
|
if err != nil {
|
|
return "", fmt.Errorf("cannot fetch %s: %w", art.URL, err)
|
|
}
|
|
defer response.Body.Close()
|
|
if response.StatusCode != http.StatusOK {
|
|
return "", fmt.Errorf("%s answered %s", art.URL, response.Status)
|
|
}
|
|
part, err := os.CreateTemp(dir, want+".*.part")
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer os.Remove(part.Name())
|
|
hash := sha256.New()
|
|
n, err := io.Copy(io.MultiWriter(part, hash), io.LimitReader(response.Body, maxUpstream+1))
|
|
if cerr := part.Close(); err == nil {
|
|
err = cerr
|
|
}
|
|
if err != nil {
|
|
return "", fmt.Errorf("fetching %s: %w", art.URL, err)
|
|
}
|
|
if n > maxUpstream {
|
|
return "", fmt.Errorf("%s is larger than %d bytes, more than a machine unpacks; nothing was kept", art.URL, maxUpstream)
|
|
}
|
|
if got := hex.EncodeToString(hash.Sum(nil)); got != want {
|
|
return "", fmt.Errorf("%s was pinned as sha256:%s and what it served is sha256:%s (%d bytes); nothing was kept. "+
|
|
"The address serves another file than the one reviewed: compare it, then move the pin in a reviewed change "+
|
|
"(novox/hq ADR 0316)", art.URL, want, got, n)
|
|
}
|
|
if err := os.Rename(part.Name(), kept); err != nil {
|
|
return "", err
|
|
}
|
|
return kept, nil
|
|
}
|
|
|
|
func sumOf(file string) (string, error) {
|
|
f, err := os.Open(file)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer f.Close()
|
|
hash := sha256.New()
|
|
if _, err := io.Copy(hash, f); err != nil {
|
|
return "", err
|
|
}
|
|
return hex.EncodeToString(hash.Sum(nil)), nil
|
|
}
|
|
|
|
// xzDictMax is the largest xz dictionary read (the build agent's): xz's own presets use at most 64 MiB (-9); a stream
|
|
// asking for more is refused rather than given the memory.
|
|
const xzDictMax = 64 << 20
|
|
|
|
// unpackUpstream unpacks a tar archive compressed with gzip, bzip2 or xz into a directory: of its entries only those
|
|
// within `within` (all, when empty), with that prefix taken off. Only files and directories, and nothing outside: the
|
|
// node-engine unpacks nothing else.
|
|
func unpackUpstream(file, into, within string) error {
|
|
f, err := os.Open(file)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
buffered := bufio.NewReader(f)
|
|
magic, _ := buffered.Peek(6)
|
|
var stream io.Reader
|
|
switch {
|
|
case bytes.HasPrefix(magic, []byte{0x1f, 0x8b}):
|
|
zipped, err := gzip.NewReader(buffered)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer zipped.Close()
|
|
stream = zipped
|
|
case bytes.HasPrefix(magic, []byte("BZh")):
|
|
stream = bzip2.NewReader(buffered)
|
|
case bytes.HasPrefix(magic, []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}):
|
|
// As the build agent reads it (mesh-controller internal/builder/upstream_file.go; issue 1590): in Go
|
|
// (github.com/therootcompany/xz, a port of xz-embedded), never by a tool the node may not carry. xzDictMax is
|
|
// a real maximum: a block whose LZMA2 dictionary is larger is refused before its dictionary is allocated. The
|
|
// reader keeps a filter decoder for every block of a stream, so the file's framing is walked first and a file
|
|
// of more blocks than xzBlocksMax is refused before anything is decoded (xz_framing.go). What the stream
|
|
// unpacks to is bounded by untar (maxUpstream, maxEntries).
|
|
if err := xzWithinLimits(file); err != nil {
|
|
return err
|
|
}
|
|
zipped, err := xz.NewReader(buffered, xzDictMax)
|
|
if err != nil {
|
|
return fmt.Errorf("it is not an xz stream this build module reads (a dictionary of %d MiB at most): %v", xzDictMax>>20, err)
|
|
}
|
|
stream = zipped
|
|
default:
|
|
return errors.New("it is not a tar archive compressed with gzip, bzip2 or xz; a single file is kept whole by naming it in `file`")
|
|
}
|
|
return untar(stream, into, within)
|
|
}
|
|
|
|
func untar(stream io.Reader, into, within string) error {
|
|
reader := tar.NewReader(stream)
|
|
prefix := ""
|
|
if within != "" {
|
|
prefix = path.Clean(within) + "/"
|
|
}
|
|
var total int64
|
|
files, entries := 0, 0
|
|
for {
|
|
header, err := reader.Next()
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if entries++; entries > maxEntries {
|
|
return fmt.Errorf("it holds more than %d entries", maxEntries)
|
|
}
|
|
if header.Typeflag == tar.TypeXGlobalHeader {
|
|
continue
|
|
}
|
|
name := path.Clean(strings.TrimPrefix(header.Name, "./"))
|
|
if path.IsAbs(name) || name == ".." || strings.HasPrefix(name, "../") {
|
|
return fmt.Errorf("%s names a path outside the archive", header.Name)
|
|
}
|
|
if name == "." || name+"/" == prefix ||
|
|
(header.Typeflag == tar.TypeDir && strings.HasPrefix(prefix, name+"/")) {
|
|
continue
|
|
}
|
|
if prefix != "" {
|
|
if !strings.HasPrefix(name, prefix) {
|
|
return fmt.Errorf("%s is outside %s, the directory the artifact is taken from", header.Name, within)
|
|
}
|
|
name = strings.TrimPrefix(name, prefix)
|
|
}
|
|
target := filepath.Join(into, filepath.FromSlash(name))
|
|
switch header.Typeflag {
|
|
case tar.TypeDir:
|
|
if err := os.MkdirAll(target, 0o755); err != nil {
|
|
return err
|
|
}
|
|
case tar.TypeReg:
|
|
total += header.Size
|
|
if total > maxUpstream {
|
|
return fmt.Errorf("it unpacks to more than %d bytes, more than a machine unpacks", maxUpstream)
|
|
}
|
|
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
|
|
return err
|
|
}
|
|
mode := os.FileMode(0o644)
|
|
if header.Mode&0o111 != 0 {
|
|
mode = 0o755
|
|
}
|
|
out, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := out.Chmod(mode); err != nil {
|
|
out.Close()
|
|
return err
|
|
}
|
|
if _, err := io.Copy(out, io.LimitReader(reader, header.Size)); err != nil {
|
|
out.Close()
|
|
return err
|
|
}
|
|
if err := out.Close(); err != nil {
|
|
return err
|
|
}
|
|
files++
|
|
default:
|
|
return fmt.Errorf("%s is a %c, and an archive carries only files and directories", header.Name, header.Typeflag)
|
|
}
|
|
}
|
|
if files == 0 {
|
|
if within != "" {
|
|
return fmt.Errorf("it holds no file within %s", within)
|
|
}
|
|
return errors.New("it holds no file")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// within says whether path is base or lies below it.
|
|
func within(base, path string) bool {
|
|
rel, err := filepath.Rel(base, path)
|
|
return err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator))
|
|
}
|
|
|
|
// resolveDeepest follows the links of the deepest part of p that exists and appends the rest, so a path not yet made
|
|
// is judged by where it would be made.
|
|
func resolveDeepest(p string) (string, error) {
|
|
rest := ""
|
|
for cur := filepath.Clean(p); ; {
|
|
if real, err := filepath.EvalSymlinks(cur); err == nil {
|
|
return filepath.Join(real, rest), nil
|
|
} else if !os.IsNotExist(err) {
|
|
return "", err
|
|
}
|
|
parent := filepath.Dir(cur)
|
|
if parent == cur {
|
|
return "", fmt.Errorf("no part of %s exists", p)
|
|
}
|
|
rest = filepath.Join(filepath.Base(cur), rest)
|
|
cur = parent
|
|
}
|
|
}
|
|
|
|
// insideModule refuses a path whose real place is not inside the module's directory, which is itself inside the
|
|
// workspace's tree: a link committed in the repository must not lead a build's writes or its pack out.
|
|
func (m *Module) insideModule(workspace, dir, path string) error {
|
|
realTree, err := filepath.EvalSymlinks(filepath.Join(m.Work, workspace, "tree"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
realDir, err := filepath.EvalSymlinks(dir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !within(realTree, realDir) {
|
|
return fmt.Errorf("the module directory resolves to %s, which lies outside the workspace's tree %s", realDir, realTree)
|
|
}
|
|
real, err := resolveDeepest(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !within(realDir, real) {
|
|
return fmt.Errorf("%s resolves to %s, which lies outside the module %s", path, real, realDir)
|
|
}
|
|
return nil
|
|
}
|