mesh-host/internal/apply/archive.go

package apply

import (
	"archive/tar"
	"compress/gzip"
	"context"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"github.com/novox/mesh-host/internal/declaration"
	"github.com/novox/mesh-host/internal/store"
)

// A set of files, fetched by digest and unpacked.
//
// For what inlining cannot serve: a theme, an icon set, a tree of configuration. Hundreds of
// files inlined would make every declaration enormous and rewrite all of them when one changed.
//
// **This is the one place the host reaches out on its own.** Everywhere else it holds a single
// outbound connection to the broker and fetches nothing; a container image is pulled by the
// runtime rather than by this process. So the discipline has to be explicit and it is the same
// one the bootstrap uses for images: **pinned by digest, and the digest is checked before
// anything is written.** What is fetched is bytes from a network the mesh does not control, and
// the only thing making them safe to unpack is that they hash to what was declared.

// maxArchive is how much will be read before giving up.
//
// Because a fetch with no limit is a machine somebody can fill up from the far end. Chosen large
// enough for a desktop theme and small enough to notice.
const maxArchive = 512 << 20

func applyArchive(ctx context.Context, r *declaration.Archive, previous store.Applied) (Outcome, error) {
	out := begin(r)
	out.Action = "unchanged"

	body, err := fetch(ctx, r.Source)
	if err != nil {
		return out, err
	}
	sum := sha256.Sum256(body)
	got := "sha256:" + hex.EncodeToString(sum[:])
	if got != r.Digest {
		// Refused before a single file is written. A digest that does not match means the thing
		// at that address is not the thing that was declared, and unpacking it would be applying
		// something nobody reviewed.
		return out, fmt.Errorf(
			"%s was declared as %s and what arrived is %s; nothing was unpacked",
			r.Source, r.Digest, got)
	}
	out.wrote = got

	// Already what it should be. The digest is the whole identity of an archive, so a matching
	// record means the unpacked tree came from these exact bytes — at this path: a record of the
	// same bytes somewhere else says nothing about what is here.
	if previous.Wrote == got && previous.Target == r.Path {
		if _, err := os.Stat(r.Path); err == nil {
			owned, err := ownedBy(ownershipProbe(r.Path, previous.Unpacked), r.Owner)
			if err == nil && owned {
				// What it unpacked is carried, or — on a record from before the host kept it — read
				// from the archive now, so the record can say it from here on (novox/hq issue 162).
				out.unpacked, err = stillUnpacked(body, r.Path, previous.Unpacked)
				if err != nil {
					return out, err
				}
				return out, nil
			}
		}
	}

	unpacked, written, err := replaceWith(body, r.Path, r.Owner, oursFrom(previous, r.Path))
	if err != nil {
		return out, err
	}
	out.unpacked = &unpacked
	out.Action = "updated"
	if previous.Wrote == "" {
		out.Action = "created"
	}
	out.Detail = fmt.Sprintf("%d file(s)", written)
	return out, nil
}

// replaceWith makes the directory exactly the archive (novox/hq issue 220), and says what it put
// there (novox/hq issue 162).
//
// **The tree on disk is the archive and nothing else — of what the mesh put there.** The digest is
// the whole identity of what is unpacked here, so a file the previous archive had and this one does
// not must go. Unpacked over the old tree, it stayed: a bundle rebuilt as one file per entrypoint
// kept the package directory of the version before, which code could still import, and a fix that
// removed a file worked on a fresh machine only. So the archive is unpacked into a fresh directory
// beside the old one, owned, and swapped in by rename. A running process keeps the files it has open,
// and the old tree is removed only once the new one is in place. A failed unpack leaves the old tree
// untouched.
//
// **What the mesh did not put there is never swapped away** (novox/hq issue 162, ADR 0030). The
// swap is for a directory that is the host's own: one it made, holding nothing but what the mesh
// put there. A directory that was there before the archive, or that something else has written
// into since, is the machine's: the archive's files are moved into it one by one, what the previous
// archive placed and this one does not is taken out, and everything else is left as it is. A file
// the archive would write over that the mesh did not put there refuses the archive before anything
// is moved — unless it already holds exactly the archive's bytes.
func replaceWith(body []byte, path, owner string, o ours) (store.Unpacked, int, error) {
	parent := filepath.Dir(path)
	madeParents, err := makeDirsSaying(parent, 0o755, owner)
	if err != nil {
		return store.Unpacked{}, 0, err
	}
	parents := joinParents(madeParents, o.parents)
	fresh := path + ".unpacking"
	replaced := path + ".replaced"
	// What an interrupted earlier attempt — or removal — left beside the directory.
	for _, leftover := range []string{fresh, replaced, path + ".removing"} {
		if err := os.RemoveAll(leftover); err != nil {
			return store.Unpacked{}, 0, err
		}
	}
	if err := os.Mkdir(fresh, 0o755); err != nil {
		return store.Unpacked{}, 0, err
	}
	written, err := unpack(body, fresh)
	if err == nil {
		err = ownAll(fresh, owner)
	}
	var files, dirs []string
	if err == nil {
		files, dirs, err = treeOf(fresh)
	}
	if err != nil {
		os.RemoveAll(fresh)
		return store.Unpacked{}, written, err
	}

	info, err := os.Lstat(path)
	existed := err == nil
	if err != nil && !os.IsNotExist(err) {
		os.RemoveAll(fresh)
		return store.Unpacked{}, written, err
	}
	if existed && !info.IsDir() {
		// Swapped, it would be deleted: a file at the path is nothing an archive put there.
		os.RemoveAll(fresh)
		return store.Unpacked{}, written, fmt.Errorf(
			"%s is there and is not a directory, and the mesh did not put it there; nothing was unpacked", path)
	}
	foreign := 0
	if existed && !o.all {
		if foreign, err = foreignIn(path, o.paths); err != nil {
			os.RemoveAll(fresh)
			return store.Unpacked{}, written, err
		}
	}
	if existed && (foreign > 0 || !(o.made || o.all)) {
		u, err := mergeInto(fresh, path, owner, files, dirs, o)
		os.RemoveAll(fresh)
		if err != nil {
			return store.Unpacked{}, written, err
		}
		u.Parents = parents
		return u, written, nil
	}

	hadOne := true
	if err := os.Rename(path, replaced); err != nil {
		if !os.IsNotExist(err) {
			os.RemoveAll(fresh)
			return store.Unpacked{}, written, err
		}
		hadOne = false
	}
	if err := os.Rename(fresh, path); err != nil {
		if hadOne {
			// Put the old tree back rather than leave nothing at the path.
			os.Rename(replaced, path)
		}
		os.RemoveAll(fresh)
		return store.Unpacked{}, written, err
	}
	// The directory is the host's own: it made it, now or before, and nothing else is in it.
	u := store.Unpacked{Files: files, Dirs: dirs, Made: true, Parents: parents}
	if hadOne {
		if err := os.RemoveAll(replaced); err != nil {
			return u, written, fmt.Errorf("%s is in place, and the tree it replaced could not be removed: %w", path, err)
		}
	}
	return u, written, nil
}

func fetch(ctx context.Context, source string) ([]byte, error) {
	request, err := http.NewRequestWithContext(ctx, http.MethodGet, source, nil)
	if err != nil {
		return nil, err
	}
	response, err := http.DefaultClient.Do(request)
	if err != nil {
		return nil, fmt.Errorf("cannot fetch %s: %w", source, err)
	}
	defer response.Body.Close()
	if response.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("%s answered %s", source, response.Status)
	}
	body, err := io.ReadAll(io.LimitReader(response.Body, maxArchive+1))
	if err != nil {
		return nil, err
	}
	if len(body) > maxArchive {
		return nil, fmt.Errorf("%s is larger than %d bytes, which is not an archive this host "+
			"will unpack", source, maxArchive)
	}
	return body, nil
}

// unpack writes a gzipped tar into a directory, refusing anything that would land outside it.
func unpack(body []byte, into string) (int, error) {
	zipped, err := gzip.NewReader(strings.NewReader(string(body)))
	if err != nil {
		return 0, fmt.Errorf("this is not a gzipped tar: %w", err)
	}
	defer zipped.Close()

	root, err := filepath.Abs(into)
	if err != nil {
		return 0, err
	}
	reader := tar.NewReader(zipped)
	written := 0
	for {
		header, err := reader.Next()
		if err == io.EOF {
			return written, nil
		}
		if err != nil {
			return written, err
		}

		// The oldest bug in unpacking: an entry named ../../etc/passwd writes outside the
		// directory it was unpacked into.
		//
		// **Refused, not sanitised.** Rewriting the name so it lands inside would put a file
		// somewhere nobody asked for and report success — the "looks configured and is not"
		// failure this host exists to prevent. An archive that names a path outside itself is
		// either hostile or broken, and both want the same answer.
		cleaned := filepath.Clean(header.Name)
		if filepath.IsAbs(cleaned) || cleaned == ".." || strings.HasPrefix(cleaned, ".."+string(os.PathSeparator)) {
			return written, fmt.Errorf(
				"%s names a path outside the archive; nothing more was unpacked", header.Name)
		}
		// And the same question asked of the result, because a name can be made to resolve
		// outside without saying so.
		target := filepath.Join(root, cleaned)
		if !strings.HasPrefix(target, root+string(os.PathSeparator)) && target != root {
			return written, fmt.Errorf(
				"%s would land outside %s; nothing more was unpacked", header.Name, into)
		}

		switch header.Typeflag {
		case tar.TypeDir:
			if err := os.MkdirAll(target, os.FileMode(header.Mode)&os.ModePerm); err != nil {
				return written, err
			}
		case tar.TypeReg:
			if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
				return written, err
			}
			file, err := os.OpenFile(target,
				os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.FileMode(header.Mode)&os.ModePerm)
			if err != nil {
				return written, err
			}
			if _, err := io.Copy(file, io.LimitReader(reader, maxArchive)); err != nil {
				file.Close()
				return written, err
			}
			if err := file.Close(); err != nil {
				return written, err
			}
			written++
		default:
			// Symlinks, devices, fifos. Refused rather than skipped: a theme that needed one
			// would silently arrive incomplete, and a device node in an archive is not something
			// to unpack quietly onto a machine.
			return written, fmt.Errorf(
				"%s is a %c, and this host unpacks only files and directories",
				header.Name, header.Typeflag)
		}
	}
}