mesh-host/internal/apply/archive_unpacked.go

package apply

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io"
	"io/fs"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/novox/mesh-host/internal/store"
)

// What an archive put on the machine, and taking exactly that away (novox/hq issue 162).
//
// An archive unpacks many files into a directory the mesh did not necessarily make, so undeclaring
// one has a real question in it: remove what the archive put there, or remove the directory? The
// second deletes whatever else lives there — for the host's own versions directory, every other
// delivered version. So the host records what each archive unpacked and whether it made the
// directory, and removal takes away exactly that: the files the archive placed, then the
// directories the host made for them once they are empty. Never a file the archive did not place,
// never a directory that was there before, never one that still holds anything else. It is the
// rule every other kind follows: the mesh gives back what it found (ADR 0118), and data outlives
// the mesh that declared it (ADR 0030).

// ours is what of the tree at an archive's path the record says is the mesh's.
type ours struct {
	// all is a record from before the host kept what an archive unpacked: since the swap of issue
	// 220 the tree at the path was the archive and nothing else, so the whole of it is taken for
	// the mesh's, as the swap that follows has always taken it.
	all bool
	// paths are the files and directories the previous archive put there, relative to the path.
	paths map[string]bool
	files []string
	dirs  []string
	made  bool
	// parents are the directories above the path the host made to reach it, deepest first.
	parents []string
}

// oursFrom reads the record of the archive before this apply, for this path only: a record of the
// same archive at a path it has moved from says nothing about what is at the new one.
func oursFrom(previous store.Applied, path string) ours {
	if previous.Wrote == "" || previous.Target != path {
		return ours{}
	}
	u := previous.Unpacked
	if u == nil {
		return ours{all: true, made: true}
	}
	o := ours{paths: map[string]bool{}, files: u.Files, dirs: u.Dirs, made: u.Made, parents: u.Parents}
	for _, rel := range append(append([]string{}, u.Files...), u.Dirs...) {
		o.paths[rel] = true
	}
	return o
}

// ownershipProbe is what says whether an archive is still its owner's. The directory, when the host
// made it; one of the archive's own files when the directory was there before — that one is held as
// found (ADR 0182), so its owner is never the archive's to judge.
func ownershipProbe(path string, u *store.Unpacked) string {
	if u != nil && !u.Made && len(u.Files) > 0 {
		return filepath.Join(path, u.Files[0])
	}
	return path
}

// stillUnpacked is what an unchanged archive has on the machine. The record's, when it has one; on
// a record from before the host kept it, read from the archive's own bytes now — and the directory
// is taken for the host's only when it holds exactly the archive and nothing else, which is what the
// swap of issue 220 leaves. Otherwise the directory is kept for somebody's, and only the archive's
// files are recorded as its.
func stillUnpacked(body []byte, path string, recorded *store.Unpacked) (*store.Unpacked, error) {
	if recorded != nil {
		kept := *recorded
		return &kept, nil
	}
	files, dirs, err := listArchive(body)
	if err != nil {
		return nil, err
	}
	u := &store.Unpacked{Files: pathsOf(files)}
	if exactly, err := holdsExactly(path, files, dirs); err == nil && exactly {
		u.Dirs = pathsOf(dirs)
		u.Made = true
	}
	return u, nil
}

// listArchive reads what an archive holds without unpacking it: each file's digest by its path, and
// every directory, named or implied, relative to where it unpacks. Refused on the same terms as
// unpack, so a listing never names a path an unpack would not write.
func listArchive(body []byte) (map[string]string, map[string]bool, error) {
	zipped, err := gzip.NewReader(bytes.NewReader(body))
	if err != nil {
		return nil, nil, fmt.Errorf("this is not a gzipped tar: %w", err)
	}
	defer zipped.Close()
	files, dirs := map[string]string{}, map[string]bool{}
	reader := tar.NewReader(zipped)
	for {
		header, err := reader.Next()
		if err == io.EOF {
			return files, dirs, nil
		}
		if err != nil {
			return nil, nil, err
		}
		rel := filepath.Clean(header.Name)
		if rel == "." {
			continue
		}
		if !insideRel(rel) {
			return nil, nil, fmt.Errorf("%s names a path outside the archive", header.Name)
		}
		for d := filepath.Dir(rel); d != "."; d = filepath.Dir(d) {
			dirs[filepath.ToSlash(d)] = true
		}
		switch header.Typeflag {
		case tar.TypeDir:
			dirs[filepath.ToSlash(rel)] = true
		case tar.TypeReg:
			sum := sha256.New()
			if _, err := io.Copy(sum, io.LimitReader(reader, maxArchive)); err != nil {
				return nil, nil, err
			}
			files[filepath.ToSlash(rel)] = hex.EncodeToString(sum.Sum(nil))
		default:
			return nil, nil, fmt.Errorf("%s is a %c, and this host unpacks only files and directories",
				header.Name, header.Typeflag)
		}
	}
}

// holdsExactly is whether a directory holds the archive's files with the archive's bytes, its
// directories, and nothing else.
func holdsExactly(root string, files map[string]string, dirs map[string]bool) (bool, error) {
	seen := 0
	exact := true
	err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		rel, err := filepath.Rel(root, path)
		if err != nil {
			return err
		}
		if rel == "." {
			return nil
		}
		rel = filepath.ToSlash(rel)
		switch {
		case d.IsDir():
			if !dirs[rel] {
				exact = false
				return filepath.SkipAll
			}
		case d.Type().IsRegular():
			want, ok := files[rel]
			if !ok || digestOfFile(path) != want {
				exact = false
				return filepath.SkipAll
			}
			seen++
		default:
			exact = false
			return filepath.SkipAll
		}
		return nil
	})
	if err != nil {
		return false, err
	}
	return exact && seen == len(files), nil
}

func digestOfFile(path string) string {
	file, err := os.Open(path)
	if err != nil {
		return ""
	}
	defer file.Close()
	sum := sha256.New()
	if _, err := io.Copy(sum, file); err != nil {
		return ""
	}
	return hex.EncodeToString(sum.Sum(nil))
}

// treeOf is every file and directory under root, relative to it, slash-separated and sorted.
func treeOf(root string) (files, dirs []string, err error) {
	err = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		rel, err := filepath.Rel(root, path)
		if err != nil || rel == "." {
			return err
		}
		if d.IsDir() {
			dirs = append(dirs, filepath.ToSlash(rel))
		} else {
			files = append(files, filepath.ToSlash(rel))
		}
		return nil
	})
	sort.Strings(files)
	sort.Strings(dirs)
	if files == nil {
		files = []string{}
	}
	return files, dirs, err
}

// foreignIn counts what under root the mesh did not put there. A directory that is not the mesh's
// counts once, with everything in it.
func foreignIn(root string, mine map[string]bool) (int, error) {
	count := 0
	err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		rel, err := filepath.Rel(root, path)
		if err != nil || rel == "." {
			return err
		}
		if !mine[filepath.ToSlash(rel)] {
			count++
			if d.IsDir() {
				return filepath.SkipDir
			}
		}
		return nil
	})
	return count, err
}

// mergeInto moves a freshly unpacked archive into a directory that is not only the mesh's, one entry
// at a time, and takes out what the previous archive placed that this one does not. Everything the
// mesh did not put there stays as it is. Collisions are looked for before anything is moved, so a
// refused archive leaves the directory exactly as it was.
func mergeInto(fresh, path, owner string, files, dirs []string, o ours) (store.Unpacked, error) {
	var collisions []string
	for _, rel := range dirs {
		info, err := os.Lstat(filepath.Join(path, filepath.FromSlash(rel)))
		if err == nil && !info.IsDir() && !o.paths[rel] {
			collisions = append(collisions, rel)
		}
	}
	for _, rel := range files {
		dest := filepath.Join(path, filepath.FromSlash(rel))
		info, err := os.Lstat(dest)
		switch {
		case err != nil:
		case o.paths[rel] && !info.IsDir():
		case info.IsDir():
			if !o.paths[rel] {
				collisions = append(collisions, rel)
			} else if n, err := foreignIn(dest, o.paths); err != nil || n > 0 {
				collisions = append(collisions, rel)
			}
		case !info.Mode().IsRegular() ||
			digestOfFile(dest) != digestOfFile(filepath.Join(fresh, filepath.FromSlash(rel))):
			// Already holding exactly the archive's bytes is not a collision: it is what an
			// interrupted earlier apply of this same archive left, or the same file either way.
			collisions = append(collisions, rel)
		}
	}
	if len(collisions) > 0 {
		shown := collisions
		if len(shown) > 5 {
			shown = shown[:5]
		}
		return store.Unpacked{}, fmt.Errorf("%s already holds %d path(s) the archive would write over "+
			"and the mesh did not put there (%s); nothing was unpacked, and what is there is left as it "+
			"is (novox/hq issue 162)", path, len(collisions), strings.Join(shown, ", "))
	}

	var made []string
	for _, rel := range dirs {
		dest := filepath.Join(path, filepath.FromSlash(rel))
		info, err := os.Lstat(dest)
		if err == nil && info.IsDir() {
			if o.paths[rel] {
				made = append(made, rel)
			}
			continue
		}
		if err == nil {
			// The previous archive's file where this one has a directory.
			if err := os.Remove(dest); err != nil {
				return store.Unpacked{}, err
			}
		}
		mode := os.FileMode(0o755)
		if from, err := os.Stat(filepath.Join(fresh, filepath.FromSlash(rel))); err == nil {
			mode = from.Mode().Perm()
		}
		if err := os.Mkdir(dest, mode); err != nil {
			return store.Unpacked{}, err
		}
		if err := own(dest, owner); err != nil {
			return store.Unpacked{}, err
		}
		made = append(made, rel)
	}
	for _, rel := range files {
		dest := filepath.Join(path, filepath.FromSlash(rel))
		if info, err := os.Lstat(dest); err == nil && info.IsDir() {
			// The previous archive's directory where this one has a file, holding nothing else.
			if err := os.RemoveAll(dest); err != nil {
				return store.Unpacked{}, err
			}
		}
		if err := os.Rename(filepath.Join(fresh, filepath.FromSlash(rel)), dest); err != nil {
			return store.Unpacked{}, err
		}
	}

	// What the previous archive placed and this one does not.
	now := map[string]bool{}
	for _, rel := range append(append([]string{}, files...), dirs...) {
		now[rel] = true
	}
	for _, rel := range o.files {
		if now[rel] {
			continue
		}
		if err := os.Remove(filepath.Join(path, filepath.FromSlash(rel))); err != nil && !os.IsNotExist(err) {
			return store.Unpacked{}, err
		}
	}
	for _, rel := range deepestFirst(o.dirs) {
		if now[rel] {
			continue
		}
		dest := filepath.Join(path, filepath.FromSlash(rel))
		if err := os.Remove(dest); err != nil && !os.IsNotExist(err) {
			// Still holding something the mesh did not put there: kept, and still the host's to
			// take away once it is empty.
			made = append(made, rel)
		}
	}
	sort.Strings(made)
	return store.Unpacked{Files: files, Dirs: made, Made: o.made}, nil
}

// removeArchive is what undeclaring an archive does (novox/hq issue 162): exactly the files it
// unpacked, then the directories the host made for them once they are empty.
//
// **Never fatal.** An archive that could not be removed stopped the whole apply, on every apply
// after, until it was declared again — so every module with tools was un-unassignable, and a race
// between two pushes froze a machine against every other change. Whatever cannot be taken away is
// said, left in place, and forgotten, as a former target is (issue 194).
//
// **Removed whole when it is the host's own, and in one step.** A directory the host made that
// holds nothing but the archive is renamed aside and then removed: a reader — the runtime serving a
// module's tools from its bundle — sees the whole tree or none of it, never half, and a file it has
// open stays readable until it closes it. The runtime is told the module went by its own membership,
// not by the files disappearing.
func removeArchive(a store.Applied) (string, string, error) {
	// First, so the parents the host made are empty by the time they are looked at.
	if !store.IsFormer(a.ID) && a.Unpacked != nil {
		os.Remove(journalPath(filepath.Clean(a.Target)))
	}
	return removeArchiveFiles(a)
}

func removeArchiveFiles(a store.Applied) (string, string, error) {
	if store.IsFormer(a.ID) {
		// The version before is what a rollback starts (ADR 0141) and what a reader may still have
		// open; the launcher retires the host's own versions, not the apply (issue 194).
		return "forgotten", "a former target left in place: only an archive the declaration dropped is " +
			"taken away (novox/hq issues 162, 194)", nil
	}
	u := a.Unpacked
	if u == nil {
		return "forgotten", "left in place: recorded before the host kept what an archive unpacked, so " +
			"its files cannot be told from anything else there (novox/hq issue 162)", nil
	}
	root := filepath.Clean(a.Target)
	mine := map[string]bool{}
	for _, rel := range append(append([]string{}, u.Files...), u.Dirs...) {
		if !insideRel(filepath.FromSlash(rel)) {
			return "forgotten", fmt.Sprintf("left in place: its record names %q, which is not inside %s",
				rel, root), nil
		}
		mine[rel] = true
	}

	info, err := os.Lstat(root)
	if os.IsNotExist(err) {
		removeParents(root, u.Parents)
		return "forgotten", "no longer there", nil
	}
	if err != nil {
		return "forgotten", fmt.Sprintf("left in place: %v", err), nil
	}
	if !info.IsDir() {
		return "forgotten", "left in place: no longer a directory, so not what the archive was unpacked into", nil
	}
	foreign, err := foreignIn(root, mine)
	if err != nil {
		return "forgotten", fmt.Sprintf("left in place: cannot read what is in it: %v", err), nil
	}

	if u.Made && foreign == 0 {
		aside := root + ".removing"
		if err := os.RemoveAll(aside); err == nil {
			if err := os.Rename(root, aside); err == nil {
				if err := os.RemoveAll(aside); err != nil {
					return "forgotten", fmt.Sprintf("taken out of place, and what it unpacked could not be "+
						"removed from %s: %v — remove it by hand", aside, err), nil
				}
				removeParents(root, u.Parents)
				return "removed", fmt.Sprintf("no longer declared; the %d file(s) it unpacked, and the "+
					"directory the host made for them", len(u.Files)), nil
			}
		}
		// A rename that could not be made is taken file by file instead.
	}

	removed := 0
	var failed []string
	for _, rel := range u.Files {
		err := os.Remove(filepath.Join(root, filepath.FromSlash(rel)))
		switch {
		case err == nil:
			removed++
		case os.IsNotExist(err):
		default:
			failed = append(failed, err.Error())
		}
	}
	for _, rel := range deepestFirst(u.Dirs) {
		// Only once empty: what is still inside is somebody's.
		_ = os.Remove(filepath.Join(root, filepath.FromSlash(rel)))
	}
	detail := fmt.Sprintf("no longer declared; %d file(s) it unpacked removed", removed)
	switch {
	case u.Made && os.Remove(root) == nil:
		removeParents(root, u.Parents)
		detail += ", and the directory the host made for them"
	case u.Made:
		left, _ := os.ReadDir(root)
		detail += fmt.Sprintf("; the directory is kept: %d item(s) inside that the mesh did not put there",
			len(left))
	default:
		detail += "; the directory is kept: it was there before the archive"
	}
	if len(failed) > 0 {
		// Said and not fatal: fatal, the record would stay and fail the same way on every apply
		// after — the very wedge this removal exists to end.
		return "forgotten", detail + "; could not remove, and left in place: " + strings.Join(failed, "; "), nil
	}
	return "removed", detail, nil
}

// removeParents takes away the directories above an archive the host made to reach it, deepest
// first, each only once it is empty and only if it is above the archive's directory.
func removeParents(root string, parents []string) {
	for _, p := range parents {
		clean := filepath.Clean(p)
		if !filepath.IsAbs(clean) || !strings.HasPrefix(root, clean+string(os.PathSeparator)) {
			continue
		}
		_ = os.Remove(clean)
	}
}

// joinParents is the parents made now and those recorded before, deepest first, once each.
func joinParents(now, before []string) []string {
	seen := map[string]bool{}
	var out []string
	for _, p := range append(append([]string{}, now...), before...) {
		if !seen[p] {
			seen[p] = true
			out = append(out, p)
		}
	}
	sort.Slice(out, func(i, j int) bool { return len(out[i]) > len(out[j]) })
	return out
}

// insideRel is whether a relative path stays inside the directory it is relative to.
func insideRel(rel string) bool {
	clean := filepath.Clean(rel)
	return clean != "." && !filepath.IsAbs(clean) && clean != ".." &&
		!strings.HasPrefix(clean, ".."+string(os.PathSeparator))
}

func deepestFirst(rels []string) []string {
	out := append([]string{}, rels...)
	sort.Slice(out, func(i, j int) bool {
		return strings.Count(out[i], "/") > strings.Count(out[j], "/") ||
			(strings.Count(out[i], "/") == strings.Count(out[j], "/") && out[i] > out[j])
	})
	return out
}

func pathsOf[V any](m map[string]V) []string {
	out := make([]string, 0, len(m))
	for k := range m {
		out = append(out, k)
	}
	sort.Strings(out)
	return out
}