mesh-host/internal/apply/hold.go

package apply

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	"github.com/novox/mesh-host/internal/declaration"
	"github.com/novox/mesh-host/internal/store"
	"github.com/novox/mesh-host/internal/system"
)

// Keep records the original of a file found on an adopted node, before anything else happens to
// it, and says where (novox/hq ADR 0100). It never overwrites an original it already kept: the
// first copy is the one that was there before the mesh.
type Keep func(path string, content []byte, mode os.FileMode) (string, error)

// KeepIn keeps originals under dir/kept, each named for the path it came from AND for what was in
// it, readable by root alone — a predecessor's configuration may carry its credentials.
//
// **By content as well as path, because a path has more than one original.** A file held, let go
// when its module was unassigned, rewritten by the predecessor and found again is a second
// original; named by path alone the second copy was silently discarded while the report said it
// was kept (novox/hq ADR 0100). The same content at the same path is kept once.
func KeepIn(dir string) Keep {
	return func(path string, content []byte, _ os.FileMode) (string, error) {
		where := sha256.Sum256([]byte(path))
		what := sha256.Sum256(content)
		kept := filepath.Join(dir, "kept", hex.EncodeToString(where[:])[:12]+"-"+
			hex.EncodeToString(what[:])[:12]+"-"+filepath.Base(path))
		if _, err := os.Lstat(kept); err == nil {
			return kept, nil
		}
		if err := os.MkdirAll(filepath.Dir(kept), 0o700); err != nil {
			return "", err
		}
		if err := writeAtomically(kept, content, 0o600); err != nil {
			return "", err
		}
		back, err := os.ReadFile(kept)
		if err != nil || string(back) != string(content) {
			return "", fmt.Errorf("kept the original of %s at %s and cannot read it back", path, kept)
		}
		return kept, nil
	}
}

// foundBefore is what an adopted apply finds on the machine before it changes anything: each
// directory, service unit and container mount source of an untaken module that is present with no
// record (novox/hq ADR 0103). Looked at first, because the apply itself makes such things — a
// file's parent directory, a unit file a module writes, a package that brings its unit — and what
// the mesh made in this apply was not found.
type foundBefore struct {
	is map[string]bool
	// trouble is what could not be asked about, by the same key, so the resource that would need
	// the answer fails rather than proceeding as if the machine had nothing there.
	trouble map[string]string
}

func (f foundBefore) has(key string) bool { return f.is[key] }

// why is the reason a key could not be settled, and empty when there was none.
func (f foundBefore) why(key string) string { return f.trouble[key] }

func lookBefore(ctx context.Context, sys system.System, d *declaration.Declaration, known store.State,
	run Runner) foundBefore {
	seen := foundBefore{is: map[string]bool{}, trouble: map[string]string{}}
	if d.Adoption == nil {
		return seen
	}
	cri, asked := "", false
	for _, r := range d.Resources {
		if _, untaken := d.Adoption.UntakenModuleOf(r.Identity()); !untaken {
			continue
		}
		if _, held := known.HeldAt(r.Identity()); held {
			continue
		}
		switch res := r.(type) {
		case *declaration.Directory:
			if present(res.Path) && !recordedPath(known, res.Path) {
				seen.is["path:"+res.Path] = true
			}
		case *declaration.Archive:
			// Unpacking over it, and re-owning it recursively, would change the predecessor's
			// files.
			if present(res.Path) && !recordedPath(known, res.Path) {
				seen.is["path:"+res.Path] = true
			}
		case *declaration.Process:
			// Its unit would be written over and restarted.
			if !known.Recorded(string(declaration.TypeProcess), res.Name) &&
				present(filepath.Join(unitDir, res.Name+".service")) {
				seen.is["unit-file:"+res.Name] = true
			}
		case *declaration.User:
			// Its shell and groups would be changed.
			if !known.Recorded(string(declaration.TypeUser), res.Name) {
				if _, exists, err := system.LookUpUser(ctx, run, res.Name); err == nil && exists {
					seen.is["user:"+res.Name] = true
				}
			}
		case *declaration.Service:
			if known.Recorded(string(declaration.TypeService), res.Unit) {
				continue
			}
			// **Found is a unit somebody put on this machine, or one the machine uses.**
			//
			// Where it comes from first: a unit the service manager loads from outside /usr —
			// /etc/systemd/system or /run/systemd/system — was installed by an administrator, so
			// it is a predecessor's whatever state it is in, and one deliberately stopped and
			// disabled must stay that way (novox/hq ADR 0103).
			//
			// A unit a package ships, under /usr, is not held by its mere presence: the private
			// network's own wg-quick@mesh0 is an instance of a template the tunnel package ships,
			// nothing had ever run it, and holding it kept the private network from ever coming up
			// (found by the adoption bed). Such a unit is held only if the machine actually uses
			// it — running, or started at boot.
			state, err := sys.ServiceState(ctx, run, res.Unit)
			if err != nil {
				continue
			}
			if from, ok := sys.(unitFiles); ok {
				if path, err := from.ServiceUnitFile(ctx, run, res.Unit); err == nil && installedByHand(path) {
					seen.is["unit:"+res.Unit] = true
					continue
				}
			}
			boot, _ := sys.ServiceBoot(ctx, run, res.Unit)
			if state == "running" || boot == "enabled" {
				seen.is["unit:"+res.Unit] = true
			}
		case *declaration.Container:
			if known.Recorded(string(declaration.TypeContainer), res.Name) {
				continue
			}
			for _, v := range res.Volumes {
				src := mountSource(v)
				switch {
				case src == "":
				case strings.HasPrefix(src, "/"):
					if !systemPath(src) && present(src) && !recordedPath(known, src) {
						seen.is["path:"+src] = true
					}
				default:
					if !asked {
						cri, _ = containerRuntime(ctx, run)
						asked = true
					}
					if cri == "" {
						continue
					}
					if _, err := run(ctx, cri, "volume", "inspect", src); err == nil {
						seen.is["volume:"+src] = true
					} else if !absent(err) {
						seen.trouble["volume:"+src] = fmt.Sprintf(
							"the container runtime could not say whether the volume %s is here: %v", src, err)
					}
				}
			}
		}
	}
	return seen
}

// unitFiles is a service manager that can say where it loads a unit from.
type unitFiles interface {
	ServiceUnitFile(ctx context.Context, run Runner, unit string) (string, error)
}

// installedByHand is whether a unit file is one somebody put on this machine rather than one a
// package ships: anywhere but /usr, where distributions keep what they install.
func installedByHand(path string) bool {
	if path == "" {
		return false
	}
	return !strings.HasPrefix(filepath.Clean(path), "/usr/")
}

func present(path string) bool {
	_, err := os.Lstat(path)
	return err == nil
}

// recordedPath is whether this host has a record of MAKING something at a path — a directory it
// created, a file it wrote, an archive it unpacked. An access record is not one of those: it says
// the mesh set permissions on a path it does not own, which is exactly what it does to a path
// somebody else's software made, so a path it only has access for is still found (novox/hq ADR 0103).
func recordedPath(known store.State, path string) bool {
	for _, kind := range []declaration.Type{declaration.TypeDirectory, declaration.TypeFile,
		declaration.TypeArchive} {
		if known.Recorded(string(kind), path) {
			return true
		}
	}
	return false
}

// systemPath is whether a bind-mount source is the machine's own plumbing — the runtime's socket,
// the kernel's filesystems, the devices, the clock — which every machine has and no predecessor's
// data lives in. Mounting it shares nothing that was found.
func systemPath(src string) bool {
	clean := filepath.Clean(src)
	for _, exact := range []string{"/etc/localtime", "/etc/timezone", "/etc/hosts", "/etc/resolv.conf",
		"/etc/machine-id", "/etc/passwd", "/etc/group"} {
		if clean == exact {
			return true
		}
	}
	for _, under := range []string{"/run", "/var/run", "/sys", "/proc", "/dev", "/usr/share/zoneinfo",
		"/etc/ssl", "/etc/ca-certificates", "/etc/pki", "/lib/modules", "/usr/lib/modules"} {
		if clean == under || strings.HasPrefix(clean, under+"/") {
			return true
		}
	}
	return false
}

// mountSource is what a volume mapping mounts: a path on the machine, or a named volume. Empty for
// an anonymous volume, which mounts nothing that could already be there.
func mountSource(mapping string) string {
	src, _, ok := strings.Cut(mapping, ":")
	if !ok {
		return ""
	}
	return src
}

// runsIn is the container a resource runs inside, if any: an action's `in`, or a run-once step
// sharing a container's namespace. An action with no `in` runs on the machine itself and is not
// held for a container: it reaches nothing a predecessor holds by running there, and holding every
// action of an untaken module would stop a module preparing itself before its cutover.
func runsIn(r declaration.Resource) string {
	switch res := r.(type) {
	case *declaration.Action:
		return res.In
	case *declaration.Container:
		if res.RunOnce {
			if name, ok := strings.CutPrefix(res.Network, "container:"); ok {
				return name
			}
		}
	}
	return ""
}

// heldContainer is what is held under a container's name.
func heldContainer(known store.State, name string) (store.Held, bool) {
	for _, h := range known.Held {
		if h.Kind == string(declaration.TypeContainer) && h.Target == name {
			return h, true
		}
	}
	return store.Held{}, false
}

// holdOnAdopted decides whether a resource of an adopted node is held rather than applied, and
// holds it (novox/hq ADR 0100, ADR 0103). For a module not yet taken, what is present with no
// record is kept as it is: a file or a container under its name, a directory, a service's unit,
// and a container that would mount a path or a volume found there. An action or a run-once step
// run inside a held container is held with it. Once held, a resource stays held — changed or gone
// — until its module is taken, and it is never recorded as applied, so never removed as an orphan.
//
// Held is false for a resource to apply as usual. News is whether the hold is new or changed,
// which is what is worth a line in the log.
func holdOnAdopted(ctx context.Context, sys system.System, r declaration.Resource, d *declaration.Declaration,
	known *store.State, before foundBefore, run Runner, keep Keep, changed map[string]bool,
	now time.Time) (held, news bool, out Outcome, err error) {
	was, already := known.HeldAt(r.Identity())

	if in := runsIn(r); in != "" {
		if container, isHeld := heldContainer(*known, in); isHeld {
			module, untaken := d.Adoption.UntakenModuleOf(r.Identity())
			if !untaken {
				module = container.Module
			}
			h := was
			if !already {
				h = store.Held{ID: r.Identity(), Kind: string(r.Kind()), Target: r.Target(), Since: now}
			}
			h.Module, h.Why = module, "runs in "+in
			known.RecordHeld(h)
			out = begin(r)
			out.Action = "held"
			out.Detail = fmt.Sprintf("runs in %s, which is held as found; not run until %s is taken", in, module)
			return true, !already, out, nil
		}
	}

	// A file written into replaces nothing that was found, so it is never held (novox/hq ADR
	// 0102) — and a hold from when it was declared whole must not keep the mesh's keys out.
	if f, ok := r.(*declaration.File); ok && f.Into != "" {
		if already {
			known.Release(r.Identity())
		}
		return false, false, out, nil
	}

	module, untaken := d.Adoption.UntakenModuleOf(r.Identity())
	if !untaken {
		return false, false, out, nil
	}
	why := was.Why
	isFound := already
	if !already {
		switch res := r.(type) {
		case *declaration.File:
			if res.Into == "" {
				if isFound, err = found(ctx, r, run, *known); err != nil {
					return false, false, begin(r), err
				}
			}
		case *declaration.Container:
			if known.Recorded(string(declaration.TypeContainer), res.Name) {
				break
			}
			_, exists, err := inspectFound(ctx, res.Name, run)
			if err != nil {
				return false, false, begin(r), err
			}
			if exists {
				if isFound, err = found(ctx, r, run, *known); err != nil {
					return false, false, begin(r), err
				}
				break
			}
			// Not there under its name, and still it would share what was found: created, it
			// would mount the predecessor's data beside the predecessor's own container.
			for _, v := range res.Volumes {
				src := mountSource(v)
				key := "volume:" + src
				if strings.HasPrefix(src, "/") {
					key = "path:" + src
				}
				if src == "" {
					continue
				}
				if trouble := before.why(key); trouble != "" {
					return false, false, begin(r), fmt.Errorf(
						"%s, so it is not safe to create a container that would mount it", trouble)
				}
				if before.has(key) {
					isFound, why = true, "would mount "+src+", found on the machine"
					break
				}
			}
		case *declaration.Directory:
			isFound = before.has("path:" + res.Path)
		case *declaration.Archive:
			isFound = before.has("path:" + res.Path)
		case *declaration.Process:
			isFound = before.has("unit-file:" + res.Name)
		case *declaration.User:
			isFound = before.has("user:" + res.Name)
		case *declaration.Service:
			isFound = before.has("unit:" + res.Unit)
		}
	}
	if !isFound {
		return false, false, out, nil
	}

	out, h, err := hold(ctx, sys, r, module, was, already, why, run, keep, now)
	if err != nil {
		return true, false, out, err
	}
	// A held service is not started, stopped, enabled or restarted — but a reload stops nothing,
	// so one the module names still happens (novox/hq ADR 0102, ADR 0103).
	if svc, ok := r.(*declaration.Service); ok && svc.State == "running" {
		if which := restartedBy(svc.ReloadOn, changed); len(which) > 0 {
			if state, err := sys.ServiceState(ctx, run, svc.Unit); err == nil && state == "running" {
				reloader, can := sys.(serviceReloader)
				if !can {
					return true, false, out, fmt.Errorf("%s must be reloaded for %s and this machine's "+
						"service manager cannot reload a unit", svc.Unit, strings.Join(which, ", "))
				}
				if err := reloader.ReloadService(ctx, run, svc.Unit); err != nil {
					return true, false, out, fmt.Errorf("reloading the held %s: %w", svc.Unit, err)
				}
				out.Detail += "; reloaded for " + strings.Join(which, ", ") + ", which stops nothing"
			}
		}
	}
	known.RecordHeld(h)
	return true, !already || h.Changed != was.Changed, out, nil
}

// found is whether a declared file or container is present on the machine with no record of this
// host making it (novox/hq ADR 0100). A container carrying the host's own spec label was made by
// a host, whatever this store says, so it is never found.
func found(ctx context.Context, r declaration.Resource, run Runner, known store.State) (bool, error) {
	if known.Recorded(string(r.Kind()), r.Target()) {
		return false, nil
	}
	switch res := r.(type) {
	case *declaration.File:
		_, err := os.Lstat(res.Path)
		if errors.Is(err, os.ErrNotExist) {
			return false, nil
		}
		return err == nil, err
	case *declaration.Container:
		seen, exists, err := inspectFound(ctx, res.Name, run)
		if err != nil || !exists {
			return false, err
		}
		return seen.spec == "", nil
	}
	return false, nil
}

type foundContainer struct {
	id      string
	running bool
	spec    string
}

// inspectFound reads a container by name the way a hold needs it: its id, whether it runs, and
// whether a host made it.
func inspectFound(ctx context.Context, name string, run Runner) (foundContainer, bool, error) {
	cri, err := containerRuntime(ctx, run)
	if err != nil {
		return foundContainer{}, false, fmt.Errorf("%w, so nothing can be said about %q", err, name)
	}
	out, err := run(ctx, cri, "inspect", "--format",
		"{{.Id}}\t{{.State.Running}}\t{{index .Config.Labels \""+specLabel+"\"}}", name)
	if err != nil {
		if absent(err) {
			return foundContainer{}, false, nil
		}
		// **A runtime that could not answer is not a machine with nothing there.** Read as
		// absence, a daemon that is down or a permission denied would let the mesh create its own
		// container over a predecessor's — the one thing an adopted node must never do
		// (novox/hq ADR 0100).
		return foundContainer{}, false, fmt.Errorf(
			"the container runtime could not say whether %s is here, so it is not safe to make one: %w",
			name, err)
	}
	parts := strings.Split(strings.TrimSpace(out), "\t")
	for len(parts) < 3 {
		parts = append(parts, "")
	}
	spec := strings.TrimSpace(parts[2])
	if spec == "<no value>" {
		spec = ""
	}
	return foundContainer{id: strings.TrimSpace(parts[0]), running: parts[1] == "true", spec: spec}, true, nil
}

// absent is whether a runtime said the thing is not there, rather than failing to answer. Its own
// words: docker and podman both say "No such object", "No such container" or "No such volume".
func absent(err error) bool {
	said := strings.ToLower(err.Error())
	for _, missing := range []string{"no such object", "no such container", "no such volume",
		"no such image"} {
		if strings.Contains(said, missing) {
			return true
		}
	}
	return false
}

// hold keeps a found file or container as it is, and reports it — the first time by recording
// what was found, every time after by comparing against that. Nothing is reverted, restarted or
// created: a held target that disappears stays held and gone until its module is taken.
func hold(ctx context.Context, sys system.System, r declaration.Resource, module string, was store.Held,
	already bool, why string, run Runner, keep Keep, now time.Time) (Outcome, store.Held, error) {
	out := begin(r)
	h := was
	if !already {
		h = store.Held{ID: r.Identity(), Module: module, Kind: string(r.Kind()),
			Target: r.Target(), Since: now, Why: why}
	}
	h.Module = module
	detail := "found on the machine; kept until " + module + " is taken"

	var changed string
	switch res := r.(type) {
	case *declaration.File:
		info, err := os.Lstat(res.Path)
		switch {
		case errors.Is(err, os.ErrNotExist):
			if !already {
				return out, h, fmt.Errorf("%s was found and is gone before it could be kept", res.Path)
			}
			changed = "gone"
		case err != nil:
			return out, h, err
		default:
			content, err := os.ReadFile(res.Path)
			if err != nil {
				return out, h, fmt.Errorf("%s was found and cannot be read to keep it: %w", res.Path, err)
			}
			if !already {
				// The original first, before anything is recorded: a hold with no kept copy
				// would be a promise the host cannot keep.
				if keep == nil {
					return out, h, fmt.Errorf(
						"%s was found on this adopted node and this host has nowhere to keep its original", res.Path)
				}
				kept, err := keep(res.Path, content, info.Mode().Perm())
				if err != nil {
					return out, h, fmt.Errorf("keeping the original of %s: %w", res.Path, err)
				}
				h.Kept = kept
				h.Digest = digestOf(string(content))
				h.Mode = fmt.Sprintf("%04o", info.Mode().Perm())
				if st, ok := info.Sys().(*syscall.Stat_t); ok {
					h.Owner = fmt.Sprintf("%d:%d", st.Uid, st.Gid)
				}
			} else if digestOf(string(content)) != h.Digest {
				changed = "rewritten"
			}
		}
	case *declaration.Directory:
		info, err := os.Lstat(res.Path)
		switch {
		case errors.Is(err, os.ErrNotExist):
			if !already {
				return out, h, fmt.Errorf("%s was found and is gone before it could be held", res.Path)
			}
			changed = "gone"
		case err != nil:
			return out, h, err
		case !already:
			// Its mode and owner as found, which the mesh leaves: a database refuses to start
			// on a data directory whose mode changed.
			h.Mode = fmt.Sprintf("%04o", info.Mode().Perm())
			if st, ok := info.Sys().(*syscall.Stat_t); ok {
				h.Owner = fmt.Sprintf("%d:%d", st.Uid, st.Gid)
			}
		}
		detail = "found on the machine; its mode, owner and contents kept until " + module + " is taken"
	case *declaration.Archive:
		if _, err := os.Lstat(res.Path); errors.Is(err, os.ErrNotExist) {
			if !already {
				return out, h, fmt.Errorf("%s was found and is gone before it could be held", res.Path)
			}
			changed = "gone"
		} else if err != nil {
			return out, h, err
		}
		detail = "something is already at " + res.Path + "; nothing unpacked over it or re-owned until " +
			module + " is taken"
	case *declaration.Process:
		unit := filepath.Join(unitDir, res.Name+".service")
		if !present(unit) {
			if !already {
				return out, h, fmt.Errorf("%s was found and is gone before it could be held", unit)
			}
			changed = "gone"
		}
		detail = "its unit " + unit + " was found on the machine; not written over or restarted until " +
			module + " is taken"
	case *declaration.User:
		_, exists, err := system.LookUpUser(ctx, run, res.Name)
		switch {
		case err != nil:
			return out, h, err
		case !exists && !already:
			return out, h, fmt.Errorf("the user %s was found and is gone before it could be held", res.Name)
		case !exists:
			changed = "gone"
		}
		detail = "the user was found on the machine; its shell and groups are kept until " + module + " is taken"
	case *declaration.Service:
		state, err := sys.ServiceState(ctx, run, res.Unit)
		switch {
		case err != nil && !already:
			return out, h, fmt.Errorf("the unit %s was found and cannot be read to hold it: %w", res.Unit, err)
		case err != nil:
			changed = "gone"
		case !already:
			h.Running = state == "running"
		case h.Running && state != "running":
			changed = "stopped"
		}
		detail = "its unit was found on the machine; its state and whether it starts at boot are " +
			"kept until " + module + " is taken"
	case *declaration.Container:
		if h.Why != "" && h.Container == "" {
			// Held for what it would mount, never created: there is nothing of it to compare.
			detail = "not created: it " + h.Why + "; kept until " + module + " is taken"
			break
		}
		seen, exists, err := inspectFound(ctx, res.Name, run)
		if err != nil {
			return out, h, err
		}
		switch {
		case !exists && !already:
			return out, h, fmt.Errorf("container %s was found and is gone before it could be held", res.Name)
		case !already:
			h.Container, h.Running = seen.id, seen.running
		case !exists:
			changed = "gone"
		case seen.id != h.Container:
			changed = "replaced"
		case h.Running && !seen.running:
			changed = "stopped"
		}
	default:
		return out, h, fmt.Errorf("a %s cannot be held", r.Kind())
	}

	if changed != h.Changed {
		h.Changed = changed
		h.ChangedAt = now
		if changed == "" {
			h.ChangedAt = time.Time{}
		}
	}
	out.Action = "held"
	out.Detail = detail
	if h.Changed != "" {
		out.Detail += "; " + h.Changed + " by something other than the mesh since it was found, and not reverted"
	}
	return out, h, nil
}

// takenDetail is what an outcome says when a module's cutover replaced what was held for it.
func takenDetail(h store.Held) string {
	if h.Kind == string(declaration.TypeAction) || (h.Why != "" && h.Container == "") {
		return "taken: no longer held (" + h.Why + ")"
	}
	if h.Kept != "" {
		return "taken: replaced what was found; original kept at " + h.Kept
	}
	return "taken: replaced what was found"
}