mesh-host/internal/apply/hold.go

package apply

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	"github.com/novox/mesh-host/internal/declaration"
	"github.com/novox/mesh-host/internal/store"
	"github.com/novox/mesh-host/internal/system"
)

// Keep records the original of a file found on an adopted node, before anything else happens to
// it, and says where (novox/hq ADR 0100). It never overwrites an original it already kept: the
// first copy is the one that was there before the mesh.
type Keep func(path string, content []byte, mode os.FileMode) (string, error)

// KeepIn keeps originals under dir/kept, each named for the path it came from, readable by root
// alone — a predecessor's configuration may carry its credentials.
func KeepIn(dir string) Keep {
	return func(path string, content []byte, _ os.FileMode) (string, error) {
		sum := sha256.Sum256([]byte(path))
		kept := filepath.Join(dir, "kept",
			hex.EncodeToString(sum[:])[:16]+"-"+filepath.Base(path))
		if _, err := os.Lstat(kept); err == nil {
			return kept, nil
		}
		if err := os.MkdirAll(filepath.Dir(kept), 0o700); err != nil {
			return "", err
		}
		if err := writeAtomically(kept, content, 0o600); err != nil {
			return "", err
		}
		back, err := os.ReadFile(kept)
		if err != nil || string(back) != string(content) {
			return "", fmt.Errorf("kept the original of %s at %s and cannot read it back", path, kept)
		}
		return kept, nil
	}
}

// foundBefore is what an adopted apply finds on the machine before it changes anything: each
// directory, service unit and container mount source of an untaken module that is present with no
// record (novox/hq ADR 0103). Looked at first, because the apply itself makes such things — a
// file's parent directory, a unit file a module writes, a package that brings its unit — and what
// the mesh made in this apply was not found.
type foundBefore map[string]bool

func lookBefore(ctx context.Context, sys system.System, d *declaration.Declaration, known store.State,
	run Runner) foundBefore {
	seen := foundBefore{}
	if d.Adoption == nil {
		return seen
	}
	cri, asked := "", false
	for _, r := range d.Resources {
		if _, untaken := d.Adoption.UntakenModuleOf(r.Identity()); !untaken {
			continue
		}
		if _, held := known.HeldAt(r.Identity()); held {
			continue
		}
		switch res := r.(type) {
		case *declaration.Directory:
			if present(res.Path) && !recordedPath(known, res.Path) {
				seen["path:"+res.Path] = true
			}
		case *declaration.Service:
			if known.Recorded(string(declaration.TypeService), res.Unit) {
				continue
			}
			// A unit the service manager cannot find is not there; one it can read is, whatever
			// state it is in.
			if _, err := sys.ServiceState(ctx, run, res.Unit); err == nil {
				seen["unit:"+res.Unit] = true
			}
		case *declaration.Container:
			if known.Recorded(string(declaration.TypeContainer), res.Name) {
				continue
			}
			for _, v := range res.Volumes {
				src := mountSource(v)
				switch {
				case src == "":
				case strings.HasPrefix(src, "/"):
					if present(src) && !recordedPath(known, src) {
						seen["path:"+src] = true
					}
				default:
					if !asked {
						cri, _ = containerRuntime(ctx, run)
						asked = true
					}
					if cri == "" {
						continue
					}
					if _, err := run(ctx, cri, "volume", "inspect", src); err == nil {
						seen["volume:"+src] = true
					}
				}
			}
		}
	}
	return seen
}

func present(path string) bool {
	_, err := os.Lstat(path)
	return err == nil
}

// recordedPath is whether this host has a record of making something at a path.
func recordedPath(known store.State, path string) bool {
	for _, kind := range []declaration.Type{declaration.TypeDirectory, declaration.TypeFile,
		declaration.TypeArchive, declaration.TypeAccess} {
		if known.Recorded(string(kind), path) {
			return true
		}
	}
	return false
}

// mountSource is what a volume mapping mounts: a path on the machine, or a named volume. Empty for
// an anonymous volume, which mounts nothing that could already be there.
func mountSource(mapping string) string {
	src, _, ok := strings.Cut(mapping, ":")
	if !ok {
		return ""
	}
	return src
}

// runsIn is the container a resource runs inside, if any: an action's `in`, or a run-once step
// sharing a container's namespace.
func runsIn(r declaration.Resource) string {
	switch res := r.(type) {
	case *declaration.Action:
		return res.In
	case *declaration.Container:
		if res.RunOnce {
			if name, ok := strings.CutPrefix(res.Network, "container:"); ok {
				return name
			}
		}
	}
	return ""
}

// heldContainer is what is held under a container's name.
func heldContainer(known store.State, name string) (store.Held, bool) {
	for _, h := range known.Held {
		if h.Kind == string(declaration.TypeContainer) && h.Target == name {
			return h, true
		}
	}
	return store.Held{}, false
}

// holdOnAdopted decides whether a resource of an adopted node is held rather than applied, and
// holds it (novox/hq ADR 0100, ADR 0103). For a module not yet taken, what is present with no
// record is kept as it is: a file or a container under its name, a directory, a service's unit,
// and a container that would mount a path or a volume found there. An action or a run-once step
// run inside a held container is held with it. Once held, a resource stays held — changed or gone
// — until its module is taken, and it is never recorded as applied, so never removed as an orphan.
//
// Held is false for a resource to apply as usual. News is whether the hold is new or changed,
// which is what is worth a line in the log.
func holdOnAdopted(ctx context.Context, sys system.System, r declaration.Resource, d *declaration.Declaration,
	known *store.State, before foundBefore, run Runner, keep Keep, changed map[string]bool,
	now time.Time) (held, news bool, out Outcome, err error) {
	was, already := known.HeldAt(r.Identity())

	if in := runsIn(r); in != "" {
		if container, isHeld := heldContainer(*known, in); isHeld {
			module, untaken := d.Adoption.UntakenModuleOf(r.Identity())
			if !untaken {
				module = container.Module
			}
			h := was
			if !already {
				h = store.Held{ID: r.Identity(), Kind: string(r.Kind()), Target: r.Target(), Since: now}
			}
			h.Module, h.Why = module, "runs in "+in
			known.RecordHeld(h)
			out = begin(r)
			out.Action = "held"
			out.Detail = fmt.Sprintf("runs in %s, which is held as found; not run until %s is taken", in, module)
			return true, !already, out, nil
		}
	}

	module, untaken := d.Adoption.UntakenModuleOf(r.Identity())
	if !untaken {
		return false, false, out, nil
	}
	why := was.Why
	isFound := already
	if !already {
		switch res := r.(type) {
		case *declaration.File:
			if res.Into == "" {
				if isFound, err = found(ctx, r, run, *known); err != nil {
					return false, false, begin(r), err
				}
			}
		case *declaration.Container:
			if known.Recorded(string(declaration.TypeContainer), res.Name) {
				break
			}
			_, exists, err := inspectFound(ctx, res.Name, run)
			if err != nil {
				return false, false, begin(r), err
			}
			if exists {
				if isFound, err = found(ctx, r, run, *known); err != nil {
					return false, false, begin(r), err
				}
				break
			}
			// Not there under its name, and still it would share what was found: created, it
			// would mount the predecessor's data beside the predecessor's own container.
			for _, v := range res.Volumes {
				src := mountSource(v)
				key := "volume:" + src
				if strings.HasPrefix(src, "/") {
					key = "path:" + src
				}
				if src != "" && before[key] {
					isFound, why = true, "would mount "+src+", found on the machine"
					break
				}
			}
		case *declaration.Directory:
			isFound = before["path:"+res.Path]
		case *declaration.Service:
			isFound = before["unit:"+res.Unit]
		}
	}
	if !isFound {
		return false, false, out, nil
	}

	out, h, err := hold(ctx, sys, r, module, was, already, why, run, keep, now)
	if err != nil {
		return true, false, out, err
	}
	// A held service is not started, stopped, enabled or restarted — but a reload stops nothing,
	// so one the module names still happens (novox/hq ADR 0102, ADR 0103).
	if svc, ok := r.(*declaration.Service); ok && svc.State == "running" {
		if which := restartedBy(svc.ReloadOn, changed); len(which) > 0 {
			if state, err := sys.ServiceState(ctx, run, svc.Unit); err == nil && state == "running" {
				reloader, can := sys.(serviceReloader)
				if !can {
					return true, false, out, fmt.Errorf("%s must be reloaded for %s and this machine's "+
						"service manager cannot reload a unit", svc.Unit, strings.Join(which, ", "))
				}
				if err := reloader.ReloadService(ctx, run, svc.Unit); err != nil {
					return true, false, out, fmt.Errorf("reloading the held %s: %w", svc.Unit, err)
				}
				out.Detail += "; reloaded for " + strings.Join(which, ", ") + ", which stops nothing"
			}
		}
	}
	known.RecordHeld(h)
	return true, !already || h.Changed != was.Changed, out, nil
}

// found is whether a declared file or container is present on the machine with no record of this
// host making it (novox/hq ADR 0100). A container carrying the host's own spec label was made by
// a host, whatever this store says, so it is never found.
func found(ctx context.Context, r declaration.Resource, run Runner, known store.State) (bool, error) {
	if known.Recorded(string(r.Kind()), r.Target()) {
		return false, nil
	}
	switch res := r.(type) {
	case *declaration.File:
		_, err := os.Lstat(res.Path)
		if errors.Is(err, os.ErrNotExist) {
			return false, nil
		}
		return err == nil, err
	case *declaration.Container:
		seen, exists, err := inspectFound(ctx, res.Name, run)
		if err != nil || !exists {
			return false, err
		}
		return seen.spec == "", nil
	}
	return false, nil
}

type foundContainer struct {
	id      string
	running bool
	spec    string
}

// inspectFound reads a container by name the way a hold needs it: its id, whether it runs, and
// whether a host made it.
func inspectFound(ctx context.Context, name string, run Runner) (foundContainer, bool, error) {
	cri, err := containerRuntime(ctx, run)
	if err != nil {
		return foundContainer{}, false, fmt.Errorf("%w, so nothing can be said about %q", err, name)
	}
	out, err := run(ctx, cri, "inspect", "--format",
		"{{.Id}}\t{{.State.Running}}\t{{index .Config.Labels \""+specLabel+"\"}}", name)
	if err != nil {
		return foundContainer{}, false, nil
	}
	parts := strings.Split(strings.TrimSpace(out), "\t")
	for len(parts) < 3 {
		parts = append(parts, "")
	}
	spec := strings.TrimSpace(parts[2])
	if spec == "<no value>" {
		spec = ""
	}
	return foundContainer{id: strings.TrimSpace(parts[0]), running: parts[1] == "true", spec: spec}, true, nil
}

// hold keeps a found file or container as it is, and reports it — the first time by recording
// what was found, every time after by comparing against that. Nothing is reverted, restarted or
// created: a held target that disappears stays held and gone until its module is taken.
func hold(ctx context.Context, sys system.System, r declaration.Resource, module string, was store.Held,
	already bool, why string, run Runner, keep Keep, now time.Time) (Outcome, store.Held, error) {
	out := begin(r)
	h := was
	if !already {
		h = store.Held{ID: r.Identity(), Module: module, Kind: string(r.Kind()),
			Target: r.Target(), Since: now, Why: why}
	}
	h.Module = module
	detail := "found on the machine; kept until " + module + " is taken"

	var changed string
	switch res := r.(type) {
	case *declaration.File:
		info, err := os.Lstat(res.Path)
		switch {
		case errors.Is(err, os.ErrNotExist):
			if !already {
				return out, h, fmt.Errorf("%s was found and is gone before it could be kept", res.Path)
			}
			changed = "gone"
		case err != nil:
			return out, h, err
		default:
			content, err := os.ReadFile(res.Path)
			if err != nil {
				return out, h, fmt.Errorf("%s was found and cannot be read to keep it: %w", res.Path, err)
			}
			if !already {
				// The original first, before anything is recorded: a hold with no kept copy
				// would be a promise the host cannot keep.
				if keep == nil {
					return out, h, fmt.Errorf(
						"%s was found on this adopted node and this host has nowhere to keep its original", res.Path)
				}
				kept, err := keep(res.Path, content, info.Mode().Perm())
				if err != nil {
					return out, h, fmt.Errorf("keeping the original of %s: %w", res.Path, err)
				}
				h.Kept = kept
				h.Digest = digestOf(string(content))
				h.Mode = fmt.Sprintf("%04o", info.Mode().Perm())
				if st, ok := info.Sys().(*syscall.Stat_t); ok {
					h.Owner = fmt.Sprintf("%d:%d", st.Uid, st.Gid)
				}
			} else if digestOf(string(content)) != h.Digest {
				changed = "rewritten"
			}
		}
	case *declaration.Directory:
		info, err := os.Lstat(res.Path)
		switch {
		case errors.Is(err, os.ErrNotExist):
			if !already {
				return out, h, fmt.Errorf("%s was found and is gone before it could be held", res.Path)
			}
			changed = "gone"
		case err != nil:
			return out, h, err
		case !already:
			// Its mode and owner as found, which the mesh leaves: a database refuses to start
			// on a data directory whose mode changed.
			h.Mode = fmt.Sprintf("%04o", info.Mode().Perm())
			if st, ok := info.Sys().(*syscall.Stat_t); ok {
				h.Owner = fmt.Sprintf("%d:%d", st.Uid, st.Gid)
			}
		}
		detail = "found on the machine; its mode, owner and contents kept until " + module + " is taken"
	case *declaration.Service:
		state, err := sys.ServiceState(ctx, run, res.Unit)
		switch {
		case err != nil && !already:
			return out, h, fmt.Errorf("the unit %s was found and cannot be read to hold it: %w", res.Unit, err)
		case err != nil:
			changed = "gone"
		case !already:
			h.Running = state == "running"
		case h.Running && state != "running":
			changed = "stopped"
		}
		detail = "its unit was found on the machine; its state and whether it starts at boot are " +
			"kept until " + module + " is taken"
	case *declaration.Container:
		if h.Why != "" && h.Container == "" {
			// Held for what it would mount, never created: there is nothing of it to compare.
			detail = "not created: it " + h.Why + "; kept until " + module + " is taken"
			break
		}
		seen, exists, err := inspectFound(ctx, res.Name, run)
		if err != nil {
			return out, h, err
		}
		switch {
		case !exists && !already:
			return out, h, fmt.Errorf("container %s was found and is gone before it could be held", res.Name)
		case !already:
			h.Container, h.Running = seen.id, seen.running
		case !exists:
			changed = "gone"
		case seen.id != h.Container:
			changed = "replaced"
		case h.Running && !seen.running:
			changed = "stopped"
		}
	default:
		return out, h, fmt.Errorf("a %s cannot be held", r.Kind())
	}

	if changed != h.Changed {
		h.Changed = changed
		h.ChangedAt = now
		if changed == "" {
			h.ChangedAt = time.Time{}
		}
	}
	out.Action = "held"
	out.Detail = detail
	if h.Changed != "" {
		out.Detail += "; " + h.Changed + " by something other than the mesh since it was found, and not reverted"
	}
	return out, h, nil
}

// takenDetail is what an outcome says when a module's cutover replaced what was held for it.
func takenDetail(h store.Held) string {
	if h.Kind == string(declaration.TypeAction) || (h.Why != "" && h.Container == "") {
		return "taken: no longer held (" + h.Why + ")"
	}
	if h.Kept != "" {
		return "taken: replaced what was found; original kept at " + h.Kept
	}
	return "taken: replaced what was found"
}