mesh-host/internal/apply/apply.go

// Package apply makes a machine match a declaration.
//
// Three properties, each following a recorded decision, and each of them the difference
// between this and a script that writes files:
//
//   - A failed step fails the apply (novox/hq ADR 0008). Not "logs and continues": a partial
//     apply that reports success is the mesh's most expensive shape.
//   - Every applier READS BACK. Setting a value is not evidence the value took.
//   - What was applied is recorded after it works, never before (ADR 0035). A failed apply
//     leaves the machine in whatever state it reached, and nothing must claim otherwise.
package apply

import (
	"context"
	"crypto/sha256"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/novox/mesh-host/internal/declaration"
	"github.com/novox/mesh-host/internal/store"
)

// Runner executes a command. The real one is used everywhere outside unit tests; behaviour
// against a real system is tested alongside rather than mocked (novox/hq ADR 0034).
type Runner func(ctx context.Context, name string, args ...string) (string, error)

// Outcome is what happened to one resource.
type Outcome struct {
	ID     string `json:"id"`
	Type   string `json:"type"`
	Target string `json:"target"`
	Action string `json:"action"` // created · updated · unchanged · removed
	Detail string `json:"detail,omitempty"`
}

// Report is what an apply did, in the order it did it.
type Report struct {
	Outcomes []Outcome `json:"outcomes"`
}

// Changed reports whether anything about the machine actually moved. An apply that changed
// nothing is the ordinary steady state, and saying so is not the same as saying it failed.
func (r Report) Changed() bool {
	for _, o := range r.Outcomes {
		if o.Action != "unchanged" {
			return true
		}
	}
	return false
}

// Error is a failure part-way through, carrying what had already been done.
//
// The outcomes matter as much as the message: the machine is in whatever state the apply
// reached, and the only honest thing to hand back is the list of what did happen.
type Error struct {
	Resource string
	Err      error
	Done     Report
}

func (e *Error) Error() string {
	return fmt.Sprintf("applying %q: %v\n\n%d resource(s) were applied before this and remain; "+
		"the machine is in whatever state that left it.", e.Resource, e.Err, len(e.Done.Outcomes))
}

func (e *Error) Unwrap() error { return e.Err }

// Apply makes the machine match the declaration, and returns what it did.
//
// Removal happens FIRST, and the order is not arbitrary. A resource that leaves a declaration
// while another arrives at the same path is an ordinary rename: removing afterwards would
// delete the file that had just been written. Removing first risks losing the old state if the
// apply then fails — a recovery concern, where the other is a correctness one.
func Apply(
	ctx context.Context,
	d *declaration.Declaration,
	known store.State,
	run Runner,
	log func(string),
) (Report, store.State, error) {
	if log == nil {
		log = func(string) {}
	}
	report := Report{}

	declared := map[string]bool{}
	for _, r := range d.Resources {
		declared[r.ID] = true
	}

	for _, orphan := range known.Orphans(declared) {
		action, detail, err := remove(ctx, orphan, run)
		if err != nil {
			return report, known, &Error{Resource: orphan.ID, Err: err, Done: report}
		}
		known.Forget(orphan.ID)
		report.Outcomes = append(report.Outcomes, Outcome{
			ID: orphan.ID, Type: orphan.Type, Target: orphan.Target,
			Action: action, Detail: detail,
		})
		log(fmt.Sprintf("  %s %s (%s)", action, orphan.ID, orphan.Target))
	}

	for _, resource := range d.Resources {
		outcome, err := applyOne(ctx, resource, run)
		if err != nil {
			return report, known, &Error{Resource: resource.ID, Err: err, Done: report}
		}

		// Only now. The record follows the fact, never leads it.
		known.Record(store.Applied{
			ID: resource.ID, Type: string(resource.Type),
			Target: outcome.Target, AppliedAt: time.Now().UTC(),
		})
		report.Outcomes = append(report.Outcomes, outcome)
		if outcome.Action != "unchanged" {
			log(fmt.Sprintf("  %s %s (%s)", outcome.Action, outcome.ID, outcome.Target))
		}
	}
	return report, known, nil
}

func applyOne(ctx context.Context, r declaration.Resource, run Runner) (Outcome, error) {
	switch r.Type {
	case declaration.TypeDirectory:
		return applyDirectory(r)
	case declaration.TypeFile:
		return applyFile(r)
	case declaration.TypeService:
		return applyService(ctx, r, run)
	case declaration.TypePackage:
		return applyPackage(ctx, r, run)
	case declaration.TypeContainer:
		return applyContainer(ctx, r, run)
	case declaration.TypeAction:
		return applyAction(ctx, r, run)
	default:
		// Unreachable: the declaration refused this already. Present because "unreachable"
		// stops being true the moment someone adds a type and forgets this switch.
		return Outcome{}, fmt.Errorf("no applier for type %q", r.Type)
	}
}

func modeOf(spec string, fallback os.FileMode) (os.FileMode, error) {
	if spec == "" {
		return fallback, nil
	}
	parsed, err := strconv.ParseUint(spec, 8, 32)
	if err != nil {
		return 0, fmt.Errorf("mode %q: %w", spec, err)
	}
	return os.FileMode(parsed), nil
}

func applyDirectory(r declaration.Resource) (Outcome, error) {
	out := Outcome{ID: r.ID, Type: string(r.Type), Target: r.Path}
	mode, err := modeOf(r.Mode, 0o755)
	if err != nil {
		return out, err
	}

	before, err := os.Stat(r.Path)
	existed := err == nil
	if err != nil && !errors.Is(err, os.ErrNotExist) {
		return out, err
	}
	if existed && !before.IsDir() {
		return out, fmt.Errorf("%s exists and is not a directory", r.Path)
	}

	if !existed {
		if err := os.MkdirAll(r.Path, mode); err != nil {
			return out, err
		}
	}
	// Set explicitly even when it existed: MkdirAll applies the mode only on creation, and a
	// permission set at creation is not a permission maintained — a lesson this repository
	// already paid for once, with world-readable environment files.
	if err := os.Chmod(r.Path, mode); err != nil {
		return out, err
	}

	// Read back.
	after, err := os.Stat(r.Path)
	if err != nil {
		return out, fmt.Errorf("made %s and cannot stat it: %w", r.Path, err)
	}
	if !after.IsDir() {
		return out, fmt.Errorf("%s is not a directory after applying", r.Path)
	}
	if after.Mode().Perm() != mode.Perm() {
		return out, fmt.Errorf("%s is mode %o after setting %o", r.Path, after.Mode().Perm(), mode.Perm())
	}

	out.Action = "unchanged"
	if !existed {
		out.Action = "created"
	} else if before.Mode().Perm() != mode.Perm() {
		out.Action = "updated"
		out.Detail = fmt.Sprintf("mode %o to %o", before.Mode().Perm(), mode.Perm())
	}
	return out, nil
}

func applyFile(r declaration.Resource) (Outcome, error) {
	out := Outcome{ID: r.ID, Type: string(r.Type), Target: r.Path}
	mode, err := modeOf(r.Mode, 0o644)
	if err != nil {
		return out, err
	}

	existing, readErr := os.ReadFile(r.Path)
	existed := readErr == nil
	if readErr != nil && !errors.Is(readErr, os.ErrNotExist) {
		return out, readErr
	}

	var beforeMode os.FileMode
	if existed {
		if info, err := os.Stat(r.Path); err == nil {
			beforeMode = info.Mode().Perm()
		}
	}

	contentSame := existed && string(existing) == r.Content
	modeSame := existed && beforeMode == mode.Perm()

	if !contentSame {
		if err := os.MkdirAll(filepath.Dir(r.Path), 0o755); err != nil {
			return out, err
		}
		if err := writeAtomically(r.Path, []byte(r.Content), mode); err != nil {
			return out, err
		}
	} else if !modeSame {
		if err := os.Chmod(r.Path, mode); err != nil {
			return out, err
		}
	}

	// Read back — the file, not the call that wrote it.
	written, err := os.ReadFile(r.Path)
	if err != nil {
		return out, fmt.Errorf("wrote %s and cannot read it back: %w", r.Path, err)
	}
	if string(written) != r.Content {
		return out, fmt.Errorf("%s does not contain what was declared after writing it", r.Path)
	}
	info, err := os.Stat(r.Path)
	if err != nil {
		return out, err
	}
	if info.Mode().Perm() != mode.Perm() {
		return out, fmt.Errorf("%s is mode %o after setting %o", r.Path, info.Mode().Perm(), mode.Perm())
	}

	switch {
	case !existed:
		out.Action = "created"
	case !contentSame && !modeSame:
		out.Action = "updated"
		out.Detail = "content and mode"
	case !contentSame:
		out.Action = "updated"
		out.Detail = "content"
	case !modeSame:
		out.Action = "updated"
		out.Detail = fmt.Sprintf("mode %o to %o", beforeMode, mode.Perm())
	default:
		out.Action = "unchanged"
	}
	return out, nil
}

// writeAtomically writes through a temporary file in the same directory.
//
// A reader of a managed file must never see half of one. The mesh's own configuration is read
// by daemons that reload on change, so a torn write is a service reading a truncated config.
func writeAtomically(path string, content []byte, mode os.FileMode) error {
	tmp, err := os.CreateTemp(filepath.Dir(path), ".mesh-host-*")
	if err != nil {
		return err
	}
	defer os.Remove(tmp.Name())

	if _, err := tmp.Write(content); err != nil {
		tmp.Close()
		return err
	}
	if err := tmp.Sync(); err != nil {
		tmp.Close()
		return err
	}
	if err := tmp.Close(); err != nil {
		return err
	}
	if err := os.Chmod(tmp.Name(), mode); err != nil {
		return err
	}
	return os.Rename(tmp.Name(), path)
}

func applyService(ctx context.Context, r declaration.Resource, run Runner) (Outcome, error) {
	out := Outcome{ID: r.ID, Type: string(r.Type), Target: r.Unit}

	before, err := serviceState(ctx, r.Unit, run)
	if err != nil {
		return out, err
	}
	if before == r.State {
		out.Action = "unchanged"
		out.Detail = before
		return out, nil
	}

	verb := "start"
	if r.State == "stopped" {
		verb = "stop"
	}
	if _, err := run(ctx, "systemctl", verb, r.Unit); err != nil {
		return out, fmt.Errorf("%s %s: %w", verb, r.Unit, err)
	}

	// Read back. `systemctl start` returning zero says the transaction was accepted, not that
	// the unit is running — a unit that starts and immediately dies satisfies the command.
	after, err := serviceState(ctx, r.Unit, run)
	if err != nil {
		return out, err
	}
	if after != r.State {
		return out, fmt.Errorf("%s was asked to be %s and is %s", r.Unit, r.State, after)
	}

	out.Action = "updated"
	out.Detail = before + " to " + after
	return out, nil
}

// serviceState reads what the service manager says about a unit.
//
// Two traps here, and both were hit before this read what it now reads.
//
// The exit code is not the answer: `is-active` exits non-zero for every state except active —
// the same shape as the capability detector reading a degraded init as no init at all.
//
// And "inactive" does not mean stopped. `systemctl is-active` says "inactive" for a unit that
// DOES NOT EXIST exactly as it does for one that is installed and stopped. Declaring a unit
// stopped therefore reported success for a unit the host cannot manage at all — absence read
// as satisfaction, which is 04-ISSUES/007 wearing a different hat. LoadState is what separates
// them, so LoadState is what is read.
func serviceState(ctx context.Context, unit string, run Runner) (string, error) {
	out, _ := run(ctx, "systemctl", "show", unit,
		"--property=LoadState", "--property=ActiveState")

	var load, active string
	for _, line := range strings.Split(out, "\n") {
		key, value, found := strings.Cut(strings.TrimSpace(line), "=")
		if !found {
			continue
		}
		switch key {
		case "LoadState":
			load = value
		case "ActiveState":
			active = value
		}
	}

	switch load {
	case "":
		return "", fmt.Errorf("the service manager said nothing about %s", unit)
	case "not-found":
		return "", fmt.Errorf(
			"%s does not exist on this machine. A declaration naming a unit that is not "+
				"installed cannot be satisfied, and reporting it stopped would be reporting "+
				"absence as success", unit)
	case "masked":
		return "", fmt.Errorf("%s is masked, so its state cannot be declared", unit)
	case "error", "bad-setting":
		return "", fmt.Errorf("%s is installed but its unit file cannot be loaded (%s)", unit, load)
	}

	switch active {
	case "active", "activating", "reloading":
		return "running", nil
	case "inactive", "failed", "deactivating":
		return "stopped", nil
	default:
		return "", fmt.Errorf(
			"the service manager reports %s as %q, which is neither running nor stopped", unit, active)
	}
}

// remove undoes one resource the host applied and the declaration no longer names, and reports
// what it actually did.
//
// Only ever called for something in the store, which is what bounds it: the host is
// authoritative over its own footprint and inert everywhere else (novox/hq ADR 0043).
//
// It returns the action rather than assuming "removed", because for half the vocabulary the
// honest word is "forgotten". A host that reported a package removed when it left the package
// installed would be describing an effect it declined to have.
func remove(ctx context.Context, a store.Applied, run Runner) (string, string, error) {
	switch declaration.Type(a.Type) {
	case declaration.TypeFile, declaration.TypeDirectory:
		if err := os.RemoveAll(a.Target); err != nil {
			return "", "", err
		}
		if _, err := os.Stat(a.Target); !errors.Is(err, os.ErrNotExist) {
			return "", "", fmt.Errorf("%s is still there after removing it", a.Target)
		}
		return "removed", "no longer declared", nil

	case declaration.TypeService:
		// A unit that is no longer declared is stopped, not deleted. The host did not install
		// it and does not own the unit file — only the state it put the unit into.
		//
		// A unit that no longer EXISTS is already in the state removal is trying to reach, and
		// saying so matters: stopping it fails, and a failure here fails the whole apply. A
		// host holding a record of an uninstalled unit would then be unable to apply anything,
		// ever, with no way out but editing its state by hand. Removal is idempotent for the
		// same reason `os.RemoveAll` is.
		if _, err := serviceState(ctx, a.Target, run); err != nil {
			if strings.Contains(err.Error(), "does not exist on this machine") {
				return "forgotten", "the unit no longer exists", nil
			}
			return "", "", err
		}
		if _, err := run(ctx, "systemctl", "stop", a.Target); err != nil {
			return "", "", fmt.Errorf("stopping %s: %w", a.Target, err)
		}
		return "removed", "stopped; the unit file is not the host's to delete", nil

	case declaration.TypeContainer:
		// The host CREATED this one, so the host removes it. That is the line: it removes what
		// it made and leaves what it merely configured.
		if _, err := run(ctx, "docker", "rm", "-f", a.Target); err != nil {
			// Already gone is the state removal wants. Anything else is a real failure.
			if _, alive := containerState(ctx, a.Target, run); alive == nil {
				return "", "", fmt.Errorf("removing container %s: %w", a.Target, err)
			}
		}
		if _, err := containerState(ctx, a.Target, run); err == nil {
			return "", "", fmt.Errorf("container %s is still there after removing it", a.Target)
		}
		return "removed", "no longer declared", nil

	case declaration.TypePackage:
		// Deliberately not uninstalled, and this is a decision rather than an omission.
		//
		// The host cannot know what else on this machine needs the package. Uninstalling a
		// container runtime because a declaration changed would stop every container on the
		// node, and the machine may have had the package before the mesh ever saw it
		// (novox/hq research 012: adopted, not installed). Undeclaring says "the mesh no
		// longer requires this", which is not the same as "remove it".
		return "forgotten", "left installed; the host does not uninstall what it cannot know is unused", nil

	case declaration.TypeAction:
		// An action has no footprint the host can undo — it ran, and whatever it did belongs
		// to whatever it acted on.
		return "forgotten", "an action leaves nothing the host owns", nil

	default:
		return "", "", fmt.Errorf("no way to remove a %q", a.Type)
	}
}

// ExecRunner runs a real command, with stdin closed and output captured.
func ExecRunner(ctx context.Context, name string, args ...string) (string, error) {
	cmd := exec.CommandContext(ctx, name, args...)
	cmd.Stdin = nil
	out, err := cmd.Output()
	if err != nil {
		var exit *exec.ExitError
		if errors.As(err, &exit) {
			return string(out), fmt.Errorf("%s exited %d: %s",
				name, exit.ExitCode(), strings.TrimSpace(string(exit.Stderr)))
		}
		return string(out), fmt.Errorf("%s: %w", name, err)
	}
	return string(out), nil
}

// applyPackage installs a package the machine does not have.
//
// It never upgrades and never removes. "Present" is the whole of what a package resource
// asserts, because version is the package manager's business and the mesh does not have a
// second opinion about it (novox/hq ADR 0041 — the host depends on nothing, and that includes
// not becoming a second package manager).
func applyPackage(ctx context.Context, r declaration.Resource, run Runner) (Outcome, error) {
	out := Outcome{ID: r.ID, Type: string(r.Type), Target: r.Package}

	installed, err := packageInstalled(ctx, r.Package, run)
	if err != nil {
		return out, err
	}
	if installed {
		out.Action = "unchanged"
		out.Detail = "already installed"
		return out, nil
	}

	if _, err := run(ctx, "pacman", "-S", "--noconfirm", "--needed", r.Package); err != nil {
		return out, fmt.Errorf("installing %s: %w", r.Package, err)
	}

	// Read back. A package manager exiting zero says the transaction was accepted.
	installed, err = packageInstalled(ctx, r.Package, run)
	if err != nil {
		return out, err
	}
	if !installed {
		return out, fmt.Errorf(
			"%s was installed without error and the package database does not have it", r.Package)
	}

	out.Action = "created"
	return out, nil
}

// packageInstalled asks the package database, having first established that it answers.
//
// The two-step is the same trap `serviceState` documents. `pacman -Q name` exits non-zero for
// a package that is not installed AND for a package database that cannot be read, so believing
// the first answer would report a broken package manager as "nothing is installed" — absence
// read as fact. Proving the tool answers about something that certainly exists separates them.
func packageInstalled(ctx context.Context, name string, run Runner) (bool, error) {
	if _, err := run(ctx, "pacman", "-Q", "pacman"); err != nil {
		return false, fmt.Errorf(
			"the package database does not answer on this machine, so nothing can be said "+
				"about %q: %w", name, err)
	}
	if _, err := run(ctx, "pacman", "-Q", name); err != nil {
		return false, nil
	}
	return true, nil
}

// Labels the host puts on every container it creates.
//
// specLabel carries a digest of the declaration that made the container. It is what lets a
// reconcile answer "is this container the one the current declaration describes" without
// comparing every field the runtime reports — which cannot be done reliably, because a runtime
// normalises, defaults and reorders what it is given, and the differences that produces are
// indistinguishable from real drift.
const (
	specLabel = "mesh-host.spec"
	idLabel   = "mesh-host.id"
)

// containerSpec is the identity of a declared container: everything that, if changed, means
// the running container is no longer what was asked for.
func containerSpec(r declaration.Resource) string {
	keys := make([]string, 0, len(r.Env))
	for k := range r.Env {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	var b strings.Builder
	b.WriteString(r.Image + "\n" + r.Name + "\n")
	for _, k := range keys {
		b.WriteString("env " + k + "=" + r.Env[k] + "\n")
	}
	for _, p := range r.Ports {
		b.WriteString("port " + p + "\n")
	}
	for _, v := range r.Volumes {
		b.WriteString("volume " + v + "\n")
	}
	for _, a := range r.Args {
		b.WriteString("arg " + a + "\n")
	}
	return fmt.Sprintf("%x", sha256.Sum256([]byte(b.String())))
}

// containerState reports whether a container is running and which spec made it.
// The error means the container does not exist.
func containerState(ctx context.Context, name string, run Runner) (state struct {
	Running bool
	Spec    string
}, err error) {
	out, err := run(ctx, "docker", "inspect", "--format",
		"{{.State.Running}}\t{{index .Config.Labels \""+specLabel+"\"}}", name)
	if err != nil {
		return state, fmt.Errorf("no container named %s", name)
	}
	running, spec, _ := strings.Cut(strings.TrimSpace(out), "\t")
	state.Running = running == "true"
	state.Spec = strings.TrimSpace(spec)
	return state, nil
}

// applyContainer makes the declared container the one that is running.
//
// There is no "update" for a container: a container's configuration is fixed when it is
// created, so any change is a replacement. Saying that plainly is better than a partial
// in-place update that leaves the running thing half-declared.
func applyContainer(ctx context.Context, r declaration.Resource, run Runner) (Outcome, error) {
	out := Outcome{ID: r.ID, Type: string(r.Type), Target: r.Name}
	want := containerSpec(r)

	if _, err := run(ctx, "docker", "version", "--format", "{{.Server.Version}}"); err != nil {
		return out, fmt.Errorf(
			"the container runtime does not answer on this machine, so nothing can be said "+
				"about %q: %w", r.Name, err)
	}

	before, err := containerState(ctx, r.Name, run)
	existed := err == nil

	switch {
	case existed && before.Spec == want && before.Running:
		out.Action = "unchanged"
		return out, nil
	case existed:
		if _, err := run(ctx, "docker", "rm", "-f", r.Name); err != nil {
			return out, fmt.Errorf("replacing container %s: %w", r.Name, err)
		}
	}

	args := []string{"run", "--detach", "--name", r.Name, "--restart", "unless-stopped",
		"--label", specLabel + "=" + want, "--label", idLabel + "=" + r.ID}
	for _, k := range sortedKeys(r.Env) {
		args = append(args, "--env", k+"="+r.Env[k])
	}
	for _, p := range r.Ports {
		args = append(args, "--publish", p)
	}
	for _, v := range r.Volumes {
		args = append(args, "--volume", v)
	}
	args = append(args, r.Image)
	args = append(args, r.Args...)

	if _, err := run(ctx, "docker", args...); err != nil {
		return out, fmt.Errorf("starting container %s: %w", r.Name, err)
	}

	// Read back. `docker run --detach` returning an id says the container was created, not
	// that it is still running — a container whose entrypoint exits immediately satisfies the
	// command exactly as one that came up does.
	after, err := containerState(ctx, r.Name, run)
	if err != nil {
		return out, fmt.Errorf("started container %s and it is not there: %w", r.Name, err)
	}
	if !after.Running {
		return out, fmt.Errorf(
			"container %s was started and is not running. It exited; ask the runtime for its "+
				"logs", r.Name)
	}
	if after.Spec != want {
		return out, fmt.Errorf("container %s is not the one that was declared after creating it", r.Name)
	}

	out.Action = "created"
	if existed {
		out.Action = "updated"
		out.Detail = "replaced; a container's configuration is fixed when it is created"
	}
	return out, nil
}

func sortedKeys(m map[string]string) []string {
	keys := make([]string, 0, len(m))
	for k := range m {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	return keys
}

// applyAction runs something the bundle declared, and never learns what it means.
//
// Verify does double duty, and that is the design rather than a convenience: it is both the
// idempotency check and the read-back. Running it first is how the host knows whether there is
// anything to do — it does not know what a database is, so "is the database there" is a
// question only the declaration can ask. Running it again afterwards is how the host knows the
// command had the effect it claimed (novox/hq ADR 0047).
func applyAction(ctx context.Context, r declaration.Resource, run Runner) (Outcome, error) {
	out := Outcome{ID: r.ID, Type: string(r.Type), Target: strings.Join(r.Command, " ")}
	if r.In != "" {
		out.Target = "in " + r.In + ": " + out.Target
	}

	if _, err := runAction(ctx, r, r.Verify, run); err == nil {
		out.Action = "unchanged"
		out.Detail = "already true"
		return out, nil
	}

	if _, err := runAction(ctx, r, r.Command, run); err != nil {
		return out, fmt.Errorf("running the action: %w", err)
	}

	if _, err := runAction(ctx, r, r.Verify, run); err != nil {
		return out, fmt.Errorf(
			"the action ran without error and its own verify still fails: %w\n\n"+
				"The command reported success and the thing it was for did not happen, which "+
				"is exactly what verify exists to catch", err)
	}

	out.Action = "created"
	out.Detail = "verify was false and is now true"
	return out, nil
}

// runAction runs one of an action's command lines, on the machine or inside a container.
func runAction(ctx context.Context, r declaration.Resource, argv []string, run Runner) (string, error) {
	if len(argv) == 0 {
		return "", errors.New("no command")
	}
	if r.In != "" {
		return run(ctx, "docker", append([]string{"exec", r.In}, argv...)...)
	}
	return run(ctx, argv[0], argv[1:]...)
}