mesh-host/internal/apply/apply.go

// Package apply makes a machine match a declaration.
//
// Three properties, each following a recorded decision, and each of them the difference
// between this and a script that writes files:
//
//   - A failed step fails the apply (novox/hq ADR 0008). Not "logs and continues": a partial
//     apply that reports success is the mesh's most expensive shape.
//   - Every applier READS BACK. Setting a value is not evidence the value took.
//   - What was applied is recorded after it works, never before (ADR 0035). A failed apply
//     leaves the machine in whatever state it reached, and nothing must claim otherwise.
package apply

import (
	"context"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/novox/mesh-host/internal/declaration"
	"github.com/novox/mesh-host/internal/store"
)

// Runner executes a command. The real one is used everywhere outside unit tests; behaviour
// against a real system is tested alongside rather than mocked (novox/hq ADR 0034).
type Runner func(ctx context.Context, name string, args ...string) (string, error)

// Outcome is what happened to one resource.
type Outcome struct {
	ID     string `json:"id"`
	Type   string `json:"type"`
	Target string `json:"target"`
	Action string `json:"action"` // created · updated · unchanged · removed
	Detail string `json:"detail,omitempty"`
}

// Report is what an apply did, in the order it did it.
type Report struct {
	Outcomes []Outcome `json:"outcomes"`
}

// Changed reports whether anything about the machine actually moved. An apply that changed
// nothing is the ordinary steady state, and saying so is not the same as saying it failed.
func (r Report) Changed() bool {
	for _, o := range r.Outcomes {
		if o.Action != "unchanged" {
			return true
		}
	}
	return false
}

// Error is a failure part-way through, carrying what had already been done.
//
// The outcomes matter as much as the message: the machine is in whatever state the apply
// reached, and the only honest thing to hand back is the list of what did happen.
type Error struct {
	Resource string
	Err      error
	Done     Report
}

func (e *Error) Error() string {
	return fmt.Sprintf("applying %q: %v\n\n%d resource(s) were applied before this and remain; "+
		"the machine is in whatever state that left it.", e.Resource, e.Err, len(e.Done.Outcomes))
}

func (e *Error) Unwrap() error { return e.Err }

// Apply makes the machine match the declaration, and returns what it did.
//
// Removal happens FIRST, and the order is not arbitrary. A resource that leaves a declaration
// while another arrives at the same path is an ordinary rename: removing afterwards would
// delete the file that had just been written. Removing first risks losing the old state if the
// apply then fails — a recovery concern, where the other is a correctness one.
func Apply(
	ctx context.Context,
	d *declaration.Declaration,
	known store.State,
	run Runner,
	log func(string),
) (Report, store.State, error) {
	if log == nil {
		log = func(string) {}
	}
	report := Report{}

	declared := map[string]bool{}
	for _, r := range d.Resources {
		declared[r.ID] = true
	}

	for _, orphan := range known.Orphans(declared) {
		if err := remove(ctx, orphan, run); err != nil {
			return report, known, &Error{Resource: orphan.ID, Err: err, Done: report}
		}
		known.Forget(orphan.ID)
		report.Outcomes = append(report.Outcomes, Outcome{
			ID: orphan.ID, Type: orphan.Type, Target: orphan.Target, Action: "removed",
			Detail: "no longer declared",
		})
		log(fmt.Sprintf("  removed %s (%s)", orphan.ID, orphan.Target))
	}

	for _, resource := range d.Resources {
		outcome, err := applyOne(ctx, resource, run)
		if err != nil {
			return report, known, &Error{Resource: resource.ID, Err: err, Done: report}
		}

		// Only now. The record follows the fact, never leads it.
		known.Record(store.Applied{
			ID: resource.ID, Type: string(resource.Type),
			Target: outcome.Target, AppliedAt: time.Now().UTC(),
		})
		report.Outcomes = append(report.Outcomes, outcome)
		if outcome.Action != "unchanged" {
			log(fmt.Sprintf("  %s %s (%s)", outcome.Action, outcome.ID, outcome.Target))
		}
	}
	return report, known, nil
}

func applyOne(ctx context.Context, r declaration.Resource, run Runner) (Outcome, error) {
	switch r.Type {
	case declaration.TypeDirectory:
		return applyDirectory(r)
	case declaration.TypeFile:
		return applyFile(r)
	case declaration.TypeService:
		return applyService(ctx, r, run)
	default:
		// Unreachable: the declaration refused this already. Present because "unreachable"
		// stops being true the moment someone adds a type and forgets this switch.
		return Outcome{}, fmt.Errorf("no applier for type %q", r.Type)
	}
}

func modeOf(spec string, fallback os.FileMode) (os.FileMode, error) {
	if spec == "" {
		return fallback, nil
	}
	parsed, err := strconv.ParseUint(spec, 8, 32)
	if err != nil {
		return 0, fmt.Errorf("mode %q: %w", spec, err)
	}
	return os.FileMode(parsed), nil
}

func applyDirectory(r declaration.Resource) (Outcome, error) {
	out := Outcome{ID: r.ID, Type: string(r.Type), Target: r.Path}
	mode, err := modeOf(r.Mode, 0o755)
	if err != nil {
		return out, err
	}

	before, err := os.Stat(r.Path)
	existed := err == nil
	if err != nil && !errors.Is(err, os.ErrNotExist) {
		return out, err
	}
	if existed && !before.IsDir() {
		return out, fmt.Errorf("%s exists and is not a directory", r.Path)
	}

	if !existed {
		if err := os.MkdirAll(r.Path, mode); err != nil {
			return out, err
		}
	}
	// Set explicitly even when it existed: MkdirAll applies the mode only on creation, and a
	// permission set at creation is not a permission maintained — a lesson this repository
	// already paid for once, with world-readable environment files.
	if err := os.Chmod(r.Path, mode); err != nil {
		return out, err
	}

	// Read back.
	after, err := os.Stat(r.Path)
	if err != nil {
		return out, fmt.Errorf("made %s and cannot stat it: %w", r.Path, err)
	}
	if !after.IsDir() {
		return out, fmt.Errorf("%s is not a directory after applying", r.Path)
	}
	if after.Mode().Perm() != mode.Perm() {
		return out, fmt.Errorf("%s is mode %o after setting %o", r.Path, after.Mode().Perm(), mode.Perm())
	}

	out.Action = "unchanged"
	if !existed {
		out.Action = "created"
	} else if before.Mode().Perm() != mode.Perm() {
		out.Action = "updated"
		out.Detail = fmt.Sprintf("mode %o to %o", before.Mode().Perm(), mode.Perm())
	}
	return out, nil
}

func applyFile(r declaration.Resource) (Outcome, error) {
	out := Outcome{ID: r.ID, Type: string(r.Type), Target: r.Path}
	mode, err := modeOf(r.Mode, 0o644)
	if err != nil {
		return out, err
	}

	existing, readErr := os.ReadFile(r.Path)
	existed := readErr == nil
	if readErr != nil && !errors.Is(readErr, os.ErrNotExist) {
		return out, readErr
	}

	var beforeMode os.FileMode
	if existed {
		if info, err := os.Stat(r.Path); err == nil {
			beforeMode = info.Mode().Perm()
		}
	}

	contentSame := existed && string(existing) == r.Content
	modeSame := existed && beforeMode == mode.Perm()

	if !contentSame {
		if err := os.MkdirAll(filepath.Dir(r.Path), 0o755); err != nil {
			return out, err
		}
		if err := writeAtomically(r.Path, []byte(r.Content), mode); err != nil {
			return out, err
		}
	} else if !modeSame {
		if err := os.Chmod(r.Path, mode); err != nil {
			return out, err
		}
	}

	// Read back — the file, not the call that wrote it.
	written, err := os.ReadFile(r.Path)
	if err != nil {
		return out, fmt.Errorf("wrote %s and cannot read it back: %w", r.Path, err)
	}
	if string(written) != r.Content {
		return out, fmt.Errorf("%s does not contain what was declared after writing it", r.Path)
	}
	info, err := os.Stat(r.Path)
	if err != nil {
		return out, err
	}
	if info.Mode().Perm() != mode.Perm() {
		return out, fmt.Errorf("%s is mode %o after setting %o", r.Path, info.Mode().Perm(), mode.Perm())
	}

	switch {
	case !existed:
		out.Action = "created"
	case !contentSame && !modeSame:
		out.Action = "updated"
		out.Detail = "content and mode"
	case !contentSame:
		out.Action = "updated"
		out.Detail = "content"
	case !modeSame:
		out.Action = "updated"
		out.Detail = fmt.Sprintf("mode %o to %o", beforeMode, mode.Perm())
	default:
		out.Action = "unchanged"
	}
	return out, nil
}

// writeAtomically writes through a temporary file in the same directory.
//
// A reader of a managed file must never see half of one. The mesh's own configuration is read
// by daemons that reload on change, so a torn write is a service reading a truncated config.
func writeAtomically(path string, content []byte, mode os.FileMode) error {
	tmp, err := os.CreateTemp(filepath.Dir(path), ".mesh-host-*")
	if err != nil {
		return err
	}
	defer os.Remove(tmp.Name())

	if _, err := tmp.Write(content); err != nil {
		tmp.Close()
		return err
	}
	if err := tmp.Sync(); err != nil {
		tmp.Close()
		return err
	}
	if err := tmp.Close(); err != nil {
		return err
	}
	if err := os.Chmod(tmp.Name(), mode); err != nil {
		return err
	}
	return os.Rename(tmp.Name(), path)
}

func applyService(ctx context.Context, r declaration.Resource, run Runner) (Outcome, error) {
	out := Outcome{ID: r.ID, Type: string(r.Type), Target: r.Unit}

	before, err := serviceState(ctx, r.Unit, run)
	if err != nil {
		return out, err
	}
	if before == r.State {
		out.Action = "unchanged"
		out.Detail = before
		return out, nil
	}

	verb := "start"
	if r.State == "stopped" {
		verb = "stop"
	}
	if _, err := run(ctx, "systemctl", verb, r.Unit); err != nil {
		return out, fmt.Errorf("%s %s: %w", verb, r.Unit, err)
	}

	// Read back. `systemctl start` returning zero says the transaction was accepted, not that
	// the unit is running — a unit that starts and immediately dies satisfies the command.
	after, err := serviceState(ctx, r.Unit, run)
	if err != nil {
		return out, err
	}
	if after != r.State {
		return out, fmt.Errorf("%s was asked to be %s and is %s", r.Unit, r.State, after)
	}

	out.Action = "updated"
	out.Detail = before + " to " + after
	return out, nil
}

// serviceState reads what the service manager says about a unit.
//
// Two traps here, and both were hit before this read what it now reads.
//
// The exit code is not the answer: `is-active` exits non-zero for every state except active —
// the same shape as the capability detector reading a degraded init as no init at all.
//
// And "inactive" does not mean stopped. `systemctl is-active` says "inactive" for a unit that
// DOES NOT EXIST exactly as it does for one that is installed and stopped. Declaring a unit
// stopped therefore reported success for a unit the host cannot manage at all — absence read
// as satisfaction, which is 04-ISSUES/007 wearing a different hat. LoadState is what separates
// them, so LoadState is what is read.
func serviceState(ctx context.Context, unit string, run Runner) (string, error) {
	out, _ := run(ctx, "systemctl", "show", unit,
		"--property=LoadState", "--property=ActiveState")

	var load, active string
	for _, line := range strings.Split(out, "\n") {
		key, value, found := strings.Cut(strings.TrimSpace(line), "=")
		if !found {
			continue
		}
		switch key {
		case "LoadState":
			load = value
		case "ActiveState":
			active = value
		}
	}

	switch load {
	case "":
		return "", fmt.Errorf("the service manager said nothing about %s", unit)
	case "not-found":
		return "", fmt.Errorf(
			"%s does not exist on this machine. A declaration naming a unit that is not "+
				"installed cannot be satisfied, and reporting it stopped would be reporting "+
				"absence as success", unit)
	case "masked":
		return "", fmt.Errorf("%s is masked, so its state cannot be declared", unit)
	case "error", "bad-setting":
		return "", fmt.Errorf("%s is installed but its unit file cannot be loaded (%s)", unit, load)
	}

	switch active {
	case "active", "activating", "reloading":
		return "running", nil
	case "inactive", "failed", "deactivating":
		return "stopped", nil
	default:
		return "", fmt.Errorf(
			"the service manager reports %s as %q, which is neither running nor stopped", unit, active)
	}
}

// remove undoes one resource the host applied and the declaration no longer names.
//
// Only ever called for something in the store, which is what bounds it: the host is
// authoritative over its own footprint and inert everywhere else (novox/hq ADR 0043).
func remove(ctx context.Context, a store.Applied, run Runner) error {
	switch declaration.Type(a.Type) {
	case declaration.TypeFile, declaration.TypeDirectory:
		if err := os.RemoveAll(a.Target); err != nil {
			return err
		}
		if _, err := os.Stat(a.Target); !errors.Is(err, os.ErrNotExist) {
			return fmt.Errorf("%s is still there after removing it", a.Target)
		}
		return nil

	case declaration.TypeService:
		// A unit that is no longer declared is stopped, not deleted. The host did not install
		// it and does not own the unit file — only the state it put the unit into.
		//
		// A unit that no longer EXISTS is already in the state removal is trying to reach, and
		// saying so matters: stopping it fails, and a failure here fails the whole apply. A
		// host holding a record of an uninstalled unit would then be unable to apply anything,
		// ever, with no way out but editing its state by hand. Removal is idempotent for the
		// same reason `os.RemoveAll` is.
		if _, err := serviceState(ctx, a.Target, run); err != nil {
			if strings.Contains(err.Error(), "does not exist on this machine") {
				return nil
			}
			return err
		}
		if _, err := run(ctx, "systemctl", "stop", a.Target); err != nil {
			return fmt.Errorf("stopping %s: %w", a.Target, err)
		}
		return nil

	default:
		return fmt.Errorf("no way to remove a %q", a.Type)
	}
}

// ExecRunner runs a real command, with stdin closed and output captured.
func ExecRunner(ctx context.Context, name string, args ...string) (string, error) {
	cmd := exec.CommandContext(ctx, name, args...)
	cmd.Stdin = nil
	out, err := cmd.Output()
	if err != nil {
		var exit *exec.ExitError
		if errors.As(err, &exit) {
			return string(out), fmt.Errorf("%s exited %d: %s",
				name, exit.ExitCode(), strings.TrimSpace(string(exit.Stderr)))
		}
		return string(out), fmt.Errorf("%s: %w", name, err)
	}
	return string(out), nil
}