mesh-host/internal/apply/apply_test.go

package apply

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/novox/mesh-host/internal/declaration"
	"github.com/novox/mesh-host/internal/store"
	"github.com/novox/mesh-host/internal/system"
)

// Each test names the decision it defends (novox/hq ADR 0017).

func parse(t *testing.T, raw string) *declaration.Declaration {
	t.Helper()
	d, err := declaration.Parse([]byte(raw))
	if err != nil {
		t.Fatalf("fixture is not a valid declaration: %v", err)
	}
	return d
}

// noServices refuses to run anything. Used where a test declares no services, so that a test
// which accidentally reaches the service manager fails loudly instead of passing quietly.
func noServices(context.Context, string, ...string) (string, error) {
	return "", errors.New("this test declares no services and should not have run a command")
}

func TestApplyingTwiceChangesNothingTheSecondTime(t *testing.T) {
	// Idempotence is what makes an apply safe to run on a schedule. Without it, a host that
	// reconciles every few minutes rewrites files forever and every reader sees churn.
	dir := t.TempDir()
	d := parse(t, `{"declaration":1,"resources":[
	  {"id":"d","type":"directory","path":"`+dir+`/etc","mode":"0755"},
	  {"id":"f","type":"file","path":"`+dir+`/etc/a.conf","content":"hello\n","mode":"0640"}
	]}`)

	first, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	if !first.Changed() {
		t.Fatal("the first apply on an empty machine changed nothing")
	}

	second, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	if second.Changed() {
		t.Errorf("the second apply changed something: %+v", second.Outcomes)
	}
}

func TestADriftedMachineIsReturned(t *testing.T) {
	// The other half of idempotence, and the half that matters: converging is not "do nothing
	// if the state file says it was done". The machine is read, not the record.
	dir := t.TempDir()
	path := filepath.Join(dir, "a.conf")
	d := parse(t, `{"declaration":1,"resources":[
	  {"id":"f","type":"file","path":"`+path+`","content":"correct\n","mode":"0644"}
	]}`)

	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(path, []byte("someone edited this\n"), 0o644); err != nil {
		t.Fatal(err)
	}

	report, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	if !report.Changed() {
		t.Fatal("a drifted file was left drifted")
	}
	got, _ := os.ReadFile(path)
	if string(got) != "correct\n" {
		t.Errorf("the file was not returned: %q", got)
	}
}

func TestADroppedResourceIsRemoved(t *testing.T) {
	// novox/hq ADR 0005: the host removes what it previously applied and is no longer
	// declared. Removing a line from a declaration is an act with an effect.
	dir := t.TempDir()
	keep := filepath.Join(dir, "keep.conf")
	drop := filepath.Join(dir, "drop.conf")

	both := parse(t, `{"declaration":1,"resources":[
	  {"id":"keep","type":"file","path":"`+keep+`","content":"a\n"},
	  {"id":"drop","type":"file","path":"`+drop+`","content":"b\n"}
	]}`)
	_, state, err := Apply(context.Background(), archHost(t), both, store.State{}, store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}

	one := parse(t, `{"declaration":1,"resources":[
	  {"id":"keep","type":"file","path":"`+keep+`","content":"a\n"}
	]}`)
	report, state, err := Apply(context.Background(), archHost(t), one, state, store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}

	if _, err := os.Stat(drop); !errors.Is(err, os.ErrNotExist) {
		t.Error("a resource dropped from the declaration was left on the machine")
	}
	if _, err := os.Stat(keep); err != nil {
		t.Error("a declared resource was removed")
	}
	if _, still := state.Find("drop"); still {
		t.Error("the host still believes it owns what it removed")
	}
	if report.Outcomes[0].Action != "removed" {
		t.Errorf("removal is not reported first: %+v", report.Outcomes)
	}
}

func TestNothingTheHostDidNotCreateIsTouched(t *testing.T) {
	// The boundary the whole removal rule turns on. A machine has things on it the mesh did
	// not put there, and a converger that treats "not declared" as "must not exist" deletes
	// them. Authoritative over its own footprint; inert everywhere else.
	dir := t.TempDir()
	stranger := filepath.Join(dir, "not-ours.conf")
	if err := os.WriteFile(stranger, []byte("someone else's\n"), 0o644); err != nil {
		t.Fatal(err)
	}

	d := parse(t, `{"declaration":1,"resources":[
	  {"id":"ours","type":"file","path":"`+filepath.Join(dir, "ours.conf")+`","content":"a\n"}
	]}`)
	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil); err != nil {
		t.Fatal(err)
	}

	got, err := os.ReadFile(stranger)
	if err != nil || string(got) != "someone else's\n" {
		t.Error("a file the host did not create was removed or changed")
	}
}

func TestARenameToTheSamePathDoesNotDeleteTheNewFile(t *testing.T) {
	// Why removal happens FIRST. A resource leaving a declaration while another arrives at the
	// same path is an ordinary rename; removing afterwards would delete the file just written.
	dir := t.TempDir()
	path := filepath.Join(dir, "shared.conf")

	before := parse(t, `{"declaration":1,"resources":[
	  {"id":"old","type":"file","path":"`+path+`","content":"old\n"}
	]}`)
	_, state, err := Apply(context.Background(), archHost(t), before, store.State{}, store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}

	after := parse(t, `{"declaration":1,"resources":[
	  {"id":"new","type":"file","path":"`+path+`","content":"new\n"}
	]}`)
	if _, _, err := Apply(context.Background(), archHost(t), after, state, store.OriginCarried, noServices, nil, nil); err != nil {
		t.Fatal(err)
	}

	got, err := os.ReadFile(path)
	if err != nil {
		t.Fatalf("the renamed resource is gone: %v", err)
	}
	if string(got) != "new\n" {
		t.Errorf("content is %q, want the new one", got)
	}
}

func TestAFailedStepFailsTheApplyAndTheRestIsStillAttempted(t *testing.T) {
	// The apply fails, names the resource, and carries what did happen — because the machine is
	// in whatever state the apply reached and the only honest thing to hand back is that list.
	//
	// **And everything is attempted.** It used to stop at the first failure, which made one
	// broken resource hold the whole machine hostage: a module declaring a package that does not
	// exist meant every module after it was never applied, for ever
	// (novox/hq 04-ISSUES/011). The case for stopping was that a later resource may depend on an
	// earlier one — and it still may, and it then fails its own check and is reported, which is
	// more information than not attempting it.
	dir := t.TempDir()
	blocker := filepath.Join(dir, "blocker")
	if err := os.WriteFile(blocker, []byte("i am a file\n"), 0o644); err != nil {
		t.Fatal(err)
	}

	d := parse(t, `{"declaration":1,"resources":[
	  {"id":"fine","type":"file","path":"`+filepath.Join(dir, "fine.conf")+`","content":"a\n"},
	  {"id":"doomed","type":"directory","path":"`+blocker+`"},
	  {"id":"after","type":"file","path":"`+filepath.Join(dir, "after.conf")+`","content":"b\n"}
	]}`)

	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil)
	if err == nil {
		t.Fatal("an impossible resource did not fail the apply")
	}

	var applyErr *Error
	if !errors.As(err, &applyErr) {
		t.Fatalf("expected an apply error, got %T", err)
	}
	if applyErr.Resource != "doomed" {
		t.Errorf("the failure names %q, not the resource that failed", applyErr.Resource)
	}
	// Everything that worked is in the report, before and after the failure.
	if len(applyErr.Done.Outcomes) != 2 {
		t.Errorf("the error does not carry what was applied: %+v", applyErr.Done.Outcomes)
	}
	if _, err := os.Stat(filepath.Join(dir, "after.conf")); err != nil {
		t.Error("a resource after the failing one was never attempted, so one broken module " +
			"still blocks every module after it")
	}
}

func TestEveryFailureIsCountedNotJustTheFirst(t *testing.T) {
	// "One thing failed" and "eleven things failed" are different machines, and the first line is
	// what somebody reads.
	dir := t.TempDir()
	for _, name := range []string{"one", "two"} {
		if err := os.WriteFile(filepath.Join(dir, name), []byte("a file\n"), 0o644); err != nil {
			t.Fatal(err)
		}
	}
	d := parse(t, `{"declaration":1,"resources":[
	  {"id":"first","type":"directory","path":"`+filepath.Join(dir, "one")+`"},
	  {"id":"second","type":"directory","path":"`+filepath.Join(dir, "two")+`"}
	]}`)

	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil)
	if err == nil {
		t.Fatal("two impossible resources did not fail the apply")
	}
	var applyErr *Error
	if !errors.As(err, &applyErr) {
		t.Fatalf("got %T", err)
	}
	if applyErr.Others != 1 {
		t.Errorf("the failure says %d others also failed, and one did", applyErr.Others)
	}
	if !strings.Contains(err.Error(), "one other resource also failed") {
		t.Errorf("the message does not say others failed: %v", err)
	}
}

func TestNothingIsRecordedUntilItWorked(t *testing.T) {
	// novox/hq ADR 0018. A record written before the fact restates the request in a new place
	// and inherits none of the authority of having happened.
	dir := t.TempDir()
	blocker := filepath.Join(dir, "blocker")
	if err := os.WriteFile(blocker, []byte("x\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	d := parse(t, `{"declaration":1,"resources":[
	  {"id":"doomed","type":"directory","path":"`+blocker+`"}
	]}`)

	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil)
	if err == nil {
		t.Fatal("expected a failure")
	}
	if _, claimed := state.Find("doomed"); claimed {
		t.Error("the host recorded owning something it failed to apply")
	}
}

func TestAModeIsMaintainedNotJustSet(t *testing.T) {
	// A permission set at creation is not a permission maintained — this repository has
	// already paid for that once, with generated files left world-readable because the mode
	// applied only when the file was first written.
	dir := t.TempDir()
	path := filepath.Join(dir, "secret.conf")
	d := parse(t, `{"declaration":1,"resources":[
	  {"id":"f","type":"file","path":"`+path+`","content":"s\n","mode":"0600"}
	]}`)

	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	if err := os.Chmod(path, 0o666); err != nil {
		t.Fatal(err)
	}

	report, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	info, _ := os.Stat(path)
	if info.Mode().Perm() != 0o600 {
		t.Errorf("mode is %o after reconciling, want 0600", info.Mode().Perm())
	}
	if !report.Changed() {
		t.Error("a mode that had drifted was reported as unchanged")
	}
}

func TestAServiceIsReadBackNotAssumed(t *testing.T) {
	// `systemctl start` returning zero says the transaction was accepted, not that the unit is
	// running. A unit that starts and immediately dies satisfies the command.
	started := false
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		if args[0] == "show" {
			if started {
				return "LoadState=loaded\nActiveState=failed\n", nil // started, then died
			}
			return "LoadState=loaded\nActiveState=inactive\n", nil
		}
		started = true
		return "", nil // `systemctl start` succeeds
	}

	d := parse(t, `{"declaration":1,"resources":[
	  {"id":"s","type":"service","unit":"doomed.service","state":"running"}
	]}`)
	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
	if err == nil {
		t.Fatal("a service that died immediately was reported as running")
	}
	if !strings.Contains(err.Error(), "asked to be running and is stopped") {
		t.Errorf("the failure does not say what was observed: %v", err)
	}
}

func TestAnUnknownServiceStateIsRefusedNotGuessed(t *testing.T) {
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		return "LoadState=loaded\nActiveState=reticent\n", nil
	}
	d := parse(t, `{"declaration":1,"resources":[
	  {"id":"s","type":"service","unit":"odd.service","state":"running"}
	]}`)
	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
	if err == nil || !strings.Contains(err.Error(), "neither running nor stopped") {
		t.Errorf("an unrecognised service state was not refused: %v", err)
	}
}

func TestADroppedServiceIsStoppedNotDeleted(t *testing.T) {
	// The host did not install the unit and does not own the unit file — only the state it put
	// the unit into.
	var commands []string
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		commands = append(commands, strings.Join(args, " "))
		if args[0] == "show" {
			return "LoadState=loaded\nActiveState=active\n", nil
		}
		return "", nil
	}
	state := store.State{Resources: []store.Applied{
		{ID: "s", Type: "service", Target: "gone.service"},
	}}
	d := parse(t, `{"declaration":1,"resources":[
	  {"id":"other","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"}
	]}`)

	if _, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, run, nil, nil); err != nil {
		t.Fatal(err)
	}
	joined := strings.Join(commands, "; ")
	if !strings.Contains(joined, "stop gone.service") {
		t.Errorf("the dropped service was not stopped: %s", joined)
	}
	if strings.Contains(joined, "disable") || strings.Contains(joined, "mask") {
		t.Errorf("the host did more than stop a unit it does not own: %s", joined)
	}
}

func TestAUnitThatDoesNotExistIsNotStopped(t *testing.T) {
	// Found by applying inside a raised machine. `systemctl is-active` says "inactive" for a
	// unit that DOES NOT EXIST exactly as it does for one that is installed and stopped, so
	// declaring a unit stopped reported success for a unit the host cannot manage at all.
	//
	// Absence read as satisfaction — 04-ISSUES/007 wearing a different hat, and the mirror of
	// the degraded-init bug the capability detector had.
	absent := func(ctx context.Context, name string, args ...string) (string, error) {
		return "LoadState=not-found\nActiveState=inactive\n", nil
	}
	d := parse(t, `{"declaration":1,"resources":[
	  {"id":"s","type":"service","unit":"never-installed.service","state":"stopped"}
	]}`)

	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, absent, nil, nil)
	if err == nil {
		t.Fatal("a unit that does not exist was reported as satisfactorily stopped")
	}
	if !strings.Contains(err.Error(), "does not exist on this machine") {
		t.Errorf("the failure does not say the unit is absent: %v", err)
	}
	if _, claimed := state.Find("s"); claimed {
		t.Error("the host recorded owning a unit that is not installed")
	}
}

func TestAMaskedUnitIsRefused(t *testing.T) {
	// Masked means someone deliberately made it unstartable. Applying over that would undo a
	// decision the host did not make and cannot see the reason for.
	masked := func(ctx context.Context, name string, args ...string) (string, error) {
		return "LoadState=masked\nActiveState=inactive\n", nil
	}
	d := parse(t, `{"declaration":1,"resources":[
	  {"id":"s","type":"service","unit":"masked.service","state":"running"}
	]}`)
	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, masked, nil, nil); err == nil {
		t.Fatal("a masked unit was accepted")
	}
}

func TestForgettingAUnitThatIsGoneDoesNotStrandTheNode(t *testing.T) {
	// Found on a real machine. Removing an orphaned service runs `systemctl stop`, which fails
	// when the unit no longer exists — and a failure there fails the whole apply. A host
	// holding a record of an uninstalled unit could then apply NOTHING, ever, with no way out
	// but editing its state by hand.
	//
	// Removal is idempotent for the same reason os.RemoveAll is: the desired end state is
	// already true.
	var stopped bool
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		if args[0] == "show" {
			return "LoadState=not-found\nActiveState=inactive\n", nil
		}
		stopped = true
		return "", errors.New("systemctl exited 5: Unit not loaded")
	}
	known := store.State{Resources: []store.Applied{
		{ID: "gone", Type: "service", Target: "uninstalled.service"},
	}}
	d := parse(t, `{"declaration":1,"resources":[
	  {"id":"f","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"}
	]}`)

	report, state, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil, nil)
	if err != nil {
		t.Fatalf("a vanished unit stranded the apply: %v", err)
	}
	if stopped {
		t.Error("the host tried to stop a unit that does not exist")
	}
	if _, still := state.Find("gone"); still {
		t.Error("the host still believes it owns a unit that is gone")
	}
	// "forgotten", not "removed": the host stopped believing it owns the unit, and did not
	// remove anything, because there was nothing there to remove. Reporting an effect it did
	// not have would be the same class of untruth as reporting a package uninstalled.
	if report.Outcomes[0].Action != "forgotten" {
		t.Errorf("the vanished unit was not reported as forgotten: %+v", report.Outcomes)
	}
}

// --- package, container and action (novox/hq 07-the-substrate.md, ADR 0006, ADR 0005) ---

func parseTrusted(t *testing.T, raw string) *declaration.Declaration {
	t.Helper()
	d, err := declaration.ParseTrusted([]byte(raw))
	if err != nil {
		t.Fatalf("fixture is not a valid declaration: %v", err)
	}
	return d
}

const pinned = "docker.io/library/postgres@sha256:" +
	"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"

func TestABrokenPackageDatabaseIsNotReadAsNotInstalled(t *testing.T) {
	// The same trap serviceState documents. `pacman -Q x` exits non-zero both for a package
	// that is not installed and for a database that cannot be read — so believing the first
	// answer would silently reinstall on a machine whose package manager is broken, or report
	// "installed nothing" as success. The apply must fail instead.
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		return "", errors.New("pacman: error: could not lock database")
	}
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"rt","type":"package","package":"docker"}
	]}`)

	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
	if err == nil {
		t.Fatal("a broken package database was read as 'not installed'")
	}
	if !strings.Contains(err.Error(), "does not answer") {
		t.Errorf("failed for the wrong reason: %v", err)
	}
}

func TestAnInstalledPackageIsNotReinstalled(t *testing.T) {
	var installed bool
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		if args[0] == "-S" {
			installed = true
		}
		return "docker 27.0-1\n", nil // -Q succeeds for everything
	}
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"rt","type":"package","package":"docker"}
	]}`)

	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
	if err != nil {
		t.Fatalf("apply failed: %v", err)
	}
	if installed {
		t.Error("a package that was already present was installed again")
	}
	if report.Changed() {
		t.Errorf("an already-installed package reported a change: %+v", report.Outcomes)
	}
}

func TestAPackageIsNeverUninstalled(t *testing.T) {
	// Deliberate: the host cannot know what else needs the package. Uninstalling a container
	// runtime because a declaration changed would stop every container on the node, and the
	// machine may have had it before the mesh ever saw it. Undeclaring is not "remove it".
	var uninstalled bool
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		if len(args) > 0 && (args[0] == "-R" || args[0] == "-Rs") {
			uninstalled = true
		}
		return "", nil
	}
	known := store.State{Resources: []store.Applied{
		{ID: "rt", Type: "package", Target: "docker"},
	}}
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"f","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"}
	]}`)

	report, state, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil, nil)
	if err != nil {
		t.Fatalf("dropping a package stranded the apply: %v", err)
	}
	if uninstalled {
		t.Fatal("the host uninstalled a package")
	}
	if _, still := state.Find("rt"); still {
		t.Error("the host still believes it owns the package")
	}
	// "forgotten", not "removed" — the host must not claim an effect it declined to have.
	if report.Outcomes[0].Action != "forgotten" {
		t.Errorf("dropping a package was not reported as forgotten: %+v", report.Outcomes[0])
	}
}

func TestAnActionThatIsAlreadyTrueDoesNotRun(t *testing.T) {
	// Verify is the idempotency check as well as the read-back. The host does not know what a
	// database is, so "is it already there" is a question only the declaration can ask.
	var ran bool
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		if name == "create-db" {
			ran = true
		}
		return "", nil // verify passes
	}
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"db","type":"action","command":["create-db","mesh"],"verify":["has-db","mesh"]}
	]}`)

	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
	if err != nil {
		t.Fatalf("apply failed: %v", err)
	}
	if ran {
		t.Error("an action whose verify already passed was run anyway")
	}
	if report.Changed() {
		t.Errorf("an already-satisfied action reported a change: %+v", report.Outcomes)
	}
}

func TestAnActionThatSucceedsAndDoesNothingFails(t *testing.T) {
	// The whole reason verify is mandatory: a command that exits zero and has no effect is
	// this repository's most expensive failure shape. Here the command "succeeds" every time
	// and verify never passes.
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		if name == "has-db" {
			return "", errors.New("no such database")
		}
		return "", nil
	}
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"db","type":"action","command":["create-db","mesh"],"verify":["has-db","mesh"]}
	]}`)

	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
	if err == nil {
		t.Fatal("an action that reported success and did nothing was accepted")
	}
	if !strings.Contains(err.Error(), "verify still fails") {
		t.Errorf("failed for the wrong reason: %v", err)
	}
	if _, recorded := state.Find("db"); recorded {
		t.Error("an action that did not work was recorded as applied")
	}
}

func TestAnActionRunsInsideTheContainerItNames(t *testing.T) {
	// Steps 2 and 3 of the bootstrap act on something inside the store's container, before
	// there is any mesh to ask.
	var sawExec bool
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		if name == "docker" && args[0] == "exec" && args[1] == "store" {
			sawExec = true
			return "", nil
		}
		return "", errors.New("not run in the container")
	}
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"db","type":"action","in":"store","command":["createdb","mesh"],"verify":["psql","-lqt"]}
	]}`)

	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil); err != nil {
		t.Fatalf("apply failed: %v", err)
	}
	if !sawExec {
		t.Error("an action naming a container did not run inside it")
	}
}

func TestAContainerThatExitsImmediatelyFailsTheApply(t *testing.T) {
	// `docker run --detach` returning an id says the container was created, not that it is
	// still running. A container whose entrypoint dies satisfies the command exactly as one
	// that came up does — which is the read-back rule, in the place it matters most.
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		switch {
		case args[0] == "info":
			return "27.0\n", nil
		case args[0] == "inspect":
			return "false\t" + "", nil // exists, not running
		case args[0] == "run":
			return "deadbeef\n", nil
		}
		return "", nil
	}
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"store","type":"container","name":"store","image":"`+pinned+`"}
	]}`)

	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
	if err == nil {
		t.Fatal("a container that exited immediately was reported as applied")
	}
	if !strings.Contains(err.Error(), "is not running") {
		t.Errorf("failed for the wrong reason: %v", err)
	}
	if _, recorded := state.Find("store"); recorded {
		t.Error("a container that is not running was recorded as applied")
	}
}

func TestAContainerWhoseDeclarationChangedIsReplaced(t *testing.T) {
	// A container's configuration is fixed when it is created, so any change is a replacement.
	// The spec label is what makes the difference visible without diffing everything the
	// runtime reports — which cannot be done reliably, because a runtime normalises what it is
	// given and that is indistinguishable from drift.
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"store","type":"container","name":"store","image":"`+pinned+`","env":{"PGDATA":"/data"}}
	]}`)
	want := containerSpec(d.Resources[0].(*declaration.Container))

	var removed, created bool
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		switch args[0] {
		case "info":
			return "27.0\n", nil
		case "inspect":
			if created {
				return "true\t" + want, nil
			}
			return "true\tsome-older-spec", nil
		case "rm":
			removed = true
			return "", nil
		case "run":
			created = true
			return "deadbeef\n", nil
		}
		return "", nil
	}

	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
	if err != nil {
		t.Fatalf("apply failed: %v", err)
	}
	if !removed || !created {
		t.Fatalf("a changed container was not replaced (removed=%v created=%v)", removed, created)
	}
	if report.Outcomes[0].Action != "updated" {
		t.Errorf("a replacement was not reported as an update: %+v", report.Outcomes[0])
	}
}

func TestAContainerThatMatchesIsLeftAlone(t *testing.T) {
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"store","type":"container","name":"store","image":"`+pinned+`","env":{"PGDATA":"/data"}}
	]}`)
	spec := containerSpec(d.Resources[0].(*declaration.Container))

	var touched bool
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		switch args[0] {
		case "info":
			return "27.0\n", nil
		case "inspect":
			return "true\t" + spec, nil
		}
		touched = true
		return "", nil
	}

	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
	if err != nil {
		t.Fatalf("apply failed: %v", err)
	}
	if touched {
		t.Error("a container that already matched was restarted")
	}
	if report.Changed() {
		t.Errorf("a matching container reported a change: %+v", report.Outcomes)
	}
}

// --- boot state (novox/hq: a unit started but not enabled stops being true at the next reboot) ---

// systemctlStub answers `show` and `is-enabled` the way systemd does, and records the verbs it
// was asked to perform. Real command shapes, because the trap being tested is what systemd
// actually says rather than what a fake would.
func systemctlStub(t *testing.T, load, active, enabled string, verbs *[]string) Runner {
	t.Helper()
	return func(ctx context.Context, name string, args ...string) (string, error) {
		switch args[0] {
		case "show":
			return "LoadState=" + load + "\nActiveState=" + active + "\n", nil
		case "is-enabled":
			// Non-zero for everything but "enabled" — the exit code says nothing useful, which
			// is the whole reason this reads the output.
			if enabled == "enabled" {
				return enabled + "\n", nil
			}
			return enabled + "\n", errors.New("exit status 1")
		case "enable":
			*verbs = append(*verbs, "enable")
			enabled = "enabled"
			return "", nil
		case "disable":
			*verbs = append(*verbs, "disable")
			enabled = "disabled"
			return "", nil
		case "start":
			*verbs = append(*verbs, "start")
			active = "active"
			return "", nil
		case "stop":
			*verbs = append(*verbs, "stop")
			active = "inactive"
			return "", nil
		}
		return "", nil
	}
}

func TestAServiceIsEnabledAtBootWhenAsked(t *testing.T) {
	// The gap this closes: the host could start a unit and never make it survive a reboot, so
	// the declaration reported success and stopped being true at the next power cut.
	var verbs []string
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"rt","type":"service","unit":"docker.service","state":"running","boot":"enabled"}
	]}`)

	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried,
		systemctlStub(t, "loaded", "inactive", "disabled", &verbs), nil, nil)
	if err != nil {
		t.Fatalf("apply failed: %v", err)
	}
	if len(verbs) != 2 || verbs[0] != "enable" || verbs[1] != "start" {
		t.Errorf("expected enable then start, got %v", verbs)
	}
	if report.Outcomes[0].Action != "updated" {
		t.Errorf("enabling and starting was not reported as an update: %+v", report.Outcomes[0])
	}
}

func TestBootIsEnabledBeforeTheUnitIsStarted(t *testing.T) {
	// Order matters when an apply fails part way. Enabled-and-stopped comes back at the next
	// boot; running-and-disabled does not. So the more durable half is made true first.
	var verbs []string
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"rt","type":"service","unit":"docker.service","state":"running","boot":"enabled"}
	]}`)
	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried,
		systemctlStub(t, "loaded", "inactive", "disabled", &verbs), nil, nil); err != nil {
		t.Fatal(err)
	}
	if len(verbs) < 2 || verbs[0] != "enable" {
		t.Errorf("boot state was not made true first: %v", verbs)
	}
}

func TestAlreadyEnabledAndRunningIsUnchanged(t *testing.T) {
	var verbs []string
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"rt","type":"service","unit":"docker.service","state":"running","boot":"enabled"}
	]}`)

	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried,
		systemctlStub(t, "loaded", "active", "enabled", &verbs), nil, nil)
	if err != nil {
		t.Fatalf("apply failed: %v", err)
	}
	if len(verbs) != 0 {
		t.Errorf("a unit already in the declared state was touched: %v", verbs)
	}
	if report.Changed() {
		t.Errorf("an unchanged service reported a change: %+v", report.Outcomes)
	}
}

func TestOmittingBootLeavesItAlone(t *testing.T) {
	// Absent means the host asserts nothing. A machine whose operator enabled something must
	// not have it silently disabled because a declaration did not mention it.
	var verbs []string
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"rt","type":"service","unit":"docker.service","state":"running"}
	]}`)
	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried,
		systemctlStub(t, "loaded", "inactive", "enabled", &verbs), nil, nil); err != nil {
		t.Fatal(err)
	}
	for _, v := range verbs {
		if v == "enable" || v == "disable" {
			t.Errorf("boot state was changed by a declaration that did not mention it: %v", verbs)
		}
	}
}

func TestAStaticUnitCannotBeEnabled(t *testing.T) {
	// `static` is neither enabled nor disabled: the unit has no install section and CANNOT be
	// enabled. Reading it as "disabled" would have the host try, fail, and blame the wrong
	// thing — the same shape as reading a missing unit as "stopped".
	var verbs []string
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"rt","type":"service","unit":"dbus.socket","state":"running","boot":"enabled"}
	]}`)

	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried,
		systemctlStub(t, "loaded", "active", "static", &verbs), nil, nil)
	if err == nil {
		t.Fatal("a static unit was accepted as enable-able")
	}
	if !strings.Contains(err.Error(), "no install section") {
		t.Errorf("failed for the wrong reason: %v", err)
	}
}

func TestAnUnknownBootStateIsRefusedNotGuessed(t *testing.T) {
	var verbs []string
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"rt","type":"service","unit":"x.service","state":"running","boot":"enabled"}
	]}`)
	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried,
		systemctlStub(t, "loaded", "active", "indirect", &verbs), nil, nil)
	if err == nil {
		t.Fatal("an unrecognised boot state was guessed at instead of refused")
	}
}

// --- more than one container runtime (novox/hq ADR 0005) ---

func TestTheRuntimeProbeIsPerRuntime(t *testing.T) {
	// Verified against a real podman 6.1.0 before this was written:
	//
	//   docker info --format '{{.ServerVersion}}'   ->  29.7.2
	//   podman info --format '{{.ServerVersion}}'   ->  Error: can't evaluate field
	//                                                   ServerVersion in type system.infoReport
	//   podman info --format '{{.Version.Version}}' ->  6.1.0
	//
	// So a single probe cannot find both, and a host that used docker's would report a machine
	// with podman as having no container runtime at all.
	for _, tc := range []struct {
		name, present, wantProbe string
	}{
		{"docker", "docker", "{{.ServerVersion}}"},
		{"podman", "podman", "{{.Version.Version}}"},
	} {
		t.Run(tc.name, func(t *testing.T) {
			var probedWith string
			run := func(ctx context.Context, name string, args ...string) (string, error) {
				if name != tc.present {
					return "", errors.New("not installed")
				}
				if args[0] == "info" {
					probedWith = args[2]
				}
				return "ok\n", nil
			}
			got, err := containerRuntime(context.Background(), run)
			if err != nil {
				t.Fatalf("%s was present and was not found: %v", tc.present, err)
			}
			if got != tc.present {
				t.Errorf("found %q, expected %q", got, tc.present)
			}
			if probedWith != tc.wantProbe {
				t.Errorf("probed %s with %q; that template does not work on it",
					tc.present, probedWith)
			}
		})
	}
}

func TestAContainerUsesTheRuntimeTheMachineHas(t *testing.T) {
	// The applier must not call `docker` on a machine that has podman. Adoption keeps what the
	// machine already has (novox/hq research 012), so hardcoding one contradicts it.
	var calledWith []string
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		if name == "docker" {
			return "", errors.New("not installed")
		}
		calledWith = append(calledWith, name)
		switch args[0] {
		case "info":
			return "6.1.0\n", nil
		case "inspect":
			return "false\t\n", errors.New("no such container")
		case "run":
			return "deadbeef\n", nil
		}
		return "", nil
	}
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"store","type":"container","name":"store","image":"`+pinned+`"}
	]}`)

	// It will fail at read-back — the stub never reports it running — and what matters is
	// WHICH binary it used getting there.
	_, _, _ = Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)

	for _, c := range calledWith {
		if c != "podman" {
			t.Errorf("called %q on a machine that only has podman", c)
		}
	}
	if len(calledWith) == 0 {
		t.Error("nothing was called; the runtime was not found")
	}
}

func TestNoRuntimeIsSaidPlainly(t *testing.T) {
	// Naming what was tried, because "docker: command not found" on a machine that deliberately
	// runs podman sends the reader looking for the wrong thing.
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		return "", errors.New("not installed")
	}
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"store","type":"container","name":"store","image":"`+pinned+`"}
	]}`)

	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
	if err == nil {
		t.Fatal("a machine with no container runtime applied a container")
	}
	for _, want := range []string{"docker", "podman", "no container runtime"} {
		if !strings.Contains(err.Error(), want) {
			t.Errorf("the failure does not mention %q: %v", want, err)
		}
	}
}

// archHost is the system these tests run against. They were written for pacman and systemd, and
// naming that is better than the implicit default it used to be.
func archHost(t *testing.T) system.System {
	t.Helper()
	s, err := system.For("arch")
	if err != nil {
		t.Fatal(err)
	}
	return s
}

func TestAServiceIsRestartedWhenWhatItReflectsChanges(t *testing.T) {
	// A running service does not re-read its configuration. Replace the file, find the service
	// already running, do nothing — and the machine keeps behaving as it did while every check
	// passes, because the file is right and the service is up.
	//
	// That is how a third node joining a mesh left the first two carrying a network that no
	// longer existed. Found in the lab; this is the shape of the fix.
	dir := t.TempDir()
	path := filepath.Join(dir, "thing.conf")

	d := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
	  {"id":"conf","type":"file","path":%q,"content":"first\n","mode":"0644"},
	  {"id":"svc","type":"service","unit":"thing.service","state":"running","restart-on":["conf"]}
	]}`, path))

	var commands []string
	run := recordingServices(&commands)

	if _, state, err := Apply(context.Background(), archHost(t), d, store.State{},
		store.OriginCarried, run, nil, nil); err != nil {
		t.Fatal(err)
	} else {
		// Second apply with the same content: nothing moved, so nothing restarts. A machine that
		// restarted its services on every reconcile would never be steady.
		commands = nil
		if _, _, err := Apply(context.Background(), archHost(t), d, state,
			store.OriginCarried, run, nil, nil); err != nil {
			t.Fatal(err)
		}
		for _, c := range commands {
			if strings.Contains(c, "stop") {
				t.Errorf("an unchanged declaration restarted the service: %s", c)
			}
		}

		// Now the file changes. The service is already running and must still be restarted.
		changedDecl := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
		  {"id":"conf","type":"file","path":%q,"content":"second\n","mode":"0644"},
		  {"id":"svc","type":"service","unit":"thing.service","state":"running","restart-on":["conf"]}
		]}`, path))
		commands = nil
		if _, _, err := Apply(context.Background(), archHost(t), changedDecl, state,
			store.OriginCarried, run, nil, nil); err != nil {
			t.Fatal(err)
		}
		var stopped, started bool
		for _, c := range commands {
			if strings.Contains(c, "stop thing.service") {
				stopped = true
			}
			if strings.Contains(c, "start thing.service") {
				started = true
			}
		}
		if !stopped || !started {
			t.Errorf("the file changed and the service was not restarted; commands were %v", commands)
		}
	}
}

func TestAServiceIsNotRestartedByAChangeItDoesNotName(t *testing.T) {
	// The list is what it reflects, not everything in the declaration. A service restarted by any
	// change anywhere would make every apply a fleet-wide bounce.
	dir := t.TempDir()
	conf := filepath.Join(dir, "thing.conf")
	other := filepath.Join(dir, "unrelated")

	first := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
	  {"id":"conf","type":"file","path":%q,"content":"same\n","mode":"0644"},
	  {"id":"other","type":"file","path":%q,"content":"one\n","mode":"0644"},
	  {"id":"svc","type":"service","unit":"thing.service","state":"running","restart-on":["conf"]}
	]}`, conf, other))

	var commands []string
	run := recordingServices(&commands)
	_, state, err := Apply(context.Background(), archHost(t), first, store.State{},
		store.OriginCarried, run, nil, nil)
	if err != nil {
		t.Fatal(err)
	}

	second := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
	  {"id":"conf","type":"file","path":%q,"content":"same\n","mode":"0644"},
	  {"id":"other","type":"file","path":%q,"content":"two\n","mode":"0644"},
	  {"id":"svc","type":"service","unit":"thing.service","state":"running","restart-on":["conf"]}
	]}`, conf, other))
	commands = nil
	if _, _, err := Apply(context.Background(), archHost(t), second, state,
		store.OriginCarried, run, nil, nil); err != nil {
		t.Fatal(err)
	}
	for _, c := range commands {
		if strings.Contains(c, "stop") {
			t.Errorf("a change to a file the service does not name restarted it: %s", c)
		}
	}
}

// recordingServices answers the way a machine with a running unit would, and remembers what it
// was asked to do — which is what a restart has to be proved by, since "running" looks the same
// before and after one.
func recordingServices(commands *[]string) Runner {
	return func(_ context.Context, name string, args ...string) (string, error) {
		line := name + " " + strings.Join(args, " ")
		*commands = append(*commands, line)
		switch {
		case strings.Contains(line, "is-enabled"):
			return "enabled", nil
		case strings.Contains(line, "show") && strings.Contains(line, "ActiveState"):
			return "LoadState=loaded\nActiveState=active\nSubState=running", nil
		}
		return "", nil
	}
}

func TestAFileChangedOnTheMachineIsCorrectedAndSaidSo(t *testing.T) {
	// The question this answers: how would anybody know somebody edited a managed file? Before
	// this they would not. It was rewritten within five minutes and reported as "updated",
	// which is what the mesh changing its mind looks like — so the person's change vanished and
	// nothing anywhere said why. They edit it again, and again.
	dir := t.TempDir()
	path := filepath.Join(dir, "thing.conf")
	d := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
	  {"id":"conf","type":"file","path":%q,"content":"from the mesh\n","mode":"0644"}
	]}`, path))

	_, state, err := Apply(context.Background(), archHost(t), d, store.State{},
		store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}

	// Somebody edits it.
	if err := os.WriteFile(path, []byte("edited by hand\n"), 0o644); err != nil {
		t.Fatal(err)
	}

	report, state, err := Apply(context.Background(), archHost(t), d, state,
		store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	if got := report.Outcomes[0].Action; got != "corrected" {
		t.Errorf("a hand edit was reported as %q; the mesh cannot tell it from changing its own "+
			"mind, and neither can anybody reading this", got)
	}

	// And it is put back, because holding the machine to what it was told is the point.
	back, err := os.ReadFile(path)
	if err != nil {
		t.Fatal(err)
	}
	if string(back) != "from the mesh\n" {
		t.Errorf("the file was left as %q", back)
	}
}

func TestTheMeshChangingItsMindIsNotDrift(t *testing.T) {
	// The other half. A new declaration is an ordinary update and must not read as somebody
	// having meddled, or every real change would look like an incident.
	dir := t.TempDir()
	path := filepath.Join(dir, "thing.conf")
	first := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
	  {"id":"conf","type":"file","path":%q,"content":"one\n","mode":"0644"}
	]}`, path))
	second := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
	  {"id":"conf","type":"file","path":%q,"content":"two\n","mode":"0644"}
	]}`, path))

	_, state, err := Apply(context.Background(), archHost(t), first, store.State{},
		store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	report, _, err := Apply(context.Background(), archHost(t), second, state,
		store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	if got := report.Outcomes[0].Action; got != "updated" {
		t.Errorf("the mesh changing what it wants was reported as %q", got)
	}
}

func TestAnUntouchedFileIsStillUnchanged(t *testing.T) {
	// And nothing about this makes a steady machine look busy.
	dir := t.TempDir()
	path := filepath.Join(dir, "thing.conf")
	d := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
	  {"id":"conf","type":"file","path":%q,"content":"steady\n","mode":"0644"}
	]}`, path))

	_, state, err := Apply(context.Background(), archHost(t), d, store.State{},
		store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	report, _, err := Apply(context.Background(), archHost(t), d, state,
		store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	if got := report.Outcomes[0].Action; got != "unchanged" {
		t.Errorf("an untouched file was reported as %q", got)
	}
}