mesh-host/internal/apply/apply_test.go

package apply

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/novox/mesh-host/internal/declaration"
	"github.com/novox/mesh-host/internal/store"
	"github.com/novox/mesh-host/internal/system"
)

// Each test names the decision it defends (novox/hq ADR 0017).

func parse(t *testing.T, raw string) *declaration.Declaration {
	t.Helper()
	d, err := declaration.Parse([]byte(raw))
	if err != nil {
		t.Fatalf("fixture is not a valid declaration: %v", err)
	}
	return d
}

// noServices refuses to run anything. Used where a test declares no services, so that a test
// which accidentally reaches the service manager fails loudly instead of passing quietly.
func noServices(context.Context, string, ...string) (string, error) {
	return "", errors.New("this test declares no services and should not have run a command")
}

func TestApplyingTwiceChangesNothingTheSecondTime(t *testing.T) {
	// Idempotence is what makes an apply safe to run on a schedule. Without it, a host that
	// reconciles every few minutes rewrites files forever and every reader sees churn.
	dir := t.TempDir()
	d := parse(t, `{"declaration":1,"resources":[
	  {"id":"d","type":"directory","path":"`+dir+`/etc","mode":"0755"},
	  {"id":"f","type":"file","path":"`+dir+`/etc/a.conf","content":"hello\n","mode":"0640"}
	]}`)

	first, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	if !first.Changed() {
		t.Fatal("the first apply on an empty machine changed nothing")
	}

	second, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	if second.Changed() {
		t.Errorf("the second apply changed something: %+v", second.Outcomes)
	}
}

// Defends novox/hq ADR 0011: a managed file is generated onto a node and never edited there.
//
// Not by overwriting silently — by noticing. An edit that vanishes without a word is how somebody
// spends an afternoon re-fixing a bug they already fixed.
func TestADriftedMachineIsReturned(t *testing.T) {
	// The other half of idempotence, and the half that matters: converging is not "do nothing
	// if the state file says it was done". The machine is read, not the record.
	dir := t.TempDir()
	path := filepath.Join(dir, "a.conf")
	d := parse(t, `{"declaration":1,"resources":[
	  {"id":"f","type":"file","path":"`+path+`","content":"correct\n","mode":"0644"}
	]}`)

	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(path, []byte("someone edited this\n"), 0o644); err != nil {
		t.Fatal(err)
	}

	report, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	if !report.Changed() {
		t.Fatal("a drifted file was left drifted")
	}
	got, _ := os.ReadFile(path)
	if string(got) != "correct\n" {
		t.Errorf("the file was not returned: %q", got)
	}
}

func TestADroppedResourceIsRemoved(t *testing.T) {
	// novox/hq ADR 0005: the host removes what it previously applied and is no longer
	// declared. Removing a line from a declaration is an act with an effect.
	dir := t.TempDir()
	keep := filepath.Join(dir, "keep.conf")
	drop := filepath.Join(dir, "drop.conf")

	both := parse(t, `{"declaration":1,"resources":[
	  {"id":"keep","type":"file","path":"`+keep+`","content":"a\n"},
	  {"id":"drop","type":"file","path":"`+drop+`","content":"b\n"}
	]}`)
	_, state, err := Apply(context.Background(), archHost(t), both, store.State{}, store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}

	one := parse(t, `{"declaration":1,"resources":[
	  {"id":"keep","type":"file","path":"`+keep+`","content":"a\n"}
	]}`)
	report, state, err := Apply(context.Background(), archHost(t), one, state, store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}

	if _, err := os.Stat(drop); !errors.Is(err, os.ErrNotExist) {
		t.Error("a resource dropped from the declaration was left on the machine")
	}
	if _, err := os.Stat(keep); err != nil {
		t.Error("a declared resource was removed")
	}
	if _, still := state.Find("drop"); still {
		t.Error("the host still believes it owns what it removed")
	}
	if report.Outcomes[0].Action != "removed" {
		t.Errorf("removal is not reported first: %+v", report.Outcomes)
	}
}

func TestNothingTheHostDidNotCreateIsTouched(t *testing.T) {
	// The boundary the whole removal rule turns on. A machine has things on it the mesh did
	// not put there, and a converger that treats "not declared" as "must not exist" deletes
	// them. Authoritative over its own footprint; inert everywhere else.
	dir := t.TempDir()
	stranger := filepath.Join(dir, "not-ours.conf")
	if err := os.WriteFile(stranger, []byte("someone else's\n"), 0o644); err != nil {
		t.Fatal(err)
	}

	d := parse(t, `{"declaration":1,"resources":[
	  {"id":"ours","type":"file","path":"`+filepath.Join(dir, "ours.conf")+`","content":"a\n"}
	]}`)
	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil); err != nil {
		t.Fatal(err)
	}

	got, err := os.ReadFile(stranger)
	if err != nil || string(got) != "someone else's\n" {
		t.Error("a file the host did not create was removed or changed")
	}
}

func TestARenameToTheSamePathDoesNotDeleteTheNewFile(t *testing.T) {
	// Why removal happens FIRST. A resource leaving a declaration while another arrives at the
	// same path is an ordinary rename; removing afterwards would delete the file just written.
	dir := t.TempDir()
	path := filepath.Join(dir, "shared.conf")

	before := parse(t, `{"declaration":1,"resources":[
	  {"id":"old","type":"file","path":"`+path+`","content":"old\n"}
	]}`)
	_, state, err := Apply(context.Background(), archHost(t), before, store.State{}, store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}

	after := parse(t, `{"declaration":1,"resources":[
	  {"id":"new","type":"file","path":"`+path+`","content":"new\n"}
	]}`)
	if _, _, err := Apply(context.Background(), archHost(t), after, state, store.OriginCarried, noServices, nil, nil); err != nil {
		t.Fatal(err)
	}

	got, err := os.ReadFile(path)
	if err != nil {
		t.Fatalf("the renamed resource is gone: %v", err)
	}
	if string(got) != "new\n" {
		t.Errorf("content is %q, want the new one", got)
	}
}

func TestAFailedStepFailsTheApplyAndTheRestIsStillAttempted(t *testing.T) {
	// The apply fails, names the resource, and carries what did happen — because the machine is
	// in whatever state the apply reached and the only honest thing to hand back is that list.
	//
	// **And everything is attempted.** It used to stop at the first failure, which made one
	// broken resource hold the whole machine hostage: a module declaring a package that does not
	// exist meant every module after it was never applied, for ever
	// (novox/hq 04-ISSUES/011). The case for stopping was that a later resource may depend on an
	// earlier one — and it still may, and it then fails its own check and is reported, which is
	// more information than not attempting it.
	dir := t.TempDir()
	blocker := filepath.Join(dir, "blocker")
	if err := os.WriteFile(blocker, []byte("i am a file\n"), 0o644); err != nil {
		t.Fatal(err)
	}

	d := parse(t, `{"declaration":1,"resources":[
	  {"id":"fine","type":"file","path":"`+filepath.Join(dir, "fine.conf")+`","content":"a\n"},
	  {"id":"doomed","type":"directory","path":"`+blocker+`"},
	  {"id":"after","type":"file","path":"`+filepath.Join(dir, "after.conf")+`","content":"b\n"}
	]}`)

	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil)
	if err == nil {
		t.Fatal("an impossible resource did not fail the apply")
	}

	var applyErr *Error
	if !errors.As(err, &applyErr) {
		t.Fatalf("expected an apply error, got %T", err)
	}
	if applyErr.Resource != "doomed" {
		t.Errorf("the failure names %q, not the resource that failed", applyErr.Resource)
	}
	// Everything that worked is in the report, before and after the failure.
	if len(applyErr.Done.Outcomes) != 2 {
		t.Errorf("the error does not carry what was applied: %+v", applyErr.Done.Outcomes)
	}
	if _, err := os.Stat(filepath.Join(dir, "after.conf")); err != nil {
		t.Error("a resource after the failing one was never attempted, so one broken module " +
			"still blocks every module after it")
	}
}

func TestEveryFailureIsCountedNotJustTheFirst(t *testing.T) {
	// "One thing failed" and "eleven things failed" are different machines, and the first line is
	// what somebody reads.
	dir := t.TempDir()
	for _, name := range []string{"one", "two"} {
		if err := os.WriteFile(filepath.Join(dir, name), []byte("a file\n"), 0o644); err != nil {
			t.Fatal(err)
		}
	}
	d := parse(t, `{"declaration":1,"resources":[
	  {"id":"first","type":"directory","path":"`+filepath.Join(dir, "one")+`"},
	  {"id":"second","type":"directory","path":"`+filepath.Join(dir, "two")+`"}
	]}`)

	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil)
	if err == nil {
		t.Fatal("two impossible resources did not fail the apply")
	}
	var applyErr *Error
	if !errors.As(err, &applyErr) {
		t.Fatalf("got %T", err)
	}
	if applyErr.Others != 1 {
		t.Errorf("the failure says %d others also failed, and one did", applyErr.Others)
	}
	if !strings.Contains(err.Error(), "one other resource also failed") {
		t.Errorf("the message does not say others failed: %v", err)
	}
}

func TestNothingIsRecordedUntilItWorked(t *testing.T) {
	// novox/hq ADR 0018. A record written before the fact restates the request in a new place
	// and inherits none of the authority of having happened.
	dir := t.TempDir()
	blocker := filepath.Join(dir, "blocker")
	if err := os.WriteFile(blocker, []byte("x\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	d := parse(t, `{"declaration":1,"resources":[
	  {"id":"doomed","type":"directory","path":"`+blocker+`"}
	]}`)

	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil)
	if err == nil {
		t.Fatal("expected a failure")
	}
	if _, claimed := state.Find("doomed"); claimed {
		t.Error("the host recorded owning something it failed to apply")
	}
}

func TestAModeIsMaintainedNotJustSet(t *testing.T) {
	// A permission set at creation is not a permission maintained — this repository has
	// already paid for that once, with generated files left world-readable because the mode
	// applied only when the file was first written.
	dir := t.TempDir()
	path := filepath.Join(dir, "secret.conf")
	d := parse(t, `{"declaration":1,"resources":[
	  {"id":"f","type":"file","path":"`+path+`","content":"s\n","mode":"0600"}
	]}`)

	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	if err := os.Chmod(path, 0o666); err != nil {
		t.Fatal(err)
	}

	report, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	info, _ := os.Stat(path)
	if info.Mode().Perm() != 0o600 {
		t.Errorf("mode is %o after reconciling, want 0600", info.Mode().Perm())
	}
	if !report.Changed() {
		t.Error("a mode that had drifted was reported as unchanged")
	}
}

func TestAServiceIsReadBackNotAssumed(t *testing.T) {
	// `systemctl start` returning zero says the transaction was accepted, not that the unit is
	// running. A unit that starts and immediately dies satisfies the command.
	started := false
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		if args[0] == "show" {
			if started {
				return "LoadState=loaded\nActiveState=failed\n", nil // started, then died
			}
			return "LoadState=loaded\nActiveState=inactive\n", nil
		}
		started = true
		return "", nil // `systemctl start` succeeds
	}

	d := parse(t, `{"declaration":1,"resources":[
	  {"id":"s","type":"service","unit":"doomed.service","state":"running"}
	]}`)
	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
	if err == nil {
		t.Fatal("a service that died immediately was reported as running")
	}
	if !strings.Contains(err.Error(), "asked to be running and is stopped") {
		t.Errorf("the failure does not say what was observed: %v", err)
	}
}

func TestAnUnknownServiceStateIsRefusedNotGuessed(t *testing.T) {
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		return "LoadState=loaded\nActiveState=reticent\n", nil
	}
	d := parse(t, `{"declaration":1,"resources":[
	  {"id":"s","type":"service","unit":"odd.service","state":"running"}
	]}`)
	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
	if err == nil || !strings.Contains(err.Error(), "neither running nor stopped") {
		t.Errorf("an unrecognised service state was not refused: %v", err)
	}
}

func TestADroppedServiceIsStoppedNotDeleted(t *testing.T) {
	// The host did not install the unit and does not own the unit file — only the state it put
	// the unit into.
	var commands []string
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		commands = append(commands, strings.Join(args, " "))
		if args[0] == "show" {
			return "LoadState=loaded\nActiveState=active\n", nil
		}
		return "", nil
	}
	state := store.State{Resources: []store.Applied{
		{ID: "s", Type: "service", Target: "gone.service"},
	}}
	d := parse(t, `{"declaration":1,"resources":[
	  {"id":"other","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"}
	]}`)

	if _, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, run, nil, nil); err != nil {
		t.Fatal(err)
	}
	joined := strings.Join(commands, "; ")
	if !strings.Contains(joined, "stop gone.service") {
		t.Errorf("the dropped service was not stopped: %s", joined)
	}
	if strings.Contains(joined, "disable") || strings.Contains(joined, "mask") {
		t.Errorf("the host did more than stop a unit it does not own: %s", joined)
	}
}

func TestAUnitThatDoesNotExistIsNotStopped(t *testing.T) {
	// Found by applying inside a raised machine. `systemctl is-active` says "inactive" for a
	// unit that DOES NOT EXIST exactly as it does for one that is installed and stopped, so
	// declaring a unit stopped reported success for a unit the host cannot manage at all.
	//
	// Absence read as satisfaction — 04-ISSUES/007 wearing a different hat, and the mirror of
	// the degraded-init bug the capability detector had.
	absent := func(ctx context.Context, name string, args ...string) (string, error) {
		return "LoadState=not-found\nActiveState=inactive\n", nil
	}
	d := parse(t, `{"declaration":1,"resources":[
	  {"id":"s","type":"service","unit":"never-installed.service","state":"stopped"}
	]}`)

	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, absent, nil, nil)
	if err == nil {
		t.Fatal("a unit that does not exist was reported as satisfactorily stopped")
	}
	if !strings.Contains(err.Error(), "does not exist on this machine") {
		t.Errorf("the failure does not say the unit is absent: %v", err)
	}
	if _, claimed := state.Find("s"); claimed {
		t.Error("the host recorded owning a unit that is not installed")
	}
}

func TestAMaskedUnitIsRefused(t *testing.T) {
	// Masked means someone deliberately made it unstartable. Applying over that would undo a
	// decision the host did not make and cannot see the reason for.
	masked := func(ctx context.Context, name string, args ...string) (string, error) {
		return "LoadState=masked\nActiveState=inactive\n", nil
	}
	d := parse(t, `{"declaration":1,"resources":[
	  {"id":"s","type":"service","unit":"masked.service","state":"running"}
	]}`)
	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, masked, nil, nil); err == nil {
		t.Fatal("a masked unit was accepted")
	}
}

func TestForgettingAUnitThatIsGoneDoesNotStrandTheNode(t *testing.T) {
	// Found on a real machine. Removing an orphaned service runs `systemctl stop`, which fails
	// when the unit no longer exists — and a failure there fails the whole apply. A host
	// holding a record of an uninstalled unit could then apply NOTHING, ever, with no way out
	// but editing its state by hand.
	//
	// Removal is idempotent for the same reason os.RemoveAll is: the desired end state is
	// already true.
	var stopped bool
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		if args[0] == "show" {
			return "LoadState=not-found\nActiveState=inactive\n", nil
		}
		stopped = true
		return "", errors.New("systemctl exited 5: Unit not loaded")
	}
	known := store.State{Resources: []store.Applied{
		{ID: "gone", Type: "service", Target: "uninstalled.service"},
	}}
	d := parse(t, `{"declaration":1,"resources":[
	  {"id":"f","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"}
	]}`)

	report, state, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil, nil)
	if err != nil {
		t.Fatalf("a vanished unit stranded the apply: %v", err)
	}
	if stopped {
		t.Error("the host tried to stop a unit that does not exist")
	}
	if _, still := state.Find("gone"); still {
		t.Error("the host still believes it owns a unit that is gone")
	}
	// "forgotten", not "removed": the host stopped believing it owns the unit, and did not
	// remove anything, because there was nothing there to remove. Reporting an effect it did
	// not have would be the same class of untruth as reporting a package uninstalled.
	if report.Outcomes[0].Action != "forgotten" {
		t.Errorf("the vanished unit was not reported as forgotten: %+v", report.Outcomes)
	}
}

// --- package, container and action (novox/hq 07-the-foundation.md, ADR 0006, ADR 0005) ---

func parseTrusted(t *testing.T, raw string) *declaration.Declaration {
	t.Helper()
	d, err := declaration.ParseTrusted([]byte(raw))
	if err != nil {
		t.Fatalf("fixture is not a valid declaration: %v", err)
	}
	return d
}

const pinned = "docker.io/library/postgres@sha256:" +
	"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"

func TestABrokenPackageDatabaseIsNotReadAsNotInstalled(t *testing.T) {
	// The same trap serviceState documents. `pacman -Q x` exits non-zero both for a package
	// that is not installed and for a database that cannot be read — so believing the first
	// answer would silently reinstall on a machine whose package manager is broken, or report
	// "installed nothing" as success. The apply must fail instead.
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		return "", errors.New("pacman: error: could not lock database")
	}
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"rt","type":"package","package":"docker"}
	]}`)

	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
	if err == nil {
		t.Fatal("a broken package database was read as 'not installed'")
	}
	if !strings.Contains(err.Error(), "does not answer") {
		t.Errorf("failed for the wrong reason: %v", err)
	}
}

func TestAnInstalledPackageIsNotReinstalled(t *testing.T) {
	var installed bool
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		if args[0] == "-S" {
			installed = true
		}
		return "docker 27.0-1\n", nil // -Q succeeds for everything
	}
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"rt","type":"package","package":"docker"}
	]}`)

	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
	if err != nil {
		t.Fatalf("apply failed: %v", err)
	}
	if installed {
		t.Error("a package that was already present was installed again")
	}
	if report.Changed() {
		t.Errorf("an already-installed package reported a change: %+v", report.Outcomes)
	}
}

func TestAPackageIsNeverUninstalled(t *testing.T) {
	// Deliberate: the host cannot know what else needs the package. Uninstalling a container
	// runtime because a declaration changed would stop every container on the node, and the
	// machine may have had it before the mesh ever saw it. Undeclaring is not "remove it".
	var uninstalled bool
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		if len(args) > 0 && (args[0] == "-R" || args[0] == "-Rs") {
			uninstalled = true
		}
		return "", nil
	}
	known := store.State{Resources: []store.Applied{
		{ID: "rt", Type: "package", Target: "docker"},
	}}
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"f","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"}
	]}`)

	report, state, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil, nil)
	if err != nil {
		t.Fatalf("dropping a package stranded the apply: %v", err)
	}
	if uninstalled {
		t.Fatal("the host uninstalled a package")
	}
	if _, still := state.Find("rt"); still {
		t.Error("the host still believes it owns the package")
	}
	// "forgotten", not "removed" — the host must not claim an effect it declined to have.
	if report.Outcomes[0].Action != "forgotten" {
		t.Errorf("dropping a package was not reported as forgotten: %+v", report.Outcomes[0])
	}
}

func TestAnActionThatIsAlreadyTrueDoesNotRun(t *testing.T) {
	// Verify is the idempotency check as well as the read-back. The host does not know what a
	// database is, so "is it already there" is a question only the declaration can ask.
	var ran bool
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		if name == "create-db" {
			ran = true
		}
		return "", nil // verify passes
	}
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"db","type":"action","command":["create-db","mesh"],"verify":["has-db","mesh"]}
	]}`)

	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
	if err != nil {
		t.Fatalf("apply failed: %v", err)
	}
	if ran {
		t.Error("an action whose verify already passed was run anyway")
	}
	if report.Changed() {
		t.Errorf("an already-satisfied action reported a change: %+v", report.Outcomes)
	}
}

func TestAnActionThatSucceedsAndDoesNothingFails(t *testing.T) {
	// The whole reason verify is mandatory: a command that exits zero and has no effect is
	// this repository's most expensive failure shape. Here the command "succeeds" every time
	// and verify never passes.
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		if name == "has-db" {
			return "", errors.New("no such database")
		}
		return "", nil
	}
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"db","type":"action","command":["create-db","mesh"],"verify":["has-db","mesh"]}
	]}`)

	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
	if err == nil {
		t.Fatal("an action that reported success and did nothing was accepted")
	}
	if !strings.Contains(err.Error(), "verify still fails") {
		t.Errorf("failed for the wrong reason: %v", err)
	}
	if _, recorded := state.Find("db"); recorded {
		t.Error("an action that did not work was recorded as applied")
	}
}

func TestAnActionRunsInsideTheContainerItNames(t *testing.T) {
	// Steps 2 and 3 of the bootstrap act on something inside the store's container, before
	// there is any mesh to ask.
	var sawExec bool
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		if name == "docker" && args[0] == "exec" && args[1] == "store" {
			sawExec = true
			return "", nil
		}
		return "", errors.New("not run in the container")
	}
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"db","type":"action","in":"store","command":["createdb","mesh"],"verify":["psql","-lqt"]}
	]}`)

	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil); err != nil {
		t.Fatalf("apply failed: %v", err)
	}
	if !sawExec {
		t.Error("an action naming a container did not run inside it")
	}
}

func TestAContainerThatExitsImmediatelyFailsTheApply(t *testing.T) {
	// `docker run --detach` returning an id says the container was created, not that it is
	// still running. A container whose entrypoint dies satisfies the command exactly as one
	// that came up does — which is the read-back rule, in the place it matters most.
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		switch {
		case args[0] == "info":
			return "27.0\n", nil
		case args[0] == "inspect":
			return "false\t" + "", nil // exists, not running
		case args[0] == "run":
			return "deadbeef\n", nil
		}
		return "", nil
	}
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"store","type":"container","name":"store","image":"`+pinned+`"}
	]}`)

	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
	if err == nil {
		t.Fatal("a container that exited immediately was reported as applied")
	}
	if !strings.Contains(err.Error(), "is not running") {
		t.Errorf("failed for the wrong reason: %v", err)
	}
	if _, recorded := state.Find("store"); recorded {
		t.Error("a container that is not running was recorded as applied")
	}
}

func TestAContainerWhoseDeclarationChangedIsReplaced(t *testing.T) {
	// A container's configuration is fixed when it is created, so any change is a replacement.
	// The spec label is what makes the difference visible without diffing everything the
	// runtime reports — which cannot be done reliably, because a runtime normalises what it is
	// given and that is indistinguishable from drift.
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"store","type":"container","name":"store","image":"`+pinned+`","env":{"PGDATA":"/data"}}
	]}`)
	want := containerSpec(d.Resources[0].(*declaration.Container), nil)

	var removed, created bool
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		switch args[0] {
		case "info":
			return "27.0\n", nil
		case "inspect":
			if created {
				return "true\t" + want, nil
			}
			return "true\tsome-older-spec", nil
		case "rm":
			removed = true
			return "", nil
		case "run":
			created = true
			return "deadbeef\n", nil
		}
		return "", nil
	}

	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
	if err != nil {
		t.Fatalf("apply failed: %v", err)
	}
	if !removed || !created {
		t.Fatalf("a changed container was not replaced (removed=%v created=%v)", removed, created)
	}
	if report.Outcomes[0].Action != "updated" {
		t.Errorf("a replacement was not reported as an update: %+v", report.Outcomes[0])
	}
}

func TestAContainerThatMatchesIsLeftAlone(t *testing.T) {
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"store","type":"container","name":"store","image":"`+pinned+`","env":{"PGDATA":"/data"}}
	]}`)
	spec := containerSpec(d.Resources[0].(*declaration.Container), nil)

	var touched bool
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		switch args[0] {
		case "info":
			return "27.0\n", nil
		case "inspect":
			return "true\t" + spec, nil
		}
		touched = true
		return "", nil
	}

	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
	if err != nil {
		t.Fatalf("apply failed: %v", err)
	}
	if touched {
		t.Error("a container that already matched was restarted")
	}
	if report.Changed() {
		t.Errorf("a matching container reported a change: %+v", report.Outcomes)
	}
}

func TestAContainerIsRecreatedWhenARestartOnResourceChanged(t *testing.T) {
	// A container reads a mounted file once, at start. When the file changed this pass but the
	// container's spec did not, the plain "spec matches, leave it" rule would keep the process
	// holding the old value for ever, with every check passing (novox/hq 04-ISSUES/009). A
	// container names the resources it must reflect in restart-on, the same as a service, and the
	// host recreates it. Here the config file is fresh, so it is written this pass, and the
	// already-running-and-matching container must still be replaced.
	dir := t.TempDir()
	conf := filepath.Join(dir, "config.json")
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"config","type":"file","path":"`+conf+`","content":"{\"token\":\"new\"}\n"},
	  {"id":"app","type":"container","name":"app","image":"`+pinned+`","restart-on":["config"]}
	]}`)
	// The spec the container was created with, back when the file said something else. Built the
	// way the host builds it, so this is the real comparison rather than a hand-written string:
	// what a container reads is part of what it is, so the old content yields a different spec.
	was := map[string]string{"config": declaredDigest(&declaration.File{Content: "{\"token\":\"old\"}\n"})}
	now := map[string]string{"config": declaredDigest(d.Resources[0].(*declaration.File))}
	stale := containerSpec(d.Resources[1].(*declaration.Container), was)
	fresh := containerSpec(d.Resources[1].(*declaration.Container), now)

	var removed, created bool
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		switch args[0] {
		case "info":
			return "27.0\n", nil
		case "inspect":
			// Already there and running, created against the file as it was. After the host
			// recreates it, the runtime holds the one it just made — as a real one would.
			if created {
				return "true\t" + fresh, nil
			}
			return "true\t" + stale, nil
		case "rm":
			removed = true
			return "", nil
		case "run":
			created = true
			return "deadbeef\n", nil
		}
		return "", nil
	}

	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
	if err != nil {
		t.Fatalf("apply failed: %v", err)
	}
	if !removed || !created {
		t.Fatalf("a container was not recreated when its restart-on file changed (removed=%v created=%v)", removed, created)
	}
	app := report.Outcomes[len(report.Outcomes)-1]
	if app.Action != "updated" || !strings.Contains(app.Detail, "config") {
		t.Errorf("the recreation did not name why it happened: %+v", app)
	}
}

func TestRestartOnFiresOnlyForResourcesThatChanged(t *testing.T) {
	// restart-on must not mean "always restart": it names resources, and only a resource that
	// changed this pass is a reason. This is the guard shared by containers and services
	// (novox/hq 04-ISSUES/009), so a container that reflects an unchanged file is left running.
	changed := map[string]bool{"other": true}
	if got := restartedBy([]string{"config"}, changed); len(got) != 0 {
		t.Errorf("an unchanged resource was treated as a reason to restart: %v", got)
	}
	changed["config"] = true
	if got := restartedBy([]string{"config", "missing"}, changed); len(got) != 1 || got[0] != "config" {
		t.Errorf("restart-on did not name exactly the changed resource: %v", got)
	}
}

// --- boot state (novox/hq: a unit started but not enabled stops being true at the next reboot) ---

// systemctlStub answers `show` and `is-enabled` the way systemd does, and records the verbs it
// was asked to perform. Real command shapes, because the trap being tested is what systemd
// actually says rather than what a fake would.
func systemctlStub(t *testing.T, load, active, enabled string, verbs *[]string) Runner {
	t.Helper()
	return func(ctx context.Context, name string, args ...string) (string, error) {
		switch args[0] {
		case "show":
			return "LoadState=" + load + "\nActiveState=" + active + "\n", nil
		case "is-enabled":
			// Non-zero for everything but "enabled" — the exit code says nothing useful, which
			// is the whole reason this reads the output.
			if enabled == "enabled" {
				return enabled + "\n", nil
			}
			return enabled + "\n", errors.New("exit status 1")
		case "enable":
			*verbs = append(*verbs, "enable")
			enabled = "enabled"
			return "", nil
		case "disable":
			*verbs = append(*verbs, "disable")
			enabled = "disabled"
			return "", nil
		case "start":
			*verbs = append(*verbs, "start")
			active = "active"
			return "", nil
		case "stop":
			*verbs = append(*verbs, "stop")
			active = "inactive"
			return "", nil
		}
		return "", nil
	}
}

func TestAServiceIsEnabledAtBootWhenAsked(t *testing.T) {
	// The gap this closes: the host could start a unit and never make it survive a reboot, so
	// the declaration reported success and stopped being true at the next power cut.
	var verbs []string
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"rt","type":"service","unit":"docker.service","state":"running","boot":"enabled"}
	]}`)

	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried,
		systemctlStub(t, "loaded", "inactive", "disabled", &verbs), nil, nil)
	if err != nil {
		t.Fatalf("apply failed: %v", err)
	}
	if len(verbs) != 2 || verbs[0] != "enable" || verbs[1] != "start" {
		t.Errorf("expected enable then start, got %v", verbs)
	}
	if report.Outcomes[0].Action != "updated" {
		t.Errorf("enabling and starting was not reported as an update: %+v", report.Outcomes[0])
	}
}

func TestBootIsEnabledBeforeTheUnitIsStarted(t *testing.T) {
	// Order matters when an apply fails part way. Enabled-and-stopped comes back at the next
	// boot; running-and-disabled does not. So the more durable half is made true first.
	var verbs []string
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"rt","type":"service","unit":"docker.service","state":"running","boot":"enabled"}
	]}`)
	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried,
		systemctlStub(t, "loaded", "inactive", "disabled", &verbs), nil, nil); err != nil {
		t.Fatal(err)
	}
	if len(verbs) < 2 || verbs[0] != "enable" {
		t.Errorf("boot state was not made true first: %v", verbs)
	}
}

func TestAlreadyEnabledAndRunningIsUnchanged(t *testing.T) {
	var verbs []string
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"rt","type":"service","unit":"docker.service","state":"running","boot":"enabled"}
	]}`)

	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried,
		systemctlStub(t, "loaded", "active", "enabled", &verbs), nil, nil)
	if err != nil {
		t.Fatalf("apply failed: %v", err)
	}
	if len(verbs) != 0 {
		t.Errorf("a unit already in the declared state was touched: %v", verbs)
	}
	if report.Changed() {
		t.Errorf("an unchanged service reported a change: %+v", report.Outcomes)
	}
}

func TestOmittingBootLeavesItAlone(t *testing.T) {
	// Absent means the host asserts nothing. A machine whose operator enabled something must
	// not have it silently disabled because a declaration did not mention it.
	var verbs []string
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"rt","type":"service","unit":"docker.service","state":"running"}
	]}`)
	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried,
		systemctlStub(t, "loaded", "inactive", "enabled", &verbs), nil, nil); err != nil {
		t.Fatal(err)
	}
	for _, v := range verbs {
		if v == "enable" || v == "disable" {
			t.Errorf("boot state was changed by a declaration that did not mention it: %v", verbs)
		}
	}
}

func TestAStaticUnitCannotBeEnabled(t *testing.T) {
	// `static` is neither enabled nor disabled: the unit has no install section and CANNOT be
	// enabled. Reading it as "disabled" would have the host try, fail, and blame the wrong
	// thing — the same shape as reading a missing unit as "stopped".
	var verbs []string
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"rt","type":"service","unit":"dbus.socket","state":"running","boot":"enabled"}
	]}`)

	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried,
		systemctlStub(t, "loaded", "active", "static", &verbs), nil, nil)
	if err == nil {
		t.Fatal("a static unit was accepted as enable-able")
	}
	if !strings.Contains(err.Error(), "no install section") {
		t.Errorf("failed for the wrong reason: %v", err)
	}
}

func TestAnUnknownBootStateIsRefusedNotGuessed(t *testing.T) {
	var verbs []string
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"rt","type":"service","unit":"x.service","state":"running","boot":"enabled"}
	]}`)
	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried,
		systemctlStub(t, "loaded", "active", "indirect", &verbs), nil, nil)
	if err == nil {
		t.Fatal("an unrecognised boot state was guessed at instead of refused")
	}
}

// --- more than one container runtime (novox/hq ADR 0005) ---

func TestTheRuntimeProbeIsPerRuntime(t *testing.T) {
	// Verified against a real podman 6.1.0 before this was written:
	//
	//   docker info --format '{{.ServerVersion}}'   ->  29.7.2
	//   podman info --format '{{.ServerVersion}}'   ->  Error: can't evaluate field
	//                                                   ServerVersion in type system.infoReport
	//   podman info --format '{{.Version.Version}}' ->  6.1.0
	//
	// So a single probe cannot find both, and a host that used docker's would report a machine
	// with podman as having no container runtime at all.
	for _, tc := range []struct {
		name, present, wantProbe string
	}{
		{"docker", "docker", "{{.ServerVersion}}"},
		{"podman", "podman", "{{.Version.Version}}"},
	} {
		t.Run(tc.name, func(t *testing.T) {
			var probedWith string
			run := func(ctx context.Context, name string, args ...string) (string, error) {
				if name != tc.present {
					return "", errors.New("not installed")
				}
				if args[0] == "info" {
					probedWith = args[2]
				}
				return "ok\n", nil
			}
			got, err := containerRuntime(context.Background(), run)
			if err != nil {
				t.Fatalf("%s was present and was not found: %v", tc.present, err)
			}
			if got != tc.present {
				t.Errorf("found %q, expected %q", got, tc.present)
			}
			if probedWith != tc.wantProbe {
				t.Errorf("probed %s with %q; that template does not work on it",
					tc.present, probedWith)
			}
		})
	}
}

func TestAContainerUsesTheRuntimeTheMachineHas(t *testing.T) {
	// The applier must not call `docker` on a machine that has podman. Adoption keeps what the
	// machine already has (novox/hq research 012), so hardcoding one contradicts it.
	var calledWith []string
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		if name == "docker" {
			return "", errors.New("not installed")
		}
		calledWith = append(calledWith, name)
		switch args[0] {
		case "info":
			return "6.1.0\n", nil
		case "inspect":
			return "false\t\n", errors.New("no such container")
		case "run":
			return "deadbeef\n", nil
		}
		return "", nil
	}
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"store","type":"container","name":"store","image":"`+pinned+`"}
	]}`)

	// It will fail at read-back — the stub never reports it running — and what matters is
	// WHICH binary it used getting there.
	_, _, _ = Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)

	for _, c := range calledWith {
		if c != "podman" {
			t.Errorf("called %q on a machine that only has podman", c)
		}
	}
	if len(calledWith) == 0 {
		t.Error("nothing was called; the runtime was not found")
	}
}

func TestNoRuntimeIsSaidPlainly(t *testing.T) {
	// Naming what was tried, because "docker: command not found" on a machine that deliberately
	// runs podman sends the reader looking for the wrong thing.
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		return "", errors.New("not installed")
	}
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"store","type":"container","name":"store","image":"`+pinned+`"}
	]}`)

	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
	if err == nil {
		t.Fatal("a machine with no container runtime applied a container")
	}
	for _, want := range []string{"docker", "podman", "no container runtime"} {
		if !strings.Contains(err.Error(), want) {
			t.Errorf("the failure does not mention %q: %v", want, err)
		}
	}
}

// archHost is the system these tests run against. They were written for pacman and systemd, and
// naming that is better than the implicit default it used to be.
func archHost(t *testing.T) system.System {
	t.Helper()
	s, err := system.For("arch")
	if err != nil {
		t.Fatal(err)
	}
	return s
}

func TestAServiceIsRestartedWhenWhatItReflectsChanges(t *testing.T) {
	// A running service does not re-read its configuration. Replace the file, find the service
	// already running, do nothing — and the machine keeps behaving as it did while every check
	// passes, because the file is right and the service is up.
	//
	// That is how a third node joining a mesh left the first two carrying a network that no
	// longer existed. Found in the lab; this is the shape of the fix.
	dir := t.TempDir()
	path := filepath.Join(dir, "thing.conf")

	d := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
	  {"id":"conf","type":"file","path":%q,"content":"first\n","mode":"0644"},
	  {"id":"svc","type":"service","unit":"thing.service","state":"running","restart-on":["conf"]}
	]}`, path))

	var commands []string
	run := recordingServices(&commands)

	if _, state, err := Apply(context.Background(), archHost(t), d, store.State{},
		store.OriginCarried, run, nil, nil); err != nil {
		t.Fatal(err)
	} else {
		// Second apply with the same content: nothing moved, so nothing restarts. A machine that
		// restarted its services on every reconcile would never be steady.
		commands = nil
		if _, _, err := Apply(context.Background(), archHost(t), d, state,
			store.OriginCarried, run, nil, nil); err != nil {
			t.Fatal(err)
		}
		for _, c := range commands {
			if strings.Contains(c, "stop") {
				t.Errorf("an unchanged declaration restarted the service: %s", c)
			}
		}

		// Now the file changes. The service is already running and must still be restarted.
		changedDecl := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
		  {"id":"conf","type":"file","path":%q,"content":"second\n","mode":"0644"},
		  {"id":"svc","type":"service","unit":"thing.service","state":"running","restart-on":["conf"]}
		]}`, path))
		commands = nil
		if _, _, err := Apply(context.Background(), archHost(t), changedDecl, state,
			store.OriginCarried, run, nil, nil); err != nil {
			t.Fatal(err)
		}
		var stopped, started bool
		for _, c := range commands {
			if strings.Contains(c, "stop thing.service") {
				stopped = true
			}
			if strings.Contains(c, "start thing.service") {
				started = true
			}
		}
		if !stopped || !started {
			t.Errorf("the file changed and the service was not restarted; commands were %v", commands)
		}
		reloaded, stop := -1, -1
		for i, c := range commands {
			if strings.Contains(c, "daemon-reload") && reloaded < 0 {
				reloaded = i
			}
			if strings.Contains(c, "stop thing.service") && stop < 0 {
				stop = i
			}
		}
		if reloaded < 0 || reloaded > stop {
			t.Errorf("the service was restarted without the unit files being read again first; commands were %v", commands)
		}
	}
}

func TestAServiceIsNotRestartedByAChangeItDoesNotName(t *testing.T) {
	// The list is what it reflects, not everything in the declaration. A service restarted by any
	// change anywhere would make every apply a fleet-wide bounce.
	dir := t.TempDir()
	conf := filepath.Join(dir, "thing.conf")
	other := filepath.Join(dir, "unrelated")

	first := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
	  {"id":"conf","type":"file","path":%q,"content":"same\n","mode":"0644"},
	  {"id":"other","type":"file","path":%q,"content":"one\n","mode":"0644"},
	  {"id":"svc","type":"service","unit":"thing.service","state":"running","restart-on":["conf"]}
	]}`, conf, other))

	var commands []string
	run := recordingServices(&commands)
	_, state, err := Apply(context.Background(), archHost(t), first, store.State{},
		store.OriginCarried, run, nil, nil)
	if err != nil {
		t.Fatal(err)
	}

	second := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
	  {"id":"conf","type":"file","path":%q,"content":"same\n","mode":"0644"},
	  {"id":"other","type":"file","path":%q,"content":"two\n","mode":"0644"},
	  {"id":"svc","type":"service","unit":"thing.service","state":"running","restart-on":["conf"]}
	]}`, conf, other))
	commands = nil
	if _, _, err := Apply(context.Background(), archHost(t), second, state,
		store.OriginCarried, run, nil, nil); err != nil {
		t.Fatal(err)
	}
	for _, c := range commands {
		if strings.Contains(c, "stop") {
			t.Errorf("a change to a file the service does not name restarted it: %s", c)
		}
	}
}

// recordingServices answers the way a machine with a running unit would, and remembers what it
// was asked to do — which is what a restart has to be proved by, since "running" looks the same
// before and after one.
func recordingServices(commands *[]string) Runner {
	return func(_ context.Context, name string, args ...string) (string, error) {
		line := name + " " + strings.Join(args, " ")
		*commands = append(*commands, line)
		switch {
		case strings.Contains(line, "is-enabled"):
			return "enabled", nil
		case strings.Contains(line, "show") && strings.Contains(line, "ActiveState"):
			return "LoadState=loaded\nActiveState=active\nSubState=running", nil
		}
		return "", nil
	}
}

func TestAFileChangedOnTheMachineIsCorrectedAndSaidSo(t *testing.T) {
	// The question this answers: how would anybody know somebody edited a managed file? Before
	// this they would not. It was rewritten within five minutes and reported as "updated",
	// which is what the mesh changing its mind looks like — so the person's change vanished and
	// nothing anywhere said why. They edit it again, and again.
	dir := t.TempDir()
	path := filepath.Join(dir, "thing.conf")
	d := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
	  {"id":"conf","type":"file","path":%q,"content":"from the mesh\n","mode":"0644"}
	]}`, path))

	_, state, err := Apply(context.Background(), archHost(t), d, store.State{},
		store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}

	// Somebody edits it.
	if err := os.WriteFile(path, []byte("edited by hand\n"), 0o644); err != nil {
		t.Fatal(err)
	}

	report, state, err := Apply(context.Background(), archHost(t), d, state,
		store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	if got := report.Outcomes[0].Action; got != "corrected" {
		t.Errorf("a hand edit was reported as %q; the mesh cannot tell it from changing its own "+
			"mind, and neither can anybody reading this", got)
	}

	// And it is put back, because holding the machine to what it was told is the point.
	back, err := os.ReadFile(path)
	if err != nil {
		t.Fatal(err)
	}
	if string(back) != "from the mesh\n" {
		t.Errorf("the file was left as %q", back)
	}
}

func TestTheMeshChangingItsMindIsNotDrift(t *testing.T) {
	// The other half. A new declaration is an ordinary update and must not read as somebody
	// having meddled, or every real change would look like an incident.
	dir := t.TempDir()
	path := filepath.Join(dir, "thing.conf")
	first := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
	  {"id":"conf","type":"file","path":%q,"content":"one\n","mode":"0644"}
	]}`, path))
	second := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
	  {"id":"conf","type":"file","path":%q,"content":"two\n","mode":"0644"}
	]}`, path))

	_, state, err := Apply(context.Background(), archHost(t), first, store.State{},
		store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	report, _, err := Apply(context.Background(), archHost(t), second, state,
		store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	if got := report.Outcomes[0].Action; got != "updated" {
		t.Errorf("the mesh changing what it wants was reported as %q", got)
	}
}

func TestAnUntouchedFileIsStillUnchanged(t *testing.T) {
	// And nothing about this makes a steady machine look busy.
	dir := t.TempDir()
	path := filepath.Join(dir, "thing.conf")
	d := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
	  {"id":"conf","type":"file","path":%q,"content":"steady\n","mode":"0644"}
	]}`, path))

	_, state, err := Apply(context.Background(), archHost(t), d, store.State{},
		store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	report, _, err := Apply(context.Background(), archHost(t), d, state,
		store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	if got := report.Outcomes[0].Action; got != "unchanged" {
		t.Errorf("an untouched file was reported as %q", got)
	}
}

func TestAFailedActionStopsWhatFollows(t *testing.T) {
	// An action is the only shape whose purpose is to make something true BEFORE the next thing
	// needs it, which is why it is the only one with a verify. The bootstrap is a row of them:
	// the store answers, then its databases exist, then their schemas, then the broker.
	//
	// Carrying on past one that did not happen starts things against a machine that is not ready
	// — and on a small machine that is how a database still initialising has its memory taken
	// away and shuts down. Observed in the lab, caused by a version of this loop that continued
	// past everything.
	dir := t.TempDir()
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"gate","type":"action","command":["false"],"verify":["false"]},
	  {"id":"after","type":"file","path":"`+filepath.Join(dir, "after.conf")+`","content":"b\n"}
	]}`)

	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried,
		ExecRunner, nil, nil)
	if err == nil {
		t.Fatal("an action that cannot succeed did not fail the apply")
	}
	var applyErr *Error
	if !errors.As(err, &applyErr) {
		t.Fatalf("got %T", err)
	}
	if !applyErr.Gated {
		t.Error("the failure does not say that nothing after it was attempted")
	}
	if _, statErr := os.Stat(filepath.Join(dir, "after.conf")); statErr == nil {
		t.Error("the apply continued past a failed action, which is a gate")
	}
	if !strings.Contains(err.Error(), "nothing after it was attempted") {
		t.Errorf("the message does not say the rest was not tried: %v", err)
	}
}

// A container is told which resolver to use, because it does not inherit the machine's names.
//
// A container gets its own `/etc/hosts` holding its own hostname, and a runtime rewrites
// `resolv.conf` — so every internal name the mesh wrote for the machine is invisible to what the
// machine is running. That was hit for real: a database client on one node could not resolve
// another node, on a mesh where both names were correct and present on both machines.
func TestAContainerIsGivenTheMeshsNames(t *testing.T) {
	var ran []string
	run := func(_ context.Context, name string, args ...string) (string, error) {
		if name != "docker" {
			return "", errors.New("not installed")
		}
		switch args[0] {
		case "info":
			return "29.0.0\n", nil
		case "inspect":
			return "false\t\n", errors.New("no such container")
		case "run":
			ran = args
			return "deadbeef\n", nil
		}
		return "", nil
	}
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"app","type":"container","name":"app","image":"`+pinned+`",
	   "hosts":["anchor.internal:10.42.0.1"]}
	]}`)
	_, _, _ = Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)

	var told bool
	for i, a := range ran {
		if a == "--add-host" && i+1 < len(ran) && ran[i+1] == "anchor.internal:10.42.0.1" {
			told = true
		}
	}
	if !told {
		t.Fatalf("the container cannot reach another machine by name: %v", ran)
	}
}

// And a container given no names is run exactly as before.
func TestAContainerGivenNoNamesIsRunAsBefore(t *testing.T) {
	var ran []string
	run := func(_ context.Context, name string, args ...string) (string, error) {
		if name != "docker" {
			return "", errors.New("not installed")
		}
		switch args[0] {
		case "info":
			return "29.0.0\n", nil
		case "inspect":
			return "false\t\n", errors.New("no such container")
		case "run":
			ran = args
			return "deadbeef\n", nil
		}
		return "", nil
	}
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"app","type":"container","name":"app","image":"`+pinned+`"}
	]}`)
	_, _, _ = Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)

	for _, a := range ran {
		if a == "--add-host" {
			t.Fatalf("a container given no names was given some anyway: %v", ran)
		}
	}
}

// Defends the ordering half of novox/hq work breakdown 1.3: resources are applied in the order
// the module declared them.
//
// **Already true, and untested until now** — the apply loop walks `d.Resources` and sorts nothing,
// so a module that needs one thing before another says so by writing it first. Worth an assertion
// because it is the kind of property a later change would break silently: sorting the resources
// for any good reason at all — by type, by identity, for a tidier report — would still pass every
// other test in this package.
//
// It is sequence, not readiness. A container started is not a container ready, and nothing here
// waits: what depends on something being *usable* retries, which is what both example
// provisioners do and is more robust than start ordering, because a dependency can also restart
// long after everything was applied.
func TestResourcesAreAppliedInTheOrderTheyWereDeclared(t *testing.T) {
	dir := t.TempDir()
	var order []string
	done := map[string]bool{}
	run := func(_ context.Context, name string, args ...string) (string, error) {
		order = append(order, name+" "+strings.Join(args, " "))
		// `verify` is the idempotency check, so it has to fail before the step and pass after —
		// a stub that always succeeds means every action is already done and nothing ever runs,
		// which is what the first version of this test measured.
		if name == "check" {
			if !done[args[0]] {
				return "", fmt.Errorf("not yet")
			}
			return "", nil
		}
		done[args[0]] = true
		return "", nil
	}

	_ = dir
	// Trusted, because the link may not carry an action and only a bundle may (novox/hq ADR 0005).
	// Actions are used here because they are the one shape whose execution is observable through
	// the runner, which is what makes the order visible at all.
	d, err := declaration.ParseTrusted([]byte(`{"declaration":1,"resources":[` +
		`{"id":"first","type":"action","command":["step","one"],"verify":["check","one"]},` +
		`{"id":"second","type":"action","command":["step","two"],"verify":["check","two"]},` +
		`{"id":"third","type":"action","command":["step","three"],"verify":["check","three"]}]}`))
	if err != nil {
		t.Fatal(err)
	}

	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{},
		store.OriginCarried, run, nil, nil); err != nil {
		t.Fatal(err)
	}

	var ran []string
	for _, line := range order {
		if strings.HasPrefix(line, "step ") {
			ran = append(ran, strings.TrimPrefix(line, "step "))
		}
	}
	want := []string{"one", "two", "three"}
	if strings.Join(ran, ",") != strings.Join(want, ",") {
		t.Fatalf("declared one, two, three and ran %v — a module that needs one thing before "+
			"another has no way to say so", ran)
	}
}

// **A data directory is never removed by the mesh.** The one failure in this system that cannot
// be undone.
//
// Unassigning a module made its directory an orphan, and an orphan directory was deleted with
// everything under it — a database's files, a mail spool, somebody's uploads — and the report
// said "removed". Reproduced before it was fixed: a module was assigned, a service wrote into
// its directory, the module was unassigned, and the file was gone.
//
// What makes the rule safe rather than merely cautious is the removal order. Everything the mesh
// puts in a directory is itself a declared resource, and orphans are removed in reverse
// declaration order — so what the mesh wrote is already gone by the time the directory is
// reached. Anything still there was put there by something else.
func TestADirectoryHoldingAnythingTheMeshDidNotPutThereIsKept(t *testing.T) {
	root := t.TempDir()
	data := filepath.Join(root, "keycloak")

	d := declare(t, `{"id":"data","type":"directory","path":"`+data+`","mode":"0700"}`)
	_, state, err := Apply(context.Background(), archHost(t), d, store.State{},
		store.OriginDeclared, nil, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	live := filepath.Join(data, "realm.db")
	if err := os.WriteFile(live, []byte("everybody's logins"), 0o600); err != nil {
		t.Fatal(err)
	}

	// The module is unassigned: the mesh declares something else entirely.
	after := declare(t, `{"id":"other","type":"directory","path":"`+filepath.Join(root, "other")+`","mode":"0700"}`)
	report, _, err := Apply(context.Background(), archHost(t), after, state,
		store.OriginDeclared, nil, nil, nil)
	if err != nil {
		t.Fatal(err)
	}

	if _, err := os.Stat(live); err != nil {
		t.Fatalf("the data was deleted by unassigning a module: %v", err)
	}
	// And it is said, rather than left for somebody to notice. A directory quietly left behind is
	// how a machine accumulates things nobody can account for.
	var said bool
	for _, o := range report.Outcomes {
		if o.Action == "kept" && strings.Contains(o.Detail, "did not put there") {
			said = true
		}
	}
	if !said {
		t.Errorf("the directory was kept and nothing reported it: %+v", report.Outcomes)
	}
}

// An empty one is the mesh's own, and goes.
func TestAnEmptyDirectoryIsStillRemoved(t *testing.T) {
	root := t.TempDir()
	mine := filepath.Join(root, "config")
	d := declare(t, `{"id":"c","type":"directory","path":"`+mine+`","mode":"0700"}`)
	_, state, err := Apply(context.Background(), archHost(t), d, store.State{},
		store.OriginDeclared, nil, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	after := declare(t, `{"id":"other","type":"directory","path":"`+filepath.Join(root, "other")+`","mode":"0700"}`)
	if _, _, err := Apply(context.Background(), archHost(t), after, state,
		store.OriginDeclared, nil, nil, nil); err != nil {
		t.Fatal(err)
	}
	if _, err := os.Stat(mine); !errors.Is(err, os.ErrNotExist) {
		t.Fatal("an empty directory the mesh made was left behind, so nothing is ever cleaned up")
	}
}

// A container holding values from before is replaced, even when nothing changed this pass.
//
// **This is the fault the restart-on tripwire could not catch** (novox/hq 04-ISSUES/045). That
// mechanism fires while a resource is being changed, so it covers the apply where the file moved
// and nothing afterwards. A file written in an earlier apply — or written before the container
// declared it as a dependency — leaves a process holding a credential the mesh has already
// replaced, and every check passes: the container is up, the spec matched, the machine reported
// success. Making what a container reads part of what it is turns that from a tripwire into a
// standing comparison.
func TestAContainerStaleFromAnEarlierApplyIsReplaced(t *testing.T) {
	dir := t.TempDir()
	conf := filepath.Join(dir, "db.env")
	// Already on disk with the current content, so this apply changes nothing at all.
	if err := os.WriteFile(conf, []byte("PASSWORD=new\n"), 0o600); err != nil {
		t.Fatal(err)
	}
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"env","type":"file","path":"`+conf+`","content":"PASSWORD=new\n","mode":"0600"},
	  {"id":"app","type":"container","name":"app","image":"`+pinned+`","restart-on":["env"]}
	]}`)

	// The container was created when the file said something else.
	was := map[string]string{"env": declaredDigest(&declaration.File{Content: "PASSWORD=old\n"})}
	now := map[string]string{"env": declaredDigest(d.Resources[0].(*declaration.File))}
	stale := containerSpec(d.Resources[1].(*declaration.Container), was)
	fresh := containerSpec(d.Resources[1].(*declaration.Container), now)

	var removed, created bool
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		switch args[0] {
		case "info":
			return "27.0\n", nil
		case "inspect":
			if created {
				return "true\t" + fresh, nil
			}
			return "true\t" + stale, nil
		case "rm":
			removed = true
			return "", nil
		case "run":
			created = true
			return "deadbeef\n", nil
		}
		return "", nil
	}

	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
	if err != nil {
		t.Fatalf("apply failed: %v", err)
	}
	if !removed || !created {
		t.Fatalf("a container running values the machine no longer holds was left alone "+
			"(removed=%v created=%v)", removed, created)
	}
}

// A seed is written once. What grows in it afterwards is somebody else's work the mesh asked for,
// and a reconcile leaves it alone — content, mode and owner — and says so (novox/hq issue 035).
func TestASeedIsCreatedOnceAndWhatGrowsInItIsKept(t *testing.T) {
	dir := t.TempDir()
	path := filepath.Join(dir, "acl.conf")
	d := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
	  {"id":"acl","type":"file","path":%q,"content":"user default on\n","mode":"0600","create-once":true}
	]}`, path))

	report, state, err := Apply(context.Background(), archHost(t), d, store.State{},
		store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	if got := report.Outcomes[0].Action; got != "created" {
		t.Fatalf("first apply: %q", got)
	}

	// The program persists into it.
	if err := os.WriteFile(path, []byte("user default on\nuser app-one on >secret\n"), 0o600); err != nil {
		t.Fatal(err)
	}

	report, _, err = Apply(context.Background(), archHost(t), d, state,
		store.OriginCarried, noServices, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	if got := report.Outcomes[0].Action; got != "kept" {
		t.Fatalf("second apply: %q — a seed was reconciled", got)
	}
	grown, _ := os.ReadFile(path)
	if string(grown) != "user default on\nuser app-one on >secret\n" {
		t.Fatalf("what grew in the seed was wiped: %q", grown)
	}
}