mesh-host/internal/apply/hold_test.go

package apply

import (
	"context"
	"errors"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/novox/mesh-host/internal/declaration"
	"github.com/novox/mesh-host/internal/store"
)

// Defends novox/hq ADR 0100: on an adopted node, what is found is kept until its module is taken.

// machine is a fake container runtime holding containers by name: id, running, and the host's spec
// label when a host made it. Every command it is asked is written down.
type machine struct {
	containers map[string]*fakeContainer
	asked      []string

	// units are service units by name, as systemd would report them; volumes are the runtime's
	// named volumes.
	units   map[string]*fakeUnit
	volumes map[string]bool
	users   map[string]bool
}

type fakeUnit struct {
	active, enabled string
	// fragment is where systemd loads the unit from; empty means /etc/systemd/system, where an
	// administrator installs one.
	fragment string
}

// systemctl answers as systemd does for the units the machine has, and "not-found" for any other.
func (m *machine) systemctl(args []string) (string, error) {
	unit := args[len(args)-1]
	if args[0] == "show" {
		unit = args[1]
	}
	u, ok := m.units[unit]
	switch args[0] {
	case "show":
		if !ok {
			if len(args) > 2 && strings.Contains(args[2], "FragmentPath") {
				return "FragmentPath=\n", nil
			}
			return "LoadState=not-found\nActiveState=inactive\nType=simple\n", nil
		}
		if len(args) > 2 && strings.Contains(args[2], "FragmentPath") {
			from := u.fragment
			if from == "" {
				from = "/etc/systemd/system/" + unit
			}
			return "FragmentPath=" + from + "\n", nil
		}
		return "LoadState=loaded\nActiveState=" + u.active + "\nType=simple\nRemainAfterExit=no\n", nil
	case "is-enabled":
		if !ok {
			return "", errors.New("not found")
		}
		return u.enabled + "\n", nil
	case "start":
		u.active = "active"
	case "stop":
		u.active = "inactive"
	case "enable":
		u.enabled = "enabled"
	case "disable":
		u.enabled = "disabled"
	}
	return "", nil
}

func (m *machine) did(prefix string) bool {
	for _, a := range m.asked {
		if strings.HasPrefix(a, prefix) {
			return true
		}
	}
	return false
}

type fakeContainer struct {
	id      string
	running bool
	spec    string
}

func (m *machine) run(_ context.Context, name string, args ...string) (string, error) {
	m.asked = append(m.asked, name+" "+strings.Join(args, " "))
	if name == "systemctl" {
		return m.systemctl(args)
	}
	if name == "getent" {
		if m.users[args[len(args)-1]] {
			return args[len(args)-1] + ":x:1500:1500::/home/" + args[len(args)-1] + ":/bin/bash\n", nil
		}
		return "", errors.New("exit status 2")
	}
	if name != "docker" {
		return "", nil
	}
	switch args[0] {
	case "volume":
		if m.volumes[args[len(args)-1]] {
			return "[]\n", nil
		}
		return "", errors.New("no such volume")
	case "info":
		return "27.0\n", nil
	case "inspect":
		c, ok := m.containers[args[len(args)-1]]
		if !ok {
			return "", errors.New("no such container")
		}
		running := "false"
		if c.running {
			running = "true"
		}
		if strings.HasPrefix(args[2], "{{.Id}}") {
			return c.id + "\t" + running + "\t" + c.spec + "\n", nil
		}
		return running + "\t" + c.spec + "\n", nil
	case "rm":
		delete(m.containers, args[len(args)-1])
		return "", nil
	case "run":
		var name, spec string
		for i, a := range args {
			if a == "--name" {
				name = args[i+1]
			}
			if a == "--label" && strings.HasPrefix(args[i+1], specLabel+"=") {
				spec = strings.TrimPrefix(args[i+1], specLabel+"=")
			}
		}
		m.containers[name] = &fakeContainer{id: "made-by-host", running: true, spec: spec}
		return "made-by-host\n", nil
	}
	return "", nil
}

func (m *machine) removed(name string) bool {
	for _, a := range m.asked {
		if strings.HasPrefix(a, "docker rm") && strings.HasSuffix(a, " "+name) {
			return true
		}
	}
	return false
}

func adopted(t *testing.T, adoption, resources string) *declaration.Declaration {
	t.Helper()
	return parse(t, `{"declaration":1,"adoption":`+adoption+`,"resources":[`+resources+`]}`)
}

const untakenWeb = `{"taken":[],"untaken":{"hello-web":["hello-web.page","hello-web.server"]}}`
const takenWeb = `{"taken":["hello-web"]}`

func webResources(page string) string {
	return `{"id":"hello-web.page","type":"file","path":"` + page + `","content":"the mesh's page\n"},
	  {"id":"hello-web.server","type":"container","name":"hello-web","image":"` + pinned + `"}`
}

func applyAdopted(t *testing.T, d *declaration.Declaration, known store.State, m *machine, keepDir string) (Report, store.State) {
	t.Helper()
	report, state, err := ApplyKeeping(context.Background(), archHost(t), d, known,
		store.OriginDeclared, m.run, nil, nil, KeepIn(keepDir))
	if err != nil {
		t.Fatalf("apply failed: %v", err)
	}
	return report, state
}

func outcomeOf(r Report, id string) Outcome {
	for _, o := range r.Outcomes {
		if o.ID == id {
			return o
		}
	}
	return Outcome{}
}

func predecessor(t *testing.T) (dir, page string, m *machine) {
	t.Helper()
	dir = t.TempDir()
	page = filepath.Join(dir, "index.html")
	if err := os.WriteFile(page, []byte("the predecessor's page\n"), 0o640); err != nil {
		t.Fatal(err)
	}
	return dir, page, &machine{containers: map[string]*fakeContainer{
		"hello-web": {id: "predecessor-id", running: true},
	}}
}

func TestAFoundFileOfAnUntakenModuleIsKeptAsItIs(t *testing.T) {
	dir, page, m := predecessor(t)
	report, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)

	got, _ := os.ReadFile(page)
	if string(got) != "the predecessor's page\n" {
		t.Fatalf("a found file was changed: %q", got)
	}
	info, _ := os.Stat(page)
	if info.Mode().Perm() != 0o640 {
		t.Errorf("a found file's mode was changed to %o", info.Mode().Perm())
	}
	if o := outcomeOf(report, "hello-web.page"); o.Action != "held" ||
		!strings.Contains(o.Detail, "kept until hello-web is taken") {
		t.Errorf("the found file was not reported held: %+v", o)
	}
	h, ok := state.HeldAt("hello-web.page")
	if !ok || h.Module != "hello-web" || h.Mode != "0640" {
		t.Fatalf("the hold was not recorded: %+v", h)
	}
	kept, err := os.ReadFile(h.Kept)
	if err != nil || string(kept) != "the predecessor's page\n" {
		t.Fatalf("the original was not kept: %q %v", kept, err)
	}
	if info, _ := os.Stat(h.Kept); info.Mode().Perm() != 0o600 {
		t.Errorf("the kept original is mode %o", info.Mode().Perm())
	}
	if _, recorded := state.Find("hello-web.page"); recorded {
		t.Error("a held file was recorded as applied, so it would be removed as an orphan")
	}
	if report.Changed() {
		t.Errorf("holding was reported as changing the machine: %+v", report.Outcomes)
	}
}

func TestAFoundContainerOfAnUntakenModuleIsNotReplaced(t *testing.T) {
	dir, page, m := predecessor(t)
	report, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)

	if m.removed("hello-web") {
		t.Fatal("a found container was removed")
	}
	for _, a := range m.asked {
		if strings.HasPrefix(a, "docker run") {
			t.Fatalf("a container was started over a found one: %s", a)
		}
	}
	if outcomeOf(report, "hello-web.server").Action != "held" {
		t.Errorf("the found container was not held: %+v", report.Outcomes)
	}
	if h, _ := state.HeldAt("hello-web.server"); h.Container != "predecessor-id" || !h.Running {
		t.Errorf("the container as found was not recorded: %+v", h)
	}
}

func TestWhatIsNotFoundIsCreatedWhenAssigned(t *testing.T) {
	// Assigning prepares: what the module declares that is not there is made.
	dir := t.TempDir()
	page := filepath.Join(dir, "index.html")
	m := &machine{containers: map[string]*fakeContainer{}}
	report, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
	if o := outcomeOf(report, "hello-web.page"); o.Action != "created" {
		t.Errorf("an absent file of an untaken module was not created: %+v", o)
	}
	if o := outcomeOf(report, "hello-web.server"); o.Action != "created" {
		t.Errorf("an absent container of an untaken module was not created: %+v", o)
	}
	if len(state.Held) != 0 {
		t.Errorf("something was held that was not found: %+v", state.Held)
	}
}

func TestAFileThisHostWroteIsNotFound(t *testing.T) {
	// Found means present with no record. A record of any origin — carried or declared, this life
	// of the node or an earlier one — means this host wrote it.
	for _, origin := range []string{store.OriginCarried, store.OriginDeclared} {
		dir, page, m := predecessor(t)
		known := store.State{Resources: []store.Applied{
			{ID: "earlier-name", Type: "file", Target: page, Origin: origin}}}
		report, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), known, m, dir)
		if o := outcomeOf(report, "hello-web.page"); o.Action == "held" {
			t.Errorf("%s: a file this host has a record of was held: %+v", origin, o)
		}
		if _, held := state.HeldAt("hello-web.page"); held {
			t.Errorf("%s: a recorded file was held", origin)
		}
	}
}

func TestAContainerAHostMadeIsNotFound(t *testing.T) {
	dir, page, m := predecessor(t)
	m.containers["hello-web"].spec = "some-spec"
	report, _ := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
	if o := outcomeOf(report, "hello-web.server"); o.Action == "held" {
		t.Errorf("a container carrying the host's spec label was held: %+v", o)
	}
}

func TestTheGenesisStoreAdoptedInPlaceIsNotFound(t *testing.T) {
	// ADR 0078: the foundation's store, raised from the bundle and recorded as carried, is adopted
	// as a module by name. It is the mesh's own and must never read as a predecessor's.
	dir := t.TempDir()
	m := &machine{containers: map[string]*fakeContainer{"mesh-store": {id: "x", running: true}}}
	known := store.State{Resources: []store.Applied{{ID: "store", Type: "container", Target: "mesh-store"}}}
	d := adopted(t, `{"taken":[],"untaken":{"postgres":["postgres.server"]}}`,
		`{"id":"postgres.server","type":"container","name":"mesh-store","image":"`+pinned+`"}`)
	report, state := applyAdopted(t, d, known, m, dir)
	if o := outcomeOf(report, "postgres.server"); o.Action == "held" {
		t.Errorf("the carried store was held: %+v", o)
	}
	if len(state.Held) != 0 {
		t.Errorf("the carried store was held: %+v", state.Held)
	}
}

func TestTakingAModuleReplacesWhatWasHeldAndTheOriginalSurvives(t *testing.T) {
	dir, page, m := predecessor(t)
	_, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
	h, _ := state.HeldAt("hello-web.page")

	report, state := applyAdopted(t, adopted(t, takenWeb, webResources(page)), state, m, dir)
	got, _ := os.ReadFile(page)
	if string(got) != "the mesh's page\n" {
		t.Fatalf("taking the module did not converge the file: %q", got)
	}
	if !m.removed("hello-web") || m.containers["hello-web"].id != "made-by-host" {
		t.Fatal("taking the module did not replace the found container")
	}
	if o := outcomeOf(report, "hello-web.page"); !strings.Contains(o.Detail, "original kept at "+h.Kept) {
		t.Errorf("the cutover does not say where the original is: %+v", o)
	}
	if len(state.Held) != 0 {
		t.Errorf("what was taken is still held: %+v", state.Held)
	}
	if _, recorded := state.Find("hello-web.page"); !recorded {
		t.Error("a taken file was not recorded as applied")
	}
	kept, err := os.ReadFile(h.Kept)
	if err != nil || string(kept) != "the predecessor's page\n" {
		t.Errorf("the kept original did not survive the cutover: %q %v", kept, err)
	}
}

func TestAHeldFileIsNeverRemovedWhenItsModuleIsUnassigned(t *testing.T) {
	dir, page, m := predecessor(t)
	_, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)

	other := filepath.Join(dir, "other")
	_, state = applyAdopted(t, adopted(t, `{"taken":[]}`,
		`{"id":"x.other","type":"file","path":"`+other+`","content":"x"}`), state, m, dir)
	if got, _ := os.ReadFile(page); string(got) != "the predecessor's page\n" {
		t.Fatalf("a held file was touched when its module left: %q", got)
	}
	if m.removed("hello-web") {
		t.Fatal("a held container was removed when its module left")
	}
	if _, still := state.HeldAt("hello-web.page"); still {
		t.Error("a hold outlived its resource leaving the declaration, so the node reports it for ever")
	}
	if _, recorded := state.Find("hello-web.page"); recorded {
		t.Error("a file let go was recorded as the mesh's")
	}
}

func TestAHoldNoLongerDeclaredIsLetGoAndFoundAgainIfItsModuleReturns(t *testing.T) {
	dir, page, m := predecessor(t)
	_, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
	first, _ := state.HeldAt("hello-web.page")

	report, state := applyAdopted(t, adopted(t, `{"taken":[]}`, withConf(dir)), state, m, dir)
	if o := outcomeOf(report, "hello-web.page"); o.Action != "forgotten" || o.Detail != "no longer declared; left as found" {
		t.Errorf("letting a hold go was not reported: %+v", o)
	}
	if len(state.Held) != 0 {
		t.Errorf("holds outlived their resources: %+v", state.Held)
	}

	// The module comes back: what is there is found again, and the original first kept stays.
	if err := os.WriteFile(page, []byte("the predecessor wrote again\n"), 0o640); err != nil {
		t.Fatal(err)
	}
	report, state = applyAdopted(t, adopted(t, untakenWeb, webResources(page)), state, m, dir)
	if o := outcomeOf(report, "hello-web.page"); o.Action != "held" {
		t.Fatalf("a returning module's found file was not held again: %+v", o)
	}
	again, _ := state.HeldAt("hello-web.page")
	if kept, _ := os.ReadFile(again.Kept); again.Kept != first.Kept || string(kept) != "the predecessor's page\n" {
		t.Errorf("the first kept original was lost: %s %q", again.Kept, kept)
	}
	if got, _ := os.ReadFile(page); string(got) != "the predecessor wrote again\n" {
		t.Errorf("the found file was touched: %q", got)
	}
}

func TestACarriedApplyLetsNoHoldGo(t *testing.T) {
	dir, page, m := predecessor(t)
	_, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
	carried := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`)
	_, state, err := ApplyKeeping(context.Background(), archHost(t), carried, state, store.OriginCarried,
		m.run, nil, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	if len(state.Held) != 2 {
		t.Errorf("a carried apply let holds go: %+v", state.Held)
	}
}

func TestAHeldFileRewrittenIsReportedAndNotReverted(t *testing.T) {
	dir, page, m := predecessor(t)
	d := adopted(t, untakenWeb, webResources(page))
	_, state := applyAdopted(t, d, store.State{}, m, dir)

	if err := os.WriteFile(page, []byte("the predecessor wrote again\n"), 0o640); err != nil {
		t.Fatal(err)
	}
	report, state := applyAdopted(t, d, state, m, dir)
	if got, _ := os.ReadFile(page); string(got) != "the predecessor wrote again\n" {
		t.Fatalf("a held file was reverted: %q", got)
	}
	if h, _ := state.HeldAt("hello-web.page"); h.Changed != "rewritten" || h.ChangedAt.IsZero() {
		t.Errorf("a rewrite was not recorded: %+v", h)
	}
	if o := outcomeOf(report, "hello-web.page"); !strings.Contains(o.Detail, "rewritten") {
		t.Errorf("a rewrite was not reported: %+v", o)
	}
	h, _ := state.HeldAt("hello-web.page")
	if kept, _ := os.ReadFile(h.Kept); string(kept) != "the predecessor's page\n" {
		t.Errorf("the kept original was overwritten by a later write: %q", kept)
	}
}

func TestAHeldContainerStoppedOrReplacedIsReportedAndNotRestarted(t *testing.T) {
	for _, c := range []struct {
		change func(*machine)
		want   string
	}{
		{func(m *machine) { m.containers["hello-web"].running = false }, "stopped"},
		{func(m *machine) { m.containers["hello-web"].id = "another" }, "replaced"},
		{func(m *machine) { delete(m.containers, "hello-web") }, "gone"},
	} {
		dir, page, m := predecessor(t)
		d := adopted(t, untakenWeb, webResources(page))
		_, state := applyAdopted(t, d, store.State{}, m, dir)
		c.change(m)
		m.asked = nil
		_, state = applyAdopted(t, d, state, m, dir)
		if h, _ := state.HeldAt("hello-web.server"); h.Changed != c.want {
			t.Errorf("%s: recorded as %q", c.want, h.Changed)
		}
		for _, a := range m.asked {
			if strings.HasPrefix(a, "docker run") || strings.HasPrefix(a, "docker rm") ||
				strings.HasPrefix(a, "docker start") {
				t.Errorf("%s: the held container was acted on: %s", c.want, a)
			}
		}
	}
}

func TestAHeldFileThatVanishesIsNotCreated(t *testing.T) {
	dir, page, m := predecessor(t)
	d := adopted(t, untakenWeb, webResources(page))
	_, state := applyAdopted(t, d, store.State{}, m, dir)
	if err := os.Remove(page); err != nil {
		t.Fatal(err)
	}
	_, state = applyAdopted(t, d, state, m, dir)
	if _, err := os.Stat(page); !errors.Is(err, os.ErrNotExist) {
		t.Fatal("a held file that vanished was created before its module was taken")
	}
	if h, _ := state.HeldAt("hello-web.page"); h.Changed != "gone" {
		t.Errorf("a vanished held file was not reported gone: %+v", h)
	}
}

func TestAConvergedNodeStillReplacesWhatItFinds(t *testing.T) {
	// No adoption, no holds: byte for byte what a converged node did before ADR 0100.
	dir, page, m := predecessor(t)
	d := parse(t, `{"declaration":1,"resources":[`+webResources(page)+`]}`)
	report, state := applyAdopted(t, d, store.State{}, m, dir)
	if got, _ := os.ReadFile(page); string(got) != "the mesh's page\n" {
		t.Errorf("a converged node kept a found file: %q", got)
	}
	if !m.removed("hello-web") {
		t.Error("a converged node kept a found container")
	}
	if len(state.Held) != 0 || outcomeOf(report, "hello-web.page").Action == "held" {
		t.Errorf("a converged node held something: %+v", state.Held)
	}
	// What it writes over that it has no record of, it keeps first — on any node.
	o := outcomeOf(report, "hello-web.page")
	kept := o.Detail[strings.Index(o.Detail, "kept at ")+len("kept at "):]
	if got, err := os.ReadFile(kept); err != nil || string(got) != "the predecessor's page\n" {
		t.Errorf("the file written over was not kept, or not named: %q (%s) %v", got, o.Detail, err)
	}
}

func TestAFileWrittenOverIsKeptOnceAndOnlyWhenTheHostHasNoRecordOfIt(t *testing.T) {
	dir := t.TempDir()
	conf := filepath.Join(dir, "nftables.conf")
	_ = os.WriteFile(conf, []byte("# the distribution's own\n"), 0o644)
	decl := func(content string) *declaration.Declaration {
		return parse(t, `{"declaration":1,"resources":[{"id":"nftables.config","type":"file","path":"`+conf+
			`","content":"`+content+`"}]}`)
	}
	m := &machine{containers: map[string]*fakeContainer{}}
	report, state := applyAdopted(t, decl("table inet mesh {}\\n"), store.State{}, m, dir)
	o := outcomeOf(report, "nftables.config")
	if !strings.Contains(o.Detail, "had no record of, was kept at ") {
		t.Fatalf("writing over an unrecorded file did not keep it: %+v", o)
	}
	kept := o.Detail[strings.Index(o.Detail, "kept at ")+len("kept at "):]
	if got, _ := os.ReadFile(kept); string(got) != "# the distribution's own\n" {
		t.Errorf("the kept original is %q", got)
	}

	// Now the mesh's: a later change keeps nothing more, and the first original stays.
	report, _ = applyAdopted(t, decl("table inet mesh { }\\n"), state, m, dir)
	if o := outcomeOf(report, "nftables.config"); o.Action != "updated" || strings.Contains(o.Detail, "kept at") {
		t.Errorf("a file the mesh wrote was kept again: %+v", o)
	}
	if got, _ := os.ReadFile(kept); string(got) != "# the distribution's own\n" {
		t.Errorf("the first original was overwritten: %q", got)
	}
}

// Defends novox/hq ADR 0103: found covers every kind that can reach what the machine already has.

// untaken is an adoption with hello-web untaken, listing these of its resources.
func untaken(ids ...string) string {
	return `{"taken":[],"untaken":{"hello-web":["` + strings.Join(ids, `","`) + `"]}}`
}

func TestAFoundDirectoryOfAnUntakenModuleKeepsItsModeOwnerAndContents(t *testing.T) {
	dir := t.TempDir()
	data := filepath.Join(dir, "data")
	if err := os.Mkdir(data, 0o700); err != nil {
		t.Fatal(err)
	}
	inside := filepath.Join(data, "PG_VERSION")
	if err := os.WriteFile(inside, []byte("16\n"), 0o600); err != nil {
		t.Fatal(err)
	}
	m := &machine{containers: map[string]*fakeContainer{}}
	report, state := applyAdopted(t, adopted(t, untaken("hello-web.data"),
		`{"id":"hello-web.data","type":"directory","path":"`+data+`","mode":"0755"}`), store.State{}, m, dir)

	if info, _ := os.Stat(data); info.Mode().Perm() != 0o700 {
		t.Errorf("a found directory was re-moded to %o", info.Mode().Perm())
	}
	if got, _ := os.ReadFile(inside); string(got) != "16\n" {
		t.Errorf("what is inside a found directory was touched: %q", got)
	}
	if o := outcomeOf(report, "hello-web.data"); o.Action != "held" || !strings.Contains(o.Detail, "mode, owner and contents") {
		t.Errorf("the found directory was not held: %+v", o)
	}
	if h, ok := state.HeldAt("hello-web.data"); !ok || h.Mode != "0700" {
		t.Errorf("the directory as found was not recorded: %+v", h)
	}
	if _, recorded := state.Find("hello-web.data"); recorded {
		t.Error("a held directory was recorded as applied")
	}
}

func TestADirectoryMadeInTheSameApplyIsNotFound(t *testing.T) {
	// A file's parent is made as the file is written; what the mesh made is not found.
	dir := t.TempDir()
	data := filepath.Join(dir, "data")
	m := &machine{containers: map[string]*fakeContainer{}}
	report, state := applyAdopted(t, adopted(t, untaken("hello-web.conf", "hello-web.data"),
		`{"id":"hello-web.conf","type":"file","path":"`+filepath.Join(data, "conf")+`","content":"x\n"},
		 {"id":"hello-web.data","type":"directory","path":"`+data+`","mode":"0750"}`), store.State{}, m, dir)
	if o := outcomeOf(report, "hello-web.data"); o.Action == "held" {
		t.Errorf("a directory the apply itself made was held: %+v", o)
	}
	if info, _ := os.Stat(data); info.Mode().Perm() != 0o750 {
		t.Errorf("the mesh's own directory was not converged: %o", info.Mode().Perm())
	}
	if len(state.Held) != 0 {
		t.Errorf("held: %+v", state.Held)
	}
}

func TestAFoundServiceOfAnUntakenModuleIsNeitherStartedNorEnabledNorRestarted(t *testing.T) {
	dir := t.TempDir()
	conf := filepath.Join(dir, "hello.conf")
	m := &machine{containers: map[string]*fakeContainer{},
		// The predecessor's unit: stopped just now, but it starts at boot, so it is the machine's.
		units: map[string]*fakeUnit{"hello.service": {active: "inactive", enabled: "enabled"}}}
	report, state := applyAdopted(t, adopted(t, untaken("hello-web.conf", "hello-web.unit"),
		`{"id":"hello-web.conf","type":"file","path":"`+conf+`","content":"x\n"},
		 {"id":"hello-web.unit","type":"service","unit":"hello.service","state":"running","boot":"enabled",
		  "restart-on":["hello-web.conf"]}`), store.State{}, m, dir)

	for _, verb := range []string{"systemctl start", "systemctl stop", "systemctl enable", "systemctl disable", "systemctl restart"} {
		if m.did(verb) {
			t.Errorf("a found service was changed: %s (%v)", verb, m.asked)
		}
	}
	if o := outcomeOf(report, "hello-web.unit"); o.Action != "held" || !strings.Contains(o.Detail, "starts at boot") {
		t.Errorf("the found service was not held: %+v", o)
	}
	if h, ok := state.HeldAt("hello-web.unit"); !ok || h.Running {
		t.Errorf("the service as found was not recorded: %+v", h)
	}
}

func TestAHeldServiceIsStillReloadedButNeverRestarted(t *testing.T) {
	// A reload stops nothing (novox/hq ADR 0102); a restart would stop the predecessor's service.
	dir := t.TempDir()
	conf := filepath.Join(dir, "daemon.json")
	m := &machine{containers: map[string]*fakeContainer{},
		units: map[string]*fakeUnit{"docker.service": {active: "active", enabled: "enabled"}}}
	report, _ := applyAdopted(t, adopted(t, untaken("hello-web.conf", "hello-web.unit"),
		`{"id":"hello-web.conf","type":"file","path":"`+conf+`","content":"{}\n"},
		 {"id":"hello-web.unit","type":"service","unit":"docker.service","state":"running","boot":"enabled",
		  "reload-on":["hello-web.conf"]}`), store.State{}, m, dir)
	if !m.did("systemctl reload docker.service") {
		t.Errorf("a held service was not reloaded for what it re-reads: %v", m.asked)
	}
	if m.did("systemctl stop") || m.did("systemctl start") {
		t.Errorf("a held service was restarted: %v", m.asked)
	}
	if o := outcomeOf(report, "hello-web.unit"); o.Action != "held" || !strings.Contains(o.Detail, "reloaded for hello-web.conf") {
		t.Errorf("the reload was not reported on the hold: %+v", o)
	}
}

func TestAUnitAPackageOnlyShipsIsNotFound(t *testing.T) {
	// The adoption bed found this: the private network's wg-quick@mesh0 is an instance of a unit
	// the tunnel package ships. Nothing had ever run it, yet it was held as a predecessor's, and
	// the private network never came up. Found is what the machine runs.
	dir := t.TempDir()
	m := &machine{containers: map[string]*fakeContainer{},
		units: map[string]*fakeUnit{"wg-quick@mesh0.service": {active: "inactive", enabled: "disabled",
			fragment: "/usr/lib/systemd/system/wg-quick@.service"}}}
	report, state := applyAdopted(t, adopted(t, untaken("mesh-wireguard.overlay-up"),
		`{"id":"mesh-wireguard.overlay-up","type":"service","unit":"wg-quick@mesh0.service","state":"running","boot":"enabled"}`),
		store.State{}, m, dir)
	if o := outcomeOf(report, "mesh-wireguard.overlay-up"); o.Action == "held" {
		t.Fatalf("a unit nothing runs was held as a predecessor's: %+v", o)
	}
	if !m.did("systemctl start wg-quick@mesh0.service") || !m.did("systemctl enable wg-quick@mesh0.service") {
		t.Errorf("the unit was not started and enabled: %v", m.asked)
	}
	if len(state.Held) != 0 {
		t.Errorf("held: %+v", state.Held)
	}
}

func TestAServiceWhoseUnitIsNotThereIsAppliedAsUsual(t *testing.T) {
	// No unit before the apply: nothing of a predecessor's to hold.
	dir := t.TempDir()
	m := &machine{containers: map[string]*fakeContainer{}, units: map[string]*fakeUnit{}}
	d := adopted(t, untaken("hello-web.unit"), `{"id":"hello-web.unit","type":"service","unit":"hello.service","state":"running"}`)
	_, _, err := ApplyKeeping(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared,
		m.run, nil, nil, KeepIn(dir))
	if err == nil || !strings.Contains(err.Error(), "does not exist") {
		t.Errorf("an absent unit was held rather than applied: %v", err)
	}
}

func TestAContainerThatWouldMountFoundDataIsNotCreated(t *testing.T) {
	dir := t.TempDir()
	data := filepath.Join(dir, "predecessor-data")
	if err := os.Mkdir(data, 0o700); err != nil {
		t.Fatal(err)
	}
	for _, c := range []struct {
		name, volume string
		m            *machine
	}{
		{"a path", data + ":/var/lib/postgresql/data", &machine{containers: map[string]*fakeContainer{}}},
		{"a named volume", "predecessor-pgdata:/var/lib/postgresql/data",
			&machine{containers: map[string]*fakeContainer{}, volumes: map[string]bool{"predecessor-pgdata": true}}},
	} {
		report, state := applyAdopted(t, adopted(t, untaken("hello-web.server"),
			`{"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`",
			  "volumes":["`+c.volume+`"]}`), store.State{}, c.m, dir)
		if c.m.did("docker run") {
			t.Errorf("%s: a container mounting found data was created: %v", c.name, c.m.asked)
		}
		o := outcomeOf(report, "hello-web.server")
		if o.Action != "held" || !strings.Contains(o.Detail, "would mount") {
			t.Errorf("%s: not held: %+v", c.name, o)
		}
		if h, ok := state.HeldAt("hello-web.server"); !ok || !strings.Contains(h.Why, "would mount") {
			t.Errorf("%s: the hold does not say why: %+v", c.name, h)
		}
		// Held, it stays held on the next pass, and is still not created.
		c.m.asked = nil
		_, state = applyAdopted(t, adopted(t, untaken("hello-web.server"),
			`{"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`",
			  "volumes":["`+c.volume+`"]}`), state, c.m, dir)
		if c.m.did("docker run") || len(state.Held) != 1 {
			t.Errorf("%s: a held container was created on the next pass: %v", c.name, c.m.asked)
		}
	}
}

func TestAContainerMountingWhatTheMeshMadeIsCreated(t *testing.T) {
	dir := t.TempDir()
	data := filepath.Join(dir, "data")
	m := &machine{containers: map[string]*fakeContainer{}}
	report, _ := applyAdopted(t, adopted(t, untaken("hello-web.data", "hello-web.server"),
		`{"id":"hello-web.data","type":"directory","path":"`+data+`"},
		 {"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`",
		  "volumes":["`+data+`:/data"]}`), store.State{}, m, dir)
	if o := outcomeOf(report, "hello-web.server"); o.Action != "created" {
		t.Errorf("a container mounting only what the mesh made was not created: %+v", o)
	}
}

func TestARunOnceStepInAHeldContainerIsHeld(t *testing.T) {
	dir, page, m := predecessor(t)
	step := `{"id":"hello-web.migrate","type":"container","name":"hello-web-migrate","image":"` + pinned + `",
	  "run-once":true,"network":"container:hello-web"}`
	report, state := applyAdopted(t, adopted(t, untaken("hello-web.page", "hello-web.server", "hello-web.migrate"), webResources(page)+","+step), store.State{}, m, dir)
	if m.did("docker run") {
		t.Errorf("a step was run inside a held container: %v", m.asked)
	}
	o := outcomeOf(report, "hello-web.migrate")
	if o.Action != "held" || !strings.Contains(o.Detail, "runs in hello-web") {
		t.Errorf("the step was not held: %+v", o)
	}
	if _, ok := state.HeldAt("hello-web.migrate"); !ok {
		t.Error("the held step is not reported held")
	}
}

func TestAnActionInAHeldContainerIsHeldUntilItsModuleIsTaken(t *testing.T) {
	// An action cannot arrive over the link today, and a bundle cannot say a node is adopted; the
	// host holds one anyway, since what it would run in is the predecessor's.
	dir, page, m := predecessor(t)
	d := adopted(t, untakenWeb, webResources(page))
	d.Resources = append(d.Resources, &declaration.Action{ID: "hello-web.seed", Type: declaration.TypeAction,
		In: "hello-web", Command: []string{"seed"}, Verify: []string{"seeded"}})
	report, state := applyAdopted(t, d, store.State{}, m, dir)
	if m.did("docker exec") {
		t.Errorf("an action was run inside a held container: %v", m.asked)
	}
	if o := outcomeOf(report, "hello-web.seed"); o.Action != "held" || !strings.Contains(o.Detail, "not run until hello-web is taken") {
		t.Errorf("the action was not held: %+v", o)
	}
	if h, ok := state.HeldAt("hello-web.seed"); !ok || h.Module != "hello-web" {
		t.Errorf("the held action is not its module's: %+v", h)
	}

	// Taken: the container is the mesh's, and the action runs in it.
	taken := adopted(t, takenWeb, webResources(page))
	taken.Resources = append(taken.Resources, d.Resources[len(d.Resources)-1])
	report, state = applyAdopted(t, taken, state, m, dir)
	if !m.did("docker exec hello-web ") {
		t.Errorf("the action did not run once its module was taken: %v", m.asked)
	}
	if _, still := state.HeldAt("hello-web.seed"); still || len(state.Held) != 0 {
		t.Errorf("holds outlived the take: %+v", state.Held)
	}
}

const sixtyFourZeros = "0000000000000000000000000000000000000000000000000000000000000000"

func TestAnArchiveOverSomethingFoundIsNotUnpacked(t *testing.T) {
	dir := t.TempDir()
	at := filepath.Join(dir, "site")
	if err := os.Mkdir(at, 0o750); err != nil {
		t.Fatal(err)
	}
	theirs := filepath.Join(at, "index.html")
	_ = os.WriteFile(theirs, []byte("the predecessor's site\n"), 0o640)
	m := &machine{containers: map[string]*fakeContainer{}}
	// The source is unreachable: fetching it would fail the apply, so a pass means it was not tried.
	report, state := applyAdopted(t, adopted(t, untaken("hello-web.site"),
		`{"id":"hello-web.site","type":"archive","source":"http://192.0.2.1/site.tar.gz",
		  "digest":"sha256:`+sixtyFourZeros+`","path":"`+at+`","owner":"root"}`), store.State{}, m, dir)
	if o := outcomeOf(report, "hello-web.site"); o.Action != "held" || !strings.Contains(o.Detail, "nothing unpacked") {
		t.Errorf("an archive over found files was not held: %+v", o)
	}
	if got, _ := os.ReadFile(theirs); string(got) != "the predecessor's site\n" {
		t.Errorf("the found files were touched: %q", got)
	}
	if _, ok := state.HeldAt("hello-web.site"); !ok {
		t.Error("the hold was not recorded")
	}
}

func TestAProcessWhoseUnitIsFoundIsNotWrittenOverOrRestarted(t *testing.T) {
	dir := t.TempDir()
	was := unitDir
	unitDir = dir
	t.Cleanup(func() { unitDir = was })
	unit := filepath.Join(dir, "hello-daemon.service")
	_ = os.WriteFile(unit, []byte("[Service]\nExecStart=/opt/predecessor/hello\n"), 0o644)
	m := &machine{containers: map[string]*fakeContainer{}}
	report, state := applyAdopted(t, adopted(t, untaken("hello-web.daemon"),
		`{"id":"hello-web.daemon","type":"process","name":"hello-daemon","source":"http://192.0.2.1/d.tar.gz",
		  "digest":"sha256:`+sixtyFourZeros+`","run":["hello"]}`), store.State{}, m, dir)
	if o := outcomeOf(report, "hello-web.daemon"); o.Action != "held" || !strings.Contains(o.Detail, "not written over or restarted") {
		t.Errorf("a process whose unit was found was not held: %+v", o)
	}
	if got, _ := os.ReadFile(unit); string(got) != "[Service]\nExecStart=/opt/predecessor/hello\n" {
		t.Errorf("the found unit was written over: %q", got)
	}
	if m.did("systemctl") {
		t.Errorf("the found unit was touched: %v", m.asked)
	}
	if _, ok := state.HeldAt("hello-web.daemon"); !ok {
		t.Error("the hold was not recorded")
	}
}

func TestAUserFoundOnTheMachineKeepsItsShellAndGroups(t *testing.T) {
	dir := t.TempDir()
	m := &machine{containers: map[string]*fakeContainer{}, users: map[string]bool{"hello": true}}
	report, _ := applyAdopted(t, adopted(t, untaken("hello-web.user"),
		`{"id":"hello-web.user","type":"user","name":"hello","shell":"/bin/zsh","groups":["docker"]}`),
		store.State{}, m, dir)
	if o := outcomeOf(report, "hello-web.user"); o.Action != "held" || !strings.Contains(o.Detail, "shell and groups") {
		t.Errorf("a found user was not held: %+v", o)
	}
	for _, a := range m.asked {
		if strings.HasPrefix(a, "usermod") || strings.HasPrefix(a, "useradd") {
			t.Errorf("a found user was changed: %s", a)
		}
	}
}

func TestMountingTheMachinesOwnPlumbingIsNotFoundData(t *testing.T) {
	// The runtime's socket, the kernel's filesystems and the clock are on every machine; a
	// container mounting them shares nothing a predecessor kept (novox/hq ADR 0103).
	dir := t.TempDir()
	m := &machine{containers: map[string]*fakeContainer{}}
	report, state := applyAdopted(t, adopted(t, untaken("hello-web.server"),
		`{"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`",
		  "volumes":["/var/run/docker.sock:/var/run/docker.sock","/etc/localtime:/etc/localtime:ro",
		             "/proc/cpuinfo:/host/cpuinfo:ro","/dev/null:/data/null"]}`), store.State{}, m, dir)
	if o := outcomeOf(report, "hello-web.server"); o.Action != "created" {
		t.Errorf("a container mounting only system paths was not created: %+v", o)
	}
	if len(state.Held) != 0 {
		t.Errorf("held: %+v", state.Held)
	}
}

// Defends novox/hq ADR 0100: a runtime that cannot answer is not a machine with nothing there.

// unreachable is a machine whose container runtime answers everything with a daemon that is down.
type unreachable struct{ asked []string }

func (u *unreachable) run(_ context.Context, name string, args ...string) (string, error) {
	u.asked = append(u.asked, name+" "+strings.Join(args, " "))
	if name == "docker" && args[0] == "info" {
		return "27.0\n", nil
	}
	if name == "docker" {
		return "", errors.New("docker exited 1: Cannot connect to the Docker daemon at unix:///var/run/docker.sock")
	}
	return "", nil
}

func TestARuntimeThatCannotAnswerNeverLetsTheMeshCreateOverAFoundContainer(t *testing.T) {
	dir := t.TempDir()
	u := &unreachable{}
	d := adopted(t, untaken("hello-web.server"),
		`{"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`"}`)
	_, state, err := ApplyKeeping(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared,
		u.run, nil, nil, KeepIn(dir))
	if err == nil || !strings.Contains(err.Error(), "not safe to make one") {
		t.Fatalf("a runtime that could not answer was read as nothing there: %v", err)
	}
	for _, a := range u.asked {
		if strings.HasPrefix(a, "docker run") || strings.HasPrefix(a, "docker rm") {
			t.Errorf("the mesh acted on a container it could not ask about: %s", a)
		}
	}
	if len(state.Held) != 0 {
		t.Errorf("something was held on an answer the machine never gave: %+v", state.Held)
	}
}

func TestAHeldContainerStaysHeldWhenTheRuntimeCannotAnswer(t *testing.T) {
	dir, page, m := predecessor(t)
	d := adopted(t, untaken("hello-web.page", "hello-web.server"), webResources(page))
	_, state := applyAdopted(t, d, store.State{}, m, dir)
	if _, held := state.HeldAt("hello-web.server"); !held {
		t.Fatal("the found container was not held to begin with")
	}
	u := &unreachable{}
	_, state, err := ApplyKeeping(context.Background(), archHost(t), d, state, store.OriginDeclared,
		u.run, nil, nil, KeepIn(dir))
	if err == nil {
		t.Fatal("a runtime that could not answer reported success")
	}
	if h, held := state.HeldAt("hello-web.server"); !held || h.Changed != "" {
		t.Errorf("a hold was let go or called changed on an answer the machine never gave: %+v", h)
	}
}

func TestAVolumeTheRuntimeCannotBeAskedAboutStopsTheContainer(t *testing.T) {
	dir := t.TempDir()
	run := func(_ context.Context, name string, args ...string) (string, error) {
		switch {
		case name == "docker" && args[0] == "info":
			return "27.0\n", nil
		case name == "docker" && args[0] == "volume":
			return "", errors.New("docker exited 1: Cannot connect to the Docker daemon")
		case name == "docker" && args[0] == "inspect":
			return "", errors.New("Error: No such object: hello-web")
		case name == "docker":
			return "", errors.New("docker run must not happen")
		}
		return "", nil
	}
	d := adopted(t, untaken("hello-web.server"),
		`{"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`",
		  "volumes":["predecessor-data:/data"]}`)
	_, _, err := ApplyKeeping(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared,
		run, nil, nil, KeepIn(dir))
	if err == nil || !strings.Contains(err.Error(), "could not say whether the volume") {
		t.Fatalf("a volume the runtime could not be asked about did not stop the container: %v", err)
	}
}

func TestAUnitSomebodyInstalledIsHeldWhateverStateItIsIn(t *testing.T) {
	// A predecessor's unit under /etc, deliberately stopped and disabled: starting it would put
	// back a service somebody took down on purpose (novox/hq ADR 0103).
	dir := t.TempDir()
	m := &machine{containers: map[string]*fakeContainer{},
		units: map[string]*fakeUnit{"hello.service": {active: "inactive", enabled: "disabled",
			fragment: "/etc/systemd/system/hello.service"}}}
	report, state := applyAdopted(t, adopted(t, untaken("hello-web.unit"),
		`{"id":"hello-web.unit","type":"service","unit":"hello.service","state":"running","boot":"enabled"}`),
		store.State{}, m, dir)
	if o := outcomeOf(report, "hello-web.unit"); o.Action != "held" {
		t.Fatalf("a unit an administrator installed was not held: %+v", o)
	}
	if m.did("systemctl start") || m.did("systemctl enable") {
		t.Errorf("a unit somebody had stopped and disabled was started: %v", m.asked)
	}
	if _, ok := state.HeldAt("hello-web.unit"); !ok {
		t.Error("the hold was not recorded")
	}
}

func TestAPackagedUnitTheMachineUsesIsStillHeld(t *testing.T) {
	// The predecessor's own service from a package, running: not the mesh's to restart.
	dir := t.TempDir()
	conf := filepath.Join(dir, "hello.conf")
	m := &machine{containers: map[string]*fakeContainer{},
		units: map[string]*fakeUnit{"nginx.service": {active: "active", enabled: "enabled",
			fragment: "/usr/lib/systemd/system/nginx.service"}}}
	report, _ := applyAdopted(t, adopted(t, untaken("hello-web.conf", "hello-web.unit"),
		`{"id":"hello-web.conf","type":"file","path":"`+conf+`","content":"x\n"},
		 {"id":"hello-web.unit","type":"service","unit":"nginx.service","state":"running","boot":"enabled",
		  "restart-on":["hello-web.conf"]}`), store.State{}, m, dir)
	if o := outcomeOf(report, "hello-web.unit"); o.Action != "held" {
		t.Fatalf("a packaged unit the machine runs was not held: %+v", o)
	}
	if m.did("systemctl stop") || m.did("systemctl restart") {
		t.Errorf("the predecessor's service was restarted: %v", m.asked)
	}
}