Hold a found directory, a found service's unit, a container that would mount found data, and a step run in a held container on an adopted node (hq ADR 0103)

internal/apply/apply.go

@@ -161,6 +161,10 @@ func ApplyKeeping(
 		return report, known, &Error{Resource: "the firewall found on this machine", Err: err, Done: report}
 	}
 
+	// What an adopted node's untaken modules find on the machine, looked at before anything in
+	// this apply — a removal included — could change it or its records (novox/hq ADR 0103).
+	before := lookBefore(ctx, sys, d, known, run)
+
 	removeOrphan := func(orphan store.Applied) error {
 		var action, detail string
 		var err error
@@ -236,36 +240,24 @@ func ApplyKeeping(
 	var failures []*Error
 	for _, resource := range d.Resources {
 		// **On an adopted node, what is found is kept until its module is taken** (novox/hq ADR
-		// 0100). Before anything is applied: a file present with no record of this host writing
+		// 0100, ADR 0103). Before anything is applied: whatever of a module not yet taken is
-		// it, or a container present under that name that no host made, is held as it is and
+		// present with no record of this host making it — or would reach what is — is held as it
-		// reported. Once held it stays held — changed or gone — until its module is taken, and
+		// is and reported. Once held it stays held until its module is taken, and it is never
-		// it is never recorded as applied, so it is never removed as an orphan either.
+		// recorded as applied, so it is never removed as an orphan either.
-		if d.Adoption != nil && holdable(resource) {
+		if d.Adoption != nil {
-			if module, untaken := d.Adoption.UntakenModuleOf(resource.Identity()); untaken {
+			isHeld, news, outcome, err := holdOnAdopted(ctx, sys, resource, d, &known, before, run, keep,
-				was, already := known.HeldAt(resource.Identity())
+				changed, time.Now().UTC())
-				isFound := false
+			if err != nil {
-				if !already {
+				failures = append(failures, &Error{Resource: resource.Identity(), Err: err, Done: report})
-					var err error
+				log(fmt.Sprintf("  failed %s (%s): %v", resource.Identity(), resource.Target(), err))
-					if isFound, err = found(ctx, resource, run, known); err != nil {
+				continue
-						failures = append(failures, &Error{Resource: resource.Identity(), Err: err, Done: report})
+			}
-						log(fmt.Sprintf("  failed %s (%s): %v", resource.Identity(), resource.Target(), err))
+			if isHeld {
-						continue
+				report.Outcomes = append(report.Outcomes, outcome)
-					}
+				if news {
-				}
+					log(fmt.Sprintf("  held %s (%s): %s", outcome.ID, outcome.Target, outcome.Detail))
-				if already || isFound {
-					outcome, held, err := hold(ctx, resource, module, was, already, run, keep, time.Now().UTC())
-					if err != nil {
-						failures = append(failures, &Error{Resource: resource.Identity(), Err: err, Done: report})
-						log(fmt.Sprintf("  failed %s (%s): %v", resource.Identity(), outcome.Target, err))
-						continue
-					}
-					known.RecordHeld(held)
-					report.Outcomes = append(report.Outcomes, outcome)
-					if !already || held.Changed != was.Changed {
-						log(fmt.Sprintf("  held %s (%s): %s", outcome.ID, outcome.Target, outcome.Detail))
-					}
-					continue
 				}
+				continue
 			}
 		}
 

internal/apply/hold.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/novox/mesh-host/internal/declaration"
 	"github.com/novox/mesh-host/internal/store"
+	"github.com/novox/mesh-host/internal/system"
 )
 
 // Keep records the original of a file found on an adopted node, before anything else happens to
@@ -45,14 +46,230 @@ func KeepIn(dir string) Keep {
 	}
 }
 
-// holdable is whether a resource is one a predecessor can already have on the machine: a file at
+// foundBefore is what an adopted apply finds on the machine before it changes anything: each
-// a path, or a container under a name. A file written into is not: it replaces nothing that was
+// directory, service unit and container mount source of an untaken module that is present with no
-// found, only adds the mesh's keys beside it (novox/hq ADR 0102).
+// record (novox/hq ADR 0103). Looked at first, because the apply itself makes such things — a
-func holdable(r declaration.Resource) bool {
+// file's parent directory, a unit file a module writes, a package that brings its unit — and what
-	if f, ok := r.(*declaration.File); ok {
+// the mesh made in this apply was not found.
-		return f.Into == ""
+type foundBefore map[string]bool
+
+func lookBefore(ctx context.Context, sys system.System, d *declaration.Declaration, known store.State,
+	run Runner) foundBefore {
+	seen := foundBefore{}
+	if d.Adoption == nil {
+		return seen
 	}
-	return r.Kind() == declaration.TypeContainer
+	cri, asked := "", false
+	for _, r := range d.Resources {
+		if _, untaken := d.Adoption.UntakenModuleOf(r.Identity()); !untaken {
+			continue
+		}
+		if _, held := known.HeldAt(r.Identity()); held {
+			continue
+		}
+		switch res := r.(type) {
+		case *declaration.Directory:
+			if present(res.Path) && !recordedPath(known, res.Path) {
+				seen["path:"+res.Path] = true
+			}
+		case *declaration.Service:
+			if known.Recorded(string(declaration.TypeService), res.Unit) {
+				continue
+			}
+			// A unit the service manager cannot find is not there; one it can read is, whatever
+			// state it is in.
+			if _, err := sys.ServiceState(ctx, run, res.Unit); err == nil {
+				seen["unit:"+res.Unit] = true
+			}
+		case *declaration.Container:
+			if known.Recorded(string(declaration.TypeContainer), res.Name) {
+				continue
+			}
+			for _, v := range res.Volumes {
+				src := mountSource(v)
+				switch {
+				case src == "":
+				case strings.HasPrefix(src, "/"):
+					if present(src) && !recordedPath(known, src) {
+						seen["path:"+src] = true
+					}
+				default:
+					if !asked {
+						cri, _ = containerRuntime(ctx, run)
+						asked = true
+					}
+					if cri == "" {
+						continue
+					}
+					if _, err := run(ctx, cri, "volume", "inspect", src); err == nil {
+						seen["volume:"+src] = true
+					}
+				}
+			}
+		}
+	}
+	return seen
+}
+
+func present(path string) bool {
+	_, err := os.Lstat(path)
+	return err == nil
+}
+
+// recordedPath is whether this host has a record of making something at a path.
+func recordedPath(known store.State, path string) bool {
+	for _, kind := range []declaration.Type{declaration.TypeDirectory, declaration.TypeFile,
+		declaration.TypeArchive, declaration.TypeAccess} {
+		if known.Recorded(string(kind), path) {
+			return true
+		}
+	}
+	return false
+}
+
+// mountSource is what a volume mapping mounts: a path on the machine, or a named volume. Empty for
+// an anonymous volume, which mounts nothing that could already be there.
+func mountSource(mapping string) string {
+	src, _, ok := strings.Cut(mapping, ":")
+	if !ok {
+		return ""
+	}
+	return src
+}
+
+// runsIn is the container a resource runs inside, if any: an action's `in`, or a run-once step
+// sharing a container's namespace.
+func runsIn(r declaration.Resource) string {
+	switch res := r.(type) {
+	case *declaration.Action:
+		return res.In
+	case *declaration.Container:
+		if res.RunOnce {
+			if name, ok := strings.CutPrefix(res.Network, "container:"); ok {
+				return name
+			}
+		}
+	}
+	return ""
+}
+
+// heldContainer is what is held under a container's name.
+func heldContainer(known store.State, name string) (store.Held, bool) {
+	for _, h := range known.Held {
+		if h.Kind == string(declaration.TypeContainer) && h.Target == name {
+			return h, true
+		}
+	}
+	return store.Held{}, false
+}
+
+// holdOnAdopted decides whether a resource of an adopted node is held rather than applied, and
+// holds it (novox/hq ADR 0100, ADR 0103). For a module not yet taken, what is present with no
+// record is kept as it is: a file or a container under its name, a directory, a service's unit,
+// and a container that would mount a path or a volume found there. An action or a run-once step
+// run inside a held container is held with it. Once held, a resource stays held — changed or gone
+// — until its module is taken, and it is never recorded as applied, so never removed as an orphan.
+//
+// Held is false for a resource to apply as usual. News is whether the hold is new or changed,
+// which is what is worth a line in the log.
+func holdOnAdopted(ctx context.Context, sys system.System, r declaration.Resource, d *declaration.Declaration,
+	known *store.State, before foundBefore, run Runner, keep Keep, changed map[string]bool,
+	now time.Time) (held, news bool, out Outcome, err error) {
+	was, already := known.HeldAt(r.Identity())
+
+	if in := runsIn(r); in != "" {
+		if container, isHeld := heldContainer(*known, in); isHeld {
+			module, untaken := d.Adoption.UntakenModuleOf(r.Identity())
+			if !untaken {
+				module = container.Module
+			}
+			h := was
+			if !already {
+				h = store.Held{ID: r.Identity(), Kind: string(r.Kind()), Target: r.Target(), Since: now}
+			}
+			h.Module, h.Why = module, "runs in "+in
+			known.RecordHeld(h)
+			out = begin(r)
+			out.Action = "held"
+			out.Detail = fmt.Sprintf("runs in %s, which is held as found; not run until %s is taken", in, module)
+			return true, !already, out, nil
+		}
+	}
+
+	module, untaken := d.Adoption.UntakenModuleOf(r.Identity())
+	if !untaken {
+		return false, false, out, nil
+	}
+	why := was.Why
+	isFound := already
+	if !already {
+		switch res := r.(type) {
+		case *declaration.File:
+			if res.Into == "" {
+				if isFound, err = found(ctx, r, run, *known); err != nil {
+					return false, false, begin(r), err
+				}
+			}
+		case *declaration.Container:
+			if known.Recorded(string(declaration.TypeContainer), res.Name) {
+				break
+			}
+			_, exists, err := inspectFound(ctx, res.Name, run)
+			if err != nil {
+				return false, false, begin(r), err
+			}
+			if exists {
+				if isFound, err = found(ctx, r, run, *known); err != nil {
+					return false, false, begin(r), err
+				}
+				break
+			}
+			// Not there under its name, and still it would share what was found: created, it
+			// would mount the predecessor's data beside the predecessor's own container.
+			for _, v := range res.Volumes {
+				src := mountSource(v)
+				key := "volume:" + src
+				if strings.HasPrefix(src, "/") {
+					key = "path:" + src
+				}
+				if src != "" && before[key] {
+					isFound, why = true, "would mount "+src+", found on the machine"
+					break
+				}
+			}
+		case *declaration.Directory:
+			isFound = before["path:"+res.Path]
+		case *declaration.Service:
+			isFound = before["unit:"+res.Unit]
+		}
+	}
+	if !isFound {
+		return false, false, out, nil
+	}
+
+	out, h, err := hold(ctx, sys, r, module, was, already, why, run, keep, now)
+	if err != nil {
+		return true, false, out, err
+	}
+	// A held service is not started, stopped, enabled or restarted — but a reload stops nothing,
+	// so one the module names still happens (novox/hq ADR 0102, ADR 0103).
+	if svc, ok := r.(*declaration.Service); ok && svc.State == "running" {
+		if which := restartedBy(svc.ReloadOn, changed); len(which) > 0 {
+			if state, err := sys.ServiceState(ctx, run, svc.Unit); err == nil && state == "running" {
+				reloader, can := sys.(serviceReloader)
+				if !can {
+					return true, false, out, fmt.Errorf("%s must be reloaded for %s and this machine's "+
+						"service manager cannot reload a unit", svc.Unit, strings.Join(which, ", "))
+				}
+				if err := reloader.ReloadService(ctx, run, svc.Unit); err != nil {
+					return true, false, out, fmt.Errorf("reloading the held %s: %w", svc.Unit, err)
+				}
+				out.Detail += "; reloaded for " + strings.Join(which, ", ") + ", which stops nothing"
+			}
+		}
+	}
+	known.RecordHeld(h)
+	return true, !already || h.Changed != was.Changed, out, nil
 }
 
 // found is whether a declared file or container is present on the machine with no record of this
@@ -111,15 +328,16 @@ func inspectFound(ctx context.Context, name string, run Runner) (foundContainer,
 // hold keeps a found file or container as it is, and reports it — the first time by recording
 // what was found, every time after by comparing against that. Nothing is reverted, restarted or
 // created: a held target that disappears stays held and gone until its module is taken.
-func hold(ctx context.Context, r declaration.Resource, module string, was store.Held, already bool,
+func hold(ctx context.Context, sys system.System, r declaration.Resource, module string, was store.Held,
-	run Runner, keep Keep, now time.Time) (Outcome, store.Held, error) {
+	already bool, why string, run Runner, keep Keep, now time.Time) (Outcome, store.Held, error) {
 	out := begin(r)
 	h := was
 	if !already {
 		h = store.Held{ID: r.Identity(), Module: module, Kind: string(r.Kind()),
-			Target: r.Target(), Since: now}
+			Target: r.Target(), Since: now, Why: why}
 	}
 	h.Module = module
+	detail := "found on the machine; kept until " + module + " is taken"
 
 	var changed string
 	switch res := r.(type) {
@@ -159,7 +377,45 @@ func hold(ctx context.Context, r declaration.Resource, module string, was store.
 				changed = "rewritten"
 			}
 		}
+	case *declaration.Directory:
+		info, err := os.Lstat(res.Path)
+		switch {
+		case errors.Is(err, os.ErrNotExist):
+			if !already {
+				return out, h, fmt.Errorf("%s was found and is gone before it could be held", res.Path)
+			}
+			changed = "gone"
+		case err != nil:
+			return out, h, err
+		case !already:
+			// Its mode and owner as found, which the mesh leaves: a database refuses to start
+			// on a data directory whose mode changed.
+			h.Mode = fmt.Sprintf("%04o", info.Mode().Perm())
+			if st, ok := info.Sys().(*syscall.Stat_t); ok {
+				h.Owner = fmt.Sprintf("%d:%d", st.Uid, st.Gid)
+			}
+		}
+		detail = "found on the machine; its mode, owner and contents kept until " + module + " is taken"
+	case *declaration.Service:
+		state, err := sys.ServiceState(ctx, run, res.Unit)
+		switch {
+		case err != nil && !already:
+			return out, h, fmt.Errorf("the unit %s was found and cannot be read to hold it: %w", res.Unit, err)
+		case err != nil:
+			changed = "gone"
+		case !already:
+			h.Running = state == "running"
+		case h.Running && state != "running":
+			changed = "stopped"
+		}
+		detail = "its unit was found on the machine; its state and whether it starts at boot are " +
+			"kept until " + module + " is taken"
 	case *declaration.Container:
+		if h.Why != "" && h.Container == "" {
+			// Held for what it would mount, never created: there is nothing of it to compare.
+			detail = "not created: it " + h.Why + "; kept until " + module + " is taken"
+			break
+		}
 		seen, exists, err := inspectFound(ctx, res.Name, run)
 		if err != nil {
 			return out, h, err
@@ -188,7 +444,7 @@ func hold(ctx context.Context, r declaration.Resource, module string, was store.
 		}
 	}
 	out.Action = "held"
-	out.Detail = "found on the machine; kept until " + module + " is taken"
+	out.Detail = detail
 	if h.Changed != "" {
 		out.Detail += "; " + h.Changed + " by something other than the mesh since it was found, and not reverted"
 	}
@@ -197,6 +453,9 @@ func hold(ctx context.Context, r declaration.Resource, module string, was store.
 
 // takenDetail is what an outcome says when a module's cutover replaced what was held for it.
 func takenDetail(h store.Held) string {
+	if h.Kind == string(declaration.TypeAction) || (h.Why != "" && h.Container == "") {
+		return "taken: no longer held (" + h.Why + ")"
+	}
 	if h.Kept != "" {
 		return "taken: replaced what was found; original kept at " + h.Kept
 	}

internal/apply/hold_test.go

@@ -19,6 +19,54 @@ import (
 type machine struct {
 	containers map[string]*fakeContainer
 	asked      []string
+
+	// units are service units by name, as systemd would report them; volumes are the runtime's
+	// named volumes.
+	units   map[string]*fakeUnit
+	volumes map[string]bool
+}
+
+type fakeUnit struct {
+	active, enabled string
+}
+
+// systemctl answers as systemd does for the units the machine has, and "not-found" for any other.
+func (m *machine) systemctl(args []string) (string, error) {
+	unit := args[len(args)-1]
+	if args[0] == "show" {
+		unit = args[1]
+	}
+	u, ok := m.units[unit]
+	switch args[0] {
+	case "show":
+		if !ok {
+			return "LoadState=not-found\nActiveState=inactive\nType=simple\n", nil
+		}
+		return "LoadState=loaded\nActiveState=" + u.active + "\nType=simple\nRemainAfterExit=no\n", nil
+	case "is-enabled":
+		if !ok {
+			return "", errors.New("not found")
+		}
+		return u.enabled + "\n", nil
+	case "start":
+		u.active = "active"
+	case "stop":
+		u.active = "inactive"
+	case "enable":
+		u.enabled = "enabled"
+	case "disable":
+		u.enabled = "disabled"
+	}
+	return "", nil
+}
+
+func (m *machine) did(prefix string) bool {
+	for _, a := range m.asked {
+		if strings.HasPrefix(a, prefix) {
+			return true
+		}
+	}
+	return false
 }
 
 type fakeContainer struct {
@@ -29,7 +77,18 @@ type fakeContainer struct {
 
 func (m *machine) run(_ context.Context, name string, args ...string) (string, error) {
 	m.asked = append(m.asked, name+" "+strings.Join(args, " "))
+	if name == "systemctl" {
+		return m.systemctl(args)
+	}
+	if name != "docker" {
+		return "", nil
+	}
 	switch args[0] {
+	case "volume":
+		if m.volumes[args[len(args)-1]] {
+			return "[]\n", nil
+		}
+		return "", errors.New("no such volume")
 	case "info":
 		return "27.0\n", nil
 	case "inspect":
@@ -278,88 +337,213 @@ func TestAHeldFileIsNeverRemovedWhenItsModuleIsUnassigned(t *testing.T) {
 	}
 }
 
-func TestAHeldFileRewrittenIsReportedAndNotReverted(t *testing.T) {
+// Defends novox/hq ADR 0103: found covers every kind that can reach what the machine already has.
-	dir, page, m := predecessor(t)
-	d := adopted(t, untakenWeb, webResources(page))
-	_, state := applyAdopted(t, d, store.State{}, m, dir)
 
-	if err := os.WriteFile(page, []byte("the predecessor wrote again\n"), 0o640); err != nil {
+// untaken is an adoption with hello-web untaken, listing these of its resources.
+func untaken(ids ...string) string {
+	return `{"taken":[],"untaken":{"hello-web":["` + strings.Join(ids, `","`) + `"]}}`
+}
+
+func TestAFoundDirectoryOfAnUntakenModuleKeepsItsModeOwnerAndContents(t *testing.T) {
+	dir := t.TempDir()
+	data := filepath.Join(dir, "data")
+	if err := os.Mkdir(data, 0o700); err != nil {
 		t.Fatal(err)
 	}
-	report, state := applyAdopted(t, d, state, m, dir)
+	inside := filepath.Join(data, "PG_VERSION")
-	if got, _ := os.ReadFile(page); string(got) != "the predecessor wrote again\n" {
+	if err := os.WriteFile(inside, []byte("16\n"), 0o600); err != nil {
-		t.Fatalf("a held file was reverted: %q", got)
+		t.Fatal(err)
 	}
-	if h, _ := state.HeldAt("hello-web.page"); h.Changed != "rewritten" || h.ChangedAt.IsZero() {
+	m := &machine{containers: map[string]*fakeContainer{}}
-		t.Errorf("a rewrite was not recorded: %+v", h)
+	report, state := applyAdopted(t, adopted(t, untaken("hello-web.data"),
+		`{"id":"hello-web.data","type":"directory","path":"`+data+`","mode":"0755"}`), store.State{}, m, dir)
+
+	if info, _ := os.Stat(data); info.Mode().Perm() != 0o700 {
+		t.Errorf("a found directory was re-moded to %o", info.Mode().Perm())
 	}
-	if o := outcomeOf(report, "hello-web.page"); !strings.Contains(o.Detail, "rewritten") {
+	if got, _ := os.ReadFile(inside); string(got) != "16\n" {
-		t.Errorf("a rewrite was not reported: %+v", o)
+		t.Errorf("what is inside a found directory was touched: %q", got)
 	}
-	h, _ := state.HeldAt("hello-web.page")
+	if o := outcomeOf(report, "hello-web.data"); o.Action != "held" || !strings.Contains(o.Detail, "mode, owner and contents") {
-	if kept, _ := os.ReadFile(h.Kept); string(kept) != "the predecessor's page\n" {
+		t.Errorf("the found directory was not held: %+v", o)
-		t.Errorf("the kept original was overwritten by a later write: %q", kept)
+	}
+	if h, ok := state.HeldAt("hello-web.data"); !ok || h.Mode != "0700" {
+		t.Errorf("the directory as found was not recorded: %+v", h)
+	}
+	if _, recorded := state.Find("hello-web.data"); recorded {
+		t.Error("a held directory was recorded as applied")
 	}
 }
 
-func TestAHeldContainerStoppedOrReplacedIsReportedAndNotRestarted(t *testing.T) {
+func TestADirectoryMadeInTheSameApplyIsNotFound(t *testing.T) {
+	// A file's parent is made as the file is written; what the mesh made is not found.
+	dir := t.TempDir()
+	data := filepath.Join(dir, "data")
+	m := &machine{containers: map[string]*fakeContainer{}}
+	report, state := applyAdopted(t, adopted(t, untaken("hello-web.conf", "hello-web.data"),
+		`{"id":"hello-web.conf","type":"file","path":"`+filepath.Join(data, "conf")+`","content":"x\n"},
+		 {"id":"hello-web.data","type":"directory","path":"`+data+`","mode":"0750"}`), store.State{}, m, dir)
+	if o := outcomeOf(report, "hello-web.data"); o.Action == "held" {
+		t.Errorf("a directory the apply itself made was held: %+v", o)
+	}
+	if info, _ := os.Stat(data); info.Mode().Perm() != 0o750 {
+		t.Errorf("the mesh's own directory was not converged: %o", info.Mode().Perm())
+	}
+	if len(state.Held) != 0 {
+		t.Errorf("held: %+v", state.Held)
+	}
+}
+
+func TestAFoundServiceOfAnUntakenModuleIsNeitherStartedNorEnabledNorRestarted(t *testing.T) {
+	dir := t.TempDir()
+	conf := filepath.Join(dir, "hello.conf")
+	m := &machine{containers: map[string]*fakeContainer{},
+		units: map[string]*fakeUnit{"hello.service": {active: "inactive", enabled: "disabled"}}}
+	report, state := applyAdopted(t, adopted(t, untaken("hello-web.conf", "hello-web.unit"),
+		`{"id":"hello-web.conf","type":"file","path":"`+conf+`","content":"x\n"},
+		 {"id":"hello-web.unit","type":"service","unit":"hello.service","state":"running","boot":"enabled",
+		  "restart-on":["hello-web.conf"]}`), store.State{}, m, dir)
+
+	for _, verb := range []string{"systemctl start", "systemctl stop", "systemctl enable", "systemctl disable", "systemctl restart"} {
+		if m.did(verb) {
+			t.Errorf("a found service was changed: %s (%v)", verb, m.asked)
+		}
+	}
+	if o := outcomeOf(report, "hello-web.unit"); o.Action != "held" || !strings.Contains(o.Detail, "starts at boot") {
+		t.Errorf("the found service was not held: %+v", o)
+	}
+	if h, ok := state.HeldAt("hello-web.unit"); !ok || h.Running {
+		t.Errorf("the service as found was not recorded: %+v", h)
+	}
+}
+
+func TestAHeldServiceIsStillReloadedButNeverRestarted(t *testing.T) {
+	// A reload stops nothing (novox/hq ADR 0102); a restart would stop the predecessor's service.
+	dir := t.TempDir()
+	conf := filepath.Join(dir, "daemon.json")
+	m := &machine{containers: map[string]*fakeContainer{},
+		units: map[string]*fakeUnit{"docker.service": {active: "active", enabled: "enabled"}}}
+	report, _ := applyAdopted(t, adopted(t, untaken("hello-web.conf", "hello-web.unit"),
+		`{"id":"hello-web.conf","type":"file","path":"`+conf+`","content":"{}\n"},
+		 {"id":"hello-web.unit","type":"service","unit":"docker.service","state":"running","boot":"enabled",
+		  "reload-on":["hello-web.conf"]}`), store.State{}, m, dir)
+	if !m.did("systemctl reload docker.service") {
+		t.Errorf("a held service was not reloaded for what it re-reads: %v", m.asked)
+	}
+	if m.did("systemctl stop") || m.did("systemctl start") {
+		t.Errorf("a held service was restarted: %v", m.asked)
+	}
+	if o := outcomeOf(report, "hello-web.unit"); o.Action != "held" || !strings.Contains(o.Detail, "reloaded for hello-web.conf") {
+		t.Errorf("the reload was not reported on the hold: %+v", o)
+	}
+}
+
+func TestAServiceWhoseUnitIsNotThereIsAppliedAsUsual(t *testing.T) {
+	// No unit before the apply: nothing of a predecessor's to hold.
+	dir := t.TempDir()
+	m := &machine{containers: map[string]*fakeContainer{}, units: map[string]*fakeUnit{}}
+	d := adopted(t, untaken("hello-web.unit"), `{"id":"hello-web.unit","type":"service","unit":"hello.service","state":"running"}`)
+	_, _, err := ApplyKeeping(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared,
+		m.run, nil, nil, KeepIn(dir))
+	if err == nil || !strings.Contains(err.Error(), "does not exist") {
+		t.Errorf("an absent unit was held rather than applied: %v", err)
+	}
+}
+
+func TestAContainerThatWouldMountFoundDataIsNotCreated(t *testing.T) {
+	dir := t.TempDir()
+	data := filepath.Join(dir, "predecessor-data")
+	if err := os.Mkdir(data, 0o700); err != nil {
+		t.Fatal(err)
+	}
 	for _, c := range []struct {
-		change func(*machine)
+		name, volume string
-		want   string
+		m            *machine
 	}{
-		{func(m *machine) { m.containers["hello-web"].running = false }, "stopped"},
+		{"a path", data + ":/var/lib/postgresql/data", &machine{containers: map[string]*fakeContainer{}}},
-		{func(m *machine) { m.containers["hello-web"].id = "another" }, "replaced"},
+		{"a named volume", "predecessor-pgdata:/var/lib/postgresql/data",
-		{func(m *machine) { delete(m.containers, "hello-web") }, "gone"},
+			&machine{containers: map[string]*fakeContainer{}, volumes: map[string]bool{"predecessor-pgdata": true}}},
 	} {
-		dir, page, m := predecessor(t)
+		report, state := applyAdopted(t, adopted(t, untaken("hello-web.server"),
-		d := adopted(t, untakenWeb, webResources(page))
+			`{"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`",
-		_, state := applyAdopted(t, d, store.State{}, m, dir)
+			  "volumes":["`+c.volume+`"]}`), store.State{}, c.m, dir)
-		c.change(m)
+		if c.m.did("docker run") {
-		m.asked = nil
+			t.Errorf("%s: a container mounting found data was created: %v", c.name, c.m.asked)
-		_, state = applyAdopted(t, d, state, m, dir)
-		if h, _ := state.HeldAt("hello-web.server"); h.Changed != c.want {
-			t.Errorf("%s: recorded as %q", c.want, h.Changed)
 		}
-		for _, a := range m.asked {
+		o := outcomeOf(report, "hello-web.server")
-			if strings.HasPrefix(a, "docker run") || strings.HasPrefix(a, "docker rm") ||
+		if o.Action != "held" || !strings.Contains(o.Detail, "would mount") {
-				strings.HasPrefix(a, "docker start") {
+			t.Errorf("%s: not held: %+v", c.name, o)
-				t.Errorf("%s: the held container was acted on: %s", c.want, a)
+		}
-			}
+		if h, ok := state.HeldAt("hello-web.server"); !ok || !strings.Contains(h.Why, "would mount") {
+			t.Errorf("%s: the hold does not say why: %+v", c.name, h)
+		}
+		// Held, it stays held on the next pass, and is still not created.
+		c.m.asked = nil
+		_, state = applyAdopted(t, adopted(t, untaken("hello-web.server"),
+			`{"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`",
+			  "volumes":["`+c.volume+`"]}`), state, c.m, dir)
+		if c.m.did("docker run") || len(state.Held) != 1 {
+			t.Errorf("%s: a held container was created on the next pass: %v", c.name, c.m.asked)
 		}
 	}
 }
 
-func TestAHeldFileThatVanishesIsNotCreated(t *testing.T) {
+func TestAContainerMountingWhatTheMeshMadeIsCreated(t *testing.T) {
+	dir := t.TempDir()
+	data := filepath.Join(dir, "data")
+	m := &machine{containers: map[string]*fakeContainer{}}
+	report, _ := applyAdopted(t, adopted(t, untaken("hello-web.data", "hello-web.server"),
+		`{"id":"hello-web.data","type":"directory","path":"`+data+`"},
+		 {"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`",
+		  "volumes":["`+data+`:/data"]}`), store.State{}, m, dir)
+	if o := outcomeOf(report, "hello-web.server"); o.Action != "created" {
+		t.Errorf("a container mounting only what the mesh made was not created: %+v", o)
+	}
+}
+
+func TestARunOnceStepInAHeldContainerIsHeld(t *testing.T) {
+	dir, page, m := predecessor(t)
+	step := `{"id":"hello-web.migrate","type":"container","name":"hello-web-migrate","image":"` + pinned + `",
+	  "run-once":true,"network":"container:hello-web"}`
+	report, state := applyAdopted(t, adopted(t, untaken("hello-web.page", "hello-web.server", "hello-web.migrate"), webResources(page)+","+step), store.State{}, m, dir)
+	if m.did("docker run") {
+		t.Errorf("a step was run inside a held container: %v", m.asked)
+	}
+	o := outcomeOf(report, "hello-web.migrate")
+	if o.Action != "held" || !strings.Contains(o.Detail, "runs in hello-web") {
+		t.Errorf("the step was not held: %+v", o)
+	}
+	if _, ok := state.HeldAt("hello-web.migrate"); !ok {
+		t.Error("the held step is not reported held")
+	}
+}
+
+func TestAnActionInAHeldContainerIsHeldUntilItsModuleIsTaken(t *testing.T) {
+	// An action cannot arrive over the link today, and a bundle cannot say a node is adopted; the
+	// host holds one anyway, since what it would run in is the predecessor's.
 	dir, page, m := predecessor(t)
 	d := adopted(t, untakenWeb, webResources(page))
-	_, state := applyAdopted(t, d, store.State{}, m, dir)
+	d.Resources = append(d.Resources, &declaration.Action{ID: "hello-web.seed", Type: declaration.TypeAction,
-	if err := os.Remove(page); err != nil {
+		In: "hello-web", Command: []string{"seed"}, Verify: []string{"seeded"}})
-		t.Fatal(err)
-	}
-	_, state = applyAdopted(t, d, state, m, dir)
-	if _, err := os.Stat(page); !errors.Is(err, os.ErrNotExist) {
-		t.Fatal("a held file that vanished was created before its module was taken")
-	}
-	if h, _ := state.HeldAt("hello-web.page"); h.Changed != "gone" {
-		t.Errorf("a vanished held file was not reported gone: %+v", h)
-	}
-}
-
-func TestAConvergedNodeStillReplacesWhatItFinds(t *testing.T) {
-	// No adoption, no holds: byte for byte what a converged node did before ADR 0100.
-	dir, page, m := predecessor(t)
-	d := parse(t, `{"declaration":1,"resources":[`+webResources(page)+`]}`)
 	report, state := applyAdopted(t, d, store.State{}, m, dir)
-	if got, _ := os.ReadFile(page); string(got) != "the mesh's page\n" {
+	if m.did("docker exec") {
-		t.Errorf("a converged node kept a found file: %q", got)
+		t.Errorf("an action was run inside a held container: %v", m.asked)
 	}
-	if !m.removed("hello-web") {
+	if o := outcomeOf(report, "hello-web.seed"); o.Action != "held" || !strings.Contains(o.Detail, "not run until hello-web is taken") {
-		t.Error("a converged node kept a found container")
+		t.Errorf("the action was not held: %+v", o)
 	}
-	if len(state.Held) != 0 || outcomeOf(report, "hello-web.page").Action == "held" {
+	if h, ok := state.HeldAt("hello-web.seed"); !ok || h.Module != "hello-web" {
-		t.Errorf("a converged node held something: %+v", state.Held)
+		t.Errorf("the held action is not its module's: %+v", h)
 	}
-	if _, err := os.Stat(filepath.Join(dir, "kept")); !errors.Is(err, os.ErrNotExist) {
+
-		t.Error("a converged node kept originals")
+	// Taken: the container is the mesh's, and the action runs in it.
+	taken := adopted(t, takenWeb, webResources(page))
+	taken.Resources = append(taken.Resources, d.Resources[len(d.Resources)-1])
+	report, state = applyAdopted(t, taken, state, m, dir)
+	if !m.did("docker exec hello-web ") {
+		t.Errorf("the action did not run once its module was taken: %v", m.asked)
+	}
+	if _, still := state.HeldAt("hello-web.seed"); still || len(state.Held) != 0 {
+		t.Errorf("holds outlived the take: %+v", state.Held)
 	}
 }

internal/declaration/adoption_test.go

@@ -57,11 +57,16 @@ func TestAnAdoptionNamingAnUnknownIDIsRefused(t *testing.T) {
 	}
 }
 
-func TestAnAdoptionMayOnlyHoldFilesAndContainers(t *testing.T) {
+func TestAnAdoptionMayNameAResourceOfAnyKind(t *testing.T) {
-	refusal := refusalFor(t, `{"adoption":{"taken":[],"untaken":{"hello-web":["hello-web.data"]}},
+	// A directory, a service or an action can reach what was found as surely as a file can, so
-	  "declaration":1,`+adoptedResources+`}`)
+	// the controller lists every resource of an untaken module (novox/hq ADR 0103).
-	if !strings.Contains(strings.Join(refusal.Problems, "\n"), "only a file or a container") {
+	d, err := Parse([]byte(`{"adoption":{"taken":[],"untaken":{"hello-web":["hello-web.data"]}},
-		t.Errorf("a directory was accepted as holdable: %v", refusal.Problems)
+	  "declaration":1,` + adoptedResources + `}`))
+	if err != nil {
+		t.Fatalf("a directory of an untaken module was refused: %v", err)
+	}
+	if module, ok := d.Adoption.UntakenModuleOf("hello-web.data"); !ok || module != "hello-web" {
+		t.Errorf("the directory is not its module's: %q %v", module, ok)
 	}
 }
 

internal/declaration/declaration.go

@@ -956,10 +956,12 @@ type Declaration struct {
 // is adopted; it is told, in every declaration, so a host restarted from the declaration it kept
 // is in the same mode it was in before.
 //
-// Untaken names, per module assigned here and not yet taken, the ids of its file and container
+// Untaken names, per module assigned here and not yet taken, the ids of its resources. Any kind
-// resources — the only shapes a predecessor can already have on the machine. The host cannot
+// may be listed: a file, a directory, a service's unit or a container can already be on the
-// split a resource id into its module, because module names may contain dots, so the controller
+// machine, and an action run inside a held container reaches what was found (novox/hq ADR 0103);
-// says which ids belong to which module rather than leaving the host to guess.
+// the host decides per kind what can be held. The host cannot split a resource id into its
+// module, because module names may contain dots, so the controller says which ids belong to which
+// module rather than leaving the host to guess.
 type Adoption struct {
 	Taken   []string            `json:"taken"`
 	Untaken map[string][]string `json:"untaken,omitempty"`
@@ -1026,15 +1028,9 @@ func checkAdoption(a *Adoption, resources []Resource, allowActions bool) []strin
 				continue
 			}
 			owner[id] = module
-			kind, declared := kinds[id]
+			if _, declared := kinds[id]; !declared {
-			switch {
-			case !declared:
 				problems = append(problems, fmt.Sprintf(
 					"adoption: %q of the untaken module %q is not in this declaration", id, module))
-			case kind != TypeFile && kind != TypeContainer:
-				problems = append(problems, fmt.Sprintf(
-					"adoption: %q of the untaken module %q is a %s, and only a file or a "+
-						"container can be found on a machine", id, module, kind))
 			}
 		}
 	}

internal/store/store.go

@@ -115,8 +115,10 @@ type FoundFirewall struct {
 	FoundAt        time.Time `json:"found_at"`
 }
 
-// Held is one file or container found on an adopted node — present at a declared path or name,
+// Held is one thing found on an adopted node — a file, directory or container present at a declared
-// with no record of this host having made it — and kept as it was found.
+// path or name, or a service's unit, with no record of this host having made it — kept as it was
+// found; or what would reach one: a container mounting found data, an action run in a held
+// container (novox/hq ADR 0100, ADR 0103).
 type Held struct {
 	ID     string `json:"id"`
 	Module string `json:"module"`
@@ -133,10 +135,15 @@ type Held struct {
 	Owner  string `json:"owner,omitempty"`
 	Kept   string `json:"kept,omitempty"`
 
-	// A container's id as found, and whether it was running.
+	// A container's id as found, and whether it was running — or a service's unit, whether it
+	// was running.
 	Container string `json:"container,omitempty"`
 	Running   bool   `json:"running,omitempty"`
 
+	// Why says what was found when it is not the resource's own target: the path or volume a
+	// container would mount, or the held container an action would run in (novox/hq ADR 0103).
+	Why string `json:"why,omitempty"`
+
 	// Changed is what something other than the mesh has done to it since it was found —
 	// rewritten, stopped, replaced or gone — and empty while it is as found. Reported, never
 	// reverted: that is how a predecessor still writing is caught.