Keep what an adopted node was found holding until its module is taken, and report it held (hq ADR 0100)

internal/apply/apply.go

@@ -61,7 +61,8 @@ type Report struct {
 // nothing is the ordinary steady state, and saying so is not the same as saying it failed.
 func (r Report) Changed() bool {
 	for _, o := range r.Outcomes {
-		if o.Action != "unchanged" {
+		// Holding is keeping the machine as it was found, which is not moving it.
+		if o.Action != "unchanged" && o.Action != "held" {
 			return true
 		}
 	}
@@ -121,6 +122,23 @@ func Apply(
 	run Runner,
 	log func(string),
 	unseal Unseal,
+) (Report, store.State, error) {
+	return ApplyKeeping(ctx, sys, d, known, origin, run, log, unseal, nil)
+}
+
+// ApplyKeeping is Apply on a node that may be adopted: keep is where the original of a file found
+// there is recorded before anything else happens to it (novox/hq ADR 0100). Nil is a caller that
+// can never be handed an adopted declaration — the carried bundle, which may not say it.
+func ApplyKeeping(
+	ctx context.Context,
+	sys system.System,
+	d *declaration.Declaration,
+	known store.State,
+	origin string,
+	run Runner,
+	log func(string),
+	unseal Unseal,
+	keep Keep,
 ) (Report, store.State, error) {
 	if log == nil {
 		log = func(string) {}
@@ -182,6 +200,40 @@ func Apply(
 	// a declaration", and they are fixed in different places.
 	var failures []*Error
 	for _, resource := range d.Resources {
+		// **On an adopted node, what is found is kept until its module is taken** (novox/hq ADR
+		// 0100). Before anything is applied: a file present with no record of this host writing
+		// it, or a container present under that name that no host made, is held as it is and
+		// reported. Once held it stays held — changed or gone — until its module is taken, and
+		// it is never recorded as applied, so it is never removed as an orphan either.
+		if d.Adoption != nil && holdable(resource) {
+			if module, untaken := d.Adoption.UntakenModuleOf(resource.Identity()); untaken {
+				was, already := known.HeldAt(resource.Identity())
+				isFound := false
+				if !already {
+					var err error
+					if isFound, err = found(ctx, resource, run, known); err != nil {
+						failures = append(failures, &Error{Resource: resource.Identity(), Err: err, Done: report})
+						log(fmt.Sprintf("  failed %s (%s): %v", resource.Identity(), resource.Target(), err))
+						continue
+					}
+				}
+				if already || isFound {
+					outcome, held, err := hold(ctx, resource, module, was, already, run, keep, time.Now().UTC())
+					if err != nil {
+						failures = append(failures, &Error{Resource: resource.Identity(), Err: err, Done: report})
+						log(fmt.Sprintf("  failed %s (%s): %v", resource.Identity(), outcome.Target, err))
+						continue
+					}
+					known.RecordHeld(held)
+					report.Outcomes = append(report.Outcomes, outcome)
+					if !already || held.Changed != was.Changed {
+						log(fmt.Sprintf("  held %s (%s): %s", outcome.ID, outcome.Target, outcome.Detail))
+					}
+					continue
+				}
+			}
+		}
+
 		was, _ := known.Find(resource.Identity())
 		outcome, err := applyOne(ctx, sys, resource, run, changed, declares, was, unseal)
 		if err != nil {
@@ -229,6 +281,11 @@ func Apply(
 			Wrote: outcome.wrote,
 			Holds: holds(resource),
 		})
+		// Its module has been taken, and what was held for it is now the mesh's.
+		if held, wasHeld := known.HeldAt(resource.Identity()); wasHeld {
+			known.Release(held.ID)
+			outcome.Detail = takenDetail(held)
+		}
 		report.Outcomes = append(report.Outcomes, outcome)
 		if outcome.Action != "unchanged" {
 			changed[resource.Identity()] = true

internal/apply/hold.go

@@ -0,0 +1,200 @@
+package apply
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/novox/mesh-host/internal/declaration"
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// Keep records the original of a file found on an adopted node, before anything else happens to
+// it, and says where (novox/hq ADR 0100). It never overwrites an original it already kept: the
+// first copy is the one that was there before the mesh.
+type Keep func(path string, content []byte, mode os.FileMode) (string, error)
+
+// KeepIn keeps originals under dir/kept, each named for the path it came from, readable by root
+// alone — a predecessor's configuration may carry its credentials.
+func KeepIn(dir string) Keep {
+	return func(path string, content []byte, _ os.FileMode) (string, error) {
+		sum := sha256.Sum256([]byte(path))
+		kept := filepath.Join(dir, "kept",
+			hex.EncodeToString(sum[:])[:16]+"-"+filepath.Base(path))
+		if _, err := os.Lstat(kept); err == nil {
+			return kept, nil
+		}
+		if err := os.MkdirAll(filepath.Dir(kept), 0o700); err != nil {
+			return "", err
+		}
+		if err := writeAtomically(kept, content, 0o600); err != nil {
+			return "", err
+		}
+		back, err := os.ReadFile(kept)
+		if err != nil || string(back) != string(content) {
+			return "", fmt.Errorf("kept the original of %s at %s and cannot read it back", path, kept)
+		}
+		return kept, nil
+	}
+}
+
+// holdable is whether a resource is one a predecessor can already have on the machine: a file at
+// a path, or a container under a name.
+func holdable(r declaration.Resource) bool {
+	return r.Kind() == declaration.TypeFile || r.Kind() == declaration.TypeContainer
+}
+
+// found is whether a declared file or container is present on the machine with no record of this
+// host making it (novox/hq ADR 0100). A container carrying the host's own spec label was made by
+// a host, whatever this store says, so it is never found.
+func found(ctx context.Context, r declaration.Resource, run Runner, known store.State) (bool, error) {
+	if known.Recorded(string(r.Kind()), r.Target()) {
+		return false, nil
+	}
+	switch res := r.(type) {
+	case *declaration.File:
+		_, err := os.Lstat(res.Path)
+		if errors.Is(err, os.ErrNotExist) {
+			return false, nil
+		}
+		return err == nil, err
+	case *declaration.Container:
+		seen, exists, err := inspectFound(ctx, res.Name, run)
+		if err != nil || !exists {
+			return false, err
+		}
+		return seen.spec == "", nil
+	}
+	return false, nil
+}
+
+type foundContainer struct {
+	id      string
+	running bool
+	spec    string
+}
+
+// inspectFound reads a container by name the way a hold needs it: its id, whether it runs, and
+// whether a host made it.
+func inspectFound(ctx context.Context, name string, run Runner) (foundContainer, bool, error) {
+	cri, err := containerRuntime(ctx, run)
+	if err != nil {
+		return foundContainer{}, false, fmt.Errorf("%w, so nothing can be said about %q", err, name)
+	}
+	out, err := run(ctx, cri, "inspect", "--format",
+		"{{.Id}}\t{{.State.Running}}\t{{index .Config.Labels \""+specLabel+"\"}}", name)
+	if err != nil {
+		return foundContainer{}, false, nil
+	}
+	parts := strings.Split(strings.TrimSpace(out), "\t")
+	for len(parts) < 3 {
+		parts = append(parts, "")
+	}
+	spec := strings.TrimSpace(parts[2])
+	if spec == "<no value>" {
+		spec = ""
+	}
+	return foundContainer{id: strings.TrimSpace(parts[0]), running: parts[1] == "true", spec: spec}, true, nil
+}
+
+// hold keeps a found file or container as it is, and reports it — the first time by recording
+// what was found, every time after by comparing against that. Nothing is reverted, restarted or
+// created: a held target that disappears stays held and gone until its module is taken.
+func hold(ctx context.Context, r declaration.Resource, module string, was store.Held, already bool,
+	run Runner, keep Keep, now time.Time) (Outcome, store.Held, error) {
+	out := begin(r)
+	h := was
+	if !already {
+		h = store.Held{ID: r.Identity(), Module: module, Kind: string(r.Kind()),
+			Target: r.Target(), Since: now}
+	}
+	h.Module = module
+
+	var changed string
+	switch res := r.(type) {
+	case *declaration.File:
+		info, err := os.Lstat(res.Path)
+		switch {
+		case errors.Is(err, os.ErrNotExist):
+			if !already {
+				return out, h, fmt.Errorf("%s was found and is gone before it could be kept", res.Path)
+			}
+			changed = "gone"
+		case err != nil:
+			return out, h, err
+		default:
+			content, err := os.ReadFile(res.Path)
+			if err != nil {
+				return out, h, fmt.Errorf("%s was found and cannot be read to keep it: %w", res.Path, err)
+			}
+			if !already {
+				// The original first, before anything is recorded: a hold with no kept copy
+				// would be a promise the host cannot keep.
+				if keep == nil {
+					return out, h, fmt.Errorf(
+						"%s was found on this adopted node and this host has nowhere to keep its original", res.Path)
+				}
+				kept, err := keep(res.Path, content, info.Mode().Perm())
+				if err != nil {
+					return out, h, fmt.Errorf("keeping the original of %s: %w", res.Path, err)
+				}
+				h.Kept = kept
+				h.Digest = digestOf(string(content))
+				h.Mode = fmt.Sprintf("%04o", info.Mode().Perm())
+				if st, ok := info.Sys().(*syscall.Stat_t); ok {
+					h.Owner = fmt.Sprintf("%d:%d", st.Uid, st.Gid)
+				}
+			} else if digestOf(string(content)) != h.Digest {
+				changed = "rewritten"
+			}
+		}
+	case *declaration.Container:
+		seen, exists, err := inspectFound(ctx, res.Name, run)
+		if err != nil {
+			return out, h, err
+		}
+		switch {
+		case !exists && !already:
+			return out, h, fmt.Errorf("container %s was found and is gone before it could be held", res.Name)
+		case !already:
+			h.Container, h.Running = seen.id, seen.running
+		case !exists:
+			changed = "gone"
+		case seen.id != h.Container:
+			changed = "replaced"
+		case h.Running && !seen.running:
+			changed = "stopped"
+		}
+	default:
+		return out, h, fmt.Errorf("a %s cannot be held", r.Kind())
+	}
+
+	if changed != h.Changed {
+		h.Changed = changed
+		h.ChangedAt = now
+		if changed == "" {
+			h.ChangedAt = time.Time{}
+		}
+	}
+	out.Action = "held"
+	out.Detail = "found on the machine; kept until " + module + " is taken"
+	if h.Changed != "" {
+		out.Detail += "; " + h.Changed + " by something other than the mesh since it was found, and not reverted"
+	}
+	return out, h, nil
+}
+
+// takenDetail is what an outcome says when a module's cutover replaced what was held for it.
+func takenDetail(h store.Held) string {
+	if h.Kept != "" {
+		return "taken: replaced what was found; original kept at " + h.Kept
+	}
+	return "taken: replaced what was found"
+}

internal/apply/hold_test.go

@@ -0,0 +1,365 @@
+package apply
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-host/internal/declaration"
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// Defends novox/hq ADR 0100: on an adopted node, what is found is kept until its module is taken.
+
+// machine is a fake container runtime holding containers by name: id, running, and the host's spec
+// label when a host made it. Every command it is asked is written down.
+type machine struct {
+	containers map[string]*fakeContainer
+	asked      []string
+}
+
+type fakeContainer struct {
+	id      string
+	running bool
+	spec    string
+}
+
+func (m *machine) run(_ context.Context, name string, args ...string) (string, error) {
+	m.asked = append(m.asked, name+" "+strings.Join(args, " "))
+	switch args[0] {
+	case "info":
+		return "27.0\n", nil
+	case "inspect":
+		c, ok := m.containers[args[len(args)-1]]
+		if !ok {
+			return "", errors.New("no such container")
+		}
+		running := "false"
+		if c.running {
+			running = "true"
+		}
+		if strings.HasPrefix(args[2], "{{.Id}}") {
+			return c.id + "\t" + running + "\t" + c.spec + "\n", nil
+		}
+		return running + "\t" + c.spec + "\n", nil
+	case "rm":
+		delete(m.containers, args[len(args)-1])
+		return "", nil
+	case "run":
+		var name, spec string
+		for i, a := range args {
+			if a == "--name" {
+				name = args[i+1]
+			}
+			if a == "--label" && strings.HasPrefix(args[i+1], specLabel+"=") {
+				spec = strings.TrimPrefix(args[i+1], specLabel+"=")
+			}
+		}
+		m.containers[name] = &fakeContainer{id: "made-by-host", running: true, spec: spec}
+		return "made-by-host\n", nil
+	}
+	return "", nil
+}
+
+func (m *machine) removed(name string) bool {
+	for _, a := range m.asked {
+		if strings.HasPrefix(a, "docker rm") && strings.HasSuffix(a, " "+name) {
+			return true
+		}
+	}
+	return false
+}
+
+func adopted(t *testing.T, adoption, resources string) *declaration.Declaration {
+	t.Helper()
+	return parse(t, `{"declaration":1,"adoption":`+adoption+`,"resources":[`+resources+`]}`)
+}
+
+const untakenWeb = `{"taken":[],"untaken":{"hello-web":["hello-web.page","hello-web.server"]}}`
+const takenWeb = `{"taken":["hello-web"]}`
+
+func webResources(page string) string {
+	return `{"id":"hello-web.page","type":"file","path":"` + page + `","content":"the mesh's page\n"},
+	  {"id":"hello-web.server","type":"container","name":"hello-web","image":"` + pinned + `"}`
+}
+
+func applyAdopted(t *testing.T, d *declaration.Declaration, known store.State, m *machine, keepDir string) (Report, store.State) {
+	t.Helper()
+	report, state, err := ApplyKeeping(context.Background(), archHost(t), d, known,
+		store.OriginDeclared, m.run, nil, nil, KeepIn(keepDir))
+	if err != nil {
+		t.Fatalf("apply failed: %v", err)
+	}
+	return report, state
+}
+
+func outcomeOf(r Report, id string) Outcome {
+	for _, o := range r.Outcomes {
+		if o.ID == id {
+			return o
+		}
+	}
+	return Outcome{}
+}
+
+func predecessor(t *testing.T) (dir, page string, m *machine) {
+	t.Helper()
+	dir = t.TempDir()
+	page = filepath.Join(dir, "index.html")
+	if err := os.WriteFile(page, []byte("the predecessor's page\n"), 0o640); err != nil {
+		t.Fatal(err)
+	}
+	return dir, page, &machine{containers: map[string]*fakeContainer{
+		"hello-web": {id: "predecessor-id", running: true},
+	}}
+}
+
+func TestAFoundFileOfAnUntakenModuleIsKeptAsItIs(t *testing.T) {
+	dir, page, m := predecessor(t)
+	report, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
+
+	got, _ := os.ReadFile(page)
+	if string(got) != "the predecessor's page\n" {
+		t.Fatalf("a found file was changed: %q", got)
+	}
+	info, _ := os.Stat(page)
+	if info.Mode().Perm() != 0o640 {
+		t.Errorf("a found file's mode was changed to %o", info.Mode().Perm())
+	}
+	if o := outcomeOf(report, "hello-web.page"); o.Action != "held" ||
+		!strings.Contains(o.Detail, "kept until hello-web is taken") {
+		t.Errorf("the found file was not reported held: %+v", o)
+	}
+	h, ok := state.HeldAt("hello-web.page")
+	if !ok || h.Module != "hello-web" || h.Mode != "0640" {
+		t.Fatalf("the hold was not recorded: %+v", h)
+	}
+	kept, err := os.ReadFile(h.Kept)
+	if err != nil || string(kept) != "the predecessor's page\n" {
+		t.Fatalf("the original was not kept: %q %v", kept, err)
+	}
+	if info, _ := os.Stat(h.Kept); info.Mode().Perm() != 0o600 {
+		t.Errorf("the kept original is mode %o", info.Mode().Perm())
+	}
+	if _, recorded := state.Find("hello-web.page"); recorded {
+		t.Error("a held file was recorded as applied, so it would be removed as an orphan")
+	}
+	if report.Changed() {
+		t.Errorf("holding was reported as changing the machine: %+v", report.Outcomes)
+	}
+}
+
+func TestAFoundContainerOfAnUntakenModuleIsNotReplaced(t *testing.T) {
+	dir, page, m := predecessor(t)
+	report, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
+
+	if m.removed("hello-web") {
+		t.Fatal("a found container was removed")
+	}
+	for _, a := range m.asked {
+		if strings.HasPrefix(a, "docker run") {
+			t.Fatalf("a container was started over a found one: %s", a)
+		}
+	}
+	if outcomeOf(report, "hello-web.server").Action != "held" {
+		t.Errorf("the found container was not held: %+v", report.Outcomes)
+	}
+	if h, _ := state.HeldAt("hello-web.server"); h.Container != "predecessor-id" || !h.Running {
+		t.Errorf("the container as found was not recorded: %+v", h)
+	}
+}
+
+func TestWhatIsNotFoundIsCreatedWhenAssigned(t *testing.T) {
+	// Assigning prepares: what the module declares that is not there is made.
+	dir := t.TempDir()
+	page := filepath.Join(dir, "index.html")
+	m := &machine{containers: map[string]*fakeContainer{}}
+	report, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
+	if o := outcomeOf(report, "hello-web.page"); o.Action != "created" {
+		t.Errorf("an absent file of an untaken module was not created: %+v", o)
+	}
+	if o := outcomeOf(report, "hello-web.server"); o.Action != "created" {
+		t.Errorf("an absent container of an untaken module was not created: %+v", o)
+	}
+	if len(state.Held) != 0 {
+		t.Errorf("something was held that was not found: %+v", state.Held)
+	}
+}
+
+func TestAFileThisHostWroteIsNotFound(t *testing.T) {
+	// Found means present with no record. A record of any origin — carried or declared, this life
+	// of the node or an earlier one — means this host wrote it.
+	for _, origin := range []string{store.OriginCarried, store.OriginDeclared} {
+		dir, page, m := predecessor(t)
+		known := store.State{Resources: []store.Applied{
+			{ID: "earlier-name", Type: "file", Target: page, Origin: origin}}}
+		report, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), known, m, dir)
+		if o := outcomeOf(report, "hello-web.page"); o.Action == "held" {
+			t.Errorf("%s: a file this host has a record of was held: %+v", origin, o)
+		}
+		if _, held := state.HeldAt("hello-web.page"); held {
+			t.Errorf("%s: a recorded file was held", origin)
+		}
+	}
+}
+
+func TestAContainerAHostMadeIsNotFound(t *testing.T) {
+	dir, page, m := predecessor(t)
+	m.containers["hello-web"].spec = "some-spec"
+	report, _ := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
+	if o := outcomeOf(report, "hello-web.server"); o.Action == "held" {
+		t.Errorf("a container carrying the host's spec label was held: %+v", o)
+	}
+}
+
+func TestTheGenesisStoreAdoptedInPlaceIsNotFound(t *testing.T) {
+	// ADR 0078: the foundation's store, raised from the bundle and recorded as carried, is adopted
+	// as a module by name. It is the mesh's own and must never read as a predecessor's.
+	dir := t.TempDir()
+	m := &machine{containers: map[string]*fakeContainer{"mesh-store": {id: "x", running: true}}}
+	known := store.State{Resources: []store.Applied{{ID: "store", Type: "container", Target: "mesh-store"}}}
+	d := adopted(t, `{"taken":[],"untaken":{"postgres":["postgres.server"]}}`,
+		`{"id":"postgres.server","type":"container","name":"mesh-store","image":"`+pinned+`"}`)
+	report, state := applyAdopted(t, d, known, m, dir)
+	if o := outcomeOf(report, "postgres.server"); o.Action == "held" {
+		t.Errorf("the carried store was held: %+v", o)
+	}
+	if len(state.Held) != 0 {
+		t.Errorf("the carried store was held: %+v", state.Held)
+	}
+}
+
+func TestTakingAModuleReplacesWhatWasHeldAndTheOriginalSurvives(t *testing.T) {
+	dir, page, m := predecessor(t)
+	_, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
+	h, _ := state.HeldAt("hello-web.page")
+
+	report, state := applyAdopted(t, adopted(t, takenWeb, webResources(page)), state, m, dir)
+	got, _ := os.ReadFile(page)
+	if string(got) != "the mesh's page\n" {
+		t.Fatalf("taking the module did not converge the file: %q", got)
+	}
+	if !m.removed("hello-web") || m.containers["hello-web"].id != "made-by-host" {
+		t.Fatal("taking the module did not replace the found container")
+	}
+	if o := outcomeOf(report, "hello-web.page"); !strings.Contains(o.Detail, "original kept at "+h.Kept) {
+		t.Errorf("the cutover does not say where the original is: %+v", o)
+	}
+	if len(state.Held) != 0 {
+		t.Errorf("what was taken is still held: %+v", state.Held)
+	}
+	if _, recorded := state.Find("hello-web.page"); !recorded {
+		t.Error("a taken file was not recorded as applied")
+	}
+	kept, err := os.ReadFile(h.Kept)
+	if err != nil || string(kept) != "the predecessor's page\n" {
+		t.Errorf("the kept original did not survive the cutover: %q %v", kept, err)
+	}
+}
+
+func TestAHeldFileIsNeverRemovedWhenItsModuleIsUnassigned(t *testing.T) {
+	dir, page, m := predecessor(t)
+	_, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
+
+	other := filepath.Join(dir, "other")
+	_, state = applyAdopted(t, adopted(t, `{"taken":[]}`,
+		`{"id":"x.other","type":"file","path":"`+other+`","content":"x"}`), state, m, dir)
+	if got, _ := os.ReadFile(page); string(got) != "the predecessor's page\n" {
+		t.Fatalf("a held file was touched when its module left: %q", got)
+	}
+	if m.removed("hello-web") {
+		t.Fatal("a held container was removed when its module left")
+	}
+	if _, still := state.HeldAt("hello-web.page"); !still {
+		t.Error("the hold was forgotten, so a return of the module would read the file as the mesh's")
+	}
+}
+
+func TestAHeldFileRewrittenIsReportedAndNotReverted(t *testing.T) {
+	dir, page, m := predecessor(t)
+	d := adopted(t, untakenWeb, webResources(page))
+	_, state := applyAdopted(t, d, store.State{}, m, dir)
+
+	if err := os.WriteFile(page, []byte("the predecessor wrote again\n"), 0o640); err != nil {
+		t.Fatal(err)
+	}
+	report, state := applyAdopted(t, d, state, m, dir)
+	if got, _ := os.ReadFile(page); string(got) != "the predecessor wrote again\n" {
+		t.Fatalf("a held file was reverted: %q", got)
+	}
+	if h, _ := state.HeldAt("hello-web.page"); h.Changed != "rewritten" || h.ChangedAt.IsZero() {
+		t.Errorf("a rewrite was not recorded: %+v", h)
+	}
+	if o := outcomeOf(report, "hello-web.page"); !strings.Contains(o.Detail, "rewritten") {
+		t.Errorf("a rewrite was not reported: %+v", o)
+	}
+	h, _ := state.HeldAt("hello-web.page")
+	if kept, _ := os.ReadFile(h.Kept); string(kept) != "the predecessor's page\n" {
+		t.Errorf("the kept original was overwritten by a later write: %q", kept)
+	}
+}
+
+func TestAHeldContainerStoppedOrReplacedIsReportedAndNotRestarted(t *testing.T) {
+	for _, c := range []struct {
+		change func(*machine)
+		want   string
+	}{
+		{func(m *machine) { m.containers["hello-web"].running = false }, "stopped"},
+		{func(m *machine) { m.containers["hello-web"].id = "another" }, "replaced"},
+		{func(m *machine) { delete(m.containers, "hello-web") }, "gone"},
+	} {
+		dir, page, m := predecessor(t)
+		d := adopted(t, untakenWeb, webResources(page))
+		_, state := applyAdopted(t, d, store.State{}, m, dir)
+		c.change(m)
+		m.asked = nil
+		_, state = applyAdopted(t, d, state, m, dir)
+		if h, _ := state.HeldAt("hello-web.server"); h.Changed != c.want {
+			t.Errorf("%s: recorded as %q", c.want, h.Changed)
+		}
+		for _, a := range m.asked {
+			if strings.HasPrefix(a, "docker run") || strings.HasPrefix(a, "docker rm") ||
+				strings.HasPrefix(a, "docker start") {
+				t.Errorf("%s: the held container was acted on: %s", c.want, a)
+			}
+		}
+	}
+}
+
+func TestAHeldFileThatVanishesIsNotCreated(t *testing.T) {
+	dir, page, m := predecessor(t)
+	d := adopted(t, untakenWeb, webResources(page))
+	_, state := applyAdopted(t, d, store.State{}, m, dir)
+	if err := os.Remove(page); err != nil {
+		t.Fatal(err)
+	}
+	_, state = applyAdopted(t, d, state, m, dir)
+	if _, err := os.Stat(page); !errors.Is(err, os.ErrNotExist) {
+		t.Fatal("a held file that vanished was created before its module was taken")
+	}
+	if h, _ := state.HeldAt("hello-web.page"); h.Changed != "gone" {
+		t.Errorf("a vanished held file was not reported gone: %+v", h)
+	}
+}
+
+func TestAConvergedNodeStillReplacesWhatItFinds(t *testing.T) {
+	// No adoption, no holds: byte for byte what a converged node did before ADR 0100.
+	dir, page, m := predecessor(t)
+	d := parse(t, `{"declaration":1,"resources":[`+webResources(page)+`]}`)
+	report, state := applyAdopted(t, d, store.State{}, m, dir)
+	if got, _ := os.ReadFile(page); string(got) != "the mesh's page\n" {
+		t.Errorf("a converged node kept a found file: %q", got)
+	}
+	if !m.removed("hello-web") {
+		t.Error("a converged node kept a found container")
+	}
+	if len(state.Held) != 0 || outcomeOf(report, "hello-web.page").Action == "held" {
+		t.Errorf("a converged node held something: %+v", state.Held)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "kept")); !errors.Is(err, os.ErrNotExist) {
+		t.Error("a converged node kept originals")
+	}
+}