README.md

@@ -62,7 +62,6 @@ mesh-host owned          what this host has applied and still owns
   --json                 machine-readable
   --state                where this node keeps what it knows
   --dry-run              say what applying would change, and change nothing
-  --dry-run              read and check the declaration, change nothing
 ```
 
 ```
@@ -119,9 +118,13 @@ make host BUNDLE=path/to/foundation.lock
 `mesh-host reconcile` then applies it, on a machine the mesh has told nothing yet. That is the
 first node's path — no mesh present, nothing fetched, nothing else copied onto the machine. Once
 the mesh has spoken, `reconcile` holds the machine to what it last said and never to the bundle,
-which genesis consumed; and any declaration — bundle, file or kept — is refused when it says the
-other mode than the node is in, or is not what the mesh last said. Both commands say what they
-would change before changing anything, and `--dry-run` is that alone. `copy it and run it` stops being true the moment
+which genesis consumed; a bundle or a file is refused when it says the other mode than the node
+is in; and `apply FILE` is refused altogether once the mesh has spoken — a file is applied as the
+bundle is, its resources recorded as the machine's own, so on an enrolled node it would plan to
+remove the foundation. `apply FILE` is for a machine the mesh has not spoken to. (hq's to-be
+node lifecycle describes `apply repair.json` as a rescue on an enrolled node; that line is being
+amended in hq, and no rescue path exists here yet.) Both commands say what they would change
+before changing anything, and `--dry-run` is that alone. `copy it and run it` stops being true the moment
 a second file has to arrive with it, which is why the bundle is embedded rather than beside it.
 
 **A default build carries nothing and refuses to reconcile**, saying so. A host that applied

cmd/mesh-host/main.go

@@ -398,10 +398,12 @@ func short(digest string) string {
 // **A declaration carries no order.** The controller signs a version — the vocabulary — the
 // node it is for, the mode, and the resources: no sequence, no issued-at. What exists is
 // identity: the mesh names what it sends by the digest of the bytes, the node keeps the last one
-// it was sent, and genesis records the digest of what it consumed. So "older" can be said of
-// exactly two things — the bundle genesis consumed, and a bundle other than the one it consumed —
-// and of anything else only that it is not what the mesh last said, which is refused too: a
-// host that cannot tell older from newer and applies anyway is the fault this issue names.
+// it was sent, and genesis records the digest of what it consumed. So the bundle is held to
+// genesis's digest, and a file is not applied at all once the mesh has spoken: a file is applied
+// as the bundle is, its resources recorded as this machine's own — so the mesh could never remove
+// them again — and everything the mesh declared read as no longer declared and removed. Even the
+// very declaration the mesh last sent, applied from a file, would plan to remove the foundation.
+// `apply FILE` is for a machine the mesh has not spoken to, and is refused saying so.
 func refuseStale(known store.State, kept store.Declared, keptErr error, digest string, from provenance) error {
 	switch from {
 	case fromDeclared:
@@ -410,43 +412,45 @@ func refuseStale(known store.State, kept store.Declared, keptErr error, digest s
 		if known.Genesis == nil || known.Genesis.Digest == digest {
 			return nil
 		}
-		how := ""
-		if known.Genesis.Rewritten {
-			how = ", rewritten for this machine — its foundation ports, root credentials and mode"
-		}
+		// By digest. Genesis applies the bundle rewritten for this machine — its foundation
+		// ports, root credentials, mode — and writes that lock out; a host built from the
+		// carried template does not match it, and one built from the written lock does.
 		return fmt.Errorf("the bundle this host carries (%s) is not the one genesis applied here on %s "+
-			"(%s%s). The bundle was consumed then, and nothing the mesh has said is kept on this node "+
-			"yet, so there is nothing to reconcile against: enrol this node, or push to it from the "+
-			"controller", short(digest), known.Genesis.At.Format(time.RFC3339), short(known.Genesis.Digest), how)
+			"(%s), which was the bundle rewritten for this machine. It was consumed then, and nothing "+
+			"the mesh has said is kept on this node yet, so there is nothing to reconcile against: "+
+			"enrol this node, or push to it from the controller",
+			short(digest), known.Genesis.At.Format(time.RFC3339), short(known.Genesis.Digest))
 	}
 	// A file.
 	switch {
 	case errors.Is(keptErr, store.ErrNothingDeclared):
-		// The mesh has said nothing here. Nothing to be older than.
+		// The mesh has said nothing here: a machine a file is for.
 		return nil
 	case keptErr != nil:
 		return fmt.Errorf("%w\n\nNothing is applied over what this node cannot read back", keptErr)
 	}
 	last := apply.DigestOf(kept.Declaration)
-	if digest == last {
-		return nil
+	what := fmt.Sprintf("this file (%s) is not it", short(digest))
+	switch {
+	case digest == last:
+		what = "this file is that declaration, and from a file it would still be applied as a bundle is"
+	case known.Genesis != nil && digest == known.Genesis.Digest:
+		what = fmt.Sprintf("this file is the bundle genesis consumed on %s (%s), older than it",
+			known.Genesis.At.Format(time.RFC3339), short(digest))
 	}
-	if known.Genesis != nil && digest == known.Genesis.Digest {
-		return fmt.Errorf("this file is the bundle genesis consumed on %s (%s), and the mesh has since told "+
-			"this node declaration %s, kept as %s. An older declaration is not applied over a newer one: "+
-			"`reconcile` applies what the mesh last said", known.Genesis.At.Format(time.RFC3339), short(digest),
-			short(last), store.DeclaredName)
-	}
-	return fmt.Errorf("this file (%s) is not the declaration the mesh last told this node (%s, kept as %s). "+
-		"A declaration carries no sequence and no issued-at, so this host cannot tell an older one from a "+
-		"newer, and it applies only what the mesh last said: `reconcile` applies that, and `push` from the "+
-		"controller changes it", short(digest), short(last), store.DeclaredName)
+	return fmt.Errorf("`apply FILE` is for a machine the mesh has not spoken to. The mesh has told this "+
+		"node declaration %s, kept as %s, and %s. A file is applied as the bundle is — its resources "+
+		"recorded as this machine's own, which the mesh could then never remove, and what the mesh "+
+		"declared read as no longer declared — so no file is applied here: `reconcile` applies what "+
+		"the mesh last said, and `push` from the controller changes it",
+		short(last), store.DeclaredName, what)
 }
 
 // applied is what an apply says in machine-readable form: what it planned, then what it did.
 type applied struct {
-	Plan   []apply.Step `json:"plan"`
-	Report apply.Report `json:"report"`
+	Plan []apply.Step `json:"plan"`
+	// Report is absent on a dry run, which is the plan and nothing more.
+	Report *apply.Report `json:"report,omitempty"`
 }
 
 // runApply makes the machine match a declaration — after refusing one this node must not apply,
@@ -465,16 +469,34 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
 	if out == nil {
 		out = os.Stdout
 	}
+	// One apply at a time on this machine, whichever process asks: the link service applies too,
+	// and two saves of the state interleaved lose what one of them recorded.
+	unlock, err := store.Lock(opts.state, func() {
+		fmt.Fprintln(out, "another apply holds this node's state — mesh-host run, or the installer; waiting for it to finish")
+	})
+	if err != nil {
+		return err
+	}
+	defer unlock()
 	known, err := store.Load(opts.state)
 	if err != nil {
 		return err
 	}
 	source, origin, digest := from.name(opts), from.origin(), apply.DigestOf(raw)
 
-	// The mode this node is in. Recorded in the state since this check existed; a state written
-	// before then has it in what the mesh last said, which this node kept.
+	// The mode this node is in. What the mesh last said is signed and was verified on load; the
+	// state's note of it is this host's own, and where they disagree the note is what is wrong
+	// — a genesis re-run over a node the mesh converged since, a state saved when the kept
+	// declaration could not be — and is repaired, not obeyed. A bundle or a file is held to the
+	// note, or to the kept declaration when the note predates it.
 	kept, keptErr := store.ReadDeclared(store.DeclaredPath(opts.state))
-	if known.Mode == "" && keptErr == nil {
+	if from == fromDeclared {
+		if known.Mode != "" && known.Mode != apply.ModeOf(d) {
+			fmt.Fprintf(out, "this node's record said %s; what the mesh last said, signed, says %s — the record is repaired\n",
+				known.Mode, apply.ModeOf(d))
+		}
+		known.Mode = apply.ModeOf(d)
+	} else if known.Mode == "" && keptErr == nil {
 		if last, err := declaration.Parse(kept.Declaration); err == nil {
 			known.Mode = apply.ModeOf(last)
 		}
@@ -489,7 +511,7 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
 	// Said before anything is done.
 	steps := apply.Plan(d, known, origin)
 	if opts.json && opts.dryRun {
-		return writeJSONTo(out, steps)
+		return writeJSONTo(out, applied{Plan: steps})
 	}
 	mode := known.Mode
 	if mode == "" {
@@ -572,7 +594,7 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
 	}
 
 	if opts.json {
-		return writeJSONTo(out, applied{Plan: steps, Report: report})
+		return writeJSONTo(out, applied{Plan: steps, Report: &report})
 	}
 	if !report.Changed() {
 		fmt.Fprintf(out, "%s: already matches — %d resource(s) checked\n", source, len(report.Outcomes))
@@ -821,7 +843,7 @@ func runLink(ctx context.Context, opts options) error {
 	go sched.Run(ctx)
 
 	applier := func(ctx context.Context, raw, signature []byte) link.Report {
-		return applyAndKeep(ctx, opts, raw, &store.Declared{Declaration: raw, Signature: signature}, sched)
+		return applyAndKeep(ctx, opts, raw, &store.Declared{Declaration: raw, Signature: signature}, sched, say)
 	}
 
 	// Two things at once, and the second is what makes disconnection ordinary. The link brings
@@ -984,7 +1006,7 @@ func holdTheMachine(ctx context.Context, opts options, mine identity.Identity, s
 			continue
 		}
 
-		report := applyDeclared(ctx, opts, declared, sched)
+		report := applyDeclared(ctx, opts, declared, sched, say)
 		// A reconcile is otherwise silent. On an adopted node it speaks when what it holds or
 		// its firewall changed, because that is how a predecessor still writing is caught
 		// (novox/hq ADR 0100); publish decides whether anything did.
@@ -1005,25 +1027,33 @@ func holdTheMachine(ctx context.Context, opts options, mine identity.Identity, s
 // Signature checking happens before this is called, in the link. By the time anything here runs,
 // the question "is this from the mesh I joined" is settled — which is why this can treat the
 // bytes as instructions.
-func applyDeclared(ctx context.Context, opts options, raw []byte, sched *apply.Scheduler) link.Report {
-	return applyAndKeep(ctx, opts, raw, nil, sched)
+func applyDeclared(ctx context.Context, opts options, raw []byte, sched *apply.Scheduler, say link.Announce) link.Report {
+	return applyAndKeep(ctx, opts, raw, nil, sched, say)
 }
 
-// applying serialises applies on this node.
+// applying serialises applies within this process.
 //
 // **Two things apply here: the link and the reconcile loop**, and each reads the node's state,
 // acts on the machine, and writes the state back. Run at the same time they interleave, and the
 // one that saves last writes a state read before the other acted — losing what the first recorded:
 // a hold, the firewall found here, a resource just applied. The machine would then be one thing
 // and its record another, which is the fault every read-back in this package exists to prevent.
+// Across processes — `reconcile` run by hand beside this service — the lock beside the state does
+// the same (store.Lock).
 var applying sync.Mutex
 
 // applyAndKeep applies a declaration and, when it came from the mesh, keeps it so this node can
-// go on obeying it while disconnected. One at a time, whoever asks.
+// go on obeying it while disconnected. One at a time, whoever asks. Given unsigned, the bytes are
+// what this node kept, already verified against the mesh's key on load.
 func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.Declared,
-	sched *apply.Scheduler) link.Report {
+	sched *apply.Scheduler, say link.Announce) link.Report {
 	applying.Lock()
 	defer applying.Unlock()
+	unlock, err := store.Lock(opts.state, nil)
+	if err != nil {
+		return link.Report{Refused: err.Error()}
+	}
+	defer unlock()
 
 	declared, err := declaration.Parse(raw)
 	if err != nil {
@@ -1034,13 +1064,19 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D
 	if err != nil {
 		return link.Report{Refused: err.Error()}
 	}
-	// A declaration the link delivered is the controller's word on the node's mode — the flip
-	// arrives as exactly that, the first converged declaration after adopted ones — and becomes
-	// the record. What this node re-applies on its own is held to the record (novox/hq issue 104).
-	if signed == nil {
-		if err := apply.CheckMode(known, declared); err != nil {
-			return link.Report{Refused: err.Error()}
+	// A declaration from the mesh is the controller's word on the node's mode — the flip arrives
+	// as exactly that, the first converged declaration after adopted ones — and becomes the
+	// record. So does what this node kept: it is signed by the mesh and verified on load, where
+	// the state's note of the mode is this host's own. Where they disagree the note is wrong — a
+	// genesis re-run over a node the mesh converged since, a state saved when the kept declaration
+	// could not be — and it is repaired and said, not obeyed: refusing would hold this node off what
+	// the mesh said until a delivery that comes only when something changes (novox/hq issue 104).
+	if signed == nil && known.Mode != "" && known.Mode != apply.ModeOf(declared) {
+		if say != nil {
+			say(fmt.Sprintf("this node's record said %s; what the mesh last said, signed, says %s — the record is repaired",
+				known.Mode, apply.ModeOf(declared)))
 		}
+		known.Mode = apply.ModeOf(declared)
 	}
 
 	built, err := system.For(builtFor)

cmd/mesh-host/main_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"crypto/ed25519"
+	"encoding/json"
 	"errors"
 	"os"
 	"path/filepath"
@@ -224,7 +225,7 @@ func TestOnlyOneApplyRunsAtATime(t *testing.T) {
 	// Whatever else is applying — the link, while this is the reconcile — this waits for it.
 	applying.Lock()
 	done := make(chan link.Report, 1)
-	go func() { done <- applyAndKeep(context.Background(), opts, raw, nil, nil) }()
+	go func() { done <- applyAndKeep(context.Background(), opts, raw, nil, nil, nil) }()
 	select {
 	case report := <-done:
 		applying.Unlock()
@@ -307,16 +308,23 @@ func TestAConvergedDeclarationIsRefusedOnAnAdoptedNode(t *testing.T) {
 	}
 }
 
-func TestAnAdoptedDeclarationIsRefusedOnAConvergedNode(t *testing.T) {
-	// Only the mesh can say a node is adopted, so this one arrives the way a disconnected node
-	// re-applies what it kept: through the reconcile loop, unsigned.
+func TestTheKeptDeclarationRepairsARecordThatDisagrees(t *testing.T) {
+	// What a node kept is signed by the mesh and verified on load; the state's note of the mode is
+	// the host's own. A genesis re-run after `converge`, or a state saved when the kept declaration
+	// could not be, leaves them apart — and a loop that refused every five minutes would hold the
+	// node off what the mesh said until a delivery that comes only when something changes. The
+	// kept declaration wins, and the repair is said.
 	opts := stateWithMode(t, store.ModeConverged)
 	raw := []byte(`{"declaration":1,"adoption":{"taken":[]},"resources":[{"id":"a","type":"file","path":"` +
 		filepath.Join(filepath.Dir(opts.state), "a.conf") + `","content":"x\n"}]}`)
-	report := applyAndKeep(context.Background(), opts, raw, nil, nil)
-	want := "this node is converged; the declaration says adopted"
-	if !strings.Contains(report.Refused, want) || !strings.Contains(report.Refused, "`adopt`") {
-		t.Errorf("refused = %q, want it to name both modes and the act that changes it", report.Refused)
+	var said []string
+	report := applyAndKeep(context.Background(), opts, raw, nil, nil, func(line string) { said = append(said, line) })
+	if strings.Contains(report.Refused, "the declaration says") {
+		t.Fatalf("what the node kept was refused against its own note: %s", report.Refused)
+	}
+	if len(said) != 1 || !strings.Contains(said[0], "record said converged") ||
+		!strings.Contains(said[0], "says adopted") || !strings.Contains(said[0], "repaired") {
+		t.Errorf("the repair was not said: %q", said)
 	}
 }
 
@@ -328,15 +336,16 @@ func TestTheMeshItselfMayChangeTheMode(t *testing.T) {
 	opts := stateWithMode(t, store.ModeAdopted)
 	raw := []byte(`{"declaration":1,"resources":[{"id":"a","type":"file","path":"` +
 		filepath.Join(filepath.Dir(opts.state), "a.conf") + `","content":"x\n"}]}`)
-	report := applyAndKeep(context.Background(), opts, raw, &store.Declared{Declaration: raw}, nil)
+	report := applyAndKeep(context.Background(), opts, raw, &store.Declared{Declaration: raw}, nil, nil)
 	if strings.Contains(report.Refused, "the declaration says") {
 		t.Errorf("the mesh's own flip was refused for its mode: %s", report.Refused)
 	}
 }
 
 // Defends novox/hq issue 104: a declaration older than what the mesh last said is refused, naming
-// both — and a declaration carries no order, so one that is merely not the last is refused too,
-// saying what is missing.
+// both — and `apply FILE` is refused altogether once the mesh has spoken, the last declaration
+// itself included: from a file it is applied as the bundle is, which would record the mesh's
+// resources as this machine's own and remove the foundation as undeclared.
 func TestAnOlderDeclarationIsRefused(t *testing.T) {
 	opts := stateWithMode(t, "")
 	genesis := []byte(`{"declaration":1,"resources":[{"id":"g","type":"file","path":"/tmp/g","content":"genesis\n"}]}`)
@@ -372,9 +381,18 @@ func TestAnOlderDeclarationIsRefused(t *testing.T) {
 	if err == nil {
 		t.Fatal("a declaration that is not what the mesh last said was applied")
 	}
-	if !strings.Contains(err.Error(), "no sequence and no issued-at") ||
+	if !strings.Contains(err.Error(), "for a machine the mesh has not spoken to") ||
 		!strings.Contains(err.Error(), short(apply.DigestOf(since))) {
-		t.Errorf("the refusal does not say what is missing and what was last said: %v", err)
+		t.Errorf("the refusal does not say what a file is for and what was last said: %v", err)
+	}
+
+	// The very declaration the mesh last sent, from a file: still a file.
+	err = runApply(context.Background(), opts, parsed(since), since, fromFile)
+	if err == nil || !strings.Contains(err.Error(), "applied as the bundle is") {
+		t.Errorf("the last declaration, from a file, was not refused as a file: %v", err)
+	}
+	if known, _ := store.Load(opts.state); len(known.Resources) != 0 {
+		t.Errorf("a refused file recorded %d resource(s)", len(known.Resources))
 	}
 
 	// A bundle other than the one genesis consumed: what genesis applied was rewritten for this
@@ -416,6 +434,20 @@ func TestADryRunChangesNothingAndListsTheActions(t *testing.T) {
 	if strings.Contains(out.String(), "applying:") {
 		t.Errorf("a dry run went on to apply:\n%s", out.String())
 	}
+
+	// Machine-readable, the same shape as an apply's: the plan, and no report.
+	out.Reset()
+	opts.json = true
+	if err := runApply(context.Background(), opts, d, raw, fromFile); err != nil {
+		t.Fatal(err)
+	}
+	var shape struct {
+		Plan   []apply.Step     `json:"plan"`
+		Report *json.RawMessage `json:"report"`
+	}
+	if err := json.Unmarshal(out.Bytes(), &shape); err != nil || len(shape.Plan) != 2 || shape.Report != nil {
+		t.Errorf("a json dry run is not {plan} alone: %v\n%s", err, out.String())
+	}
 }
 
 // Defends novox/hq issue 104: once the mesh has told this node anything, `reconcile` holds it to

internal/apply/plan.go

@@ -118,18 +118,54 @@ func Plan(d *declaration.Declaration, known store.State, origin string) []Step {
 }
 
 // planned is what one declared resource would come to.
+//
+// **In the order holdOnAdopted decides it**, because the one cutover ADR 0100 says must be
+// previewed is the one a plan gets backwards if it looks at the record first: a resource this
+// node holds for a module the declaration now says is taken is not held any longer — it is
+// applied, and what was found is replaced. The declaration's word on which modules are untaken
+// comes first; the record of what is held only says what that replacement replaces.
 func planned(r declaration.Resource, d *declaration.Declaration, known store.State) Step {
 	step := Step{Type: string(r.Kind()), ID: r.Identity(), Target: r.Target()}
 
 	if d.Adoption != nil {
-		if h, held := known.HeldAt(r.Identity()); held {
-			step.Verb, step.Why = "hold", "found on this machine and kept as it is until "+h.Module+" is taken"
-			return step
+		// Something run inside a held container is held with it, while that container's module
+		// is untaken; once the module is taken the container is replaced before this runs.
+		if in := runsIn(r); in != "" {
+			if container, isHeld := heldContainer(known, in); isHeld {
+				if _, untaken := d.Adoption.Untaken[container.Module]; untaken {
+					step.Verb = "hold"
+					step.Why = "runs in " + in + ", which is held as found; not run until " + container.Module + " is taken"
+					return step
+				}
+			}
 		}
-		if module, untaken := d.Adoption.UntakenModuleOf(r.Identity()); untaken {
+		// A file written into replaces nothing that was found, so it is never held (ADR 0102).
+		into := false
+		if f, ok := r.(*declaration.File); ok && f.Into != "" {
+			into = true
+		}
+		h, held := known.HeldAt(r.Identity())
+		module, untaken := d.Adoption.UntakenModuleOf(r.Identity())
+		switch {
+		case into:
+		case untaken && held:
+			step.Verb, step.Why = "hold", "found on this machine and kept as it is until "+module+" is taken"
+			return step
+		case untaken:
 			step.Verb = "create"
 			step.Why = "unless it is found on this machine — then held as it is until " + module + " is taken"
 			return step
+		case held:
+			// The cutover: the module is taken, and what was held for it is replaced.
+			step.Verb = "create"
+			if _, recorded := known.Find(r.Identity()); recorded {
+				step.Verb = "update"
+			}
+			step.Why = h.Module + " is taken: replaces what was found and held"
+			if h.Kept != "" {
+				step.Why += "; the original stays at " + h.Kept
+			}
+			return step
 		}
 	}
 

internal/apply/plan_test.go

@@ -1,6 +1,9 @@
 package apply
 
 import (
+	"context"
+	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 	"time"
@@ -92,3 +95,133 @@ func TestAPlanHoldsWhatAnAdoptedNodeFound(t *testing.T) {
 		t.Errorf("the plan does not say an untaken module's resource is held if found: %q", steps[1].Why)
 	}
 }
+
+// both is a machine with a container runtime and ufw on it at once.
+type both struct {
+	m *machine
+	u *ufwMachine
+}
+
+func (b *both) run(ctx context.Context, name string, args ...string) (string, error) {
+	switch name {
+	case "nft", "ufw", "iptables", "ip6tables", "firewall-cmd":
+		return b.u.run(ctx, name, args...)
+	}
+	return b.m.run(ctx, name, args...)
+}
+
+func ids(steps []Step) []string {
+	var out []string
+	for _, s := range steps {
+		if s.Type == "firewall" {
+			continue // said, not an outcome
+		}
+		out = append(out, s.ID)
+	}
+	return out
+}
+
+func outcomeIDs(r Report) []string {
+	var out []string
+	for _, o := range r.Outcomes {
+		out = append(out, o.ID)
+	}
+	return out
+}
+
+func write(t *testing.T, path, content string) {
+	t.Helper()
+	if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Defends novox/hq issue 104: the plan is the apply, said first — the same resources in the same
+// order, and the one cutover ADR 0100 says must be previewed said as a cutover, not a hold.
+func TestThePlanIsTheApplyInOrder(t *testing.T) {
+	dir := t.TempDir()
+	guard, page, keep, old := filepath.Join(dir, "guard.nft"), filepath.Join(dir, "index.html"),
+		filepath.Join(dir, "keep.html"), filepath.Join(dir, "old.conf")
+	for _, p := range []string{page, keep, old} {
+		write(t, p, "the predecessor's\n")
+	}
+
+	// Returning to adopted, with a guard to raise, an orphan, a hold that stays, a hold whose
+	// module is now taken, and a hold no longer declared.
+	back := adopted(t, `{"taken":["hello-web"],"untaken":{"keep":["keep.page"]}}`,
+		`{"id":"adoption.guard.table","type":"file","path":"`+guard+`","content":"table inet mesh-guard {}\n"},
+		 {"id":"hello-web.page","type":"file","path":"`+page+`","content":"the mesh's page\n"},
+		 {"id":"keep.page","type":"file","path":"`+keep+`","content":"the mesh's keep\n"}`)
+	known := store.State{Firewall: &store.FoundFirewall{Kind: "ufw", WasActive: true, DisabledByMesh: true, FoundAt: time.Now()}}
+	known.Record(store.Applied{ID: "old.conf", Type: "file", Target: old, Origin: store.OriginDeclared})
+	for id, module := range map[string]string{"hello-web.page": "hello-web", "keep.page": "keep", "gone.page": "gone"} {
+		known.RecordHeld(store.Held{ID: id, Module: module, Kind: "file", Target: filepath.Join(dir, id), Since: time.Now()})
+	}
+	fake := &both{m: &machine{containers: map[string]*fakeContainer{}}, u: &ufwMachine{installed: true}}
+
+	plan := Plan(back, known, store.OriginDeclared)
+	report, state, err := ApplyKeeping(context.Background(), archHost(t), back, known, store.OriginDeclared,
+		fake.run, nil, nil, KeepIn(dir))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got, want := verbs(plan), "enable ufw, forget gone.page, create adoption.guard.table, remove old.conf, "+
+		"create hello-web.page, hold keep.page"; got != want {
+		t.Errorf("planned %q, want %q", got, want)
+	}
+	if got, want := strings.Join(ids(plan), " "), strings.Join(outcomeIDs(report), " "); got != want {
+		t.Errorf("the plan said %q and the apply did %q", got, want)
+	}
+	taken := outcomeOf(report, "hello-web.page")
+	if !strings.Contains(taken.Detail, "taken") || !strings.Contains(plan[4].Why, "hello-web is taken: replaces what was found") {
+		t.Errorf("the cutover was applied as %q and planned as %q", taken.Detail, plan[4].Why)
+	}
+	if !fake.u.active || state.Firewall.DisabledByMesh {
+		t.Error("returning to adopted did not enable ufw again")
+	}
+
+	// Converged, from the mesh: an orphan, a new file, ufw retired, and what protected the node
+	// removed last.
+	flip := parse(t, `{"declaration":1,"resources":[{"id":"a","type":"file","path":"`+filepath.Join(dir, "a.conf")+`","content":"a\n"}]}`)
+	write(t, old, "again\n")
+	known = store.State{Firewall: &store.FoundFirewall{Kind: "ufw", WasActive: true, FoundAt: time.Now()}}
+	known.Record(store.Applied{ID: "adoption.guard.table", Type: "file", Target: guard, Origin: store.OriginDeclared})
+	known.Record(store.Applied{ID: "old.conf", Type: "file", Target: old, Origin: store.OriginDeclared})
+	fake = &both{m: &machine{containers: map[string]*fakeContainer{}}, u: &ufwMachine{installed: true, active: true,
+		ruleset: "table inet mesh {\n}\n"}}
+
+	plan = Plan(flip, known, store.OriginDeclared)
+	report, state, err = ApplyKeeping(context.Background(), archHost(t), flip, known, store.OriginDeclared,
+		fake.run, nil, nil, KeepIn(dir))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got, want := verbs(plan), "remove old.conf, create a, disable ufw, remove adoption.guard.table"; got != want {
+		t.Errorf("planned %q, want %q", got, want)
+	}
+	if got, want := strings.Join(ids(plan), " "), strings.Join(outcomeIDs(report), " "); got != want {
+		t.Errorf("the plan said %q and the apply did %q", got, want)
+	}
+	if fake.u.active || !state.Firewall.DisabledByMesh {
+		t.Error("converging did not retire ufw")
+	}
+}
+
+func TestAResourceRunInAHeldContainerIsPlannedAsItIsApplied(t *testing.T) {
+	// A run-once step sharing a container's namespace runs in it, as an action's `in` does.
+	img := "example/x@sha256:0000000000000000000000000000000000000000000000000000000000000000"
+	d := parse(t, `{"declaration":1,"adoption":{"taken":["web"],"untaken":{"db":["db.server"]}},"resources":[
+	  {"id":"web.server","type":"container","name":"web","image":"`+img+`"},
+	  {"id":"db.server","type":"container","name":"db","image":"`+img+`"},
+	  {"id":"db.init","type":"container","name":"db-init","image":"`+img+`","run-once":true,"network":"container:db"},
+	  {"id":"web.warm","type":"container","name":"web-warm","image":"`+img+`","run-once":true,"network":"container:web"}]}`)
+	known := store.State{}
+	known.RecordHeld(store.Held{ID: "db.server", Module: "db", Kind: "container", Target: "db", Container: "predecessor"})
+	known.RecordHeld(store.Held{ID: "web.server", Module: "web", Kind: "container", Target: "web", Container: "predecessor"})
+	got := verbs(Plan(d, known, store.OriginDeclared))
+	// db is untaken, so what runs in it waits; web is taken, so its held container is replaced (the
+	// cutover) and what runs in it runs.
+	if got != "create web.server, hold db.server, hold db.init, create web.warm" {
+		t.Errorf("planned %q", got)
+	}
+}

internal/bootstrap/apply.go

@@ -37,10 +37,11 @@ type Runner = apply.Runner
 //
 // What it does record is that the bundle was consumed, and in which mode the operator raised
 // the machine. `raw` is the exact bytes of what is applied — the bundle as rewritten for this
-// machine, not as carried — and its digest is what `mesh-host reconcile` later holds the carried
-// bundle against: what genesis applied had its ports, root credentials and adoption rewritten,
-// so the carried bytes are never it, and applying them on a raised node recreated the store and
-// loaded the converged filter on an adopted one (novox/hq issue 104).
+// machine — and its digest is what `mesh-host reconcile` later holds the carried bundle against,
+// by digest: a host built from the carried template does not match it, since genesis rewrote the
+// ports, root credentials and adoption; a host built from the lock genesis wrote out does.
+// Applying the unrewritten template on a raised node recreated the store and loaded the converged
+// filter on an adopted one (novox/hq issue 104).
 func ApplyBundle(ctx context.Context, o Options, sys system.System, d *declaration.Declaration,
 	raw []byte, run Runner, say func(string)) (apply.Report, error) {
 
@@ -50,6 +51,12 @@ func ApplyBundle(ctx context.Context, o Options, sys system.System, d *declarati
 		return apply.Report{}, err
 	}
 
+	// One apply at a time on this machine: a host already running here applies too.
+	unlock, err := store.Lock(o.State, func() { say("  waiting           another apply holds this node's state") })
+	if err != nil {
+		return apply.Report{}, err
+	}
+	defer unlock()
 	known, err := store.Load(o.State)
 	if err != nil {
 		return apply.Report{}, err
@@ -65,12 +72,16 @@ func ApplyBundle(ctx context.Context, o Options, sys system.System, d *declarati
 
 	// The bundle is consumed, whichever way the apply went: what is on the machine came from these
 	// bytes, and the carried ones must not be applied over it. The mode is the operator's word at
-	// genesis; the controller records the same and says it in every declaration from then on
-	// (novox/hq ADR 0100).
+	// genesis, and only at genesis: the controller records the same and says it in every
+	// declaration from then on (novox/hq ADR 0100), so a node the mesh has spoken to — this
+	// installer re-run on it, or finishing a pivot — keeps the mode it has, which may since have
+	// been flipped.
 	updated.Genesis = &store.Genesis{Digest: apply.DigestOf(raw), At: time.Now().UTC(), Rewritten: true}
-	updated.Mode = store.ModeConverged
-	if o.Adopted {
-		updated.Mode = store.ModeAdopted
+	if updated.Mode == "" {
+		updated.Mode = store.ModeConverged
+		if o.Adopted {
+			updated.Mode = store.ModeAdopted
+		}
 	}
 
 	// Saved whichever way it went, for the reason `mesh-host` gives: what was applied before a

internal/bootstrap/apply_test.go

@@ -166,4 +166,17 @@ func TestGenesisConsumesTheBundleAndRecordsTheMode(t *testing.T) {
 			t.Errorf("adopted=%v: genesis recorded the mode as %q, want %q", adopted, known.Mode, want)
 		}
 	}
+
+	// Re-run on a node the mesh has spoken to since — and converged — genesis leaves the mode
+	// alone: it is the operator's word at genesis, and the controller's from then on.
+	o := Options{State: filepath.Join(dir, "state-flipped.json"), Adopted: true}
+	if err := store.Save(o.State, store.State{Mode: store.ModeConverged}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ApplyBundle(context.Background(), o, sys, d, raw, nil, quietly); err != nil {
+		t.Fatal(err)
+	}
+	if known, _ := store.Load(o.State); known.Mode != store.ModeConverged {
+		t.Errorf("a genesis re-run set the mode back to %q over the mesh's converged", known.Mode)
+	}
 }

internal/store/lock.go

@@ -0,0 +1,58 @@
+package store
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"syscall"
+)
+
+// One apply at a time on a machine, whoever asks.
+//
+// Three things apply here and each reads the state, acts, and writes the state back: the link
+// and the reconcile loop inside `mesh-host run`, the host's own `reconcile` and `apply` run by
+// hand, and the installer at genesis. Inside one process a mutex serialises them; across
+// processes nothing did, and two applies interleaved leave the last saver writing a state read
+// before the other acted — a hold, a firewall record, a resource just applied, lost (novox/hq
+// issue 104 review). A lock on a file beside the state is what every one of them can take, however
+// it was started: a `reconcile` run against a service, or against a `mesh-host run` somebody
+// started by hand, waits the same way. Refusing while a named service is active would have known
+// the service's name and missed the hand-started one.
+//
+// An advisory lock, held for the life of the open file and released by the kernel when the
+// process ends, so a host that dies mid-apply leaves no lock behind for the next one to clear.
+
+// LockName is the file the lock is taken on, beside the state.
+const LockName = "state.lock"
+
+// Lock takes the machine's apply lock and returns what releases it. When another process holds
+// it, wait is told once — so a person running a command knows what they are waiting for — and
+// Lock blocks until it is free.
+func Lock(statePath string, wait func()) (func(), error) {
+	dir := filepath.Dir(statePath)
+	if err := os.MkdirAll(dir, 0o700); err != nil {
+		return nil, err
+	}
+	f, err := os.OpenFile(filepath.Join(dir, LockName), os.O_CREATE|os.O_RDWR, 0o600)
+	if err != nil {
+		return nil, fmt.Errorf("cannot take this node's apply lock: %w", err)
+	}
+	if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
+		if !errors.Is(err, syscall.EWOULDBLOCK) {
+			f.Close()
+			return nil, fmt.Errorf("cannot take this node's apply lock: %w", err)
+		}
+		if wait != nil {
+			wait()
+		}
+		if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil {
+			f.Close()
+			return nil, fmt.Errorf("waiting for this node's apply lock: %w", err)
+		}
+	}
+	return func() {
+		_ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
+		f.Close()
+	}, nil
+}

internal/store/lock_test.go

@@ -0,0 +1,45 @@
+package store
+
+import (
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+// Defends the node's record across processes: a `reconcile` run by hand beside the link service,
+// or the installer beside either, waits for the other's apply rather than saving over it.
+func TestASecondApplyWaitsForTheFirst(t *testing.T) {
+	state := filepath.Join(t.TempDir(), "state.json")
+	release, err := Lock(state, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	waited := make(chan struct{}, 1)
+	got := make(chan struct{})
+	go func() {
+		unlock, err := Lock(state, func() { waited <- struct{}{} })
+		if err != nil {
+			t.Error(err)
+		}
+		close(got)
+		unlock()
+	}()
+
+	select {
+	case <-waited:
+	case <-time.After(5 * time.Second):
+		t.Fatal("the second apply was not told it is waiting")
+	}
+	select {
+	case <-got:
+		t.Fatal("the second apply took the lock while the first held it")
+	case <-time.After(50 * time.Millisecond):
+	}
+	release()
+	select {
+	case <-got:
+	case <-time.After(5 * time.Second):
+		t.Fatal("the second apply never got the lock once the first let go")
+	}
+}