The bundle and the mesh stop removing each other

04-ISSUES/010. The store now records where each resource came from -- carried,
or declared -- and each origin removes only its own. A declaration removes what
the mesh previously declared and never what the bundle raised.

State written before the field existed reads as carried, because everything a
host had applied by then came from its bundle: there was no other way to tell
it anything. Guessing the other way would have the first upgrade remove the
substrate, which is this fault arriving through the change that fixes it.

Verified on the scenario that caused it, and on the property that had to
survive it: a later declaration dropping a resource still removes that
resource, so removal by omission still means what it meant.

Also stops swallowing a publish failure. A node that applied a declaration and
could not tell the mesh looked exactly like one that had -- the mesh believing
it never answered, the node believing it did, and nothing anywhere saying so.
Reports are published mandatory now, so anything the broker cannot route comes
back and is said out loud rather than dropped in silence.

cmd/mesh-host/main.go

@@ -331,7 +331,7 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, sou
 		return err
 	}
 
-	report, updated, applyErr := apply.Apply(ctx, sys, d, known, apply.ExecRunner, func(line string) {
+	report, updated, applyErr := apply.Apply(ctx, sys, d, known, store.OriginCarried, apply.ExecRunner, func(line string) {
 		if !opts.json {
 			fmt.Println(line)
 		}
@@ -538,7 +538,7 @@ func runLink(ctx context.Context, opts options) error {
 		Fingerprint: mine.Membership.Fingerprint,
 		Password:    mine.Membership.Password,
 		Signer:      mine.Membership.Signer,
-	}, apply, opts.timeout)
+	}, apply, func(line string) { fmt.Println(line) }, opts.timeout)
 }
 
 // applyDeclared applies a declaration that has already been proved to come from the mesh.
@@ -569,7 +569,10 @@ func applyDeclared(ctx context.Context, opts options, raw []byte) link.Report {
 		return link.Report{Refused: err.Error()}
 	}
 
-	outcome, updated, applyErr := apply.Apply(ctx, built, declared, known, apply.ExecRunner, nil)
+	// Declared, not carried. A declaration from the mesh removes only what the mesh previously
+	// declared — never what this machine raised for itself from its bundle (04-ISSUES/010).
+	outcome, updated, applyErr := apply.Apply(ctx, built, declared, known, store.OriginDeclared,
+		apply.ExecRunner, nil)
 
 	// Saved whichever way it went. Recording only on success would lose the footprint of a
 	// failed apply, and that footprint is on the machine either way.

examples/substrate-first-node.lock

@@ -74,7 +74,7 @@
       "command": ["docker", "run", "--rm", "--network", "container:mesh-store",
                   "-e", "MESH_STORE_INVENTORY=postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable",
                   "-e", "MESH_STORE_IDENTITY=postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable",
-                  "192.0.2.250:5000/mesh-control@sha256:1dfcf6a879e16e671d4d6459271fd2630e30560afa1211508ab3994494786893",
+                  "192.0.2.250:5000/mesh-control@sha256:b14e0a9765445b56f21aa8cf25be6a614b46974de710ddad92ee577074065858",
                   "migrate"],
       "verify": ["sh", "-c", "docker exec mesh-store psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec mesh-store psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key"]
     },
@@ -108,7 +108,7 @@
       "id": "control-plane",
       "type": "container",
       "name": "mesh-control",
-      "image": "192.0.2.250:5000/mesh-control@sha256:1dfcf6a879e16e671d4d6459271fd2630e30560afa1211508ab3994494786893",
+      "image": "192.0.2.250:5000/mesh-control@sha256:b14e0a9765445b56f21aa8cf25be6a614b46974de710ddad92ee577074065858",
       "network": "host",
       "args": ["serve"],
       "volumes": ["mesh-broker-tls:/broker-tls:ro"],

internal/apply/apply.go

@@ -85,6 +85,7 @@ func Apply(
 	sys system.System,
 	d *declaration.Declaration,
 	known store.State,
+	origin string,
 	run Runner,
 	log func(string),
 ) (Report, store.State, error) {
@@ -98,7 +99,7 @@ func Apply(
 		declared[r.Identity()] = true
 	}
 
-	for _, orphan := range known.Orphans(declared) {
+	for _, orphan := range known.Orphans(declared, origin) {
 		action, detail, err := remove(ctx, sys, orphan, run)
 		if err != nil {
 			return report, known, &Error{Resource: orphan.ID, Err: err, Done: report}
@@ -119,7 +120,8 @@ func Apply(
 
 		// Only now. The record follows the fact, never leads it.
 		known.Record(store.Applied{
-			ID: resource.Identity(), Type: string(resource.Kind()),
+			Origin: origin,
+			ID:     resource.Identity(), Type: string(resource.Kind()),
 			Target: outcome.Target, AppliedAt: time.Now().UTC(),
 		})
 		report.Outcomes = append(report.Outcomes, outcome)

internal/apply/apply_test.go

@@ -39,7 +39,7 @@ func TestApplyingTwiceChangesNothingTheSecondTime(t *testing.T) {
 	  {"id":"f","type":"file","path":"`+dir+`/etc/a.conf","content":"hello\n","mode":"0640"}
 	]}`)
 
-	first, state, err := Apply(context.Background(), archHost(t), d, store.State{}, noServices, nil)
+	first, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -47,7 +47,7 @@ func TestApplyingTwiceChangesNothingTheSecondTime(t *testing.T) {
 		t.Fatal("the first apply on an empty machine changed nothing")
 	}
 
-	second, _, err := Apply(context.Background(), archHost(t), d, state, noServices, nil)
+	second, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -65,7 +65,7 @@ func TestADriftedMachineIsReturned(t *testing.T) {
 	  {"id":"f","type":"file","path":"`+path+`","content":"correct\n","mode":"0644"}
 	]}`)
 
-	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, noServices, nil)
+	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -73,7 +73,7 @@ func TestADriftedMachineIsReturned(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	report, _, err := Apply(context.Background(), archHost(t), d, state, noServices, nil)
+	report, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -97,7 +97,7 @@ func TestADroppedResourceIsRemoved(t *testing.T) {
 	  {"id":"keep","type":"file","path":"`+keep+`","content":"a\n"},
 	  {"id":"drop","type":"file","path":"`+drop+`","content":"b\n"}
 	]}`)
-	_, state, err := Apply(context.Background(), archHost(t), both, store.State{}, noServices, nil)
+	_, state, err := Apply(context.Background(), archHost(t), both, store.State{}, store.OriginCarried, noServices, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -105,7 +105,7 @@ func TestADroppedResourceIsRemoved(t *testing.T) {
 	one := parse(t, `{"declaration":1,"resources":[
 	  {"id":"keep","type":"file","path":"`+keep+`","content":"a\n"}
 	]}`)
-	report, state, err := Apply(context.Background(), archHost(t), one, state, noServices, nil)
+	report, state, err := Apply(context.Background(), archHost(t), one, state, store.OriginCarried, noServices, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -137,7 +137,7 @@ func TestNothingTheHostDidNotCreateIsTouched(t *testing.T) {
 	d := parse(t, `{"declaration":1,"resources":[
 	  {"id":"ours","type":"file","path":"`+filepath.Join(dir, "ours.conf")+`","content":"a\n"}
 	]}`)
-	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, noServices, nil); err != nil {
+	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil); err != nil {
 		t.Fatal(err)
 	}
 
@@ -156,7 +156,7 @@ func TestARenameToTheSamePathDoesNotDeleteTheNewFile(t *testing.T) {
 	before := parse(t, `{"declaration":1,"resources":[
 	  {"id":"old","type":"file","path":"`+path+`","content":"old\n"}
 	]}`)
-	_, state, err := Apply(context.Background(), archHost(t), before, store.State{}, noServices, nil)
+	_, state, err := Apply(context.Background(), archHost(t), before, store.State{}, store.OriginCarried, noServices, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -164,7 +164,7 @@ func TestARenameToTheSamePathDoesNotDeleteTheNewFile(t *testing.T) {
 	after := parse(t, `{"declaration":1,"resources":[
 	  {"id":"new","type":"file","path":"`+path+`","content":"new\n"}
 	]}`)
-	if _, _, err := Apply(context.Background(), archHost(t), after, state, noServices, nil); err != nil {
+	if _, _, err := Apply(context.Background(), archHost(t), after, state, store.OriginCarried, noServices, nil); err != nil {
 		t.Fatal(err)
 	}
 
@@ -192,7 +192,7 @@ func TestAFailedStepFailsTheApply(t *testing.T) {
 	  {"id":"never","type":"file","path":"`+filepath.Join(dir, "never.conf")+`","content":"b\n"}
 	]}`)
 
-	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, noServices, nil)
+	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil)
 	if err == nil {
 		t.Fatal("an impossible resource did not fail the apply")
 	}
@@ -225,7 +225,7 @@ func TestNothingIsRecordedUntilItWorked(t *testing.T) {
 	  {"id":"doomed","type":"directory","path":"`+blocker+`"}
 	]}`)
 
-	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, noServices, nil)
+	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil)
 	if err == nil {
 		t.Fatal("expected a failure")
 	}
@@ -244,7 +244,7 @@ func TestAModeIsMaintainedNotJustSet(t *testing.T) {
 	  {"id":"f","type":"file","path":"`+path+`","content":"s\n","mode":"0600"}
 	]}`)
 
-	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, noServices, nil)
+	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -252,7 +252,7 @@ func TestAModeIsMaintainedNotJustSet(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	report, _, err := Apply(context.Background(), archHost(t), d, state, noServices, nil)
+	report, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -283,7 +283,7 @@ func TestAServiceIsReadBackNotAssumed(t *testing.T) {
 	d := parse(t, `{"declaration":1,"resources":[
 	  {"id":"s","type":"service","unit":"doomed.service","state":"running"}
 	]}`)
-	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, run, nil)
+	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil)
 	if err == nil {
 		t.Fatal("a service that died immediately was reported as running")
 	}
@@ -299,7 +299,7 @@ func TestAnUnknownServiceStateIsRefusedNotGuessed(t *testing.T) {
 	d := parse(t, `{"declaration":1,"resources":[
 	  {"id":"s","type":"service","unit":"odd.service","state":"running"}
 	]}`)
-	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, run, nil)
+	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil)
 	if err == nil || !strings.Contains(err.Error(), "neither running nor stopped") {
 		t.Errorf("an unrecognised service state was not refused: %v", err)
 	}
@@ -323,7 +323,7 @@ func TestADroppedServiceIsStoppedNotDeleted(t *testing.T) {
 	  {"id":"other","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"}
 	]}`)
 
-	if _, _, err := Apply(context.Background(), archHost(t), d, state, run, nil); err != nil {
+	if _, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, run, nil); err != nil {
 		t.Fatal(err)
 	}
 	joined := strings.Join(commands, "; ")
@@ -349,7 +349,7 @@ func TestAUnitThatDoesNotExistIsNotStopped(t *testing.T) {
 	  {"id":"s","type":"service","unit":"never-installed.service","state":"stopped"}
 	]}`)
 
-	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, absent, nil)
+	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, absent, nil)
 	if err == nil {
 		t.Fatal("a unit that does not exist was reported as satisfactorily stopped")
 	}
@@ -370,7 +370,7 @@ func TestAMaskedUnitIsRefused(t *testing.T) {
 	d := parse(t, `{"declaration":1,"resources":[
 	  {"id":"s","type":"service","unit":"masked.service","state":"running"}
 	]}`)
-	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, masked, nil); err == nil {
+	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, masked, nil); err == nil {
 		t.Fatal("a masked unit was accepted")
 	}
 }
@@ -398,7 +398,7 @@ func TestForgettingAUnitThatIsGoneDoesNotStrandTheNode(t *testing.T) {
 	  {"id":"f","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"}
 	]}`)
 
-	report, state, err := Apply(context.Background(), archHost(t), d, known, run, nil)
+	report, state, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil)
 	if err != nil {
 		t.Fatalf("a vanished unit stranded the apply: %v", err)
 	}
@@ -442,7 +442,7 @@ func TestABrokenPackageDatabaseIsNotReadAsNotInstalled(t *testing.T) {
 	  {"id":"rt","type":"package","package":"docker"}
 	]}`)
 
-	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, run, nil)
+	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil)
 	if err == nil {
 		t.Fatal("a broken package database was read as 'not installed'")
 	}
@@ -463,7 +463,7 @@ func TestAnInstalledPackageIsNotReinstalled(t *testing.T) {
 	  {"id":"rt","type":"package","package":"docker"}
 	]}`)
 
-	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, run, nil)
+	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil)
 	if err != nil {
 		t.Fatalf("apply failed: %v", err)
 	}
@@ -493,7 +493,7 @@ func TestAPackageIsNeverUninstalled(t *testing.T) {
 	  {"id":"f","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"}
 	]}`)
 
-	report, state, err := Apply(context.Background(), archHost(t), d, known, run, nil)
+	report, state, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil)
 	if err != nil {
 		t.Fatalf("dropping a package stranded the apply: %v", err)
 	}
@@ -523,7 +523,7 @@ func TestAnActionThatIsAlreadyTrueDoesNotRun(t *testing.T) {
 	  {"id":"db","type":"action","command":["create-db","mesh"],"verify":["has-db","mesh"]}
 	]}`)
 
-	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, run, nil)
+	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil)
 	if err != nil {
 		t.Fatalf("apply failed: %v", err)
 	}
@@ -549,7 +549,7 @@ func TestAnActionThatSucceedsAndDoesNothingFails(t *testing.T) {
 	  {"id":"db","type":"action","command":["create-db","mesh"],"verify":["has-db","mesh"]}
 	]}`)
 
-	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, run, nil)
+	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil)
 	if err == nil {
 		t.Fatal("an action that reported success and did nothing was accepted")
 	}
@@ -576,7 +576,7 @@ func TestAnActionRunsInsideTheContainerItNames(t *testing.T) {
 	  {"id":"db","type":"action","in":"store","command":["createdb","mesh"],"verify":["psql","-lqt"]}
 	]}`)
 
-	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, run, nil); err != nil {
+	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil); err != nil {
 		t.Fatalf("apply failed: %v", err)
 	}
 	if !sawExec {
@@ -603,7 +603,7 @@ func TestAContainerThatExitsImmediatelyFailsTheApply(t *testing.T) {
 	  {"id":"store","type":"container","name":"store","image":"`+pinned+`"}
 	]}`)
 
-	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, run, nil)
+	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil)
 	if err == nil {
 		t.Fatal("a container that exited immediately was reported as applied")
 	}
@@ -645,7 +645,7 @@ func TestAContainerWhoseDeclarationChangedIsReplaced(t *testing.T) {
 		return "", nil
 	}
 
-	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, run, nil)
+	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil)
 	if err != nil {
 		t.Fatalf("apply failed: %v", err)
 	}
@@ -675,7 +675,7 @@ func TestAContainerThatMatchesIsLeftAlone(t *testing.T) {
 		return "", nil
 	}
 
-	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, run, nil)
+	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil)
 	if err != nil {
 		t.Fatalf("apply failed: %v", err)
 	}
@@ -734,7 +734,7 @@ func TestAServiceIsEnabledAtBootWhenAsked(t *testing.T) {
 	  {"id":"rt","type":"service","unit":"docker.service","state":"running","boot":"enabled"}
 	]}`)
 
-	report, _, err := Apply(context.Background(), archHost(t), d, store.State{},
+	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried,
 		systemctlStub(t, "loaded", "inactive", "disabled", &verbs), nil)
 	if err != nil {
 		t.Fatalf("apply failed: %v", err)
@@ -754,7 +754,7 @@ func TestBootIsEnabledBeforeTheUnitIsStarted(t *testing.T) {
 	d := parseTrusted(t, `{"declaration":1,"resources":[
 	  {"id":"rt","type":"service","unit":"docker.service","state":"running","boot":"enabled"}
 	]}`)
-	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{},
+	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried,
 		systemctlStub(t, "loaded", "inactive", "disabled", &verbs), nil); err != nil {
 		t.Fatal(err)
 	}
@@ -769,7 +769,7 @@ func TestAlreadyEnabledAndRunningIsUnchanged(t *testing.T) {
 	  {"id":"rt","type":"service","unit":"docker.service","state":"running","boot":"enabled"}
 	]}`)
 
-	report, _, err := Apply(context.Background(), archHost(t), d, store.State{},
+	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried,
 		systemctlStub(t, "loaded", "active", "enabled", &verbs), nil)
 	if err != nil {
 		t.Fatalf("apply failed: %v", err)
@@ -789,7 +789,7 @@ func TestOmittingBootLeavesItAlone(t *testing.T) {
 	d := parseTrusted(t, `{"declaration":1,"resources":[
 	  {"id":"rt","type":"service","unit":"docker.service","state":"running"}
 	]}`)
-	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{},
+	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried,
 		systemctlStub(t, "loaded", "inactive", "enabled", &verbs), nil); err != nil {
 		t.Fatal(err)
 	}
@@ -809,7 +809,7 @@ func TestAStaticUnitCannotBeEnabled(t *testing.T) {
 	  {"id":"rt","type":"service","unit":"dbus.socket","state":"running","boot":"enabled"}
 	]}`)
 
-	_, _, err := Apply(context.Background(), archHost(t), d, store.State{},
+	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried,
 		systemctlStub(t, "loaded", "active", "static", &verbs), nil)
 	if err == nil {
 		t.Fatal("a static unit was accepted as enable-able")
@@ -824,7 +824,7 @@ func TestAnUnknownBootStateIsRefusedNotGuessed(t *testing.T) {
 	d := parseTrusted(t, `{"declaration":1,"resources":[
 	  {"id":"rt","type":"service","unit":"x.service","state":"running","boot":"enabled"}
 	]}`)
-	_, _, err := Apply(context.Background(), archHost(t), d, store.State{},
+	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried,
 		systemctlStub(t, "loaded", "active", "indirect", &verbs), nil)
 	if err == nil {
 		t.Fatal("an unrecognised boot state was guessed at instead of refused")
@@ -900,7 +900,7 @@ func TestAContainerUsesTheRuntimeTheMachineHas(t *testing.T) {
 
 	// It will fail at read-back — the stub never reports it running — and what matters is
 	// WHICH binary it used getting there.
-	_, _, _ = Apply(context.Background(), archHost(t), d, store.State{}, run, nil)
+	_, _, _ = Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil)
 
 	for _, c := range calledWith {
 		if c != "podman" {
@@ -922,7 +922,7 @@ func TestNoRuntimeIsSaidPlainly(t *testing.T) {
 	  {"id":"store","type":"container","name":"store","image":"`+pinned+`"}
 	]}`)
 
-	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, run, nil)
+	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil)
 	if err == nil {
 		t.Fatal("a machine with no container runtime applied a container")
 	}

internal/link/run.go

@@ -31,13 +31,20 @@ type Membership struct {
 // Applier is what the host does with a declaration that has been proved to come from the mesh.
 type Applier func(ctx context.Context, declaration []byte) Report
 
+// Announce is how the link says what is happening, so a node running unattended leaves an
+// account of it. Nil is allowed and means say nothing.
+type Announce func(string)
+
 // Run holds the link open, applying what arrives and reporting what happened.
 //
 // Outbound only, and nothing listens on this machine. The connection is the node's presence in
 // the mesh: while it is up the node is enrolled, and while it is down the node is disconnected —
 // which is an ordinary situation and not a failure, so this returns rather than panicking and
 // leaves restarting to whatever supervises it.
-func Run(ctx context.Context, m Membership, apply Applier, timeout time.Duration) error {
+func Run(ctx context.Context, m Membership, apply Applier, say Announce, timeout time.Duration) error {
+	if say == nil {
+		say = func(string) {}
+	}
 	config, err := PinnedConfig(m.Fingerprint)
 	if err != nil {
 		return err
@@ -84,6 +91,18 @@ func Run(ctx context.Context, m Membership, apply Applier, timeout time.Duration
 	}
 	closed := conn.NotifyClose(make(chan *amqp.Error, 1))
 
+	// Published mandatory, so the broker hands back anything it cannot route rather than
+	// dropping it. Without this a report goes to an exchange with no matching binding, the
+	// publisher is told nothing, and the mesh believes this node never answered while the node
+	// believes it did — which is what happened when `report` was left unbound on the other side.
+	returned := channel.NotifyReturn(make(chan amqp.Return, 4))
+	go func() {
+		for r := range returned {
+			say(fmt.Sprintf("the broker could not route this node's %s: %s (%d %s)",
+				r.RoutingKey, r.Exchange, r.ReplyCode, r.ReplyText))
+		}
+	}()
+
 	for {
 		select {
 		case <-ctx.Done():
@@ -95,7 +114,15 @@ func Run(ctx context.Context, m Membership, apply Applier, timeout time.Duration
 				return errors.New("the broker stopped delivering")
 			}
 			report := handle(ctx, m, apply, delivery)
-			publishReport(ctx, channel, m, report, timeout)
+			switch {
+			case report.Refused != "":
+				say("refused a declaration: " + report.Refused)
+			case len(report.Failed) > 0:
+				say(fmt.Sprintf("applied %d and failed: %v", len(report.Applied), report.Failed))
+			default:
+				say(fmt.Sprintf("applied %d resource(s)", len(report.Applied)))
+			}
+			publishReport(ctx, channel, m, report, say, timeout)
 			// Acknowledged after the report is published. A node that dies between applying and
 			// reporting leaves the declaration on the broker and applies it again on return,
 			// which is safe because applying is reconciliation — it converges rather than
@@ -127,14 +154,21 @@ func handleBody(ctx context.Context, m Membership, body []byte, apply Applier) R
 }
 
 func publishReport(ctx context.Context, channel *amqp.Channel, m Membership, report Report,
-	timeout time.Duration) {
+	say Announce, timeout time.Duration) {
 	report.Node = m.Node
 	body, err := json.Marshal(report)
 	if err != nil {
+		say("cannot encode this node's own report: " + err.Error())
 		return
 	}
 	publish, cancel := context.WithTimeout(ctx, timeout)
 	defer cancel()
-	_ = channel.PublishWithContext(publish, Exchange, KeyReport, false, false,
-		amqp.Publishing{ContentType: "application/json", Body: body})
+
+	// Said rather than swallowed. A report that fails to publish leaves the mesh believing this
+	// node never answered, while the node believes it did — and the two would go on disagreeing
+	// with nothing anywhere saying so. That shape of fault is the one this project keeps finding.
+	if err := channel.PublishWithContext(publish, Exchange, KeyReport, true, false,
+		amqp.Publishing{ContentType: "application/json", Body: body}); err != nil {
+		say(fmt.Sprintf("applied, and could not tell the mesh: %v", err))
+	}
 }

internal/store/store.go

@@ -33,6 +33,16 @@ const DefaultPath = "/var/lib/mesh-host/state.json"
 type Applied struct {
 	ID   string `json:"id"`
 	Type string `json:"type"`
+	// Origin is who asked for this: the bundle this host carries, or the mesh.
+	//
+	// Recorded because the two must not remove each other. A node raises its own substrate from
+	// the bundle before any mesh exists, then enrols and is sent declarations — and a
+	// declaration naming two resources would otherwise remove the store, the broker and the
+	// control plane, which is 04-ISSUES/010 and happened on the first end-to-end run.
+	//
+	// Empty means carried, for state written before this field existed: everything a host had
+	// applied at that point came from its bundle.
+	Origin string `json:"origin,omitempty"`
 	// Target is what was changed — a path, a unit — so removal knows what to undo without
 	// re-reading a declaration that may no longer exist.
 	Target    string    `json:"target"`
@@ -165,12 +175,39 @@ func (s *State) Forget(id string) {
 // Reverse order because undoing in the order things were made undoes a directory before the
 // file inside it. Reversing is the only ordering the host can derive without deciding
 // anything, which is the line novox/hq ADR 0005 draws.
-func (s State) Orphans(declared map[string]bool) []Applied {
+func (s State) Orphans(declared map[string]bool, origin string) []Applied {
 	var out []Applied
 	for i := len(s.Resources) - 1; i >= 0; i-- {
-		if !declared[s.Resources[i].ID] {
-			out = append(out, s.Resources[i])
+		r := s.Resources[i]
+		// Only this origin's own. A mesh declaration says nothing about what the bundle raised,
+		// and a bundle says nothing about what the mesh assigned — so neither may remove the
+		// other's by omission, which is the only way either could express removal.
+		if originOf(r) != origin {
+			continue
+		}
+		if !declared[r.ID] {
+			out = append(out, r)
 		}
 	}
 	return out
 }
+
+// Origins a resource can have.
+const (
+	// Carried is the bundle this host was built with.
+	OriginCarried = "carried"
+	// Declared is the mesh, over the link.
+	OriginDeclared = "declared"
+)
+
+// originOf reads a record's origin, treating absence as carried.
+//
+// State written before origins existed was all bundle-applied: a host had no other way to be
+// told anything. Guessing wrong in the other direction would have a first upgrade remove the
+// substrate, which is the fault this field exists to prevent.
+func originOf(r Applied) string {
+	if r.Origin == "" {
+		return OriginCarried
+	}
+	return r.Origin
+}

internal/store/store_test.go

@@ -116,7 +116,7 @@ func TestOrphansAreWhatWasAppliedAndIsNoLongerDeclared(t *testing.T) {
 		{ID: "file", Type: "file", Target: "/etc/mesh/a.conf"},
 		{ID: "kept", Type: "file", Target: "/etc/mesh/b.conf"},
 	}}
-	orphans := s.Orphans(map[string]bool{"kept": true})
+	orphans := s.Orphans(map[string]bool{"kept": true}, OriginCarried)
 
 	if len(orphans) != 2 {
 		t.Fatalf("expected two orphans, got %d: %+v", len(orphans), orphans)
@@ -130,7 +130,59 @@ func TestOrphansAreWhatWasAppliedAndIsNoLongerDeclared(t *testing.T) {
 
 func TestNothingIsAnOrphanWhenEverythingIsDeclared(t *testing.T) {
 	s := State{Resources: []Applied{{ID: "a", Type: "file", Target: "/a"}}}
-	if got := s.Orphans(map[string]bool{"a": true}); len(got) != 0 {
+	if got := s.Orphans(map[string]bool{"a": true}, OriginCarried); len(got) != 0 {
 		t.Errorf("a declared resource was treated as an orphan: %+v", got)
 	}
 }
+
+func TestADeclarationDoesNotOrphanWhatTheBundleRaised(t *testing.T) {
+	// 04-ISSUES/010. A first node raises its substrate from the bundle it carries, then enrols
+	// and is sent a declaration naming two resources. Before origins, that removed the store, the
+	// broker and the control plane that had sent it — the mesh deleting itself over the link the
+	// message arrived on, in under a second, on the first end-to-end run.
+	s := State{Resources: []Applied{
+		{ID: "store", Type: "container", Target: "mesh-store", Origin: OriginCarried},
+		{ID: "broker", Type: "container", Target: "mesh-broker", Origin: OriginCarried},
+		{ID: "greeting", Type: "directory", Target: "/var/lib/demo", Origin: OriginDeclared},
+	}}
+
+	// The mesh declares nothing at all. Everything it previously declared is an orphan; nothing
+	// the bundle raised is.
+	orphans := s.Orphans(map[string]bool{}, OriginDeclared)
+	if len(orphans) != 1 || orphans[0].ID != "greeting" {
+		var got []string
+		for _, o := range orphans {
+			got = append(got, o.ID)
+		}
+		t.Fatalf("a declaration would remove %v; it may only remove what the mesh declared", got)
+	}
+}
+
+func TestTheBundleDoesNotOrphanWhatTheMeshDeclared(t *testing.T) {
+	// The same rule the other way. A host reconciling its carried bundle must not remove what the
+	// mesh assigned to this node, or every restart would undo the node's actual work.
+	s := State{Resources: []Applied{
+		{ID: "store", Type: "container", Target: "mesh-store", Origin: OriginCarried},
+		{ID: "workload", Type: "container", Target: "some-app", Origin: OriginDeclared},
+	}}
+
+	orphans := s.Orphans(map[string]bool{"store": true}, OriginCarried)
+	if len(orphans) != 0 {
+		t.Errorf("reconciling the bundle would remove %s, which the mesh declared", orphans[0].ID)
+	}
+}
+
+func TestStateWrittenBeforeOriginsExistedIsTreatedAsCarried(t *testing.T) {
+	// Every resource a host had applied before this field existed came from its bundle, because
+	// there was no other way to tell it anything. Guessing the other way would have the first
+	// declaration remove the substrate — which is the fault this exists to prevent, arriving
+	// through the upgrade that fixes it.
+	s := State{Resources: []Applied{{ID: "store", Type: "container", Target: "mesh-store"}}}
+
+	if got := s.Orphans(map[string]bool{}, OriginDeclared); len(got) != 0 {
+		t.Errorf("a declaration would remove %s, recorded before origins existed", got[0].ID)
+	}
+	if got := s.Orphans(map[string]bool{}, OriginCarried); len(got) != 1 {
+		t.Error("the bundle cannot remove its own resource, so nothing could ever remove it")
+	}
+}