The bundle and the mesh stop removing each other

04-ISSUES/010. The store now records where each resource came from -- carried,
or declared -- and each origin removes only its own. A declaration removes what
the mesh previously declared and never what the bundle raised.

State written before the field existed reads as carried, because everything a
host had applied by then came from its bundle: there was no other way to tell
it anything. Guessing the other way would have the first upgrade remove the
substrate, which is this fault arriving through the change that fixes it.

Verified on the scenario that caused it, and on the property that had to
survive it: a later declaration dropping a resource still removes that
resource, so removal by omission still means what it meant.

Also stops swallowing a publish failure. A node that applied a declaration and
could not tell the mesh looked exactly like one that had -- the mesh believing
it never answered, the node believing it did, and nothing anywhere saying so.
Reports are published mandatory now, so anything the broker cannot route comes
back and is said out loud rather than dropped in silence.

internal/store/store.go

@@ -33,6 +33,16 @@ const DefaultPath = "/var/lib/mesh-host/state.json"
 type Applied struct {
 	ID   string `json:"id"`
 	Type string `json:"type"`
+	// Origin is who asked for this: the bundle this host carries, or the mesh.
+	//
+	// Recorded because the two must not remove each other. A node raises its own substrate from
+	// the bundle before any mesh exists, then enrols and is sent declarations — and a
+	// declaration naming two resources would otherwise remove the store, the broker and the
+	// control plane, which is 04-ISSUES/010 and happened on the first end-to-end run.
+	//
+	// Empty means carried, for state written before this field existed: everything a host had
+	// applied at that point came from its bundle.
+	Origin string `json:"origin,omitempty"`
 	// Target is what was changed — a path, a unit — so removal knows what to undo without
 	// re-reading a declaration that may no longer exist.
 	Target    string    `json:"target"`
@@ -165,12 +175,39 @@ func (s *State) Forget(id string) {
 // Reverse order because undoing in the order things were made undoes a directory before the
 // file inside it. Reversing is the only ordering the host can derive without deciding
 // anything, which is the line novox/hq ADR 0005 draws.
-func (s State) Orphans(declared map[string]bool) []Applied {
+func (s State) Orphans(declared map[string]bool, origin string) []Applied {
 	var out []Applied
 	for i := len(s.Resources) - 1; i >= 0; i-- {
-		if !declared[s.Resources[i].ID] {
-			out = append(out, s.Resources[i])
+		r := s.Resources[i]
+		// Only this origin's own. A mesh declaration says nothing about what the bundle raised,
+		// and a bundle says nothing about what the mesh assigned — so neither may remove the
+		// other's by omission, which is the only way either could express removal.
+		if originOf(r) != origin {
+			continue
+		}
+		if !declared[r.ID] {
+			out = append(out, r)
 		}
 	}
 	return out
 }
+
+// Origins a resource can have.
+const (
+	// Carried is the bundle this host was built with.
+	OriginCarried = "carried"
+	// Declared is the mesh, over the link.
+	OriginDeclared = "declared"
+)
+
+// originOf reads a record's origin, treating absence as carried.
+//
+// State written before origins existed was all bundle-applied: a host had no other way to be
+// told anything. Guessing wrong in the other direction would have a first upgrade remove the
+// substrate, which is the fault this field exists to prevent.
+func originOf(r Applied) string {
+	if r.Origin == "" {
+		return OriginCarried
+	}
+	return r.Origin
+}

internal/store/store_test.go

@@ -116,7 +116,7 @@ func TestOrphansAreWhatWasAppliedAndIsNoLongerDeclared(t *testing.T) {
 		{ID: "file", Type: "file", Target: "/etc/mesh/a.conf"},
 		{ID: "kept", Type: "file", Target: "/etc/mesh/b.conf"},
 	}}
-	orphans := s.Orphans(map[string]bool{"kept": true})
+	orphans := s.Orphans(map[string]bool{"kept": true}, OriginCarried)
 
 	if len(orphans) != 2 {
 		t.Fatalf("expected two orphans, got %d: %+v", len(orphans), orphans)
@@ -130,7 +130,59 @@ func TestOrphansAreWhatWasAppliedAndIsNoLongerDeclared(t *testing.T) {
 
 func TestNothingIsAnOrphanWhenEverythingIsDeclared(t *testing.T) {
 	s := State{Resources: []Applied{{ID: "a", Type: "file", Target: "/a"}}}
-	if got := s.Orphans(map[string]bool{"a": true}); len(got) != 0 {
+	if got := s.Orphans(map[string]bool{"a": true}, OriginCarried); len(got) != 0 {
 		t.Errorf("a declared resource was treated as an orphan: %+v", got)
 	}
 }
+
+func TestADeclarationDoesNotOrphanWhatTheBundleRaised(t *testing.T) {
+	// 04-ISSUES/010. A first node raises its substrate from the bundle it carries, then enrols
+	// and is sent a declaration naming two resources. Before origins, that removed the store, the
+	// broker and the control plane that had sent it — the mesh deleting itself over the link the
+	// message arrived on, in under a second, on the first end-to-end run.
+	s := State{Resources: []Applied{
+		{ID: "store", Type: "container", Target: "mesh-store", Origin: OriginCarried},
+		{ID: "broker", Type: "container", Target: "mesh-broker", Origin: OriginCarried},
+		{ID: "greeting", Type: "directory", Target: "/var/lib/demo", Origin: OriginDeclared},
+	}}
+
+	// The mesh declares nothing at all. Everything it previously declared is an orphan; nothing
+	// the bundle raised is.
+	orphans := s.Orphans(map[string]bool{}, OriginDeclared)
+	if len(orphans) != 1 || orphans[0].ID != "greeting" {
+		var got []string
+		for _, o := range orphans {
+			got = append(got, o.ID)
+		}
+		t.Fatalf("a declaration would remove %v; it may only remove what the mesh declared", got)
+	}
+}
+
+func TestTheBundleDoesNotOrphanWhatTheMeshDeclared(t *testing.T) {
+	// The same rule the other way. A host reconciling its carried bundle must not remove what the
+	// mesh assigned to this node, or every restart would undo the node's actual work.
+	s := State{Resources: []Applied{
+		{ID: "store", Type: "container", Target: "mesh-store", Origin: OriginCarried},
+		{ID: "workload", Type: "container", Target: "some-app", Origin: OriginDeclared},
+	}}
+
+	orphans := s.Orphans(map[string]bool{"store": true}, OriginCarried)
+	if len(orphans) != 0 {
+		t.Errorf("reconciling the bundle would remove %s, which the mesh declared", orphans[0].ID)
+	}
+}
+
+func TestStateWrittenBeforeOriginsExistedIsTreatedAsCarried(t *testing.T) {
+	// Every resource a host had applied before this field existed came from its bundle, because
+	// there was no other way to tell it anything. Guessing the other way would have the first
+	// declaration remove the substrate — which is the fault this exists to prevent, arriving
+	// through the upgrade that fixes it.
+	s := State{Resources: []Applied{{ID: "store", Type: "container", Target: "mesh-store"}}}
+
+	if got := s.Orphans(map[string]bool{}, OriginDeclared); len(got) != 0 {
+		t.Errorf("a declaration would remove %s, recorded before origins existed", got[0].ID)
+	}
+	if got := s.Orphans(map[string]bool{}, OriginCarried); len(got) != 1 {
+		t.Error("the bundle cannot remove its own resource, so nothing could ever remove it")
+	}
+}