Say that the guard names the runtime's bridges where the filter names their addresses (hq ADR 0103)

cmd/mesh-controller/acts.go

@@ -39,6 +39,13 @@ import (
 // It costs a resolution per machine. Assignment is a person typing a command, and being told which
 // machines this just blocked is worth more than the milliseconds.
 func assign(ctx context.Context, open *stores, node, module string) (string, error) {
+	// Held while it is recorded, so it cannot land between a converge's preview and its flip and
+	// be taken without ever having been previewed (novox/hq ADR 0100).
+	ctx, release, err := holdNodes(ctx, open, []string{node})
+	if err != nil {
+		return "", err
+	}
+	defer release()
 	if err := open.inventory.Assign(ctx, node, module); err != nil {
 		return "", err
 	}
@@ -71,6 +78,11 @@ func assign(ctx context.Context, open *stores, node, module string) (string, err
 // module off one machine is the ordinary way to stop providing something to another, and nothing
 // about the command's own output would ever have said so.
 func unassign(ctx context.Context, open *stores, node, module string) (string, error) {
+	ctx, release, err := holdNodes(ctx, open, []string{node})
+	if err != nil {
+		return "", err
+	}
+	defer release()
 	if err := open.inventory.Unassign(ctx, node, module); err != nil {
 		return "", err
 	}

cmd/mesh-controller/adopting_test.go

@@ -473,3 +473,62 @@ func TestThePreviewNamesEveryHeldKind(t *testing.T) {
 		t.Fatal("the digest does not change with what is held")
 	}
 }
+
+// novox/hq ADR 0100: the flip acts on what the node said is reachable, so an account naming nothing
+// is refused. Every machine that is up answers on ssh; nothing reported means the host's collectors
+// did not, and flipping would close ports the preview never named.
+func TestTheFlipIsRefusedOnAnAccountNamingNothingReachable(t *testing.T) {
+	open, sent := anAdoptedAnchor(t)
+	ctx := t.Context()
+	if _, err := take(ctx, open, "anchor", "hello-web"); err != nil {
+		t.Fatal(err)
+	}
+	// Only a loopback listener: nothing off the machine, which is the same silence.
+	reportsReaching(t, open, []link.Reach{
+		{Protocol: "tcp", Address: "127.0.0.1", Port: 15672, By: "mesh-broker"},
+	}, heldFile)
+	preview, err := converge(ctx, open, "anchor", false, "", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(preview, "this account looks partial") {
+		t.Errorf("the preview does not mark a partial account:\n%s", preview)
+	}
+	_, err = converge(ctx, open, "anchor", true, digestIn(t, preview), "")
+	if err == nil || !strings.Contains(err.Error(), "says nothing is reachable on it") {
+		t.Fatalf("the flip was not refused on an account naming nothing: %v", err)
+	}
+	if n, _ := open.inventory.NodeByName(ctx, "anchor"); !n.Adopted || len(*sent) != 0 {
+		t.Fatal("a refused flip changed something")
+	}
+}
+
+// An assignment cannot land between a preview and the flip that takes every module: assigning
+// holds the node, so it waits for whatever is converging it.
+func TestAssigningWaitsForWhateverIsConvergingTheNode(t *testing.T) {
+	open, _ := anAdoptedAnchor(t)
+	ctx := t.Context()
+	if _, err := take(ctx, open, "anchor", "hello-web"); err != nil {
+		t.Fatal(err)
+	}
+	reportsHolding(t, open, heldFile)
+	preview, err := converge(ctx, open, "anchor", false, "", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	saved, savedPoll := inventory.HoldWaitFor, inventory.HoldPoll
+	inventory.HoldWaitFor, inventory.HoldPoll = time.Second, 50*time.Millisecond
+	defer func() { inventory.HoldWaitFor, inventory.HoldPoll = saved, savedPoll }()
+
+	var whileFlipping error
+	sendNodes = func(context.Context, *stores, []string) error {
+		_, whileFlipping = assign(ctx, open, "anchor", "notes")
+		return nil
+	}
+	if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil {
+		t.Fatal(err)
+	}
+	if !errors.Is(whileFlipping, inventory.ErrNodeBusy) {
+		t.Fatalf("an assignment landed while the node was being converged: %v", whileFlipping)
+	}
+}

cmd/mesh-controller/adoption.go

@@ -189,6 +189,10 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s
 			"converge once it has applied", node, node)
 	}
 
+	assignedWhenPreviewed, err := inv.Assigned(ctx, node)
+	if err != nil {
+		return "", err
+	}
 	plan, settings, err := planFor(ctx, open, node)
 	if err != nil {
 		return "", err
@@ -248,6 +252,16 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s
 		return preview + fmt.Sprintf("\n\nNothing has changed. Run `converge %s --yes %s` to do "+
 			"it.", node, saw), nil
 	}
+	// An account naming nothing reachable is not an account of a machine: every machine answers
+	// on ssh, and the host's collectors failing — `ss` refusing, or the container runtime not
+	// answering, which drops every published port at once — leaves exactly this. Flipping on it
+	// would close ports the preview never named.
+	if yes && countReachable(reported) == 0 {
+		return preview, fmt.Errorf("%s says nothing is reachable on it, which no machine that is "+
+			"up ever is: its account looks partial — whatever reads what is listening, or what "+
+			"the container runtime publishes, did not answer. Fix that on the machine and run "+
+			"`push %s --wait 2m`, then preview again", node, node)
+	}
 	if age := time.Since(reported.At); age > reportFreshFor {
 		return preview, fmt.Errorf("%s last said what is reachable on it %s ago, and the flip acts "+
 			"only on an account newer than %s: wait for its next report, or run `push %s --wait 2m`, "+
@@ -269,6 +283,13 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s
 	if err != nil {
 		return "", err
 	}
+	// And nothing assigned since the preview was composed: the flip takes every module the node
+	// runs, and one assigned in between would be taken without ever having been previewed.
+	if !slices.Equal(assigned, assignedWhenPreviewed) {
+		return preview, fmt.Errorf("what %s runs changed while this was converging (it is now %s): "+
+			"the flip takes every module on the node, so read the preview again", node,
+			strings.Join(assigned, ", "))
+	}
 	if !slices.Contains(assigned, filter) {
 		if err := inv.Assign(ctx, node, filter); err != nil {
 			return "", err
@@ -317,8 +338,12 @@ func previewOf(node string, reported inventory.Adoption, derived derivedFilter,
 		fmt.Fprintf(&b, "    %-44s %s\n", what, fate)
 		said = append(said, fmt.Sprintf("reach %s %s %s", r.Address, what, fate))
 	}
-	if len(reported.Reachable) == 0 {
-		b.WriteString("    nothing reported\n")
+	if countReachable(reported) == 0 {
+		// Said as what it is: no machine that is up is reachable on nothing, so this is an
+		// account that did not come back, not a machine with nothing on it.
+		b.WriteString("    nothing reported — this account looks partial, and the flip is " +
+			"refused on it\n")
+		said = append(said, "reach nothing reported")
 	}
 	// What the machine routes for others is not a listener and not a published port, so nothing
 	// above can show it; the derived filter's forward chain drops it all the same.
@@ -431,6 +456,19 @@ func (d derivedFilter) fate(r inventory.Reach) string {
 	return "WILL CLOSE — no module assigned here declares it"
 }
 
+// countReachable is how much of a node's account of itself names something off the machine.
+// Loopback is left out for the same reason the preview leaves it out: nothing outside reaches it,
+// so a report of loopback alone says nothing about what the filter would close.
+func countReachable(reported inventory.Adoption) int {
+	n := 0
+	for _, r := range reported.Reachable {
+		if !loopback(r.Address) {
+			n++
+		}
+	}
+	return n
+}
+
 // heldLine is one thing a node holds as found, as take and the converge preview both say it.
 func heldLine(h inventory.Held) string {
 	return fmt.Sprintf("%s %s (%s)", h.Kind, h.Target, h.ID)

cmd/mesh-controller/sendable_test.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/novox/mesh-controller/internal/catalogue"
 	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/overlay"
 )
 
 // novox/hq ADR 0100: every declaration an adopted node is sent says it is adopted and which modules
@@ -41,13 +42,24 @@ func composed(t *testing.T, open *stores, node string) sendable {
 	return declared
 }
 
+// convergedBefore is the envelope a converged node was sent before adoption existed, captured by
+// running this same composition at the commit this branch left main (0a39b7d). The mesh here is
+// aMesh's laptop with the private network taken off it, so nothing in the declaration is random:
+// what changes this string is a change to what a converged machine is sent, which is the thing an
+// older host would refuse.
+const convergedBefore = `{"declaration":1,"resources":[{"content":"hello","id":"hello-web.page","path":"/var/lib/hello-web/index.html","type":"file"},{"hosts":["anchor.internal:10.77.0.1"],"id":"hello-web.server","image":"registry.example/hello@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","name":"hello-web","type":"container"},{"id":"hello-web.served","path":"/var/lib/hello-web","type":"directory"}]}`
+
 func TestAConvergedDeclarationIsByteForByteWhatItWas(t *testing.T) {
 	open := aMesh(t)
+	ctx := t.Context()
 	register(t, open, helloWeb())
-	if _, err := assign(t.Context(), open, "anchor", "hello-web"); err != nil {
+	if _, err := unassign(ctx, open, "laptop", overlay.Name); err != nil {
 		t.Fatal(err)
 	}
-	declared := composed(t, open, "anchor")
+	if _, err := assign(ctx, open, "laptop", "hello-web"); err != nil {
+		t.Fatal(err)
+	}
+	declared := composed(t, open, "laptop")
 	if declared.Adoption != nil {
 		t.Fatal("a converged node was given an adoption envelope")
 	}
@@ -55,13 +67,9 @@ func TestAConvergedDeclarationIsByteForByteWhatItWas(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	// The envelope exactly as every send site marshalled it before adoption existed.
-	before, err := json.Marshal(map[string]any{"declaration": 1, "resources": declared.Resources})
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !bytes.Equal(body, before) {
-		t.Fatalf("a converged declaration changed:\n%s\n%s", body, before)
+	if string(body) != convergedBefore {
+		t.Fatalf("a converged declaration changed; an older host parses this strictly:\n%s\n%s",
+			body, convergedBefore)
 	}
 	if bytes.Contains(body, []byte(`"adoption"`)) {
 		t.Fatal("a converged declaration names adoption; an older host would refuse it")
@@ -214,17 +222,19 @@ func TestConsumersAreToldTheGivenPort(t *testing.T) {
 		}
 	}
 
+	// A port for the whole mesh is refused where it is set, not stored to refuse every node's
+	// declaration afterwards.
 	if err := open.inventory.SetSettings(ctx, "", "store",
-		map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5434}}); err != nil {
-		t.Fatal(err)
+		map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5434}}); err == nil ||
+		!strings.Contains(err.Error(), "per node") {
+		t.Fatalf("a port given for the whole mesh was not refused: %v", err)
 	}
 	plan, settings, err := planFor(ctx, open, "anchor")
 	if err != nil {
 		t.Fatal(err)
 	}
-	if _, err := declarationFor(ctx, open, "anchor", plan, settings); err == nil ||
-		!strings.Contains(err.Error(), "per node") {
-		t.Fatalf("a port given for the whole mesh was not refused: %v", err)
+	if _, err := declarationFor(ctx, open, "anchor", plan, settings); err != nil {
+		t.Fatalf("the refused mesh-wide layer was stored anyway: %v", err)
 	}
 }
 

internal/catalogue/adoption.go

@@ -159,6 +159,16 @@ func Published(resources []map[string]any) map[string]map[int]int {
 // prerouting, ahead of the runtime's destination translation, so it matches the port the packet
 // was sent to; in the inet family, so both address families.
 //
+// **The machine's own interfaces are named, where the derived filter names address ranges.** The
+// filter accepts the container runtime's networks by CIDR; this excludes its bridges by name — lo,
+// docker0, the br-* a compose network gets, and the mesh's own mesh0. A runtime whose bridge is
+// named anything else (a podman or libvirt bridge, or a docker network created with a fixed name)
+// would have its containers' traffic to a guarded port refused, which reads as the port being
+// down. Names rather than addresses is deliberate: a source address can be claimed by whoever
+// sends the packet, and this table exists to refuse what the found firewall never sees. Widening
+// it means adding names here and in the installer's copy together, which the golden test holds to
+// one text.
+//
 // The same text the installer raises on an adopted genesis; a test holds both to it.
 func AsGuard(ports []int) string {
 	sorted := append([]int{}, ports...)

internal/catalogue/adoption_test.go

@@ -397,3 +397,35 @@ func TestPublishedLeavesOutLoopbackInBothFamilies(t *testing.T) {
 		t.Fatalf("published is %v, want %v", got, want)
 	}
 }
+
+// novox/hq ADR 0038: only a published port is the mesh's to move. A module that binds the machine
+// itself listens where its software was told to, so giving it a machine port is refused.
+func TestAGivenPortIsRefusedForAPortNoContainerPublishes(t *testing.T) {
+	onTheMachine := Manifest{Module: "daemon",
+		Listens: []Listening{{Port: 9000, From: FromMesh}}, Guards: []int{9000}}
+	layers := []Layer{{From: "node anchor",
+		Values: map[string]any{PortsSetting: map[string]any{"9000": float64(9100)}}}}
+	_, err := GivenPorts(onTheMachine, layers)
+	if err == nil || !strings.Contains(err.Error(), "does not publish") {
+		t.Fatalf("a port no container publishes was given: %v", err)
+	}
+}
+
+// novox/hq ADR 0103: a guarded port this node is told to open to everyone is opened, not guarded.
+// An opening from everywhere beside a guard dropping it is one statement refusing the other.
+func TestAGuardedPortOpenedToEveryoneIsNotGuarded(t *testing.T) {
+	with := anchorRendering(true)
+	with.Settings["postgres"] = []Layer{{From: "node anchor",
+		Values: map[string]any{ExposeSetting: map[string]any{"5432": FromEverywhere}}}}
+	composed, err := anAdoptedAnchor().Compose(with)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := byID(composed.Resources)
+	if o := got["adoption.opening-tcp-5432-forwarded"]; o == nil || o["from"] != OpeningFromEverywhere {
+		t.Fatalf("the store's port is not opened to everyone: %v", o)
+	}
+	if guard, _ := got[GuardID()]["content"].(string); guard != AsGuard([]int{5672, 15672}) {
+		t.Fatalf("a port opened to everyone is still guarded:\n%s", guard)
+	}
+}

internal/catalogue/declaration.go

@@ -610,13 +610,21 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri
 func (r Resolution) guarded(out []map[string]any, owner map[string]string, rules []Rule,
 	with Rendering) []int {
 	meshOnly := map[int]bool{}
+	fromEverywhere := map[int]bool{}
 	for _, rule := range rules {
-		if rule.Protocol == "tcp" && rule.From == FromMesh {
+		if rule.Protocol != "tcp" {
+			continue
+		}
+		switch rule.From {
+		case FromMesh:
 			meshOnly[rule.Port] = true
+		case FromEverywhere:
+			fromEverywhere[rule.Port] = true
 		}
 	}
 	for _, port := range with.Foundation {
 		delete(meshOnly, port)
+		fromEverywhere[port] = true
 	}
 	seen := map[int]bool{}
 	var ports []int
@@ -655,7 +663,12 @@ func (r Resolution) guarded(out []map[string]any, owner map[string]string, rules
 					}
 				}
 			}
-			guard(at)
+			// Not what this node is told to open to everyone: a per-node exposure setting that
+			// widens a guarded port is the operator saying so, and declaring an opening for it
+			// and a guard dropping it would be one statement refusing the other.
+			if !fromEverywhere[at] {
+				guard(at)
+			}
 		}
 	}
 	sort.Ints(ports)

internal/catalogue/filtering.go

@@ -475,17 +475,16 @@ const MeshWideLayer = "the mesh"
 // GivenPorts reads a module's given machine ports from its settings: software port → machine port.
 //
 // Refused from a mesh-wide layer — a port is a fact about one machine, and one number for every
-// machine is the collision this exists to avoid — and for a port the module neither listens on,
-// publishes from a container, nor guards: a given port that reaches nothing is a setting somebody
-// believes changed something.
+// machine is the collision this exists to avoid — and for a port the module's containers do not
+// publish.
+//
+// **Only a published port is the mesh's to move** (novox/hq ADR 0038). A container's mapping is
+// what translates; a module binding the machine's network directly binds the number its software
+// was configured with, and moving that number would put it in the filter, in the openings and in
+// what consumers are told while the software still listens on the old one — a port that reads as
+// moved and is not.
 func GivenPorts(m Manifest, layers []Layer) (map[int]int, error) {
 	known := map[int]bool{}
-	for _, l := range m.Listens {
-		known[l.Port] = true
-	}
-	for _, p := range m.Guards {
-		known[p] = true
-	}
 	for _, p := range containerPorts(m) {
 		known[p] = true
 	}
@@ -510,8 +509,9 @@ func GivenPorts(m Manifest, layers []Layer) (map[int]int, error) {
 				return nil, fmt.Errorf("%s gives %q a port, which is not a port", m.Module, portText)
 			}
 			if !known[port] {
-				return nil, fmt.Errorf("%s gives port %d a machine port, and it neither listens "+
-					"on, publishes nor guards %d — the setting reaches nothing", m.Module, port, port)
+				return nil, fmt.Errorf("%s gives port %d a machine port, and no container of its "+
+					"publishes %d — the mesh cannot move a port the module does not publish; the "+
+					"software would go on listening where it was told to", m.Module, port, port)
 			}
 			at, ok := asPort(value)
 			if !ok || at < 1 || at > 65535 {

internal/catalogue/foundation_manifests_test.go

@@ -70,8 +70,15 @@ func TestTheFilterModuleNeverFlushesTheRuleset(t *testing.T) {
 			"[Service]\nExecStop=\nExecStop=nft delete table inet mesh\n") {
 		t.Fatalf("the stock unit's stop is not replaced with deleting the mesh's table: %v", stock)
 	}
-	if !reflect.DeepEqual(load["restart-on"], []any{"filtering", "unit", "stock-unit-stop"}) {
-		t.Fatalf("the filter is not reloaded when its rules, its unit or the stock unit's drop-in "+
-			"change: %v", load["restart-on"])
+	// A changed rule set is RELOADED — ExecReload replaces the table in one `nft -f`, so the node
+	// is never unfiltered — and only the units themselves restart it, which is the one change a
+	// reload cannot carry.
+	if !reflect.DeepEqual(load["reload-on"], []any{"filtering"}) {
+		t.Fatalf("the filter is restarted rather than reloaded when its rules change, leaving the "+
+			"node unfiltered in between: %v", load)
+	}
+	if !reflect.DeepEqual(load["restart-on"], []any{"unit", "stock-unit-stop"}) {
+		t.Fatalf("the filter is not restarted when its unit or the stock unit's drop-in changes: %v",
+			load["restart-on"])
 	}
 }

internal/inventory/adoption.go

@@ -118,7 +118,7 @@ func (i *Inventory) Converge(ctx context.Context, nodeName string) ([]string, er
 	}
 	if _, err := tx.Exec(ctx,
 		`update node set adopted = false, adopted_since = null, converged_at = now(),
-		        held = null, reachable = null
+		        held = null, reachable = null, firewall = null, adoption_reported = null
 		  where id = $1`,
 		node.ID); err != nil {
 		return nil, err

internal/inventory/adoption_test.go

@@ -132,3 +132,30 @@ func TestATakenModuleOutlivesItsAssignmentAndReturningToAdopted(t *testing.T) {
 		t.Fatalf("returning to adopted lost what was taken: %v", taken)
 	}
 }
+
+// Converging clears the whole of a node's account of itself: what it held, what was reachable, the
+// firewall it found and when it said so. Keeping any of it would have `node show` report an
+// adopted machine's account of a converged one.
+func TestConvergingClearsTheAccountTheNodeGave(t *testing.T) {
+	inv := fresh(t)
+	ctx := t.Context()
+	made, err := inv.AddNodeAs(ctx, "anchor", true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RecordAdoption(ctx, made.ID,
+		[]Held{{ID: "notes.conf", Module: "notes", Kind: "file", Target: "/etc/notes.conf"}},
+		"ufw", []Reach{{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"}}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.Converge(ctx, "anchor"); err != nil {
+		t.Fatal(err)
+	}
+	said, err := inv.AdoptionOf(ctx, "anchor")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(said.Held) != 0 || len(said.Reachable) != 0 || said.Firewall != "" || !said.At.IsZero() {
+		t.Fatalf("converging kept the adopted machine's account: %+v", said)
+	}
+}

internal/inventory/catalogue.go

@@ -585,6 +585,17 @@ func (i *Inventory) SetSettings(ctx context.Context, nodeName, module string, va
 		return err
 	}
 	if nodeName == "" {
+		// A port is a fact about one machine (novox/hq ADR 0100). Refused here, in composition's
+		// words: stored, it refuses every node running the module at composition, and the mesh
+		// cannot be pushed at all until somebody finds the layer that did it.
+		given, err := givenIn(module, raw)
+		if err != nil {
+			return err
+		}
+		if len(given) > 0 {
+			return fmt.Errorf("%s: %s is given per node — a port is a fact about one machine; "+
+				"set it with --node", module, catalogue.PortsSetting)
+		}
 		_, err = i.store.Pool().Exec(ctx,
 			`insert into settings (node, module, values) values (null, $1, $2)
 			 on conflict (module) where node is null

internal/inventory/ports_test.go

@@ -2,6 +2,7 @@ package inventory
 
 import (
 	"errors"
+	"strings"
 	"testing"
 
 	"github.com/novox/mesh-controller/internal/catalogue"
@@ -318,3 +319,28 @@ func TestAGivenPortHasOneHolderAndReplacesTheAssignment(t *testing.T) {
 		t.Fatalf("the released port %d was not free again (got %d)", assigned.Machine, other.Machine)
 	}
 }
+
+// novox/hq ADR 0100: a port is a fact about one machine, so a layer for the whole mesh cannot give
+// one. Refused where it is set — stored, it refuses every node running the module at composition,
+// and the mesh cannot be pushed until somebody finds the layer that did it.
+func TestAPortGivenForTheWholeMeshIsRefusedWhereItIsSet(t *testing.T) {
+	inv, node := aNodeWithModules(t, "postgres")
+	ctx := t.Context()
+	err := inv.SetSettings(ctx, "", "postgres",
+		map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5433}})
+	if err == nil || !strings.Contains(err.Error(), "per node") {
+		t.Fatalf("a port given for the whole mesh was accepted: %v", err)
+	}
+	layers, err := inv.SettingsFor(ctx, node, "postgres")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(layers) != 0 {
+		t.Fatalf("the refused layer was stored: %v", layers)
+	}
+	// The same values for one machine are the ordinary setting.
+	if err := inv.SetSettings(ctx, node, "postgres",
+		map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5433}}); err != nil {
+		t.Fatal(err)
+	}
+}