cmd/mesh-controller/push.go

@@ -9,6 +9,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"sort"
 	"strings"
 	"time"
 
@@ -321,6 +322,50 @@ func pushCommand(ctx context.Context, args []string) error {
 	}
 	fmt.Printf("\n%d node(s) told\n", len(sending))
 
+	// **A named push leaves the mesh consistent, not just the machine it named** (novox/hq
+	// issue 057, ADR 0083). Assigning a cross-node consumer mints a provision, and the PROVIDER's
+	// grant list is a pure read of secrets already issued — so after the named node is current,
+	// other machines can be behind *as a consequence*: their declaration now differs from what
+	// they were last sent. Those are flushed too, by name, in the push's own output.
+	//
+	// Compared against what each machine was last SENT, not against a before/after of this push:
+	// the mint usually happened at `assign` or `module issue`, before this command ran, so the
+	// only durable signal is "what it should be" versus "what it last received". A machine behind
+	// for an unrelated reason is caught here too, which is not a cost — a named push that knew a
+	// machine was behind and left it so would be the very silence this removes. Bounded: a
+	// flushed send may itself mint, so this converges over a few rounds.
+	if len(args) == 1 {
+		flushed := map[string]bool{args[0]: true}
+		for round := 0; round < 4; round++ {
+			would, err := wouldSend(ctx, open, nodes)
+			if err != nil {
+				return err
+			}
+			behind, err := inv.Waiting(ctx, would)
+			if err != nil {
+				return err
+			}
+			var also []string
+			for _, m := range behind {
+				if !flushed[m.Node] {
+					also = append(also, m.Node)
+				}
+			}
+			if len(also) == 0 {
+				break
+			}
+			sort.Strings(also)
+			fmt.Printf("\nthis push left %s behind — a provision granted from there, or a "+
+				"declaration since changed; sending it too\n", strings.Join(also, ", "))
+			if err := sendTo(ctx, open, also); err != nil {
+				return err
+			}
+			for _, name := range also {
+				flushed[name] = true
+			}
+		}
+	}
+
 	// A named node is a request to make THAT node current now, so it waits for the node to say it
 	// applied exactly this. A whole-mesh or --behind push does not wait: it is a sweep, and blocking
 	// on the slowest machine would hold back the report on all the others.

internal/catalogue/filtering.go

@@ -409,6 +409,21 @@ func AsNftables(rules []Rule, mesh []string, outward bool, foundation []int) str
 			b.WriteString(fmt.Sprintf("\t\tct original proto-dst %d accept\n", rule.Port))
 		}
 	}
+
+	// The foundation ports, forwarded — for the same reason they are in the input chain, and the
+	// same reason the module rules above are here too: the broker is a published container port,
+	// so a cross-node dial to it is redirected and *forwarded*, never reaching the input chain
+	// (novox/hq issue 047's lesson, applied to the foundation this time). Without this a joined
+	// node reaches the broker only while its first connection's conntrack entry survives — and a
+	// broker restart, which adopting the foundation's own broker causes, drops the entry and the
+	// node can never receive another declaration (novox/hq issue 063). From anywhere, matching
+	// the input rule: a node enrolling has no overlay address yet.
+	if len(foundation) > 0 {
+		b.WriteString("\n\t\t# the mesh's own — never derived, never closed\n")
+		for _, port := range foundation {
+			b.WriteString(fmt.Sprintf("\t\tct original proto-dst %d accept\n", port))
+		}
+	}
 	b.WriteString("\t}\n")
 	b.WriteString("}\n")
 	return b.String()

internal/catalogue/filtering_foundation_test.go

@@ -33,6 +33,16 @@ func TestTheBrokersPortIsOpenedThoughNoModuleDeclaresIt(t *testing.T) {
 				"has not yet enrolled is not on:\n%s", line)
 		}
 	}
+
+	// And forwarded, not only accepted on input: the broker is a published container port, so a
+	// cross-node dial is redirected and forwarded and never reaches the input chain. Without this
+	// a joined node reaches the broker only until its first connection's conntrack entry drops —
+	// which a broker restart (adopting the foundation's broker) causes — and then never receives
+	// another declaration (novox/hq issue 063).
+	if !strings.Contains(out, "ct original proto-dst 5671 accept") {
+		t.Fatalf("the broker's port is not forwarded, so a DNAT'd broker is unreachable "+
+			"cross-node after it restarts:\n%s", out)
+	}
 }
 
 // And a mesh that was never told about a broker still gets a ruleset, rather than an empty one or