diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 656d683..2713cd1 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -9,6 +9,7 @@ import ( "flag" "fmt" "os" + "sort" "strings" "time" @@ -321,6 +322,50 @@ func pushCommand(ctx context.Context, args []string) error { } fmt.Printf("\n%d node(s) told\n", len(sending)) + // **A named push leaves the mesh consistent, not just the machine it named** (novox/hq + // issue 057, ADR 0083). Assigning a cross-node consumer mints a provision, and the PROVIDER's + // grant list is a pure read of secrets already issued — so after the named node is current, + // other machines can be behind *as a consequence*: their declaration now differs from what + // they were last sent. Those are flushed too, by name, in the push's own output. + // + // Compared against what each machine was last SENT, not against a before/after of this push: + // the mint usually happened at `assign` or `module issue`, before this command ran, so the + // only durable signal is "what it should be" versus "what it last received". A machine behind + // for an unrelated reason is caught here too, which is not a cost — a named push that knew a + // machine was behind and left it so would be the very silence this removes. Bounded: a + // flushed send may itself mint, so this converges over a few rounds. + if len(args) == 1 { + flushed := map[string]bool{args[0]: true} + for round := 0; round < 4; round++ { + would, err := wouldSend(ctx, open, nodes) + if err != nil { + return err + } + behind, err := inv.Waiting(ctx, would) + if err != nil { + return err + } + var also []string + for _, m := range behind { + if !flushed[m.Node] { + also = append(also, m.Node) + } + } + if len(also) == 0 { + break + } + sort.Strings(also) + fmt.Printf("\nthis push left %s behind — a provision granted from there, or a "+ + "declaration since changed; sending it too\n", strings.Join(also, ", ")) + if err := sendTo(ctx, open, also); err != nil { + return err + } + for _, name := range also { + flushed[name] = true + } + } + } + // A named node is a request to make THAT node current now, so it waits for the node to say it // applied exactly this. A whole-mesh or --behind push does not wait: it is a sweep, and blocking // on the slowest machine would hold back the report on all the others. diff --git a/internal/catalogue/filtering.go b/internal/catalogue/filtering.go index 773a434..3726e87 100644 --- a/internal/catalogue/filtering.go +++ b/internal/catalogue/filtering.go @@ -409,6 +409,21 @@ func AsNftables(rules []Rule, mesh []string, outward bool, foundation []int) str b.WriteString(fmt.Sprintf("\t\tct original proto-dst %d accept\n", rule.Port)) } } + + // The foundation ports, forwarded — for the same reason they are in the input chain, and the + // same reason the module rules above are here too: the broker is a published container port, + // so a cross-node dial to it is redirected and *forwarded*, never reaching the input chain + // (novox/hq issue 047's lesson, applied to the foundation this time). Without this a joined + // node reaches the broker only while its first connection's conntrack entry survives — and a + // broker restart, which adopting the foundation's own broker causes, drops the entry and the + // node can never receive another declaration (novox/hq issue 063). From anywhere, matching + // the input rule: a node enrolling has no overlay address yet. + if len(foundation) > 0 { + b.WriteString("\n\t\t# the mesh's own — never derived, never closed\n") + for _, port := range foundation { + b.WriteString(fmt.Sprintf("\t\tct original proto-dst %d accept\n", port)) + } + } b.WriteString("\t}\n") b.WriteString("}\n") return b.String() diff --git a/internal/catalogue/filtering_foundation_test.go b/internal/catalogue/filtering_foundation_test.go index 96d4263..c98da09 100644 --- a/internal/catalogue/filtering_foundation_test.go +++ b/internal/catalogue/filtering_foundation_test.go @@ -33,6 +33,16 @@ func TestTheBrokersPortIsOpenedThoughNoModuleDeclaresIt(t *testing.T) { "has not yet enrolled is not on:\n%s", line) } } + + // And forwarded, not only accepted on input: the broker is a published container port, so a + // cross-node dial is redirected and forwarded and never reaches the input chain. Without this + // a joined node reaches the broker only until its first connection's conntrack entry drops — + // which a broker restart (adopting the foundation's broker) causes — and then never receives + // another declaration (novox/hq issue 063). + if !strings.Contains(out, "ct original proto-dst 5671 accept") { + t.Fatalf("the broker's port is not forwarded, so a DNAT'd broker is unreachable "+ + "cross-node after it restarts:\n%s", out) + } } // And a mesh that was never told about a broker still gets a ruleset, rather than an empty one or