From f47b6e1c31bd0da7875a987c66773aa6ee75bacd Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 18 Sep 2026 02:05:55 +0200 Subject: [PATCH 1/3] One push leaves the mesh consistent (issue 057) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A provision is minted while composing the consumer's node, and the provider's grant list is a pure read of secrets already issued — so pushing the consumer left the provider blind until somebody pushed it again, with no signal to. A named push now captures what every machine should be before composing, recomputes after, and sends the machines whose declaration changed because of this push — by name, never silently, converging over bounded rounds. --- cmd/mesh-controller/push.go | 48 +++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 656d683..ba1fd80 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -9,6 +9,7 @@ import ( "flag" "fmt" "os" + "sort" "strings" "time" @@ -249,6 +250,18 @@ func pushCommand(ctx context.Context, args []string) error { return err } + // What every machine should be BEFORE this push composes anything. Composing a named node + // mints the provisions its consumers need, and a minted provision changes what the PROVIDER + // machine should be — its grant list is a pure read of secrets already issued (novox/hq + // issue 057). Captured now so that, after the send, "what changed because of this push" is + // a comparison rather than a guess. + var before map[string]string + if len(args) == 1 { + if before, err = wouldSend(ctx, open, nodes); err != nil { + return err + } + } + server, err := link.Connect(nil, nil) if err != nil { return err @@ -321,6 +334,41 @@ func pushCommand(ctx context.Context, args []string) error { } fmt.Printf("\n%d node(s) told\n", len(sending)) + // **A push leaves the mesh consistent, not just the machine it named** (novox/hq issue 057). + // The machines whose declaration changed because of THIS push — providers a provision was + // just minted from — are sent theirs too, by name, never silently: the alternative was a + // consumer that retries forever while nothing says the provider was left blind, and a rule + // ("push the provider node too") enforced by nothing. Bounded: a cascade send may itself + // mint, so this converges over a few rounds, each naming what changed and why. + if len(args) == 1 && before != nil { + delivered := map[string]bool{args[0]: true} + for round := 0; round < 3; round++ { + after, err := wouldSend(ctx, open, nodes) + if err != nil { + return err + } + var also []string + for name, digest := range after { + if !delivered[name] && before[name] != "" && before[name] != digest { + also = append(also, name) + } + } + if len(also) == 0 { + break + } + sort.Strings(also) + fmt.Printf("\nthis push changed what %s should be — a provision granted from "+ + "there; sending it too\n", strings.Join(also, ", ")) + if err := sendTo(ctx, open, also); err != nil { + return err + } + for _, name := range also { + delivered[name] = true + } + before = after + } + } + // A named node is a request to make THAT node current now, so it waits for the node to say it // applied exactly this. A whole-mesh or --behind push does not wait: it is a sweep, and blocking // on the slowest machine would hold back the report on all the others. From 25e42b3ad684a0b8944e50f4c534e0c9a6e3fc52 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 18 Sep 2026 02:19:34 +0200 Subject: [PATCH 2/3] The foundation's ports are forwarded, not only accepted on input (issue 063) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The broker's amqps port is opened from anywhere so a node can enrol before it has an overlay address — but only in the input chain. The broker is a published container port, so a cross-node dial is DNAT'd and forwarded, never reaching input; it survived on the first connection's conntrack entry and no more. Adopting the foundation's own broker restarts it, dropping that entry, after which a joined node could never receive another declaration. The forward chain now carries the foundation ports too, from anywhere, matching their input rule. Intermittent in the built-store-cross-node bed: it passed whenever the broker did not happen to restart after the joined node first connected. --- internal/catalogue/filtering.go | 15 +++++++++++++++ internal/catalogue/filtering_foundation_test.go | 10 ++++++++++ 2 files changed, 25 insertions(+) diff --git a/internal/catalogue/filtering.go b/internal/catalogue/filtering.go index 773a434..3726e87 100644 --- a/internal/catalogue/filtering.go +++ b/internal/catalogue/filtering.go @@ -409,6 +409,21 @@ func AsNftables(rules []Rule, mesh []string, outward bool, foundation []int) str b.WriteString(fmt.Sprintf("\t\tct original proto-dst %d accept\n", rule.Port)) } } + + // The foundation ports, forwarded — for the same reason they are in the input chain, and the + // same reason the module rules above are here too: the broker is a published container port, + // so a cross-node dial to it is redirected and *forwarded*, never reaching the input chain + // (novox/hq issue 047's lesson, applied to the foundation this time). Without this a joined + // node reaches the broker only while its first connection's conntrack entry survives — and a + // broker restart, which adopting the foundation's own broker causes, drops the entry and the + // node can never receive another declaration (novox/hq issue 063). From anywhere, matching + // the input rule: a node enrolling has no overlay address yet. + if len(foundation) > 0 { + b.WriteString("\n\t\t# the mesh's own — never derived, never closed\n") + for _, port := range foundation { + b.WriteString(fmt.Sprintf("\t\tct original proto-dst %d accept\n", port)) + } + } b.WriteString("\t}\n") b.WriteString("}\n") return b.String() diff --git a/internal/catalogue/filtering_foundation_test.go b/internal/catalogue/filtering_foundation_test.go index 96d4263..c98da09 100644 --- a/internal/catalogue/filtering_foundation_test.go +++ b/internal/catalogue/filtering_foundation_test.go @@ -33,6 +33,16 @@ func TestTheBrokersPortIsOpenedThoughNoModuleDeclaresIt(t *testing.T) { "has not yet enrolled is not on:\n%s", line) } } + + // And forwarded, not only accepted on input: the broker is a published container port, so a + // cross-node dial is redirected and forwarded and never reaches the input chain. Without this + // a joined node reaches the broker only until its first connection's conntrack entry drops — + // which a broker restart (adopting the foundation's broker) causes — and then never receives + // another declaration (novox/hq issue 063). + if !strings.Contains(out, "ct original proto-dst 5671 accept") { + t.Fatalf("the broker's port is not forwarded, so a DNAT'd broker is unreachable "+ + "cross-node after it restarts:\n%s", out) + } } // And a mesh that was never told about a broker still gets a ruleset, rather than an empty one or From 97c076ea4841f14b3a6a65cfcf146b0393ff1bb0 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 18 Sep 2026 02:37:26 +0200 Subject: [PATCH 3/3] The one-push cascade compares against what was last sent (issue 057) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first cut compared a before/after snapshot of the named push — but the provision is minted at assign or module-issue, before push runs, so by push time the provider is already behind with no delta to detect. Fixed to flush machines whose declaration differs from what they were last SENT (the same Waiting path --behind uses), which is the honest meaning of 'one push leaves the mesh consistent' (ADR 0083). Verified live on a kept two-node mesh: pushing the consumer populates the provider's grant and the vhost is minted. --- cmd/mesh-controller/push.go | 55 ++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index ba1fd80..2713cd1 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -250,18 +250,6 @@ func pushCommand(ctx context.Context, args []string) error { return err } - // What every machine should be BEFORE this push composes anything. Composing a named node - // mints the provisions its consumers need, and a minted provision changes what the PROVIDER - // machine should be — its grant list is a pure read of secrets already issued (novox/hq - // issue 057). Captured now so that, after the send, "what changed because of this push" is - // a comparison rather than a guess. - var before map[string]string - if len(args) == 1 { - if before, err = wouldSend(ctx, open, nodes); err != nil { - return err - } - } - server, err := link.Connect(nil, nil) if err != nil { return err @@ -334,38 +322,47 @@ func pushCommand(ctx context.Context, args []string) error { } fmt.Printf("\n%d node(s) told\n", len(sending)) - // **A push leaves the mesh consistent, not just the machine it named** (novox/hq issue 057). - // The machines whose declaration changed because of THIS push — providers a provision was - // just minted from — are sent theirs too, by name, never silently: the alternative was a - // consumer that retries forever while nothing says the provider was left blind, and a rule - // ("push the provider node too") enforced by nothing. Bounded: a cascade send may itself - // mint, so this converges over a few rounds, each naming what changed and why. - if len(args) == 1 && before != nil { - delivered := map[string]bool{args[0]: true} - for round := 0; round < 3; round++ { - after, err := wouldSend(ctx, open, nodes) + // **A named push leaves the mesh consistent, not just the machine it named** (novox/hq + // issue 057, ADR 0083). Assigning a cross-node consumer mints a provision, and the PROVIDER's + // grant list is a pure read of secrets already issued — so after the named node is current, + // other machines can be behind *as a consequence*: their declaration now differs from what + // they were last sent. Those are flushed too, by name, in the push's own output. + // + // Compared against what each machine was last SENT, not against a before/after of this push: + // the mint usually happened at `assign` or `module issue`, before this command ran, so the + // only durable signal is "what it should be" versus "what it last received". A machine behind + // for an unrelated reason is caught here too, which is not a cost — a named push that knew a + // machine was behind and left it so would be the very silence this removes. Bounded: a + // flushed send may itself mint, so this converges over a few rounds. + if len(args) == 1 { + flushed := map[string]bool{args[0]: true} + for round := 0; round < 4; round++ { + would, err := wouldSend(ctx, open, nodes) + if err != nil { + return err + } + behind, err := inv.Waiting(ctx, would) if err != nil { return err } var also []string - for name, digest := range after { - if !delivered[name] && before[name] != "" && before[name] != digest { - also = append(also, name) + for _, m := range behind { + if !flushed[m.Node] { + also = append(also, m.Node) } } if len(also) == 0 { break } sort.Strings(also) - fmt.Printf("\nthis push changed what %s should be — a provision granted from "+ - "there; sending it too\n", strings.Join(also, ", ")) + fmt.Printf("\nthis push left %s behind — a provision granted from there, or a "+ + "declaration since changed; sending it too\n", strings.Join(also, ", ")) if err := sendTo(ctx, open, also); err != nil { return err } for _, name := range also { - delivered[name] = true + flushed[name] = true } - before = after } }