From 05b90f966ac3562b858f7bc705e589c47f4827a3 Mon Sep 17 00:00:00 2001
From: jochen <jschoubben@gmail.com>
Date: Thu, 1 Oct 2026 15:30:04 +0200
Subject: [PATCH] A refused membership does not stop the controller
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A stream publish waits for its acknowledgement as long as its context lives, and the server never
acknowledges a publish it refuses. Issuing memberships after a push used the daemon's own context, so
the one refused membership of 2026-10-01 (hq issue 183) held the controller's receive loop for good:
no report, no build outcome, no merge was heard until a restart (hq issue 185). Issuing one
membership is now bounded to ten seconds, and a push says how many could not be issued and stands —
the machines keep the shape they derive until the next push.
---
 cmd/mesh-controller/push.go | 16 ++++++++++++++--
 internal/link/bus.go        |  8 ++++++++
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go
index 095086f..4449d00 100644
--- a/cmd/mesh-controller/push.go
+++ b/cmd/mesh-controller/push.go
@@ -708,7 +708,11 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, na
 	if !ok {
 		return nil
 	}
-	issued := 0
+	// The declarations are sent and recorded by now; a membership that cannot be issued is said
+	// and does not unsay them. Every runtime without one serves the shape it derives (ADR 0160), so
+	// the push stands, the first failure is named once, and the next push tries again.
+	issued, failed := 0, 0
+	var first error
 	for _, node := range names {
 		for _, d := range records.Assigned[node] {
 			body, err := json.Marshal(broker.MembershipFor(node, d, where))
@@ -716,7 +720,11 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, na
 				return err
 			}
 			if err := bus.PublishMembership(ctx, node, d.Module, body); err != nil {
-				return err
+				if first == nil {
+					first = err
+				}
+				failed++
+				continue
 			}
 			issued++
 		}
@@ -724,6 +732,10 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, na
 	if issued > 0 {
 		fmt.Printf("  issued %d membership(s)\n", issued)
 	}
+	if failed > 0 {
+		fmt.Printf("  %d membership(s) could not be issued; the first: %v — the machines keep what "+
+			"they derive until the next push\n", failed, first)
+	}
 	return nil
 }
 
diff --git a/internal/link/bus.go b/internal/link/bus.go
index 3e83ce7..cc0ead6 100644
--- a/internal/link/bus.go
+++ b/internal/link/bus.go
@@ -148,7 +148,15 @@ func (b OverNATS) PublishSeatEvent(ctx context.Context, seat, event string, body
 
 // PublishMembership issues one assignment what it serves and reaches (novox/hq ADR 0160), last per
 // subject, so the runtime that connects later reads the current one and one that is running follows.
+// MembershipWait bounds how long issuing one membership may take. A publish the server refuses is
+// never acknowledged, and a stream publish waits for its acknowledgement for as long as its
+// context lives: on 2026-10-01 the daemon's own context was that long, and one refused membership
+// held the controller's receive loop for good (novox/hq issue 185).
+const MembershipWait = 10 * time.Second
+
 func (b OverNATS) PublishMembership(ctx context.Context, node, module string, body []byte) error {
+	ctx, cancel := context.WithTimeout(ctx, MembershipWait)
+	defer cancel()
 	_, err := b.JS.Publish(broker.MembershipSubject(node, module), body, nats.Context(ctx))
 	if err != nil {
 		return fmt.Errorf("issuing %s on %s its membership: %w", module, node, err)
-- 
2.54.0