Add the replay of issue 348, and refuse bind-dynamic now the catalogue binds once (hq issue 348)

The replay holds the operator's acceptance: a build sent after the resolver
stopped answering is not put back for that fault, and a restarted engine's
undecided statement does not forget it. Written only with what the controller
had before mesh-controller #179, so mesh-lab's prover lays it over the commit
before: it fails there and passes on the fix and on main.

The resolver's manifest test took either binding until the catalogue's main
bound once; mesh-catalog #161 (105cc7c0) did, so bind-dynamic is refused.

cmd/mesh-controller/replay348_test.go

@@ -0,0 +1,94 @@
+package main
+
+import (
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// Replay of novox/hq issue 348 (2026-10-09): the control node's resolver stopped answering on its private
+// address at 10:57:57 UTC, and the machine's network condition was raised at 10:58:45. The controller and the
+// node-engine were sent to their first machines at 10:59:34. The restarted node-engine's first statement judged
+// the names part once — unknown, "one look failed; a second decides" — and that statement cleared the condition,
+// though the resolver still refused. Its next look raised it again at 11:00:23, after the send, and at 11:10:29
+// both builds failed their gate with "raised since it was sent: machine.<control node>.network" and were put
+// back, for a fault that began before they were sent.
+//
+// Written only with what the controller had before its fix (mesh-controller #179): a mesh, the machine's
+// statements as stateHealth keeps them, the keeper's open conditions and the gate's aboutTheMachine — so the
+// prover (mesh-lab replays/cmd/prove) lays it over the commit before. Its outcome is the operator's acceptance:
+// a build is never put back for a problem that started before it was sent; and the fault is not forgotten
+// while the resolver still refuses. Needs a store: MESH_TEST_POSTGRES.
+func TestReplay348AFaultFromBeforeTheSendFailsNoGate(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	inv, k := open.inventory, conditionsFrom
+	const key = "machine.anchor.network"
+	names := func(state string, streak int) link.NetworkPart {
+		p := link.NetworkPart{Part: link.PartNames, State: state, Since: h0, Streak: streak}
+		if state == link.StateUnhealthy {
+			p.Reason = "1 of its 2 resolvers do not answer as the mesh's do"
+			p.Said = "10.77.0.1 — anchor.internal (IPv4): read udp 10.77.0.1:35244->10.77.0.1:53: read: connection refused"
+			p.Toward = []string{"10.77.0.1"}
+		}
+		return p
+	}
+	say := func(at time.Time, state string, streak int) {
+		t.Helper()
+		n := &link.NetworkHealth{State: state, Since: h0, Parts: []link.NetworkPart{names(state, streak)}}
+		if err := stateHealth(ctx, inv, k, "anchor", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil {
+			t.Fatal(err)
+		}
+	}
+	network := func() (conditions.Condition, bool) {
+		t.Helper()
+		list, err := k.Open(ctx)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for _, c := range list {
+			if c.Key == key {
+				return c, true
+			}
+		}
+		return conditions.Condition{}, false
+	}
+
+	// 10:58:45 — the node-engine's second failing look: raised.
+	say(h0, link.StateUnhealthy, 2)
+	if _, ok := network(); !ok {
+		t.Fatal("the resolver refusing every question on the control node raised nothing")
+	}
+	// 10:59:34 — the controller and the node-engine are sent.
+	time.Sleep(10 * time.Millisecond)
+	sent := time.Now().UTC()
+	time.Sleep(10 * time.Millisecond)
+
+	// 10:59:42 — the restarted engine's first statement: one look failed, a second decides.
+	say(h0.Add(time.Minute), link.StateUnknown, 1)
+	if _, ok := network(); !ok {
+		t.Error("the restarted node-engine's first, undecided statement cleared the network condition while " +
+			"the resolver still refused")
+	}
+	// 11:00:23 — its second look: unhealthy again.
+	time.Sleep(10 * time.Millisecond)
+	say(h0.Add(2*time.Minute), link.StateUnhealthy, 2)
+	if _, ok := network(); !ok {
+		t.Fatal("the resolver still refusing raised nothing")
+	}
+
+	// 11:10:29 — the gate on the control node, for the builds sent after the fault began.
+	list, err := k.Open(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, moved := range [][]string{{"mesh-host"}, {"mesh-controller"}} {
+		w := aboutTheMachine("anchor", moved, sent, gateFacts{judged: true, open: list})
+		if w.whole != "" || len(w.on) != 0 {
+			t.Errorf("%s, sent after the resolver stopped answering, was held for that fault: whole %q, on %v",
+				moved[0], w.whole, w.on)
+		}
+	}
+}

internal/catalogue/resolver_manifests_test.go

@@ -63,13 +63,17 @@ func TestTheResolverForwardsToFixedUpstreamsAndNeverReadsResolvConf(t *testing.T
 			t.Errorf("the resolver's configuration lacks %q:\n%s", strings.TrimSpace(want), config)
 		}
 	}
-	// Bound to its addresses, one way or the other. mesh-catalog PR 161 (novox/hq issue 348) moves it from
-	// bind-dynamic, which closed the private address's listener when a bridge teardown failed its re-read
-	// of the machine's addresses, to bind-interfaces. This test reads the catalogue beside it, which may be
-	// on either side of that merge, so it takes both; once the catalogue's main has it, bind-dynamic is
-	// refused here.
-	if !strings.Contains(config, "\nbind-interfaces\n") && !strings.Contains(config, "\nbind-dynamic\n") {
-		t.Errorf("the resolver's configuration binds neither by bind-interfaces nor by bind-dynamic:\n%s", config)
+	// Bound to each address once, at start, and never by bind-dynamic (novox/hq issue 348): bind-dynamic
+	// re-reads the machine's addresses on every change of any link and closes each listener that reading
+	// did not find, and a reading made while the container runtime tears down a bridge's virtual links
+	// fails, so the private address was left unbound for twenty minutes, the daemon running and silent.
+	// The catalogue's main binds once since mesh-catalog PR 161 (105cc7c0).
+	if !strings.Contains(config, "\nbind-interfaces\n") {
+		t.Errorf("the resolver's configuration does not bind its addresses once (bind-interfaces):\n%s", config)
+	}
+	if strings.Contains(config, "\nbind-dynamic\n") {
+		t.Errorf("the resolver's configuration says bind-dynamic, which closes its listeners when a bridge is "+
+			"torn down (novox/hq issue 348):\n%s", config)
 	}
 	// By address and never by interface: dnsmasq admits a query by the interface it arrives on
 	// when told one, and a container's query to the private address arrives on the runtime's

testdata/store-tests

@@ -42,3 +42,4 @@
 ./cmd/mesh-controller TestReplay340
 ./cmd/mesh-controller TestOnlyATrustedFilesNamedKeyIsRefusedThroughAVerb
 ./cmd/mesh-controller TestReplay349
+./cmd/mesh-controller TestReplay348AFaultFromBeforeTheSendFailsNoGate