From 6eea92c323f4862d4162f9d977c8783d8a62efc3 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 13 Oct 2026 06:09:09 +0200 Subject: [PATCH] Add the replay of issue 348, and refuse bind-dynamic now the catalogue binds once (hq issue 348) The replay holds the operator's acceptance: a build sent after the resolver stopped answering is not put back for that fault, and a restarted engine's undecided statement does not forget it. Written only with what the controller had before mesh-controller #179, so mesh-lab's prover lays it over the commit before: it fails there and passes on the fix and on main. The resolver's manifest test took either binding until the catalogue's main bound once; mesh-catalog #161 (105cc7c0) did, so bind-dynamic is refused. --- cmd/mesh-controller/replay348_test.go | 94 +++++++++++++++++++ internal/catalogue/resolver_manifests_test.go | 18 ++-- testdata/store-tests | 1 + 3 files changed, 106 insertions(+), 7 deletions(-) create mode 100644 cmd/mesh-controller/replay348_test.go diff --git a/cmd/mesh-controller/replay348_test.go b/cmd/mesh-controller/replay348_test.go new file mode 100644 index 00000000..97fcfdb4 --- /dev/null +++ b/cmd/mesh-controller/replay348_test.go @@ -0,0 +1,94 @@ +package main + +import ( + "testing" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/link" +) + +// Replay of novox/hq issue 348 (2026-10-09): the control node's resolver stopped answering on its private +// address at 10:57:57 UTC, and the machine's network condition was raised at 10:58:45. The controller and the +// node-engine were sent to their first machines at 10:59:34. The restarted node-engine's first statement judged +// the names part once — unknown, "one look failed; a second decides" — and that statement cleared the condition, +// though the resolver still refused. Its next look raised it again at 11:00:23, after the send, and at 11:10:29 +// both builds failed their gate with "raised since it was sent: machine..network" and were put +// back, for a fault that began before they were sent. +// +// Written only with what the controller had before its fix (mesh-controller #179): a mesh, the machine's +// statements as stateHealth keeps them, the keeper's open conditions and the gate's aboutTheMachine — so the +// prover (mesh-lab replays/cmd/prove) lays it over the commit before. Its outcome is the operator's acceptance: +// a build is never put back for a problem that started before it was sent; and the fault is not forgotten +// while the resolver still refuses. Needs a store: MESH_TEST_POSTGRES. +func TestReplay348AFaultFromBeforeTheSendFailsNoGate(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv, k := open.inventory, conditionsFrom + const key = "machine.anchor.network" + names := func(state string, streak int) link.NetworkPart { + p := link.NetworkPart{Part: link.PartNames, State: state, Since: h0, Streak: streak} + if state == link.StateUnhealthy { + p.Reason = "1 of its 2 resolvers do not answer as the mesh's do" + p.Said = "10.77.0.1 — anchor.internal (IPv4): read udp 10.77.0.1:35244->10.77.0.1:53: read: connection refused" + p.Toward = []string{"10.77.0.1"} + } + return p + } + say := func(at time.Time, state string, streak int) { + t.Helper() + n := &link.NetworkHealth{State: state, Since: h0, Parts: []link.NetworkPart{names(state, streak)}} + if err := stateHealth(ctx, inv, k, "anchor", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil { + t.Fatal(err) + } + } + network := func() (conditions.Condition, bool) { + t.Helper() + list, err := k.Open(ctx) + if err != nil { + t.Fatal(err) + } + for _, c := range list { + if c.Key == key { + return c, true + } + } + return conditions.Condition{}, false + } + + // 10:58:45 — the node-engine's second failing look: raised. + say(h0, link.StateUnhealthy, 2) + if _, ok := network(); !ok { + t.Fatal("the resolver refusing every question on the control node raised nothing") + } + // 10:59:34 — the controller and the node-engine are sent. + time.Sleep(10 * time.Millisecond) + sent := time.Now().UTC() + time.Sleep(10 * time.Millisecond) + + // 10:59:42 — the restarted engine's first statement: one look failed, a second decides. + say(h0.Add(time.Minute), link.StateUnknown, 1) + if _, ok := network(); !ok { + t.Error("the restarted node-engine's first, undecided statement cleared the network condition while " + + "the resolver still refused") + } + // 11:00:23 — its second look: unhealthy again. + time.Sleep(10 * time.Millisecond) + say(h0.Add(2*time.Minute), link.StateUnhealthy, 2) + if _, ok := network(); !ok { + t.Fatal("the resolver still refusing raised nothing") + } + + // 11:10:29 — the gate on the control node, for the builds sent after the fault began. + list, err := k.Open(ctx) + if err != nil { + t.Fatal(err) + } + for _, moved := range [][]string{{"mesh-host"}, {"mesh-controller"}} { + w := aboutTheMachine("anchor", moved, sent, gateFacts{judged: true, open: list}) + if w.whole != "" || len(w.on) != 0 { + t.Errorf("%s, sent after the resolver stopped answering, was held for that fault: whole %q, on %v", + moved[0], w.whole, w.on) + } + } +} diff --git a/internal/catalogue/resolver_manifests_test.go b/internal/catalogue/resolver_manifests_test.go index 7d149596..c3dfadd6 100644 --- a/internal/catalogue/resolver_manifests_test.go +++ b/internal/catalogue/resolver_manifests_test.go @@ -63,13 +63,17 @@ func TestTheResolverForwardsToFixedUpstreamsAndNeverReadsResolvConf(t *testing.T t.Errorf("the resolver's configuration lacks %q:\n%s", strings.TrimSpace(want), config) } } - // Bound to its addresses, one way or the other. mesh-catalog PR 161 (novox/hq issue 348) moves it from - // bind-dynamic, which closed the private address's listener when a bridge teardown failed its re-read - // of the machine's addresses, to bind-interfaces. This test reads the catalogue beside it, which may be - // on either side of that merge, so it takes both; once the catalogue's main has it, bind-dynamic is - // refused here. - if !strings.Contains(config, "\nbind-interfaces\n") && !strings.Contains(config, "\nbind-dynamic\n") { - t.Errorf("the resolver's configuration binds neither by bind-interfaces nor by bind-dynamic:\n%s", config) + // Bound to each address once, at start, and never by bind-dynamic (novox/hq issue 348): bind-dynamic + // re-reads the machine's addresses on every change of any link and closes each listener that reading + // did not find, and a reading made while the container runtime tears down a bridge's virtual links + // fails, so the private address was left unbound for twenty minutes, the daemon running and silent. + // The catalogue's main binds once since mesh-catalog PR 161 (105cc7c0). + if !strings.Contains(config, "\nbind-interfaces\n") { + t.Errorf("the resolver's configuration does not bind its addresses once (bind-interfaces):\n%s", config) + } + if strings.Contains(config, "\nbind-dynamic\n") { + t.Errorf("the resolver's configuration says bind-dynamic, which closes its listeners when a bridge is "+ + "torn down (novox/hq issue 348):\n%s", config) } // By address and never by interface: dnsmasq admits a query by the interface it arrives on // when told one, and a container's query to the private address arrives on the runtime's diff --git a/testdata/store-tests b/testdata/store-tests index 2c024822..936e5928 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -42,3 +42,4 @@ ./cmd/mesh-controller TestReplay340 ./cmd/mesh-controller TestOnlyATrustedFilesNamedKeyIsRefusedThroughAVerb ./cmd/mesh-controller TestReplay349 +./cmd/mesh-controller TestReplay348AFaultFromBeforeTheSendFailsNoGate -- 2.54.0