package main import ( "testing" "time" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/link" ) // Replay of novox/hq issue 348 (2026-10-09): the control node's resolver stopped answering on its private // address at 10:57:57 UTC, and the machine's network condition was raised at 10:58:45. The controller and the // node-engine were sent to their first machines at 10:59:34. The restarted node-engine's first statement judged // the names part once — unknown, "one look failed; a second decides" — and that statement cleared the condition, // though the resolver still refused. Its next look raised it again at 11:00:23, after the send, and at 11:10:29 // both builds failed their gate with "raised since it was sent: machine..network" and were put // back, for a fault that began before they were sent. // // Written only with what the controller had before its fix (mesh-controller #179): a mesh, the machine's // statements as stateHealth keeps them, the keeper's open conditions and the gate's aboutTheMachine — so the // prover (mesh-lab replays/cmd/prove) lays it over the commit before. Its outcome is the operator's acceptance: // a build is never put back for a problem that started before it was sent; and the fault is not forgotten // while the resolver still refuses. Needs a store: MESH_TEST_POSTGRES. func TestReplay348AFaultFromBeforeTheSendFailsNoGate(t *testing.T) { open := aMesh(t) ctx := t.Context() inv, k := open.inventory, conditionsFrom const key = "machine.anchor.network" names := func(state string, streak int) link.NetworkPart { p := link.NetworkPart{Part: link.PartNames, State: state, Since: h0, Streak: streak} if state == link.StateUnhealthy { p.Reason = "1 of its 2 resolvers do not answer as the mesh's do" p.Said = "10.77.0.1 — anchor.internal (IPv4): read udp 10.77.0.1:35244->10.77.0.1:53: read: connection refused" p.Toward = []string{"10.77.0.1"} } return p } say := func(at time.Time, state string, streak int) { t.Helper() n := &link.NetworkHealth{State: state, Since: h0, Parts: []link.NetworkPart{names(state, streak)}} if err := stateHealth(ctx, inv, k, "anchor", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil { t.Fatal(err) } } network := func() (conditions.Condition, bool) { t.Helper() list, err := k.Open(ctx) if err != nil { t.Fatal(err) } for _, c := range list { if c.Key == key { return c, true } } return conditions.Condition{}, false } // 10:58:45 — the node-engine's second failing look: raised. say(h0, link.StateUnhealthy, 2) if _, ok := network(); !ok { t.Fatal("the resolver refusing every question on the control node raised nothing") } // 10:59:34 — the controller and the node-engine are sent. time.Sleep(10 * time.Millisecond) sent := time.Now().UTC() time.Sleep(10 * time.Millisecond) // 10:59:42 — the restarted engine's first statement: one look failed, a second decides. say(h0.Add(time.Minute), link.StateUnknown, 1) if _, ok := network(); !ok { t.Error("the restarted node-engine's first, undecided statement cleared the network condition while " + "the resolver still refused") } // 11:00:23 — its second look: unhealthy again. time.Sleep(10 * time.Millisecond) say(h0.Add(2*time.Minute), link.StateUnhealthy, 2) if _, ok := network(); !ok { t.Fatal("the resolver still refusing raised nothing") } // 11:10:29 — the gate on the control node, for the builds sent after the fault began. list, err := k.Open(ctx) if err != nil { t.Fatal(err) } for _, moved := range [][]string{{"mesh-host"}, {"mesh-controller"}} { w := aboutTheMachine("anchor", moved, sent, gateFacts{judged: true, open: list}) if w.whole != "" || len(w.on) != 0 { t.Errorf("%s, sent after the resolver stopped answering, was held for that fault: whole %q, on %v", moved[0], w.whole, w.on) } } }