mesh-controller/cmd/mesh-controller/replay348_test.go

package main

import (
	"testing"
	"time"

	"github.com/novox/mesh-controller/internal/conditions"
	"github.com/novox/mesh-controller/internal/link"
)

// Replay of novox/hq issue 348 (2026-10-09): the control node's resolver stopped answering on its private
// address at 10:57:57 UTC, and the machine's network condition was raised at 10:58:45. The controller and the
// node-engine were sent to their first machines at 10:59:34. The restarted node-engine's first statement judged
// the names part once — unknown, "one look failed; a second decides" — and that statement cleared the condition,
// though the resolver still refused. Its next look raised it again at 11:00:23, after the send, and at 11:10:29
// both builds failed their gate with "raised since it was sent: machine.<control node>.network" and were put
// back, for a fault that began before they were sent.
//
// Written only with what the controller had before its fix (mesh-controller #179): a mesh, the machine's
// statements as stateHealth keeps them, the keeper's open conditions and the gate's aboutTheMachine — so the
// prover (mesh-lab replays/cmd/prove) lays it over the commit before. Its outcome is the operator's acceptance:
// a build is never put back for a problem that started before it was sent; and the fault is not forgotten
// while the resolver still refuses. Needs a store: MESH_TEST_POSTGRES.
func TestReplay348AFaultFromBeforeTheSendFailsNoGate(t *testing.T) {
	open := aMesh(t)
	ctx := t.Context()
	inv, k := open.inventory, conditionsFrom
	const key = "machine.anchor.network"
	names := func(state string, streak int) link.NetworkPart {
		p := link.NetworkPart{Part: link.PartNames, State: state, Since: h0, Streak: streak}
		if state == link.StateUnhealthy {
			p.Reason = "1 of its 2 resolvers do not answer as the mesh's do"
			p.Said = "10.77.0.1 — anchor.internal (IPv4): read udp 10.77.0.1:35244->10.77.0.1:53: read: connection refused"
			p.Toward = []string{"10.77.0.1"}
		}
		return p
	}
	say := func(at time.Time, state string, streak int) {
		t.Helper()
		n := &link.NetworkHealth{State: state, Since: h0, Parts: []link.NetworkPart{names(state, streak)}}
		if err := stateHealth(ctx, inv, k, "anchor", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil {
			t.Fatal(err)
		}
	}
	network := func() (conditions.Condition, bool) {
		t.Helper()
		list, err := k.Open(ctx)
		if err != nil {
			t.Fatal(err)
		}
		for _, c := range list {
			if c.Key == key {
				return c, true
			}
		}
		return conditions.Condition{}, false
	}

	// 10:58:45 — the node-engine's second failing look: raised.
	say(h0, link.StateUnhealthy, 2)
	if _, ok := network(); !ok {
		t.Fatal("the resolver refusing every question on the control node raised nothing")
	}
	// 10:59:34 — the controller and the node-engine are sent.
	time.Sleep(10 * time.Millisecond)
	sent := time.Now().UTC()
	time.Sleep(10 * time.Millisecond)

	// 10:59:42 — the restarted engine's first statement: one look failed, a second decides.
	say(h0.Add(time.Minute), link.StateUnknown, 1)
	if _, ok := network(); !ok {
		t.Error("the restarted node-engine's first, undecided statement cleared the network condition while " +
			"the resolver still refused")
	}
	// 11:00:23 — its second look: unhealthy again.
	time.Sleep(10 * time.Millisecond)
	say(h0.Add(2*time.Minute), link.StateUnhealthy, 2)
	if _, ok := network(); !ok {
		t.Fatal("the resolver still refusing raised nothing")
	}

	// 11:10:29 — the gate on the control node, for the builds sent after the fault began.
	list, err := k.Open(ctx)
	if err != nil {
		t.Fatal(err)
	}
	for _, moved := range [][]string{{"mesh-host"}, {"mesh-controller"}} {
		w := aboutTheMachine("anchor", moved, sent, gateFacts{judged: true, open: list})
		if w.whole != "" || len(w.on) != 0 {
			t.Errorf("%s, sent after the resolver stopped answering, was held for that fault: whole %q, on %v",
				moved[0], w.whole, w.on)
		}
	}
}