Judge a send by when a fault began, not when it was last raised (hq issue 348)
On 2026-10-09 the control node's resolver refused from 10:57:57 UTC. A node-engine restarted by the 10:59:34 send said its names undecided, that statement cleared the network condition, the next look raised it again after the send, and the gate put back two builds for a fault older than them. - A condition keeps First across a reopening; the gate reads Began. - An undecided network statement (unknown, starting) clears nothing. - D10 counts what a release's tier names as rolling, so a walked node-engine is not core-behind on its own first machine. - D2 raises a resolver that refuses every try at once: a refusal is an answer, not a loaded resolver (issue 277).
This commit is contained in:
jochen
@@ -0,0 +1,212 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net"
|
||||
"os"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// novox/hq issue 348: on 2026-10-09 the control node's resolver stopped answering on its private address
|
||||
// at 10:57:57 UTC; the machine's network condition was raised at 10:58:45. The node-engine and the
|
||||
// controller were sent at 10:59:34. The new node-engine's first statement judged the names once — unknown,
|
||||
// "one look failed; a second decides" — and that statement cleared the condition, though its last evidence
|
||||
// still said "connection refused". The next look raised it again at 11:00:23, after the send, and both
|
||||
// builds failed their gate at 11:10 with "raised since it was sent" and were put back, for a fault that
|
||||
// began before they were sent.
|
||||
|
||||
// namesRefused is the control node's names part as its node-engine said it in the outage: its own
|
||||
// resolver, at its own address, refusing.
|
||||
func namesRefused(state string, streak int) link.NetworkPart {
|
||||
p := link.NetworkPart{Part: link.PartNames, State: state, Since: h0, Streak: streak}
|
||||
if state == link.StateUnhealthy {
|
||||
p.Reason = "1 of its 2 resolvers do not answer as the mesh's do"
|
||||
p.Said = "10.77.0.1 — anchor.internal (IPv4): read udp 10.77.0.1:35244->10.77.0.1:53: read: connection refused"
|
||||
p.Toward = []string{"10.77.0.1"}
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func networkSaying(parts ...link.NetworkPart) *link.NetworkHealth {
|
||||
state := link.StateHealthy
|
||||
for _, p := range parts {
|
||||
switch {
|
||||
case p.State == link.StateUnhealthy:
|
||||
state = link.StateUnhealthy
|
||||
case p.State == link.StateUnknown && state == link.StateHealthy:
|
||||
state = link.StateUnknown
|
||||
}
|
||||
}
|
||||
return &link.NetworkHealth{State: state, Since: h0, Parts: parts}
|
||||
}
|
||||
|
||||
// TestReplay348 replays the statements of the outage: the condition raised before the send is not
|
||||
// cleared by the restarted engine's first, undecided statement, and the gate does not count it against
|
||||
// the builds sent after it began.
|
||||
func TestReplay348(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv, k := open.inventory, conditionsFrom
|
||||
say := func(at time.Time, n *link.NetworkHealth) {
|
||||
t.Helper()
|
||||
if err := stateHealth(ctx, inv, k, "anchor", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
network := func() (conditions.Condition, bool) {
|
||||
t.Helper()
|
||||
list, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range list {
|
||||
if c.Key == "machine.anchor.network" {
|
||||
return c, true
|
||||
}
|
||||
}
|
||||
return conditions.Condition{}, false
|
||||
}
|
||||
|
||||
// 10:58:45 — the second failing look: raised.
|
||||
say(h0, networkSaying(namesRefused(link.StateUnhealthy, 2)))
|
||||
raised, ok := network()
|
||||
if !ok {
|
||||
t.Fatal("the resolver refusing on the control node raised nothing")
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
sent := time.Now().UTC()
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
|
||||
// 10:59:42 — the restarted engine's first statement: one look failed, a second decides.
|
||||
say(h0.Add(time.Minute), networkSaying(namesRefused(link.StateUnknown, 1)))
|
||||
if _, ok := network(); !ok {
|
||||
t.Fatal("a statement that judged nothing yet cleared the condition: the restarted engine's first look " +
|
||||
"said the fault was gone while it still refused")
|
||||
}
|
||||
// 11:00:23 — its second look: unhealthy again, the same raising.
|
||||
say(h0.Add(2*time.Minute), networkSaying(namesRefused(link.StateUnhealthy, 2)))
|
||||
again, ok := network()
|
||||
if !ok || !again.Began().Equal(raised.Raised) {
|
||||
t.Fatalf("the same fault was said as one that began at %s (raised first at %s)", again.Began(), raised.Raised)
|
||||
}
|
||||
|
||||
// The gate on the control node, for a build sent after the fault began.
|
||||
open2, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := gateFacts{judged: true, open: open2}
|
||||
if w := aboutTheMachine("anchor", []string{"mesh-host"}, sent, f); w.whole != "" || len(w.on) != 0 {
|
||||
t.Fatalf("a fault from before the send held the build: %+v", w)
|
||||
}
|
||||
|
||||
// Decided healthy: cleared.
|
||||
say(h0.Add(3*time.Minute), networkSaying(link.NetworkPart{Part: link.PartNames, State: link.StateHealthy, Since: h0}))
|
||||
if c, ok := network(); ok {
|
||||
t.Fatalf("a statement that decides the names healthy left %s open", c.Key)
|
||||
}
|
||||
}
|
||||
|
||||
// A reopening is the same fault: it began when it was first raised, and the gate reads when it began. The
|
||||
// controller that cleared it on an undecided statement — or any clearing and raising within ReopenWithin —
|
||||
// no longer makes a fault from before a send into one raised since it.
|
||||
func TestAReopenedFaultFromBeforeTheSendIsNotRaisedSinceIt(t *testing.T) {
|
||||
since := h0
|
||||
c := conditions.Condition{Key: "machine.anchor.network", Kind: kindMachineNetwork,
|
||||
Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor"},
|
||||
Summary: "anchor's network is not healthy", Source: sourceNetwork, Count: 2,
|
||||
First: since.Add(-time.Minute), Raised: since.Add(time.Minute)}
|
||||
f := gateFacts{judged: true, open: []conditions.Condition{c}}
|
||||
if w := aboutTheMachine("anchor", []string{"mesh-controller"}, since, f); w.whole != "" || len(w.on) != 0 {
|
||||
t.Fatalf("a fault reopened after the send, first raised before it, held the machine: %+v", w)
|
||||
}
|
||||
// The same raised first after the send is the send's to answer for.
|
||||
c.First = since.Add(30 * time.Second)
|
||||
f.open = []conditions.Condition{c}
|
||||
if w := aboutTheMachine("anchor", []string{"mesh-controller"}, since, f); w.whole == "" {
|
||||
t.Fatalf("a fault that began after the send held nothing: %+v", w)
|
||||
}
|
||||
// And a module's own: judgeHealth reads it the same way.
|
||||
at := since.Add(2 * time.Minute)
|
||||
g := gateFacts{judged: true, now: at, reports: map[string]inventory.Reported{"anchor": {Node: "anchor",
|
||||
Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, engines: map[string]string{},
|
||||
served: map[string]served{}, rolledBack: map[string][]lease.Rollback{},
|
||||
open: []conditions.Condition{{Key: "provider.app.anchor.x.failing", Subject: conditions.Subject{
|
||||
Scope: conditions.ScopeProvider, ID: "app.anchor.x", Machine: "anchor"}, Summary: "failing",
|
||||
First: since.Add(-time.Hour), Raised: since.Add(time.Minute)}}}
|
||||
if h, why := judgeHealth("app", "", catalogue.Manifest{Module: "app"}, "anchor", since, g); strings.HasPrefix(why, "raised since it was sent") {
|
||||
t.Fatalf("a module's own fault, reopened after its send and first raised before it: %v (%s)", h, why)
|
||||
}
|
||||
g.open[0].First = time.Time{}
|
||||
if _, why := judgeHealth("app", "", catalogue.Manifest{Module: "app"}, "anchor", since, g); !strings.HasPrefix(why, "raised since it was sent") {
|
||||
t.Fatalf("a module's own fault raised after its send was not counted: %s", why)
|
||||
}
|
||||
}
|
||||
|
||||
// An undecided statement — unknown or starting — keeps the machine's network conditions; a decided one
|
||||
// does not. Pure.
|
||||
func TestAnUndecidedStatementKeepsTheMachinesNetworkConditions(t *testing.T) {
|
||||
f := netFacts(map[string]*inventory.NetworkHealth{
|
||||
"anchor": aNetwork(link.StateUnknown, inventory.NetworkPart{Part: link.PartNames, State: link.StateUnknown, Streak: 1}),
|
||||
"laptop": aNetwork(link.StateHealthy, inventory.NetworkPart{Part: link.PartNames, State: link.StateHealthy}),
|
||||
"spare": aNetwork(link.StateUnhealthy, noRoute),
|
||||
})
|
||||
got := undecidedMachines(f)
|
||||
if !got["anchor"] || got["laptop"] || got["spare"] || len(got) != 1 {
|
||||
t.Fatalf("undecided: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A release walks its modules without a record per module: D10 counts what its tier names as rolling,
|
||||
// so the node-engine a release walks is not "behind, and no plan is rolling it out" on its first machine.
|
||||
func TestAReleaseRollsOutWhatItsTierNames(t *testing.T) {
|
||||
plans := []inventory.Plan{
|
||||
{ID: "release-1", State: inventory.PlanRolling, Tiers: [][]string{{"mesh-host"}}, Modules: map[string]*inventory.PlanModule{}},
|
||||
{ID: "plan-2", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{"letta": {}}},
|
||||
}
|
||||
got := rollingModules(plans)
|
||||
if !got["mesh-host"] || !got["letta"] || len(got) != 2 {
|
||||
t.Fatalf("rolling: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A resolver that refuses every try — nothing listens on its port — is said at once, not held for the
|
||||
// next run five minutes later: a refusal is the machine's answer, not a loaded resolver (issue 277).
|
||||
func TestAResolverThatRefusesEveryTryIsSaidAtOnce(t *testing.T) {
|
||||
conn, err := net.ListenPacket("udp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, port, _ := net.SplitHostPort(conn.LocalAddr().String())
|
||||
_ = conn.Close() // nothing listens there now: every question is refused
|
||||
quickResolvers(t, port)
|
||||
got := askEveryResolver(t.Context(), map[string]string{"anchor.internal": "127.0.0.1"}, anchorOnTheNetwork, "internal")
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
if got[0].Confirm || !strings.Contains(got[0].Said, "refused") {
|
||||
t.Fatalf("a resolver refusing every try was held for the next run: %+v", got[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyARefusalOnEveryTryIsAllRefused(t *testing.T) {
|
||||
refused := &net.OpError{Op: "read", Net: "udp", Err: os.NewSyscallError("read", syscall.ECONNREFUSED)}
|
||||
if !(resolverAsked{errs: []error{refused, refused}}).allRefused() {
|
||||
t.Fatal("refused on every try is not all refused")
|
||||
}
|
||||
if (resolverAsked{errs: []error{refused, errors.New("i/o timeout")}}).allRefused() {
|
||||
t.Fatal("a timeout among the tries is all refused")
|
||||
}
|
||||
if (resolverAsked{answered: true, errs: []error{refused}}).allRefused() || (resolverAsked{}).allRefused() {
|
||||
t.Fatal("an answered question, or one never asked, is all refused")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user