Judge a send by when a fault began, not when it was last raised (hq issue 348)

On 2026-10-09 the control node's resolver refused from 10:57:57 UTC. A
node-engine restarted by the 10:59:34 send said its names undecided, that
statement cleared the network condition, the next look raised it again
after the send, and the gate put back two builds for a fault older than
them.

- A condition keeps First across a reopening; the gate reads Began.
- An undecided network statement (unknown, starting) clears nothing.
- D10 counts what a release's tier names as rolling, so a walked
  node-engine is not core-behind on its own first machine.
- D2 raises a resolver that refuses every try at once: a refusal is an
  answer, not a loaded resolver (issue 277).
This commit is contained in:
2026-10-09 15:00:56 +02:00
committed by jochen
parent c58516f819
commit 077ddf0eb8
7 changed files with 370 additions and 17 deletions
+43 -8
View File
@@ -11,6 +11,7 @@ import (
"sort"
"strings"
"sync"
"syscall"
"time"
"github.com/nats-io/nats.go"
@@ -216,6 +217,8 @@ func askEveryResolver(ctx context.Context, resolvers map[string]string, places [
type verdict struct {
wrong, unanswered, said []string
asked int
// refused says every try of every unanswered question was refused (nothing listens there).
refused bool
}
byHolder := map[string]*verdict{}
for _, q := range questions {
@@ -232,6 +235,7 @@ func askEveryResolver(ctx context.Context, resolvers map[string]string, places [
v.wrong = append(v.wrong, words)
v.said = append(v.said, said)
default:
v.refused = (len(v.unanswered) == 0 || v.refused) && q.asked.allRefused()
v.unanswered = append(v.unanswered, words)
v.said = append(v.said, said)
}
@@ -251,8 +255,11 @@ func askEveryResolver(ctx context.Context, resolvers map[string]string, places [
o.Summary = fmt.Sprintf("the mesh's resolver on %s does not answer machine names as it must: %s%s", node,
v.wrong[0], andMore(len(v.wrong)-1))
} else {
// Nothing but silence: held for the next run, which raises it if the resolver is still silent.
o.Confirm = true
// Nothing but silence: held for the next run, which raises it if the resolver is still silent —
// unless every try of every question was refused. A refusal is an answer from the machine:
// nothing listens there. It is not a loaded resolver slow to answer (issue 277), and holding it
// for the next run cost the control node's resolver five minutes on 2026-10-09 (issue 348).
o.Confirm = !v.refused
o.Summary = fmt.Sprintf("the mesh's resolver on %s does not answer: %d of the %d question(s) about the "+
"machines' names went unanswered, each asked %d times — the first, %s", node, len(v.unanswered), v.asked,
resolverTries, v.unanswered[0])
@@ -341,6 +348,19 @@ type resolverAsked struct {
errs []error
}
// allRefused says every try was refused: the resolver's machine said nothing listens on its port.
func (a resolverAsked) allRefused() bool {
if a.answered || len(a.errs) == 0 {
return false
}
for _, err := range a.errs {
if !errors.Is(err, syscall.ECONNREFUSED) {
return false
}
}
return true
}
// askResolverPatiently asks one question until it is answered as right says, at most resolverTries
// times. A wrong answer is asked again too — a resolver restarting may say NXDOMAIN for a moment — and
// stands if no try answers rightly.
@@ -1040,12 +1060,7 @@ func probeCoreBuilds(ctx context.Context, d *doctor) ([]conditions.Observation,
return nil, err
}
hostVersions := deliveredVersions(shelf[hostModule])
rolling := map[string]bool{}
for _, p := range plans {
for m := range p.Modules {
rolling[m] = true
}
}
rolling := rollingModules(plans)
nodes, err := inv.Nodes(ctx)
if err != nil {
return nil, err
@@ -1103,6 +1118,26 @@ func probeCoreBuilds(ctx context.Context, d *doctor) ([]conditions.Observation,
return out, nil
}
// rollingModules is every module an open plan is rolling out: those it keeps a record of, and those its
// tiers name. **A release keeps no record per module** — its walk is per machine, its modules only in its
// tier — so reading the records alone, D10 said "no plan is rolling them out" about the node-engine while
// a release walked it, and the gate on that release's first machine waited on what its own send causes
// (novox/hq issue 348). Pure.
func rollingModules(plans []inventory.Plan) map[string]bool {
rolling := map[string]bool{}
for _, p := range plans {
for m := range p.Modules {
rolling[m] = true
}
for _, tier := range p.Tiers {
for _, m := range tier {
rolling[m] = true
}
}
}
return rolling
}
// deliveredVersions are the versions a module's registered build is delivered as: the last element
// of every resource path under a `versions/` directory, which registration filled from the artifact's
// digest (catalogue `${version}`). The node-engine names itself by that directory.