Judge a send by when a fault began, not when it was last raised (hq issue 348)
On 2026-10-09 the control node's resolver refused from 10:57:57 UTC. A node-engine restarted by the 10:59:34 send said its names undecided, that statement cleared the network condition, the next look raised it again after the send, and the gate put back two builds for a fault older than them. - A condition keeps First across a reopening; the gate reads Began. - An undecided network statement (unknown, starting) clears nothing. - D10 counts what a release's tier names as rolling, so a walked node-engine is not core-behind on its own first machine. - D2 raises a resolver that refuses every try at once: a refusal is an answer, not a loaded resolver (issue 277).
This commit is contained in:
@@ -11,6 +11,7 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
@@ -216,6 +217,8 @@ func askEveryResolver(ctx context.Context, resolvers map[string]string, places [
|
||||
type verdict struct {
|
||||
wrong, unanswered, said []string
|
||||
asked int
|
||||
// refused says every try of every unanswered question was refused (nothing listens there).
|
||||
refused bool
|
||||
}
|
||||
byHolder := map[string]*verdict{}
|
||||
for _, q := range questions {
|
||||
@@ -232,6 +235,7 @@ func askEveryResolver(ctx context.Context, resolvers map[string]string, places [
|
||||
v.wrong = append(v.wrong, words)
|
||||
v.said = append(v.said, said)
|
||||
default:
|
||||
v.refused = (len(v.unanswered) == 0 || v.refused) && q.asked.allRefused()
|
||||
v.unanswered = append(v.unanswered, words)
|
||||
v.said = append(v.said, said)
|
||||
}
|
||||
@@ -251,8 +255,11 @@ func askEveryResolver(ctx context.Context, resolvers map[string]string, places [
|
||||
o.Summary = fmt.Sprintf("the mesh's resolver on %s does not answer machine names as it must: %s%s", node,
|
||||
v.wrong[0], andMore(len(v.wrong)-1))
|
||||
} else {
|
||||
// Nothing but silence: held for the next run, which raises it if the resolver is still silent.
|
||||
o.Confirm = true
|
||||
// Nothing but silence: held for the next run, which raises it if the resolver is still silent —
|
||||
// unless every try of every question was refused. A refusal is an answer from the machine:
|
||||
// nothing listens there. It is not a loaded resolver slow to answer (issue 277), and holding it
|
||||
// for the next run cost the control node's resolver five minutes on 2026-10-09 (issue 348).
|
||||
o.Confirm = !v.refused
|
||||
o.Summary = fmt.Sprintf("the mesh's resolver on %s does not answer: %d of the %d question(s) about the "+
|
||||
"machines' names went unanswered, each asked %d times — the first, %s", node, len(v.unanswered), v.asked,
|
||||
resolverTries, v.unanswered[0])
|
||||
@@ -341,6 +348,19 @@ type resolverAsked struct {
|
||||
errs []error
|
||||
}
|
||||
|
||||
// allRefused says every try was refused: the resolver's machine said nothing listens on its port.
|
||||
func (a resolverAsked) allRefused() bool {
|
||||
if a.answered || len(a.errs) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, err := range a.errs {
|
||||
if !errors.Is(err, syscall.ECONNREFUSED) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// askResolverPatiently asks one question until it is answered as right says, at most resolverTries
|
||||
// times. A wrong answer is asked again too — a resolver restarting may say NXDOMAIN for a moment — and
|
||||
// stands if no try answers rightly.
|
||||
@@ -1040,12 +1060,7 @@ func probeCoreBuilds(ctx context.Context, d *doctor) ([]conditions.Observation,
|
||||
return nil, err
|
||||
}
|
||||
hostVersions := deliveredVersions(shelf[hostModule])
|
||||
rolling := map[string]bool{}
|
||||
for _, p := range plans {
|
||||
for m := range p.Modules {
|
||||
rolling[m] = true
|
||||
}
|
||||
}
|
||||
rolling := rollingModules(plans)
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1103,6 +1118,26 @@ func probeCoreBuilds(ctx context.Context, d *doctor) ([]conditions.Observation,
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// rollingModules is every module an open plan is rolling out: those it keeps a record of, and those its
|
||||
// tiers name. **A release keeps no record per module** — its walk is per machine, its modules only in its
|
||||
// tier — so reading the records alone, D10 said "no plan is rolling them out" about the node-engine while
|
||||
// a release walked it, and the gate on that release's first machine waited on what its own send causes
|
||||
// (novox/hq issue 348). Pure.
|
||||
func rollingModules(plans []inventory.Plan) map[string]bool {
|
||||
rolling := map[string]bool{}
|
||||
for _, p := range plans {
|
||||
for m := range p.Modules {
|
||||
rolling[m] = true
|
||||
}
|
||||
for _, tier := range p.Tiers {
|
||||
for _, m := range tier {
|
||||
rolling[m] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return rolling
|
||||
}
|
||||
|
||||
// deliveredVersions are the versions a module's registered build is delivered as: the last element
|
||||
// of every resource path under a `versions/` directory, which registration filled from the artifact's
|
||||
// digest (catalogue `${version}`). The node-engine names itself by that directory.
|
||||
|
||||
Reference in New Issue
Block a user