Page an answer larger than one message of the bus, and keep overviews brief (hq issue 314)
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery delivered
mesh/delivery-group group fix/314-a-large-answer-is-paged-not-lost delivered: every member is delivered

The client library refuses to send a reply over the bus's max_payload, and the
controller only logged it: conditions and status answered nobody for hours on
2026-10-08 while calls said each was answered in 130 ms, and the operator's
channel read nothing. An answer too large is now held under its call and paged
to the caller that asks, on the same subject; a caller that does not page is
told in words, and calls says it. The overviews no longer carry every finding:
conditions and status list each condition with its newest evidence, doctor at
most twenty findings a probe (probe= gives one whole), and the JSON overviews
are sent once, as data, instead of twice.
This commit is contained in:
jochen
2026-10-08 12:19:16 +02:00
parent 4d337ff897
commit 175b28ee42
12 changed files with 793 additions and 27 deletions
+71 -8
View File
@@ -151,6 +151,50 @@ type probeVerdict struct {
Unconfirmed []string `json:"unconfirmed,omitempty"`
Error string `json:"error,omitempty"`
Took string `json:"took,omitempty"`
// FoundCount and More are an overview's: how many findings the probe had when the overview shows
// fewer, and how to read them all (novox/hq issue 314). Never in a heartbeat.
FoundCount int `json:"found-count,omitempty"`
More string `json:"more,omitempty"`
}
// overviewFindings is how many findings of one probe an overview carries. On 2026-10-08 D14 found 689
// stalled deliveries, and an overview carrying every one of them, beside the conditions they raised,
// was more than the bus carries in one message (novox/hq issue 314); one probe's verdict whole is
// `doctor probe=<id>`.
const overviewFindings = 20
// inOverview is a run as an overview carries it: each probe's findings at most overviewFindings, with
// their count and how to read them all. The run itself is not changed.
func inOverview(run doctorRun) doctorRun {
probes := make([]probeVerdict, len(run.Probes))
for i, p := range run.Probes {
total := len(p.Found) + len(p.Unconfirmed)
if total > overviewFindings {
p.FoundCount = len(p.Found)
if len(p.Found) > overviewFindings {
p.Found = p.Found[:overviewFindings:overviewFindings]
}
if room := overviewFindings - len(p.Found); len(p.Unconfirmed) > room {
p.Unconfirmed = p.Unconfirmed[:room:room]
}
p.More = fmt.Sprintf("%d found and %d unconfirmed, %d of them shown — `doctor probe=%s` gives them all",
p.FoundCount, total-p.FoundCount, len(p.Found)+len(p.Unconfirmed), p.ID)
}
probes[i] = p
}
run.Probes = probes
return run
}
// oneProbe is a run with only one probe's verdict, whole.
func oneProbe(run doctorRun, id string) (doctorRun, error) {
for _, p := range run.Probes {
if strings.EqualFold(p.ID, id) {
run.Probes = []probeVerdict{p}
return run, nil
}
}
return doctorRun{}, fmt.Errorf("the run %s has no probe %q — `doctor probes=true` lists them", run.Run, id)
}
// doctorCounts are a run's verdicts, counted.
@@ -386,12 +430,13 @@ func doctorCommand(ctx context.Context, args []string) error {
}
set := flag.NewFlagSet("doctor", flag.ContinueOnError)
asJSON := set.Bool("json", false, "as data")
probe := set.String("probe", "", "one probe's verdict, with every finding")
if rest, err := parseAround(set, args); err != nil {
return err
} else if len(rest) > 0 {
return errors.New("doctor [run|probes|signals] [--json]")
return errors.New("doctor [run|probes|signals] [--probe <id>] [--json]")
}
answer, err := doctorAnswer(ctx, sub)
answer, err := doctorAnswer(ctx, sub, *probe)
if err != nil {
return err
}
@@ -403,12 +448,18 @@ func doctorCommand(ctx context.Context, args []string) error {
}
// doctorAnswer is what the verb answers, as data.
func doctorAnswer(ctx context.Context, sub string) (any, error) {
//
// A verdict is an overview — each probe's findings at most overviewFindings — unless probe names one,
// which is then answered alone and whole (novox/hq issue 314).
func doctorAnswer(ctx context.Context, sub, probe string) (any, error) {
if probe != "" && sub != "" && sub != "run" {
return nil, fmt.Errorf("probe names one probe of a verdict; %s has none", sub)
}
switch sub {
case "":
if doctorFrom != nil {
if run := doctorFrom.lastRun(); run != nil {
return verdictAnswer(*run, time.Now()), nil
return verdictAnswer(*run, time.Now(), probe)
}
return nil, fmt.Errorf("the self-check has not finished its first run yet: it runs %s after the "+
"controller starts, then every %s — `doctor run` runs it now", doctorFirstAfter, doctorEvery)
@@ -417,7 +468,7 @@ func doctorAnswer(ctx context.Context, sub string) (any, error) {
if err != nil {
return nil, err
}
return verdictAnswer(run, time.Now()), nil
return verdictAnswer(run, time.Now(), probe)
case "run":
d := doctorFrom
if d == nil {
@@ -428,7 +479,7 @@ func doctorAnswer(ctx context.Context, sub string) (any, error) {
defer closeIt()
d = local
}
return verdictAnswer(d.runOnce(ctx, "asked by "+link.Caller()), time.Now()), nil
return verdictAnswer(d.runOnce(ctx, "asked by "+link.Caller()), time.Now(), probe)
case "probes":
return probesAnswer(), nil
case "signals":
@@ -442,11 +493,20 @@ func doctorAnswer(ctx context.Context, sub string) (any, error) {
}
// verdictAnswer is a run as the verb answers it, with its age.
func verdictAnswer(run doctorRun, now time.Time) map[string]any {
func verdictAnswer(run doctorRun, now time.Time, probe string) (map[string]any, error) {
if probe != "" {
one, err := oneProbe(run, probe)
if err != nil {
return nil, err
}
run = one
} else {
run = inOverview(run)
}
return map[string]any{"run": run, "age": now.Sub(run.At).Round(time.Second).String(),
"note": "a probe that could not run is never a pass; each failure is an open condition until a run passes it, " +
"and one a single look can be wrong about — an unanswered question, a slow answer — is raised when two " +
"runs in a row see it"}
"runs in a row see it"}, nil
}
// probesAnswer is the registry.
@@ -515,6 +575,9 @@ func doctorText(answer any) string {
for _, f := range p.Unconfirmed {
fmt.Fprintf(&b, " unconfirmed, raised if the next run sees it too: %s\n", f)
}
if p.More != "" {
fmt.Fprintf(&b, " … %s\n", p.More)
}
if p.Error != "" {
fmt.Fprintf(&b, " %s\n", p.Error)
}