Look twice before saying a probe failed, and say conditions in machine names (hq issue 277)

D2 raised a resolver urgent on one query that timed out while its machine was
loaded, and its summary carried the resolver's address and socket text, so the
operator channel withheld the whole alert.

- D2 asks every question up to three times, all at once; a resolver that
  answers nothing is held for the next run and raised urgent when two runs
  in a row find it silent. A wrong answer is still raised at once.
- Findings a single look can be wrong about carry Confirm: raised on the
  second look in a row, kept while open, never cleared-and-reraised. Used by
  D2 silence, D3 (also asks discovery twice), D6 behind, D9, D13 unmeasured,
  probe-failed of the doctor, and blind watchdog rows.
- Probe seat asks (D8, D13) are asked again when the bus brought no answer.
- Summaries name machines and say things in words; addresses, paths,
  domains and raw errors move to the evidence (D2, D5, D8, D9, D13, S12).
- internal/outward mirrors the messenger's content rule, allowing the mesh's
  machine names; the keeper rewords a summary that would be withheld and keeps
  it whole in the evidence; a TestMain lint fails the suite on any raised or
  linted finding that would be withheld.

cmd/mesh-controller/doctor.go

@@ -120,8 +120,11 @@ type probeVerdict struct {
 	ID      string   `json:"id"`
 	Verdict string   `json:"verdict"`
 	Found   []string `json:"found,omitempty"`
-	Error   string   `json:"error,omitempty"`
-	Took    string   `json:"took,omitempty"`
+	// Unconfirmed are findings one look can be wrong about, seen by this run and not the one before:
+	// raised if the next run sees them too (confirm.go). Not a pass, and not yet a condition.
+	Unconfirmed []string `json:"unconfirmed,omitempty"`
+	Error       string   `json:"error,omitempty"`
+	Took        string   `json:"took,omitempty"`
 }
 
 // doctorCounts are a run's verdicts, counted.
@@ -156,6 +159,8 @@ type doctor struct {
 	teller    conditions.Teller
 	watchdogs *watchdogs
 	host      string
+	// confirm holds back what one run alone saw of a finding a single look can be wrong about.
+	confirm confirming
 
 	running sync.Mutex
 	mu      sync.Mutex
@@ -267,21 +272,27 @@ func (d *doctor) runOnce(ctx context.Context, why string) doctorRun {
 		case r.err != nil:
 			v.Verdict, v.Error = verdictFailedToRun, r.err.Error()
 			run.Counts.FailedToRun++
+			// A probe that could not run once — a question timed out on a loaded machine — is said in
+			// the verdict at once, and raised as a condition when the next run cannot run it either.
 			blind = append(blind, conditions.Observation{Scope: conditions.ScopeProbe, ID: p.ID, Kind: "probe-failed",
-				Token: "failed", Severity: conditions.Warning,
+				Token: "failed", Severity: conditions.Warning, Confirm: true,
 				Summary: fmt.Sprintf("the probe %s (%s) could not run: what it checks is not known — never a pass", p.ID, p.Asserts),
 				Said:    firstLine(r.err.Error())})
 		default:
-			if err := d.keeper.Reconcile(ctx, p.ID, kindedAs(r.obs, p.Kind)); err != nil {
+			raise, held := d.confirm.pass(ctx, d.keeper, p.ID, kindedAs(r.obs, p.Kind))
+			if err := d.keeper.Reconcile(ctx, p.ID, raise); err != nil {
 				v.Error = "what it found could not be kept: " + err.Error()
 			}
-			if len(r.obs) == 0 {
+			for _, o := range held {
+				v.Unconfirmed = append(v.Unconfirmed, o.Summary)
+			}
+			if len(raise) == 0 {
 				v.Verdict = verdictPass
 				run.Counts.Passed++
 			} else {
 				v.Verdict = verdictFail
 				run.Counts.Failed++
-				for _, o := range r.obs {
+				for _, o := range raise {
 					v.Found = append(v.Found, o.Summary)
 				}
 			}
@@ -291,6 +302,7 @@ func (d *doctor) runOnce(ctx context.Context, why string) doctorRun {
 		}
 		run.Probes = append(run.Probes, v)
 	}
+	blind, _ = d.confirm.pass(ctx, d.keeper, sourceDoctor, blind)
 	if err := d.keeper.Reconcile(ctx, sourceDoctor, blind); err != nil {
 		fmt.Printf("the self-check's own failures could not be kept: %v\n", err)
 	}
@@ -406,7 +418,9 @@ func doctorAnswer(ctx context.Context, sub string) (any, error) {
 // verdictAnswer is a run as the verb answers it, with its age.
 func verdictAnswer(run doctorRun, now time.Time) map[string]any {
 	return map[string]any{"run": run, "age": now.Sub(run.At).Round(time.Second).String(),
-		"note": "a probe that could not run is never a pass; each failure is an open condition until a run passes it"}
+		"note": "a probe that could not run is never a pass; each failure is an open condition until a run passes it, " +
+			"and one a single look can be wrong about — an unanswered question, a slow answer — is raised when two " +
+			"runs in a row see it"}
 }
 
 // probesAnswer is the registry.
@@ -472,6 +486,9 @@ func doctorText(answer any) string {
 			for _, f := range p.Found {
 				fmt.Fprintf(&b, "         %s\n", f)
 			}
+			for _, f := range p.Unconfirmed {
+				fmt.Fprintf(&b, "         unconfirmed, raised if the next run sees it too: %s\n", f)
+			}
 			if p.Error != "" {
 				fmt.Fprintf(&b, "         %s\n", p.Error)
 			}