Look twice before saying a probe failed, and say conditions in machine names (hq issue 277)

D2 raised a resolver urgent on one query that timed out while its machine was
loaded, and its summary carried the resolver's address and socket text, so the
operator channel withheld the whole alert.

- D2 asks every question up to three times, all at once; a resolver that
  answers nothing is held for the next run and raised urgent when two runs
  in a row find it silent. A wrong answer is still raised at once.
- Findings a single look can be wrong about carry Confirm: raised on the
  second look in a row, kept while open, never cleared-and-reraised. Used by
  D2 silence, D3 (also asks discovery twice), D6 behind, D9, D13 unmeasured,
  probe-failed of the doctor, and blind watchdog rows.
- Probe seat asks (D8, D13) are asked again when the bus brought no answer.
- Summaries name machines and say things in words; addresses, paths,
  domains and raw errors move to the evidence (D2, D5, D8, D9, D13, S12).
- internal/outward mirrors the messenger's content rule, allowing the mesh's
  machine names; the keeper rewords a summary that would be withheld and keeps
  it whole in the evidence; a TestMain lint fails the suite on any raised or
  linted finding that would be withheld.

cmd/mesh-controller/data.go

@@ -220,9 +220,9 @@ func probeData(ctx context.Context, d *doctor) ([]conditions.Observation, error)
 	for _, m := range machines {
 		if m.askErr != nil {
 			out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name, Token: kindDataUnmeasured,
-				Kind: kindDataUnmeasured, Machine: m.name, Severity: conditions.Warning,
+				Kind: kindDataUnmeasured, Machine: m.name, Severity: conditions.Warning, Confirm: true,
 				Summary: fmt.Sprintf("%s's backup holder did not say what it measured of the data declared there, so "+
-					"nothing about that data is known this run: %s", m.name, firstLine(m.askErr.Error())),
+					"nothing about that data is known", m.name),
 				Said: firstLine(m.askErr.Error())})
 		}
 		why := fmt.Sprintf("no longer declared on %s: its module was unassigned there, or is no longer pulled in", m.name)
@@ -407,10 +407,11 @@ func dataFindings(records []inventory.DataRecord, peaks map[string]int64, shelf
 			if now.Sub(*r.RetiredAt) > cleanupAfter {
 				out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: "cleanup",
 					Kind: kindCleanupWaiting, Machine: r.Machine, Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
-					Summary: fmt.Sprintf("%s of %s on %s (%s, %s) has been retired %d days — kept at %s since %s; `cleanup "+
-						"delete %s %s %s --why …` once a person has decided, or assign %s there again",
+					Summary: fmt.Sprintf("%s of %s on %s (%s, %s) has been retired %d days — kept where it was since %s; "+
+						"`cleanup delete %s %s %s --why …` once a person has decided, or assign %s there again",
 						r.Item, r.Module, r.Machine, r.Class, sizeWords(r.Size), int(now.Sub(*r.RetiredAt).Hours()/24),
-						orUnknownPath(r.Path), r.RetiredWhy, r.Machine, r.Module, r.Item, r.Module)})
+						r.RetiredWhy, r.Machine, r.Module, r.Item, r.Module),
+					Said: "kept at " + orUnknownPath(r.Path)})
 			}
 			continue
 		}
@@ -425,14 +426,16 @@ func dataFindings(records []inventory.DataRecord, peaks map[string]int64, shelf
 			out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindProtectionMissing,
 				Kind: kindProtectionMissing, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator,
 				Summary: fmt.Sprintf("%s of %s on %s (%s) is said to be protected by the redundancy of the storage it is on, "+
-					"and %s is on nothing the backup holder can read as redundant: it has no protection the mesh can see",
-					r.Item, r.Module, r.Machine, class, orUnknownPath(r.Path))})
+					"and it is on nothing the backup holder can read as redundant: it has no protection the mesh can see",
+					r.Item, r.Module, r.Machine, class),
+				Said: orUnknownPath(r.Path) + " is on no redundant storage the backup holder can read"})
 		}
 		if r.MeasureError != "" && strings.Contains(r.MeasureError, "does not exist") {
 			out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataMissing,
 				Kind: kindDataMissing, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator,
-				Summary: fmt.Sprintf("%s of %s on %s (%s) is gone: %s does not exist any more", r.Item, r.Module, r.Machine,
-					class, orUnknownPath(r.Path))})
+				Summary: fmt.Sprintf("%s of %s on %s (%s) is gone: where it was declared, nothing exists any more", r.Item,
+					r.Module, r.Machine, class),
+				Said: orUnknownPath(r.Path) + " does not exist: " + firstLine(r.MeasureError)})
 		} else if peak, ok := peaks[r.Key()]; ok && r.Size != nil && inventory.Comparable(r.Precision) &&
 			*r.Size*2 < peak && peak-*r.Size >= shrinkFloor && inventory.Dataset(r.Precision) != "" {
 			// Several items on one dataset share its size: one condition for the dataset, as loud as the
@@ -486,9 +489,10 @@ func dataFindings(records []inventory.DataRecord, peaks map[string]int64, shelf
 		out = append(out, conditions.Observation{Scope: conditions.ScopeMachine,
 			ID: ds.machine + ".dataset." + strings.ReplaceAll(ds.dataset, "/", "-"), Token: kindDataShrank,
 			Kind: kindDataShrank, Machine: ds.machine, Severity: severityOf(ds.class), Resolver: conditions.ResolverOperator,
-			Summary: fmt.Sprintf("the dataset %s on %s shrank to %s from %s within %d days — more than half of what it held "+
-				"is gone; it holds %s", ds.dataset, ds.machine, sizeWords(&ds.size), sizeWords(&ds.peak),
-				int(shrinkWindow.Hours()/24), strings.Join(ds.items, ", "))})
+			Summary: fmt.Sprintf("the dataset on %s that holds %s shrank to %s from %s within %d days — more than half of "+
+				"what it held is gone", ds.machine, strings.Join(ds.items, ", "), sizeWords(&ds.size), sizeWords(&ds.peak),
+				int(shrinkWindow.Hours()/24)),
+			Said: "the dataset " + ds.dataset})
 	}
 	// The redundant storage watched data is on: one condition per array, as loud as the most precious
 	// item on it — the array, not each item, is what degrades.
@@ -512,8 +516,9 @@ func dataFindings(records []inventory.DataRecord, peaks map[string]int64, shelf
 			ID:    machine + ".array." + strings.NewReplacer("/", "-", ":", "-").Replace(red.Kind+"-"+red.Where),
 			Token: kindArrayDegraded, Kind: kindArrayDegraded, Machine: machine, Severity: severityOf(class),
 			Resolver: conditions.ResolverOperator,
-			Summary: fmt.Sprintf("the %s storage %s on %s %s: %s — and it is what protects %s", red.Kind, red.Where, machine,
-				state, firstLine(red.Said), strings.Join(names, ", "))})
+			Summary: fmt.Sprintf("the %s storage on %s that protects %s %s", red.Kind, machine, strings.Join(names, ", "),
+				state),
+			Said: fmt.Sprintf("the %s storage %s %s: %s", red.Kind, red.Where, state, firstLine(red.Said))})
 	}
 	// The same item on several machines: a copy that is in use and far smaller than one kept elsewhere is
 	// an empty replacement. Only against a retired copy — a module running on two machines on purpose