Status says when each machine last reported, beside when it was sent

"Not waiting" says the declaration is current, not that the machine
finished applying it: the sent digest is recorded at send. So a test
that pushed, saw waiting clear, and asked the machine what it was
running found containers that did not exist yet — the certificate fix
made compositions stable, and the settling that used to fail first had
been hiding the gap behind it.

The mesh already held the missing half: every machine's last report,
with its time. It just was not in the JSON. `reported` now sets each
machine's last word beside when the current declaration went to it, and
"has it caught up" becomes a comparison of two timestamps the mesh
recorded itself — a report newer than the send means the machine acted
on what was sent; older means it is still working, which waiting alone
cannot distinguish.

cmd/mesh-control/board.go

@@ -95,7 +95,7 @@ func board() http.Handler {
 			return
 		}
 		body, err := statusAsJSON(asked.wrong, asked.nodes, asked.quiet, asked.behind, asked.sources,
-			asked.waiting)
+			asked.waiting, asked.reported)
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return

cmd/mesh-control/build.go

@@ -438,4 +438,8 @@ type answers struct {
 	sources map[string]inventory.Source
 	// waiting is every machine not running what the mesh would send it.
 	waiting []inventory.Machine
+	// reported is every machine's last word beside when it was last sent a declaration — the
+	// pair that answers "has it caught up", which waiting alone cannot (the sent digest is
+	// recorded at send, not at apply).
+	reported []inventory.Reported
 }

cmd/mesh-control/readable.go

@@ -35,6 +35,11 @@ type meshStatus struct {
 	// Behind one level down: that says the catalogue is old, this says a machine is — and only
 	// this one has somebody's change waiting inside it.
 	Waiting []machineWaiting `json:"waiting"`
+	// Reported is every machine's last word beside when it was last sent a declaration. A
+	// machine whose report is newer than its send has acted on the current declaration; one
+	// whose is older is still working — and Waiting cannot tell those apart, because the sent
+	// digest is recorded at send, not at apply.
+	Reported []machineReported `json:"reported"`
 	// Machines is how many the mesh knows about, so a reader can tell "none wrong" from
 	// "none at all".
 	Machines int `json:"machines"`
@@ -54,6 +59,13 @@ type machineDoing struct {
 	At      time.Time `json:"at"`
 }
 
+type machineReported struct {
+	Node    string     `json:"node"`
+	Outcome string     `json:"outcome"`
+	At      *time.Time `json:"at,omitempty"`
+	Sent    *time.Time `json:"sent,omitempty"`
+}
+
 type machineWaiting struct {
 	Node string `json:"node"`
 	// Never is true when nothing has ever been sent to it. Not out of date: nobody has ever asked
@@ -79,10 +91,15 @@ type moduleBehind struct {
 // statusAsJSON answers the same three questions as the text form, from the same calls.
 func statusAsJSON(wrong []inventory.Doing, nodes []inventory.Node, quiet []inventory.Node,
 	behind map[string][]string, sources map[string]inventory.Source,
-	waiting []inventory.Machine) ([]byte, error) {
+	waiting []inventory.Machine, reported []inventory.Reported) ([]byte, error) {
 
 	out := meshStatus{Machines: len(nodes), Wrong: []machineDoing{},
-		Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{}}
+		Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{},
+		Reported: []machineReported{}}
+	for _, r := range reported {
+		out.Reported = append(out.Reported, machineReported{
+			Node: r.Node, Outcome: r.Outcome, At: r.At, Sent: r.Sent})
+	}
 
 	for _, m := range waiting {
 		out.Waiting = append(out.Waiting, machineWaiting{

cmd/mesh-control/readable_test.go

@@ -17,7 +17,7 @@ import (
 func statusOf(t *testing.T, wrong []inventory.Doing, nodes, quiet []inventory.Node,
 	behind map[string][]string, sources map[string]inventory.Source) map[string]any {
 	t.Helper()
-	body, err := statusAsJSON(wrong, nodes, quiet, behind, sources, nil)
+	body, err := statusAsJSON(wrong, nodes, quiet, behind, sources, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -126,7 +126,7 @@ func TestNoSecretIsInWhatABoardReads(t *testing.T) {
 	body, err := statusAsJSON(
 		[]inventory.Doing{{Node: "a", Outcome: inventory.OutcomeRefused,
 			Refused: "resource \"x\": a file needs a path"}},
-		[]inventory.Node{{Name: "a"}}, nil, nil, nil, nil)
+		[]inventory.Node{{Name: "a"}}, nil, nil, nil, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/mesh-control/status.go

@@ -54,7 +54,8 @@ func statusCommand(ctx context.Context, args []string) error {
 	behind, sources := asked.behind, asked.sources
 
 	if *asJSON {
-		body, err := statusAsJSON(wrong, nodes, quiet, behind, sources, asked.waiting)
+		body, err := statusAsJSON(wrong, nodes, quiet, behind, sources, asked.waiting,
+			asked.reported)
 		if err != nil {
 			return err
 		}
@@ -207,6 +208,10 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
 	if err != nil {
 		return answers{}, err
 	}
+	out.reported, err = inv.LastReports(ctx)
+	if err != nil {
+		return answers{}, err
+	}
 	out.sources = map[string]inventory.Source{}
 	for module := range out.behind {
 		from, err := inv.SourceOf(ctx, module)

internal/inventory/nodes.go

@@ -514,6 +514,43 @@ func (i *Inventory) DoingOf(ctx context.Context, name string) (Doing, bool, erro
 	return d, true, nil
 }
 
+// Reported is one machine's last word set beside what was last asked of it.
+//
+// The pair is what makes "has it caught up" answerable: a machine that reported *after* it was
+// sent the current declaration has acted on it; one that reported before is still working, or
+// has not started — and "not waiting" alone cannot tell those apart, because the sent digest is
+// recorded at send, not at apply. The lab asserted on a machine the moment its declaration was
+// current and found containers that did not exist yet.
+type Reported struct {
+	Node    string
+	Outcome string
+	// At is when it last reported; nil if it never has.
+	At *time.Time
+	// Sent is when the current declaration went to it; nil if nothing ever did.
+	Sent *time.Time
+}
+
+// LastReports is every machine's last report beside when it was last sent a declaration.
+func (i *Inventory) LastReports(ctx context.Context) ([]Reported, error) {
+	rows, err := i.store.Pool().Query(ctx,
+		`select n.name, coalesce(r.outcome, ''), r.at, n.sent_at
+		 from node n left join node_report r on r.node = n.id
+		 order by n.name`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []Reported
+	for rows.Next() {
+		var r Reported
+		if err := rows.Scan(&r.Node, &r.Outcome, &r.At, &r.Sent); err != nil {
+			return nil, err
+		}
+		out = append(out, r)
+	}
+	return out, rows.Err()
+}
+
 // RecordSent keeps a digest of the declaration a machine was last sent.
 //
 // **A digest rather than the declaration.** The mesh can compute what a machine should be at any