Caught up is an equality, not an ordering

The report carries the digest of the declaration it applied (mesh-host
8211d8b), and the mesh stores it beside the outcome. `reported` rows in
the status JSON now say `current`: whether the machine's last word
names the declaration last sent.

Not derivable from the timestamps beside it, which is why they were
not enough: an apply begun under the previous declaration reports
after the next send — newer, and still about the old words. The lab
lost exactly that race between one test's closing push and the next
test's opening one.

Empty digests — every host from before reports carried one — read as
not current, which errs toward waiting rather than toward asserting on
files that are not there yet.

cmd/mesh-control/readable.go

@@ -64,6 +64,10 @@ type machineReported struct {
 	Outcome string     `json:"outcome"`
 	At      *time.Time `json:"at,omitempty"`
 	Sent    *time.Time `json:"sent,omitempty"`
+	// Current is whether the last report names the declaration last sent. Not derivable from
+	// the timestamps beside it: an apply begun under the previous declaration reports after the
+	// next send, newer and still about the old words.
+	Current bool `json:"current"`
 }
 
 type machineWaiting struct {
@@ -98,7 +102,7 @@ func statusAsJSON(wrong []inventory.Doing, nodes []inventory.Node, quiet []inven
 		Reported: []machineReported{}}
 	for _, r := range reported {
 		out.Reported = append(out.Reported, machineReported{
-			Node: r.Node, Outcome: r.Outcome, At: r.At, Sent: r.Sent})
+			Node: r.Node, Outcome: r.Outcome, At: r.At, Sent: r.Sent, Current: r.Current})
 	}
 
 	for _, m := range waiting {

internal/inventory/migrations/0018-a-report-names-its-declaration.sql

@@ -0,0 +1,12 @@
+-- Which declaration a machine's report is about.
+--
+-- The mesh decided "has this machine caught up" by comparing its send time to the report's
+-- arrival, and lost the race it invited: an apply started under the previous declaration
+-- finishes after the next one is sent, its report lands newer than the send, and the machine
+-- reads as caught up with words it has not read yet.
+--
+-- The report now carries the digest of the exact bytes it applied -- the same bytes, hashed the
+-- same way, that `node.sent` records. Caught up is `declared = sent`: an equality the mesh
+-- checks rather than an ordering it hopes. Empty means the machine has not said, which is every
+-- report from before this column existed.
+alter table node_report add column declared text not null default '';

internal/inventory/nodes.go

@@ -429,6 +429,9 @@ type Doing struct {
 	Failed  []FailedResource
 	Applied int
 	At      time.Time
+	// Declared is the digest of the declaration the report was about; empty when the machine
+	// did not say.
+	Declared string
 }
 
 // FailedResource is one thing a node could not do.
@@ -451,11 +454,12 @@ func (i *Inventory) RecordDoing(ctx context.Context, node string, d Doing) error
 		return err
 	}
 	_, err = i.store.Pool().Exec(ctx,
-		`insert into node_report (node, outcome, refused, failed, applied, at)
+		`insert into node_report (node, outcome, refused, failed, applied, at, declared)
-		 values ($1, $2, $3, $4, $5, now())
+		 values ($1, $2, $3, $4, $5, now(), $6)
 		 on conflict (node) do update set outcome = excluded.outcome, refused = excluded.refused,
-		   failed = excluded.failed, applied = excluded.applied, at = excluded.at`,
+		   failed = excluded.failed, applied = excluded.applied, at = excluded.at,
-		node, d.Outcome, d.Refused, failed, d.Applied)
+		   declared = excluded.declared`,
+		node, d.Outcome, d.Refused, failed, d.Applied, d.Declared)
 	return err
 }
 
@@ -528,12 +532,17 @@ type Reported struct {
 	At *time.Time
 	// Sent is when the current declaration went to it; nil if nothing ever did.
 	Sent *time.Time
+	// Current is whether the last report names the declaration last sent — the machine has
+	// acted on the current words, not merely spoken after they were written. False also covers
+	// a machine that has not said which, which is every host from before reports carried it.
+	Current bool
 }
 
 // LastReports is every machine's last report beside when it was last sent a declaration.
 func (i *Inventory) LastReports(ctx context.Context) ([]Reported, error) {
 	rows, err := i.store.Pool().Query(ctx,
-		`select n.name, coalesce(r.outcome, ''), r.at, n.sent_at
+		`select n.name, coalesce(r.outcome, ''), r.at, n.sent_at,
+		        r.declared is not null and r.declared <> '' and r.declared = n.sent
 		 from node n left join node_report r on r.node = n.id
 		 order by n.name`)
 	if err != nil {
@@ -543,7 +552,7 @@ func (i *Inventory) LastReports(ctx context.Context) ([]Reported, error) {
 	var out []Reported
 	for rows.Next() {
 		var r Reported
-		if err := rows.Scan(&r.Node, &r.Outcome, &r.At, &r.Sent); err != nil {
+		if err := rows.Scan(&r.Node, &r.Outcome, &r.At, &r.Sent, &r.Current); err != nil {
 			return nil, err
 		}
 		out = append(out, r)

internal/link/enrolment.go

@@ -155,9 +155,10 @@ func (e Enrolment) Heard(ctx context.Context, report Report) error {
 	// last_seen and the reason went to a log line, so "which machine is not doing what it was
 	// told" had no answer the next morning — which is the question a mesh exists to answer.
 	doing := inventory.Doing{
-		Outcome: inventory.OutcomeApplied,
+		Outcome:  inventory.OutcomeApplied,
-		Refused: report.Refused,
+		Refused:  report.Refused,
-		Applied: len(report.Applied),
+		Applied:  len(report.Applied),
+		Declared: report.Declared,
 	}
 	switch {
 	case report.Refused != "":

internal/link/protocol.go

@@ -94,6 +94,9 @@ type Report struct {
 	//
 	// The node states and this context writes, which is the shape of every message here.
 	Carried []int `json:"carried,omitempty"`
+	// Declared is the digest of the declaration this report is about — the same bytes, hashed
+	// the same way, as the `sent` digest the mesh recorded. Which declaration, not when.
+	Declared string `json:"declared,omitempty"`
 }
 
 // EnrolReply is what the mesh says back.