A failure that repeats is said to be stuck

The mesh kept one report per machine, replaced, so a resource nothing can ever apply
looked like a failure that had just happened, every reconcile interval, for ever.
The row now keeps when the current failure began and how many reports in a row have
said it — the same outcome, refusal and failed resources; anything different starts
again and a clean apply clears it. Three make the machine stuck, and status says so
beside the failure, in words and in JSON (novox/hq 04-ISSUES/065, ADR 0090).

internal/inventory/nodes.go

@@ -466,6 +466,15 @@ type Doing struct {
 	// Declared is the digest of the declaration the report was about; empty when the machine
 	// did not say.
 	Declared string
+
+	// Since is when the machine first reported THIS failure — the same outcome, the same refusal,
+	// the same failed resources — and Times is how many reports in a row have said it. A node
+	// re-applies on a steady interval and reports each time (novox/hq ADR 0010), so a failure
+	// that will never succeed arrives as the same report over and over, indistinguishable from
+	// one that just happened until somebody counts (novox/hq 04-ISSUES/065). Nil and zero for a
+	// machine doing what it was told.
+	Since *time.Time
+	Times int
 }
 
 // FailedResource is one thing a node could not do.
@@ -477,23 +486,54 @@ type FailedResource struct {
 // Wrong reports whether this machine needs somebody to look at it.
 func (d Doing) Wrong() bool { return d.Outcome != OutcomeApplied }
 
+// StuckAfter is how many identical reports in a row make a failure one that will not fix itself.
+//
+// Three, because a node reports after every apply and applies on its reconcile interval: one
+// failure is an event, two may be the same event still under way, three separate applies saying
+// the same words is a machine looping on something that is not going to change (novox/hq
+// 04-ISSUES/065). Not a duration: a laptop that was shut for a week has had one attempt.
+const StuckAfter = 3
+
+// Stuck reports whether this machine has been failing the same way for long enough that waiting
+// is no longer a plan. The failure is still the host's own words; this only says it is not new.
+func (d Doing) Stuck() bool { return d.Wrong() && d.Times >= StuckAfter }
+
 // RecordDoing keeps what a node said it did.
 //
 // One row per node, replaced. The question is the machine's current state — "this failed an hour
 // ago and then succeeded" is not something anybody needs to look at, and a table of every report
 // would bury the ones that matter.
+//
+// **What the row also keeps is whether this failure is the one before.** The same outcome, the
+// same refusal, the same failed resources with the same words: then the failure did not just
+// happen, it is still happening, and the row keeps when it began and counts one more report. Any
+// difference starts again — a machine failing on a new resource is a new situation, not a longer
+// one — and a clean apply clears both (novox/hq 04-ISSUES/065).
 func (i *Inventory) RecordDoing(ctx context.Context, node string, d Doing) error {
 	failed, err := json.Marshal(d.Failed)
 	if err != nil {
 		return err
 	}
 	_, err = i.store.Pool().Exec(ctx,
-		`insert into node_report (node, outcome, refused, failed, applied, at, declared)
-		 values ($1, $2, $3, $4, $5, now(), $6)
+		`insert into node_report (node, outcome, refused, failed, applied, at, declared,
+		                          failing_since, failures)
+		 values ($1, $2, $3, $4, $5, now(), $6,
+		         case when $2 <> $7 then now() end,
+		         case when $2 <> $7 then 1 else 0 end)
 		 on conflict (node) do update set outcome = excluded.outcome, refused = excluded.refused,
 		   failed = excluded.failed, applied = excluded.applied, at = excluded.at,
-		   declared = excluded.declared`,
-		node, d.Outcome, d.Refused, failed, d.Applied, d.Declared)
+		   declared = excluded.declared,
+		   failing_since = case
+		     when excluded.outcome = $7 then null
+		     when node_report.outcome = excluded.outcome and node_report.refused = excluded.refused
+		          and node_report.failed = excluded.failed then node_report.failing_since
+		     else excluded.at end,
+		   failures = case
+		     when excluded.outcome = $7 then 0
+		     when node_report.outcome = excluded.outcome and node_report.refused = excluded.refused
+		          and node_report.failed = excluded.failed then node_report.failures + 1
+		     else 1 end`,
+		node, d.Outcome, d.Refused, failed, d.Applied, d.Declared, OutcomeApplied)
 	return err
 }
 
@@ -505,7 +545,7 @@ func (i *Inventory) RecordDoing(ctx context.Context, node string, d Doing) error
 // from.
 func (i *Inventory) NotDoingWhatTheyWereTold(ctx context.Context) ([]Doing, error) {
 	rows, err := i.store.Pool().Query(ctx,
-		`select n.name, r.outcome, r.refused, r.failed, r.applied, r.at
+		`select n.name, r.outcome, r.refused, r.failed, r.applied, r.at, r.failing_since, r.failures
 		 from node_report r join node n on n.id = r.node
 		 where r.outcome <> $1 order by r.at desc`, OutcomeApplied)
 	if err != nil {
@@ -517,7 +557,8 @@ func (i *Inventory) NotDoingWhatTheyWereTold(ctx context.Context) ([]Doing, erro
 	for rows.Next() {
 		var d Doing
 		var failed []byte
-		if err := rows.Scan(&d.Node, &d.Outcome, &d.Refused, &failed, &d.Applied, &d.At); err != nil {
+		if err := rows.Scan(&d.Node, &d.Outcome, &d.Refused, &failed, &d.Applied, &d.At,
+			&d.Since, &d.Times); err != nil {
 			return nil, err
 		}
 		if err := json.Unmarshal(failed, &d.Failed); err != nil {
@@ -537,8 +578,9 @@ func (i *Inventory) DoingOf(ctx context.Context, name string) (Doing, bool, erro
 	var d Doing
 	var failed []byte
 	err = i.store.Pool().QueryRow(ctx,
-		`select outcome, refused, failed, applied, at from node_report where node = $1`, node.ID).
-		Scan(&d.Outcome, &d.Refused, &failed, &d.Applied, &d.At)
+		`select outcome, refused, failed, applied, at, failing_since, failures
+		 from node_report where node = $1`, node.ID).
+		Scan(&d.Outcome, &d.Refused, &failed, &d.Applied, &d.At, &d.Since, &d.Times)
 	if errors.Is(err, pgx.ErrNoRows) {
 		return Doing{}, false, nil
 	}