Judge apply-failing by the client's failing-since, so a failure said in new words is measured from when it began (issue 774)

mesh-provisioner-client's report now carries failing-since (mesh-catalog 3aae6c08), kept while a module
stays failed however its words change. S21 measures a failed placing from it, falling back to the
report's time from an older client; the hysteresis of 6ffcea4f (an open condition stays while any
failure stands) is kept. This is the thorough fix that commit named.

Research 072 piece: milestone 3 (health leaves the controller): the judgement reads the owner's state.

cmd/mesh-controller/apply_failing.go

@@ -57,7 +57,7 @@ func watchApplyFailing(f *signalFacts) []conditions.Observation {
 	failed := map[string][]inventory.ProvReport{}
 	for _, r := range f.provFailed {
 		if r.Status != inventory.ProvFailed || !heard[r.Node] || said[r.Node] ||
-			(f.now.Sub(r.At) <= applyEvery && !f.applyFailingOpen[r.Node]) {
+			(f.now.Sub(r.Since()) <= applyEvery && !f.applyFailingOpen[r.Node]) {
 			continue
 		}
 		failed[r.Node] = append(failed[r.Node], r)
@@ -84,12 +84,12 @@ func applyFailingOpen(open []conditions.Condition) map[string]bool {
 func provisioningFailing(m string, failed []inventory.ProvReport, now time.Time) conditions.Observation {
 	sort.Slice(failed, func(a, b int) bool { return failed[a].Module < failed[b].Module })
 	var names, words []string
-	since := failed[0].At
+	since := failed[0].Since()
 	for _, r := range failed {
 		names = append(names, r.Module)
 		words = append(words, r.Module+": "+oneLine(orNoReason(r.Reason)))
-		if r.At.Before(since) {
+		if r.Since().Before(since) {
-			since = r.At
+			since = r.Since()
 		}
 	}
 	o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m,

cmd/mesh-controller/provisioned_test.go

@@ -84,12 +84,14 @@ func TestApplyFailingStaysRaisedWhileAnyFailureStands(t *testing.T) {
 		prov   string
 		doing  bool
 		raised bool
+		since  time.Duration // the client's failing-since, before now; zero when it says none
 	}{
-		"a fresh placing failure, nothing open":   {false, inventory.ProvFailed, false, false},
+		"a fresh placing failure, nothing open":                     {false, inventory.ProvFailed, false, false, 0},
-		"a fresh placing failure, condition open": {true, inventory.ProvFailed, false, true},
+		"a fresh placing failure, condition open":                   {true, inventory.ProvFailed, false, true, 0},
-		"placing applied again, condition open":   {true, inventory.ProvApplied, false, false},
+		"placing applied again, condition open":                     {true, inventory.ProvApplied, false, false, 0},
-		"a fresh failed apply, condition open":    {true, "", true, true},
+		"a fresh failed apply, condition open":                      {true, "", true, true, 0},
-		"a fresh failed apply, nothing open":      {false, "", true, false},
+		"a fresh failed apply, nothing open":                        {false, "", true, false, 0},
+		"new words of a failure standing ten minutes, nothing open": {false, inventory.ProvFailed, false, true, 10 * time.Minute},
 	} {
 		f := calm(now)
 		if c.open {
@@ -99,6 +101,9 @@ func TestApplyFailingStaysRaisedWhileAnyFailureStands(t *testing.T) {
 		if c.prov != "" {
 			f.provFailed = []inventory.ProvReport{{Node: "anchor", Module: "postgres", Status: c.prov,
 				Reason: "the vault refused it", At: now.Add(-time.Minute)}}
+			if c.since > 0 {
+				f.provFailed[0].FailingSince = now.Add(-c.since)
+			}
 		}
 		if c.doing {
 			f.doing = []inventory.Doing{{Node: "anchor", Outcome: inventory.OutcomeRefused, Refused: "no", At: since,

internal/inventory/provisioned.go

@@ -26,6 +26,17 @@ type ProvReport struct {
 	WaitsFor  []string  `json:"waits-for,omitempty"`
 	Templates string    `json:"templates,omitempty"`
 	At        time.Time `json:"at"`
+	// FailingSince is, on a failed report, when the module's standing failure began, kept by the client while only its
+	// words change (mesh-catalog 3aae6c08); zero from an older client.
+	FailingSince time.Time `json:"failing-since,omitzero"`
+}
+
+// Since is when a failed report's failure began: FailingSince when the client says it, else At.
+func (r ProvReport) Since() time.Time {
+	if !r.FailingSince.IsZero() {
+		return r.FailingSince
+	}
+	return r.At
 }
 
 // The words a client's report says.