From cf4ca555960e5e274e313ca2e5976622a35b9e5c Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 30 Oct 2026 18:35:37 +0100 Subject: [PATCH] Judge apply-failing by the client's failing-since, so a failure said in new words is measured from when it began (issue 774) mesh-provisioner-client's report now carries failing-since (mesh-catalog 3aae6c08), kept while a module stays failed however its words change. S21 measures a failed placing from it, falling back to the report's time from an older client; the hysteresis of 6ffcea4f (an open condition stays while any failure stands) is kept. This is the thorough fix that commit named. Research 072 piece: milestone 3 (health leaves the controller): the judgement reads the owner's state. --- cmd/mesh-controller/apply_failing.go | 8 ++++---- cmd/mesh-controller/provisioned_test.go | 15 ++++++++++----- internal/inventory/provisioned.go | 11 +++++++++++ 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/cmd/mesh-controller/apply_failing.go b/cmd/mesh-controller/apply_failing.go index b9698b10..4fb2eb13 100644 --- a/cmd/mesh-controller/apply_failing.go +++ b/cmd/mesh-controller/apply_failing.go @@ -57,7 +57,7 @@ func watchApplyFailing(f *signalFacts) []conditions.Observation { failed := map[string][]inventory.ProvReport{} for _, r := range f.provFailed { if r.Status != inventory.ProvFailed || !heard[r.Node] || said[r.Node] || - (f.now.Sub(r.At) <= applyEvery && !f.applyFailingOpen[r.Node]) { + (f.now.Sub(r.Since()) <= applyEvery && !f.applyFailingOpen[r.Node]) { continue } failed[r.Node] = append(failed[r.Node], r) @@ -84,12 +84,12 @@ func applyFailingOpen(open []conditions.Condition) map[string]bool { func provisioningFailing(m string, failed []inventory.ProvReport, now time.Time) conditions.Observation { sort.Slice(failed, func(a, b int) bool { return failed[a].Module < failed[b].Module }) var names, words []string - since := failed[0].At + since := failed[0].Since() for _, r := range failed { names = append(names, r.Module) words = append(words, r.Module+": "+oneLine(orNoReason(r.Reason))) - if r.At.Before(since) { - since = r.At + if r.Since().Before(since) { + since = r.Since() } } o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m, diff --git a/cmd/mesh-controller/provisioned_test.go b/cmd/mesh-controller/provisioned_test.go index 67539647..f86e963c 100644 --- a/cmd/mesh-controller/provisioned_test.go +++ b/cmd/mesh-controller/provisioned_test.go @@ -84,12 +84,14 @@ func TestApplyFailingStaysRaisedWhileAnyFailureStands(t *testing.T) { prov string doing bool raised bool + since time.Duration // the client's failing-since, before now; zero when it says none }{ - "a fresh placing failure, nothing open": {false, inventory.ProvFailed, false, false}, - "a fresh placing failure, condition open": {true, inventory.ProvFailed, false, true}, - "placing applied again, condition open": {true, inventory.ProvApplied, false, false}, - "a fresh failed apply, condition open": {true, "", true, true}, - "a fresh failed apply, nothing open": {false, "", true, false}, + "a fresh placing failure, nothing open": {false, inventory.ProvFailed, false, false, 0}, + "a fresh placing failure, condition open": {true, inventory.ProvFailed, false, true, 0}, + "placing applied again, condition open": {true, inventory.ProvApplied, false, false, 0}, + "a fresh failed apply, condition open": {true, "", true, true, 0}, + "a fresh failed apply, nothing open": {false, "", true, false, 0}, + "new words of a failure standing ten minutes, nothing open": {false, inventory.ProvFailed, false, true, 10 * time.Minute}, } { f := calm(now) if c.open { @@ -99,6 +101,9 @@ func TestApplyFailingStaysRaisedWhileAnyFailureStands(t *testing.T) { if c.prov != "" { f.provFailed = []inventory.ProvReport{{Node: "anchor", Module: "postgres", Status: c.prov, Reason: "the vault refused it", At: now.Add(-time.Minute)}} + if c.since > 0 { + f.provFailed[0].FailingSince = now.Add(-c.since) + } } if c.doing { f.doing = []inventory.Doing{{Node: "anchor", Outcome: inventory.OutcomeRefused, Refused: "no", At: since, diff --git a/internal/inventory/provisioned.go b/internal/inventory/provisioned.go index ba2c8f0e..4423a6b2 100644 --- a/internal/inventory/provisioned.go +++ b/internal/inventory/provisioned.go @@ -26,6 +26,17 @@ type ProvReport struct { WaitsFor []string `json:"waits-for,omitempty"` Templates string `json:"templates,omitempty"` At time.Time `json:"at"` + // FailingSince is, on a failed report, when the module's standing failure began, kept by the client while only its + // words change (mesh-catalog 3aae6c08); zero from an older client. + FailingSince time.Time `json:"failing-since,omitzero"` +} + +// Since is when a failed report's failure began: FailingSince when the client says it, else At. +func (r ProvReport) Since() time.Time { + if !r.FailingSince.IsZero() { + return r.FailingSince + } + return r.At } // The words a client's report says.