From cde0dc287bf5b7ca26cc399b6dd936c8145f918e Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 10:46:38 +0200 Subject: [PATCH] Say a restart count runs from when the node-engine began watching it The engine reports its own count, kept across recreates and starts by hand, not systemd's NRestarts or the runtime's RestartCount (novox/hq issue 458). --- cmd/mesh-controller/module_health.go | 14 ++++++++------ cmd/mesh-controller/restarts_words_test.go | 8 ++++---- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/cmd/mesh-controller/module_health.go b/cmd/mesh-controller/module_health.go index 2177a8c5..41c027e3 100644 --- a/cmd/mesh-controller/module_health.go +++ b/cmd/mesh-controller/module_health.go @@ -497,7 +497,8 @@ func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHeal continue } words = append(words, fmt.Sprintf("its %s %s %s", r.Kind, r.Resource, reasonWords(r))) - // The restart count runs from the last start or creation, not from the state's since (novox/hq issue 458). + // The restart count runs from when the engine began watching it, not from the state's since (novox/hq + // issue 458). said = append(said, fmt.Sprintf("%s (%s %s): %s, since %s, %d look(s) in a row, %s", r.Resource, r.Kind, r.Target, orNotSaid(r.Reason), r.Since.UTC().Format("2006-01-02 15:04:05 MST"), r.Streak, restartsWords(r.Restarts))) @@ -514,12 +515,13 @@ func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHeal Resolved: fmt.Sprintf("%s works again on %s", module, node)} } -// restartsWords is a restart count as what it is (novox/hq issue 458): the engine reads systemd's NRestarts, -// which counts only restarts its Restart= setting caused and resets when the unit is started by hand, or the -// runtime's RestartCount, which counts only restart-policy restarts and is lost when the container is -// recreated. So it counts automatic restarts since the last start or creation, never since a state's "since". +// restartsWords is a restart count as what it is (novox/hq issue 458). The engine reports its own count +// (mesh-host's liveness judge, kept.Counted): every rise of systemd's NRestarts or the runtime's RestartCount +// after a start's grace, so only restarts by the Restart= setting or the restart policy. It keeps the count +// across a recreate, a start by hand and its own restarts, and forgets it only when the resource is no longer +// declared. So it counts automatic restarts since the engine began watching it, never since a state's "since". func restartsWords(n int) string { - return fmt.Sprintf("%d automatic restart(s) since it was last started or created", n) + return fmt.Sprintf("%d automatic restart(s) since the node-engine began watching it", n) } // reasonWords is why a resource is unhealthy, as a person reads it. diff --git a/cmd/mesh-controller/restarts_words_test.go b/cmd/mesh-controller/restarts_words_test.go index 71e39f53..fe1dfb44 100644 --- a/cmd/mesh-controller/restarts_words_test.go +++ b/cmd/mesh-controller/restarts_words_test.go @@ -9,9 +9,9 @@ import ( "github.com/novox/mesh-controller/internal/link" ) -// A restart count runs from the unit's last start or creation, so it is never said beside "since" as if it -// counted from there (novox/hq issue 458): 3890 restarts beside "since 00:58" sent a reader after a crash loop that was not. -func TestARestartCountSaysItCountsFromTheLastStart(t *testing.T) { +// A restart count runs from when the node-engine began watching the resource, so it is never said beside +// "since" as if it counted from there (novox/hq issue 458): 3890 restarts beside "since 00:58" sent a reader after a crash loop that was not. +func TestARestartCountSaysItCountsFromWhenTheEngineBeganWatching(t *testing.T) { since := time.Date(2026, 10, 11, 0, 58, 0, 0, time.UTC) r := inventory.ResourceHealth{Module: "messenger", Resource: "messenger.own-runtime", Kind: "process", Target: "messenger-runtime.service", State: link.StateUnhealthy, Reason: "restarting", Since: since, @@ -21,7 +21,7 @@ func TestARestartCountSaysItCountsFromTheLastStart(t *testing.T) { lines := strings.Join(healthLines(h, true, since.Add(time.Minute)), "\n") o := moduleUnhealthyObservation("messenger", "anchor", []inventory.ResourceHealth{r}) for name, said := range map[string]string{"node show": lines, "evidence": o.Said, "summary": o.Summary} { - if !strings.Contains(said, "3890 automatic restart(s) since it was last started or created") || strings.Contains(said, "counted") { + if !strings.Contains(said, "3890 automatic restart(s) since the node-engine began watching it") || strings.Contains(said, "counted") { t.Errorf("%s says the restart count as if it were since then:\n%s", name, said) } }