From 0d08d5c730270bbc3eb6b1360ac204d68c7bbc39 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 10:42:19 +0200 Subject: [PATCH 1/3] Say a unit's restart count is its whole life, not since its state began The count comes from systemd's NRestarts or the runtime's restart count; beside "since" it read as thousands of restarts tonight (novox/hq issue 458). --- cmd/mesh-controller/module_health.go | 18 ++++++++++---- cmd/mesh-controller/restarts_words_test.go | 28 ++++++++++++++++++++++ 2 files changed, 41 insertions(+), 5 deletions(-) create mode 100644 cmd/mesh-controller/restarts_words_test.go diff --git a/cmd/mesh-controller/module_health.go b/cmd/mesh-controller/module_health.go index 58f0d0ac..cc54e7c8 100644 --- a/cmd/mesh-controller/module_health.go +++ b/cmd/mesh-controller/module_health.go @@ -497,9 +497,10 @@ func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHeal continue } words = append(words, fmt.Sprintf("its %s %s %s", r.Kind, r.Resource, reasonWords(r))) - said = append(said, fmt.Sprintf("%s (%s %s): %s, %d look(s) in a row, %d restart(s) counted, since %s", - r.Resource, r.Kind, r.Target, orNotSaid(r.Reason), r.Streak, r.Restarts, - r.Since.UTC().Format("2006-01-02 15:04:05 MST"))) + // The restart count is the unit's or container's whole life, never the time since (novox/hq issue 458). + said = append(said, fmt.Sprintf("%s (%s %s): %s, since %s, %d look(s) in a row, %s", + r.Resource, r.Kind, r.Target, orNotSaid(r.Reason), r.Since.UTC().Format("2006-01-02 15:04:05 MST"), + r.Streak, restartsWords(r.Restarts))) } return conditions.Observation{Scope: conditions.ScopeModule, ID: module + "." + node, Token: "unhealthy", Kind: kindModuleUnhealthy, Machine: node, Severity: conditions.Warning, Source: sourceHealth, @@ -513,6 +514,13 @@ func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHeal Resolved: fmt.Sprintf("%s works again on %s", module, node)} } +// restartsWords is a restart count as what it is: the engine reads it from systemd's NRestarts or the +// runtime's restart count, which run over the unit's or container's whole life, so it is never said beside a +// "since" as if it counted from there (novox/hq issue 458). +func restartsWords(n int) string { + return fmt.Sprintf("%d restart(s) over its whole life", n) +} + // reasonWords is why a resource is unhealthy, as a person reads it. func reasonWords(r inventory.ResourceHealth) string { // An account waiting for a new login (ADR 0252) is said in the mesh's words, not the engine's. @@ -521,7 +529,7 @@ func reasonWords(r inventory.ResourceHealth) string { } switch r.Reason { case "restarting": - return fmt.Sprintf("keeps restarting (%d restart(s) counted)", r.Restarts) + return fmt.Sprintf("keeps restarting (%s)", restartsWords(r.Restarts)) case "down": return "is not running" case "": @@ -725,7 +733,7 @@ func healthLines(h inventory.NodeHealth, had bool, now time.Time) []string { line += " — " + r.Reason } if r.Restarts > 0 { - line += fmt.Sprintf(", %d restart(s) counted", r.Restarts) + line += ", " + restartsWords(r.Restarts) + ", not since then" } if r.State == link.StateUnhealthy && h.Streaks[r.Module] < moduleUnhealthyAfter { line += " (unconfirmed: said once)" diff --git a/cmd/mesh-controller/restarts_words_test.go b/cmd/mesh-controller/restarts_words_test.go new file mode 100644 index 00000000..02b6856a --- /dev/null +++ b/cmd/mesh-controller/restarts_words_test.go @@ -0,0 +1,28 @@ +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// A restart count runs over the unit's whole life, so it is never said beside "since" as if it counted from +// there (novox/hq issue 458): 3890 restarts beside "since 00:58" sent a reader after a crash loop that was not. +func TestARestartCountSaysItIsTheWholeLife(t *testing.T) { + since := time.Date(2026, 10, 11, 0, 58, 0, 0, time.UTC) + r := inventory.ResourceHealth{Module: "messenger", Resource: "messenger.own-runtime", Kind: "process", + Target: "messenger-runtime.service", State: link.StateUnhealthy, Reason: "restarting", Since: since, + Streak: 2, Restarts: 3890} + h := inventory.NodeHealth{HeardAt: since, Resources: []inventory.ResourceHealth{r}, + Streaks: map[string]int{"messenger": 2}} + lines := strings.Join(healthLines(h, true, since.Add(time.Minute)), "\n") + o := moduleUnhealthyObservation("messenger", "anchor", []inventory.ResourceHealth{r}) + for name, said := range map[string]string{"node show": lines, "evidence": o.Said, "summary": o.Summary} { + if !strings.Contains(said, "3890 restart(s) over its whole life") || strings.Contains(said, "counted") { + t.Errorf("%s says the restart count as if it were since then:\n%s", name, said) + } + } +} From 63079e2c671b40013e692f714600c7432f67b4dd Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 10:45:40 +0200 Subject: [PATCH 2/3] Say a restart count is automatic restarts since the last start or creation NRestarts and RestartCount reset on a start by hand or a recreation, so "whole life" was as wrong as "since" (novox/hq issue 458). --- cmd/mesh-controller/module_health.go | 13 +++++++------ cmd/mesh-controller/restarts_words_test.go | 8 ++++---- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/cmd/mesh-controller/module_health.go b/cmd/mesh-controller/module_health.go index cc54e7c8..2177a8c5 100644 --- a/cmd/mesh-controller/module_health.go +++ b/cmd/mesh-controller/module_health.go @@ -497,7 +497,7 @@ func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHeal continue } words = append(words, fmt.Sprintf("its %s %s %s", r.Kind, r.Resource, reasonWords(r))) - // The restart count is the unit's or container's whole life, never the time since (novox/hq issue 458). + // The restart count runs from the last start or creation, not from the state's since (novox/hq issue 458). said = append(said, fmt.Sprintf("%s (%s %s): %s, since %s, %d look(s) in a row, %s", r.Resource, r.Kind, r.Target, orNotSaid(r.Reason), r.Since.UTC().Format("2006-01-02 15:04:05 MST"), r.Streak, restartsWords(r.Restarts))) @@ -514,11 +514,12 @@ func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHeal Resolved: fmt.Sprintf("%s works again on %s", module, node)} } -// restartsWords is a restart count as what it is: the engine reads it from systemd's NRestarts or the -// runtime's restart count, which run over the unit's or container's whole life, so it is never said beside a -// "since" as if it counted from there (novox/hq issue 458). +// restartsWords is a restart count as what it is (novox/hq issue 458): the engine reads systemd's NRestarts, +// which counts only restarts its Restart= setting caused and resets when the unit is started by hand, or the +// runtime's RestartCount, which counts only restart-policy restarts and is lost when the container is +// recreated. So it counts automatic restarts since the last start or creation, never since a state's "since". func restartsWords(n int) string { - return fmt.Sprintf("%d restart(s) over its whole life", n) + return fmt.Sprintf("%d automatic restart(s) since it was last started or created", n) } // reasonWords is why a resource is unhealthy, as a person reads it. @@ -733,7 +734,7 @@ func healthLines(h inventory.NodeHealth, had bool, now time.Time) []string { line += " — " + r.Reason } if r.Restarts > 0 { - line += ", " + restartsWords(r.Restarts) + ", not since then" + line += ", " + restartsWords(r.Restarts) } if r.State == link.StateUnhealthy && h.Streaks[r.Module] < moduleUnhealthyAfter { line += " (unconfirmed: said once)" diff --git a/cmd/mesh-controller/restarts_words_test.go b/cmd/mesh-controller/restarts_words_test.go index 02b6856a..71e39f53 100644 --- a/cmd/mesh-controller/restarts_words_test.go +++ b/cmd/mesh-controller/restarts_words_test.go @@ -9,9 +9,9 @@ import ( "github.com/novox/mesh-controller/internal/link" ) -// A restart count runs over the unit's whole life, so it is never said beside "since" as if it counted from -// there (novox/hq issue 458): 3890 restarts beside "since 00:58" sent a reader after a crash loop that was not. -func TestARestartCountSaysItIsTheWholeLife(t *testing.T) { +// A restart count runs from the unit's last start or creation, so it is never said beside "since" as if it +// counted from there (novox/hq issue 458): 3890 restarts beside "since 00:58" sent a reader after a crash loop that was not. +func TestARestartCountSaysItCountsFromTheLastStart(t *testing.T) { since := time.Date(2026, 10, 11, 0, 58, 0, 0, time.UTC) r := inventory.ResourceHealth{Module: "messenger", Resource: "messenger.own-runtime", Kind: "process", Target: "messenger-runtime.service", State: link.StateUnhealthy, Reason: "restarting", Since: since, @@ -21,7 +21,7 @@ func TestARestartCountSaysItIsTheWholeLife(t *testing.T) { lines := strings.Join(healthLines(h, true, since.Add(time.Minute)), "\n") o := moduleUnhealthyObservation("messenger", "anchor", []inventory.ResourceHealth{r}) for name, said := range map[string]string{"node show": lines, "evidence": o.Said, "summary": o.Summary} { - if !strings.Contains(said, "3890 restart(s) over its whole life") || strings.Contains(said, "counted") { + if !strings.Contains(said, "3890 automatic restart(s) since it was last started or created") || strings.Contains(said, "counted") { t.Errorf("%s says the restart count as if it were since then:\n%s", name, said) } } From cde0dc287bf5b7ca26cc399b6dd936c8145f918e Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 10:46:38 +0200 Subject: [PATCH 3/3] Say a restart count runs from when the node-engine began watching it The engine reports its own count, kept across recreates and starts by hand, not systemd's NRestarts or the runtime's RestartCount (novox/hq issue 458). --- cmd/mesh-controller/module_health.go | 14 ++++++++------ cmd/mesh-controller/restarts_words_test.go | 8 ++++---- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/cmd/mesh-controller/module_health.go b/cmd/mesh-controller/module_health.go index 2177a8c5..41c027e3 100644 --- a/cmd/mesh-controller/module_health.go +++ b/cmd/mesh-controller/module_health.go @@ -497,7 +497,8 @@ func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHeal continue } words = append(words, fmt.Sprintf("its %s %s %s", r.Kind, r.Resource, reasonWords(r))) - // The restart count runs from the last start or creation, not from the state's since (novox/hq issue 458). + // The restart count runs from when the engine began watching it, not from the state's since (novox/hq + // issue 458). said = append(said, fmt.Sprintf("%s (%s %s): %s, since %s, %d look(s) in a row, %s", r.Resource, r.Kind, r.Target, orNotSaid(r.Reason), r.Since.UTC().Format("2006-01-02 15:04:05 MST"), r.Streak, restartsWords(r.Restarts))) @@ -514,12 +515,13 @@ func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHeal Resolved: fmt.Sprintf("%s works again on %s", module, node)} } -// restartsWords is a restart count as what it is (novox/hq issue 458): the engine reads systemd's NRestarts, -// which counts only restarts its Restart= setting caused and resets when the unit is started by hand, or the -// runtime's RestartCount, which counts only restart-policy restarts and is lost when the container is -// recreated. So it counts automatic restarts since the last start or creation, never since a state's "since". +// restartsWords is a restart count as what it is (novox/hq issue 458). The engine reports its own count +// (mesh-host's liveness judge, kept.Counted): every rise of systemd's NRestarts or the runtime's RestartCount +// after a start's grace, so only restarts by the Restart= setting or the restart policy. It keeps the count +// across a recreate, a start by hand and its own restarts, and forgets it only when the resource is no longer +// declared. So it counts automatic restarts since the engine began watching it, never since a state's "since". func restartsWords(n int) string { - return fmt.Sprintf("%d automatic restart(s) since it was last started or created", n) + return fmt.Sprintf("%d automatic restart(s) since the node-engine began watching it", n) } // reasonWords is why a resource is unhealthy, as a person reads it. diff --git a/cmd/mesh-controller/restarts_words_test.go b/cmd/mesh-controller/restarts_words_test.go index 71e39f53..fe1dfb44 100644 --- a/cmd/mesh-controller/restarts_words_test.go +++ b/cmd/mesh-controller/restarts_words_test.go @@ -9,9 +9,9 @@ import ( "github.com/novox/mesh-controller/internal/link" ) -// A restart count runs from the unit's last start or creation, so it is never said beside "since" as if it -// counted from there (novox/hq issue 458): 3890 restarts beside "since 00:58" sent a reader after a crash loop that was not. -func TestARestartCountSaysItCountsFromTheLastStart(t *testing.T) { +// A restart count runs from when the node-engine began watching the resource, so it is never said beside +// "since" as if it counted from there (novox/hq issue 458): 3890 restarts beside "since 00:58" sent a reader after a crash loop that was not. +func TestARestartCountSaysItCountsFromWhenTheEngineBeganWatching(t *testing.T) { since := time.Date(2026, 10, 11, 0, 58, 0, 0, time.UTC) r := inventory.ResourceHealth{Module: "messenger", Resource: "messenger.own-runtime", Kind: "process", Target: "messenger-runtime.service", State: link.StateUnhealthy, Reason: "restarting", Since: since, @@ -21,7 +21,7 @@ func TestARestartCountSaysItCountsFromTheLastStart(t *testing.T) { lines := strings.Join(healthLines(h, true, since.Add(time.Minute)), "\n") o := moduleUnhealthyObservation("messenger", "anchor", []inventory.ResourceHealth{r}) for name, said := range map[string]string{"node show": lines, "evidence": o.Said, "summary": o.Summary} { - if !strings.Contains(said, "3890 automatic restart(s) since it was last started or created") || strings.Contains(said, "counted") { + if !strings.Contains(said, "3890 automatic restart(s) since the node-engine began watching it") || strings.Contains(said, "counted") { t.Errorf("%s says the restart count as if it were since then:\n%s", name, said) } }