diff --git a/cmd/mesh-controller/module_health.go b/cmd/mesh-controller/module_health.go index 58f0d0ac..41c027e3 100644 --- a/cmd/mesh-controller/module_health.go +++ b/cmd/mesh-controller/module_health.go @@ -497,9 +497,11 @@ func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHeal continue } words = append(words, fmt.Sprintf("its %s %s %s", r.Kind, r.Resource, reasonWords(r))) - said = append(said, fmt.Sprintf("%s (%s %s): %s, %d look(s) in a row, %d restart(s) counted, since %s", - r.Resource, r.Kind, r.Target, orNotSaid(r.Reason), r.Streak, r.Restarts, - r.Since.UTC().Format("2006-01-02 15:04:05 MST"))) + // The restart count runs from when the engine began watching it, not from the state's since (novox/hq + // issue 458). + said = append(said, fmt.Sprintf("%s (%s %s): %s, since %s, %d look(s) in a row, %s", + r.Resource, r.Kind, r.Target, orNotSaid(r.Reason), r.Since.UTC().Format("2006-01-02 15:04:05 MST"), + r.Streak, restartsWords(r.Restarts))) } return conditions.Observation{Scope: conditions.ScopeModule, ID: module + "." + node, Token: "unhealthy", Kind: kindModuleUnhealthy, Machine: node, Severity: conditions.Warning, Source: sourceHealth, @@ -513,6 +515,15 @@ func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHeal Resolved: fmt.Sprintf("%s works again on %s", module, node)} } +// restartsWords is a restart count as what it is (novox/hq issue 458). The engine reports its own count +// (mesh-host's liveness judge, kept.Counted): every rise of systemd's NRestarts or the runtime's RestartCount +// after a start's grace, so only restarts by the Restart= setting or the restart policy. It keeps the count +// across a recreate, a start by hand and its own restarts, and forgets it only when the resource is no longer +// declared. So it counts automatic restarts since the engine began watching it, never since a state's "since". +func restartsWords(n int) string { + return fmt.Sprintf("%d automatic restart(s) since the node-engine began watching it", n) +} + // reasonWords is why a resource is unhealthy, as a person reads it. func reasonWords(r inventory.ResourceHealth) string { // An account waiting for a new login (ADR 0252) is said in the mesh's words, not the engine's. @@ -521,7 +532,7 @@ func reasonWords(r inventory.ResourceHealth) string { } switch r.Reason { case "restarting": - return fmt.Sprintf("keeps restarting (%d restart(s) counted)", r.Restarts) + return fmt.Sprintf("keeps restarting (%s)", restartsWords(r.Restarts)) case "down": return "is not running" case "": @@ -725,7 +736,7 @@ func healthLines(h inventory.NodeHealth, had bool, now time.Time) []string { line += " — " + r.Reason } if r.Restarts > 0 { - line += fmt.Sprintf(", %d restart(s) counted", r.Restarts) + line += ", " + restartsWords(r.Restarts) } if r.State == link.StateUnhealthy && h.Streaks[r.Module] < moduleUnhealthyAfter { line += " (unconfirmed: said once)" diff --git a/cmd/mesh-controller/restarts_words_test.go b/cmd/mesh-controller/restarts_words_test.go new file mode 100644 index 00000000..fe1dfb44 --- /dev/null +++ b/cmd/mesh-controller/restarts_words_test.go @@ -0,0 +1,28 @@ +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// A restart count runs from when the node-engine began watching the resource, so it is never said beside +// "since" as if it counted from there (novox/hq issue 458): 3890 restarts beside "since 00:58" sent a reader after a crash loop that was not. +func TestARestartCountSaysItCountsFromWhenTheEngineBeganWatching(t *testing.T) { + since := time.Date(2026, 10, 11, 0, 58, 0, 0, time.UTC) + r := inventory.ResourceHealth{Module: "messenger", Resource: "messenger.own-runtime", Kind: "process", + Target: "messenger-runtime.service", State: link.StateUnhealthy, Reason: "restarting", Since: since, + Streak: 2, Restarts: 3890} + h := inventory.NodeHealth{HeardAt: since, Resources: []inventory.ResourceHealth{r}, + Streaks: map[string]int{"messenger": 2}} + lines := strings.Join(healthLines(h, true, since.Add(time.Minute)), "\n") + o := moduleUnhealthyObservation("messenger", "anchor", []inventory.ResourceHealth{r}) + for name, said := range map[string]string{"node show": lines, "evidence": o.Said, "summary": o.Summary} { + if !strings.Contains(said, "3890 automatic restart(s) since the node-engine began watching it") || strings.Contains(said, "counted") { + t.Errorf("%s says the restart count as if it were since then:\n%s", name, said) + } + } +}