A failure that repeats is said to be stuck

The mesh kept one report per machine, replaced, so a resource nothing can ever apply
looked like a failure that had just happened, every reconcile interval, for ever.
The row now keeps when the current failure began and how many reports in a row have
said it — the same outcome, refusal and failed resources; anything different starts
again and a clean apply clears it. Three make the machine stuck, and status says so
beside the failure, in words and in JSON (novox/hq 04-ISSUES/065, ADR 0090).
This commit is contained in:
2026-09-21 17:43:24 +02:00
parent 69d0b94395
commit fcaad7271a
6 changed files with 183 additions and 8 deletions
+6
View File
@@ -75,6 +75,11 @@ type machineDoing struct {
} `json:"failed,omitempty"`
Applied int `json:"applied"`
At time.Time `json:"at"`
// Since is when this same failure was first reported and Times how many reports in a row
// have said it; Stuck is the mesh's word for "enough of them" (novox/hq 04-ISSUES/065).
Since *time.Time `json:"since,omitempty"`
Times int `json:"times"`
Stuck bool `json:"stuck"`
}
type machineReported struct {
@@ -149,6 +154,7 @@ func statusAsJSON(asked answers) ([]byte, error) {
for _, d := range wrong {
row := machineDoing{
Node: d.Node, Outcome: d.Outcome, Refused: d.Refused, Applied: d.Applied, At: d.At,
Since: d.Since, Times: d.Times, Stuck: d.Stuck(),
}
for _, f := range d.Failed {
row.Failed = append(row.Failed, struct {
+28
View File
@@ -139,3 +139,31 @@ func TestNoSecretIsInWhatABoardReads(t *testing.T) {
}
}
}
// A failure that has been reported identically enough times is said to be stuck, with when it
// began and how many times — so a board can tell a machine looping on something that will never
// apply from one that failed a minute ago (novox/hq 04-ISSUES/065, ADR 0090).
func TestAMachineFailingTheSameWayIsSaidToBeStuck(t *testing.T) {
began := time.Date(2026, 9, 21, 9, 0, 0, 0, time.UTC)
got := statusOf(t,
[]inventory.Doing{
{Node: "looping", Outcome: inventory.OutcomeFailed, Since: &began, Times: inventory.StuckAfter,
Failed: []inventory.FailedResource{{ID: "img", Error: "no such image"}}},
{Node: "once", Outcome: inventory.OutcomeFailed, Since: &began, Times: 1,
Failed: []inventory.FailedResource{{ID: "img", Error: "no such image"}}},
},
[]inventory.Node{{Name: "looping"}, {Name: "once"}}, nil, nil, nil)
wrong, _ := got["wrong"].([]any)
looping, _ := wrong[0].(map[string]any)
if looping["stuck"] != true || looping["times"] != float64(inventory.StuckAfter) {
t.Fatalf("three identical failures are stuck: %v", looping)
}
if since, _ := looping["since"].(string); !strings.HasPrefix(since, "2026-09-21T09:00:00") {
t.Fatalf("when it began was not carried: %v", looping)
}
once, _ := wrong[1].(map[string]any)
if once["stuck"] != false || once["times"] != float64(1) {
t.Fatalf("one failure is not stuck: %v", once)
}
}
+8
View File
@@ -99,6 +99,14 @@ func statusCommand(ctx context.Context, args []string) error {
for _, f := range d.Failed {
fmt.Printf(" %-18s %s: %s\n", "", f.ID, firstLine(f.Error))
}
if d.Stuck() {
// Said apart from the failure itself. The host's words say what is wrong; this
// says it is not new — the machine has applied, failed the same way and reported
// so this many times, and will keep doing exactly that until something changes
// (novox/hq 04-ISSUES/065).
fmt.Printf(" %-18s stuck: the same failure %d times since %s — it will not fix itself\n",
"", d.Times, d.Since.Local().Format("2006-01-02 15:04"))
}
}
fmt.Println()
}