A failure that repeats is said to be stuck
The mesh kept one report per machine, replaced, so a resource nothing can ever apply looked like a failure that had just happened, every reconcile interval, for ever. The row now keeps when the current failure began and how many reports in a row have said it — the same outcome, refusal and failed resources; anything different starts again and a clean apply clears it. Three make the machine stuck, and status says so beside the failure, in words and in JSON (novox/hq 04-ISSUES/065, ADR 0090).
This commit is contained in:
@@ -75,6 +75,11 @@ type machineDoing struct {
|
||||
} `json:"failed,omitempty"`
|
||||
Applied int `json:"applied"`
|
||||
At time.Time `json:"at"`
|
||||
// Since is when this same failure was first reported and Times how many reports in a row
|
||||
// have said it; Stuck is the mesh's word for "enough of them" (novox/hq 04-ISSUES/065).
|
||||
Since *time.Time `json:"since,omitempty"`
|
||||
Times int `json:"times"`
|
||||
Stuck bool `json:"stuck"`
|
||||
}
|
||||
|
||||
type machineReported struct {
|
||||
@@ -149,6 +154,7 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
||||
for _, d := range wrong {
|
||||
row := machineDoing{
|
||||
Node: d.Node, Outcome: d.Outcome, Refused: d.Refused, Applied: d.Applied, At: d.At,
|
||||
Since: d.Since, Times: d.Times, Stuck: d.Stuck(),
|
||||
}
|
||||
for _, f := range d.Failed {
|
||||
row.Failed = append(row.Failed, struct {
|
||||
|
||||
@@ -139,3 +139,31 @@ func TestNoSecretIsInWhatABoardReads(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A failure that has been reported identically enough times is said to be stuck, with when it
|
||||
// began and how many times — so a board can tell a machine looping on something that will never
|
||||
// apply from one that failed a minute ago (novox/hq 04-ISSUES/065, ADR 0090).
|
||||
func TestAMachineFailingTheSameWayIsSaidToBeStuck(t *testing.T) {
|
||||
began := time.Date(2026, 9, 21, 9, 0, 0, 0, time.UTC)
|
||||
got := statusOf(t,
|
||||
[]inventory.Doing{
|
||||
{Node: "looping", Outcome: inventory.OutcomeFailed, Since: &began, Times: inventory.StuckAfter,
|
||||
Failed: []inventory.FailedResource{{ID: "img", Error: "no such image"}}},
|
||||
{Node: "once", Outcome: inventory.OutcomeFailed, Since: &began, Times: 1,
|
||||
Failed: []inventory.FailedResource{{ID: "img", Error: "no such image"}}},
|
||||
},
|
||||
[]inventory.Node{{Name: "looping"}, {Name: "once"}}, nil, nil, nil)
|
||||
|
||||
wrong, _ := got["wrong"].([]any)
|
||||
looping, _ := wrong[0].(map[string]any)
|
||||
if looping["stuck"] != true || looping["times"] != float64(inventory.StuckAfter) {
|
||||
t.Fatalf("three identical failures are stuck: %v", looping)
|
||||
}
|
||||
if since, _ := looping["since"].(string); !strings.HasPrefix(since, "2026-09-21T09:00:00") {
|
||||
t.Fatalf("when it began was not carried: %v", looping)
|
||||
}
|
||||
once, _ := wrong[1].(map[string]any)
|
||||
if once["stuck"] != false || once["times"] != float64(1) {
|
||||
t.Fatalf("one failure is not stuck: %v", once)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -99,6 +99,14 @@ func statusCommand(ctx context.Context, args []string) error {
|
||||
for _, f := range d.Failed {
|
||||
fmt.Printf(" %-18s %s: %s\n", "", f.ID, firstLine(f.Error))
|
||||
}
|
||||
if d.Stuck() {
|
||||
// Said apart from the failure itself. The host's words say what is wrong; this
|
||||
// says it is not new — the machine has applied, failed the same way and reported
|
||||
// so this many times, and will keep doing exactly that until something changes
|
||||
// (novox/hq 04-ISSUES/065).
|
||||
fmt.Printf(" %-18s stuck: the same failure %d times since %s — it will not fix itself\n",
|
||||
"", d.Times, d.Since.Local().Format("2006-01-02 15:04"))
|
||||
}
|
||||
}
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user