package main import ( "context" "encoding/json" "fmt" "regexp" "sort" "strconv" "strings" "time" ) // Kill is one thing the machine killed for memory, from the journal. type Kill struct { Time time.Time `json:"time"` By string `json:"by"` Victim string `json:"victim,omitempty"` Message string `json:"message"` } // The three voices that report a kill: the kernel's OOM killer, systemd-oomd, and the service manager // saying one of its units lost a process to the OOM killer. var ( kernelKill = regexp.MustCompile(`Out of memory: Killed process (\d+) \(([^)]*)\)`) oomdKill = regexp.MustCompile(`Killed (\S+) due to (.+)`) unitKilled = regexp.MustCompile(`^(\S+): A process of this unit has been killed by the OOM killer`) ) // sinceArg is a value journalctl understands after --since; anything else is refused before it // reaches the command line. var sinceArg = regexp.MustCompile(`^(-?\d+[smhdw]|today|yesterday|\d{4}-\d{2}-\d{2}( \d{2}:\d{2}(:\d{2})?)?)$`) // ParseJournal reads `journalctl -o json` lines into kills, newest first. func ParseJournal(text string) []Kill { var out []Kill for _, line := range strings.Split(text, "\n") { if strings.TrimSpace(line) == "" { continue } var e map[string]any if json.Unmarshal([]byte(line), &e) != nil { continue } msg, ok := e["MESSAGE"].(string) // a message that is not UTF-8 arrives as bytes; not one of these if !ok { continue } var k Kill if us, err := strconv.ParseInt(fmt.Sprint(e["__REALTIME_TIMESTAMP"]), 10, 64); err == nil { k.Time = time.UnixMicro(us).UTC() } switch { case kernelKill.MatchString(msg): g := kernelKill.FindStringSubmatch(msg) k.By, k.Victim = "kernel", g[2]+" (pid "+g[1]+")" case oomdKill.MatchString(msg) && strings.Contains(fmt.Sprint(e["_SYSTEMD_UNIT"], e["SYSLOG_IDENTIFIER"]), "oomd"): g := oomdKill.FindStringSubmatch(msg) k.By, k.Victim = "systemd-oomd", g[1] case unitKilled.MatchString(msg): k.By, k.Victim = "service manager", unitKilled.FindStringSubmatch(msg)[1] default: continue } k.Message = msg out = append(out, k) } sort.SliceStable(out, func(i, j int) bool { return out[i].Time.After(out[j].Time) }) return out } // OOMHistory searches the journal since a time for every kill. func (m *Machine) OOMHistory(ctx context.Context, since string, limit int) ([]Kill, error) { if !sinceArg.MatchString(since) { return nil, fmt.Errorf("since %q is not -7d, 12h, today, yesterday or a date (2026-10-01)", since) } since = expandRelative(since) out, err := m.Run(ctx, "journalctl", "--no-pager", "-q", "-o", "json", "--since", since, "-g", "Out of memory: Killed process|Killed .* due to|has been killed by the OOM killer") if err != nil && strings.TrimSpace(out) == "" { if strings.HasSuffix(err.Error(), "exit status 1") { return []Kill{}, nil // journalctl exits 1 when a grep matches nothing } return nil, err } kills := ParseJournal(out) if kills == nil { kills = []Kill{} } if len(kills) > limit { kills = kills[:limit] } return kills, nil } // expandRelative turns -7d (or 7d) into the "-7 days" form journalctl's --since reads. func expandRelative(s string) string { g := regexp.MustCompile(`^-?(\d+)([smhdw])$`).FindStringSubmatch(s) if g == nil { return s } unit := map[string]string{"s": "seconds", "m": "minutes", "h": "hours", "d": "days", "w": "weeks"}[g[2]] return "-" + g[1] + " " + unit } // Oomd is what systemd-oomd says it watches. type Oomd struct { Active string `json:"active"` Enabled string `json:"enabled"` Report []string `json:"oomctl"` Note string `json:"note,omitempty"` } func (m *Machine) Oomd(ctx context.Context) Oomd { o := Oomd{Report: []string{}} a, _ := m.Run(ctx, "systemctl", "is-active", "systemd-oomd.service") e, _ := m.Run(ctx, "systemctl", "is-enabled", "systemd-oomd.service") o.Active, o.Enabled = strings.TrimSpace(a), strings.TrimSpace(e) out, err := m.Run(ctx, "oomctl") if err != nil { o.Note = "oomctl: " + err.Error() } for _, line := range strings.Split(out, "\n") { if strings.TrimSpace(line) != "" { o.Report = append(o.Report, strings.TrimRight(strings.ReplaceAll(line, "\t", " "), " ")) } } return o }