mesh-catalog/modules/memory-pressure/cmd/memory-pressure/oom.go

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"
)

// Kill is one thing the machine killed for memory, from the journal.
type Kill struct {
	Time    time.Time `json:"time"`
	By      string    `json:"by"`
	Victim  string    `json:"victim,omitempty"`
	Message string    `json:"message"`
}

// The three voices that report a kill: the kernel's OOM killer, systemd-oomd, and the service manager
// saying one of its units lost a process to the OOM killer.
var (
	kernelKill = regexp.MustCompile(`Out of memory: Killed process (\d+) \(([^)]*)\)`)
	oomdKill   = regexp.MustCompile(`Killed (\S+) due to (.+)`)
	unitKilled = regexp.MustCompile(`^(\S+): A process of this unit has been killed by the OOM killer`)
)

// sinceArg is a value journalctl understands after --since; anything else is refused before it
// reaches the command line.
var sinceArg = regexp.MustCompile(`^(-?\d+[smhdw]|today|yesterday|\d{4}-\d{2}-\d{2}( \d{2}:\d{2}(:\d{2})?)?)$`)

// ParseJournal reads `journalctl -o json` lines into kills, newest first.
func ParseJournal(text string) []Kill {
	var out []Kill
	for _, line := range strings.Split(text, "\n") {
		if strings.TrimSpace(line) == "" {
			continue
		}
		var e map[string]any
		if json.Unmarshal([]byte(line), &e) != nil {
			continue
		}
		msg, ok := e["MESSAGE"].(string) // a message that is not UTF-8 arrives as bytes; not one of these
		if !ok {
			continue
		}
		var k Kill
		if us, err := strconv.ParseInt(fmt.Sprint(e["__REALTIME_TIMESTAMP"]), 10, 64); err == nil {
			k.Time = time.UnixMicro(us).UTC()
		}
		switch {
		case kernelKill.MatchString(msg):
			g := kernelKill.FindStringSubmatch(msg)
			k.By, k.Victim = "kernel", g[2]+" (pid "+g[1]+")"
		case oomdKill.MatchString(msg) && strings.Contains(fmt.Sprint(e["_SYSTEMD_UNIT"], e["SYSLOG_IDENTIFIER"]), "oomd"):
			g := oomdKill.FindStringSubmatch(msg)
			k.By, k.Victim = "systemd-oomd", g[1]
		case unitKilled.MatchString(msg):
			k.By, k.Victim = "service manager", unitKilled.FindStringSubmatch(msg)[1]
		default:
			continue
		}
		k.Message = msg
		out = append(out, k)
	}
	sort.SliceStable(out, func(i, j int) bool { return out[i].Time.After(out[j].Time) })
	return out
}

// OOMHistory searches the journal since a time for every kill.
func (m *Machine) OOMHistory(ctx context.Context, since string, limit int) ([]Kill, error) {
	if !sinceArg.MatchString(since) {
		return nil, fmt.Errorf("since %q is not -7d, 12h, today, yesterday or a date (2026-10-01)", since)
	}
	since = expandRelative(since)
	out, err := m.Run(ctx, "journalctl", "--no-pager", "-q", "-o", "json", "--since", since,
		"-g", "Out of memory: Killed process|Killed .* due to|has been killed by the OOM killer")
	if err != nil && strings.TrimSpace(out) == "" {
		if strings.HasSuffix(err.Error(), "exit status 1") {
			return []Kill{}, nil // journalctl exits 1 when a grep matches nothing
		}
		return nil, err
	}
	kills := ParseJournal(out)
	if kills == nil {
		kills = []Kill{}
	}
	if len(kills) > limit {
		kills = kills[:limit]
	}
	return kills, nil
}

// expandRelative turns -7d (or 7d) into the "-7 days" form journalctl's --since reads.
func expandRelative(s string) string {
	g := regexp.MustCompile(`^-?(\d+)([smhdw])$`).FindStringSubmatch(s)
	if g == nil {
		return s
	}
	unit := map[string]string{"s": "seconds", "m": "minutes", "h": "hours", "d": "days", "w": "weeks"}[g[2]]
	return "-" + g[1] + " " + unit
}

// Oomd is what systemd-oomd says it watches.
type Oomd struct {
	Active  string   `json:"active"`
	Enabled string   `json:"enabled"`
	Report  []string `json:"oomctl"`
	Note    string   `json:"note,omitempty"`
}

func (m *Machine) Oomd(ctx context.Context) Oomd {
	o := Oomd{Report: []string{}}
	a, _ := m.Run(ctx, "systemctl", "is-active", "systemd-oomd.service")
	e, _ := m.Run(ctx, "systemctl", "is-enabled", "systemd-oomd.service")
	o.Active, o.Enabled = strings.TrimSpace(a), strings.TrimSpace(e)
	out, err := m.Run(ctx, "oomctl")
	if err != nil {
		o.Note = "oomctl: " + err.Error()
	}
	for _, line := range strings.Split(out, "\n") {
		if strings.TrimSpace(line) != "" {
			o.Report = append(o.Report, strings.TrimRight(strings.ReplaceAll(line, "\t", "  "), " "))
		}
	}
	return o
}