mesh-catalog/modules/logrotate/cmd/logrotate-tools/logrotate.go

package main

// Log rotation, on every machine (novox/hq to-be 42 Phase 1, research 027/01: "rotation running on
// one machine of four"). Three machines carried rules in /etc/logrotate.d — put there by their
// packages and by the mesh's own fail2ban module — and no logrotate to read them, so those logs
// grew without bound. The module installs logrotate, owns its base configuration and enables its
// timer; these tools read what it did, find what grows, force one rule set, and do the same for the
// journal, which is the other place a machine's logs fill its disk.
//
// The status file and much of /var/log are root's, so reading them goes through sudo -n.

import (
	"fmt"
	"path"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"
)

// The files logrotate reads and keeps.
const (
	BaseConf   = "/etc/logrotate.conf"
	RulesDir   = "/etc/logrotate.d"
	StateFile  = "/var/lib/logrotate.status"
	LogRoot    = "/var/log"
	forcedConf = "/run/mesh-logrotate-force.conf"
)

// Rotation is one log and when logrotate last rotated it.
type Rotation struct {
	Log         string `json:"log"`
	LastRotated string `json:"last_rotated"`
}

var stateLine = regexp.MustCompile(`^"(.*)" (\d+)-(\d+)-(\d+)(?:-(\d+):(\d+)(?::(\d+))?)?$`)

// ParseState reads logrotate's status file: `"<log>" Y-M-D-h:m:s` per line.
func ParseState(text string) []Rotation {
	out := []Rotation{}
	for _, l := range lines(text) {
		s := stateLine.FindStringSubmatch(strings.TrimSpace(l))
		if s == nil {
			continue
		}
		n := make([]int, 6)
		for i := range n {
			n[i], _ = strconv.Atoi(s[i+2])
		}
		t := time.Date(n[0], time.Month(n[1]), n[2], n[3], n[4], n[5], 0, time.Local)
		out = append(out, Rotation{Log: s[1], LastRotated: t.Format(time.RFC3339)})
	}
	sort.Slice(out, func(i, j int) bool { return out[i].Log < out[j].Log })
	return out
}

// Status is each log's last rotation and the timer that rotates them.
func (m *Machine) Status(match string) (map[string]any, error) {
	out := map[string]any{"state_file": StateFile}
	r, err := m.RootRan("cat", StateFile)
	if err != nil {
		return nil, err
	}
	switch {
	case r.Status == 0:
		rot := []Rotation{}
		for _, x := range ParseState(r.Stdout) {
			if match == "" || strings.Contains(x.Log, match) {
				rot = append(rot, x)
			}
		}
		out["logs"], out["state_file_present"] = rot, true
	case strings.Contains(r.Stderr, "No such file"):
		out["logs"], out["state_file_present"] = []Rotation{}, false
		out["note"] = "logrotate has never run here"
	default:
		return nil, failure("cat", "sudo", r)
	}
	if t, err := m.unitProps("logrotate.timer", "LoadState", "ActiveState", "UnitFileState", "LastTriggerUSec", "NextElapseUSecRealtime"); err == nil {
		out["timer"] = t
	}
	if s, err := m.unitProps("logrotate.service", "LoadState", "Result", "ExecMainExitTimestamp", "ExecMainStatus"); err == nil && s["LoadState"] == "loaded" {
		out["last_run"] = s
	}
	return out, nil
}

// Rule is one rule file and the logs it rotates.
type Rule struct {
	File string   `json:"file"`
	Logs []string `json:"logs"`
	Mesh bool     `json:"mesh_owned,omitempty"`
}

// RulesIn reads the log patterns a logrotate file names: the paths before each `{`.
func RulesIn(text string) []string {
	logs := []string{}
	depth := 0
	var pending []string
	for _, l := range lines(text) {
		l = strings.TrimSpace(l)
		if strings.HasPrefix(l, "#") {
			continue
		}
		if depth == 0 {
			before, _, opens := strings.Cut(l, "{")
			fields := strings.Fields(before)
			if len(fields) > 0 && !strings.HasPrefix(fields[0], "/") && !strings.HasPrefix(fields[0], "\"") {
				// A directive (olddir, include …), not a log.
				fields = nil
			}
			for _, f := range fields {
				if strings.HasPrefix(f, "/") || strings.HasPrefix(f, "\"/") {
					pending = append(pending, strings.Trim(f, "\""))
				}
			}
			if opens {
				logs = append(logs, pending...)
				pending = nil
				depth++
				if strings.Contains(l[strings.Index(l, "{"):], "}") {
					depth--
				}
			}
			continue
		}
		if strings.HasPrefix(l, "}") || strings.HasSuffix(l, "}") && !strings.Contains(l, "{") {
			depth--
		}
	}
	return logs
}

// Configs is the base configuration's own logs and every rule file with the logs it rotates.
func (m *Machine) Configs() (map[string]any, error) {
	base, err := m.ReadFile(BaseConf)
	if err != nil {
		return nil, fmt.Errorf("reading %s: %w (logrotate is not installed, or the module has not been applied)", BaseConf, err)
	}
	names, err := m.Out("find", RulesDir, "-mindepth", "1", "-maxdepth", "1", "-type", "f", "-printf", "%f\n")
	if err != nil {
		return nil, err
	}
	rules := []Rule{{File: BaseConf, Logs: RulesIn(string(base)), Mesh: strings.HasPrefix(string(base), "# The mesh's (module logrotate")}}
	sorted := lines(names)
	sort.Strings(sorted)
	for _, n := range sorted {
		p := path.Join(RulesDir, n)
		text, err := m.ReadFile(p)
		if err != nil {
			rules = append(rules, Rule{File: p, Logs: []string{"(unreadable: " + err.Error() + ")"}})
			continue
		}
		rules = append(rules, Rule{File: p, Logs: RulesIn(string(text))})
	}
	return map[string]any{"globals": Globals(string(base)), "rules": rules}, nil
}

// Globals is the base configuration without its includes and its per-log blocks: what every rule
// file inherits. Forcing one rule file is done with these before it, so it rotates as it would in
// the whole run — without them, a rule that names no count would keep no old log at all.
func Globals(text string) []string {
	out := []string{}
	depth := 0
	for _, l := range strings.Split(text, "\n") {
		t := strings.TrimSpace(l)
		switch {
		case depth > 0:
			if strings.Contains(t, "}") {
				depth--
			}
		case strings.Contains(t, "{"):
			if !strings.Contains(t, "}") {
				depth++
			}
		case t == "" || strings.HasPrefix(t, "#"), strings.HasPrefix(t, "include"):
		default:
			out = append(out, t)
		}
	}
	return out
}

// Check is a dry run of the whole configuration (logrotate -d, which changes nothing): its errors
// and warnings, so a broken rule is found before the night it was meant to run.
func (m *Machine) Check() (map[string]any, error) {
	r, err := m.RootRan("logrotate", "-d", BaseConf)
	if err != nil {
		return nil, err
	}
	errs, warns := []string{}, []string{}
	for _, l := range lines(r.Stdout + "\n" + r.Stderr) {
		l = strings.TrimSpace(l)
		switch {
		case strings.HasPrefix(l, "error:"):
			errs = append(errs, l)
		case strings.HasPrefix(l, "warning:") && !strings.Contains(l, "debug mode does nothing"):
			warns = append(warns, l)
		}
	}
	return map[string]any{"ok": len(errs) == 0 && r.Status == 0, "status": r.Status, "errors": errs, "warnings": warns}, nil
}

// LogFile is one file under /var/log and its size.
type LogFile struct {
	Path     string `json:"path"`
	Bytes    int64  `json:"bytes"`
	Size     string `json:"size"`
	Modified string `json:"modified"`
	Journal  bool   `json:"journal"`
}

// BigLogs is the largest files under /var/log, on its own filesystem, read as root. Journal files
// are counted and, unless asked for, not listed: journald bounds them, and the journal tools speak
// for them.
func (m *Machine) BigLogs(limit int, journals bool) (map[string]any, error) {
	r, err := m.RootRan("find", LogRoot, "-xdev", "-type", "f", "-printf", "%s\t%TY-%Tm-%Td %TH:%TM\t%p\n")
	if err != nil {
		return nil, err
	}
	if r.Status != 0 && strings.TrimSpace(r.Stdout) == "" {
		return nil, failure("find", "sudo", r)
	}
	files := []LogFile{}
	var total, journalBytes int64
	for _, l := range lines(r.Stdout) {
		f := strings.SplitN(l, "\t", 3)
		if len(f) != 3 {
			continue
		}
		n, _ := strconv.ParseInt(f[0], 10, 64)
		total += n
		journal := strings.HasSuffix(f[2], ".journal") || strings.HasSuffix(f[2], ".journal~")
		if journal {
			journalBytes += n
			if !journals {
				continue
			}
		}
		files = append(files, LogFile{Path: f[2], Bytes: n, Size: human(n), Modified: f[1], Journal: journal})
	}
	sort.Slice(files, func(i, j int) bool { return files[i].Bytes > files[j].Bytes })
	count := len(files)
	if len(files) > limit {
		files = files[:limit]
	}
	return map[string]any{"under": LogRoot, "files": count, "total_bytes": total, "total": human(total),
		"journal_bytes": journalBytes, "journal": human(journalBytes), "journals_listed": journals, "largest": files}, nil
}

func human(n int64) string {
	units := []string{"B", "K", "M", "G", "T"}
	f := float64(n)
	i := 0
	for f >= 1024 && i < len(units)-1 {
		f /= 1024
		i++
	}
	if i == 0 {
		return fmt.Sprintf("%d%s", n, units[0])
	}
	return fmt.Sprintf("%.1f%s", f, units[i])
}

var ruleName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._@+-]*$`)

// Force rotates the logs of one rule file now (logrotate -f -v), with the base configuration's
// globals before it; or every log, given the base configuration's own name.
func (m *Machine) Force(config string, writeTemp func(string) (string, func(), error)) (map[string]any, error) {
	var args []string
	switch {
	case config == path.Base(BaseConf) || config == BaseConf:
		args = []string{"-f", "-v", BaseConf}
	case ruleName.MatchString(config):
		rule := path.Join(RulesDir, config)
		if _, err := m.ReadFile(rule); err != nil {
			return nil, fmt.Errorf("%s is not a rule file here: %w", rule, err)
		}
		base, err := m.ReadFile(BaseConf)
		if err != nil {
			return nil, fmt.Errorf("reading %s: %w", BaseConf, err)
		}
		temp, done, err := writeTemp("# The globals of " + BaseConf + ", for forcing " + rule + " alone.\n" + strings.Join(Globals(string(base)), "\n") + "\n")
		if err != nil {
			return nil, err
		}
		defer done()
		// logrotate running as root reads only a configuration root owns.
		if _, err := m.Root("install", "-m", "0644", "-o", "root", "-g", "root", temp, forcedConf); err != nil {
			return nil, err
		}
		defer m.Root("rm", "-f", forcedConf) //nolint:errcheck
		args = []string{"-f", "-v", forcedConf, rule}
	default:
		return nil, fmt.Errorf("%q is neither a file of %s nor %s", config, RulesDir, path.Base(BaseConf))
	}
	r, err := m.RootRan("logrotate", args...)
	if err != nil {
		return nil, err
	}
	said := lines(r.Stdout + "\n" + r.Stderr)
	rotated, errs := []string{}, []string{}
	for _, l := range said {
		l = strings.TrimSpace(l)
		switch {
		case strings.HasPrefix(l, "rotating log "):
			rotated = append(rotated, strings.TrimSuffix(strings.Fields(strings.TrimPrefix(l, "rotating log "))[0], ","))
		case strings.HasPrefix(l, "error:"):
			errs = append(errs, l)
		}
	}
	if len(said) > 200 {
		said = said[len(said)-200:]
	}
	return map[string]any{"config": config, "ok": r.Status == 0 && len(errs) == 0, "rotated": rotated, "errors": errs, "log": said}, nil
}

func contains(list []string, want string) bool {
	for _, s := range list {
		if s == want {
			return true
		}
	}
	return false
}