package main // Log rotation, on every machine (novox/hq to-be 42 Phase 1, research 027/01: "rotation running on // one machine of four"). Three machines carried rules in /etc/logrotate.d — put there by their // packages and by the mesh's own fail2ban module — and no logrotate to read them, so those logs // grew without bound. The module installs logrotate, owns its base configuration and enables its // timer; these tools read what it did, find what grows, force one rule set, and do the same for the // journal, which is the other place a machine's logs fill its disk. // // The status file and much of /var/log are root's, so reading them goes through sudo -n. import ( "fmt" "path" "regexp" "sort" "strconv" "strings" "time" ) // The files logrotate reads and keeps. const ( BaseConf = "/etc/logrotate.conf" RulesDir = "/etc/logrotate.d" StateFile = "/var/lib/logrotate.status" LogRoot = "/var/log" forcedConf = "/run/mesh-logrotate-force.conf" ) // Rotation is one log and when logrotate last rotated it. type Rotation struct { Log string `json:"log"` LastRotated string `json:"last_rotated"` } var stateLine = regexp.MustCompile(`^"(.*)" (\d+)-(\d+)-(\d+)(?:-(\d+):(\d+)(?::(\d+))?)?$`) // ParseState reads logrotate's status file: `"" Y-M-D-h:m:s` per line. func ParseState(text string) []Rotation { out := []Rotation{} for _, l := range lines(text) { s := stateLine.FindStringSubmatch(strings.TrimSpace(l)) if s == nil { continue } n := make([]int, 6) for i := range n { n[i], _ = strconv.Atoi(s[i+2]) } t := time.Date(n[0], time.Month(n[1]), n[2], n[3], n[4], n[5], 0, time.Local) out = append(out, Rotation{Log: s[1], LastRotated: t.Format(time.RFC3339)}) } sort.Slice(out, func(i, j int) bool { return out[i].Log < out[j].Log }) return out } // Status is each log's last rotation and the timer that rotates them. func (m *Machine) Status(match string) (map[string]any, error) { out := map[string]any{"state_file": StateFile} r, err := m.RootRan("cat", StateFile) if err != nil { return nil, err } switch { case r.Status == 0: rot := []Rotation{} for _, x := range ParseState(r.Stdout) { if match == "" || strings.Contains(x.Log, match) { rot = append(rot, x) } } out["logs"], out["state_file_present"] = rot, true case strings.Contains(r.Stderr, "No such file"): out["logs"], out["state_file_present"] = []Rotation{}, false out["note"] = "logrotate has never run here" default: return nil, failure("cat", "sudo", r) } if t, err := m.unitProps("logrotate.timer", "LoadState", "ActiveState", "UnitFileState", "LastTriggerUSec", "NextElapseUSecRealtime"); err == nil { out["timer"] = t } if s, err := m.unitProps("logrotate.service", "LoadState", "Result", "ExecMainExitTimestamp", "ExecMainStatus"); err == nil && s["LoadState"] == "loaded" { out["last_run"] = s } return out, nil } // Rule is one rule file and the logs it rotates. type Rule struct { File string `json:"file"` Logs []string `json:"logs"` Mesh bool `json:"mesh_owned,omitempty"` } // RulesIn reads the log patterns a logrotate file names: the paths before each `{`. func RulesIn(text string) []string { logs := []string{} depth := 0 var pending []string for _, l := range lines(text) { l = strings.TrimSpace(l) if strings.HasPrefix(l, "#") { continue } if depth == 0 { before, _, opens := strings.Cut(l, "{") fields := strings.Fields(before) if len(fields) > 0 && !strings.HasPrefix(fields[0], "/") && !strings.HasPrefix(fields[0], "\"") { // A directive (olddir, include …), not a log. fields = nil } for _, f := range fields { if strings.HasPrefix(f, "/") || strings.HasPrefix(f, "\"/") { pending = append(pending, strings.Trim(f, "\"")) } } if opens { logs = append(logs, pending...) pending = nil depth++ if strings.Contains(l[strings.Index(l, "{"):], "}") { depth-- } } continue } if strings.HasPrefix(l, "}") || strings.HasSuffix(l, "}") && !strings.Contains(l, "{") { depth-- } } return logs } // Configs is the base configuration's own logs and every rule file with the logs it rotates. func (m *Machine) Configs() (map[string]any, error) { base, err := m.ReadFile(BaseConf) if err != nil { return nil, fmt.Errorf("reading %s: %w (logrotate is not installed, or the module has not been applied)", BaseConf, err) } names, err := m.Out("find", RulesDir, "-mindepth", "1", "-maxdepth", "1", "-type", "f", "-printf", "%f\n") if err != nil { return nil, err } rules := []Rule{{File: BaseConf, Logs: RulesIn(string(base)), Mesh: strings.HasPrefix(string(base), "# The mesh's (module logrotate")}} sorted := lines(names) sort.Strings(sorted) for _, n := range sorted { p := path.Join(RulesDir, n) text, err := m.ReadFile(p) if err != nil { rules = append(rules, Rule{File: p, Logs: []string{"(unreadable: " + err.Error() + ")"}}) continue } rules = append(rules, Rule{File: p, Logs: RulesIn(string(text))}) } return map[string]any{"globals": Globals(string(base)), "rules": rules}, nil } // Globals is the base configuration without its includes and its per-log blocks: what every rule // file inherits. Forcing one rule file is done with these before it, so it rotates as it would in // the whole run — without them, a rule that names no count would keep no old log at all. func Globals(text string) []string { out := []string{} depth := 0 for _, l := range strings.Split(text, "\n") { t := strings.TrimSpace(l) switch { case depth > 0: if strings.Contains(t, "}") { depth-- } case strings.Contains(t, "{"): if !strings.Contains(t, "}") { depth++ } case t == "" || strings.HasPrefix(t, "#"), strings.HasPrefix(t, "include"): default: out = append(out, t) } } return out } // Check is a dry run of the whole configuration (logrotate -d, which changes nothing): its errors // and warnings, so a broken rule is found before the night it was meant to run. func (m *Machine) Check() (map[string]any, error) { r, err := m.RootRan("logrotate", "-d", BaseConf) if err != nil { return nil, err } errs, warns := []string{}, []string{} for _, l := range lines(r.Stdout + "\n" + r.Stderr) { l = strings.TrimSpace(l) switch { case strings.HasPrefix(l, "error:"): errs = append(errs, l) case strings.HasPrefix(l, "warning:") && !strings.Contains(l, "debug mode does nothing"): warns = append(warns, l) } } return map[string]any{"ok": len(errs) == 0 && r.Status == 0, "status": r.Status, "errors": errs, "warnings": warns}, nil } // LogFile is one file under /var/log and its size. type LogFile struct { Path string `json:"path"` Bytes int64 `json:"bytes"` Size string `json:"size"` Modified string `json:"modified"` Journal bool `json:"journal"` } // BigLogs is the largest files under /var/log, on its own filesystem, read as root. Journal files // are counted and, unless asked for, not listed: journald bounds them, and the journal tools speak // for them. func (m *Machine) BigLogs(limit int, journals bool) (map[string]any, error) { r, err := m.RootRan("find", LogRoot, "-xdev", "-type", "f", "-printf", "%s\t%TY-%Tm-%Td %TH:%TM\t%p\n") if err != nil { return nil, err } if r.Status != 0 && strings.TrimSpace(r.Stdout) == "" { return nil, failure("find", "sudo", r) } files := []LogFile{} var total, journalBytes int64 for _, l := range lines(r.Stdout) { f := strings.SplitN(l, "\t", 3) if len(f) != 3 { continue } n, _ := strconv.ParseInt(f[0], 10, 64) total += n journal := strings.HasSuffix(f[2], ".journal") || strings.HasSuffix(f[2], ".journal~") if journal { journalBytes += n if !journals { continue } } files = append(files, LogFile{Path: f[2], Bytes: n, Size: human(n), Modified: f[1], Journal: journal}) } sort.Slice(files, func(i, j int) bool { return files[i].Bytes > files[j].Bytes }) count := len(files) if len(files) > limit { files = files[:limit] } return map[string]any{"under": LogRoot, "files": count, "total_bytes": total, "total": human(total), "journal_bytes": journalBytes, "journal": human(journalBytes), "journals_listed": journals, "largest": files}, nil } func human(n int64) string { units := []string{"B", "K", "M", "G", "T"} f := float64(n) i := 0 for f >= 1024 && i < len(units)-1 { f /= 1024 i++ } if i == 0 { return fmt.Sprintf("%d%s", n, units[0]) } return fmt.Sprintf("%.1f%s", f, units[i]) } var ruleName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._@+-]*$`) // Force rotates the logs of one rule file now (logrotate -f -v), with the base configuration's // globals before it; or every log, given the base configuration's own name. func (m *Machine) Force(config string, writeTemp func(string) (string, func(), error)) (map[string]any, error) { var args []string switch { case config == path.Base(BaseConf) || config == BaseConf: args = []string{"-f", "-v", BaseConf} case ruleName.MatchString(config): rule := path.Join(RulesDir, config) if _, err := m.ReadFile(rule); err != nil { return nil, fmt.Errorf("%s is not a rule file here: %w", rule, err) } base, err := m.ReadFile(BaseConf) if err != nil { return nil, fmt.Errorf("reading %s: %w", BaseConf, err) } temp, done, err := writeTemp("# The globals of " + BaseConf + ", for forcing " + rule + " alone.\n" + strings.Join(Globals(string(base)), "\n") + "\n") if err != nil { return nil, err } defer done() // logrotate running as root reads only a configuration root owns. if _, err := m.Root("install", "-m", "0644", "-o", "root", "-g", "root", temp, forcedConf); err != nil { return nil, err } defer m.Root("rm", "-f", forcedConf) //nolint:errcheck args = []string{"-f", "-v", forcedConf, rule} default: return nil, fmt.Errorf("%q is neither a file of %s nor %s", config, RulesDir, path.Base(BaseConf)) } r, err := m.RootRan("logrotate", args...) if err != nil { return nil, err } said := lines(r.Stdout + "\n" + r.Stderr) rotated, errs := []string{}, []string{} for _, l := range said { l = strings.TrimSpace(l) switch { case strings.HasPrefix(l, "rotating log "): rotated = append(rotated, strings.TrimSuffix(strings.Fields(strings.TrimPrefix(l, "rotating log "))[0], ",")) case strings.HasPrefix(l, "error:"): errs = append(errs, l) } } if len(said) > 200 { said = said[len(said)-200:] } return map[string]any{"config": config, "ok": r.Status == 0 && len(errs) == 0, "rotated": rotated, "errors": errs, "log": said}, nil } func contains(list []string, want string) bool { for _, s := range list { if s == want { return true } } return false }