mesh-catalog/modules/fail2ban/cmd/fail2ban-tools/watch.go

package main

// The watch (novox/hq ADR 0339 §2 and §4, issue 238): the module's long-running process, as root. Every look it
//
//   - writes the operator's own addresses from the setting into the list every jail ignores (own.go);
//   - asks the daemon whether every jail's ignore list names that list, so no jail leaves it out;
//   - reads what the daemon logged since the last look, and keeps every failure a jail ignored because the
//     address is one of the operator's own, for a day.
//
// What it saw goes to StateFile, which the health tool reads as the operator's account. A failure from one
// of the operator's own addresses is never banned — that is the point — and never silent: the module's health
// says it, so the controller raises it as a warning naming the node, the jail, the address and the count, and
// it clears a day after the last one. A loop like the one of 2026-10-04 is then seen within minutes.

import (
	"bufio"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"time"
)

// The watch's timing.
var (
	// LookEvery is how often the watch looks.
	LookEvery = 30 * time.Second
	// Remembered is how long an ignored failure is said: a day, as the record has it.
	Remembered = 24 * time.Hour
	// StaleAfter is how old the watch's last look may be before the health says it stopped looking.
	StaleAfter = 3 * time.Minute
	// keptPerAddress bounds what is remembered of one address in one jail; the count says when it met it.
	keptPerAddress = 10000
)

// IgnoredFailures is the failures one jail ignored from one of the operator's own addresses, in the last day.
type IgnoredFailures struct {
	Jail    string    `json:"jail"`
	Address string    `json:"address"`
	Count   int       `json:"count"`
	First   time.Time `json:"first"`
	Last    time.Time `json:"last"`
	// AtLeast says the count met what is remembered of one address and is a floor.
	AtLeast bool `json:"at-least,omitempty"`
}

// State is what the watch saw at its last look.
type State struct {
	Looked time.Time `json:"looked"`
	// Own is the operator's own addresses as written into the list, and Problems the entries left out.
	Own      []string `json:"own"`
	Problems []string `json:"problems,omitempty"`
	// Unread is why something the watch reads could not be read at this look.
	Unread []string `json:"unread,omitempty"`
	// Jails is how many jails the daemon runs; Missing those whose ignore list does not name the list.
	Jails   int      `json:"jails"`
	Missing []string `json:"missing,omitempty"`
	// Ignored is every jail and address of the operator's own with a failure ignored in the last day.
	Ignored []IgnoredFailures `json:"ignored"`
}

// ignoredLine is the daemon's line for a failure it ignored: `<time>,<ms> fail2ban.filter [<pid>]: INFO
// [<jail>] Ignore <address> by <source>` (fail2ban 1.1, server/filter.py logIgnoreIp).
var ignoredLine = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}),\d+ .*?\]:\s+\S+\s+\[([^\]]+)\] Ignore (\S+) by (\S+)\s*$`)

// Watcher is the watch's memory between looks: where it is in the log and what it saw there.
type Watcher struct {
	F    Fail2ban
	Root string
	Now  func() time.Time

	logFile os.FileInfo
	offset  int64
	partial string
	seen    map[[2]string][]time.Time
}

// Look is one look: the list written, the jails asked, the log read, the state answered.
func (w *Watcher) Look(ctx context.Context) State {
	now := w.Now()
	s := State{Looked: now, Own: []string{}, Ignored: []IgnoredFailures{}}
	own, err := ReadOwn(w.Root)
	if err != nil {
		s.Unread = append(s.Unread, err.Error())
	} else {
		s.Own, s.Problems = own.Listed, own.Problems
		if _, err := WriteList(w.Root, own); err != nil {
			s.Unread = append(s.Unread, fmt.Sprintf("the list every jail ignores (%s) cannot be written: %v", ListFile, err))
		}
	}
	if jails, err := w.F.Jails(ctx); err != nil {
		s.Unread = append(s.Unread, fmt.Sprintf("the daemon cannot be asked for its jails: %v", err))
	} else {
		s.Jails = len(jails)
		for _, j := range jails {
			out, err := w.F.client(ctx, "get", j, "ignoreip")
			if err != nil {
				s.Unread = append(s.Unread, fmt.Sprintf("the jail %s's ignore list cannot be read: %v", j, err))
				continue
			}
			if !namesOurList(listed(out)) {
				s.Missing = append(s.Missing, j)
			}
		}
	}
	if err := w.readLog(now); err != nil {
		s.Unread = append(s.Unread, err.Error())
	}
	s.Ignored = w.ignored(now)
	return s
}

func namesOurList(entries []string) bool {
	for _, e := range entries {
		if isOurList(e) {
			return true
		}
	}
	return false
}

func contains(list []string, want string) bool {
	for _, l := range list {
		if l == want {
			return true
		}
	}
	return false
}

// readLog reads what the daemon logged since the last look. A log rotated away is read from its start; the
// first look reads the whole of it, so a watch started again still knows the last day.
func (w *Watcher) readLog(now time.Time) error {
	path := filepath.Join(w.Root, LogFile)
	f, err := os.Open(path)
	if err != nil {
		return fmt.Errorf("the daemon's log %s cannot be read: %w", LogFile, err)
	}
	defer f.Close()
	info, err := f.Stat()
	if err != nil {
		return fmt.Errorf("the daemon's log %s cannot be read: %w", LogFile, err)
	}
	if w.logFile == nil || !os.SameFile(w.logFile, info) || info.Size() < w.offset {
		w.offset, w.partial = 0, ""
	}
	w.logFile = info
	if _, err := f.Seek(w.offset, io.SeekStart); err != nil {
		return fmt.Errorf("the daemon's log %s cannot be read: %w", LogFile, err)
	}
	r := bufio.NewReader(f)
	for {
		line, err := r.ReadString('\n')
		w.offset += int64(len(line))
		if err != nil {
			// A line the daemon is still writing is kept for the next look.
			w.partial += line
			break
		}
		w.take(w.partial+strings.TrimRight(line, "\n"), now)
		w.partial = ""
	}
	return nil
}

// take keeps one line of the log when it is a failure ignored because of the operator's own addresses.
func (w *Watcher) take(line string, now time.Time) {
	m := ignoredLine.FindStringSubmatch(line)
	if m == nil || !isOurList(m[4]) {
		return
	}
	at, err := time.ParseInLocation("2006-01-02 15:04:05", m[1], time.Local)
	if err != nil || now.Sub(at) > Remembered {
		return
	}
	if w.seen == nil {
		w.seen = map[[2]string][]time.Time{}
	}
	key := [2]string{m[2], m[3]}
	times := append(w.seen[key], at)
	if len(times) > keptPerAddress {
		times = times[len(times)-keptPerAddress:]
	}
	w.seen[key] = times
}

// ignored is what was seen in the last day, by jail and address; anything older is forgotten.
func (w *Watcher) ignored(now time.Time) []IgnoredFailures {
	out := []IgnoredFailures{}
	for key, times := range w.seen {
		kept := times[:0]
		for _, at := range times {
			if now.Sub(at) <= Remembered {
				kept = append(kept, at)
			}
		}
		if len(kept) == 0 {
			delete(w.seen, key)
			continue
		}
		w.seen[key] = kept
		sort.Slice(kept, func(i, j int) bool { return kept[i].Before(kept[j]) })
		out = append(out, IgnoredFailures{Jail: key[0], Address: key[1], Count: len(kept), First: kept[0],
			Last: kept[len(kept)-1], AtLeast: len(kept) >= keptPerAddress})
	}
	sort.Slice(out, func(i, j int) bool {
		if out[i].Jail != out[j].Jail {
			return out[i].Jail < out[j].Jail
		}
		return out[i].Address < out[j].Address
	})
	return out
}

// WriteState writes the state for the health tool, readable by the operator's account, by a rename.
func WriteState(root string, s State) error {
	raw, err := json.MarshalIndent(s, "", "  ")
	if err != nil {
		return err
	}
	path := filepath.Join(root, StateFile)
	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
		return err
	}
	tmp := path + ".new"
	if err := os.WriteFile(tmp, append(raw, '\n'), 0o644); err != nil {
		return err
	}
	return os.Rename(tmp, path)
}

// Watch looks until it is stopped.
func (w *Watcher) Watch(ctx context.Context) error {
	for {
		if err := WriteState(w.Root, w.Look(ctx)); err != nil {
			fmt.Fprintf(os.Stderr, "[fail2ban] what the watch saw cannot be written to %s: %v\n", StateFile, err)
		}
		select {
		case <-ctx.Done():
			return nil
		case <-time.After(LookEvery):
		}
	}
}

// Health is the module's health as the node-engine reads it (novox/hq ADR 0240): healthy, and why.
type Health struct {
	Healthy bool   `json:"healthy"`
	Why     string `json:"why"`
	State   *State `json:"seen,omitempty"`
}

// ReadHealth judges what the watch last saw. Not healthy when the watch is not looking, when something it
// reads cannot be read, when the setting holds an entry that is not an address, when a jail's ignore list
// leaves the operator's own addresses out, and when a jail ignored a failure from one in the last day — each
// said, so the controller's warning names the jail, the address and the count (ADR 0339 §4).
func ReadHealth(root string, now time.Time) Health {
	raw, err := os.ReadFile(filepath.Join(root, StateFile))
	if err != nil {
		return Health{Why: fmt.Sprintf("the watch has written nothing this tool can read (%v): it is not looking", err)}
	}
	var s State
	if err := json.Unmarshal(raw, &s); err != nil {
		return Health{Why: fmt.Sprintf("what the watch wrote is not its state: %v", err)}
	}
	if age := now.Sub(s.Looked); age > StaleAfter {
		return Health{State: &s, Why: fmt.Sprintf("the watch last looked at %s, %s ago: it is not looking",
			s.Looked.Local().Format("2006-01-02 15:04:05"), age.Round(time.Second))}
	}
	var wrong []string
	wrong = append(wrong, s.Unread...)
	if len(s.Problems) > 0 {
		wrong = append(wrong, fmt.Sprintf("the setting %s holds what is not an address or a range, left out: %s",
			SettingKey, strings.Join(s.Problems, ", ")))
	}
	if len(s.Missing) > 0 {
		wrong = append(wrong, fmt.Sprintf("the jail(s) %s do not ignore the operator's own addresses: their ignore "+
			"list lacks %s (novox/hq ADR 0339 §2)", strings.Join(s.Missing, ", "), IgnoredBy))
	}
	for _, i := range s.Ignored {
		count := fmt.Sprint(i.Count)
		if i.AtLeast {
			count = "at least " + count
		}
		wrong = append(wrong, fmt.Sprintf("the jail %s counted %s failure(s) from %s, one of the operator's own "+
			"addresses, in the last day, the last at %s, and banned none (novox/hq ADR 0339); it clears a day after "+
			"the last", i.Jail, count, i.Address, i.Last.Local().Format("2006-01-02 15:04:05")))
	}
	if len(wrong) > 0 {
		return Health{State: &s, Why: strings.Join(wrong, "; ")}
	}
	return Health{Healthy: true, State: &s, Why: fmt.Sprintf("every jail (%d) ignores the operator's own addresses "+
		"(%d entr%s), and none failed in the last day", s.Jails, len(s.Own), plural(len(s.Own)))}
}

func plural(n int) string {
	if n == 1 {
		return "y"
	}
	return "ies"
}