mesh-catalog/modules/dnsmasq/cmd/dnsmasq-tools/watch.go

package main

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
)

// The watch is the module's long-running process, as root. Every LookEvery it asks the resolver its real
// question on every listen address (look.go); after Confirm failing looks in a row it restarts the
// resolver's unit, within a budget, and writes what it saw and did for the tools, which say it to the mesh
// (novox/hq issue 348).
//
// **Why the module restarts its own resolver.** A resolver that stops answering on a machine can cut that
// machine off from the bus by name, and then nothing outside the machine — neither the gate's rollback nor
// a healer in the controller — reaches it (ADR 0236 on the network path). The repair has to be made where
// the resolver runs, as the identity provider repairs its own administrator (ADR 0224 §5). It is braked,
// and it never says it worked: its next look does. A restart that did not help is said by the health
// tool as the resolver's failure, which the node-engine states and the controller raises — the operator
// hears of it only then.

// The watch's clocks and its budget.
var (
	LookEvery = 15 * time.Second
	// Confirm failing looks in a row are a resolver that does not answer: one look can be wrong (issue 277).
	Confirm = 2
	// RestartGap is the least time between two restarts; RestartsPerHour the most within an hour.
	RestartGap      = 2 * time.Minute
	RestartsPerHour = 3
	// HelpedWithin is how soon after a restart a healthy look says the restart helped.
	HelpedWithin = time.Minute
)

// Unit is the resolver's service.
const Unit = "dnsmasq.service"

// StateFile is where the watch writes what it saw, for the tools to read: readable by the operator
// account the tools run as, written only by the watch.
const StateFile = "/var/lib/dnsmasq-watch/watch.json"

// Restart is one restart the watch made.
type Restart struct {
	At  time.Time `json:"at"`
	Why string    `json:"why"`
	// Error is what the service manager said when the restart itself failed.
	Error string `json:"error,omitempty"`
	// Helped is the next looks' word: true once a look within HelpedWithin answered rightly, false once
	// HelpedWithin passed without one, absent until then.
	Helped *bool `json:"helped,omitempty"`
}

// State is the watch's last look and what it did, as the tools read it.
type State struct {
	Last Look `json:"last"`
	// Failing is how many looks in a row failed; Since when the first of them looked.
	Failing int        `json:"failing"`
	Since   *time.Time `json:"since,omitempty"`
	// Restarts are the restarts within the last hour and the last one ever, newest last.
	Restarts []Restart `json:"restarts,omitempty"`
	// Held is why a restart that was due was not made: the budget is spent, or the unit is not running.
	Held string `json:"held,omitempty"`
	// Unit is the resolver unit's state at the last look, as the service manager said it: active,
	// activating, inactive, failed …
	Unit string `json:"unit,omitempty"`
}

// Runner runs one command; injected, so the watch is tested over recorded answers.
type Runner func(ctx context.Context, name string, args ...string) (string, error)

// ExecRunner runs a command on the machine, bounded to a minute.
func ExecRunner(ctx context.Context, name string, args ...string) (string, error) {
	ctx, cancel := context.WithTimeout(ctx, time.Minute)
	defer cancel()
	var out bytes.Buffer
	cmd := exec.CommandContext(ctx, name, args...)
	cmd.Stdout, cmd.Stderr = &out, &out
	err := cmd.Run()
	return strings.TrimSpace(out.String()), err
}

// Step is one look of the watch and what follows from it: the state after, and whether to restart and why.
// unit is the resolver unit's state at the look. **Only a unit that runs is restarted** (review of
// mesh-catalog PR 161): one stopped or disabled on purpose is a person's, one still starting — waiting at
// boot for the private address — or failed is the service manager's, and none of them is a resolver that
// runs and does not answer. Pure.
func Step(s State, l Look, unit string) (State, string) {
	now := l.At
	s.Last, s.Unit = l, unit
	// What the earlier restarts are now known to have done.
	for i := range s.Restarts {
		r := &s.Restarts[i]
		if r.Helped != nil || r.Error != "" {
			continue
		}
		switch {
		case l.Healthy && now.Sub(r.At) <= HelpedWithin:
			yes := true
			r.Helped = &yes
		case now.Sub(r.At) > HelpedWithin:
			no := false
			r.Helped = &no
		}
	}
	// The budget's window: the last hour, and always the newest.
	kept := s.Restarts[:0]
	for i, r := range s.Restarts {
		if now.Sub(r.At) <= time.Hour || i == len(s.Restarts)-1 {
			kept = append(kept, r)
		}
	}
	s.Restarts = kept
	if l.Healthy {
		s.Failing, s.Since, s.Held = 0, nil, ""
		return s, ""
	}
	if s.Failing == 0 {
		at := now
		s.Since = &at
	}
	s.Failing++
	if s.Failing < Confirm || len(l.Problems) > 0 && len(l.Answers) == 0 {
		// Not yet confirmed; or the look could not ask at all — a restart answers nothing a missing
		// configuration or roster is missing.
		return s, ""
	}
	if unit != "active" {
		s.Held = fmt.Sprintf("the unit is %s, not running: the watch restarts only a resolver that runs and does not "+
			"answer — one stopped on purpose, still starting or failed is left to the service manager", orUnknown(unit))
		return s, ""
	}
	within := 0
	var newest *Restart
	for i, r := range s.Restarts {
		if now.Sub(r.At) <= time.Hour {
			within++
		}
		newest = &s.Restarts[i]
	}
	switch {
	case newest != nil && now.Sub(newest.At) < RestartGap:
		s.Held = ""
		return s, "" // the last restart is still settling
	case within >= RestartsPerHour:
		s.Held = fmt.Sprintf("%d restart(s) within the last hour, the most the watch makes; it restarts nothing "+
			"more until the oldest is an hour old, and the resolver's failure is said as its health", within)
		return s, ""
	}
	s.Held = ""
	return s, fmt.Sprintf("%d look(s) in a row since %s failed: %s", s.Failing, s.Since.Format("15:04:05 UTC"), l.Why())
}

// Watch runs the watch until ctx ends.
func (m *Machine) Watch(ctx context.Context, run Runner, write func(State) error) error {
	var s State
	if raw, err := m.read(StateFile); err == nil {
		_ = json.Unmarshal([]byte(raw), &s) // the budget survives the watch's own restart
	}
	tick := time.NewTicker(LookEvery)
	defer tick.Stop()
	for {
		l := m.Look(ctx)
		unit, err := run(ctx, "systemctl", "show", "--property=ActiveState", "--value", Unit)
		if err != nil {
			unit = ""
		}
		next, why := Step(s, l, strings.TrimSpace(unit))
		if why != "" {
			r := Restart{At: m.Now().UTC(), Why: why}
			// Not waited for: a start that waits for the private address is the service manager's to time,
			// and the next looks say whether it answers.
			if out, err := run(ctx, "systemctl", "restart", "--no-block", Unit); err != nil {
				r.Error = strings.TrimSpace(err.Error() + " " + out)
			}
			fmt.Fprintf(os.Stderr, "dnsmasq-watch: restarted %s: %s%s\n", Unit, why, orEmpty(" — the restart failed: ", r.Error))
			next.Restarts = append(next.Restarts, r)
		} else if !l.Healthy && next.Failing == Confirm {
			fmt.Fprintf(os.Stderr, "dnsmasq-watch: the resolver does not answer: %s%s\n", l.Why(), orEmpty("; ", next.Held))
		}
		s = next
		if err := write(s); err != nil {
			fmt.Fprintln(os.Stderr, "dnsmasq-watch: what it saw could not be written:", err)
		}
		select {
		case <-ctx.Done():
			return nil
		case <-tick.C:
		}
	}
}

func orUnknown(s string) string {
	if s == "" {
		return "not known (the service manager did not say)"
	}
	return s
}

func orEmpty(prefix, s string) string {
	if s == "" {
		return ""
	}
	return prefix + s
}

// WriteState writes the state whole, replacing the file in one rename.
func (m *Machine) WriteState(s State) error {
	path := filepath.Join(m.Root, StateFile)
	body, err := json.MarshalIndent(s, "", "  ")
	if err != nil {
		return err
	}
	tmp := path + ".new"
	if err := os.WriteFile(tmp, body, 0o644); err != nil {
		return err
	}
	return os.Rename(tmp, path)
}

// ReadState is the watch's last state, and false when it has written none.
func (m *Machine) ReadState() (State, bool, error) {
	raw, err := m.read(StateFile)
	if errors.Is(err, os.ErrNotExist) {
		return State{}, false, nil
	}
	if err != nil {
		return State{}, false, err
	}
	var s State
	if err := json.Unmarshal([]byte(raw), &s); err != nil {
		return State{}, false, fmt.Errorf("%s is not the watch's: %w", StateFile, err)
	}
	return s, true, nil
}