package main import ( "bytes" "context" "encoding/json" "errors" "fmt" "os" "os/exec" "path/filepath" "strings" "time" ) // The watch is the module's long-running process, as root. Every LookEvery it asks the resolver its real // question on every listen address (look.go); after Confirm failing looks in a row it restarts the // resolver's unit, within a budget, and writes what it saw and did for the tools, which say it to the mesh // (novox/hq issue 348). // // **Why the module restarts its own resolver.** A resolver that stops answering on a machine can cut that // machine off from the bus by name, and then nothing outside the machine — neither the gate's rollback nor // a healer in the controller — reaches it (ADR 0236 on the network path). The repair has to be made where // the resolver runs, as the identity provider repairs its own administrator (ADR 0224 §5). It is braked, // and it never says it worked: its next look does. A restart that did not help is said by the health // tool as the resolver's failure, which the node-engine states and the controller raises — the operator // hears of it only then. // The watch's clocks and its budget. var ( LookEvery = 15 * time.Second // Confirm failing looks in a row are a resolver that does not answer: one look can be wrong (issue 277). Confirm = 2 // RestartGap is the least time between two restarts; RestartsPerHour the most within an hour. RestartGap = 2 * time.Minute RestartsPerHour = 3 // HelpedWithin is how soon after a restart a healthy look says the restart helped. HelpedWithin = time.Minute ) // Unit is the resolver's service. const Unit = "dnsmasq.service" // StateFile is where the watch writes what it saw, for the tools to read: readable by the operator // account the tools run as, written only by the watch. const StateFile = "/var/lib/dnsmasq-watch/watch.json" // Restart is one restart the watch made. type Restart struct { At time.Time `json:"at"` Why string `json:"why"` // Error is what the service manager said when the restart itself failed. Error string `json:"error,omitempty"` // Helped is the next looks' word: true once a look within HelpedWithin answered rightly, false once // HelpedWithin passed without one, absent until then. Helped *bool `json:"helped,omitempty"` } // State is the watch's last look and what it did, as the tools read it. type State struct { Last Look `json:"last"` // Failing is how many looks in a row failed; Since when the first of them looked. Failing int `json:"failing"` Since *time.Time `json:"since,omitempty"` // Restarts are the restarts within the last hour and the last one ever, newest last. Restarts []Restart `json:"restarts,omitempty"` // Held is why a restart that was due was not made: the budget is spent, or the unit is not running. Held string `json:"held,omitempty"` // Unit is the resolver unit's state at the last look, as the service manager said it: active, // activating, inactive, failed … Unit string `json:"unit,omitempty"` } // Runner runs one command; injected, so the watch is tested over recorded answers. type Runner func(ctx context.Context, name string, args ...string) (string, error) // ExecRunner runs a command on the machine, bounded to a minute. func ExecRunner(ctx context.Context, name string, args ...string) (string, error) { ctx, cancel := context.WithTimeout(ctx, time.Minute) defer cancel() var out bytes.Buffer cmd := exec.CommandContext(ctx, name, args...) cmd.Stdout, cmd.Stderr = &out, &out err := cmd.Run() return strings.TrimSpace(out.String()), err } // Step is one look of the watch and what follows from it: the state after, and whether to restart and why. // unit is the resolver unit's state at the look. **Only a unit that runs is restarted** (review of // mesh-catalog PR 161): one stopped or disabled on purpose is a person's, one still starting — waiting at // boot for the private address — or failed is the service manager's, and none of them is a resolver that // runs and does not answer. Pure. func Step(s State, l Look, unit string) (State, string) { now := l.At s.Last, s.Unit = l, unit // What the earlier restarts are now known to have done. for i := range s.Restarts { r := &s.Restarts[i] if r.Helped != nil || r.Error != "" { continue } switch { case l.Healthy && now.Sub(r.At) <= HelpedWithin: yes := true r.Helped = &yes case now.Sub(r.At) > HelpedWithin: no := false r.Helped = &no } } // The budget's window: the last hour, and always the newest. kept := s.Restarts[:0] for i, r := range s.Restarts { if now.Sub(r.At) <= time.Hour || i == len(s.Restarts)-1 { kept = append(kept, r) } } s.Restarts = kept if l.Healthy { s.Failing, s.Since, s.Held = 0, nil, "" return s, "" } if s.Failing == 0 { at := now s.Since = &at } s.Failing++ if s.Failing < Confirm || len(l.Problems) > 0 && len(l.Answers) == 0 { // Not yet confirmed; or the look could not ask at all — a restart answers nothing a missing // configuration or roster is missing. return s, "" } if unit != "active" { s.Held = fmt.Sprintf("the unit is %s, not running: the watch restarts only a resolver that runs and does not "+ "answer — one stopped on purpose, still starting or failed is left to the service manager", orUnknown(unit)) return s, "" } within := 0 var newest *Restart for i, r := range s.Restarts { if now.Sub(r.At) <= time.Hour { within++ } newest = &s.Restarts[i] } switch { case newest != nil && now.Sub(newest.At) < RestartGap: s.Held = "" return s, "" // the last restart is still settling case within >= RestartsPerHour: s.Held = fmt.Sprintf("%d restart(s) within the last hour, the most the watch makes; it restarts nothing "+ "more until the oldest is an hour old, and the resolver's failure is said as its health", within) return s, "" } s.Held = "" return s, fmt.Sprintf("%d look(s) in a row since %s failed: %s", s.Failing, s.Since.Format("15:04:05 UTC"), l.Why()) } // Watch runs the watch until ctx ends. func (m *Machine) Watch(ctx context.Context, run Runner, write func(State) error) error { var s State if raw, err := m.read(StateFile); err == nil { _ = json.Unmarshal([]byte(raw), &s) // the budget survives the watch's own restart } tick := time.NewTicker(LookEvery) defer tick.Stop() for { l := m.Look(ctx) unit, err := run(ctx, "systemctl", "show", "--property=ActiveState", "--value", Unit) if err != nil { unit = "" } next, why := Step(s, l, strings.TrimSpace(unit)) if why != "" { r := Restart{At: m.Now().UTC(), Why: why} // Not waited for: a start that waits for the private address is the service manager's to time, // and the next looks say whether it answers. if out, err := run(ctx, "systemctl", "restart", "--no-block", Unit); err != nil { r.Error = strings.TrimSpace(err.Error() + " " + out) } fmt.Fprintf(os.Stderr, "dnsmasq-watch: restarted %s: %s%s\n", Unit, why, orEmpty(" — the restart failed: ", r.Error)) next.Restarts = append(next.Restarts, r) } else if !l.Healthy && next.Failing == Confirm { fmt.Fprintf(os.Stderr, "dnsmasq-watch: the resolver does not answer: %s%s\n", l.Why(), orEmpty("; ", next.Held)) } s = next if err := write(s); err != nil { fmt.Fprintln(os.Stderr, "dnsmasq-watch: what it saw could not be written:", err) } select { case <-ctx.Done(): return nil case <-tick.C: } } } func orUnknown(s string) string { if s == "" { return "not known (the service manager did not say)" } return s } func orEmpty(prefix, s string) string { if s == "" { return "" } return prefix + s } // WriteState writes the state whole, replacing the file in one rename. func (m *Machine) WriteState(s State) error { path := filepath.Join(m.Root, StateFile) body, err := json.MarshalIndent(s, "", " ") if err != nil { return err } tmp := path + ".new" if err := os.WriteFile(tmp, body, 0o644); err != nil { return err } return os.Rename(tmp, path) } // ReadState is the watch's last state, and false when it has written none. func (m *Machine) ReadState() (State, bool, error) { raw, err := m.read(StateFile) if errors.Is(err, os.ErrNotExist) { return State{}, false, nil } if err != nil { return State{}, false, err } var s State if err := json.Unmarshal([]byte(raw), &s); err != nil { return State{}, false, fmt.Errorf("%s is not the watch's: %w", StateFile, err) } return s, true, nil }