jochen
8bcf787258
A VPN client rewrote the laptop's resolver file and nothing said so. The engine now states its machine's networking; the controller keeps it with the machine's health (migration 0077) and raises the rewrite as its own finding naming the writer, the machine's own faults as machine.<m>.network, and what several machines cannot reach once, there. The gate waits on a rewrite it did not make rather than putting back a good build.
360 lines
12 KiB
Go
360 lines
12 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// A machine says how its network is (novox/hq ADR 0241, which extends ADR 0240 from what a module runs
|
|
// to the machine it runs on).
|
|
//
|
|
// **Every machine's node-engine judges its own networking** — the resolver file the uplink holder
|
|
// declared, the names through every resolver it lists, the tunnel's handshake with the hub, the bus, the
|
|
// default route — on the two-look rule, and states it beside its resources. The controller keeps the
|
|
// newest statement per machine and raises from all of them together:
|
|
//
|
|
// - **an outside writer of the resolver file is its own finding**, `machine.<m>.<owner>.rewritten`:
|
|
// the file the uplink holder declares was rewritten by another program, named where the engine could
|
|
// name it. The names failing through what that program wrote are that finding's consequence, said in
|
|
// it — never a second condition;
|
|
// - **what is the machine's own** — its route, its tunnel, its resolvers answering wrong, a resolver
|
|
// that is no mesh machine — is `machine.<m>.network`;
|
|
// - **what points at another machine is said once, there** (the provider hold of ADR 0240 rule 5, for
|
|
// the network): a failure toward the hub or toward a mesh resolver is held under that machine when it
|
|
// is down on the record — silent, or its own network unhealthy — or when a second machine finds the
|
|
// same; then `machine.<x>.unreachable` names every machine that cannot reach it, and none of them
|
|
// raises anything of its own for it. One machine alone failing toward a healthy one is its own.
|
|
//
|
|
// Each is a warning; urgent on the control node, or when the bus cannot be reached, or for the hub.
|
|
// Cleared on the first statement that no longer says it. An engine older than this judging says nothing
|
|
// of its network, and nothing is raised for it.
|
|
|
|
// The conditions a machine's network raises.
|
|
const (
|
|
kindMachineNetwork = "machine-network"
|
|
kindNetworkRewritten = "network-rewritten"
|
|
kindNetworkUnreachable = "network-unreachable"
|
|
sourceNetwork = "network"
|
|
)
|
|
|
|
// networkKinds are the kinds this judging owns: every open one it no longer says, it clears.
|
|
var networkKinds = []string{kindMachineNetwork, kindNetworkRewritten, kindNetworkUnreachable}
|
|
|
|
// networkFacts is what one judging of every machine's network reads.
|
|
type networkFacts struct {
|
|
healths map[string]inventory.NodeHealth
|
|
// byAddress is each machine's address on the private network; hub the hub's name; control the
|
|
// control node's.
|
|
byAddress map[string]string
|
|
hub string
|
|
control string
|
|
// silent is every machine whose silence is an open condition.
|
|
silent map[string]bool
|
|
}
|
|
|
|
// judgeNetworks raises and clears every machine's network conditions from every machine's newest
|
|
// statement, after one machine's statement was kept.
|
|
func judgeNetworks(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, now time.Time) error {
|
|
healths, err := inv.Healths(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
overlays, err := inv.Overlays(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
open, err := k.Open(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
f := networkFacts{healths: healths, byAddress: map[string]string{}, control: controlHost(ctx, inv),
|
|
silent: map[string]bool{}}
|
|
for _, o := range overlays {
|
|
if o.Address != "" {
|
|
f.byAddress[o.Address] = o.Name
|
|
}
|
|
if o.Hub {
|
|
f.hub = o.Name
|
|
}
|
|
}
|
|
for _, c := range open {
|
|
if c.Subject.Scope == conditions.ScopeMachine && c.Kind == "silent" {
|
|
f.silent[c.Subject.ID] = true
|
|
}
|
|
}
|
|
var problems []string
|
|
said := map[string]bool{}
|
|
for _, o := range networkObservations(f) {
|
|
said[o.Key()] = true
|
|
if _, err := k.Observe(ctx, o); err != nil {
|
|
problems = append(problems, err.Error())
|
|
}
|
|
}
|
|
for _, c := range open {
|
|
if !slices.Contains(networkKinds, c.Kind) || said[c.Key] {
|
|
continue
|
|
}
|
|
why := "no machine says it any more"
|
|
if c.Subject.Machine != "" {
|
|
why = c.Subject.Machine + "'s network no longer says it"
|
|
}
|
|
if _, err := k.Clear(ctx, c.Key, why); err != nil {
|
|
problems = append(problems, err.Error())
|
|
}
|
|
}
|
|
if len(problems) > 0 {
|
|
return fmt.Errorf("%s", strings.Join(problems, "; "))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// pointed is one machine's failing part that points at another machine.
|
|
type pointed struct {
|
|
from string
|
|
part inventory.NetworkPart
|
|
}
|
|
|
|
// networkObservations is every network condition the statements say now. Pure.
|
|
func networkObservations(f networkFacts) []conditions.Observation {
|
|
machines := make([]string, 0, len(f.healths))
|
|
for m := range f.healths {
|
|
machines = append(machines, m)
|
|
}
|
|
sort.Strings(machines)
|
|
|
|
unhealthy := func(m string) []inventory.NetworkPart {
|
|
h := f.healths[m]
|
|
if h.Network == nil {
|
|
return nil
|
|
}
|
|
var out []inventory.NetworkPart
|
|
for _, p := range h.Network.Parts {
|
|
if p.State == link.StateUnhealthy {
|
|
out = append(out, p)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
// The machines a part points at, other than its own: the hub, and each mesh resolver by its address.
|
|
// An address that is no mesh machine's — the resolver a VPN client wrote in — is the machine's own.
|
|
targets := func(m string, p inventory.NetworkPart) ([]string, bool) {
|
|
if len(p.Toward) == 0 {
|
|
return nil, false
|
|
}
|
|
var out []string
|
|
for _, t := range p.Toward {
|
|
x := f.byAddress[t]
|
|
if t == link.TowardHub {
|
|
x = f.hub
|
|
}
|
|
if x == "" || x == m {
|
|
return nil, false
|
|
}
|
|
if !slices.Contains(out, x) {
|
|
out = append(out, x)
|
|
}
|
|
}
|
|
return out, true
|
|
}
|
|
|
|
// First pass: what points at whom.
|
|
pointing := map[string][]pointed{}
|
|
for _, m := range machines {
|
|
for _, p := range unhealthy(m) {
|
|
if xs, ok := targets(m, p); ok {
|
|
for _, x := range xs {
|
|
pointing[x] = append(pointing[x], pointed{from: m, part: p})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
from := func(x string) []string {
|
|
var out []string
|
|
for _, pt := range pointing[x] {
|
|
if !slices.Contains(out, pt.from) {
|
|
out = append(out, pt.from)
|
|
}
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
// A machine is down on the record when its silence is open or its own network is unhealthy, or when
|
|
// two machines find it unreachable: then what points at it is held there.
|
|
down := func(x string) bool {
|
|
return f.silent[x] || len(unhealthy(x)) > 0 || len(from(x)) >= 2
|
|
}
|
|
|
|
var out []conditions.Observation
|
|
saysOwn := map[string]bool{}
|
|
for _, m := range machines {
|
|
parts := unhealthy(m)
|
|
if len(parts) == 0 {
|
|
continue
|
|
}
|
|
var own []inventory.NetworkPart
|
|
var rewritten *inventory.NetworkPart
|
|
for i, p := range parts {
|
|
if p.Part == link.PartResolvConf {
|
|
rewritten = &parts[i]
|
|
continue
|
|
}
|
|
if xs, ok := targets(m, p); ok && allDown(xs, down) {
|
|
continue // held at the machines it points at
|
|
}
|
|
own = append(own, p)
|
|
}
|
|
if rewritten != nil {
|
|
out = append(out, rewrittenObservation(m, *rewritten, parts, f))
|
|
// The names failing through what another program wrote are that finding's, said in it.
|
|
kept := own[:0]
|
|
for _, p := range own {
|
|
if p.Part != link.PartNames {
|
|
kept = append(kept, p)
|
|
}
|
|
}
|
|
own = kept
|
|
}
|
|
if len(own) > 0 {
|
|
out = append(out, machineNetworkObservation(m, own, f, from(m)))
|
|
saysOwn[m] = true
|
|
}
|
|
}
|
|
// Said once, at the machine everybody points at — unless its own network condition already says it
|
|
// (listed there), or its silence does.
|
|
targetsSorted := make([]string, 0, len(pointing))
|
|
for x := range pointing {
|
|
targetsSorted = append(targetsSorted, x)
|
|
}
|
|
sort.Strings(targetsSorted)
|
|
for _, x := range targetsSorted {
|
|
if !down(x) || saysOwn[x] || f.silent[x] {
|
|
continue
|
|
}
|
|
out = append(out, unreachableObservation(x, pointing[x], from(x), f))
|
|
}
|
|
return out
|
|
}
|
|
|
|
func allDown(xs []string, down func(string) bool) bool {
|
|
for _, x := range xs {
|
|
if !down(x) {
|
|
return false
|
|
}
|
|
}
|
|
return len(xs) > 0
|
|
}
|
|
|
|
// rewrittenObservation is the resolver file rewritten by another program: its own finding, naming the
|
|
// writer where the engine could, and what it costs the machine.
|
|
func rewrittenObservation(m string, p inventory.NetworkPart, all []inventory.NetworkPart, f networkFacts) conditions.Observation {
|
|
writer := ""
|
|
if p.Writer != "" {
|
|
writer = " (" + p.Writer + ")"
|
|
}
|
|
cost := "the names through it are not yet judged"
|
|
said := []string{p.Part + ": " + p.Said}
|
|
for _, q := range all {
|
|
if q.Part == link.PartNames {
|
|
cost = strings.TrimSuffix(q.Reason, ".")
|
|
said = append(said, q.Part+": "+q.Said)
|
|
}
|
|
}
|
|
if cost == "the names through it are not yet judged" {
|
|
cost = "the names still resolve through what it wrote"
|
|
}
|
|
id := m
|
|
if p.Owner != "" {
|
|
// Named by the module whose file it is: a send that moved that module is what the gate
|
|
// holds it on (issue 281's rule — what names a moved module is that module's).
|
|
id = m + "." + p.Owner
|
|
}
|
|
severity := conditions.Warning
|
|
if m == f.control {
|
|
severity = conditions.Urgent
|
|
}
|
|
summary := fmt.Sprintf("the resolver file on %s was rewritten by another program%s — %s until the "+
|
|
"node-engine writes it back at its next reconcile, or that program gives it back", m, writer, cost)
|
|
if p.Owner != "" {
|
|
summary = fmt.Sprintf("the resolver file %s writes on %s was rewritten by another program%s — %s until "+
|
|
"the node-engine writes it back at its next reconcile, or that program gives it back", p.Owner, m, writer, cost)
|
|
}
|
|
return conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: "rewritten", Kind: kindNetworkRewritten,
|
|
Machine: m, Severity: severity, Source: sourceNetwork, Summary: summary,
|
|
Said: fmt.Sprintf("since %s: %s", p.Since.UTC().Format("2006-01-02 15:04:05 MST"), strings.Join(said, " | "))}
|
|
}
|
|
|
|
// machineNetworkObservation is what is wrong with a machine's own networking.
|
|
func machineNetworkObservation(m string, parts []inventory.NetworkPart, f networkFacts, waiting []string) conditions.Observation {
|
|
var words, said []string
|
|
severity := conditions.Warning
|
|
for _, p := range parts {
|
|
words = append(words, p.Reason)
|
|
said = append(said, fmt.Sprintf("%s since %s: %s", p.Part, p.Since.UTC().Format("2006-01-02 15:04:05 MST"), p.Said))
|
|
if p.Part == link.PartBus {
|
|
severity = conditions.Urgent
|
|
}
|
|
}
|
|
if m == f.control || m == f.hub {
|
|
severity = conditions.Urgent
|
|
}
|
|
summary := fmt.Sprintf("%s's network is not healthy: %s", m, strings.Join(words, "; "))
|
|
if len(waiting) > 0 {
|
|
severity = conditions.Urgent
|
|
summary += fmt.Sprintf("; %s cannot reach it", strings.Join(waiting, ", "))
|
|
}
|
|
return conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Token: "network", Kind: kindMachineNetwork,
|
|
Machine: m, Severity: severity, Source: sourceNetwork, Summary: summary, Said: strings.Join(said, " | ")}
|
|
}
|
|
|
|
// unreachableObservation is one machine others cannot reach, said once there.
|
|
func unreachableObservation(x string, pts []pointed, from []string, f networkFacts) conditions.Observation {
|
|
var what []string
|
|
var said []string
|
|
for _, pt := range pts {
|
|
w := map[string]string{link.PartTunnel: "the tunnel to it", link.PartBus: "the bus on it",
|
|
link.PartNames: "its resolver"}[pt.part.Part]
|
|
if w == "" {
|
|
w = pt.part.Part
|
|
}
|
|
if !slices.Contains(what, w) {
|
|
what = append(what, w)
|
|
}
|
|
said = append(said, fmt.Sprintf("%s: %s: %s", pt.from, pt.part.Part, pt.part.Said))
|
|
}
|
|
severity := conditions.Warning
|
|
if x == f.hub || x == f.control || slices.Contains(what, "the bus on it") {
|
|
severity = conditions.Urgent
|
|
}
|
|
return conditions.Observation{Scope: conditions.ScopeMachine, ID: x, Token: "unreachable", Kind: kindNetworkUnreachable,
|
|
Machine: x, Also: from, Severity: severity, Source: sourceNetwork,
|
|
Summary: fmt.Sprintf("%s cannot be reached from %s: %s", x, strings.Join(from, ", "), strings.Join(what, ", ")),
|
|
Said: strings.Join(said, " | ")}
|
|
}
|
|
|
|
// networkLines is what `node show` says of a machine's networking.
|
|
func networkLines(h inventory.NodeHealth, had bool, now time.Time) []string {
|
|
if !had || h.Network == nil {
|
|
return []string{" its node-engine does not say how its network is — it is older than that judging (ADR 0241)"}
|
|
}
|
|
out := []string{fmt.Sprintf(" its network: %s since %s", h.Network.State, h.Network.Since.Local().Format("2006-01-02 15:04"))}
|
|
for _, p := range h.Network.Parts {
|
|
line := fmt.Sprintf(" %-10s %s", p.State, p.Part)
|
|
if p.Reason != "" {
|
|
line += " — " + p.Reason
|
|
}
|
|
if p.Writer != "" {
|
|
line += " (" + p.Writer + ")"
|
|
}
|
|
out = append(out, line)
|
|
}
|
|
return out
|
|
}
|