Raise a machine's network from what its engine says, once (hq ADR 0241)
A VPN client rewrote the laptop's resolver file and nothing said so. The engine now states its machine's networking; the controller keeps it with the machine's health (migration 0077) and raises the rewrite as its own finding naming the writer, the machine's own faults as machine.<m>.network, and what several machines cannot reach once, there. The gate waits on a rewrite it did not make rather than putting back a good build.
This commit is contained in:
jochen
@@ -0,0 +1,359 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A machine says how its network is (novox/hq ADR 0241, which extends ADR 0240 from what a module runs
|
||||
// to the machine it runs on).
|
||||
//
|
||||
// **Every machine's node-engine judges its own networking** — the resolver file the uplink holder
|
||||
// declared, the names through every resolver it lists, the tunnel's handshake with the hub, the bus, the
|
||||
// default route — on the two-look rule, and states it beside its resources. The controller keeps the
|
||||
// newest statement per machine and raises from all of them together:
|
||||
//
|
||||
// - **an outside writer of the resolver file is its own finding**, `machine.<m>.<owner>.rewritten`:
|
||||
// the file the uplink holder declares was rewritten by another program, named where the engine could
|
||||
// name it. The names failing through what that program wrote are that finding's consequence, said in
|
||||
// it — never a second condition;
|
||||
// - **what is the machine's own** — its route, its tunnel, its resolvers answering wrong, a resolver
|
||||
// that is no mesh machine — is `machine.<m>.network`;
|
||||
// - **what points at another machine is said once, there** (the provider hold of ADR 0240 rule 5, for
|
||||
// the network): a failure toward the hub or toward a mesh resolver is held under that machine when it
|
||||
// is down on the record — silent, or its own network unhealthy — or when a second machine finds the
|
||||
// same; then `machine.<x>.unreachable` names every machine that cannot reach it, and none of them
|
||||
// raises anything of its own for it. One machine alone failing toward a healthy one is its own.
|
||||
//
|
||||
// Each is a warning; urgent on the control node, or when the bus cannot be reached, or for the hub.
|
||||
// Cleared on the first statement that no longer says it. An engine older than this judging says nothing
|
||||
// of its network, and nothing is raised for it.
|
||||
|
||||
// The conditions a machine's network raises.
|
||||
const (
|
||||
kindMachineNetwork = "machine-network"
|
||||
kindNetworkRewritten = "network-rewritten"
|
||||
kindNetworkUnreachable = "network-unreachable"
|
||||
sourceNetwork = "network"
|
||||
)
|
||||
|
||||
// networkKinds are the kinds this judging owns: every open one it no longer says, it clears.
|
||||
var networkKinds = []string{kindMachineNetwork, kindNetworkRewritten, kindNetworkUnreachable}
|
||||
|
||||
// networkFacts is what one judging of every machine's network reads.
|
||||
type networkFacts struct {
|
||||
healths map[string]inventory.NodeHealth
|
||||
// byAddress is each machine's address on the private network; hub the hub's name; control the
|
||||
// control node's.
|
||||
byAddress map[string]string
|
||||
hub string
|
||||
control string
|
||||
// silent is every machine whose silence is an open condition.
|
||||
silent map[string]bool
|
||||
}
|
||||
|
||||
// judgeNetworks raises and clears every machine's network conditions from every machine's newest
|
||||
// statement, after one machine's statement was kept.
|
||||
func judgeNetworks(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, now time.Time) error {
|
||||
healths, err := inv.Healths(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
overlays, err := inv.Overlays(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
open, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f := networkFacts{healths: healths, byAddress: map[string]string{}, control: controlHost(ctx, inv),
|
||||
silent: map[string]bool{}}
|
||||
for _, o := range overlays {
|
||||
if o.Address != "" {
|
||||
f.byAddress[o.Address] = o.Name
|
||||
}
|
||||
if o.Hub {
|
||||
f.hub = o.Name
|
||||
}
|
||||
}
|
||||
for _, c := range open {
|
||||
if c.Subject.Scope == conditions.ScopeMachine && c.Kind == "silent" {
|
||||
f.silent[c.Subject.ID] = true
|
||||
}
|
||||
}
|
||||
var problems []string
|
||||
said := map[string]bool{}
|
||||
for _, o := range networkObservations(f) {
|
||||
said[o.Key()] = true
|
||||
if _, err := k.Observe(ctx, o); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
for _, c := range open {
|
||||
if !slices.Contains(networkKinds, c.Kind) || said[c.Key] {
|
||||
continue
|
||||
}
|
||||
why := "no machine says it any more"
|
||||
if c.Subject.Machine != "" {
|
||||
why = c.Subject.Machine + "'s network no longer says it"
|
||||
}
|
||||
if _, err := k.Clear(ctx, c.Key, why); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
return fmt.Errorf("%s", strings.Join(problems, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// pointed is one machine's failing part that points at another machine.
|
||||
type pointed struct {
|
||||
from string
|
||||
part inventory.NetworkPart
|
||||
}
|
||||
|
||||
// networkObservations is every network condition the statements say now. Pure.
|
||||
func networkObservations(f networkFacts) []conditions.Observation {
|
||||
machines := make([]string, 0, len(f.healths))
|
||||
for m := range f.healths {
|
||||
machines = append(machines, m)
|
||||
}
|
||||
sort.Strings(machines)
|
||||
|
||||
unhealthy := func(m string) []inventory.NetworkPart {
|
||||
h := f.healths[m]
|
||||
if h.Network == nil {
|
||||
return nil
|
||||
}
|
||||
var out []inventory.NetworkPart
|
||||
for _, p := range h.Network.Parts {
|
||||
if p.State == link.StateUnhealthy {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
// The machines a part points at, other than its own: the hub, and each mesh resolver by its address.
|
||||
// An address that is no mesh machine's — the resolver a VPN client wrote in — is the machine's own.
|
||||
targets := func(m string, p inventory.NetworkPart) ([]string, bool) {
|
||||
if len(p.Toward) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
var out []string
|
||||
for _, t := range p.Toward {
|
||||
x := f.byAddress[t]
|
||||
if t == link.TowardHub {
|
||||
x = f.hub
|
||||
}
|
||||
if x == "" || x == m {
|
||||
return nil, false
|
||||
}
|
||||
if !slices.Contains(out, x) {
|
||||
out = append(out, x)
|
||||
}
|
||||
}
|
||||
return out, true
|
||||
}
|
||||
|
||||
// First pass: what points at whom.
|
||||
pointing := map[string][]pointed{}
|
||||
for _, m := range machines {
|
||||
for _, p := range unhealthy(m) {
|
||||
if xs, ok := targets(m, p); ok {
|
||||
for _, x := range xs {
|
||||
pointing[x] = append(pointing[x], pointed{from: m, part: p})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
from := func(x string) []string {
|
||||
var out []string
|
||||
for _, pt := range pointing[x] {
|
||||
if !slices.Contains(out, pt.from) {
|
||||
out = append(out, pt.from)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
// A machine is down on the record when its silence is open or its own network is unhealthy, or when
|
||||
// two machines find it unreachable: then what points at it is held there.
|
||||
down := func(x string) bool {
|
||||
return f.silent[x] || len(unhealthy(x)) > 0 || len(from(x)) >= 2
|
||||
}
|
||||
|
||||
var out []conditions.Observation
|
||||
saysOwn := map[string]bool{}
|
||||
for _, m := range machines {
|
||||
parts := unhealthy(m)
|
||||
if len(parts) == 0 {
|
||||
continue
|
||||
}
|
||||
var own []inventory.NetworkPart
|
||||
var rewritten *inventory.NetworkPart
|
||||
for i, p := range parts {
|
||||
if p.Part == link.PartResolvConf {
|
||||
rewritten = &parts[i]
|
||||
continue
|
||||
}
|
||||
if xs, ok := targets(m, p); ok && allDown(xs, down) {
|
||||
continue // held at the machines it points at
|
||||
}
|
||||
own = append(own, p)
|
||||
}
|
||||
if rewritten != nil {
|
||||
out = append(out, rewrittenObservation(m, *rewritten, parts, f))
|
||||
// The names failing through what another program wrote are that finding's, said in it.
|
||||
kept := own[:0]
|
||||
for _, p := range own {
|
||||
if p.Part != link.PartNames {
|
||||
kept = append(kept, p)
|
||||
}
|
||||
}
|
||||
own = kept
|
||||
}
|
||||
if len(own) > 0 {
|
||||
out = append(out, machineNetworkObservation(m, own, f, from(m)))
|
||||
saysOwn[m] = true
|
||||
}
|
||||
}
|
||||
// Said once, at the machine everybody points at — unless its own network condition already says it
|
||||
// (listed there), or its silence does.
|
||||
targetsSorted := make([]string, 0, len(pointing))
|
||||
for x := range pointing {
|
||||
targetsSorted = append(targetsSorted, x)
|
||||
}
|
||||
sort.Strings(targetsSorted)
|
||||
for _, x := range targetsSorted {
|
||||
if !down(x) || saysOwn[x] || f.silent[x] {
|
||||
continue
|
||||
}
|
||||
out = append(out, unreachableObservation(x, pointing[x], from(x), f))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func allDown(xs []string, down func(string) bool) bool {
|
||||
for _, x := range xs {
|
||||
if !down(x) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(xs) > 0
|
||||
}
|
||||
|
||||
// rewrittenObservation is the resolver file rewritten by another program: its own finding, naming the
|
||||
// writer where the engine could, and what it costs the machine.
|
||||
func rewrittenObservation(m string, p inventory.NetworkPart, all []inventory.NetworkPart, f networkFacts) conditions.Observation {
|
||||
writer := ""
|
||||
if p.Writer != "" {
|
||||
writer = " (" + p.Writer + ")"
|
||||
}
|
||||
cost := "the names through it are not yet judged"
|
||||
said := []string{p.Part + ": " + p.Said}
|
||||
for _, q := range all {
|
||||
if q.Part == link.PartNames {
|
||||
cost = strings.TrimSuffix(q.Reason, ".")
|
||||
said = append(said, q.Part+": "+q.Said)
|
||||
}
|
||||
}
|
||||
if cost == "the names through it are not yet judged" {
|
||||
cost = "the names still resolve through what it wrote"
|
||||
}
|
||||
id := m
|
||||
if p.Owner != "" {
|
||||
// Named by the module whose file it is: a send that moved that module is what the gate
|
||||
// holds it on (issue 281's rule — what names a moved module is that module's).
|
||||
id = m + "." + p.Owner
|
||||
}
|
||||
severity := conditions.Warning
|
||||
if m == f.control {
|
||||
severity = conditions.Urgent
|
||||
}
|
||||
summary := fmt.Sprintf("the resolver file on %s was rewritten by another program%s — %s until the "+
|
||||
"node-engine writes it back at its next reconcile, or that program gives it back", m, writer, cost)
|
||||
if p.Owner != "" {
|
||||
summary = fmt.Sprintf("the resolver file %s writes on %s was rewritten by another program%s — %s until "+
|
||||
"the node-engine writes it back at its next reconcile, or that program gives it back", p.Owner, m, writer, cost)
|
||||
}
|
||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: "rewritten", Kind: kindNetworkRewritten,
|
||||
Machine: m, Severity: severity, Source: sourceNetwork, Summary: summary,
|
||||
Said: fmt.Sprintf("since %s: %s", p.Since.UTC().Format("2006-01-02 15:04:05 MST"), strings.Join(said, " | "))}
|
||||
}
|
||||
|
||||
// machineNetworkObservation is what is wrong with a machine's own networking.
|
||||
func machineNetworkObservation(m string, parts []inventory.NetworkPart, f networkFacts, waiting []string) conditions.Observation {
|
||||
var words, said []string
|
||||
severity := conditions.Warning
|
||||
for _, p := range parts {
|
||||
words = append(words, p.Reason)
|
||||
said = append(said, fmt.Sprintf("%s since %s: %s", p.Part, p.Since.UTC().Format("2006-01-02 15:04:05 MST"), p.Said))
|
||||
if p.Part == link.PartBus {
|
||||
severity = conditions.Urgent
|
||||
}
|
||||
}
|
||||
if m == f.control || m == f.hub {
|
||||
severity = conditions.Urgent
|
||||
}
|
||||
summary := fmt.Sprintf("%s's network is not healthy: %s", m, strings.Join(words, "; "))
|
||||
if len(waiting) > 0 {
|
||||
severity = conditions.Urgent
|
||||
summary += fmt.Sprintf("; %s cannot reach it", strings.Join(waiting, ", "))
|
||||
}
|
||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Token: "network", Kind: kindMachineNetwork,
|
||||
Machine: m, Severity: severity, Source: sourceNetwork, Summary: summary, Said: strings.Join(said, " | ")}
|
||||
}
|
||||
|
||||
// unreachableObservation is one machine others cannot reach, said once there.
|
||||
func unreachableObservation(x string, pts []pointed, from []string, f networkFacts) conditions.Observation {
|
||||
var what []string
|
||||
var said []string
|
||||
for _, pt := range pts {
|
||||
w := map[string]string{link.PartTunnel: "the tunnel to it", link.PartBus: "the bus on it",
|
||||
link.PartNames: "its resolver"}[pt.part.Part]
|
||||
if w == "" {
|
||||
w = pt.part.Part
|
||||
}
|
||||
if !slices.Contains(what, w) {
|
||||
what = append(what, w)
|
||||
}
|
||||
said = append(said, fmt.Sprintf("%s: %s: %s", pt.from, pt.part.Part, pt.part.Said))
|
||||
}
|
||||
severity := conditions.Warning
|
||||
if x == f.hub || x == f.control || slices.Contains(what, "the bus on it") {
|
||||
severity = conditions.Urgent
|
||||
}
|
||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: x, Token: "unreachable", Kind: kindNetworkUnreachable,
|
||||
Machine: x, Also: from, Severity: severity, Source: sourceNetwork,
|
||||
Summary: fmt.Sprintf("%s cannot be reached from %s: %s", x, strings.Join(from, ", "), strings.Join(what, ", ")),
|
||||
Said: strings.Join(said, " | ")}
|
||||
}
|
||||
|
||||
// networkLines is what `node show` says of a machine's networking.
|
||||
func networkLines(h inventory.NodeHealth, had bool, now time.Time) []string {
|
||||
if !had || h.Network == nil {
|
||||
return []string{" its node-engine does not say how its network is — it is older than that judging (ADR 0241)"}
|
||||
}
|
||||
out := []string{fmt.Sprintf(" its network: %s since %s", h.Network.State, h.Network.Since.Local().Format("2006-01-02 15:04"))}
|
||||
for _, p := range h.Network.Parts {
|
||||
line := fmt.Sprintf(" %-10s %s", p.State, p.Part)
|
||||
if p.Reason != "" {
|
||||
line += " — " + p.Reason
|
||||
}
|
||||
if p.Writer != "" {
|
||||
line += " (" + p.Writer + ")"
|
||||
}
|
||||
out = append(out, line)
|
||||
}
|
||||
return out
|
||||
}
|
||||
Reference in New Issue
Block a user