package main import ( "context" "fmt" "slices" "sort" "strings" "time" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) // A machine says how its network is (novox/hq ADR 0241, which extends ADR 0240 from what a module runs // to the machine it runs on). // // **Every machine's node-engine judges its own networking** — the resolver file the uplink holder // declared, the names through every resolver it lists, the tunnel's handshake with the hub, the bus, the // default route — on the two-look rule, and states it beside its resources. The controller keeps the // newest statement per machine and raises from all of them together: // // - **an outside writer of the resolver file is its own finding**, `machine...rewritten`: // the file the uplink holder declares was rewritten by another program, named where the engine could // name it. The names failing through what that program wrote are that finding's consequence, said in // it — never a second condition; // - **what is the machine's own** — its route, its tunnel, its resolvers answering wrong, a resolver // that is no mesh machine — is `machine..network`; // - **what points at another machine is said once, there** (the provider hold of ADR 0240 rule 5, for // the network): a failure toward the hub or toward a mesh resolver is held under that machine when it // is down on the record — silent, or its own network unhealthy — or when a second machine finds the // same; then `machine..unreachable` names every machine that cannot reach it, and none of them // raises anything of its own for it. One machine alone failing toward a healthy one is its own. // // Each is a warning; urgent on the control node, or when the bus cannot be reached, or for the hub. // Cleared on the first statement that no longer says it. An engine older than this judging says nothing // of its network, and nothing is raised for it. // The conditions a machine's network raises. const ( kindMachineNetwork = "machine-network" kindNetworkRewritten = "network-rewritten" kindNetworkUnreachable = "network-unreachable" sourceNetwork = "network" ) // networkKinds are the kinds this judging owns: every open one it no longer says, it clears. var networkKinds = []string{kindMachineNetwork, kindNetworkRewritten, kindNetworkUnreachable} // networkFacts is what one judging of every machine's network reads. type networkFacts struct { healths map[string]inventory.NodeHealth // byAddress is each machine's address on the private network; hub the hub's name; control the // control node's. byAddress map[string]string hub string control string // silent is every machine whose silence is an open condition. silent map[string]bool } // judgeNetworks raises and clears every machine's network conditions from every machine's newest // statement, after one machine's statement was kept. func judgeNetworks(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, now time.Time) error { healths, err := inv.Healths(ctx) if err != nil { return err } overlays, err := inv.Overlays(ctx) if err != nil { return err } open, err := k.Open(ctx) if err != nil { return err } f := networkFacts{healths: healths, byAddress: map[string]string{}, control: controlHost(ctx, inv), silent: map[string]bool{}} for _, o := range overlays { if o.Address != "" { f.byAddress[o.Address] = o.Name } if o.Hub { f.hub = o.Name } } for _, c := range open { if c.Subject.Scope == conditions.ScopeMachine && c.Kind == "silent" { f.silent[c.Subject.ID] = true } } var problems []string said := map[string]bool{} for _, o := range networkObservations(f) { said[o.Key()] = true if _, err := k.Observe(ctx, o); err != nil { problems = append(problems, err.Error()) } } for _, c := range open { if !slices.Contains(networkKinds, c.Kind) || said[c.Key] { continue } why := "no machine says it any more" if c.Subject.Machine != "" { why = c.Subject.Machine + "'s network no longer says it" } if _, err := k.Clear(ctx, c.Key, why); err != nil { problems = append(problems, err.Error()) } } if len(problems) > 0 { return fmt.Errorf("%s", strings.Join(problems, "; ")) } return nil } // pointed is one machine's failing part that points at another machine. type pointed struct { from string part inventory.NetworkPart } // networkObservations is every network condition the statements say now. Pure. func networkObservations(f networkFacts) []conditions.Observation { machines := make([]string, 0, len(f.healths)) for m := range f.healths { machines = append(machines, m) } sort.Strings(machines) unhealthy := func(m string) []inventory.NetworkPart { h := f.healths[m] if h.Network == nil { return nil } var out []inventory.NetworkPart for _, p := range h.Network.Parts { if p.State == link.StateUnhealthy { out = append(out, p) } } return out } // The machines a part points at, other than its own: the hub, and each mesh resolver by its address. // An address that is no mesh machine's — the resolver a VPN client wrote in — is the machine's own. targets := func(m string, p inventory.NetworkPart) ([]string, bool) { if len(p.Toward) == 0 { return nil, false } var out []string for _, t := range p.Toward { x := f.byAddress[t] if t == link.TowardHub { x = f.hub } if x == "" || x == m { return nil, false } if !slices.Contains(out, x) { out = append(out, x) } } return out, true } // First pass: what points at whom. pointing := map[string][]pointed{} for _, m := range machines { for _, p := range unhealthy(m) { if xs, ok := targets(m, p); ok { for _, x := range xs { pointing[x] = append(pointing[x], pointed{from: m, part: p}) } } } } from := func(x string) []string { var out []string for _, pt := range pointing[x] { if !slices.Contains(out, pt.from) { out = append(out, pt.from) } } sort.Strings(out) return out } // A machine is down on the record when its silence is open or its own network is unhealthy, or when // two machines find it unreachable: then what points at it is held there. down := func(x string) bool { return f.silent[x] || len(unhealthy(x)) > 0 || len(from(x)) >= 2 } var out []conditions.Observation saysOwn := map[string]bool{} for _, m := range machines { parts := unhealthy(m) if len(parts) == 0 { continue } var own []inventory.NetworkPart var rewritten *inventory.NetworkPart for i, p := range parts { if p.Part == link.PartResolvConf { rewritten = &parts[i] continue } if xs, ok := targets(m, p); ok && allDown(xs, down) { continue // held at the machines it points at } own = append(own, p) } if rewritten != nil { out = append(out, rewrittenObservation(m, *rewritten, parts, f)) // The names failing through what another program wrote are that finding's, said in it. kept := own[:0] for _, p := range own { if p.Part != link.PartNames { kept = append(kept, p) } } own = kept } if len(own) > 0 { out = append(out, machineNetworkObservation(m, own, f, from(m))) saysOwn[m] = true } } // Said once, at the machine everybody points at — unless its own network condition already says it // (listed there), or its silence does. targetsSorted := make([]string, 0, len(pointing)) for x := range pointing { targetsSorted = append(targetsSorted, x) } sort.Strings(targetsSorted) for _, x := range targetsSorted { if !down(x) || saysOwn[x] || f.silent[x] { continue } out = append(out, unreachableObservation(x, pointing[x], from(x), f)) } return out } func allDown(xs []string, down func(string) bool) bool { for _, x := range xs { if !down(x) { return false } } return len(xs) > 0 } // rewrittenObservation is the resolver file rewritten by another program: its own finding, naming the // writer where the engine could, and what it costs the machine. func rewrittenObservation(m string, p inventory.NetworkPart, all []inventory.NetworkPart, f networkFacts) conditions.Observation { writer := "" if p.Writer != "" { writer = " (" + p.Writer + ")" } cost := "the names through it are not yet judged" said := []string{p.Part + ": " + p.Said} for _, q := range all { if q.Part == link.PartNames { cost = strings.TrimSuffix(q.Reason, ".") said = append(said, q.Part+": "+q.Said) } } if cost == "the names through it are not yet judged" { cost = "the names still resolve through what it wrote" } id := m if p.Owner != "" { // Named by the module whose file it is: a send that moved that module is what the gate // holds it on (issue 281's rule — what names a moved module is that module's). id = m + "." + p.Owner } severity := conditions.Warning if m == f.control { severity = conditions.Urgent } summary := fmt.Sprintf("the resolver file on %s was rewritten by another program%s — %s until the "+ "node-engine writes it back at its next reconcile, or that program gives it back", m, writer, cost) if p.Owner != "" { summary = fmt.Sprintf("the resolver file %s writes on %s was rewritten by another program%s — %s until "+ "the node-engine writes it back at its next reconcile, or that program gives it back", p.Owner, m, writer, cost) } return conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: "rewritten", Kind: kindNetworkRewritten, Machine: m, Severity: severity, Source: sourceNetwork, Summary: summary, Said: fmt.Sprintf("since %s: %s", p.Since.UTC().Format("2006-01-02 15:04:05 MST"), strings.Join(said, " | "))} } // machineNetworkObservation is what is wrong with a machine's own networking. func machineNetworkObservation(m string, parts []inventory.NetworkPart, f networkFacts, waiting []string) conditions.Observation { var words, said []string severity := conditions.Warning for _, p := range parts { words = append(words, p.Reason) said = append(said, fmt.Sprintf("%s since %s: %s", p.Part, p.Since.UTC().Format("2006-01-02 15:04:05 MST"), p.Said)) if p.Part == link.PartBus { severity = conditions.Urgent } } if m == f.control || m == f.hub { severity = conditions.Urgent } summary := fmt.Sprintf("%s's network is not healthy: %s", m, strings.Join(words, "; ")) if len(waiting) > 0 { severity = conditions.Urgent summary += fmt.Sprintf("; %s cannot reach it", strings.Join(waiting, ", ")) } return conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Token: "network", Kind: kindMachineNetwork, Machine: m, Severity: severity, Source: sourceNetwork, Summary: summary, Said: strings.Join(said, " | ")} } // unreachableObservation is one machine others cannot reach, said once there. func unreachableObservation(x string, pts []pointed, from []string, f networkFacts) conditions.Observation { var what []string var said []string for _, pt := range pts { w := map[string]string{link.PartTunnel: "the tunnel to it", link.PartBus: "the bus on it", link.PartNames: "its resolver"}[pt.part.Part] if w == "" { w = pt.part.Part } if !slices.Contains(what, w) { what = append(what, w) } said = append(said, fmt.Sprintf("%s: %s: %s", pt.from, pt.part.Part, pt.part.Said)) } severity := conditions.Warning if x == f.hub || x == f.control || slices.Contains(what, "the bus on it") { severity = conditions.Urgent } return conditions.Observation{Scope: conditions.ScopeMachine, ID: x, Token: "unreachable", Kind: kindNetworkUnreachable, Machine: x, Also: from, Severity: severity, Source: sourceNetwork, Summary: fmt.Sprintf("%s cannot be reached from %s: %s", x, strings.Join(from, ", "), strings.Join(what, ", ")), Said: strings.Join(said, " | ")} } // networkLines is what `node show` says of a machine's networking. func networkLines(h inventory.NodeHealth, had bool, now time.Time) []string { if !had || h.Network == nil { return []string{" its node-engine does not say how its network is — it is older than that judging (ADR 0241)"} } out := []string{fmt.Sprintf(" its network: %s since %s", h.Network.State, h.Network.Since.Local().Format("2006-01-02 15:04"))} for _, p := range h.Network.Parts { line := fmt.Sprintf(" %-10s %s", p.State, p.Part) if p.Reason != "" { line += " — " + p.Reason } if p.Writer != "" { line += " (" + p.Writer + ")" } out = append(out, line) } return out }