package main import ( "encoding/json" "os" "strings" "testing" "time" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) // A machine says how its network is (novox/hq ADR 0241, "how it is checked"): the resolver file rewritten // by another program is one finding, naming the writer, with the names it costs said in it; what is the // machine's own is `machine..network`; what points at another machine that is down is said once, there; // one machine alone failing toward a healthy one is its own; the control node, the hub and the bus are // urgent; an engine that says nothing of its network raises nothing; and the gate waits on what is shown to // be another's. func aNetwork(state string, parts ...inventory.NetworkPart) *inventory.NetworkHealth { for i := range parts { if parts[i].State == "" { parts[i].State = link.StateUnhealthy } parts[i].Since = h0 } return &inventory.NetworkHealth{State: state, Since: h0, Parts: parts} } func netFacts(healths map[string]*inventory.NetworkHealth) networkFacts { f := networkFacts{healths: map[string]inventory.NodeHealth{}, hub: "anchor", control: "anchor", byAddress: map[string]string{"10.77.0.1": "anchor", "10.77.0.2": "laptop", "10.77.0.3": "spare"}, silent: map[string]bool{}} for m, n := range healths { f.healths[m] = inventory.NodeHealth{Node: m, Network: n} } return f } var ( rewrittenByVPN = inventory.NetworkPart{Part: link.PartResolvConf, Reason: "the resolver file was rewritten by another program", Said: "/etc/resolv.conf lists 172.16.5.5 where 10.77.0.1 is declared", Writer: "FortiClient", Owner: "networkmanager"} namesThroughVPN = inventory.NetworkPart{Part: link.PartNames, Reason: "mesh names do not resolve", Said: "172.16.5.5 — anchor.internal (IPv4): says no such name", Toward: []string{"172.16.5.5"}} tunnelDown = inventory.NetworkPart{Part: link.PartTunnel, Reason: "the tunnel to the hub has not handshaken for over five minutes", Said: "mesh0's newest handshake with the hub was 9m0s ago", Toward: []string{link.TowardHub}} noRoute = inventory.NetworkPart{Part: link.PartRoute, Reason: "the machine has no default route", Said: "no default route"} ) func keysOf(obs []conditions.Observation) []string { var keys []string for _, o := range obs { keys = append(keys, o.Key()) } return keys } func TestAResolverFileRewrittenIsOneFindingNamingItsWriterAndWhatItCosts(t *testing.T) { obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ "anchor": aNetwork(link.StateHealthy), "laptop": aNetwork(link.StateUnhealthy, rewrittenByVPN, namesThroughVPN), })) if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" { t.Fatalf("want one finding, the rewrite, got %v", keys) } o := obs[0] for _, want := range []string{"laptop", "rewritten by another program (FortiClient)", "mesh names do not resolve", "next reconcile"} { if !strings.Contains(o.Summary, want) { t.Errorf("the summary does not say %q: %s", want, o.Summary) } } if strings.Contains(o.Summary, "172.16.") || strings.Contains(o.Summary, "/etc/") { t.Errorf("an address or a path reached the summary: %s", o.Summary) } if !strings.Contains(o.Said, "172.16.5.5") || o.Severity != conditions.Warning || o.Machine != "laptop" { t.Errorf("the evidence or the severity is wrong: %+v", o) } } func TestOneMachineFailingTowardAHealthyHubIsItsOwn(t *testing.T) { obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ "anchor": aNetwork(link.StateHealthy), "laptop": aNetwork(link.StateUnhealthy, tunnelDown), "spare": aNetwork(link.StateHealthy), })) if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" { t.Fatalf("want the laptop's own, got %v", keys) } if obs[0].Severity != conditions.Warning { t.Fatalf("a laptop's own tunnel is a warning, got %s", obs[0].Severity) } } func TestTwoMachinesThatCannotReachTheHubAreSaidOnceAtTheHub(t *testing.T) { obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ "anchor": aNetwork(link.StateHealthy), "laptop": aNetwork(link.StateUnhealthy, tunnelDown), "spare": aNetwork(link.StateUnhealthy, tunnelDown), })) if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.unreachable" { t.Fatalf("want one condition at the hub, got %v", keys) } o := obs[0] if o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop, spare") || len(o.Also) != 2 { t.Fatalf("the hub's condition is %+v", o) } } func TestWhatPointsAtASilentHubIsHeldUnderItsSilence(t *testing.T) { f := netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, tunnelDown)}) f.silent["anchor"] = true if obs := networkObservations(f); len(obs) != 0 { t.Fatalf("the hub's silence says it; got %v", keysOf(obs)) } } func TestAHubWhoseOwnNetworkIsUnhealthyListsWhoCannotReachIt(t *testing.T) { obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ "anchor": aNetwork(link.StateUnhealthy, noRoute), "laptop": aNetwork(link.StateUnhealthy, tunnelDown), })) if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.network" { t.Fatalf("want the hub's own, holding the laptop's, got %v", keys) } if o := obs[0]; o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop cannot reach it") { t.Fatalf("the hub's condition is %+v", o) } } func TestOneMachineFailingAHealthyMeshResolverIsItsOwnAndTwoAreTheResolvers(t *testing.T) { silentResolver := inventory.NetworkPart{Part: link.PartNames, Reason: "1 of its 2 resolvers do not answer as the mesh's do", Said: "10.77.0.3 — no answer within 1s", Toward: []string{"10.77.0.3"}} obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ "anchor": aNetwork(link.StateHealthy), "spare": aNetwork(link.StateHealthy), "laptop": aNetwork(link.StateUnhealthy, silentResolver), })) if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" { t.Fatalf("one machine alone: want its own, got %v", keys) } obs = networkObservations(netFacts(map[string]*inventory.NetworkHealth{ "anchor": aNetwork(link.StateUnhealthy, silentResolver), "spare": aNetwork(link.StateHealthy), "laptop": aNetwork(link.StateUnhealthy, silentResolver), })) if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.spare.unreachable" { t.Fatalf("two machines: want it said once at the resolver's machine, got %v", keys) } if !strings.Contains(obs[0].Summary, "its resolver") { t.Fatalf("the resolver's machine is said %s", obs[0].Summary) } } func TestTheControlNodeAndTheBusAreUrgent(t *testing.T) { f := netFacts(map[string]*inventory.NetworkHealth{"anchor": aNetwork(link.StateUnhealthy, rewrittenByVPN)}) if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent { t.Fatalf("the control node's rewritten file: %+v", obs) } bus := inventory.NetworkPart{Part: link.PartBus, Reason: "the bus cannot be reached", Said: "no link"} f = netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, bus, noRoute)}) if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent { t.Fatalf("the bus unreachable from the laptop: %+v", obs) } } func TestAnEngineThatSaysNothingOfItsNetworkRaisesNothing(t *testing.T) { f := netFacts(map[string]*inventory.NetworkHealth{"laptop": nil, "anchor": aNetwork(link.StateHealthy)}) if obs := networkObservations(f); len(obs) != 0 { t.Fatalf("got %v", keysOf(obs)) } } // Through the store and the keeper: the statement kept, the rewrite raised from it, cleared when the file // is written back, and node show saying it. func TestARewrittenResolverFileIsRaisedFromTheStatementAndClearedWhenWrittenBack(t *testing.T) { open := aMesh(t) ctx := t.Context() inv, k := open.inventory, conditionsFrom say := func(at time.Time, n *link.NetworkHealth) { t.Helper() if err := stateHealth(ctx, inv, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil { t.Fatal(err) } } openKeys := func() []string { t.Helper() list, err := k.Open(ctx) if err != nil { t.Fatal(err) } var keys []string for _, c := range list { keys = append(keys, c.Key) } return keys } healthy := &link.NetworkHealth{State: link.StateHealthy, Since: h0, Parts: []link.NetworkPart{ {Part: link.PartResolvConf, State: link.StateHealthy, Since: h0}}} rewritten := &link.NetworkHealth{State: link.StateUnhealthy, Since: h0.Add(time.Minute), Parts: []link.NetworkPart{ {Part: link.PartResolvConf, State: link.StateUnhealthy, Reason: rewrittenByVPN.Reason, Said: rewrittenByVPN.Said, Writer: "FortiClient", Owner: "networkmanager", Since: h0.Add(time.Minute)}, {Part: link.PartNames, State: link.StateUnhealthy, Reason: namesThroughVPN.Reason, Said: namesThroughVPN.Said, Toward: namesThroughVPN.Toward, Since: h0.Add(time.Minute)}}} say(h0, healthy) if keys := openKeys(); len(keys) != 0 { t.Fatalf("a healthy network raised %v", keys) } say(h0.Add(time.Minute), rewritten) if keys := openKeys(); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" { t.Fatalf("the rewrite raised %v", keys) } kept, had, err := inv.HealthOf(ctx, "laptop") if err != nil || !had || kept.Network == nil || kept.Network.Parts[0].Writer != "FortiClient" { t.Fatalf("the statement's network was not kept: %+v %v", kept.Network, err) } if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "FortiClient") || !strings.Contains(lines, "unhealthy resolv-conf") { t.Fatalf("node show says:\n%s", lines) } say(h0.Add(2*time.Minute), healthy) if keys := openKeys(); len(keys) != 0 { t.Fatalf("written back, still open: %v", keys) } // An engine older than the judging says no network: nothing raised, and node show says it is not known. say(h0.Add(3*time.Minute), nil) kept, had, _ = inv.HealthOf(ctx, "laptop") if keys := openKeys(); len(keys) != 0 || kept.Network != nil { t.Fatalf("an older engine: %v %+v", keys, kept.Network) } if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "older than that judging") { t.Fatalf("node show says:\n%s", lines) } } // The gate: a resolver file another program rewrote waits the judging rather than failing it at the // bound; the same, when the send moved the module whose file it is, is that module's; a machine's own // network fault holds the machine as a whole, as before. func TestTheGateWaitsOnARewriteItDidNotMakeAndHoldsTheOwnerOnOneItMoved(t *testing.T) { since := h0 rewrite := conditions.Condition{Key: "machine.laptop.networkmanager.rewritten", Kind: kindNetworkRewritten, Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop.networkmanager", Machine: "laptop"}, Summary: "the resolver file was rewritten", Raised: since.Add(time.Minute), Source: sourceNetwork} f := gateFacts{judged: true, open: []conditions.Condition{rewrite}} if w := aboutTheMachine("laptop", []string{"letta"}, since, f); w.waiting == "" || w.whole != "" || len(w.on) != 0 { t.Fatalf("a rewrite the send did not make: %+v", w) } if w := aboutTheMachine("laptop", []string{"networkmanager", "letta"}, since, f); w.on["networkmanager"] == "" || w.on["letta"] != "" || w.waiting != "" { t.Fatalf("a rewrite of the file a moved module owns: %+v", w) } own := conditions.Condition{Key: "machine.laptop.network", Kind: kindMachineNetwork, Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop", Machine: "laptop"}, Summary: "laptop's network is not healthy", Raised: since.Add(time.Minute), Source: sourceNetwork} if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true, open: []conditions.Condition{own}}); w.whole == "" || w.waiting != "" { t.Fatalf("the machine's own network: %+v", w) } unreachable := conditions.Condition{Key: "machine.anchor.unreachable", Kind: kindNetworkUnreachable, Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor", Also: []string{"laptop", "spare"}}, Summary: "anchor cannot be reached", Raised: since.Add(time.Minute), Source: sourceNetwork} if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true, open: []conditions.Condition{unreachable}}); w.waiting == "" || w.whole != "" { t.Fatalf("a machine that cannot reach the hub: %+v", w) } } // TestTheDrillsStatementsRaiseAndClearTheRewrite replays the drill of ADR 0241 (mesh-host // internal/network TestDrill…): what a node-engine said in a throwaway container while its resolver file // was declared, rewritten as a VPN client rewrites it, and written back — recorded, with the mesh's names // and addresses replaced by this test mesh's. Healthy raises nothing; the rewrite, on its second look, is // raised as one finding naming the writer; written back, it clears. func TestTheDrillsStatementsRaiseAndClearTheRewrite(t *testing.T) { raw, err := os.ReadFile("testdata/network-drill.json") if err != nil { t.Fatal(err) } var said []link.Health if err := json.Unmarshal(raw, &said); err != nil { t.Fatal(err) } open := aMesh(t) ctx := t.Context() k := conditionsFrom var raisedAt []int for i, h := range said { if err := stateHealth(ctx, open.inventory, k, "laptop", h, h.At); err != nil { t.Fatal(err) } list, err := k.Open(ctx) if err != nil { t.Fatal(err) } for _, c := range list { if c.Key != "machine.laptop.networkmanager.rewritten" || !strings.Contains(c.Summary, "(FortiClient)") { t.Fatalf("statement %d raised %s: %s", i, c.Key, c.Summary) } raisedAt = append(raisedAt, i) } } // Five statements: healthy, healthy, one failing look (still healthy), unhealthy, written back. if len(raisedAt) != 1 || raisedAt[0] != 3 { t.Fatalf("the rewrite was open after statements %v; want after the fourth alone", raisedAt) } }