mesh-controller/cmd/mesh-controller/machine_network_test.go

package main

import (
	"encoding/json"
	"os"
	"strings"
	"testing"
	"time"

	"github.com/novox/mesh-controller/internal/conditions"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/link"
)

// A machine says how its network is (novox/hq ADR 0241, "how it is checked"): the resolver file rewritten
// by another program is one finding, naming the writer, with the names it costs said in it; what is the
// machine's own is `machine.<m>.network`; what points at another machine that is down is said once, there;
// one machine alone failing toward a healthy one is its own; the control node, the hub and the bus are
// urgent; an engine that says nothing of its network raises nothing; and the gate waits on what is shown to
// be another's.

func aNetwork(state string, parts ...inventory.NetworkPart) *inventory.NetworkHealth {
	for i := range parts {
		if parts[i].State == "" {
			parts[i].State = link.StateUnhealthy
		}
		parts[i].Since = h0
	}
	return &inventory.NetworkHealth{State: state, Since: h0, Parts: parts}
}

func netFacts(healths map[string]*inventory.NetworkHealth) networkFacts {
	f := networkFacts{healths: map[string]inventory.NodeHealth{}, hub: "anchor", control: "anchor",
		byAddress: map[string]string{"10.77.0.1": "anchor", "10.77.0.2": "laptop", "10.77.0.3": "spare"},
		silent:    map[string]bool{}}
	for m, n := range healths {
		f.healths[m] = inventory.NodeHealth{Node: m, Network: n}
	}
	return f
}

var (
	rewrittenByVPN = inventory.NetworkPart{Part: link.PartResolvConf, Reason: "the resolver file was rewritten by another program",
		Said: "/etc/resolv.conf lists 172.16.5.5 where 10.77.0.1 is declared", Writer: "FortiClient", Owner: "networkmanager"}
	namesThroughVPN = inventory.NetworkPart{Part: link.PartNames, Reason: "mesh names do not resolve",
		Said: "172.16.5.5 — anchor.internal (IPv4): says no such name", Toward: []string{"172.16.5.5"}}
	tunnelDown = inventory.NetworkPart{Part: link.PartTunnel, Reason: "the tunnel to the hub has not handshaken for over five minutes",
		Said: "mesh0's newest handshake with the hub was 9m0s ago", Toward: []string{link.TowardHub}}
	noRoute = inventory.NetworkPart{Part: link.PartRoute, Reason: "the machine has no default route", Said: "no default route"}
)

func keysOf(obs []conditions.Observation) []string {
	var keys []string
	for _, o := range obs {
		keys = append(keys, o.Key())
	}
	return keys
}

func TestAResolverFileRewrittenIsOneFindingNamingItsWriterAndWhatItCosts(t *testing.T) {
	obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
		"anchor": aNetwork(link.StateHealthy),
		"laptop": aNetwork(link.StateUnhealthy, rewrittenByVPN, namesThroughVPN),
	}))
	if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" {
		t.Fatalf("want one finding, the rewrite, got %v", keys)
	}
	o := obs[0]
	for _, want := range []string{"laptop", "rewritten by another program (FortiClient)", "mesh names do not resolve",
		"next reconcile"} {
		if !strings.Contains(o.Summary, want) {
			t.Errorf("the summary does not say %q: %s", want, o.Summary)
		}
	}
	if strings.Contains(o.Summary, "172.16.") || strings.Contains(o.Summary, "/etc/") {
		t.Errorf("an address or a path reached the summary: %s", o.Summary)
	}
	if !strings.Contains(o.Said, "172.16.5.5") || o.Severity != conditions.Warning || o.Machine != "laptop" {
		t.Errorf("the evidence or the severity is wrong: %+v", o)
	}
}

func TestOneMachineFailingTowardAHealthyHubIsItsOwn(t *testing.T) {
	obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
		"anchor": aNetwork(link.StateHealthy),
		"laptop": aNetwork(link.StateUnhealthy, tunnelDown),
		"spare":  aNetwork(link.StateHealthy),
	}))
	if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" {
		t.Fatalf("want the laptop's own, got %v", keys)
	}
	if obs[0].Severity != conditions.Warning {
		t.Fatalf("a laptop's own tunnel is a warning, got %s", obs[0].Severity)
	}
}

func TestTwoMachinesThatCannotReachTheHubAreSaidOnceAtTheHub(t *testing.T) {
	obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
		"anchor": aNetwork(link.StateHealthy),
		"laptop": aNetwork(link.StateUnhealthy, tunnelDown),
		"spare":  aNetwork(link.StateUnhealthy, tunnelDown),
	}))
	if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.unreachable" {
		t.Fatalf("want one condition at the hub, got %v", keys)
	}
	o := obs[0]
	if o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop, spare") || len(o.Also) != 2 {
		t.Fatalf("the hub's condition is %+v", o)
	}
}

func TestWhatPointsAtASilentHubIsHeldUnderItsSilence(t *testing.T) {
	f := netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, tunnelDown)})
	f.silent["anchor"] = true
	if obs := networkObservations(f); len(obs) != 0 {
		t.Fatalf("the hub's silence says it; got %v", keysOf(obs))
	}
}

func TestAHubWhoseOwnNetworkIsUnhealthyListsWhoCannotReachIt(t *testing.T) {
	obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
		"anchor": aNetwork(link.StateUnhealthy, noRoute),
		"laptop": aNetwork(link.StateUnhealthy, tunnelDown),
	}))
	if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.network" {
		t.Fatalf("want the hub's own, holding the laptop's, got %v", keys)
	}
	if o := obs[0]; o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop cannot reach it") {
		t.Fatalf("the hub's condition is %+v", o)
	}
}

func TestOneMachineFailingAHealthyMeshResolverIsItsOwnAndTwoAreTheResolvers(t *testing.T) {
	silentResolver := inventory.NetworkPart{Part: link.PartNames, Reason: "1 of its 2 resolvers do not answer as the mesh's do",
		Said: "10.77.0.3 — no answer within 1s", Toward: []string{"10.77.0.3"}}
	obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
		"anchor": aNetwork(link.StateHealthy), "spare": aNetwork(link.StateHealthy),
		"laptop": aNetwork(link.StateUnhealthy, silentResolver),
	}))
	if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" {
		t.Fatalf("one machine alone: want its own, got %v", keys)
	}
	obs = networkObservations(netFacts(map[string]*inventory.NetworkHealth{
		"anchor": aNetwork(link.StateUnhealthy, silentResolver), "spare": aNetwork(link.StateHealthy),
		"laptop": aNetwork(link.StateUnhealthy, silentResolver),
	}))
	if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.spare.unreachable" {
		t.Fatalf("two machines: want it said once at the resolver's machine, got %v", keys)
	}
	if !strings.Contains(obs[0].Summary, "its resolver") {
		t.Fatalf("the resolver's machine is said %s", obs[0].Summary)
	}
}

func TestTheControlNodeAndTheBusAreUrgent(t *testing.T) {
	f := netFacts(map[string]*inventory.NetworkHealth{"anchor": aNetwork(link.StateUnhealthy, rewrittenByVPN)})
	if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent {
		t.Fatalf("the control node's rewritten file: %+v", obs)
	}
	bus := inventory.NetworkPart{Part: link.PartBus, Reason: "the bus cannot be reached", Said: "no link"}
	f = netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, bus, noRoute)})
	if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent {
		t.Fatalf("the bus unreachable from the laptop: %+v", obs)
	}
}

func TestAnEngineThatSaysNothingOfItsNetworkRaisesNothing(t *testing.T) {
	f := netFacts(map[string]*inventory.NetworkHealth{"laptop": nil, "anchor": aNetwork(link.StateHealthy)})
	if obs := networkObservations(f); len(obs) != 0 {
		t.Fatalf("got %v", keysOf(obs))
	}
}

// Through the store and the keeper: the statement kept, the rewrite raised from it, cleared when the file
// is written back, and node show saying it.
func TestARewrittenResolverFileIsRaisedFromTheStatementAndClearedWhenWrittenBack(t *testing.T) {
	open := aMesh(t)
	ctx := t.Context()
	inv, k := open.inventory, conditionsFrom
	say := func(at time.Time, n *link.NetworkHealth) {
		t.Helper()
		if err := stateHealth(ctx, inv, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil {
			t.Fatal(err)
		}
	}
	openKeys := func() []string {
		t.Helper()
		list, err := k.Open(ctx)
		if err != nil {
			t.Fatal(err)
		}
		var keys []string
		for _, c := range list {
			keys = append(keys, c.Key)
		}
		return keys
	}
	healthy := &link.NetworkHealth{State: link.StateHealthy, Since: h0, Parts: []link.NetworkPart{
		{Part: link.PartResolvConf, State: link.StateHealthy, Since: h0}}}
	rewritten := &link.NetworkHealth{State: link.StateUnhealthy, Since: h0.Add(time.Minute), Parts: []link.NetworkPart{
		{Part: link.PartResolvConf, State: link.StateUnhealthy, Reason: rewrittenByVPN.Reason, Said: rewrittenByVPN.Said,
			Writer: "FortiClient", Owner: "networkmanager", Since: h0.Add(time.Minute)},
		{Part: link.PartNames, State: link.StateUnhealthy, Reason: namesThroughVPN.Reason, Said: namesThroughVPN.Said,
			Toward: namesThroughVPN.Toward, Since: h0.Add(time.Minute)}}}

	say(h0, healthy)
	if keys := openKeys(); len(keys) != 0 {
		t.Fatalf("a healthy network raised %v", keys)
	}
	say(h0.Add(time.Minute), rewritten)
	if keys := openKeys(); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" {
		t.Fatalf("the rewrite raised %v", keys)
	}
	kept, had, err := inv.HealthOf(ctx, "laptop")
	if err != nil || !had || kept.Network == nil || kept.Network.Parts[0].Writer != "FortiClient" {
		t.Fatalf("the statement's network was not kept: %+v %v", kept.Network, err)
	}
	if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "FortiClient") ||
		!strings.Contains(lines, "unhealthy  resolv-conf") {
		t.Fatalf("node show says:\n%s", lines)
	}
	say(h0.Add(2*time.Minute), healthy)
	if keys := openKeys(); len(keys) != 0 {
		t.Fatalf("written back, still open: %v", keys)
	}
	// An engine older than the judging says no network: nothing raised, and node show says it is not known.
	say(h0.Add(3*time.Minute), nil)
	kept, had, _ = inv.HealthOf(ctx, "laptop")
	if keys := openKeys(); len(keys) != 0 || kept.Network != nil {
		t.Fatalf("an older engine: %v %+v", keys, kept.Network)
	}
	if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "older than that judging") {
		t.Fatalf("node show says:\n%s", lines)
	}
}

// The gate: a resolver file another program rewrote waits the judging rather than failing it at the
// bound; the same, when the send moved the module whose file it is, is that module's; a machine's own
// network fault holds the machine as a whole, as before.
func TestTheGateWaitsOnARewriteItDidNotMakeAndHoldsTheOwnerOnOneItMoved(t *testing.T) {
	since := h0
	rewrite := conditions.Condition{Key: "machine.laptop.networkmanager.rewritten", Kind: kindNetworkRewritten,
		Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop.networkmanager", Machine: "laptop"},
		Summary: "the resolver file was rewritten", Raised: since.Add(time.Minute), Source: sourceNetwork}
	f := gateFacts{judged: true, open: []conditions.Condition{rewrite}}
	if w := aboutTheMachine("laptop", []string{"letta"}, since, f); w.waiting == "" || w.whole != "" || len(w.on) != 0 {
		t.Fatalf("a rewrite the send did not make: %+v", w)
	}
	if w := aboutTheMachine("laptop", []string{"networkmanager", "letta"}, since, f); w.on["networkmanager"] == "" ||
		w.on["letta"] != "" || w.waiting != "" {
		t.Fatalf("a rewrite of the file a moved module owns: %+v", w)
	}
	own := conditions.Condition{Key: "machine.laptop.network", Kind: kindMachineNetwork,
		Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop", Machine: "laptop"},
		Summary: "laptop's network is not healthy", Raised: since.Add(time.Minute), Source: sourceNetwork}
	if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true,
		open: []conditions.Condition{own}}); w.whole == "" || w.waiting != "" {
		t.Fatalf("the machine's own network: %+v", w)
	}
	unreachable := conditions.Condition{Key: "machine.anchor.unreachable", Kind: kindNetworkUnreachable,
		Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor", Also: []string{"laptop", "spare"}},
		Summary: "anchor cannot be reached", Raised: since.Add(time.Minute), Source: sourceNetwork}
	if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true,
		open: []conditions.Condition{unreachable}}); w.waiting == "" || w.whole != "" {
		t.Fatalf("a machine that cannot reach the hub: %+v", w)
	}
}

// TestTheDrillsStatementsRaiseAndClearTheRewrite replays the drill of ADR 0241 (mesh-host
// internal/network TestDrill…): what a node-engine said in a throwaway container while its resolver file
// was declared, rewritten as a VPN client rewrites it, and written back — recorded, with the mesh's names
// and addresses replaced by this test mesh's. Healthy raises nothing; the rewrite, on its second look, is
// raised as one finding naming the writer; written back, it clears.
func TestTheDrillsStatementsRaiseAndClearTheRewrite(t *testing.T) {
	raw, err := os.ReadFile("testdata/network-drill.json")
	if err != nil {
		t.Fatal(err)
	}
	var said []link.Health
	if err := json.Unmarshal(raw, &said); err != nil {
		t.Fatal(err)
	}
	open := aMesh(t)
	ctx := t.Context()
	k := conditionsFrom
	var raisedAt []int
	for i, h := range said {
		if err := stateHealth(ctx, open.inventory, k, "laptop", h, h.At); err != nil {
			t.Fatal(err)
		}
		list, err := k.Open(ctx)
		if err != nil {
			t.Fatal(err)
		}
		for _, c := range list {
			if c.Key != "machine.laptop.networkmanager.rewritten" || !strings.Contains(c.Summary, "(FortiClient)") {
				t.Fatalf("statement %d raised %s: %s", i, c.Key, c.Summary)
			}
			raisedAt = append(raisedAt, i)
		}
	}
	// Five statements: healthy, healthy, one failing look (still healthy), unhealthy, written back.
	if len(raisedAt) != 1 || raisedAt[0] != 3 {
		t.Fatalf("the rewrite was open after statements %v; want after the fourth alone", raisedAt)
	}
}