mesh-controller/cmd/mesh-controller/sayable_test.go

package main

import (
	"context"
	"fmt"
	"net"
	"os"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"golang.org/x/net/dns/dnsmessage"

	"github.com/novox/mesh-controller/internal/conditions"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/outward"
)

// **Every condition this suite raises says itself in machine names and words** (novox/hq issue 277,
// ADR 0234 §6). Every observation a keeper takes in any test of this package — every signals row
// suppressed past its bound, every probe's finding, every event's condition — is held to the operator
// channel's content rule; one whose summary or key carries an address, a domain, a path or a secret's
// shape fails the suite, naming its source. The keeper would say such a summary in words at run time;
// this is where the producer is made to say it rightly in the first place.
func TestMain(m *testing.M) {
	// **A command line this suite runs ends when it is done** (novox/hq issue 591). Many tests run this test
	// binary as a verb's command line, and a binary built with the race detector sleeps a second before it
	// exits (GORACE's atexit_sleep_ms), which was more than the rest of each such test. The children still
	// run under the race detector and still fail on a race they find; they only stop waiting at their end.
	// Set before any child starts, and only where nobody chose GORACE.
	if _, chosen := os.LookupEnv("GORACE"); !chosen {
		_ = os.Setenv("GORACE", "atexit_sleep_ms=0")
	}
	// The process a mesh-cli test runs as a command line: it says the verb and the caller it was given, and
	// ends (meshcli_test.go).
	// The process a mesh-cli test runs as `secret accept`: it reads its value exactly as `secret accept` does
	// (valueFor) and says whether it is the one the test gave, never the value (meshcli_stdin_test.go).
	if want := os.Getenv(secretAcceptWants); want != "" {
		os.Exit(readAsSecretAccept(want, os.Args[1:]))
	}
	if os.Getenv(echoEnvironment) != "" {
		fmt.Printf("verb=%q caller=%q terminal=%v\n", os.Getenv("MESH_VERB"), os.Getenv("MESH_CALLER"), startedAtTheTerminal())
		os.Exit(0)
	}
	conditions.Unsayable = func(o conditions.Observation, field string, r outward.Refusal) {
		unsaid.note(o, field, r)
	}
	conditions.Unworded = func(o conditions.Observation, why string) {
		if o.Source != "test" { // a kind a test makes up to exercise the keeper
			unworded.add(o, why)
		}
	}
	code := m.Run()
	if said := unworded.all(); len(said) > 0 {
		fmt.Fprintf(os.Stderr, "FAIL: %d condition(s) raised in these tests have no plain words of their own, or "+
			"words that are not plain (novox/hq ADR 0253: a headline, an explanation and a resolved line the operator "+
			"reads at a glance — plain_words.go):\n  %s\n", len(said), strings.Join(said, "\n  "))
		if code == 0 {
			code = 1
		}
	}
	if said := unsaid.all(); len(said) > 0 {
		fmt.Fprintf(os.Stderr, "FAIL: %d condition(s) raised in these tests say what the operator's channel withholds "+
			"(an address, a domain, a path or a secret's shape belongs in the evidence, not the summary):\n  %s\n",
			len(said), strings.Join(said, "\n  "))
		if code == 0 {
			code = 1
		}
	}
	os.Exit(code)
}

// unsaid collects, across the suite, every finding whose words the operator's channel would withhold.
var unsaid unsayable

type unsayable struct {
	mu   sync.Mutex
	seen map[string]bool
}

func (u *unsayable) note(o conditions.Observation, field string, r outward.Refusal) {
	text := o.Summary
	if field == "key" {
		text = o.Key()
	}
	u.mu.Lock()
	defer u.mu.Unlock()
	if u.seen == nil {
		u.seen = map[string]bool{}
	}
	u.seen[fmt.Sprintf("%s (source %s, kind %s): its %s carries %s — %q", o.Key(), o.Source, o.Kind, field, r.What,
		text)] = true
}

func (u *unsayable) all() []string {
	u.mu.Lock()
	defer u.mu.Unlock()
	var out []string
	for s := range u.seen {
		out = append(out, s)
	}
	sort.Strings(out)
	return out
}

// unworded collects, across the suite, every finding said in borrowed words (ADR 0253).
var unworded wordless

type wordless struct {
	mu   sync.Mutex
	seen map[string]bool
}

func (u *wordless) add(o conditions.Observation, why string) {
	u.mu.Lock()
	defer u.mu.Unlock()
	if u.seen == nil {
		u.seen = map[string]bool{}
	}
	u.seen[fmt.Sprintf("kind %s (scope %s, source %s): %s", o.Kind, o.Scope, o.Source, why)] = true
}

func (u *wordless) all() []string {
	u.mu.Lock()
	defer u.mu.Unlock()
	var out []string
	for s := range u.seen {
		out = append(out, s)
	}
	sort.Strings(out)
	return out
}

// linted is a producer's findings, held to the content rule as a keeper would hold them: for a test
// that reads a producer's findings without raising them.
func linted(obs []conditions.Observation) []conditions.Observation {
	for _, o := range obs {
		machines := append([]string{o.Machine}, o.Also...)
		if r, ok := outward.Check(o.Key(), machines...); !ok {
			unsaid.note(o, "key", r)
		}
		if r, ok := outward.Check(o.Summary, machines...); !ok {
			unsaid.note(o, "summary", r)
		}
		switch {
		case o.Headline != "":
			w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved}
			if w.Resolved == "" {
				w.Resolved = "Resolved"
			}
			if why, ok := conditions.PlainWords(w, machines...); !ok {
				unworded.add(o, "its source's words are not plain: "+why)
			}
		case o.Kind != "" && !conditions.Worded(o.Kind):
			unworded.add(o, "the kind "+o.Kind+" has no plain words")
		}
	}
	return obs
}

// resolverStandIn is a resolver on loopback that answers as answer says; nil answers nothing.
func resolverStandIn(t *testing.T, answer func(q dnsmessage.Question, n int64) *dnsmessage.Message) (port string, asked *atomic.Int64) {
	t.Helper()
	conn, err := net.ListenPacket("udp", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = conn.Close() })
	asked = &atomic.Int64{}
	go func() {
		buf := make([]byte, 1500)
		for {
			n, from, err := conn.ReadFrom(buf)
			if err != nil {
				return
			}
			var q dnsmessage.Message
			if q.Unpack(buf[:n]) != nil || len(q.Questions) == 0 {
				continue
			}
			reply := answer(q.Questions[0], asked.Add(1))
			if reply == nil {
				continue
			}
			reply.Header.ID, reply.Header.Response, reply.Questions = q.ID, true, q.Questions
			packed, _ := reply.Pack()
			_, _ = conn.WriteTo(packed, from)
		}
	}()
	_, port, _ = net.SplitHostPort(conn.LocalAddr().String())
	return port, asked
}

// answersRightly answers A with the address the test's machine has, and NODATA for AAAA.
func answersRightly(q dnsmessage.Question) *dnsmessage.Message {
	reply := &dnsmessage.Message{}
	if q.Type == dnsmessage.TypeA {
		reply.Answers = []dnsmessage.Resource{{Header: dnsmessage.ResourceHeader{Name: q.Name, Type: dnsmessage.TypeA,
			Class: dnsmessage.ClassINET}, Body: &dnsmessage.AResource{A: [4]byte{10, 77, 0, 1}}}}
	}
	return reply
}

// quickResolvers makes D2's patience short for a test.
func quickResolvers(t *testing.T, port string) {
	t.Helper()
	before, within, pause := resolverPort, resolverWithin, resolverPause
	resolverPort, resolverWithin, resolverPause = port, 150*time.Millisecond, 10*time.Millisecond
	t.Cleanup(func() { resolverPort, resolverWithin, resolverPause = before, within, pause })
}

var anchorOnTheNetwork = []inventory.Overlay{{Name: "anchor", Address: "10.77.0.1"}}

// **D2 asks again before it says anything** (novox/hq issue 277): a resolver that misses a question on
// a loaded machine and answers the next try is a resolver that answers.
func TestAResolverThatMissesOneTryAndAnswersTheNextIsWell(t *testing.T) {
	port, asked := resolverStandIn(t, func(q dnsmessage.Question, n int64) *dnsmessage.Message {
		if n <= 2 {
			return nil // the first question of each kind lost, as under a push and a build starting
		}
		return answersRightly(q)
	})
	quickResolvers(t, port)
	got := askEveryResolver(t.Context(), map[string]string{"anchor.internal": "127.0.0.1"}, anchorOnTheNetwork, "internal")
	if len(got) != 0 {
		t.Fatalf("a resolver that answered its second try was said: %+v", got)
	}
	if asked.Load() < 3 {
		t.Fatalf("asked %d times", asked.Load())
	}
}

// **A resolver that answers nothing is held for the next run, and said in machine names**: the address
// it was asked at and the socket's words are the evidence, never the summary the operator reads.
func TestAResolverThatAnswersNothingIsHeldAndSaidInMachineNames(t *testing.T) {
	port, asked := resolverStandIn(t, func(dnsmessage.Question, int64) *dnsmessage.Message { return nil })
	quickResolvers(t, port)
	got := askEveryResolver(t.Context(), map[string]string{"anchor.internal": "127.0.0.1"}, anchorOnTheNetwork, "internal")
	if len(got) != 1 {
		t.Fatalf("%+v", got)
	}
	o := got[0]
	if !o.Confirm || o.Key() != "seat.mesh-dns-resolver.anchor.wrong" || o.Severity != conditions.Urgent {
		t.Fatalf("not held for a second look, or not the resolver's condition: %+v", o)
	}
	if r, ok := outward.Check(o.Summary, "anchor"); !ok {
		t.Fatalf("the summary carries %s: %q", r, o.Summary)
	}
	if !strings.Contains(o.Summary, "anchor's") || !strings.Contains(o.Said, "127.0.0.1") ||
		!strings.Contains(o.Said, "anchor.internal") {
		t.Fatalf("summary %q, evidence %q", o.Summary, o.Said)
	}
	if want := int64(2 * resolverTries); asked.Load() != want {
		t.Fatalf("asked %d times, want %d: each question %d times", asked.Load(), want, resolverTries)
	}
}

// **A resolver that answers wrongly is said at once**, in machine names: an answer is not the absence
// of one (issue 262's NXDOMAIN for IPv6).
func TestAResolverAnsweringWronglyIsSaidAtOnceInMachineNames(t *testing.T) {
	port, _ := resolverStandIn(t, func(q dnsmessage.Question, _ int64) *dnsmessage.Message {
		if q.Type == dnsmessage.TypeAAAA {
			return &dnsmessage.Message{Header: dnsmessage.Header{RCode: dnsmessage.RCodeNameError}}
		}
		return answersRightly(q)
	})
	quickResolvers(t, port)
	got := askEveryResolver(t.Context(), map[string]string{"anchor.internal": "127.0.0.1"}, anchorOnTheNetwork, "internal")
	if len(got) != 1 || got[0].Confirm {
		t.Fatalf("%+v", got)
	}
	if r, ok := outward.Check(got[0].Summary, "anchor"); !ok || !strings.Contains(got[0].Summary, "NXDOMAIN") ||
		!strings.Contains(got[0].Summary, "anchor's IPv6 address") {
		t.Fatalf("summary %q (%v)", got[0].Summary, r)
	}
}

// **The self-check raises a finding one look can be wrong about on the second run in a row**, keeps it
// open while it is seen, and clears it when it is not — and a probe that cannot run is said as a
// condition only when the next run cannot run it either. Neither is ever a pass in the verdict.
func TestASingleLookIsHeldAndTheSecondInARowRaises(t *testing.T) {
	var unanswered, broken atomic.Bool
	held := conditions.Observation{Scope: conditions.ScopeSeat, ID: "mesh-dns-resolver.anchor", Token: "wrong",
		Machine: "anchor", Severity: conditions.Urgent, Confirm: true,
		Summary: "the mesh's resolver on anchor does not answer", Said: "no answer from 10.77.0.1"}
	withProbes(t,
		probe{ID: "P1", Asserts: "asks over the network", Kind: "resolver-wrong", Phase: 1,
			run: func(context.Context, *doctor) ([]conditions.Observation, error) {
				if unanswered.Load() {
					return []conditions.Observation{held}, nil
				}
				return nil, nil
			}},
		probe{ID: "P2", Asserts: "sometimes cannot run", Kind: "x", Phase: 1,
			run: func(context.Context, *doctor) ([]conditions.Observation, error) {
				if broken.Load() {
					return nil, fmt.Errorf("a question timed out")
				}
				return nil, nil
			}},
	)
	store := conditions.NewInMemory()
	k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
	defer k.Close(context.Background())
	d := &doctor{keeper: k, host: "anchor"}
	keys := func() []string {
		open, err := k.Open(t.Context())
		if err != nil {
			t.Fatal(err)
		}
		var out []string
		for _, c := range open {
			out = append(out, c.Key)
		}
		sort.Strings(out)
		return out
	}

	unanswered.Store(true)
	broken.Store(true)
	run := d.runOnce(t.Context(), "a test")
	if open := keys(); len(open) != 0 {
		t.Fatalf("one look raised %v", open)
	}
	if run.Probes[0].Verdict != verdictPass || len(run.Probes[0].Unconfirmed) != 1 || run.Probes[1].Verdict != verdictFailedToRun {
		t.Fatalf("the first run's verdict hides what it saw: %+v", run.Probes)
	}

	// Seen twice in a row: raised, and so is the probe that could not run twice.
	run = d.runOnce(t.Context(), "a test")
	if open := keys(); strings.Join(open, " ") != "probe.P2.failed seat.mesh-dns-resolver.anchor.wrong" {
		t.Fatalf("two looks in a row left open %v", open)
	}
	if run.Probes[0].Verdict != verdictFail {
		t.Fatalf("%+v", run.Probes[0])
	}

	// Open, and seen again: kept, never cleared and raised again.
	d.runOnce(t.Context(), "a test")
	if c, open, _ := k.Get(t.Context(), "seat.mesh-dns-resolver.anchor.wrong"); !open || c.Count != 1 || c.Observations != 2 {
		t.Fatalf("the open condition was not kept as it was: %+v", c)
	}

	// Answered again: cleared. Then one look alone raises nothing again.
	unanswered.Store(false)
	broken.Store(false)
	d.runOnce(t.Context(), "a test")
	if open := keys(); len(open) != 0 {
		t.Fatalf("a passing run left open %v", open)
	}
	unanswered.Store(true)
	d.runOnce(t.Context(), "a test")
	if open := keys(); len(open) != 0 {
		t.Fatalf("one look after a pass raised %v", open)
	}
}