package main import ( "context" "fmt" "net" "os" "sort" "strings" "sync" "sync/atomic" "testing" "time" "golang.org/x/net/dns/dnsmessage" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/outward" ) // **Every condition this suite raises says itself in machine names and words** (novox/hq issue 277, // ADR 0234 §6). Every observation a keeper takes in any test of this package — every signals row // suppressed past its bound, every probe's finding, every event's condition — is held to the operator // channel's content rule; one whose summary or key carries an address, a domain, a path or a secret's // shape fails the suite, naming its source. The keeper would say such a summary in words at run time; // this is where the producer is made to say it rightly in the first place. func TestMain(m *testing.M) { // **A command line this suite runs ends when it is done** (novox/hq issue 591). Many tests run this test // binary as a verb's command line, and a binary built with the race detector sleeps a second before it // exits (GORACE's atexit_sleep_ms), which was more than the rest of each such test. The children still // run under the race detector and still fail on a race they find; they only stop waiting at their end. // Set before any child starts, and only where nobody chose GORACE. if _, chosen := os.LookupEnv("GORACE"); !chosen { _ = os.Setenv("GORACE", "atexit_sleep_ms=0") } // The process a mesh-cli test runs as a command line: it says the verb and the caller it was given, and // ends (meshcli_test.go). // The process a mesh-cli test runs as `secret accept`: it reads its value exactly as `secret accept` does // (valueFor) and says whether it is the one the test gave, never the value (meshcli_stdin_test.go). if want := os.Getenv(secretAcceptWants); want != "" { os.Exit(readAsSecretAccept(want, os.Args[1:])) } if os.Getenv(echoEnvironment) != "" { fmt.Printf("verb=%q caller=%q terminal=%v\n", os.Getenv("MESH_VERB"), os.Getenv("MESH_CALLER"), startedAtTheTerminal()) os.Exit(0) } conditions.Unsayable = func(o conditions.Observation, field string, r outward.Refusal) { unsaid.note(o, field, r) } conditions.Unworded = func(o conditions.Observation, why string) { if o.Source != "test" { // a kind a test makes up to exercise the keeper unworded.add(o, why) } } code := m.Run() if said := unworded.all(); len(said) > 0 { fmt.Fprintf(os.Stderr, "FAIL: %d condition(s) raised in these tests have no plain words of their own, or "+ "words that are not plain (novox/hq ADR 0253: a headline, an explanation and a resolved line the operator "+ "reads at a glance — plain_words.go):\n %s\n", len(said), strings.Join(said, "\n ")) if code == 0 { code = 1 } } if said := unsaid.all(); len(said) > 0 { fmt.Fprintf(os.Stderr, "FAIL: %d condition(s) raised in these tests say what the operator's channel withholds "+ "(an address, a domain, a path or a secret's shape belongs in the evidence, not the summary):\n %s\n", len(said), strings.Join(said, "\n ")) if code == 0 { code = 1 } } os.Exit(code) } // unsaid collects, across the suite, every finding whose words the operator's channel would withhold. var unsaid unsayable type unsayable struct { mu sync.Mutex seen map[string]bool } func (u *unsayable) note(o conditions.Observation, field string, r outward.Refusal) { text := o.Summary if field == "key" { text = o.Key() } u.mu.Lock() defer u.mu.Unlock() if u.seen == nil { u.seen = map[string]bool{} } u.seen[fmt.Sprintf("%s (source %s, kind %s): its %s carries %s — %q", o.Key(), o.Source, o.Kind, field, r.What, text)] = true } func (u *unsayable) all() []string { u.mu.Lock() defer u.mu.Unlock() var out []string for s := range u.seen { out = append(out, s) } sort.Strings(out) return out } // unworded collects, across the suite, every finding said in borrowed words (ADR 0253). var unworded wordless type wordless struct { mu sync.Mutex seen map[string]bool } func (u *wordless) add(o conditions.Observation, why string) { u.mu.Lock() defer u.mu.Unlock() if u.seen == nil { u.seen = map[string]bool{} } u.seen[fmt.Sprintf("kind %s (scope %s, source %s): %s", o.Kind, o.Scope, o.Source, why)] = true } func (u *wordless) all() []string { u.mu.Lock() defer u.mu.Unlock() var out []string for s := range u.seen { out = append(out, s) } sort.Strings(out) return out } // linted is a producer's findings, held to the content rule as a keeper would hold them: for a test // that reads a producer's findings without raising them. func linted(obs []conditions.Observation) []conditions.Observation { for _, o := range obs { machines := append([]string{o.Machine}, o.Also...) if r, ok := outward.Check(o.Key(), machines...); !ok { unsaid.note(o, "key", r) } if r, ok := outward.Check(o.Summary, machines...); !ok { unsaid.note(o, "summary", r) } switch { case o.Headline != "": w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved} if w.Resolved == "" { w.Resolved = "Resolved" } if why, ok := conditions.PlainWords(w, machines...); !ok { unworded.add(o, "its source's words are not plain: "+why) } case o.Kind != "" && !conditions.Worded(o.Kind): unworded.add(o, "the kind "+o.Kind+" has no plain words") } } return obs } // resolverStandIn is a resolver on loopback that answers as answer says; nil answers nothing. func resolverStandIn(t *testing.T, answer func(q dnsmessage.Question, n int64) *dnsmessage.Message) (port string, asked *atomic.Int64) { t.Helper() conn, err := net.ListenPacket("udp", "127.0.0.1:0") if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = conn.Close() }) asked = &atomic.Int64{} go func() { buf := make([]byte, 1500) for { n, from, err := conn.ReadFrom(buf) if err != nil { return } var q dnsmessage.Message if q.Unpack(buf[:n]) != nil || len(q.Questions) == 0 { continue } reply := answer(q.Questions[0], asked.Add(1)) if reply == nil { continue } reply.Header.ID, reply.Header.Response, reply.Questions = q.ID, true, q.Questions packed, _ := reply.Pack() _, _ = conn.WriteTo(packed, from) } }() _, port, _ = net.SplitHostPort(conn.LocalAddr().String()) return port, asked } // answersRightly answers A with the address the test's machine has, and NODATA for AAAA. func answersRightly(q dnsmessage.Question) *dnsmessage.Message { reply := &dnsmessage.Message{} if q.Type == dnsmessage.TypeA { reply.Answers = []dnsmessage.Resource{{Header: dnsmessage.ResourceHeader{Name: q.Name, Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET}, Body: &dnsmessage.AResource{A: [4]byte{10, 77, 0, 1}}}} } return reply } // quickResolvers makes D2's patience short for a test. func quickResolvers(t *testing.T, port string) { t.Helper() before, within, pause := resolverPort, resolverWithin, resolverPause resolverPort, resolverWithin, resolverPause = port, 150*time.Millisecond, 10*time.Millisecond t.Cleanup(func() { resolverPort, resolverWithin, resolverPause = before, within, pause }) } var anchorOnTheNetwork = []inventory.Overlay{{Name: "anchor", Address: "10.77.0.1"}} // **D2 asks again before it says anything** (novox/hq issue 277): a resolver that misses a question on // a loaded machine and answers the next try is a resolver that answers. func TestAResolverThatMissesOneTryAndAnswersTheNextIsWell(t *testing.T) { port, asked := resolverStandIn(t, func(q dnsmessage.Question, n int64) *dnsmessage.Message { if n <= 2 { return nil // the first question of each kind lost, as under a push and a build starting } return answersRightly(q) }) quickResolvers(t, port) got := askEveryResolver(t.Context(), map[string]string{"anchor.internal": "127.0.0.1"}, anchorOnTheNetwork, "internal") if len(got) != 0 { t.Fatalf("a resolver that answered its second try was said: %+v", got) } if asked.Load() < 3 { t.Fatalf("asked %d times", asked.Load()) } } // **A resolver that answers nothing is held for the next run, and said in machine names**: the address // it was asked at and the socket's words are the evidence, never the summary the operator reads. func TestAResolverThatAnswersNothingIsHeldAndSaidInMachineNames(t *testing.T) { port, asked := resolverStandIn(t, func(dnsmessage.Question, int64) *dnsmessage.Message { return nil }) quickResolvers(t, port) got := askEveryResolver(t.Context(), map[string]string{"anchor.internal": "127.0.0.1"}, anchorOnTheNetwork, "internal") if len(got) != 1 { t.Fatalf("%+v", got) } o := got[0] if !o.Confirm || o.Key() != "seat.mesh-dns-resolver.anchor.wrong" || o.Severity != conditions.Urgent { t.Fatalf("not held for a second look, or not the resolver's condition: %+v", o) } if r, ok := outward.Check(o.Summary, "anchor"); !ok { t.Fatalf("the summary carries %s: %q", r, o.Summary) } if !strings.Contains(o.Summary, "anchor's") || !strings.Contains(o.Said, "127.0.0.1") || !strings.Contains(o.Said, "anchor.internal") { t.Fatalf("summary %q, evidence %q", o.Summary, o.Said) } if want := int64(2 * resolverTries); asked.Load() != want { t.Fatalf("asked %d times, want %d: each question %d times", asked.Load(), want, resolverTries) } } // **A resolver that answers wrongly is said at once**, in machine names: an answer is not the absence // of one (issue 262's NXDOMAIN for IPv6). func TestAResolverAnsweringWronglyIsSaidAtOnceInMachineNames(t *testing.T) { port, _ := resolverStandIn(t, func(q dnsmessage.Question, _ int64) *dnsmessage.Message { if q.Type == dnsmessage.TypeAAAA { return &dnsmessage.Message{Header: dnsmessage.Header{RCode: dnsmessage.RCodeNameError}} } return answersRightly(q) }) quickResolvers(t, port) got := askEveryResolver(t.Context(), map[string]string{"anchor.internal": "127.0.0.1"}, anchorOnTheNetwork, "internal") if len(got) != 1 || got[0].Confirm { t.Fatalf("%+v", got) } if r, ok := outward.Check(got[0].Summary, "anchor"); !ok || !strings.Contains(got[0].Summary, "NXDOMAIN") || !strings.Contains(got[0].Summary, "anchor's IPv6 address") { t.Fatalf("summary %q (%v)", got[0].Summary, r) } } // **The self-check raises a finding one look can be wrong about on the second run in a row**, keeps it // open while it is seen, and clears it when it is not — and a probe that cannot run is said as a // condition only when the next run cannot run it either. Neither is ever a pass in the verdict. func TestASingleLookIsHeldAndTheSecondInARowRaises(t *testing.T) { var unanswered, broken atomic.Bool held := conditions.Observation{Scope: conditions.ScopeSeat, ID: "mesh-dns-resolver.anchor", Token: "wrong", Machine: "anchor", Severity: conditions.Urgent, Confirm: true, Summary: "the mesh's resolver on anchor does not answer", Said: "no answer from 10.77.0.1"} withProbes(t, probe{ID: "P1", Asserts: "asks over the network", Kind: "resolver-wrong", Phase: 1, run: func(context.Context, *doctor) ([]conditions.Observation, error) { if unanswered.Load() { return []conditions.Observation{held}, nil } return nil, nil }}, probe{ID: "P2", Asserts: "sometimes cannot run", Kind: "x", Phase: 1, run: func(context.Context, *doctor) ([]conditions.Observation, error) { if broken.Load() { return nil, fmt.Errorf("a question timed out") } return nil, nil }}, ) store := conditions.NewInMemory() k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store}) defer k.Close(context.Background()) d := &doctor{keeper: k, host: "anchor"} keys := func() []string { open, err := k.Open(t.Context()) if err != nil { t.Fatal(err) } var out []string for _, c := range open { out = append(out, c.Key) } sort.Strings(out) return out } unanswered.Store(true) broken.Store(true) run := d.runOnce(t.Context(), "a test") if open := keys(); len(open) != 0 { t.Fatalf("one look raised %v", open) } if run.Probes[0].Verdict != verdictPass || len(run.Probes[0].Unconfirmed) != 1 || run.Probes[1].Verdict != verdictFailedToRun { t.Fatalf("the first run's verdict hides what it saw: %+v", run.Probes) } // Seen twice in a row: raised, and so is the probe that could not run twice. run = d.runOnce(t.Context(), "a test") if open := keys(); strings.Join(open, " ") != "probe.P2.failed seat.mesh-dns-resolver.anchor.wrong" { t.Fatalf("two looks in a row left open %v", open) } if run.Probes[0].Verdict != verdictFail { t.Fatalf("%+v", run.Probes[0]) } // Open, and seen again: kept, never cleared and raised again. d.runOnce(t.Context(), "a test") if c, open, _ := k.Get(t.Context(), "seat.mesh-dns-resolver.anchor.wrong"); !open || c.Count != 1 || c.Observations != 2 { t.Fatalf("the open condition was not kept as it was: %+v", c) } // Answered again: cleared. Then one look alone raises nothing again. unanswered.Store(false) broken.Store(false) d.runOnce(t.Context(), "a test") if open := keys(); len(open) != 0 { t.Fatalf("a passing run left open %v", open) } unanswered.Store(true) d.runOnce(t.Context(), "a test") if open := keys(); len(open) != 0 { t.Fatalf("one look after a pass raised %v", open) } }