jochen
8e8712e352
D2 raised a resolver urgent on one query that timed out while its machine was loaded, and its summary carried the resolver's address and socket text, so the operator channel withheld the whole alert. - D2 asks every question up to three times, all at once; a resolver that answers nothing is held for the next run and raised urgent when two runs in a row find it silent. A wrong answer is still raised at once. - Findings a single look can be wrong about carry Confirm: raised on the second look in a row, kept while open, never cleared-and-reraised. Used by D2 silence, D3 (also asks discovery twice), D6 behind, D9, D13 unmeasured, probe-failed of the doctor, and blind watchdog rows. - Probe seat asks (D8, D13) are asked again when the bus brought no answer. - Summaries name machines and say things in words; addresses, paths, domains and raw errors move to the evidence (D2, D5, D8, D9, D13, S12). - internal/outward mirrors the messenger's content rule, allowing the mesh's machine names; the keeper rewords a summary that would be withheld and keeps it whole in the evidence; a TestMain lint fails the suite on any raised or linted finding that would be withheld.
288 lines
10 KiB
Go
288 lines
10 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net"
|
|
"os"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"golang.org/x/net/dns/dnsmessage"
|
|
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/outward"
|
|
)
|
|
|
|
// **Every condition this suite raises says itself in machine names and words** (novox/hq issue 277,
|
|
// ADR 0234 §6). Every observation a keeper takes in any test of this package — every signals row
|
|
// suppressed past its bound, every probe's finding, every event's condition — is held to the operator
|
|
// channel's content rule; one whose summary or key carries an address, a domain, a path or a secret's
|
|
// shape fails the suite, naming its source. The keeper would say such a summary in words at run time;
|
|
// this is where the producer is made to say it rightly in the first place.
|
|
func TestMain(m *testing.M) {
|
|
conditions.Unsayable = func(o conditions.Observation, field string, r outward.Refusal) {
|
|
unsaid.note(o, field, r)
|
|
}
|
|
code := m.Run()
|
|
if said := unsaid.all(); len(said) > 0 {
|
|
fmt.Fprintf(os.Stderr, "FAIL: %d condition(s) raised in these tests say what the operator's channel withholds "+
|
|
"(an address, a domain, a path or a secret's shape belongs in the evidence, not the summary):\n %s\n",
|
|
len(said), strings.Join(said, "\n "))
|
|
if code == 0 {
|
|
code = 1
|
|
}
|
|
}
|
|
os.Exit(code)
|
|
}
|
|
|
|
// unsaid collects, across the suite, every finding whose words the operator's channel would withhold.
|
|
var unsaid unsayable
|
|
|
|
type unsayable struct {
|
|
mu sync.Mutex
|
|
seen map[string]bool
|
|
}
|
|
|
|
func (u *unsayable) note(o conditions.Observation, field string, r outward.Refusal) {
|
|
text := o.Summary
|
|
if field == "key" {
|
|
text = o.Key()
|
|
}
|
|
u.mu.Lock()
|
|
defer u.mu.Unlock()
|
|
if u.seen == nil {
|
|
u.seen = map[string]bool{}
|
|
}
|
|
u.seen[fmt.Sprintf("%s (source %s, kind %s): its %s carries %s — %q", o.Key(), o.Source, o.Kind, field, r.What,
|
|
text)] = true
|
|
}
|
|
|
|
func (u *unsayable) all() []string {
|
|
u.mu.Lock()
|
|
defer u.mu.Unlock()
|
|
var out []string
|
|
for s := range u.seen {
|
|
out = append(out, s)
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
// linted is a producer's findings, held to the content rule as a keeper would hold them: for a test
|
|
// that reads a producer's findings without raising them.
|
|
func linted(obs []conditions.Observation) []conditions.Observation {
|
|
for _, o := range obs {
|
|
machines := append([]string{o.Machine}, o.Also...)
|
|
if r, ok := outward.Check(o.Key(), machines...); !ok {
|
|
unsaid.note(o, "key", r)
|
|
}
|
|
if r, ok := outward.Check(o.Summary, machines...); !ok {
|
|
unsaid.note(o, "summary", r)
|
|
}
|
|
}
|
|
return obs
|
|
}
|
|
|
|
// resolverStandIn is a resolver on loopback that answers as answer says; nil answers nothing.
|
|
func resolverStandIn(t *testing.T, answer func(q dnsmessage.Question, n int64) *dnsmessage.Message) (port string, asked *atomic.Int64) {
|
|
t.Helper()
|
|
conn, err := net.ListenPacket("udp", "127.0.0.1:0")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = conn.Close() })
|
|
asked = &atomic.Int64{}
|
|
go func() {
|
|
buf := make([]byte, 1500)
|
|
for {
|
|
n, from, err := conn.ReadFrom(buf)
|
|
if err != nil {
|
|
return
|
|
}
|
|
var q dnsmessage.Message
|
|
if q.Unpack(buf[:n]) != nil || len(q.Questions) == 0 {
|
|
continue
|
|
}
|
|
reply := answer(q.Questions[0], asked.Add(1))
|
|
if reply == nil {
|
|
continue
|
|
}
|
|
reply.Header.ID, reply.Header.Response, reply.Questions = q.ID, true, q.Questions
|
|
packed, _ := reply.Pack()
|
|
_, _ = conn.WriteTo(packed, from)
|
|
}
|
|
}()
|
|
_, port, _ = net.SplitHostPort(conn.LocalAddr().String())
|
|
return port, asked
|
|
}
|
|
|
|
// answersRightly answers A with the address the test's machine has, and NODATA for AAAA.
|
|
func answersRightly(q dnsmessage.Question) *dnsmessage.Message {
|
|
reply := &dnsmessage.Message{}
|
|
if q.Type == dnsmessage.TypeA {
|
|
reply.Answers = []dnsmessage.Resource{{Header: dnsmessage.ResourceHeader{Name: q.Name, Type: dnsmessage.TypeA,
|
|
Class: dnsmessage.ClassINET}, Body: &dnsmessage.AResource{A: [4]byte{10, 77, 0, 1}}}}
|
|
}
|
|
return reply
|
|
}
|
|
|
|
// quickResolvers makes D2's patience short for a test.
|
|
func quickResolvers(t *testing.T, port string) {
|
|
t.Helper()
|
|
before, within, pause := resolverPort, resolverWithin, resolverPause
|
|
resolverPort, resolverWithin, resolverPause = port, 150*time.Millisecond, 10*time.Millisecond
|
|
t.Cleanup(func() { resolverPort, resolverWithin, resolverPause = before, within, pause })
|
|
}
|
|
|
|
var anchorOnTheNetwork = []inventory.Overlay{{Name: "anchor", Address: "10.77.0.1"}}
|
|
|
|
// **D2 asks again before it says anything** (novox/hq issue 277): a resolver that misses a question on
|
|
// a loaded machine and answers the next try is a resolver that answers.
|
|
func TestAResolverThatMissesOneTryAndAnswersTheNextIsWell(t *testing.T) {
|
|
port, asked := resolverStandIn(t, func(q dnsmessage.Question, n int64) *dnsmessage.Message {
|
|
if n <= 2 {
|
|
return nil // the first question of each kind lost, as under a push and a build starting
|
|
}
|
|
return answersRightly(q)
|
|
})
|
|
quickResolvers(t, port)
|
|
got := askEveryResolver(t.Context(), map[string]string{"anchor.internal": "127.0.0.1"}, anchorOnTheNetwork, "internal")
|
|
if len(got) != 0 {
|
|
t.Fatalf("a resolver that answered its second try was said: %+v", got)
|
|
}
|
|
if asked.Load() < 3 {
|
|
t.Fatalf("asked %d times", asked.Load())
|
|
}
|
|
}
|
|
|
|
// **A resolver that answers nothing is held for the next run, and said in machine names**: the address
|
|
// it was asked at and the socket's words are the evidence, never the summary the operator reads.
|
|
func TestAResolverThatAnswersNothingIsHeldAndSaidInMachineNames(t *testing.T) {
|
|
port, asked := resolverStandIn(t, func(dnsmessage.Question, int64) *dnsmessage.Message { return nil })
|
|
quickResolvers(t, port)
|
|
got := askEveryResolver(t.Context(), map[string]string{"anchor.internal": "127.0.0.1"}, anchorOnTheNetwork, "internal")
|
|
if len(got) != 1 {
|
|
t.Fatalf("%+v", got)
|
|
}
|
|
o := got[0]
|
|
if !o.Confirm || o.Key() != "seat.mesh-dns-resolver.anchor.wrong" || o.Severity != conditions.Urgent {
|
|
t.Fatalf("not held for a second look, or not the resolver's condition: %+v", o)
|
|
}
|
|
if r, ok := outward.Check(o.Summary, "anchor"); !ok {
|
|
t.Fatalf("the summary carries %s: %q", r, o.Summary)
|
|
}
|
|
if !strings.Contains(o.Summary, "anchor's") || !strings.Contains(o.Said, "127.0.0.1") ||
|
|
!strings.Contains(o.Said, "anchor.internal") {
|
|
t.Fatalf("summary %q, evidence %q", o.Summary, o.Said)
|
|
}
|
|
if want := int64(2 * resolverTries); asked.Load() != want {
|
|
t.Fatalf("asked %d times, want %d: each question %d times", asked.Load(), want, resolverTries)
|
|
}
|
|
}
|
|
|
|
// **A resolver that answers wrongly is said at once**, in machine names: an answer is not the absence
|
|
// of one (issue 262's NXDOMAIN for IPv6).
|
|
func TestAResolverAnsweringWronglyIsSaidAtOnceInMachineNames(t *testing.T) {
|
|
port, _ := resolverStandIn(t, func(q dnsmessage.Question, _ int64) *dnsmessage.Message {
|
|
if q.Type == dnsmessage.TypeAAAA {
|
|
return &dnsmessage.Message{Header: dnsmessage.Header{RCode: dnsmessage.RCodeNameError}}
|
|
}
|
|
return answersRightly(q)
|
|
})
|
|
quickResolvers(t, port)
|
|
got := askEveryResolver(t.Context(), map[string]string{"anchor.internal": "127.0.0.1"}, anchorOnTheNetwork, "internal")
|
|
if len(got) != 1 || got[0].Confirm {
|
|
t.Fatalf("%+v", got)
|
|
}
|
|
if r, ok := outward.Check(got[0].Summary, "anchor"); !ok || !strings.Contains(got[0].Summary, "NXDOMAIN") ||
|
|
!strings.Contains(got[0].Summary, "anchor's IPv6 address") {
|
|
t.Fatalf("summary %q (%v)", got[0].Summary, r)
|
|
}
|
|
}
|
|
|
|
// **The self-check raises a finding one look can be wrong about on the second run in a row**, keeps it
|
|
// open while it is seen, and clears it when it is not — and a probe that cannot run is said as a
|
|
// condition only when the next run cannot run it either. Neither is ever a pass in the verdict.
|
|
func TestASingleLookIsHeldAndTheSecondInARowRaises(t *testing.T) {
|
|
var unanswered, broken atomic.Bool
|
|
held := conditions.Observation{Scope: conditions.ScopeSeat, ID: "mesh-dns-resolver.anchor", Token: "wrong",
|
|
Machine: "anchor", Severity: conditions.Urgent, Confirm: true,
|
|
Summary: "the mesh's resolver on anchor does not answer", Said: "no answer from 10.77.0.1"}
|
|
withProbes(t,
|
|
probe{ID: "P1", Asserts: "asks over the network", Kind: "resolver-wrong", Phase: 1,
|
|
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
|
|
if unanswered.Load() {
|
|
return []conditions.Observation{held}, nil
|
|
}
|
|
return nil, nil
|
|
}},
|
|
probe{ID: "P2", Asserts: "sometimes cannot run", Kind: "x", Phase: 1,
|
|
run: func(context.Context, *doctor) ([]conditions.Observation, error) {
|
|
if broken.Load() {
|
|
return nil, fmt.Errorf("a question timed out")
|
|
}
|
|
return nil, nil
|
|
}},
|
|
)
|
|
store := conditions.NewInMemory()
|
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
|
defer k.Close(context.Background())
|
|
d := &doctor{keeper: k, host: "anchor"}
|
|
keys := func() []string {
|
|
open, err := k.Open(t.Context())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var out []string
|
|
for _, c := range open {
|
|
out = append(out, c.Key)
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
unanswered.Store(true)
|
|
broken.Store(true)
|
|
run := d.runOnce(t.Context(), "a test")
|
|
if open := keys(); len(open) != 0 {
|
|
t.Fatalf("one look raised %v", open)
|
|
}
|
|
if run.Probes[0].Verdict != verdictPass || len(run.Probes[0].Unconfirmed) != 1 || run.Probes[1].Verdict != verdictFailedToRun {
|
|
t.Fatalf("the first run's verdict hides what it saw: %+v", run.Probes)
|
|
}
|
|
|
|
// Seen twice in a row: raised, and so is the probe that could not run twice.
|
|
run = d.runOnce(t.Context(), "a test")
|
|
if open := keys(); strings.Join(open, " ") != "probe.P2.failed seat.mesh-dns-resolver.anchor.wrong" {
|
|
t.Fatalf("two looks in a row left open %v", open)
|
|
}
|
|
if run.Probes[0].Verdict != verdictFail {
|
|
t.Fatalf("%+v", run.Probes[0])
|
|
}
|
|
|
|
// Open, and seen again: kept, never cleared and raised again.
|
|
d.runOnce(t.Context(), "a test")
|
|
if c, open, _ := k.Get(t.Context(), "seat.mesh-dns-resolver.anchor.wrong"); !open || c.Count != 1 || c.Observations != 2 {
|
|
t.Fatalf("the open condition was not kept as it was: %+v", c)
|
|
}
|
|
|
|
// Answered again: cleared. Then one look alone raises nothing again.
|
|
unanswered.Store(false)
|
|
broken.Store(false)
|
|
d.runOnce(t.Context(), "a test")
|
|
if open := keys(); len(open) != 0 {
|
|
t.Fatalf("a passing run left open %v", open)
|
|
}
|
|
unanswered.Store(true)
|
|
d.runOnce(t.Context(), "a test")
|
|
if open := keys(); len(open) != 0 {
|
|
t.Fatalf("one look after a pass raised %v", open)
|
|
}
|
|
}
|