diff --git a/modules/netcheck/README.md b/modules/netcheck/README.md new file mode 100644 index 00000000..1e48e07f --- /dev/null +++ b/modules/netcheck/README.md @@ -0,0 +1,3 @@ +# netcheck + +The machine's own network checks as tools: a TCP connect (`netcheck_tcp`), a DNS lookup of the machine's resolver or of one named (`netcheck_dns`), what listens here (`netcheck_listening`), the private network's peers (`netcheck_tunnel`), and whether a URL answers (`netcheck_http`). `netcheck_request` makes one HTTP request (GET, HEAD or POST, with headers and a body of at most 256 KiB) and answers the status line, every response header and the body (at most 1 MiB, with a flag saying when it was cut), reporting redirects without following them and speaking only http and https, so a service's live answers can be captured (hq issue 1643). diff --git a/modules/netcheck/cmd/netcheck/checks_test.go b/modules/netcheck/cmd/netcheck/checks_test.go index 4912b663..c11621b5 100644 --- a/modules/netcheck/cmd/netcheck/checks_test.go +++ b/modules/netcheck/cmd/netcheck/checks_test.go @@ -62,7 +62,7 @@ func TestTheToolsAreListedUnprefixed(t *testing.T) { for _, tool := range tools() { names[tool.Name] = true } - if !names["netcheck_tcp"] || !names["netcheck_dns"] || !names["netcheck_listening"] || !names["netcheck_tunnel"] || len(names) != 4 { + if !names["netcheck_tcp"] || !names["netcheck_dns"] || !names["netcheck_listening"] || !names["netcheck_tunnel"] || !names["netcheck_request"] || len(names) != 5 { t.Fatalf("tools: %v", names) } } diff --git a/modules/netcheck/cmd/netcheck/main.go b/modules/netcheck/cmd/netcheck/main.go index 67bf9cf8..0ed7362a 100644 --- a/modules/netcheck/cmd/netcheck/main.go +++ b/modules/netcheck/cmd/netcheck/main.go @@ -88,6 +88,22 @@ func tools() []stdio.Tool { return CheckTCP(host, port, timeout) }, }, + { + Name: "netcheck_request", + Description: "Make one HTTP request from this machine and answer what comes back: the status line, every response " + + "header and the body as text (base64 when it is not UTF-8), cut at 1 MiB with body_cut saying so. method is GET, " + + "HEAD or POST; headers and a body (at most 256 KiB) are sent as given. A redirect is reported (status and redirect), " + + "never followed. Only http and https; TLS is verified against this machine's trust store. For what a service " + + "answers, not only whether it is up (that is netcheck_http).", + Input: map[string]any{ + "url": map[string]any{"type": "string", "description": "http:// or https:// URL"}, + "method": map[string]any{"type": "string", "enum": []string{"GET", "HEAD", "POST"}, "description": "default GET"}, + "headers": map[string]any{"type": "object", "additionalProperties": map[string]any{"type": "string"}, "description": "request headers, name to value (optional)"}, + "body": map[string]any{"type": "string", "description": "request body, at most 262144 bytes (optional)"}, + "timeout_ms": map[string]any{"type": "integer", "description": "give up after this long (default 10000, at most 30000)"}, + }, + Run: runRequest, + }, { Name: "netcheck_dns", Description: "Look a name up with this machine's resolver (its /etc/resolv.conf and /etc/hosts), or — with server — ask " + diff --git a/modules/netcheck/cmd/netcheck/request.go b/modules/netcheck/cmd/netcheck/request.go new file mode 100644 index 00000000..49f34808 --- /dev/null +++ b/modules/netcheck/cmd/netcheck/request.go @@ -0,0 +1,164 @@ +package main + +// One HTTP request and its whole answer (hq issue 1643). netcheck_http answers whether a URL is up; +// this answers what it says: the status line, every response header and the body, so a service's live +// answers (a mail autoconfiguration file, an API's reply) can be captured from the machine that +// reaches it. The request is made once: redirects are reported, never followed, and only http and +// https are spoken. TLS is verified against the machine's own trust store (which holds the mesh's CA). + +import ( + "bytes" + "context" + "encoding/base64" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + "unicode/utf8" +) + +// Bounds: a request body of at most 256 KiB is sent, a response body of at most 1 MiB is read. +const ( + MostRequestBody = 256 * 1024 + MostResponseBody = 1024 * 1024 + DefaultRequestTTL = 10 * time.Second +) + +// RequestResult is what netcheck_request answers. +type RequestResult struct { + URL string `json:"url"` + Method string `json:"method"` + Status int `json:"status"` + StatusLine string `json:"status_line"` + Proto string `json:"proto"` + Headers map[string][]string `json:"headers"` + Redirect string `json:"redirect,omitempty"` + Body string `json:"body"` + Encoding string `json:"body_encoding"` + BodyBytes int `json:"body_bytes"` + Cut bool `json:"body_cut"` + ElapsedMS int64 `json:"elapsed_ms"` +} + +func runRequest(args map[string]any) (any, error) { + raw, err := text(args, "url") + if err != nil { + return nil, err + } + u, err := url.Parse(raw) + if err != nil { + return nil, fmt.Errorf("url %q is not one: %v", raw, err) + } + if u.Scheme != "http" && u.Scheme != "https" { + return nil, fmt.Errorf("url %q: only http and https are spoken, not %q", raw, u.Scheme) + } + if u.Host == "" { + return nil, fmt.Errorf("url %q has no host", raw) + } + method := "GET" + if m, _ := args["method"].(string); strings.TrimSpace(m) != "" { + method = strings.ToUpper(strings.TrimSpace(m)) + } + if method != "GET" && method != "HEAD" && method != "POST" { + return nil, fmt.Errorf("method %q is not GET, HEAD or POST", method) + } + body, _ := args["body"].(string) + if len(body) > MostRequestBody { + return nil, fmt.Errorf("body is %d bytes, more than the %d this tool sends", len(body), MostRequestBody) + } + timeout := DefaultRequestTTL + if _, given := args["timeout_ms"]; given { + ms, err := whole(args, "timeout_ms", int(DefaultRequestTTL/time.Millisecond)) + if err != nil { + return nil, err + } + if ms < 1 { + return nil, fmt.Errorf("timeout_ms must be at least 1, not %d", ms) + } + timeout = time.Duration(ms) * time.Millisecond + if timeout > MostTimeout { + timeout = MostTimeout + } + } + var headers map[string]any + if h, given := args["headers"]; given && h != nil { + var ok bool + if headers, ok = h.(map[string]any); !ok { + return nil, errors.New("headers must be an object of header name to value") + } + } + + cx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + var reader io.Reader + if body != "" { + reader = strings.NewReader(body) + } + req, err := http.NewRequestWithContext(cx, method, u.String(), reader) + if err != nil { + return nil, err + } + for name, v := range headers { + s, ok := v.(string) + if !ok { + return nil, fmt.Errorf("header %q must be a string", name) + } + if strings.EqualFold(name, "Host") { + req.Host = s + continue + } + req.Header.Set(name, s) + } + client := &http.Client{ + Timeout: timeout, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + } + start := time.Now() + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("%s %s: %v", method, raw, err) + } + defer resp.Body.Close() + data, err := io.ReadAll(io.LimitReader(resp.Body, MostResponseBody+1)) + if err != nil { + return nil, fmt.Errorf("reading the answer of %s: %v", raw, err) + } + cut := len(data) > MostResponseBody + if cut { + data = data[:MostResponseBody] + } + res := RequestResult{ + URL: raw, Method: method, Status: resp.StatusCode, StatusLine: resp.Status, Proto: resp.Proto, + Headers: map[string][]string(resp.Header), BodyBytes: len(data), Cut: cut, Encoding: "text", + ElapsedMS: time.Since(start).Milliseconds(), + } + if resp.StatusCode >= 300 && resp.StatusCode < 400 { + if loc, err := resp.Location(); err == nil { + res.Redirect = loc.String() + if l := resp.Header.Get("Location"); !strings.Contains(l, "://") { + res.Redirect = l + } + } else { + res.Redirect = resp.Header.Get("Location") + } + } + if utf8.Valid(data) || (cut && validBeforeCut(data)) { + res.Body = string(data) + } else { + res.Body, res.Encoding = base64.StdEncoding.EncodeToString(bytes.Clone(data)), "base64" + } + return res, nil +} + +// validBeforeCut: a body cut inside a multi-byte character is still text. +func validBeforeCut(b []byte) bool { + for i := 0; i < utf8.UTFMax && i < len(b); i++ { + if utf8.Valid(b[:len(b)-i]) { + return true + } + } + return false +} diff --git a/modules/netcheck/cmd/netcheck/request_test.go b/modules/netcheck/cmd/netcheck/request_test.go new file mode 100644 index 00000000..b4e91740 --- /dev/null +++ b/modules/netcheck/cmd/netcheck/request_test.go @@ -0,0 +1,153 @@ +package main + +import ( + "encoding/base64" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func asked(t *testing.T, args map[string]any) (RequestResult, error) { + t.Helper() + got, err := runRequest(args) + if err != nil { + return RequestResult{}, err + } + return got.(RequestResult), nil +} + +func TestRequestGet(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Seen", r.Method+" "+r.URL.RequestURI()) + _, _ = io.WriteString(w, "hello") + })) + defer srv.Close() + res, err := asked(t, map[string]any{"url": srv.URL + "/mail/config-v1.1.xml?a=1"}) + if err != nil { + t.Fatal(err) + } + if res.Status != 200 || res.StatusLine != "200 OK" || res.Body != "hello" || res.Cut || res.Encoding != "text" { + t.Fatalf("unexpected answer %+v", res) + } + if res.Headers["X-Seen"][0] != "GET /mail/config-v1.1.xml?a=1" { + t.Fatalf("headers %+v", res.Headers) + } +} + +func TestRequestHeadHasNoBody(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Method", r.Method) + })) + defer srv.Close() + res, err := asked(t, map[string]any{"url": srv.URL, "method": "HEAD"}) + if err != nil || res.Headers["X-Method"][0] != "HEAD" || res.Body != "" { + t.Fatalf("%+v %v", res, err) + } +} + +func TestRequestPostWithBodyAndHeaders(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + w.WriteHeader(http.StatusCreated) + _, _ = io.WriteString(w, r.Method+"|"+r.Header.Get("Content-Type")+"|"+string(b)) + })) + defer srv.Close() + res, err := asked(t, map[string]any{ + "url": srv.URL, "method": "POST", "body": "", + "headers": map[string]any{"Content-Type": "text/xml"}, + }) + if err != nil { + t.Fatal(err) + } + if res.Status != 201 || res.Body != "POST|text/xml|" { + t.Fatalf("%+v", res) + } +} + +func TestRequestBodyTooLarge(t *testing.T) { + _, err := asked(t, map[string]any{"url": "http://127.0.0.1:1", "method": "POST", "body": strings.Repeat("x", MostRequestBody+1)}) + if err == nil || !strings.Contains(err.Error(), "body") { + t.Fatalf("want a body error, got %v", err) + } +} + +func TestRequestCutBody(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, strings.Repeat("a", MostResponseBody+10)) + })) + defer srv.Close() + res, err := asked(t, map[string]any{"url": srv.URL}) + if err != nil { + t.Fatal(err) + } + if !res.Cut || len(res.Body) != MostResponseBody { + t.Fatalf("cut=%v len=%d", res.Cut, len(res.Body)) + } +} + +func TestRequestBinaryBodyIsBase64(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte{0xff, 0xfe, 0x00, 0x01}) + })) + defer srv.Close() + res, err := asked(t, map[string]any{"url": srv.URL}) + if err != nil { + t.Fatal(err) + } + raw, derr := base64.StdEncoding.DecodeString(res.Body) + if res.Encoding != "base64" || derr != nil || len(raw) != 4 { + t.Fatalf("%+v", res) + } +} + +func TestRequestRedirectReportedNotFollowed(t *testing.T) { + hit := false + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/there" { + hit = true + } + http.Redirect(w, r, "/there", http.StatusFound) + })) + defer srv.Close() + res, err := asked(t, map[string]any{"url": srv.URL}) + if err != nil { + t.Fatal(err) + } + if res.Status != 302 || res.Redirect != "/there" || hit { + t.Fatalf("%+v hit=%v", res, hit) + } +} + +func TestRequestRefusesOtherSchemes(t *testing.T) { + for _, u := range []string{"file:///etc/passwd", "ftp://example.org/x", "gopher://x", "//x/y", "example.org"} { + if _, err := asked(t, map[string]any{"url": u}); err == nil { + t.Errorf("%q was not refused", u) + } + } +} + +func TestRequestRefusesOtherMethods(t *testing.T) { + if _, err := asked(t, map[string]any{"url": "http://127.0.0.1:1", "method": "DELETE"}); err == nil { + t.Fatal("DELETE was not refused") + } +} + +func TestRequestTimeout(t *testing.T) { + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case <-release: + case <-r.Context().Done(): + } + })) + defer srv.Close() + defer close(release) + start := time.Now() + _, err := asked(t, map[string]any{"url": srv.URL, "timeout_ms": 100}) + if err == nil || time.Since(start) > 3*time.Second { + t.Fatalf("want a timeout error quickly, got %v after %v", err, time.Since(start)) + } +} diff --git a/modules/netcheck/module.json b/modules/netcheck/module.json index ec01003a..bb2f0f32 100644 --- a/modules/netcheck/module.json +++ b/modules/netcheck/module.json @@ -11,6 +11,7 @@ "netcheck_tcp", "netcheck_dns", "netcheck_http", + "netcheck_request", "netcheck_listening" ], "replaces": {