diff --git a/modules/netcheck/README.md b/modules/netcheck/README.md
new file mode 100644
index 00000000..76cd4994
--- /dev/null
+++ b/modules/netcheck/README.md
@@ -0,0 +1,3 @@
+# netcheck
+
+The machine's own network checks as tools: a TCP connect (`netcheck_tcp`), a DNS lookup of the machine's resolver or of one named (`netcheck_dns`), what listens here (`netcheck_listening`), the private network's peers (`netcheck_tunnel`), and whether a URL answers (`netcheck_http`). `netcheck_request` makes one HTTP request (GET, HEAD or POST, with headers and a body of at most 256 KiB) and answers the status line, every response header and the body (at most 1 MiB, with a flag saying when it was cut), reporting redirects without following them and speaking only http and https, refusing any connection to this machine's own loopback, unspecified or link-local addresses (checked on the address dialled, so names that resolve there are refused too) and using no proxy, so a service's live answers can be captured (hq issue 1643).
diff --git a/modules/netcheck/cmd/netcheck/checks_test.go b/modules/netcheck/cmd/netcheck/checks_test.go
index 4912b663..c11621b5 100644
--- a/modules/netcheck/cmd/netcheck/checks_test.go
+++ b/modules/netcheck/cmd/netcheck/checks_test.go
@@ -62,7 +62,7 @@ func TestTheToolsAreListedUnprefixed(t *testing.T) {
for _, tool := range tools() {
names[tool.Name] = true
}
- if !names["netcheck_tcp"] || !names["netcheck_dns"] || !names["netcheck_listening"] || !names["netcheck_tunnel"] || len(names) != 4 {
+ if !names["netcheck_tcp"] || !names["netcheck_dns"] || !names["netcheck_listening"] || !names["netcheck_tunnel"] || !names["netcheck_request"] || len(names) != 5 {
t.Fatalf("tools: %v", names)
}
}
diff --git a/modules/netcheck/cmd/netcheck/main.go b/modules/netcheck/cmd/netcheck/main.go
index 67bf9cf8..349f2aa8 100644
--- a/modules/netcheck/cmd/netcheck/main.go
+++ b/modules/netcheck/cmd/netcheck/main.go
@@ -88,6 +88,22 @@ func tools() []stdio.Tool {
return CheckTCP(host, port, timeout)
},
},
+ {
+ Name: "netcheck_request",
+ Description: "Make one HTTP request from this machine and answer what comes back: the status line, every response " +
+ "header and the body as text (base64 when it is not UTF-8), cut at 1 MiB with body_cut saying so. method is GET, " +
+ "HEAD or POST; headers and a body (at most 256 KiB) are sent as given. A redirect is reported (status and redirect), " +
+ "never followed. Only http and https; never this machine's own loopback, unspecified or link-local addresses (refused on the address dialled, so names and rebinding are caught too), and no proxy; TLS is verified against this machine's trust store. For what a service " +
+ "answers, not only whether it is up (that is netcheck_http).",
+ Input: map[string]any{
+ "url": map[string]any{"type": "string", "description": "http:// or https:// URL"},
+ "method": map[string]any{"type": "string", "enum": []string{"GET", "HEAD", "POST"}, "description": "default GET"},
+ "headers": map[string]any{"type": "object", "additionalProperties": map[string]any{"type": "string"}, "description": "request headers, name to value (optional)"},
+ "body": map[string]any{"type": "string", "description": "request body, at most 262144 bytes (optional)"},
+ "timeout_ms": map[string]any{"type": "integer", "description": "give up after this long (default 10000, at most 30000)"},
+ },
+ Run: runRequest,
+ },
{
Name: "netcheck_dns",
Description: "Look a name up with this machine's resolver (its /etc/resolv.conf and /etc/hosts), or — with server — ask " +
diff --git a/modules/netcheck/cmd/netcheck/request.go b/modules/netcheck/cmd/netcheck/request.go
new file mode 100644
index 00000000..af6327d1
--- /dev/null
+++ b/modules/netcheck/cmd/netcheck/request.go
@@ -0,0 +1,245 @@
+package main
+
+// One HTTP request and its whole answer (hq issue 1643). netcheck_http answers whether a URL is up;
+// this answers what it says: the status line, every response header and the body, so a service's live
+// answers (a mail autoconfiguration file, an API's reply) can be captured from the machine that
+// reaches it. The request is made once: redirects are reported, never followed, and only http and
+// https are spoken. TLS is verified against the machine's own trust store (which holds the mesh's CA).
+
+import (
+ "bytes"
+ "context"
+ "encoding/base64"
+ "errors"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "net/netip"
+ "net/url"
+ "strings"
+ "syscall"
+ "time"
+ "unicode/utf8"
+)
+
+// Bounds: a request body of at most 256 KiB is sent, a response body of at most 1 MiB is read.
+const (
+ MostRequestBody = 256 * 1024
+ MostResponseBody = 1024 * 1024
+ DefaultRequestTTL = 10 * time.Second
+)
+
+// RequestResult is what netcheck_request answers.
+type RequestResult struct {
+ URL string `json:"url"`
+ Method string `json:"method"`
+ Status int `json:"status"`
+ StatusLine string `json:"status_line"`
+ Proto string `json:"proto"`
+ Headers map[string][]string `json:"headers"`
+ Redirect string `json:"redirect,omitempty"`
+ Body string `json:"body"`
+ Encoding string `json:"body_encoding"`
+ BodyBytes int `json:"body_bytes"`
+ Cut bool `json:"body_cut"`
+ ElapsedMS int64 `json:"elapsed_ms"`
+}
+
+// allowLoopback is a seam for the tests only (their servers listen on loopback); nothing sets it otherwise.
+var allowLoopback = false
+
+// refusedAddress says why an address may not be dialled, or "". The node's own loopback carries
+// unauthenticated consoles (the tool console, the bus monitor), and link-local carries instance metadata;
+// a caller on another node must not reach them through this tool.
+func refusedAddress(ip netip.Addr) string {
+ ip = ip.Unmap()
+ switch {
+ case ip.IsLoopback() && !allowLoopback:
+ return "loopback"
+ case ip.IsUnspecified():
+ return "unspecified"
+ case ip.IsLinkLocalUnicast(), ip.IsLinkLocalMulticast():
+ return "link-local"
+ }
+ return ""
+}
+
+// guardDial runs on the address actually about to be connected, after name resolution, so a name that
+// resolves to a refused address (or is rebound to one) is refused as well as an address in the URL.
+func guardDial(_, address string, _ syscall.RawConn) error {
+ host, _, err := net.SplitHostPort(address)
+ if err != nil {
+ return fmt.Errorf("refused: %q is not an address", address)
+ }
+ ip, err := netip.ParseAddr(strings.SplitN(host, "%", 2)[0])
+ if err != nil {
+ return fmt.Errorf("refused: %q is not an address", host)
+ }
+ if why := refusedAddress(ip); why != "" {
+ return fmt.Errorf("refused: %s is a %s address; this tool does not reach this machine's own services", host, why)
+ }
+ return nil
+}
+
+// shortFormAddress: a host made only of numeric labels (decimal, octal or 0x hex) that net/netip does not
+// take as an address. The C resolver expands these (127.1 is 127.0.0.1, 2130706433 too), so the tool
+// refuses them on its own words before any lookup rather than rely on the resolver.
+func shortFormAddress(host string) bool {
+ if host == "" || strings.Contains(host, ":") {
+ return false
+ }
+ if _, err := netip.ParseAddr(host); err == nil {
+ return false
+ }
+ for _, label := range strings.Split(strings.TrimSuffix(host, "."), ".") {
+ digits := label
+ if len(label) > 2 && (label[:2] == "0x" || label[:2] == "0X") {
+ digits = label[2:]
+ for _, c := range digits {
+ if !strings.ContainsRune("0123456789abcdefABCDEF", c) {
+ return false
+ }
+ }
+ continue
+ }
+ if digits == "" {
+ return false
+ }
+ for _, c := range digits {
+ if c < '0' || c > '9' {
+ return false
+ }
+ }
+ }
+ return true
+}
+
+func runRequest(args map[string]any) (any, error) {
+ raw, err := text(args, "url")
+ if err != nil {
+ return nil, err
+ }
+ u, err := url.Parse(raw)
+ if err != nil {
+ return nil, fmt.Errorf("url %q is not one: %v", raw, err)
+ }
+ if u.Scheme != "http" && u.Scheme != "https" {
+ return nil, fmt.Errorf("url %q: only http and https are spoken, not %q", raw, u.Scheme)
+ }
+ if u.Host == "" {
+ return nil, fmt.Errorf("url %q has no host", raw)
+ }
+ if shortFormAddress(u.Hostname()) {
+ return nil, fmt.Errorf("refused: host %q is an address in a short or legacy form (as 127.1, 2130706433 or 0x7f.0.0.1), which the resolver may read as loopback, unspecified or link-local; write the full address", u.Hostname())
+ }
+ method := "GET"
+ if m, _ := args["method"].(string); strings.TrimSpace(m) != "" {
+ method = strings.ToUpper(strings.TrimSpace(m))
+ }
+ if method != "GET" && method != "HEAD" && method != "POST" {
+ return nil, fmt.Errorf("method %q is not GET, HEAD or POST", method)
+ }
+ body, _ := args["body"].(string)
+ if len(body) > MostRequestBody {
+ return nil, fmt.Errorf("body is %d bytes, more than the %d this tool sends", len(body), MostRequestBody)
+ }
+ timeout := DefaultRequestTTL
+ if _, given := args["timeout_ms"]; given {
+ ms, err := whole(args, "timeout_ms", int(DefaultRequestTTL/time.Millisecond))
+ if err != nil {
+ return nil, err
+ }
+ if ms < 1 {
+ return nil, fmt.Errorf("timeout_ms must be at least 1, not %d", ms)
+ }
+ timeout = time.Duration(ms) * time.Millisecond
+ if timeout > MostTimeout {
+ timeout = MostTimeout
+ }
+ }
+ var headers map[string]any
+ if h, given := args["headers"]; given && h != nil {
+ var ok bool
+ if headers, ok = h.(map[string]any); !ok {
+ return nil, errors.New("headers must be an object of header name to value")
+ }
+ }
+
+ cx, cancel := context.WithTimeout(context.Background(), timeout)
+ defer cancel()
+ var reader io.Reader
+ if body != "" {
+ reader = strings.NewReader(body)
+ }
+ req, err := http.NewRequestWithContext(cx, method, u.String(), reader)
+ if err != nil {
+ return nil, err
+ }
+ for name, v := range headers {
+ s, ok := v.(string)
+ if !ok {
+ return nil, fmt.Errorf("header %q must be a string", name)
+ }
+ if strings.EqualFold(name, "Host") {
+ req.Host = s
+ continue
+ }
+ req.Header.Set(name, s)
+ }
+ transport := &http.Transport{
+ Proxy: nil,
+ DialContext: (&net.Dialer{Timeout: timeout, Control: guardDial}).DialContext,
+ }
+ defer transport.CloseIdleConnections()
+ client := &http.Client{
+ Transport: transport,
+ Timeout: timeout,
+ CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
+ }
+ start := time.Now()
+ resp, err := client.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("%s %s: %v", method, raw, err)
+ }
+ defer resp.Body.Close()
+ data, err := io.ReadAll(io.LimitReader(resp.Body, MostResponseBody+1))
+ if err != nil {
+ return nil, fmt.Errorf("reading the answer of %s: %v", raw, err)
+ }
+ cut := len(data) > MostResponseBody
+ if cut {
+ data = data[:MostResponseBody]
+ }
+ res := RequestResult{
+ URL: raw, Method: method, Status: resp.StatusCode, StatusLine: resp.Status, Proto: resp.Proto,
+ Headers: map[string][]string(resp.Header), BodyBytes: len(data), Cut: cut, Encoding: "text",
+ ElapsedMS: time.Since(start).Milliseconds(),
+ }
+ if resp.StatusCode >= 300 && resp.StatusCode < 400 {
+ if loc, err := resp.Location(); err == nil {
+ res.Redirect = loc.String()
+ if l := resp.Header.Get("Location"); !strings.Contains(l, "://") {
+ res.Redirect = l
+ }
+ } else {
+ res.Redirect = resp.Header.Get("Location")
+ }
+ }
+ if utf8.Valid(data) || (cut && validBeforeCut(data)) {
+ res.Body = string(data)
+ } else {
+ res.Body, res.Encoding = base64.StdEncoding.EncodeToString(bytes.Clone(data)), "base64"
+ }
+ return res, nil
+}
+
+// validBeforeCut: a body cut inside a multi-byte character is still text.
+func validBeforeCut(b []byte) bool {
+ for i := 0; i < utf8.UTFMax && i < len(b); i++ {
+ if utf8.Valid(b[:len(b)-i]) {
+ return true
+ }
+ }
+ return false
+}
diff --git a/modules/netcheck/cmd/netcheck/request_test.go b/modules/netcheck/cmd/netcheck/request_test.go
new file mode 100644
index 00000000..2bcb41e7
--- /dev/null
+++ b/modules/netcheck/cmd/netcheck/request_test.go
@@ -0,0 +1,212 @@
+package main
+
+import (
+ "encoding/base64"
+ "io"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "strconv"
+ "strings"
+ "testing"
+ "time"
+)
+
+// The httptest servers listen on loopback, which the tool refuses; the tests allow it, and the refusal
+// tests put the refusal back.
+func TestMain(m *testing.M) {
+ allowLoopback = true
+ os.Exit(m.Run())
+}
+
+func refusing(t *testing.T) {
+ t.Helper()
+ allowLoopback = false
+ t.Cleanup(func() { allowLoopback = true })
+}
+
+func TestRequestRefusesLocalAddresses(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ t.Error("the server was reached")
+ }))
+ defer srv.Close()
+ port := srv.Listener.Addr().(*net.TCPAddr).Port
+ refusing(t)
+ for _, host := range []string{
+ "127.0.0.1", "localhost", "127.1", "2130706433", "0x7f.0.0.1", "0x7f.1", "0177.0.0.1", "[::1]", "[::ffff:127.0.0.1]", "0.0.0.0", "[::]",
+ "169.254.169.254", "[fe80::1]",
+ } {
+ u := "http://" + host + ":" + strconv.Itoa(port) + "/"
+ _, err := asked(t, map[string]any{"url": u, "timeout_ms": float64(2000)})
+ if err == nil || !strings.Contains(err.Error(), "refus") {
+ t.Errorf("%s: want a refusal, got %v", u, err)
+ }
+ }
+}
+
+func TestShortFormAddressIsRefusedOnItsOwnWords(t *testing.T) {
+ for _, host := range []string{"127.1", "2130706433", "0x7f.0.0.1", "0x7f.1", "0177.0.0.1"} {
+ _, err := asked(t, map[string]any{"url": "http://" + host + ":1/", "timeout_ms": float64(2000)})
+ if err == nil || !strings.Contains(err.Error(), "short or legacy form") {
+ t.Errorf("%s: want the short-form refusal, got %v", host, err)
+ }
+ }
+ for _, host := range []string{"example.com", "127.0.0.1", "1.example.com", "[::1]", "10.0.0.1"} {
+ if shortFormAddress(strings.Trim(host, "[]")) {
+ t.Errorf("%s taken for a short form", host)
+ }
+ }
+}
+
+func TestRequestIgnoresProxyEnvironment(t *testing.T) {
+ t.Setenv("HTTP_PROXY", "http://127.0.0.1:1")
+ t.Setenv("http_proxy", "http://127.0.0.1:1")
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+ defer srv.Close()
+ if _, err := asked(t, map[string]any{"url": "http://example.invalid:80/"}); err == nil || strings.Contains(err.Error(), "127.0.0.1:1") {
+ t.Fatalf("a proxy seems to have been used: %v", err)
+ }
+}
+
+func asked(t *testing.T, args map[string]any) (RequestResult, error) {
+ t.Helper()
+ got, err := runRequest(args)
+ if err != nil {
+ return RequestResult{}, err
+ }
+ return got.(RequestResult), nil
+}
+
+func TestRequestGet(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("X-Seen", r.Method+" "+r.URL.RequestURI())
+ _, _ = io.WriteString(w, "hello")
+ }))
+ defer srv.Close()
+ res, err := asked(t, map[string]any{"url": srv.URL + "/mail/config-v1.1.xml?a=1"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if res.Status != 200 || res.StatusLine != "200 OK" || res.Body != "hello" || res.Cut || res.Encoding != "text" {
+ t.Fatalf("unexpected answer %+v", res)
+ }
+ if res.Headers["X-Seen"][0] != "GET /mail/config-v1.1.xml?a=1" {
+ t.Fatalf("headers %+v", res.Headers)
+ }
+}
+
+func TestRequestHeadHasNoBody(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("X-Method", r.Method)
+ }))
+ defer srv.Close()
+ res, err := asked(t, map[string]any{"url": srv.URL, "method": "HEAD"})
+ if err != nil || res.Headers["X-Method"][0] != "HEAD" || res.Body != "" {
+ t.Fatalf("%+v %v", res, err)
+ }
+}
+
+func TestRequestPostWithBodyAndHeaders(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ b, _ := io.ReadAll(r.Body)
+ w.WriteHeader(http.StatusCreated)
+ _, _ = io.WriteString(w, r.Method+"|"+r.Header.Get("Content-Type")+"|"+string(b))
+ }))
+ defer srv.Close()
+ res, err := asked(t, map[string]any{
+ "url": srv.URL, "method": "POST", "body": "",
+ "headers": map[string]any{"Content-Type": "text/xml"},
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if res.Status != 201 || res.Body != "POST|text/xml|" {
+ t.Fatalf("%+v", res)
+ }
+}
+
+func TestRequestBodyTooLarge(t *testing.T) {
+ _, err := asked(t, map[string]any{"url": "http://127.0.0.1:1", "method": "POST", "body": strings.Repeat("x", MostRequestBody+1)})
+ if err == nil || !strings.Contains(err.Error(), "body") {
+ t.Fatalf("want a body error, got %v", err)
+ }
+}
+
+func TestRequestCutBody(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _, _ = io.WriteString(w, strings.Repeat("a", MostResponseBody+10))
+ }))
+ defer srv.Close()
+ res, err := asked(t, map[string]any{"url": srv.URL})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !res.Cut || len(res.Body) != MostResponseBody {
+ t.Fatalf("cut=%v len=%d", res.Cut, len(res.Body))
+ }
+}
+
+func TestRequestBinaryBodyIsBase64(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _, _ = w.Write([]byte{0xff, 0xfe, 0x00, 0x01})
+ }))
+ defer srv.Close()
+ res, err := asked(t, map[string]any{"url": srv.URL})
+ if err != nil {
+ t.Fatal(err)
+ }
+ raw, derr := base64.StdEncoding.DecodeString(res.Body)
+ if res.Encoding != "base64" || derr != nil || len(raw) != 4 {
+ t.Fatalf("%+v", res)
+ }
+}
+
+func TestRequestRedirectReportedNotFollowed(t *testing.T) {
+ hit := false
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path == "/there" {
+ hit = true
+ }
+ http.Redirect(w, r, "/there", http.StatusFound)
+ }))
+ defer srv.Close()
+ res, err := asked(t, map[string]any{"url": srv.URL})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if res.Status != 302 || res.Redirect != "/there" || hit {
+ t.Fatalf("%+v hit=%v", res, hit)
+ }
+}
+
+func TestRequestRefusesOtherSchemes(t *testing.T) {
+ for _, u := range []string{"file:///etc/passwd", "ftp://example.org/x", "gopher://x", "//x/y", "example.org"} {
+ if _, err := asked(t, map[string]any{"url": u}); err == nil {
+ t.Errorf("%q was not refused", u)
+ }
+ }
+}
+
+func TestRequestRefusesOtherMethods(t *testing.T) {
+ if _, err := asked(t, map[string]any{"url": "http://127.0.0.1:1", "method": "DELETE"}); err == nil {
+ t.Fatal("DELETE was not refused")
+ }
+}
+
+func TestRequestTimeout(t *testing.T) {
+ release := make(chan struct{})
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ select {
+ case <-release:
+ case <-r.Context().Done():
+ }
+ }))
+ defer srv.Close()
+ defer close(release)
+ start := time.Now()
+ _, err := asked(t, map[string]any{"url": srv.URL, "timeout_ms": float64(100)})
+ if err == nil || time.Since(start) > 3*time.Second {
+ t.Fatalf("want a timeout error quickly, got %v after %v", err, time.Since(start))
+ }
+}
diff --git a/modules/netcheck/module.json b/modules/netcheck/module.json
index ec01003a..bb2f0f32 100644
--- a/modules/netcheck/module.json
+++ b/modules/netcheck/module.json
@@ -11,6 +11,7 @@
"netcheck_tcp",
"netcheck_dns",
"netcheck_http",
+ "netcheck_request",
"netcheck_listening"
],
"replaces": {