Merge pull request 'Add netcheck_request: one HTTP request and its whole answer, so a service's live answers can be captured, refusing loopback, unspecified and link-local addresses (issue 1643)' (#412) from fix/1643-http-request into main

modules/netcheck/README.md

@@ -0,0 +1,3 @@
+# netcheck
+
+The machine's own network checks as tools: a TCP connect (`netcheck_tcp`), a DNS lookup of the machine's resolver or of one named (`netcheck_dns`), what listens here (`netcheck_listening`), the private network's peers (`netcheck_tunnel`), and whether a URL answers (`netcheck_http`). `netcheck_request` makes one HTTP request (GET, HEAD or POST, with headers and a body of at most 256 KiB) and answers the status line, every response header and the body (at most 1 MiB, with a flag saying when it was cut), reporting redirects without following them and speaking only http and https, refusing any connection to this machine's own loopback, unspecified or link-local addresses (checked on the address dialled, so names that resolve there are refused too) and using no proxy, so a service's live answers can be captured (hq issue 1643).

modules/netcheck/cmd/netcheck/checks_test.go

@@ -62,7 +62,7 @@ func TestTheToolsAreListedUnprefixed(t *testing.T) {
 	for _, tool := range tools() {
 		names[tool.Name] = true
 	}
-	if !names["netcheck_tcp"] || !names["netcheck_dns"] || !names["netcheck_listening"] || !names["netcheck_tunnel"] || len(names) != 4 {
+	if !names["netcheck_tcp"] || !names["netcheck_dns"] || !names["netcheck_listening"] || !names["netcheck_tunnel"] || !names["netcheck_request"] || len(names) != 5 {
 		t.Fatalf("tools: %v", names)
 	}
 }

modules/netcheck/cmd/netcheck/main.go

@@ -88,6 +88,22 @@ func tools() []stdio.Tool {
 				return CheckTCP(host, port, timeout)
 			},
 		},
+		{
+			Name: "netcheck_request",
+			Description: "Make one HTTP request from this machine and answer what comes back: the status line, every response " +
+				"header and the body as text (base64 when it is not UTF-8), cut at 1 MiB with body_cut saying so. method is GET, " +
+				"HEAD or POST; headers and a body (at most 256 KiB) are sent as given. A redirect is reported (status and redirect), " +
+				"never followed. Only http and https; never this machine's own loopback, unspecified or link-local addresses (refused on the address dialled, so names and rebinding are caught too), and no proxy; TLS is verified against this machine's trust store. For what a service " +
+				"answers, not only whether it is up (that is netcheck_http).",
+			Input: map[string]any{
+				"url":        map[string]any{"type": "string", "description": "http:// or https:// URL"},
+				"method":     map[string]any{"type": "string", "enum": []string{"GET", "HEAD", "POST"}, "description": "default GET"},
+				"headers":    map[string]any{"type": "object", "additionalProperties": map[string]any{"type": "string"}, "description": "request headers, name to value (optional)"},
+				"body":       map[string]any{"type": "string", "description": "request body, at most 262144 bytes (optional)"},
+				"timeout_ms": map[string]any{"type": "integer", "description": "give up after this long (default 10000, at most 30000)"},
+			},
+			Run: runRequest,
+		},
 		{
 			Name: "netcheck_dns",
 			Description: "Look a name up with this machine's resolver (its /etc/resolv.conf and /etc/hosts), or — with server — ask " +

modules/netcheck/cmd/netcheck/request.go

@@ -0,0 +1,245 @@
+package main
+
+// One HTTP request and its whole answer (hq issue 1643). netcheck_http answers whether a URL is up;
+// this answers what it says: the status line, every response header and the body, so a service's live
+// answers (a mail autoconfiguration file, an API's reply) can be captured from the machine that
+// reaches it. The request is made once: redirects are reported, never followed, and only http and
+// https are spoken. TLS is verified against the machine's own trust store (which holds the mesh's CA).
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/netip"
+	"net/url"
+	"strings"
+	"syscall"
+	"time"
+	"unicode/utf8"
+)
+
+// Bounds: a request body of at most 256 KiB is sent, a response body of at most 1 MiB is read.
+const (
+	MostRequestBody   = 256 * 1024
+	MostResponseBody  = 1024 * 1024
+	DefaultRequestTTL = 10 * time.Second
+)
+
+// RequestResult is what netcheck_request answers.
+type RequestResult struct {
+	URL        string              `json:"url"`
+	Method     string              `json:"method"`
+	Status     int                 `json:"status"`
+	StatusLine string              `json:"status_line"`
+	Proto      string              `json:"proto"`
+	Headers    map[string][]string `json:"headers"`
+	Redirect   string              `json:"redirect,omitempty"`
+	Body       string              `json:"body"`
+	Encoding   string              `json:"body_encoding"`
+	BodyBytes  int                 `json:"body_bytes"`
+	Cut        bool                `json:"body_cut"`
+	ElapsedMS  int64               `json:"elapsed_ms"`
+}
+
+// allowLoopback is a seam for the tests only (their servers listen on loopback); nothing sets it otherwise.
+var allowLoopback = false
+
+// refusedAddress says why an address may not be dialled, or "". The node's own loopback carries
+// unauthenticated consoles (the tool console, the bus monitor), and link-local carries instance metadata;
+// a caller on another node must not reach them through this tool.
+func refusedAddress(ip netip.Addr) string {
+	ip = ip.Unmap()
+	switch {
+	case ip.IsLoopback() && !allowLoopback:
+		return "loopback"
+	case ip.IsUnspecified():
+		return "unspecified"
+	case ip.IsLinkLocalUnicast(), ip.IsLinkLocalMulticast():
+		return "link-local"
+	}
+	return ""
+}
+
+// guardDial runs on the address actually about to be connected, after name resolution, so a name that
+// resolves to a refused address (or is rebound to one) is refused as well as an address in the URL.
+func guardDial(_, address string, _ syscall.RawConn) error {
+	host, _, err := net.SplitHostPort(address)
+	if err != nil {
+		return fmt.Errorf("refused: %q is not an address", address)
+	}
+	ip, err := netip.ParseAddr(strings.SplitN(host, "%", 2)[0])
+	if err != nil {
+		return fmt.Errorf("refused: %q is not an address", host)
+	}
+	if why := refusedAddress(ip); why != "" {
+		return fmt.Errorf("refused: %s is a %s address; this tool does not reach this machine's own services", host, why)
+	}
+	return nil
+}
+
+// shortFormAddress: a host made only of numeric labels (decimal, octal or 0x hex) that net/netip does not
+// take as an address. The C resolver expands these (127.1 is 127.0.0.1, 2130706433 too), so the tool
+// refuses them on its own words before any lookup rather than rely on the resolver.
+func shortFormAddress(host string) bool {
+	if host == "" || strings.Contains(host, ":") {
+		return false
+	}
+	if _, err := netip.ParseAddr(host); err == nil {
+		return false
+	}
+	for _, label := range strings.Split(strings.TrimSuffix(host, "."), ".") {
+		digits := label
+		if len(label) > 2 && (label[:2] == "0x" || label[:2] == "0X") {
+			digits = label[2:]
+			for _, c := range digits {
+				if !strings.ContainsRune("0123456789abcdefABCDEF", c) {
+					return false
+				}
+			}
+			continue
+		}
+		if digits == "" {
+			return false
+		}
+		for _, c := range digits {
+			if c < '0' || c > '9' {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+func runRequest(args map[string]any) (any, error) {
+	raw, err := text(args, "url")
+	if err != nil {
+		return nil, err
+	}
+	u, err := url.Parse(raw)
+	if err != nil {
+		return nil, fmt.Errorf("url %q is not one: %v", raw, err)
+	}
+	if u.Scheme != "http" && u.Scheme != "https" {
+		return nil, fmt.Errorf("url %q: only http and https are spoken, not %q", raw, u.Scheme)
+	}
+	if u.Host == "" {
+		return nil, fmt.Errorf("url %q has no host", raw)
+	}
+	if shortFormAddress(u.Hostname()) {
+		return nil, fmt.Errorf("refused: host %q is an address in a short or legacy form (as 127.1, 2130706433 or 0x7f.0.0.1), which the resolver may read as loopback, unspecified or link-local; write the full address", u.Hostname())
+	}
+	method := "GET"
+	if m, _ := args["method"].(string); strings.TrimSpace(m) != "" {
+		method = strings.ToUpper(strings.TrimSpace(m))
+	}
+	if method != "GET" && method != "HEAD" && method != "POST" {
+		return nil, fmt.Errorf("method %q is not GET, HEAD or POST", method)
+	}
+	body, _ := args["body"].(string)
+	if len(body) > MostRequestBody {
+		return nil, fmt.Errorf("body is %d bytes, more than the %d this tool sends", len(body), MostRequestBody)
+	}
+	timeout := DefaultRequestTTL
+	if _, given := args["timeout_ms"]; given {
+		ms, err := whole(args, "timeout_ms", int(DefaultRequestTTL/time.Millisecond))
+		if err != nil {
+			return nil, err
+		}
+		if ms < 1 {
+			return nil, fmt.Errorf("timeout_ms must be at least 1, not %d", ms)
+		}
+		timeout = time.Duration(ms) * time.Millisecond
+		if timeout > MostTimeout {
+			timeout = MostTimeout
+		}
+	}
+	var headers map[string]any
+	if h, given := args["headers"]; given && h != nil {
+		var ok bool
+		if headers, ok = h.(map[string]any); !ok {
+			return nil, errors.New("headers must be an object of header name to value")
+		}
+	}
+
+	cx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+	var reader io.Reader
+	if body != "" {
+		reader = strings.NewReader(body)
+	}
+	req, err := http.NewRequestWithContext(cx, method, u.String(), reader)
+	if err != nil {
+		return nil, err
+	}
+	for name, v := range headers {
+		s, ok := v.(string)
+		if !ok {
+			return nil, fmt.Errorf("header %q must be a string", name)
+		}
+		if strings.EqualFold(name, "Host") {
+			req.Host = s
+			continue
+		}
+		req.Header.Set(name, s)
+	}
+	transport := &http.Transport{
+		Proxy:       nil,
+		DialContext: (&net.Dialer{Timeout: timeout, Control: guardDial}).DialContext,
+	}
+	defer transport.CloseIdleConnections()
+	client := &http.Client{
+		Transport:     transport,
+		Timeout:       timeout,
+		CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
+	}
+	start := time.Now()
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("%s %s: %v", method, raw, err)
+	}
+	defer resp.Body.Close()
+	data, err := io.ReadAll(io.LimitReader(resp.Body, MostResponseBody+1))
+	if err != nil {
+		return nil, fmt.Errorf("reading the answer of %s: %v", raw, err)
+	}
+	cut := len(data) > MostResponseBody
+	if cut {
+		data = data[:MostResponseBody]
+	}
+	res := RequestResult{
+		URL: raw, Method: method, Status: resp.StatusCode, StatusLine: resp.Status, Proto: resp.Proto,
+		Headers: map[string][]string(resp.Header), BodyBytes: len(data), Cut: cut, Encoding: "text",
+		ElapsedMS: time.Since(start).Milliseconds(),
+	}
+	if resp.StatusCode >= 300 && resp.StatusCode < 400 {
+		if loc, err := resp.Location(); err == nil {
+			res.Redirect = loc.String()
+			if l := resp.Header.Get("Location"); !strings.Contains(l, "://") {
+				res.Redirect = l
+			}
+		} else {
+			res.Redirect = resp.Header.Get("Location")
+		}
+	}
+	if utf8.Valid(data) || (cut && validBeforeCut(data)) {
+		res.Body = string(data)
+	} else {
+		res.Body, res.Encoding = base64.StdEncoding.EncodeToString(bytes.Clone(data)), "base64"
+	}
+	return res, nil
+}
+
+// validBeforeCut: a body cut inside a multi-byte character is still text.
+func validBeforeCut(b []byte) bool {
+	for i := 0; i < utf8.UTFMax && i < len(b); i++ {
+		if utf8.Valid(b[:len(b)-i]) {
+			return true
+		}
+	}
+	return false
+}

modules/netcheck/cmd/netcheck/request_test.go

@@ -0,0 +1,212 @@
+package main
+
+import (
+	"encoding/base64"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+)
+
+// The httptest servers listen on loopback, which the tool refuses; the tests allow it, and the refusal
+// tests put the refusal back.
+func TestMain(m *testing.M) {
+	allowLoopback = true
+	os.Exit(m.Run())
+}
+
+func refusing(t *testing.T) {
+	t.Helper()
+	allowLoopback = false
+	t.Cleanup(func() { allowLoopback = true })
+}
+
+func TestRequestRefusesLocalAddresses(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		t.Error("the server was reached")
+	}))
+	defer srv.Close()
+	port := srv.Listener.Addr().(*net.TCPAddr).Port
+	refusing(t)
+	for _, host := range []string{
+		"127.0.0.1", "localhost", "127.1", "2130706433", "0x7f.0.0.1", "0x7f.1", "0177.0.0.1", "[::1]", "[::ffff:127.0.0.1]", "0.0.0.0", "[::]",
+		"169.254.169.254", "[fe80::1]",
+	} {
+		u := "http://" + host + ":" + strconv.Itoa(port) + "/"
+		_, err := asked(t, map[string]any{"url": u, "timeout_ms": float64(2000)})
+		if err == nil || !strings.Contains(err.Error(), "refus") {
+			t.Errorf("%s: want a refusal, got %v", u, err)
+		}
+	}
+}
+
+func TestShortFormAddressIsRefusedOnItsOwnWords(t *testing.T) {
+	for _, host := range []string{"127.1", "2130706433", "0x7f.0.0.1", "0x7f.1", "0177.0.0.1"} {
+		_, err := asked(t, map[string]any{"url": "http://" + host + ":1/", "timeout_ms": float64(2000)})
+		if err == nil || !strings.Contains(err.Error(), "short or legacy form") {
+			t.Errorf("%s: want the short-form refusal, got %v", host, err)
+		}
+	}
+	for _, host := range []string{"example.com", "127.0.0.1", "1.example.com", "[::1]", "10.0.0.1"} {
+		if shortFormAddress(strings.Trim(host, "[]")) {
+			t.Errorf("%s taken for a short form", host)
+		}
+	}
+}
+
+func TestRequestIgnoresProxyEnvironment(t *testing.T) {
+	t.Setenv("HTTP_PROXY", "http://127.0.0.1:1")
+	t.Setenv("http_proxy", "http://127.0.0.1:1")
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+	defer srv.Close()
+	if _, err := asked(t, map[string]any{"url": "http://example.invalid:80/"}); err == nil || strings.Contains(err.Error(), "127.0.0.1:1") {
+		t.Fatalf("a proxy seems to have been used: %v", err)
+	}
+}
+
+func asked(t *testing.T, args map[string]any) (RequestResult, error) {
+	t.Helper()
+	got, err := runRequest(args)
+	if err != nil {
+		return RequestResult{}, err
+	}
+	return got.(RequestResult), nil
+}
+
+func TestRequestGet(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("X-Seen", r.Method+" "+r.URL.RequestURI())
+		_, _ = io.WriteString(w, "hello")
+	}))
+	defer srv.Close()
+	res, err := asked(t, map[string]any{"url": srv.URL + "/mail/config-v1.1.xml?a=1"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if res.Status != 200 || res.StatusLine != "200 OK" || res.Body != "hello" || res.Cut || res.Encoding != "text" {
+		t.Fatalf("unexpected answer %+v", res)
+	}
+	if res.Headers["X-Seen"][0] != "GET /mail/config-v1.1.xml?a=1" {
+		t.Fatalf("headers %+v", res.Headers)
+	}
+}
+
+func TestRequestHeadHasNoBody(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("X-Method", r.Method)
+	}))
+	defer srv.Close()
+	res, err := asked(t, map[string]any{"url": srv.URL, "method": "HEAD"})
+	if err != nil || res.Headers["X-Method"][0] != "HEAD" || res.Body != "" {
+		t.Fatalf("%+v %v", res, err)
+	}
+}
+
+func TestRequestPostWithBodyAndHeaders(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		b, _ := io.ReadAll(r.Body)
+		w.WriteHeader(http.StatusCreated)
+		_, _ = io.WriteString(w, r.Method+"|"+r.Header.Get("Content-Type")+"|"+string(b))
+	}))
+	defer srv.Close()
+	res, err := asked(t, map[string]any{
+		"url": srv.URL, "method": "POST", "body": "<a/>",
+		"headers": map[string]any{"Content-Type": "text/xml"},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if res.Status != 201 || res.Body != "POST|text/xml|<a/>" {
+		t.Fatalf("%+v", res)
+	}
+}
+
+func TestRequestBodyTooLarge(t *testing.T) {
+	_, err := asked(t, map[string]any{"url": "http://127.0.0.1:1", "method": "POST", "body": strings.Repeat("x", MostRequestBody+1)})
+	if err == nil || !strings.Contains(err.Error(), "body") {
+		t.Fatalf("want a body error, got %v", err)
+	}
+}
+
+func TestRequestCutBody(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = io.WriteString(w, strings.Repeat("a", MostResponseBody+10))
+	}))
+	defer srv.Close()
+	res, err := asked(t, map[string]any{"url": srv.URL})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !res.Cut || len(res.Body) != MostResponseBody {
+		t.Fatalf("cut=%v len=%d", res.Cut, len(res.Body))
+	}
+}
+
+func TestRequestBinaryBodyIsBase64(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte{0xff, 0xfe, 0x00, 0x01})
+	}))
+	defer srv.Close()
+	res, err := asked(t, map[string]any{"url": srv.URL})
+	if err != nil {
+		t.Fatal(err)
+	}
+	raw, derr := base64.StdEncoding.DecodeString(res.Body)
+	if res.Encoding != "base64" || derr != nil || len(raw) != 4 {
+		t.Fatalf("%+v", res)
+	}
+}
+
+func TestRequestRedirectReportedNotFollowed(t *testing.T) {
+	hit := false
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/there" {
+			hit = true
+		}
+		http.Redirect(w, r, "/there", http.StatusFound)
+	}))
+	defer srv.Close()
+	res, err := asked(t, map[string]any{"url": srv.URL})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if res.Status != 302 || res.Redirect != "/there" || hit {
+		t.Fatalf("%+v hit=%v", res, hit)
+	}
+}
+
+func TestRequestRefusesOtherSchemes(t *testing.T) {
+	for _, u := range []string{"file:///etc/passwd", "ftp://example.org/x", "gopher://x", "//x/y", "example.org"} {
+		if _, err := asked(t, map[string]any{"url": u}); err == nil {
+			t.Errorf("%q was not refused", u)
+		}
+	}
+}
+
+func TestRequestRefusesOtherMethods(t *testing.T) {
+	if _, err := asked(t, map[string]any{"url": "http://127.0.0.1:1", "method": "DELETE"}); err == nil {
+		t.Fatal("DELETE was not refused")
+	}
+}
+
+func TestRequestTimeout(t *testing.T) {
+	release := make(chan struct{})
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		select {
+		case <-release:
+		case <-r.Context().Done():
+		}
+	}))
+	defer srv.Close()
+	defer close(release)
+	start := time.Now()
+	_, err := asked(t, map[string]any{"url": srv.URL, "timeout_ms": float64(100)})
+	if err == nil || time.Since(start) > 3*time.Second {
+		t.Fatalf("want a timeout error quickly, got %v after %v", err, time.Since(start))
+	}
+}

modules/netcheck/module.json

@@ -11,6 +11,7 @@
     "netcheck_tcp",
     "netcheck_dns",
     "netcheck_http",
+    "netcheck_request",
     "netcheck_listening"
   ],
   "replaces": {