mesh-catalog/modules/netcheck/cmd/netcheck/request.go

package main

// One HTTP request and its whole answer (hq issue 1643). netcheck_http answers whether a URL is up;
// this answers what it says: the status line, every response header and the body, so a service's live
// answers (a mail autoconfiguration file, an API's reply) can be captured from the machine that
// reaches it. The request is made once: redirects are reported, never followed, and only http and
// https are spoken. TLS is verified against the machine's own trust store (which holds the mesh's CA).

import (
	"bytes"
	"context"
	"encoding/base64"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/netip"
	"net/url"
	"strings"
	"syscall"
	"time"
	"unicode/utf8"
)

// Bounds: a request body of at most 256 KiB is sent, a response body of at most 1 MiB is read.
const (
	MostRequestBody   = 256 * 1024
	MostResponseBody  = 1024 * 1024
	DefaultRequestTTL = 10 * time.Second
)

// RequestResult is what netcheck_request answers.
type RequestResult struct {
	URL        string              `json:"url"`
	Method     string              `json:"method"`
	Status     int                 `json:"status"`
	StatusLine string              `json:"status_line"`
	Proto      string              `json:"proto"`
	Headers    map[string][]string `json:"headers"`
	Redirect   string              `json:"redirect,omitempty"`
	Body       string              `json:"body"`
	Encoding   string              `json:"body_encoding"`
	BodyBytes  int                 `json:"body_bytes"`
	Cut        bool                `json:"body_cut"`
	ElapsedMS  int64               `json:"elapsed_ms"`
}

// allowLoopback is a seam for the tests only (their servers listen on loopback); nothing sets it otherwise.
var allowLoopback = false

// refusedAddress says why an address may not be dialled, or "". The node's own loopback carries
// unauthenticated consoles (the tool console, the bus monitor), and link-local carries instance metadata;
// a caller on another node must not reach them through this tool.
func refusedAddress(ip netip.Addr) string {
	ip = ip.Unmap()
	switch {
	case ip.IsLoopback() && !allowLoopback:
		return "loopback"
	case ip.IsUnspecified():
		return "unspecified"
	case ip.IsLinkLocalUnicast(), ip.IsLinkLocalMulticast():
		return "link-local"
	}
	return ""
}

// guardDial runs on the address actually about to be connected, after name resolution, so a name that
// resolves to a refused address (or is rebound to one) is refused as well as an address in the URL.
func guardDial(_, address string, _ syscall.RawConn) error {
	host, _, err := net.SplitHostPort(address)
	if err != nil {
		return fmt.Errorf("refused: %q is not an address", address)
	}
	ip, err := netip.ParseAddr(strings.SplitN(host, "%", 2)[0])
	if err != nil {
		return fmt.Errorf("refused: %q is not an address", host)
	}
	if why := refusedAddress(ip); why != "" {
		return fmt.Errorf("refused: %s is a %s address; this tool does not reach this machine's own services", host, why)
	}
	return nil
}

// shortFormAddress: a host made only of numeric labels (decimal, octal or 0x hex) that net/netip does not
// take as an address. The C resolver expands these (127.1 is 127.0.0.1, 2130706433 too), so the tool
// refuses them on its own words before any lookup rather than rely on the resolver.
func shortFormAddress(host string) bool {
	if host == "" || strings.Contains(host, ":") {
		return false
	}
	if _, err := netip.ParseAddr(host); err == nil {
		return false
	}
	for _, label := range strings.Split(strings.TrimSuffix(host, "."), ".") {
		digits := label
		if len(label) > 2 && (label[:2] == "0x" || label[:2] == "0X") {
			digits = label[2:]
			for _, c := range digits {
				if !strings.ContainsRune("0123456789abcdefABCDEF", c) {
					return false
				}
			}
			continue
		}
		if digits == "" {
			return false
		}
		for _, c := range digits {
			if c < '0' || c > '9' {
				return false
			}
		}
	}
	return true
}

func runRequest(args map[string]any) (any, error) {
	raw, err := text(args, "url")
	if err != nil {
		return nil, err
	}
	u, err := url.Parse(raw)
	if err != nil {
		return nil, fmt.Errorf("url %q is not one: %v", raw, err)
	}
	if u.Scheme != "http" && u.Scheme != "https" {
		return nil, fmt.Errorf("url %q: only http and https are spoken, not %q", raw, u.Scheme)
	}
	if u.Host == "" {
		return nil, fmt.Errorf("url %q has no host", raw)
	}
	if shortFormAddress(u.Hostname()) {
		return nil, fmt.Errorf("refused: host %q is an address in a short or legacy form (as 127.1, 2130706433 or 0x7f.0.0.1), which the resolver may read as loopback, unspecified or link-local; write the full address", u.Hostname())
	}
	method := "GET"
	if m, _ := args["method"].(string); strings.TrimSpace(m) != "" {
		method = strings.ToUpper(strings.TrimSpace(m))
	}
	if method != "GET" && method != "HEAD" && method != "POST" {
		return nil, fmt.Errorf("method %q is not GET, HEAD or POST", method)
	}
	body, _ := args["body"].(string)
	if len(body) > MostRequestBody {
		return nil, fmt.Errorf("body is %d bytes, more than the %d this tool sends", len(body), MostRequestBody)
	}
	timeout := DefaultRequestTTL
	if _, given := args["timeout_ms"]; given {
		ms, err := whole(args, "timeout_ms", int(DefaultRequestTTL/time.Millisecond))
		if err != nil {
			return nil, err
		}
		if ms < 1 {
			return nil, fmt.Errorf("timeout_ms must be at least 1, not %d", ms)
		}
		timeout = time.Duration(ms) * time.Millisecond
		if timeout > MostTimeout {
			timeout = MostTimeout
		}
	}
	var headers map[string]any
	if h, given := args["headers"]; given && h != nil {
		var ok bool
		if headers, ok = h.(map[string]any); !ok {
			return nil, errors.New("headers must be an object of header name to value")
		}
	}

	cx, cancel := context.WithTimeout(context.Background(), timeout)
	defer cancel()
	var reader io.Reader
	if body != "" {
		reader = strings.NewReader(body)
	}
	req, err := http.NewRequestWithContext(cx, method, u.String(), reader)
	if err != nil {
		return nil, err
	}
	for name, v := range headers {
		s, ok := v.(string)
		if !ok {
			return nil, fmt.Errorf("header %q must be a string", name)
		}
		if strings.EqualFold(name, "Host") {
			req.Host = s
			continue
		}
		req.Header.Set(name, s)
	}
	transport := &http.Transport{
		Proxy:       nil,
		DialContext: (&net.Dialer{Timeout: timeout, Control: guardDial}).DialContext,
	}
	defer transport.CloseIdleConnections()
	client := &http.Client{
		Transport:     transport,
		Timeout:       timeout,
		CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
	}
	start := time.Now()
	resp, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("%s %s: %v", method, raw, err)
	}
	defer resp.Body.Close()
	data, err := io.ReadAll(io.LimitReader(resp.Body, MostResponseBody+1))
	if err != nil {
		return nil, fmt.Errorf("reading the answer of %s: %v", raw, err)
	}
	cut := len(data) > MostResponseBody
	if cut {
		data = data[:MostResponseBody]
	}
	res := RequestResult{
		URL: raw, Method: method, Status: resp.StatusCode, StatusLine: resp.Status, Proto: resp.Proto,
		Headers: map[string][]string(resp.Header), BodyBytes: len(data), Cut: cut, Encoding: "text",
		ElapsedMS: time.Since(start).Milliseconds(),
	}
	if resp.StatusCode >= 300 && resp.StatusCode < 400 {
		if loc, err := resp.Location(); err == nil {
			res.Redirect = loc.String()
			if l := resp.Header.Get("Location"); !strings.Contains(l, "://") {
				res.Redirect = l
			}
		} else {
			res.Redirect = resp.Header.Get("Location")
		}
	}
	if utf8.Valid(data) || (cut && validBeforeCut(data)) {
		res.Body = string(data)
	} else {
		res.Body, res.Encoding = base64.StdEncoding.EncodeToString(bytes.Clone(data)), "base64"
	}
	return res, nil
}

// validBeforeCut: a body cut inside a multi-byte character is still text.
func validBeforeCut(b []byte) bool {
	for i := 0; i < utf8.UTFMax && i < len(b); i++ {
		if utf8.Valid(b[:len(b)-i]) {
			return true
		}
	}
	return false
}