Add netcheck_request: one HTTP request and its whole answer, so a service's live answers can be captured (issue 1643)

2026-10-21 10:08:59 +00:00
parent f0c8958dd2
commit ff849d9f62
6 changed files with 338 additions and 1 deletions
@@ -0,0 +1,164 @@
+package main
+
+// One HTTP request and its whole answer (hq issue 1643). netcheck_http answers whether a URL is up;
+// this answers what it says: the status line, every response header and the body, so a service's live
+// answers (a mail autoconfiguration file, an API's reply) can be captured from the machine that
+// reaches it. The request is made once: redirects are reported, never followed, and only http and
+// https are spoken. TLS is verified against the machine's own trust store (which holds the mesh's CA).
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+	"unicode/utf8"
+)
+
+// Bounds: a request body of at most 256 KiB is sent, a response body of at most 1 MiB is read.
+const (
+	MostRequestBody   = 256 * 1024
+	MostResponseBody  = 1024 * 1024
+	DefaultRequestTTL = 10 * time.Second
+)
+
+// RequestResult is what netcheck_request answers.
+type RequestResult struct {
+	URL        string              `json:"url"`
+	Method     string              `json:"method"`
+	Status     int                 `json:"status"`
+	StatusLine string              `json:"status_line"`
+	Proto      string              `json:"proto"`
+	Headers    map[string][]string `json:"headers"`
+	Redirect   string              `json:"redirect,omitempty"`
+	Body       string              `json:"body"`
+	Encoding   string              `json:"body_encoding"`
+	BodyBytes  int                 `json:"body_bytes"`
+	Cut        bool                `json:"body_cut"`
+	ElapsedMS  int64               `json:"elapsed_ms"`
+}
+
+func runRequest(args map[string]any) (any, error) {
+	raw, err := text(args, "url")
+	if err != nil {
+		return nil, err
+	}
+	u, err := url.Parse(raw)
+	if err != nil {
+		return nil, fmt.Errorf("url %q is not one: %v", raw, err)
+	}
+	if u.Scheme != "http" && u.Scheme != "https" {
+		return nil, fmt.Errorf("url %q: only http and https are spoken, not %q", raw, u.Scheme)
+	}
+	if u.Host == "" {
+		return nil, fmt.Errorf("url %q has no host", raw)
+	}
+	method := "GET"
+	if m, _ := args["method"].(string); strings.TrimSpace(m) != "" {
+		method = strings.ToUpper(strings.TrimSpace(m))
+	}
+	if method != "GET" && method != "HEAD" && method != "POST" {
+		return nil, fmt.Errorf("method %q is not GET, HEAD or POST", method)
+	}
+	body, _ := args["body"].(string)
+	if len(body) > MostRequestBody {
+		return nil, fmt.Errorf("body is %d bytes, more than the %d this tool sends", len(body), MostRequestBody)
+	}
+	timeout := DefaultRequestTTL
+	if _, given := args["timeout_ms"]; given {
+		ms, err := whole(args, "timeout_ms", int(DefaultRequestTTL/time.Millisecond))
+		if err != nil {
+			return nil, err
+		}
+		if ms < 1 {
+			return nil, fmt.Errorf("timeout_ms must be at least 1, not %d", ms)
+		}
+		timeout = time.Duration(ms) * time.Millisecond
+		if timeout > MostTimeout {
+			timeout = MostTimeout
+		}
+	}
+	var headers map[string]any
+	if h, given := args["headers"]; given && h != nil {
+		var ok bool
+		if headers, ok = h.(map[string]any); !ok {
+			return nil, errors.New("headers must be an object of header name to value")
+		}
+	}
+
+	cx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+	var reader io.Reader
+	if body != "" {
+		reader = strings.NewReader(body)
+	}
+	req, err := http.NewRequestWithContext(cx, method, u.String(), reader)
+	if err != nil {
+		return nil, err
+	}
+	for name, v := range headers {
+		s, ok := v.(string)
+		if !ok {
+			return nil, fmt.Errorf("header %q must be a string", name)
+		}
+		if strings.EqualFold(name, "Host") {
+			req.Host = s
+			continue
+		}
+		req.Header.Set(name, s)
+	}
+	client := &http.Client{
+		Timeout:       timeout,
+		CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
+	}
+	start := time.Now()
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("%s %s: %v", method, raw, err)
+	}
+	defer resp.Body.Close()
+	data, err := io.ReadAll(io.LimitReader(resp.Body, MostResponseBody+1))
+	if err != nil {
+		return nil, fmt.Errorf("reading the answer of %s: %v", raw, err)
+	}
+	cut := len(data) > MostResponseBody
+	if cut {
+		data = data[:MostResponseBody]
+	}
+	res := RequestResult{
+		URL: raw, Method: method, Status: resp.StatusCode, StatusLine: resp.Status, Proto: resp.Proto,
+		Headers: map[string][]string(resp.Header), BodyBytes: len(data), Cut: cut, Encoding: "text",
+		ElapsedMS: time.Since(start).Milliseconds(),
+	}
+	if resp.StatusCode >= 300 && resp.StatusCode < 400 {
+		if loc, err := resp.Location(); err == nil {
+			res.Redirect = loc.String()
+			if l := resp.Header.Get("Location"); !strings.Contains(l, "://") {
+				res.Redirect = l
+			}
+		} else {
+			res.Redirect = resp.Header.Get("Location")
+		}
+	}
+	if utf8.Valid(data) || (cut && validBeforeCut(data)) {
+		res.Body = string(data)
+	} else {
+		res.Body, res.Encoding = base64.StdEncoding.EncodeToString(bytes.Clone(data)), "base64"
+	}
+	return res, nil
+}
+
+// validBeforeCut: a body cut inside a multi-byte character is still text.
+func validBeforeCut(b []byte) bool {
+	for i := 0; i < utf8.UTFMax && i < len(b); i++ {
+		if utf8.Valid(b[:len(b)-i]) {
+			return true
+		}
+	}
+	return false
+}