mesh-lab/replays/bed.go

package replays

import (
	"bufio"
	"context"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"
)

// The bed (novox/hq ADR 0240 rule 7, to-be 48 §8, Phase D): **a declaration is proved before it is trusted.**
//
// Two of the nineteen image checks the catalogue's containers ship read *unhealthy* while working in the
// mesh's hands — the studio's asked an address its program does not bind, the flow editor's read settings
// from a path the module mounted elsewhere. Read without proof, they would have put back two good builds.
// So the catalogue's merge check starts every resource whose declared check or image the change touches,
// **alone**, on a throwaway runtime, and requires its check to see the program within its grace.
//
// **What it proves is the check's wiring**: that it can see the program working — a property of the image
// and the declaration, not of the mesh. A resource is started with what the module declares and nothing of
// the mesh's: its literal environment, its arguments, its files where the module writes them in full, and an
// empty directory for every other place it mounts; no secret, no binding, no provider. So:
//
//   - a check naming a provision in `needs` gets no provider here, and must only reach the program — an
//     http answer of any status, a connect;
//   - a program that does not stay up alone — it needs what the mesh gives it — is said as not proved here,
//     and judged on the first machine's gate, never passed as proved and never failed;
//   - a program that stays up while its check does not see it **fails**: that is the studio's fault.
//
// A tool check and a unit's own readiness need the mesh and a machine; they are said, not proved.

// BedResource is one resource the bed proves: its module, its id, and the resource as the manifest says it.
type BedResource struct {
	Module   string
	ID       string
	Resource map[string]any
	// Listens are the module's endpoints, by name: the container's own port for each.
	Listens map[string]int
}

// BedVerdict is what the bed found of one resource.
type BedVerdict struct {
	Proved bool
	// Failed says the program stayed up and its check did not see it: the change's fault.
	Failed bool
	Said   string
}

// BedLook is how often the bed looks: often, because it proves the wiring, not the timing.
var BedLook = 2 * time.Second

// BedFloor is the least the bed waits for a check to pass, whatever the grace: a program needs a moment to
// listen even when the module declared no grace.
var BedFloor = 30 * time.Second

// ChangedHealth is every long-running container resource of the catalogue at root that declares `health`
// and whose resource changed against base (a git ref in that checkout): its declaration, its image, or
// anything else of it. A manifest new on this branch counts whole. Read through git; an error when the
// base cannot be read.
func ChangedHealth(root, base string, show func(ref, path string) ([]byte, error)) ([]BedResource, error) {
	paths, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
	if err != nil {
		return nil, err
	}
	var out []BedResource
	for _, p := range paths {
		raw, err := os.ReadFile(p)
		if err != nil {
			return nil, err
		}
		rel, _ := filepath.Rel(root, p)
		was := map[string]string{}
		if before, err := show(base, rel); err == nil {
			for _, r := range resourcesOf(before) {
				was[fmt.Sprint(r["id"])] = canonical(r)
			}
		}
		module, listens, resources := manifestParts(raw)
		for _, r := range resources {
			if _, declared := r["health"]; !declared || fmt.Sprint(r["type"]) != "container" || !stays(r) {
				continue
			}
			if was[fmt.Sprint(r["id"])] == canonical(r) {
				continue
			}
			out = append(out, BedResource{Module: module, ID: fmt.Sprint(r["id"]), Resource: r, Listens: listens})
		}
	}
	sort.Slice(out, func(a, b int) bool { return out[a].Module+"."+out[a].ID < out[b].Module+"."+out[b].ID })
	return out, nil
}

func stays(r map[string]any) bool {
	once, _ := r["run-once"].(bool)
	return !once && r["schedule"] == nil
}

func canonical(r map[string]any) string {
	raw, _ := json.Marshal(r)
	return string(raw)
}

func resourcesOf(raw []byte) []map[string]any {
	_, _, rs := manifestParts(raw)
	return rs
}

func manifestParts(raw []byte) (string, map[string]int, []map[string]any) {
	var m struct {
		Module  string `json:"module"`
		Listens []struct {
			Name string `json:"name"`
			Port int    `json:"port"`
		} `json:"listens"`
		Resources []map[string]any `json:"resources"`
	}
	_ = json.Unmarshal(raw, &m)
	listens := map[string]int{}
	for _, l := range m.Listens {
		if l.Name != "" {
			listens[l.Name] = l.Port
		}
	}
	return m.Module, listens, m.Resources
}

// placeholder is anything the mesh fills in on a machine: a value holding one is not the module's alone.
var placeholder = regexp.MustCompile(`\$\{[^}]*\}`)

// Prove starts one resource alone on the runtime and looks at it with its declared check until the check
// sees it, the program stops, or its grace (at least BedFloor) runs out. Everything it made is removed.
func (d *Docker) Prove(ctx context.Context, r BedResource, files map[string]string) BedVerdict {
	h, _ := r.Resource["health"].(map[string]any)
	kind := fmt.Sprint(h["kind"])
	switch kind {
	case "tool", "unit":
		return BedVerdict{Said: fmt.Sprintf("%s.%s: a %s check needs the mesh and a machine; judged on the first "+
			"machine's gate", r.Module, r.ID, kind)}
	}
	image, _ := r.Resource["image"].(string)
	if image == "" || placeholder.MatchString(image) {
		return BedVerdict{Said: fmt.Sprintf("%s.%s: its image is built by the mesh (%v); judged on the first machine's gate",
			r.Module, r.ID, r.Resource["artifact"])}
	}
	if err := d.Pull(ctx, image); err != nil {
		return BedVerdict{Said: fmt.Sprintf("%s.%s: its image could not be fetched here: %v", r.Module, r.ID, oneLine(err.Error()))}
	}
	scratch, err := os.MkdirTemp("", "mesh-bed-")
	if err != nil {
		return BedVerdict{Said: err.Error()}
	}
	defer os.RemoveAll(scratch)
	// Its literal environment, and every env-file the module writes in full.
	var env []string
	if e, ok := r.Resource["env"].(map[string]any); ok {
		for k, v := range e {
			if s := fmt.Sprint(v); !placeholder.MatchString(s) {
				env = append(env, k+"="+s)
			}
		}
	}
	if list, ok := r.Resource["env-file"].([]any); ok {
		for _, f := range list {
			if content, ok := files[fmt.Sprint(f)]; ok {
				env = append(env, envLines(content)...)
			}
		}
	}
	sort.Strings(env)
	// Its mounts: a file the module writes in full, or an empty place of its own.
	var binds []string
	if list, ok := r.Resource["volumes"].([]any); ok {
		for i, v := range list {
			src, dst, rest := splitMount(fmt.Sprint(v))
			if dst == "" || !strings.HasPrefix(dst, "/") {
				continue
			}
			local := filepath.Join(scratch, "m"+strconv.Itoa(i))
			if content, ok := files[src]; ok {
				if err := os.WriteFile(local, []byte(content), 0o644); err != nil {
					continue
				}
			} else if err := os.MkdirAll(local, 0o777); err != nil {
				continue
			}
			_ = os.Chmod(local, 0o777)
			binds = append(binds, local+":"+dst+rest)
		}
	}
	var args []string
	if list, ok := r.Resource["args"].([]any); ok {
		for _, a := range list {
			args = append(args, fmt.Sprint(a))
		}
	}
	network := fmt.Sprintf("mesh-bed-%d", time.Now().UnixNano())
	netID, err := d.Network(ctx, network)
	if err != nil {
		return BedVerdict{Said: "the bed's network could not be made: " + err.Error()}
	}
	defer d.RemoveNetwork(context.Background(), netID)

	grace, _ := time.ParseDuration(fmt.Sprint(h["grace"]))
	if _, said := h["grace"]; !said {
		grace = 60 * time.Second
	}
	if grace < BedFloor {
		grace = BedFloor
	}
	config := map[string]any{"Image": image, "Env": env, "Labels": map[string]string{d.Label: "bed"}}
	if len(args) > 0 {
		config["Cmd"] = args
	}
	switch kind {
	case "exec":
		config["Healthcheck"] = map[string]any{"Test": []string{"CMD-SHELL", fmt.Sprint(h["command"])},
			"Interval": BedLook.Nanoseconds(), "Timeout": BedLook.Nanoseconds() - 1, "Retries": 1}
	case "runtime":
		// The image's own command, adopted by name: an empty Test keeps it.
		config["Healthcheck"] = map[string]any{"Interval": BedLook.Nanoseconds(), "Timeout": BedLook.Nanoseconds() - 1,
			"Retries": 1}
	}
	config["HostConfig"] = map[string]any{"Binds": binds, "NetworkMode": network}
	name := fmt.Sprintf("mesh-bed-%s-%s-%d", r.Module, r.ID, time.Now().UnixNano())
	var made struct{ ID string }
	if err := d.call(ctx, http.MethodPost, "/containers/create?name="+name, config, &made); err != nil {
		return BedVerdict{Said: fmt.Sprintf("%s.%s could not be made here: %v", r.Module, r.ID, oneLine(err.Error()))}
	}
	defer d.Remove(context.Background(), made.ID)
	if err := d.call(ctx, http.MethodPost, "/containers/"+made.ID+"/start", nil, nil); err != nil {
		return BedVerdict{Said: fmt.Sprintf("%s.%s could not be started here: %v", r.Module, r.ID, oneLine(err.Error()))}
	}

	port := r.Listens[fmt.Sprint(h["endpoint"])]
	needs := fmt.Sprint(h["needs"]) != "" && h["needs"] != nil
	deadline := time.Now().Add(grace)
	last := "it was never looked at"
	for {
		var seen struct {
			State struct {
				Running bool
				Health  *struct {
					Status string
					Log    []struct{ Output string }
				}
			}
			NetworkSettings struct {
				Networks map[string]struct{ IPAddress string }
			}
		}
		if err := d.call(ctx, http.MethodGet, "/containers/"+made.ID+"/json", nil, &seen); err != nil {
			return BedVerdict{Said: err.Error()}
		}
		if !seen.State.Running {
			return BedVerdict{Said: fmt.Sprintf("%s.%s does not stay up alone — it needs what the mesh gives it — so its "+
				"check is judged on the first machine's gate: %s", r.Module, r.ID, oneLine(lastLine(d.Logs(ctx, made.ID))))}
		}
		ok := false
		switch kind {
		case "exec", "runtime":
			hs := seen.State.Health
			switch {
			case hs == nil:
				last = "the container carries no check: its image ships none to adopt"
			case hs.Status == "healthy":
				ok = true
			default:
				last = "the runtime says " + hs.Status
				if n := len(hs.Log); n > 0 {
					last += ": " + oneLine(hs.Log[n-1].Output)
				}
			}
		case "http", "tcp":
			address := ""
			for _, n := range seen.NetworkSettings.Networks {
				address = n.IPAddress
			}
			ok, last = look(ctx, kind, address, port, h, needs)
		default:
			return BedVerdict{Said: fmt.Sprintf("%s.%s declares a %s check the bed does not know", r.Module, r.ID, kind)}
		}
		if ok {
			return BedVerdict{Proved: true, Said: fmt.Sprintf("%s.%s: its %s check sees the program", r.Module, r.ID, kind)}
		}
		if time.Now().After(deadline) || ctx.Err() != nil {
			return BedVerdict{Failed: true, Said: fmt.Sprintf("%s.%s stays up and its %s check does not see it within %s: %s",
				r.Module, r.ID, kind, grace, last)}
		}
		select {
		case <-ctx.Done():
		case <-time.After(BedLook):
		}
	}
}

// look is one http or tcp look at the program, from outside it, as the node-engine makes it. A check that
// needs a provider only has to reach the program: any answer, a connect.
func look(ctx context.Context, kind, address string, port int, h map[string]any, needs bool) (bool, string) {
	if address == "" || port == 0 {
		return false, "it has no address on the bed's network yet"
	}
	at := net.JoinHostPort(address, strconv.Itoa(port))
	ctx, cancel := context.WithTimeout(ctx, BedLook)
	defer cancel()
	if kind == "tcp" {
		c, err := (&net.Dialer{}).DialContext(ctx, "tcp", at)
		if err != nil {
			return false, oneLine(err.Error())
		}
		c.Close()
		return true, ""
	}
	scheme, _ := h["scheme"].(string)
	if scheme == "" {
		scheme = "http"
	}
	path, _ := h["path"].(string)
	if path == "" {
		path = "/"
	}
	req, _ := http.NewRequestWithContext(ctx, http.MethodGet, scheme+"://"+at+path, nil)
	res, err := (&http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
		Transport: insecure()}).Do(req)
	if err != nil {
		return false, oneLine(err.Error())
	}
	res.Body.Close()
	if needs {
		return true, ""
	}
	want, _ := h["status"].(float64)
	switch {
	case want != 0 && res.StatusCode != int(want):
		return false, fmt.Sprintf("answered %d, expected %d", res.StatusCode, int(want))
	case want == 0 && res.StatusCode >= 400:
		return false, fmt.Sprintf("answered %d", res.StatusCode)
	}
	return true, ""
}

func splitMount(v string) (src, dst, rest string) {
	parts := strings.SplitN(v, ":", 3)
	if len(parts) < 2 {
		return "", "", ""
	}
	if len(parts) == 3 {
		rest = ":" + parts[2]
	}
	return parts[0], parts[1], rest
}

func envLines(content string) []string {
	var out []string
	s := bufio.NewScanner(strings.NewReader(content))
	for s.Scan() {
		line := strings.TrimSpace(s.Text())
		if line == "" || strings.HasPrefix(line, "#") || !strings.Contains(line, "=") || placeholder.MatchString(line) {
			continue
		}
		out = append(out, line)
	}
	return out
}

func lastLine(s string) string {
	lines := strings.Split(strings.TrimSpace(s), "\n")
	return lines[len(lines)-1]
}

// FilesOf is every file a module writes in full — its content declared, nothing the mesh fills in — by the
// path it writes it at, as the module names it: what the bed gives a container that mounts or reads it.
func FilesOf(manifest []byte) map[string]string {
	_, _, rs := manifestParts(manifest)
	out := map[string]string{}
	for _, r := range rs {
		if fmt.Sprint(r["type"]) != "file" {
			continue
		}
		path, _ := r["path"].(string)
		content, ok := r["content"].(string)
		if path == "" || !ok || placeholder.MatchString(content) {
			continue
		}
		out[path] = content
	}
	return out
}

// insecure is a transport that asks whether a program answers, not whom to trust.
func insecure() *http.Transport {
	return &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, DisableKeepAlives: true}
}

// oneLine is the first line of what was said, short.
func oneLine(s string) string {
	line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
	if len(line) > 300 {
		line = line[:300] + "…"
	}
	return line
}