mesh-host/internal/liveness/runtime.go

package liveness

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"strconv"
	"strings"
	"sync"

	"github.com/novox/mesh-host/internal/system"
)

// Runner runs a command and answers what it printed — the apply's own (apply.ExecRunner), so a look
// reads the machine exactly as an apply does.
type Runner func(ctx context.Context, name string, args ...string) (string, error)

// Exec reads the container runtime and the service manager through their command lines: one inspect of
// every container, one show of every unit per manager. **Reads only** — the whole of what this file may
// ask either of them is `container inspect`, `version` and `show`, and of the machine's manager whether an
// account's own is active (ADR 0240 rule 6, and a test holds it).
type Exec struct {
	Run Runner
	// System is what the host was built for: it says how the machine's manager is asked whether an
	// account's own runs (system.ManagerRunning).
	System string

	mu sync.Mutex
	// cri is the runtime that answered, once one has.
	cri string
}

// runtimes are the container runtimes the apply knows, asked for their version in the form each
// understands (apply.containerRuntimes): the first that answers is the machine's.
var runtimes = []struct {
	command string
	probe   []string
}{
	{"docker", []string{"version", "--format", "{{.Server.Version}}"}},
	{"podman", []string{"version", "--format", "{{.Version}}"}},
}

func (e *Exec) runtime(ctx context.Context) (string, error) {
	e.mu.Lock()
	defer e.mu.Unlock()
	if e.cri != "" {
		return e.cri, nil
	}
	var tried []string
	for _, rt := range runtimes {
		if _, err := e.Run(ctx, rt.command, rt.probe...); err == nil {
			e.cri = rt.command
			return rt.command, nil
		}
		tried = append(tried, rt.command)
	}
	return "", fmt.Errorf("no container runtime answers on this machine (tried %s)", strings.Join(tried, ", "))
}

// inspectFormat is what one inspect says of each container: its name, id, status, the runtime's restart
// count, when its current run started — and the state of the check the runtime runs as the container's
// own, read out of the whole state as one line of JSON, for a check declared exec or runtime (Phase B).
// The whole state and not its Health: a container that carries no check has no such key, and the runtime
// refuses a template naming a key that is not there — for every container in the read.
const inspectFormat = "{{.Name}}\t{{.Id}}\t{{.State.Status}}\t{{.RestartCount}}\t{{.State.StartedAt}}\t{{json .State}}"

// Containers is one inspect of every container named. A name the runtime does not have is not found;
// a runtime that does not answer is an error — unknown, never down.
func (e *Exec) Containers(ctx context.Context, names []string) (map[string]Observed, error) {
	out := map[string]Observed{}
	if len(names) == 0 {
		return out, nil
	}
	cri, err := e.runtime(ctx)
	if err != nil {
		return nil, err
	}
	args := append([]string{"container", "inspect", "--format", inspectFormat}, names...)
	said, err := e.Run(ctx, cri, args...)
	if err != nil && !missing(err) {
		// Asked again next look, in case another runtime is what answers now.
		e.mu.Lock()
		e.cri = ""
		e.mu.Unlock()
		return nil, fmt.Errorf("the container runtime could not be read: %w", err)
	}
	for _, line := range strings.Split(strings.TrimSpace(said), "\n") {
		parts := strings.Split(line, "\t")
		if len(parts) < 5 {
			continue
		}
		name := strings.TrimPrefix(strings.TrimSpace(parts[0]), "/")
		restarts, _ := strconv.ParseInt(strings.TrimSpace(parts[3]), 10, 64)
		status := strings.TrimSpace(parts[2])
		o := Observed{Found: true, Identity: strings.TrimSpace(parts[1]), Running: status == "running",
			Restarting: status == "restarting", Restarts: restarts, Started: strings.TrimSpace(parts[4])}
		if len(parts) > 5 {
			o.Health, o.HealthSaid = runtimeHealth(strings.Join(parts[5:], "\t"))
		}
		out[name] = o
	}
	return out, nil
}

// runtimeHealth reads, from a container's state, the runtime's state of its own check: its status and what its last look
// printed, in a line. Nothing when the container carries no check.
func runtimeHealth(said string) (string, string) {
	var state struct {
		Health *struct {
			Status string
			Log    []struct {
				ExitCode int
				Output   string
			}
		}
	}
	if err := json.Unmarshal([]byte(strings.TrimSpace(said)), &state); err != nil || state.Health == nil ||
		state.Health.Status == "" {
		return "", ""
	}
	h := state.Health
	last := ""
	if n := len(h.Log); n > 0 {
		l := h.Log[n-1]
		last = firstLine(l.Output)
		if last == "" && l.ExitCode != 0 {
			last = fmt.Sprintf("its check exited %d", l.ExitCode)
		}
		if len(last) > 200 {
			last = last[:200] + "…"
		}
	}
	return strings.ToLower(h.Status), last
}

// missing is an inspect that failed only because a name is not there: what it printed for the others is
// still the answer.
func missing(err error) bool {
	s := strings.ToLower(err.Error())
	return strings.Contains(s, "no such container") || strings.Contains(s, "no such object")
}

// unitProperties are what one show says of each unit.
const unitProperties = "Id,LoadState,ActiveState,SubState,NRestarts,InvocationID"

// Units is one show of every unit named, in the manager it is in: the machine's, or an account's own.
func (e *Exec) Units(ctx context.Context, scope, user string, units []string) (map[string]Observed, error) {
	out := map[string]Observed{}
	if len(units) == 0 {
		return out, nil
	}
	args := []string{"show", "--property=" + unitProperties, "--"}
	if scope == "user" && user != "" {
		// **A look starts no account's manager** (novox/hq issue 764, ADR 0177): reading it is a login,
		// which starts it when none runs. The machine's manager is asked first; a manager not running is
		// away — its units unread, said unknown with why, never down — as the apply says them waiting.
		running, exists, err := system.ManagerRunning(ctx, e.System, system.Runner(e.Run), user)
		switch {
		case err != nil:
			return nil, fmt.Errorf("whether %s's own service manager runs could not be read: %w", user, err)
		case !exists:
			return nil, fmt.Errorf("the machine has no account %q, whose own manager these units are in", user)
		case !running:
			return nil, fmt.Errorf("%s's own service manager is not running: the account is not logged in "+
				"and does not linger", user)
		}
		args = append([]string{"--user", "--machine=" + user + "@"}, args...)
	}
	said, err := e.Run(ctx, "systemctl", append(args, units...)...)
	if err != nil {
		return nil, fmt.Errorf("the service manager could not be read: %w", err)
	}
	blocks := strings.Split(strings.TrimSpace(said), "\n\n")
	if len(blocks) != len(units) {
		return nil, errors.New("the service manager answered for a different number of units than were asked")
	}
	for i, block := range blocks {
		props := map[string]string{}
		for _, line := range strings.Split(block, "\n") {
			if k, v, ok := strings.Cut(line, "="); ok {
				props[strings.TrimSpace(k)] = strings.TrimSpace(v)
			}
		}
		restarts, _ := strconv.ParseInt(props["NRestarts"], 10, 64)
		out[units[i]] = Observed{Found: props["LoadState"] != "not-found", Identity: props["InvocationID"],
			Running:    props["ActiveState"] == "active",
			Restarting: props["ActiveState"] == "activating" && props["SubState"] == "auto-restart",
			Restarts:   restarts}
	}
	return out, nil
}