mesh-host/internal/apply/busy.go

package apply

import (
	"context"
	"fmt"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/novox/mesh-host/internal/declaration"
	"github.com/novox/mesh-host/internal/store"
)

// A container that says it is busy is replaced once it is idle, within a bound (novox/hq ADR 0335).
//
// **Asked, never waited on.** The apply asks the container once, and while it says busy leaves it running
// as it is and goes on with everything else of the declaration: the machine's lock is held for one ask,
// never for the wait. The wait is carried between passes in the container's record (store.Applied's
// BusySince), so the next pass — the node-engine asks again sooner than its reconcile while anything waits
// — asks again, and so does a node-engine started again in between: the bound runs from the first apply
// that would have replaced the container, whoever applied it.

// waitsBusy is the action of a container left running because it said busy (novox/hq ADR 0335 §2). Its own
// word rather than "unchanged": its declaration moved, and the machine is not yet what was declared.
const waitsBusy = "waits-busy"

// What a report says of a busy container (novox/hq ADR 0335 §2, §3).
const (
	// BusyWaiting is a container left running because it said busy, waiting to be replaced.
	BusyWaiting = "waiting"
	// BusyReplacedIdle is a container replaced on the first pass it said idle, after it had waited.
	BusyReplacedIdle = "replaced-when-idle"
	// BusyReplacedAtBound is a container still busy at the bound, replaced anyway.
	BusyReplacedAtBound = "replaced-at-bound"
)

// BusyWait is one container a busy command held, as a report says it.
type BusyWait struct {
	// ID is the resource's id (`<module>.<resource>`), Container its runtime name.
	ID        string `json:"id"`
	Container string `json:"container"`
	// State is BusyWaiting, BusyReplacedIdle or BusyReplacedAtBound.
	State string `json:"state"`
	// Since is when the first apply that would have replaced it ran; Until when the bound replaces it.
	Since time.Time `json:"since"`
	Until time.Time `json:"until"`
	// Said is the last output its busy command gave.
	Said string `json:"said,omitempty"`
}

// Words is the wait in a sentence, for an outcome's detail and the console.
func (b BusyWait) Words() string {
	since := b.Since.UTC().Format(time.RFC3339)
	said := ""
	if b.Said != "" {
		said = "; its busy command last said: " + b.Said
	}
	switch b.State {
	case BusyReplacedAtBound:
		return fmt.Sprintf("replaced while still busy: it said busy since %s and the bound of %s ran out "+
			"(novox/hq ADR 0335 §3)%s", since, declaration.BusyBound, said)
	case BusyReplacedIdle:
		return fmt.Sprintf("replaced once it said idle, after waiting since %s (novox/hq ADR 0335 §2)%s", since, said)
	}
	return fmt.Sprintf("busy: left running and replaced on the first pass it says idle, or at %s at the latest; "+
		"waiting since %s (novox/hq ADR 0335 §2)%s", b.Until.UTC().Format(time.RFC3339), since, said)
}

// busyClock is the clock the bound is read on. A test moves it.
var busyClock = time.Now

// busySaidMost is how much of a busy command's output a report carries: its last lines, enough to say what
// it held.
const busySaidMost = 400

// askBeforeReplacing asks a running container that declares busy whether it may be replaced now, and says
// in out what came of it. True when it waits: out is then the whole outcome of this pass. False when the
// caller replaces it — idle, or past the bound — with out carrying the wait it ended, if there was one.
func askBeforeReplacing(ctx context.Context, r *declaration.Container, run Runner, cri string,
	previous store.Applied, out *Outcome) bool {
	now := busyClock().UTC()
	busy, said := askBusy(ctx, r, run, cri)
	var since time.Time
	waited := previous.BusySince != nil && previous.Target == r.Name
	if waited {
		since = previous.BusySince.UTC()
	}
	wait := BusyWait{ID: r.ID, Container: r.Name, Since: since, Said: said}
	switch {
	case !busy && !waited:
		// Idle at the first ask: replaced at once, as a container that declares nothing is.
		return false
	case !busy:
		wait.State, wait.Until = BusyReplacedIdle, since.Add(declaration.BusyBound)
		out.busy = &wait
		return false
	}
	if !waited {
		since = now
		wait.Since = now
	}
	wait.Until = since.Add(declaration.BusyBound)
	if !now.Before(wait.Until) {
		wait.State = BusyReplacedAtBound
		out.busy = &wait
		return false
	}
	wait.State = BusyWaiting
	out.Action, out.Detail = waitsBusy, wait.Words()
	out.busySince, out.busy = &since, &wait
	return true
}

// askBusy runs a container's busy command inside it, by its shell, as an exec health check is run: busy is
// an exit of 0 and nothing else. An ask that could not be made, or did not answer within its timeout, is
// not busy: the bound is what keeps a container whose command never answers from being held.
func askBusy(ctx context.Context, r *declaration.Container, run Runner, cri string) (bool, string) {
	ask, cancel := context.WithTimeout(ctx, r.Busy.Within())
	defer cancel()
	output, err := run(ask, cri, "exec", r.Name, "/bin/sh", "-c", r.Busy.Command)
	said := strings.TrimSpace(output)
	if err != nil && said == "" {
		said = err.Error()
	}
	return err == nil, lastOf(said, busySaidMost)
}

// lastOf is the end of a text, at most n bytes, on one line.
func lastOf(s string, n int) string {
	s = strings.Join(strings.Fields(s), " ")
	if len(s) <= n {
		return s
	}
	from := len(s) - n
	for from < len(s) && !utf8.RuneStart(s[from]) {
		from++
	}
	return "…" + s[from:]
}