mesh-host/internal/apply/busy_test.go

package apply

import (
	"context"
	"errors"
	"path/filepath"
	"slices"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/novox/mesh-host/internal/declaration"
	"github.com/novox/mesh-host/internal/store"
)

// A container that says it is busy is replaced once it is idle, within a bound (novox/hq ADR 0335). The
// machine of these tests runs two containers, a build agent that declares `busy` and a web server that
// does not; every call that replaces a container is counted.

type aBusyMachine struct {
	mu sync.Mutex
	// containers is each container's name and the spec it was created with; running whether it runs.
	containers map[string]string
	busy       bool   // what the agent's busy command answers
	said       string // what it prints
	asked      int
	calls      []string
}

func newBusyMachine() *aBusyMachine {
	return &aBusyMachine{containers: map[string]string{}}
}

func (m *aBusyMachine) run(_ context.Context, _ string, args ...string) (string, error) {
	m.mu.Lock()
	defer m.mu.Unlock()
	switch args[0] {
	case "info", "image":
		return "27.0\n", nil
	case "container": // inspect
		name := args[len(args)-1]
		spec, there := m.containers[name]
		if !there {
			return "", errors.New("no such container")
		}
		return "true\t" + spec, nil
	case "exec":
		m.asked++
		m.calls = append(m.calls, "ask "+args[1])
		if strings.Join(args[2:], " ") != "/bin/sh -c mesh-builder busy" {
			return "", errors.New("asked something else: " + strings.Join(args[2:], " "))
		}
		if m.busy {
			return m.said + "\n", nil
		}
		return m.said + "\n", errors.New("exited 1")
	case "rm":
		name := args[len(args)-1]
		m.calls = append(m.calls, "rm "+name)
		delete(m.containers, name)
	case "run":
		name := args[slices.Index(args, "--name")+1]
		m.calls = append(m.calls, "create "+name)
		for i, a := range args {
			if a == "--label" && strings.HasPrefix(args[i+1], specLabel+"=") {
				m.containers[name] = strings.TrimPrefix(args[i+1], specLabel+"=")
			}
		}
	}
	return "", nil
}

func (m *aBusyMachine) mark() int {
	m.mu.Lock()
	defer m.mu.Unlock()
	return len(m.calls)
}

// since is every call since mark that replaced a container, or asked one.
func (m *aBusyMachine) since(mark int) []string {
	m.mu.Lock()
	defer m.mu.Unlock()
	return append([]string(nil), m.calls[mark:]...)
}

func (m *aBusyMachine) answer(busy bool, said string) {
	m.mu.Lock()
	defer m.mu.Unlock()
	m.busy, m.said = busy, said
}

// twoContainers is the declaration at a version: v moves the agent's image tag through its env, w the web's.
func twoContainers(t *testing.T, agent, web string) *declaration.Declaration {
	t.Helper()
	return parse(t, `{"declaration":1,"resources":[
	  {"id":"build-agent.server","type":"container","name":"mesh-build-agent","image":"`+pinned+`",
	   "env":{"V":"`+agent+`"},"busy":{"command":"mesh-builder busy","timeout":"5s"}},
	  {"id":"web.server","type":"container","name":"web","image":"`+pinned+`","env":{"V":"`+web+`"}}
	]}`)
}

// atClock moves the bound's clock for one test.
func atClock(t *testing.T, at *time.Time) {
	t.Helper()
	was := busyClock
	busyClock = func() time.Time { return *at }
	t.Cleanup(func() { busyClock = was })
}

func busyOf(r Report, id string) (BusyWait, bool) {
	for _, b := range r.Busy {
		if b.ID == id {
			return b, true
		}
	}
	return BusyWait{}, false
}

func applyBusy(t *testing.T, m *aBusyMachine, d *declaration.Declaration, known store.State) (Report, store.State) {
	t.Helper()
	report, state, err := Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, m.run, nil, nil)
	if err != nil {
		t.Fatalf("the apply failed: %v", err)
	}
	return report, state
}

// The decision's first check: an apply that would replace a container whose busy command exits 0 leaves it
// running, applies the rest, and replaces it on the first pass the command exits non-zero.
func TestABusyContainerIsLeftRunningAndReplacedOnTheFirstPassItSaysIdle(t *testing.T) {
	now := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
	atClock(t, &now)
	m := newBusyMachine()
	_, state := applyBusy(t, m, twoContainers(t, "1", "1"), store.State{})

	// A merge moves both: the agent holds a build, the web server holds nothing.
	m.answer(true, "building novox/mesh-controller (check of #280) since 00:58")
	mark := m.mark()
	report, state := applyBusy(t, m, twoContainers(t, "2", "2"), state)
	if got := m.since(mark); !slices.Equal(got, []string{"ask mesh-build-agent", "rm web", "create web"}) {
		t.Fatalf("a busy agent was not left running while the rest applied: %v", got)
	}
	o := outcomeOf(report, "build-agent.server")
	if o.Action != waitsBusy {
		t.Fatalf("a busy container was reported %q, want %q: %+v", o.Action, waitsBusy, o)
	}
	for _, says := range []string{"busy", "first pass it says idle", "check of #280", "ADR 0335"} {
		if !strings.Contains(o.Detail, says) {
			t.Errorf("the outcome does not say %q: %s", says, o.Detail)
		}
	}
	if (Report{Outcomes: []Outcome{o}}).Changed() {
		t.Errorf("a container left running reads as a change: %+v", o)
	}
	b, said := busyOf(report, "build-agent.server")
	if !said || b.State != BusyWaiting || !b.Since.Equal(now) || !b.Until.Equal(now.Add(15*time.Minute)) ||
		b.Container != "mesh-build-agent" || !strings.Contains(b.Said, "check of #280") {
		t.Fatalf("the report does not say the agent waits, since when and until when: %+v", report.Busy)
	}
	if !report.Waiting() {
		t.Error("a report with a container waiting does not say so")
	}
	if rec, _ := state.Find("build-agent.server"); rec.BusySince == nil || !rec.BusySince.Equal(now) {
		t.Fatalf("the wait is not kept in the container's record: %+v", rec)
	}

	// Asked again a minute later, still busy: still left running, the wait counted from the first apply.
	now = now.Add(time.Minute)
	mark = m.mark()
	report, state = applyBusy(t, m, twoContainers(t, "2", "2"), state)
	if got := m.since(mark); !slices.Equal(got, []string{"ask mesh-build-agent"}) {
		t.Fatalf("a second pass while busy did more than ask: %v", got)
	}
	if b, _ := busyOf(report, "build-agent.server"); b.State != BusyWaiting || !b.Since.Equal(now.Add(-time.Minute)) {
		t.Fatalf("the second pass does not keep the wait from the first apply: %+v", report.Busy)
	}

	// The build ends: the next pass replaces it, and says it waited.
	now = now.Add(time.Minute)
	m.answer(false, "idle")
	mark = m.mark()
	report, state = applyBusy(t, m, twoContainers(t, "2", "2"), state)
	if got := m.since(mark); !slices.Equal(got, []string{"ask mesh-build-agent", "rm mesh-build-agent", "create mesh-build-agent"}) {
		t.Fatalf("the agent was not replaced on the first pass it said idle: %v", got)
	}
	if o := outcomeOf(report, "build-agent.server"); o.Action != "updated" || !strings.Contains(o.Detail, "once it said idle") {
		t.Errorf("the replacement does not say it waited: %+v", o)
	}
	if b, _ := busyOf(report, "build-agent.server"); b.State != BusyReplacedIdle {
		t.Errorf("the report does not say the agent was replaced when idle: %+v", report.Busy)
	}
	if report.Waiting() {
		t.Error("a report with nothing waiting says something waits")
	}
	if rec, _ := state.Find("build-agent.server"); rec.BusySince != nil {
		t.Errorf("the wait is still kept after the replacement: %+v", rec)
	}

	// And the pass after that has nothing to say about it.
	report, _ = applyBusy(t, m, twoContainers(t, "2", "2"), state)
	if len(report.Busy) != 0 {
		t.Errorf("a container replaced is still said: %+v", report.Busy)
	}
}

// An idle container is replaced at once, as one that declares nothing is: asked once, nothing waits.
func TestAnIdleContainerIsReplacedAtOnce(t *testing.T) {
	m := newBusyMachine()
	_, state := applyBusy(t, m, twoContainers(t, "1", "1"), store.State{})
	m.answer(false, "")
	mark := m.mark()
	report, _ := applyBusy(t, m, twoContainers(t, "2", "1"), state)
	if got := m.since(mark); !slices.Equal(got, []string{"ask mesh-build-agent", "rm mesh-build-agent", "create mesh-build-agent"}) {
		t.Fatalf("an idle agent was not replaced at once: %v", got)
	}
	if len(report.Busy) != 0 {
		t.Errorf("an idle container replaced at once is said as a wait: %+v", report.Busy)
	}
	// And a container whose declaration did not move is never asked.
	mark = m.mark()
	applyBusy(t, m, twoContainers(t, "2", "1"), state)
	if got := m.since(mark); slices.Contains(got, "ask mesh-build-agent") {
		t.Errorf("a container nothing replaces was asked: %v", got)
	}
}

// Past the bound it is replaced anyway, the report carrying the command's last output — and the bound holds
// across a restart of the node-engine: the wait is read back from the state on disk.
func TestABusyContainerIsReplacedAtTheBoundEvenAfterARestart(t *testing.T) {
	now := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
	atClock(t, &now)
	m := newBusyMachine()
	_, state := applyBusy(t, m, twoContainers(t, "1", "1"), store.State{})
	m.answer(true, "building novox/mesh-host (a check that never ends)")
	_, state = applyBusy(t, m, twoContainers(t, "2", "1"), state)

	// The node-engine stops and starts again: what it knows is what it saved.
	path := filepath.Join(t.TempDir(), "state.json")
	if err := store.Save(path, state); err != nil {
		t.Fatal(err)
	}
	reloaded, err := store.Load(path)
	if err != nil {
		t.Fatal(err)
	}

	now = now.Add(14 * time.Minute)
	mark := m.mark()
	report, reloaded := applyBusy(t, m, twoContainers(t, "2", "1"), reloaded)
	if got := m.since(mark); slices.Contains(got, "rm mesh-build-agent") {
		t.Fatalf("a container busy within the bound was replaced after a restart: %v", got)
	}
	if b, _ := busyOf(report, "build-agent.server"); b.State != BusyWaiting || !b.Since.Equal(now.Add(-14*time.Minute)) {
		t.Fatalf("a restart started the wait again: %+v", report.Busy)
	}

	now = now.Add(time.Minute)
	mark = m.mark()
	report, _ = applyBusy(t, m, twoContainers(t, "2", "1"), reloaded)
	if got := m.since(mark); !slices.Equal(got, []string{"ask mesh-build-agent", "rm mesh-build-agent", "create mesh-build-agent"}) {
		t.Fatalf("a container still busy at the bound was not replaced: %v", got)
	}
	b, _ := busyOf(report, "build-agent.server")
	if b.State != BusyReplacedAtBound || !strings.Contains(b.Said, "never ends") {
		t.Fatalf("the report does not say it replaced a busy container, with its last output: %+v", report.Busy)
	}
	if o := outcomeOf(report, "build-agent.server"); !strings.Contains(o.Detail, "still busy") ||
		!strings.Contains(o.Detail, "never ends") {
		t.Errorf("the outcome does not say a busy container was replaced, with what it said: %s", o.Detail)
	}
}

// While one container waits, a send that changes another module on the same machine is applied at once: the
// apply returns after one ask, and holds nothing for the wait.
func TestASendToAnotherModuleIsAppliedWhileOneWaits(t *testing.T) {
	now := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
	atClock(t, &now)
	m := newBusyMachine()
	_, state := applyBusy(t, m, twoContainers(t, "1", "1"), store.State{})
	m.answer(true, "building")
	_, state = applyBusy(t, m, twoContainers(t, "2", "1"), state)

	// The next send moves only the web server.
	mark := m.mark()
	done := make(chan struct{})
	var report Report
	go func() {
		defer close(done)
		report, _ = applyBusy(t, m, twoContainers(t, "2", "2"), state)
	}()
	select {
	case <-done:
	case <-time.After(5 * time.Second):
		t.Fatal("an apply while a container waits did not return")
	}
	got := m.since(mark)
	if !slices.Contains(got, "create web") || slices.Contains(got, "rm mesh-build-agent") {
		t.Fatalf("the other module's send was not applied at once beside the wait: %v", got)
	}
	if o := outcomeOf(report, "web.server"); o.Action != "updated" {
		t.Errorf("the web server was reported %q: %+v", o.Action, o)
	}
}

// A container's busy is held to its shape where it arrives.
func TestBusyIsReadAndHeldToItsShape(t *testing.T) {
	d := twoContainers(t, "1", "1")
	c := d.Resources[0].(*declaration.Container)
	if c.Busy == nil || c.Busy.Command != "mesh-builder busy" || c.Busy.Within() != 5*time.Second {
		t.Fatalf("busy was not read: %+v", c.Busy)
	}
	for name, raw := range map[string]string{
		"no command":     `"busy":{"command":" "}`,
		"a long timeout": `"busy":{"command":"x","timeout":"5m"}`,
		"not a duration": `"busy":{"command":"x","timeout":"soon"}`,
		"an unknown key": `"busy":{"command":"x","every":"1m"}`,
		"on a step":      `"run-once":true,"busy":{"command":"x"}`,
	} {
		_, err := declaration.Parse([]byte(`{"declaration":1,"resources":[{"id":"a.b","type":"container",` +
			`"name":"b","image":"` + pinned + `",` + raw + `}]}`))
		if err == nil {
			t.Errorf("%s: a busy of that shape was accepted", name)
		}
	}
}