mesh-controller/cmd/mesh-builder/busy_test.go

package main

import (
	"os"
	"strings"
	"sync"
	"testing"
	"time"
)

// `mesh-builder busy` answers 0 while a build is in hand and non-zero when idle, and after its first ask the
// agent takes no new build request (novox/hq ADR 0335 §5).
func TestBusyAnswersWhileABuildIsInHandAndDrainsTheAgent(t *testing.T) {
	t.Setenv("TMPDIR", t.TempDir())
	if err := undrained(); err != nil {
		t.Fatal(err)
	}
	if drained() {
		t.Fatal("an agent never asked is drained")
	}
	// A build is claimed and runs.
	if !claim() {
		t.Fatal("an agent never asked refused a build")
	}
	holding("novox/mesh-controller (check-280)")
	said, err := busy()
	if err != nil || !strings.Contains(said, "novox/mesh-controller (check-280)") {
		t.Fatalf("an agent with a build in hand did not answer busy with what it holds: %q %v", said, err)
	}
	// From that ask on it takes nothing new: the fetch is not made, and one fetched is handed back.
	if !drained() {
		t.Fatal("the ask did not drain the agent")
	}
	if claim() {
		t.Fatal("a drained agent took a new build")
	}
	// The build in hand is still in hand after the refused claim, and busy goes on saying so.
	if _, err := busy(); err != nil {
		t.Fatalf("a refused claim ended the build in hand: %v", err)
	}
	// It ends: idle, and still drained.
	released()
	if _, err := busy(); err == nil {
		t.Fatal("an agent with nothing in hand answered busy")
	}
	if claim() {
		t.Fatal("a drained agent took a build after its last one ended")
	}
	// Started again, it is asked afresh.
	if err := undrained(); err != nil {
		t.Fatal(err)
	}
	if drained() || !claim() {
		t.Fatal("a holder started again is still drained")
	}
	released()
}

// An ask racing a fetch sees the build in hand, or the agent sees the drain and hands the request back: never
// neither, which would replace the container under a build just taken.
func TestAnAskRacingAFetchNeverMissesTheBuild(t *testing.T) {
	t.Setenv("TMPDIR", t.TempDir())
	for i := 0; i < 200; i++ {
		if err := undrained(); err != nil {
			t.Fatal(err)
		}
		var claimed bool
		var askErr error
		var wg sync.WaitGroup
		wg.Add(2)
		go func() { defer wg.Done(); claimed = claim() }()
		go func() { defer wg.Done(); _, askErr = busy() }()
		wg.Wait()
		if claimed && askErr != nil {
			t.Fatalf("round %d: the agent took a build and the ask said idle", i)
		}
		released()
	}
}

// A drain no longer asked lapses: an agent whose replacement was withdrawn after the first ask (a put-back, a
// revert, a stopped walk) takes work again after drainLapse, rather than staying drained with its health green.
func TestADrainNoLongerAskedLapses(t *testing.T) {
	t.Setenv("TMPDIR", t.TempDir())
	if err := undrained(); err != nil {
		t.Fatal(err)
	}
	if _, err := busy(); err == nil {
		t.Fatal("an idle agent answered busy")
	}
	if !drained() || claim() {
		t.Fatal("an agent just asked takes work")
	}
	// The last ask was three minutes ago.
	old := time.Now().Add(-3 * time.Minute)
	if err := os.Chtimes(drainMark(), old, old); err != nil {
		t.Fatal(err)
	}
	if drained() {
		t.Fatal("a drain no longer asked for three minutes still holds")
	}
	if !claim() {
		t.Fatal("an agent whose drain lapsed takes no work")
	}
	released()
}

// An agent drained, and asked all along, for longer than its node-engine's bound and the slack is not ready: the
// replacement it was drained for never came. One asked again after a lapse starts a new drain.
func TestAnAgentDrainedTooLongIsNotReady(t *testing.T) {
	t.Setenv("TMPDIR", t.TempDir())
	if err := undrained(); err != nil {
		t.Fatal(err)
	}
	taking("mesh-build-machine")
	if _, err := busy(); err == nil {
		t.Fatal("an idle agent answered busy")
	}
	now := time.Now()
	if err := drainedTooLong(now); err != nil {
		t.Fatalf("an agent drained a moment ago is not ready: %v", err)
	}
	// The drain began 25 minutes ago, and the node-engine asked a moment ago.
	began := now.Add(-25 * time.Minute).UTC().Format(time.RFC3339)
	if err := os.WriteFile(drainMark(), []byte(began+"\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	err := drainedTooLong(time.Now())
	if err == nil || !strings.Contains(err.Error(), "has not come") {
		t.Fatalf("an agent drained for 25 minutes reads ready: %v", err)
	}
	// Each ask keeps when the drain began.
	if _, err := busy(); err == nil {
		t.Fatal("an idle agent answered busy")
	}
	if err := drainedTooLong(time.Now()); err == nil {
		t.Fatal("an ask started the drain again")
	}
	// The asks stopped: the drain lapsed, and so did the fault.
	old := time.Now().Add(-3 * time.Minute)
	if err := os.Chtimes(drainMark(), old, old); err != nil {
		t.Fatal(err)
	}
	if err := drainedTooLong(time.Now()); err != nil {
		t.Fatalf("a lapsed drain still reads not ready: %v", err)
	}
}

// An ask this build does not know exits non-zero: an older image asked `busy` reads idle, not busy for the
// node-engine's whole bound.
func TestAnUnknownCommandFails(t *testing.T) {
	was := os.Args
	t.Cleanup(func() { os.Args = was })
	os.Args = []string{"mesh-builder", "some-later-word"}
	if err := run(); err == nil {
		t.Fatal("an unknown command exited 0")
	}
	os.Args = []string{"mesh-builder", "help"}
	if err := run(); err != nil {
		t.Fatalf("help failed: %v", err)
	}
}

// 1768/AC2: a mark naming no build of this agent (left by a crash or a fault) is stale: the next ask removes it and is
// run, never fetched and handed back in a loop.
func TestAStaleInHandMarkIsRemovedAndTheAskRuns(t *testing.T) {
	t.Setenv("TMPDIR", t.TempDir())
	inHand.Store(false)
	if err := undrained(); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(inHandMark(), []byte("2026-10-21T18:45:06Z\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	if !claim() {
		t.Fatal("a stale in-hand mark made the agent hand back an ask")
	}
	// A live one — this agent's own build in hand — is never taken over.
	if claim() {
		t.Fatal("a second build was taken while one is in hand")
	}
	if _, err := busy(); err != nil {
		t.Fatalf("the build in hand was lost: %v", err)
	}
	released()
}

// 1768/AC3: an agent that cannot mark a build in hand for another reason takes no work, instead of fetching asks and
// handing them back; it takes work again once marking works.
func TestAnAgentThatCannotMarkABuildTakesNoWorkUntilItCan(t *testing.T) {
	dir := t.TempDir()
	t.Setenv("TMPDIR", dir)
	inHand.Store(false)
	markFault, markSaid = time.Time{}, ""
	t.Cleanup(func() { markFault, markSaid = time.Time{}, "" })
	if err := undrained(); err != nil {
		t.Fatal(err)
	}
	// The mark's place is a directory that cannot be removed or made over: a directory named as the mark.
	if err := os.MkdirAll(inHandMark()+"/x", 0o755); err != nil {
		t.Fatal(err)
	}
	if claim() {
		t.Fatal("an agent that cannot mark a build took it")
	}
	if !markFaulty(time.Now()) {
		t.Fatal("an agent that cannot mark a build still takes work")
	}
	// Still failing after the retry time: still takes nothing, and no ask was needed to find out.
	if !markFaulty(time.Now().Add(2 * markRetry)) {
		t.Fatal("an agent whose mark still cannot be made takes work")
	}
	if err := os.RemoveAll(inHandMark()); err != nil {
		t.Fatal(err)
	}
	if markFaulty(time.Now().Add(4 * markRetry)) {
		t.Fatal("an agent whose mark can be made again takes no work")
	}
	if !claim() {
		t.Fatal("an agent that can mark takes no build")
	}
	released()
}