Let a drain lapse, bound the gate's wait on the controller's clock, keep a busy wait out of apply time

Review of #283: a drain whose replacement was withdrawn held the agent for
ever with its health green; the gate trusted the engine's until; a machine
of the rest waiting at the bound read as silent and its apply time held the
wait; an unknown mesh-builder word exited 0; plan said nothing of a busy an
older engine is not sent; and a node with a report and no send broke the
reading of every last report (the store test's NULL).

cmd/mesh-builder/busy_test.go

@@ -1,9 +1,11 @@
 package main
 
 import (
+	"os"
 	"strings"
 	"sync"
 	"testing"
+	"time"
 )
 
 // `mesh-builder busy` answers 0 while a build is in hand and non-zero when idle, and after its first ask the
@@ -75,3 +77,86 @@ func TestAnAskRacingAFetchNeverMissesTheBuild(t *testing.T) {
 		released()
 	}
 }
+
+// A drain no longer asked lapses: an agent whose replacement was withdrawn after the first ask (a put-back, a
+// revert, a stopped walk) takes work again after drainLapse, rather than staying drained with its health green.
+func TestADrainNoLongerAskedLapses(t *testing.T) {
+	t.Setenv("TMPDIR", t.TempDir())
+	if err := undrained(); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := busy(); err == nil {
+		t.Fatal("an idle agent answered busy")
+	}
+	if !drained() || claim() {
+		t.Fatal("an agent just asked takes work")
+	}
+	// The last ask was three minutes ago.
+	old := time.Now().Add(-3 * time.Minute)
+	if err := os.Chtimes(drainMark(), old, old); err != nil {
+		t.Fatal(err)
+	}
+	if drained() {
+		t.Fatal("a drain no longer asked for three minutes still holds")
+	}
+	if !claim() {
+		t.Fatal("an agent whose drain lapsed takes no work")
+	}
+	released()
+}
+
+// An agent drained, and asked all along, for longer than its node-engine's bound and the slack is not ready: the
+// replacement it was drained for never came. One asked again after a lapse starts a new drain.
+func TestAnAgentDrainedTooLongIsNotReady(t *testing.T) {
+	t.Setenv("TMPDIR", t.TempDir())
+	if err := undrained(); err != nil {
+		t.Fatal(err)
+	}
+	taking("mesh-build-machine")
+	if _, err := busy(); err == nil {
+		t.Fatal("an idle agent answered busy")
+	}
+	now := time.Now()
+	if err := drainedTooLong(now); err != nil {
+		t.Fatalf("an agent drained a moment ago is not ready: %v", err)
+	}
+	// The drain began 25 minutes ago, and the node-engine asked a moment ago.
+	began := now.Add(-25 * time.Minute).UTC().Format(time.RFC3339)
+	if err := os.WriteFile(drainMark(), []byte(began+"\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	err := drainedTooLong(time.Now())
+	if err == nil || !strings.Contains(err.Error(), "has not come") {
+		t.Fatalf("an agent drained for 25 minutes reads ready: %v", err)
+	}
+	// Each ask keeps when the drain began.
+	if _, err := busy(); err == nil {
+		t.Fatal("an idle agent answered busy")
+	}
+	if err := drainedTooLong(time.Now()); err == nil {
+		t.Fatal("an ask started the drain again")
+	}
+	// The asks stopped: the drain lapsed, and so did the fault.
+	old := time.Now().Add(-3 * time.Minute)
+	if err := os.Chtimes(drainMark(), old, old); err != nil {
+		t.Fatal(err)
+	}
+	if err := drainedTooLong(time.Now()); err != nil {
+		t.Fatalf("a lapsed drain still reads not ready: %v", err)
+	}
+}
+
+// An ask this build does not know exits non-zero: an older image asked `busy` reads idle, not busy for the
+// node-engine's whole bound.
+func TestAnUnknownCommandFails(t *testing.T) {
+	was := os.Args
+	t.Cleanup(func() { os.Args = was })
+	os.Args = []string{"mesh-builder", "some-later-word"}
+	if err := run(); err == nil {
+		t.Fatal("an unknown command exited 0")
+	}
+	os.Args = []string{"mesh-builder", "help"}
+	if err := run(); err != nil {
+		t.Fatalf("help failed: %v", err)
+	}
+}