Let a drain lapse, bound the gate's wait on the controller's clock, keep a busy wait out of apply time

Review of #283: a drain whose replacement was withdrawn held the agent for
ever with its health green; the gate trusted the engine's until; a machine
of the rest waiting at the bound read as silent and its apply time held the
wait; an unknown mesh-builder word exited 0; plan said nothing of a busy an
older engine is not sent; and a node with a report and no send broke the
reading of every last report (the store test's NULL).

cmd/mesh-builder/busy.go

@@ -28,10 +28,49 @@ import (
 func inHandMark() string { return filepath.Join(os.TempDir(), "mesh-builder.in-hand") }
 func drainMark() string  { return filepath.Join(os.TempDir(), "mesh-builder.drained") }
 
-// drained says the agent was asked whether it is busy: it takes no new build request.
-func drained() bool {
-	_, err := os.Stat(drainMark())
-	return err == nil
+// The drain lapses when no ask has come for drainLapse: the node-engine asks every 20 seconds for as long as it
+// really waits, so an agent whose replacement was withdrawn — the declaration put back, the walk stopped —
+// takes work again rather than staying drained for ever (review of ADR 0335). And an agent drained for longer
+// than the node-engine's bound and the slack says it is not ready: the replacement it was drained for never
+// came, which is a fault to see.
+const (
+	drainLapse   = 2 * time.Minute
+	drainTooLong = 15*time.Minute + 5*time.Minute
+)
+
+// drainedAt is when the drain began and when it was last asked, from its mark; false when there is no mark.
+func drainedAt() (began, asked time.Time, ok bool) {
+	info, err := os.Stat(drainMark())
+	if err != nil {
+		return time.Time{}, time.Time{}, false
+	}
+	asked = info.ModTime()
+	began = asked
+	if raw, err := os.ReadFile(drainMark()); err == nil {
+		if t, err := time.Parse(time.RFC3339, strings.TrimSpace(string(raw))); err == nil && !t.After(asked) {
+			began = t
+		}
+	}
+	return began, asked, true
+}
+
+// drained says the agent was asked whether it is busy within drainLapse: it takes no new build request.
+func drained() bool { return drainedSince(time.Now()) }
+
+func drainedSince(now time.Time) bool {
+	_, asked, ok := drainedAt()
+	return ok && now.Sub(asked) < drainLapse
+}
+
+// drainedTooLong is the health check's second half: nil unless the agent has been drained, asked all along, for
+// longer than its node-engine's bound and the slack.
+func drainedTooLong(now time.Time) error {
+	began, asked, ok := drainedAt()
+	if !ok || now.Sub(asked) >= drainLapse || now.Sub(began) <= drainTooLong {
+		return nil
+	}
+	return fmt.Errorf("drained since %s and still asked: the replacement its node-engine waits for has not come "+
+		"within %s (novox/hq ADR 0335)", began.UTC().Format(time.RFC3339), drainTooLong)
 }
 
 // claim marks a request in hand, unless the agent is drained: then nothing is in hand, and the request goes
@@ -85,7 +124,13 @@ func undrained() error {
 
 // busy is the ask: it drains the agent, then says whether a build is in hand. Nil — exit 0 — while one is.
 func busy() (string, error) {
-	if err := os.WriteFile(drainMark(), []byte(time.Now().UTC().Format(time.RFC3339)+"\n"), 0o644); err != nil {
+	// The mark keeps when the drain began while it is asked within drainLapse, and is written again on every
+	// ask: its time is the last ask.
+	began := time.Now().UTC()
+	if was, _, ok := drainedAt(); ok && drained() {
+		began = was.UTC()
+	}
+	if err := os.WriteFile(drainMark(), []byte(began.Format(time.RFC3339)+"\n"), 0o644); err != nil {
 		// Not drained, it could take the next build: said, and answered busy only for what is in hand now.
 		fmt.Fprintf(os.Stderr, "cannot mark this agent drained (%s): %v\n", drainMark(), err)
 	}

cmd/mesh-builder/busy_test.go

@@ -1,9 +1,11 @@
 package main
 
 import (
+	"os"
 	"strings"
 	"sync"
 	"testing"
+	"time"
 )
 
 // `mesh-builder busy` answers 0 while a build is in hand and non-zero when idle, and after its first ask the
@@ -75,3 +77,86 @@ func TestAnAskRacingAFetchNeverMissesTheBuild(t *testing.T) {
 		released()
 	}
 }
+
+// A drain no longer asked lapses: an agent whose replacement was withdrawn after the first ask (a put-back, a
+// revert, a stopped walk) takes work again after drainLapse, rather than staying drained with its health green.
+func TestADrainNoLongerAskedLapses(t *testing.T) {
+	t.Setenv("TMPDIR", t.TempDir())
+	if err := undrained(); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := busy(); err == nil {
+		t.Fatal("an idle agent answered busy")
+	}
+	if !drained() || claim() {
+		t.Fatal("an agent just asked takes work")
+	}
+	// The last ask was three minutes ago.
+	old := time.Now().Add(-3 * time.Minute)
+	if err := os.Chtimes(drainMark(), old, old); err != nil {
+		t.Fatal(err)
+	}
+	if drained() {
+		t.Fatal("a drain no longer asked for three minutes still holds")
+	}
+	if !claim() {
+		t.Fatal("an agent whose drain lapsed takes no work")
+	}
+	released()
+}
+
+// An agent drained, and asked all along, for longer than its node-engine's bound and the slack is not ready: the
+// replacement it was drained for never came. One asked again after a lapse starts a new drain.
+func TestAnAgentDrainedTooLongIsNotReady(t *testing.T) {
+	t.Setenv("TMPDIR", t.TempDir())
+	if err := undrained(); err != nil {
+		t.Fatal(err)
+	}
+	taking("mesh-build-machine")
+	if _, err := busy(); err == nil {
+		t.Fatal("an idle agent answered busy")
+	}
+	now := time.Now()
+	if err := drainedTooLong(now); err != nil {
+		t.Fatalf("an agent drained a moment ago is not ready: %v", err)
+	}
+	// The drain began 25 minutes ago, and the node-engine asked a moment ago.
+	began := now.Add(-25 * time.Minute).UTC().Format(time.RFC3339)
+	if err := os.WriteFile(drainMark(), []byte(began+"\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	err := drainedTooLong(time.Now())
+	if err == nil || !strings.Contains(err.Error(), "has not come") {
+		t.Fatalf("an agent drained for 25 minutes reads ready: %v", err)
+	}
+	// Each ask keeps when the drain began.
+	if _, err := busy(); err == nil {
+		t.Fatal("an idle agent answered busy")
+	}
+	if err := drainedTooLong(time.Now()); err == nil {
+		t.Fatal("an ask started the drain again")
+	}
+	// The asks stopped: the drain lapsed, and so did the fault.
+	old := time.Now().Add(-3 * time.Minute)
+	if err := os.Chtimes(drainMark(), old, old); err != nil {
+		t.Fatal(err)
+	}
+	if err := drainedTooLong(time.Now()); err != nil {
+		t.Fatalf("a lapsed drain still reads not ready: %v", err)
+	}
+}
+
+// An ask this build does not know exits non-zero: an older image asked `busy` reads idle, not busy for the
+// node-engine's whole bound.
+func TestAnUnknownCommandFails(t *testing.T) {
+	was := os.Args
+	t.Cleanup(func() { os.Args = was })
+	os.Args = []string{"mesh-builder", "some-later-word"}
+	if err := run(); err == nil {
+		t.Fatal("an unknown command exited 0")
+	}
+	os.Args = []string{"mesh-builder", "help"}
+	if err := run(); err != nil {
+		t.Fatalf("help failed: %v", err)
+	}
+}

cmd/mesh-builder/main.go

@@ -92,8 +92,12 @@ func run() error {
 		case "build":
 			return buildOnce(context.Background(), os.Args[2:])
 		case "ready":
-			// The container's health check (novox/hq issue 544): exits non-zero until the agent takes work.
-			return ready()
+			// The container's health check (novox/hq issue 544): exits non-zero until the agent takes work, and
+			// once it has been drained longer than its node-engine would wait to replace it (ADR 0335).
+			if err := ready(); err != nil {
+				return err
+			}
+			return drainedTooLong(time.Now())
 		case "busy":
 			// The container's busy command (novox/hq ADR 0335): drains the agent, and exits non-zero once
 			// nothing is in hand.
@@ -103,9 +107,14 @@ func run() error {
 			}
 			fmt.Println(said)
 			return nil
-		default:
+		case "help", "-h", "--help":
 			fmt.Print(usage)
 			return nil
+		default:
+			// **An ask this build does not know fails** (novox/hq ADR 0335): an image older than `busy`, asked
+			// it, must read as idle — exiting 0 would hold its container for the node-engine's whole bound.
+			fmt.Fprint(os.Stderr, usage)
+			return fmt.Errorf("%q is not a command this mesh-builder knows", os.Args[1])
 		}
 	}