Let two controllers overlap safely while one hands over to the other (hq issue 213)

The controller's machine moves it from the container to a process by
starting the process first and removing the container once the process
is up (mesh-host's `replaces`). For that moment two controllers share the
store and the bus. Checked what each does:

- the seat's verbs: a queue group per seat, each call answered once. Safe.
- the controller's consumers on CONTROL and EVENTS: push consumers with
  no delivery group, so the second bind is refused with "consumer is
  already bound" and serve exited. The process would restart for ever,
  the host would never see it up, and the container would never go. The
  second controller now stands by and binds when the first lets go
  (tested on a real bus; fails without the change).
- plans: read, changed and saved whole by the 30s timer, by build
  outcomes, by a merge and by `plans stop`. Two timers would each ask a
  tier the other had just asked. Working the plans now takes a
  session-level advisory lock on the inventory: the timer skips while
  another holds it, the other paths wait for it. Build asks happen only
  inside plan work and are covered by the same lock.

internal/inventory/hold.go

@@ -87,3 +87,62 @@ func (i *Inventory) tryHold(ctx context.Context, sorted []string) (func(), strin
 	}
 	return release, "", nil
 }
+
+// ErrPlansBusy is the plans held by another act — on a machine replacing its controller, the other
+// controller — for longer than a caller waits, or at all for one that does not wait.
+var ErrPlansBusy = errors.New("another controller is working the plans")
+
+// HoldPlans makes working the plans one act at a time, across every controller on the store
+// (novox/hq issue 213). A plan is read, changed and written whole; two controllers doing that at
+// once — the old and the new for the moment a machine hands its controller over, or a controller
+// and a person's `plans stop` — each act on what the other has not saved yet: a tier asked twice,
+// an outcome written over. A session-level advisory lock on one connection, released by the
+// returned function and by the session ending, so a controller that dies holding it holds nothing.
+//
+// wait false gives ErrPlansBusy at once when another holds them — the timer's way: the holder is
+// moving the plans already. wait true looks again every HoldPoll for up to HoldWaitFor — an
+// outcome's or a merge's way, which must be written.
+func (i *Inventory) HoldPlans(ctx context.Context, wait bool) (func(), error) {
+	deadline := time.Now().Add(HoldWaitFor)
+	for {
+		release, took, err := i.tryLock(ctx, "mesh-plans")
+		if err != nil || took {
+			return release, err
+		}
+		if !wait || time.Now().After(deadline) {
+			return nil, ErrPlansBusy
+		}
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case <-time.After(HoldPoll):
+		}
+	}
+}
+
+// tryLock takes one named advisory lock on a connection of its own, or gives the connection back.
+func (i *Inventory) tryLock(ctx context.Context, key string) (func(), bool, error) {
+	conn, err := i.store.Pool().Acquire(ctx)
+	if err != nil {
+		return nil, false, err
+	}
+	var once sync.Once
+	release := func() {
+		once.Do(func() {
+			if _, err := conn.Exec(context.WithoutCancel(ctx), `select pg_advisory_unlock_all()`); err != nil {
+				_ = conn.Conn().Close(context.WithoutCancel(ctx))
+			}
+			conn.Release()
+		})
+	}
+	var took bool
+	if err := conn.QueryRow(ctx, `select pg_try_advisory_lock(hashtext($1)::bigint)`, key).Scan(&took); err != nil {
+		release()
+		return nil, false, err
+	}
+	if !took {
+		release()
+		return nil, false, nil
+	}
+	return release, true, nil
+}

internal/inventory/hold_plans_test.go

@@ -0,0 +1,38 @@
+package inventory
+
+import (
+	"errors"
+	"testing"
+	"time"
+)
+
+// novox/hq issue 213: while a machine hands its controller over from the container to the process,
+// two controllers run on one store for a moment. Working the plans is one act at a time across them.
+func TestThePlansAreWorkedByOneControllerAtATime(t *testing.T) {
+	first := ForTest(t)
+	// A second controller: its own connections to the same store.
+	second, err := Open(t.Context())
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(second.Close)
+
+	release, err := first.HoldPlans(t.Context(), false)
+	if err != nil {
+		t.Fatalf("the plans could not be held when nobody held them: %v", err)
+	}
+	t.Cleanup(release) // a pool closing waits for a connection still held; release is idempotent
+	if _, err := second.HoldPlans(t.Context(), false); !errors.Is(err, ErrPlansBusy) {
+		t.Fatalf("a second controller held the plans while the first did: %v", err)
+	}
+	// A waiter gets them once they are let go.
+	was := HoldPoll
+	HoldPoll = 10 * time.Millisecond
+	defer func() { HoldPoll = was }()
+	go func() { time.Sleep(50 * time.Millisecond); release() }()
+	again, err := second.HoldPlans(t.Context(), true)
+	if err != nil {
+		t.Fatalf("a waiting controller never got the plans once they were let go: %v", err)
+	}
+	again()
+}