Let two controllers overlap safely while one hands over to the other (hq issue 213)

The controller's machine moves it from the container to a process by
starting the process first and removing the container once the process
is up (mesh-host's `replaces`). For that moment two controllers share the
store and the bus. Checked what each does:

- the seat's verbs: a queue group per seat, each call answered once. Safe.
- the controller's consumers on CONTROL and EVENTS: push consumers with
  no delivery group, so the second bind is refused with "consumer is
  already bound" and serve exited. The process would restart for ever,
  the host would never see it up, and the container would never go. The
  second controller now stands by and binds when the first lets go
  (tested on a real bus; fails without the change).
- plans: read, changed and saved whole by the 30s timer, by build
  outcomes, by a merge and by `plans stop`. Two timers would each ask a
  tier the other had just asked. Working the plans now takes a
  session-level advisory lock on the inventory: the timer skips while
  another holds it, the other paths wait for it. Build asks happen only
  inside plan work and are covered by the same lock.

cmd/mesh-controller/plans_one_at_a_time_test.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// novox/hq issue 213: for the moment a machine hands its controller over, the container and the
+// process both run the plan timer on one store. Only the one holding the plans moves them; the other
+// leaves them alone, and moves them once they are let go.
+func TestAControllerLeavesThePlansToTheOneHoldingThem(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	now := time.Now().UTC()
+	// Every tier done: the next step is the plan's last, and needs nothing but the store.
+	plan := inventory.Plan{ID: "plan-213", Repository: "r", Commit: "abc", Created: now, Updated: now,
+		State: inventory.PlanRolling, Tier: 1, Tiers: [][]string{{"app"}},
+		Modules: map[string]*inventory.PlanModule{"app": {State: "built"}}}
+	if err := open.inventory.SavePlan(ctx, plan); err != nil {
+		t.Fatal(err)
+	}
+
+	// The other controller: its own connections to the same store, holding the plans.
+	other, err := inventory.Open(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(other.Close)
+	release, err := other.HoldPlans(ctx, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(release) // before the close above: a pool waits for a connection still held
+
+	advancePlans(ctx, open)
+	if p, err := open.inventory.PlanByID(ctx, "plan-213"); err != nil || !p.Open() {
+		t.Fatalf("a controller moved a plan another held: %+v %v", p, err)
+	}
+
+	release()
+	advancePlans(ctx, open)
+	if p, err := open.inventory.PlanByID(ctx, "plan-213"); err != nil || p.State != inventory.PlanDone {
+		t.Fatalf("the plan did not move once it was let go: %+v %v", p, err)
+	}
+}

cmd/mesh-controller/release_plan.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"flag"
 	"fmt"
 	"sort"
@@ -296,6 +297,15 @@ func askTier(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) e
 // build's request time is not known, and such an outcome is taken as before.
 func planBuilt(ctx context.Context, open *stores, module, commit, failed string, asked time.Time) {
 	inv := open.inventory
+	// One controller works the plans at a time (novox/hq issue 213); an outcome waits its turn rather
+	// than write over what the holder is about to save. Not taken, it is still in the build records,
+	// which the holder settles the plan from (issue 214).
+	release, err := inv.HoldPlans(ctx, true)
+	if err != nil {
+		fmt.Printf("plans: %s's outcome is left to the build records: %v\n", module, err)
+		return
+	}
+	defer release()
 	plans, err := inv.OpenPlans(ctx)
 	if err != nil {
 		fmt.Printf("plans: cannot read them: %v\n", err)
@@ -342,13 +352,30 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string,
 			fmt.Printf("%s: %s; the tiers after it are not asked\n", p.ID, p.Note)
 		}
 	}
-	advancePlans(ctx, open)
+	advanceHeld(ctx, open)
 }
 
 // advancePlans moves every open plan as far as the facts allow: a tier whose modules are all built
 // and whose gates are applied gives way to the next; the last tier done is the plan done. Called
 // after every outcome and on a timer, so a plan waiting on a machine's report moves when it comes.
+//
+// **One controller at a time** (novox/hq issue 213). A plan is read, changed and saved whole; two
+// controllers — the old and the new while a machine hands its controller over — would each ask a
+// tier the other had just asked. Taken without waiting: whoever holds the plans is moving them.
 func advancePlans(ctx context.Context, open *stores) {
+	release, err := open.inventory.HoldPlans(ctx, false)
+	if err != nil {
+		if !errors.Is(err, inventory.ErrPlansBusy) {
+			fmt.Printf("plans: cannot hold them: %v\n", err)
+		}
+		return
+	}
+	defer release()
+	advanceHeld(ctx, open)
+}
+
+// advanceHeld is advancePlans for a caller already holding the plans.
+func advanceHeld(ctx context.Context, open *stores) {
 	inv := open.inventory
 	plans, err := inv.OpenPlans(ctx)
 	if err != nil {
@@ -676,6 +703,19 @@ func plansCommand(ctx context.Context, args []string) error {
 		}
 		p.State = inventory.PlanFailed
 		p.Note = "stopped by hand at tier " + fmt.Sprint(p.Tier)
+		release, err := inv.HoldPlans(ctx, true)
+		if err != nil {
+			return err
+		}
+		defer release()
+		if p, err = inv.PlanByID(ctx, positionals[1]); err != nil {
+			return err
+		}
+		if !p.Open() {
+			return fmt.Errorf("%s is already %s", p.ID, p.State)
+		}
+		p.State = inventory.PlanFailed
+		p.Note = "stopped by hand at tier " + fmt.Sprint(p.Tier)
 		if err := inv.SavePlan(ctx, p); err != nil {
 			return err
 		}

cmd/mesh-controller/upgrades.go

@@ -315,6 +315,13 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
 	for _, e := range moved {
 		movedNames = append(movedNames, e.Manifest.Module)
 	}
+	// Written and its first tier asked as one act on the plans (novox/hq issue 213): a timer on
+	// another controller reading it between the two would ask the tier again.
+	release, err := inv.HoldPlans(ctx, true)
+	if err != nil {
+		return notNow(err)
+	}
+	defer release()
 	plan := planOfMerge(m, movedNames, edges)
 	if hasCycle(plan.Tiers, edges) {
 		fmt.Printf("  the last tier depends on itself: %s — built together, in no order\n",