package main import ( "context" "errors" "testing" "time" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/lease" "github.com/novox/mesh-controller/internal/testbus" ) // Two controllers at once (novox/hq to-be 45 §6, issue 204; the half of replay R1 that lives here): two // serving controllers over one store and one bus. The second waits while the first holds the lease; on // a handover it takes it at a higher epoch, the record says which held what and how each ended; and the // one that lost it acts no more — no declaration composed, no plan and no condition written — the // moment it lost it. // // MESH_TEST_POSTGRES=… go test ./cmd/mesh-controller/ -run Controllers (each test on a bus of its own: internal/testbus) // aBusForTheLease is the test bus with the controller's lease bucket new. func aBusForTheLease(t *testing.T) (string, jetstream.KeyValue) { t.Helper() url := testbus.URL(t) conn, err := nats.Connect(url) if err != nil { t.Fatal(err) } t.Cleanup(conn.Close) js, err := jetstream.New(conn) if err != nil { t.Fatal(err) } _ = js.DeleteKeyValue(t.Context(), broker.LeaseBucket) if err := broker.EnsureLeaseBucket(t.Context(), js); err != nil { t.Fatal(err) } kv, err := js.KeyValue(t.Context(), broker.LeaseBucket) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = js.DeleteKeyValue(context.Background(), broker.LeaseBucket) }) return url, kv } func TestTwoControllersOneActs(t *testing.T) { url, kv := aBusForTheLease(t) inv := inventory.ForTest(t) ctx := t.Context() // Controller A takes the lease. a := &actor{} aCtx, stopA := context.WithCancel(ctx) defer stopA() lostA, err := a.serveUnderTheLease(aCtx, inv, url) if err != nil { t.Fatal(err) } epochA, err := a.epoch(ctx) if err != nil || epochA == 0 { t.Fatalf("A holds no epoch: %d, %v", epochA, err) } // Controller B starts while A holds it, and waits — acting on nothing meanwhile. b := &actor{} bCtx, stopB := context.WithCancel(ctx) defer stopB() tookB := make(chan (<-chan struct{}), 1) go func() { lost, err := b.serveUnderTheLease(bCtx, inv, url) if err != nil { t.Errorf("B: %v", err) close(tookB) return } tookB <- lost }() select { case <-tookB: t.Fatal("B took the lease while A held it") case <-time.After(3 * time.Second): } if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) { t.Fatalf("B, waiting, may act: %v", err) } // A hands over, as a controller being replaced does: B takes the lease at once, at a higher epoch. stopA() a.release() var lostB <-chan struct{} select { case lostB = <-tookB: case <-time.After(10 * time.Second): t.Fatal("B did not take the lease A gave back") } select { case <-lostA: default: t.Fatal("A, having given the lease back, is not told it no longer holds it") } epochB, err := b.epoch(ctx) if err != nil || epochB <= epochA { t.Fatalf("B acts as epoch %d after A's %d (%v): an epoch only grows", epochB, epochA, err) } if _, err := a.epoch(ctx); err == nil { t.Fatal("A acts after giving the lease back") } ea, _, _ := inv.EpochOf(ctx, epochA) eb, _, _ := inv.EpochOf(ctx, epochB) if ea.How != inventory.EpochReleased || eb.Ended != nil || eb.Instance != instance { t.Fatalf("the record of the handover reads %+v then %+v", ea, eb) } // Something else writes the lease's key — a third controller on a clock that read it as expired: // B's next renewal is refused, and B stops acting at once. if _, err := kv.Put(ctx, lease.Key, []byte(`{"instance":"a third controller","epoch":1}`)); err != nil { t.Fatal(err) } select { case <-lostB: case <-time.After(2 * lease.RenewEvery): t.Fatal("B was not told it lost the lease") } if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) { t.Fatalf("B acts after losing the lease: %v", err) } // Nothing B does is written: a declaration's number is not taken, a plan is not saved, a condition // is not raised. inv.ActsUnder(b.epoch) if _, err := inv.AddNode(ctx, "anchor"); err != nil { t.Fatal(err) } saved := inventory.Plan{ID: "plan-after-loss", Repository: "novox/app", Commit: "c0ffee00", Created: time.Now(), State: inventory.PlanBuilding} if err := inv.SavePlan(ctx, &saved); err == nil { t.Fatal("B wrote a plan after losing the lease") } store := conditions.NewInMemory() keeper := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store, Epoch: func() (uint64, error) { return b.epoch(ctx) }}) defer keeper.Close(context.Background()) if _, err := keeper.Observe(ctx, conditions.Observation{Scope: conditions.ScopeCore, ID: "x", Kind: "x", Severity: conditions.Warning, Summary: "x", Source: "test"}); err == nil { t.Fatal("B raised a condition after losing the lease") } if ended, _, _ := inv.EpochOf(ctx, epochB); ended.How != inventory.EpochLost { t.Fatalf("B's epoch does not say it was lost: %+v", ended) } } // A controller whose bus refuses it the lease's key, with nobody holding it, serves without the lease: // it acts with no epoch — refused by no node-engine — says so, and takes the lease once it can. func TestAControllerTheBusRefusesTheLeaseServesUnleasedAndSaysSo(t *testing.T) { a := &actor{serving: true, unleased: "the bus refused the lease's key"} if epoch, err := a.epoch(context.Background()); err != nil || epoch != 0 { t.Fatalf("an unleased controller answers %d, %v: it acts, claiming no epoch", epoch, err) } if st := a.standing(); st.Unleased == "" || st.Held { t.Fatalf("its standing says %+v", st) } // One that neither holds nor is unleased — still waiting — acts on nothing. waiting := &actor{serving: true} if _, err := waiting.epoch(context.Background()); !errors.Is(err, lease.ErrNotHeld) { t.Fatalf("a controller waiting for the lease may act: %v", err) } }