Act under a lease, keep accounts by order, one writer at composition (hq to-be 45 Phase 2)
Two controllers could both act (issue 204), a reconcile's report could overtake the apply after it and the digest decided (issue 267), and a grant could make a second writer of a machine's report. - The lease (internal/lease, ADR 0229): mesh-controller_lease key `holder`, 15 s age, renewed every 5 s by compare-and-set; the epoch is the revision it was taken at. The gate is the clock (stops 3 s before expiry); a refused renewal is a loss and the process exits; a holder that stops gives it back. serve takes it before asserting the bus. Epochs kept in the store (migration 0068 controller_epoch) as a floor: a bucket raised from nothing is compacted past it. Unleased (no epoch, S12 urgent) only when nobody holds it and the bus will not let it be written. A shell command acts under the holder's epoch, or its own lease when none. - Declarations carry `epoch` inside the signed envelope, only to a machine whose latest account carried a report_sequence (mesh-host #35); would-send is composed with the epoch last sent. Allot and the send both pass the gate. - Reports: contract in internal/link/order.go (epoch, sequence, report_sequence, older_than, refused_older). Accounts kept by epoch, then sequence, then report sequence; older refused, counted; unordered reports keep the digest rule. Plans by compare-and-set on a revision, with epoch. Conditions and calls carry the epoch and are not written off the lease. - S12 and S13 (naming the writer by epoch) watched, D5 run; reset of the bucket said. Writers table compiled in and enforced in PermissionsFor; the controller no longer publishes mesh.control.>. A contract per consumed kind, and the empty-on-error lint over the repository. - mesh-host pinned to its main with the epoch in the validator (D1 validates the envelope as sent). Needs mesh-host's genesis lock with the lease grant (mesh-host PR) for TestTheInstallersFirstUserListIsWhatTheControllerWouldCompose.
This commit is contained in:
jochen
@@ -0,0 +1,175 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
)
|
||||
|
||||
// Two controllers at once (novox/hq to-be 45 §6, issue 204; the half of replay R1 that lives here): two
|
||||
// serving controllers over one store and one bus. The second waits while the first holds the lease; on
|
||||
// a handover it takes it at a higher epoch, the record says which held what and how each ended; and the
|
||||
// one that lost it acts no more — no declaration composed, no plan and no condition written — the
|
||||
// moment it lost it.
|
||||
//
|
||||
// MESH_TEST_POSTGRES=… MESH_TEST_NATS=nats://127.0.0.1:14222 go test ./cmd/mesh-controller/ -run Controllers
|
||||
|
||||
// aBusForTheLease is the test bus with the controller's lease bucket new.
|
||||
func aBusForTheLease(t *testing.T) (string, jetstream.KeyValue) {
|
||||
t.Helper()
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(conn.Close)
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = js.DeleteKeyValue(t.Context(), broker.LeaseBucket)
|
||||
if err := broker.EnsureLeaseBucket(t.Context(), js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv, err := js.KeyValue(t.Context(), broker.LeaseBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = js.DeleteKeyValue(context.Background(), broker.LeaseBucket) })
|
||||
return url, kv
|
||||
}
|
||||
|
||||
func TestTwoControllersOneActs(t *testing.T) {
|
||||
url, kv := aBusForTheLease(t)
|
||||
inv := inventory.ForTest(t)
|
||||
ctx := t.Context()
|
||||
|
||||
// Controller A takes the lease.
|
||||
a := &actor{}
|
||||
aCtx, stopA := context.WithCancel(ctx)
|
||||
defer stopA()
|
||||
lostA, err := a.serveUnderTheLease(aCtx, inv, url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
epochA, err := a.epoch(ctx)
|
||||
if err != nil || epochA == 0 {
|
||||
t.Fatalf("A holds no epoch: %d, %v", epochA, err)
|
||||
}
|
||||
|
||||
// Controller B starts while A holds it, and waits — acting on nothing meanwhile.
|
||||
b := &actor{}
|
||||
bCtx, stopB := context.WithCancel(ctx)
|
||||
defer stopB()
|
||||
tookB := make(chan (<-chan struct{}), 1)
|
||||
go func() {
|
||||
lost, err := b.serveUnderTheLease(bCtx, inv, url)
|
||||
if err != nil {
|
||||
t.Errorf("B: %v", err)
|
||||
close(tookB)
|
||||
return
|
||||
}
|
||||
tookB <- lost
|
||||
}()
|
||||
select {
|
||||
case <-tookB:
|
||||
t.Fatal("B took the lease while A held it")
|
||||
case <-time.After(3 * time.Second):
|
||||
}
|
||||
if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) {
|
||||
t.Fatalf("B, waiting, may act: %v", err)
|
||||
}
|
||||
|
||||
// A hands over, as a controller being replaced does: B takes the lease at once, at a higher epoch.
|
||||
stopA()
|
||||
a.release()
|
||||
var lostB <-chan struct{}
|
||||
select {
|
||||
case lostB = <-tookB:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("B did not take the lease A gave back")
|
||||
}
|
||||
select {
|
||||
case <-lostA:
|
||||
default:
|
||||
t.Fatal("A, having given the lease back, is not told it no longer holds it")
|
||||
}
|
||||
epochB, err := b.epoch(ctx)
|
||||
if err != nil || epochB <= epochA {
|
||||
t.Fatalf("B acts as epoch %d after A's %d (%v): an epoch only grows", epochB, epochA, err)
|
||||
}
|
||||
if _, err := a.epoch(ctx); err == nil {
|
||||
t.Fatal("A acts after giving the lease back")
|
||||
}
|
||||
ea, _, _ := inv.EpochOf(ctx, epochA)
|
||||
eb, _, _ := inv.EpochOf(ctx, epochB)
|
||||
if ea.How != inventory.EpochReleased || eb.Ended != nil || eb.Instance != instance {
|
||||
t.Fatalf("the record of the handover reads %+v then %+v", ea, eb)
|
||||
}
|
||||
|
||||
// Something else writes the lease's key — a third controller on a clock that read it as expired:
|
||||
// B's next renewal is refused, and B stops acting at once.
|
||||
if _, err := kv.Put(ctx, lease.Key, []byte(`{"instance":"a third controller","epoch":1}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-lostB:
|
||||
case <-time.After(2 * lease.RenewEvery):
|
||||
t.Fatal("B was not told it lost the lease")
|
||||
}
|
||||
if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) {
|
||||
t.Fatalf("B acts after losing the lease: %v", err)
|
||||
}
|
||||
// Nothing B does is written: a declaration's number is not taken, a plan is not saved, a condition
|
||||
// is not raised.
|
||||
inv.ActsUnder(b.epoch)
|
||||
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saved := inventory.Plan{ID: "plan-after-loss", Repository: "novox/app", Commit: "c0ffee00", Created: time.Now(),
|
||||
State: inventory.PlanBuilding}
|
||||
if err := inv.SavePlan(ctx, &saved); err == nil {
|
||||
t.Fatal("B wrote a plan after losing the lease")
|
||||
}
|
||||
store := conditions.NewInMemory()
|
||||
keeper := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store,
|
||||
Epoch: func() (uint64, error) { return b.epoch(ctx) }})
|
||||
defer keeper.Close(context.Background())
|
||||
if _, err := keeper.Observe(ctx, conditions.Observation{Scope: conditions.ScopeCore, ID: "x", Kind: "x",
|
||||
Severity: conditions.Warning, Summary: "x", Source: "test"}); err == nil {
|
||||
t.Fatal("B raised a condition after losing the lease")
|
||||
}
|
||||
if ended, _, _ := inv.EpochOf(ctx, epochB); ended.How != inventory.EpochLost {
|
||||
t.Fatalf("B's epoch does not say it was lost: %+v", ended)
|
||||
}
|
||||
}
|
||||
|
||||
// A controller whose bus refuses it the lease's key, with nobody holding it, serves without the lease:
|
||||
// it acts with no epoch — refused by no node-engine — says so, and takes the lease once it can.
|
||||
func TestAControllerTheBusRefusesTheLeaseServesUnleasedAndSaysSo(t *testing.T) {
|
||||
a := &actor{serving: true, unleased: "the bus refused the lease's key"}
|
||||
if epoch, err := a.epoch(context.Background()); err != nil || epoch != 0 {
|
||||
t.Fatalf("an unleased controller answers %d, %v: it acts, claiming no epoch", epoch, err)
|
||||
}
|
||||
if st := a.standing(); st.Unleased == "" || st.Held {
|
||||
t.Fatalf("its standing says %+v", st)
|
||||
}
|
||||
// One that neither holds nor is unleased — still waiting — acts on nothing.
|
||||
waiting := &actor{serving: true}
|
||||
if _, err := waiting.epoch(context.Background()); !errors.Is(err, lease.ErrNotHeld) {
|
||||
t.Fatalf("a controller waiting for the lease may act: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user