Split the unit level from the integration level outside cmd, builder, link, catalogue, inventory; merge-check.sh times and bounds the unit level (issue 895)

internal/broker/callers_live_test.go

@@ -1,3 +1,5 @@
+//go:build integration
+
 package broker
 
 import (

internal/broker/config_owner_live_test.go

@@ -1,3 +1,5 @@
+//go:build integration
+
 package broker
 
 import (

internal/broker/consumer_from_now_test.go

@@ -1,3 +1,5 @@
+//go:build integration
+
 package broker
 
 import (

internal/broker/consumer_upgrade_test.go

@@ -1,3 +1,5 @@
+//go:build integration
+
 package broker
 
 import (

internal/broker/controller_buckets_integration_test.go

@@ -0,0 +1,261 @@
+//go:build integration
+
+package broker
+
+import (
+	"context"
+	"slices"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/novox/mesh-controller/internal/testbus"
+)
+
+func TestWhatTheControllerAskedIsBounded(t *testing.T) {
+	js, err := Dial(testbus.URL(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer js.Close()
+	if err := js.EnsureControllerBuckets(); err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	kv, err := jetstream.New(js.Conn())
+	if err != nil {
+		t.Fatal(err)
+	}
+	bucket, err := kv.KeyValue(ctx, AskedBucket)
+	if err != nil {
+		t.Fatal(err)
+	}
+	status, err := bucket.Status(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	info := status.(*jetstream.KeyValueBucketStatus).StreamInfo()
+	if status.History() != 1 || status.TTL() != AskedKeptFor || info.Config.MaxBytes <= 0 || info.Config.MaxBytes > 64<<20 {
+		t.Errorf("history %d, age %s, bytes %d", status.History(), status.TTL(), info.Config.MaxBytes)
+	}
+}
+
+func TestTheMergeWindowsKeysExpireAndAWatcherHearsIt(t *testing.T) {
+	controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !slices.Contains(controller.Publish, "$KV."+MergeWindowBucket+".>") {
+		t.Error("the controller may not write the merge window's bucket")
+	}
+	if !IsControllerBucket(MergeWindowBucket) || !slices.Contains(ControllerBuckets(), MergeWindowBucket) {
+		t.Error("the merge window's bucket is not asserted as the controller's own")
+	}
+	js, err := Dial(testbus.URL(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer js.Close()
+	if err := js.EnsureControllerBuckets(); err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	defer cancel()
+	api, err := jetstream.New(js.Conn())
+	if err != nil {
+		t.Fatal(err)
+	}
+	kv, err := api.KeyValue(ctx, MergeWindowBucket)
+	if err != nil {
+		t.Fatal(err)
+	}
+	w, err := kv.WatchAll(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer w.Stop()
+	if _, err := kv.Create(ctx, "plan-1.quiet", []byte("x"), jetstream.KeyTTL(time.Second)); err != nil {
+		t.Fatal(err)
+	}
+	for {
+		select {
+		case e := <-w.Updates():
+			if e != nil && e.Key() == "plan-1.quiet" && e.Operation() != jetstream.KeyValuePut {
+				return
+			}
+		case <-ctx.Done():
+			t.Fatal("no watcher heard the key expire")
+		}
+	}
+}
+
+func TestTheRolloutsSilentKeysExpireAsAPurgeAndAReportsDeleteIsNot(t *testing.T) {
+	controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !slices.Contains(controller.Publish, "$KV."+RolloutSilentBucket+".>") {
+		t.Error("the controller may not write the rollout's silent keys' bucket")
+	}
+	if !IsControllerBucket(RolloutSilentBucket) || !slices.Contains(ControllerBuckets(), RolloutSilentBucket) {
+		t.Error("the rollout's silent keys' bucket is not asserted as the controller's own")
+	}
+	js, err := Dial(testbus.URL(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer js.Close()
+	if err := js.EnsureControllerBuckets(); err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	defer cancel()
+	api, err := jetstream.New(js.Conn())
+	if err != nil {
+		t.Fatal(err)
+	}
+	kv, err := api.KeyValue(ctx, RolloutSilentBucket)
+	if err != nil {
+		t.Fatal(err)
+	}
+	w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer w.Stop()
+	if _, err := kv.Create(ctx, "plan-1.0.ace", []byte("x"), jetstream.KeyTTL(time.Hour)); err != nil {
+		t.Fatal(err)
+	}
+	if err := kv.Delete(ctx, "plan-1.0.ace"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := kv.Create(ctx, "plan-1.0.g14", []byte("x"), jetstream.KeyTTL(time.Second)); err != nil {
+		t.Fatal(err)
+	}
+	for {
+		select {
+		case e := <-w.Updates():
+			if e == nil || e.Operation() == jetstream.KeyValuePut {
+				continue
+			}
+			if e.Key() == "plan-1.0.ace" && e.Operation() != jetstream.KeyValueDelete {
+				t.Fatalf("a report's delete was heard as %v", e.Operation())
+			}
+			if e.Key() == "plan-1.0.g14" {
+				if e.Operation() != jetstream.KeyValuePurge {
+					t.Fatalf("an expiry was heard as %v, not a purge", e.Operation())
+				}
+				return
+			}
+		case <-ctx.Done():
+			t.Fatal("no watcher heard the key expire")
+		}
+	}
+}
+
+func TestTheSilentKeysBucketAgesAndAnOlderOneIsBroughtToIt(t *testing.T) {
+	js, err := Dial(testbus.URL(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer js.Close()
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	defer cancel()
+	api, err := jetstream.New(js.Conn())
+	if err != nil {
+		t.Fatal(err)
+	}
+	// As the first build made it: no age.
+	if _, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: RolloutSilentBucket, History: 1,
+		MaxValueSize: 4 << 10, MaxBytes: 8 << 20, Storage: jetstream.FileStorage, LimitMarkerTTL: RolloutSilentMarkerTTL}); err != nil {
+		t.Fatal(err)
+	}
+	if err := js.EnsureControllerBuckets(); err != nil {
+		t.Fatalf("a bucket made without the age is not brought to it: %v", err)
+	}
+	kv, err := api.KeyValue(ctx, RolloutSilentBucket)
+	if err != nil {
+		t.Fatal(err)
+	}
+	status, err := kv.Status(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if status.TTL() != RolloutSilentKeptFor {
+		t.Errorf("the bucket ages at %v; want %v", status.TTL(), RolloutSilentKeptFor)
+	}
+}
+
+func TestAResetOfTheEventsConsumerLeavesTheMergesAndTheirConsumerAlone(t *testing.T) {
+	js, err := Dial(testbus.URL(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer js.Close()
+	if err := AssertMeshStreams(js); err != nil {
+		t.Fatal(err)
+	}
+	if err := AssertMeshConsumers(js); err != nil {
+		t.Fatal(err)
+	}
+	for _, c := range MeshConsumers() {
+		if c.Stream == EventsStream && c.Name == MergesConsumerName {
+			if c.Resettable != "" || c.MaxDeliver != 0 || len(c.Filters) != 1 || c.Filters[0] != ControllerFollows[3] {
+				t.Errorf("the merges consumer is %+v; want one filter, no max-deliver, not resettable", c)
+			}
+		}
+		if c.Stream == EventsStream && c.Name == ControllerName {
+			for _, f := range c.Filters {
+				if f == ControllerFollows[3] {
+					t.Errorf("the events consumer still follows the merges, which the merges consumer takes")
+				}
+			}
+		}
+	}
+	if _, err := js.Context().Publish(ControllerFollows[3], []byte(`{"commit":"c1"}`)); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := js.Context().Publish(ControllerFollows[0], []byte(`{}`)); err != nil {
+		t.Fatal(err)
+	}
+	merges, err := js.Context().ConsumerInfo(EventsStream, MergesConsumerName)
+	if err != nil || merges.NumPending != 1 {
+		t.Fatalf("the merges consumer holds %v (%v); want the merge", merges, err)
+	}
+	if _, _, err := js.ResetConsumer(EventsStream, ControllerName); err != nil {
+		t.Fatal(err)
+	}
+	merges, err = js.Context().ConsumerInfo(EventsStream, MergesConsumerName)
+	if err != nil || merges.NumPending != 1 || merges.Delivered.Stream != 0 {
+		t.Fatalf("after the reset the merges consumer holds %+v (%v); want its merge left alone", merges, err)
+	}
+}
+
+func TestAMergeTheOldConsumerHadNotAcknowledgedIsTheMergesConsumers(t *testing.T) {
+	js, err := Dial(testbus.URL(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer js.Close()
+	if err := AssertMeshStreams(js); err != nil {
+		t.Fatal(err)
+	}
+	old := &nats.ConsumerConfig{Durable: ControllerName, AckPolicy: nats.AckExplicitPolicy, AckWait: 30 * time.Second,
+		MaxDeliver: 5, MaxAckPending: 1, DeliverSubject: "_DELIVER.controller.EVENTS", DeliverPolicy: nats.DeliverNewPolicy,
+		FilterSubjects: append(append([]string(nil), ControllerFollows...), AgainFilter(ControllerName))}
+	if _, err := js.Context().AddConsumer(EventsStream, old); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := js.Context().Publish(ControllerFollows[3], []byte(`{"commit":"c1"}`)); err != nil {
+		t.Fatal(err)
+	}
+	if err := AssertMeshConsumers(js); err != nil {
+		t.Fatal(err)
+	}
+	merges, err := js.Context().ConsumerInfo(EventsStream, MergesConsumerName)
+	if err != nil || merges.NumPending != 1 {
+		t.Fatalf("the merges consumer holds %v (%v); want the merge the old consumer never acknowledged", merges, err)
+	}
+}

internal/broker/controller_buckets_test.go

@@ -1,16 +1,9 @@
 package broker
 
 import (
-	"context"
-	"github.com/nats-io/nats.go"
 	"slices"
 	"strings"
 	"testing"
-	"time"
-
-	"github.com/nats-io/nats.go/jetstream"
-
-	"github.com/novox/mesh-controller/internal/testbus"
 )
 
 // **The controller may write every bucket it writes** (novox/hq to-be 45 §1, issue 269). Writing a
@@ -100,34 +93,6 @@ func TestTheWatchedSignalsMayBeSaidAndHeard(t *testing.T) {
 
 // The controller's record of what it asked the operator is bounded (correctness review of 2026-10-08): one
 // value a key, a month's age, and a size it cannot outgrow.
-func TestWhatTheControllerAskedIsBounded(t *testing.T) {
-	js, err := Dial(testbus.URL(t))
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer js.Close()
-	if err := js.EnsureControllerBuckets(); err != nil {
-		t.Fatal(err)
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	kv, err := jetstream.New(js.Conn())
-	if err != nil {
-		t.Fatal(err)
-	}
-	bucket, err := kv.KeyValue(ctx, AskedBucket)
-	if err != nil {
-		t.Fatal(err)
-	}
-	status, err := bucket.Status(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	info := status.(*jetstream.KeyValueBucketStatus).StreamInfo()
-	if status.History() != 1 || status.TTL() != AskedKeptFor || info.Config.MaxBytes <= 0 || info.Config.MaxBytes > 64<<20 {
-		t.Errorf("history %d, age %s, bytes %d", status.History(), status.TTL(), info.Config.MaxBytes)
-	}
-}
 
 // **A listed mirror is no undeclared bucket** (ADR 0356 §3, ADR 0201 §16): the controller's list of its mirrors
 // stands for their declaration, so the report of state nothing declares never names one.
@@ -273,232 +238,19 @@ func TestTheControllerMayAskMeshConfigInItsOwnName(t *testing.T) {
 // **The merge window is a bucket whose keys expire** (novox/hq ADR 0407 decision 1, issue 973): a key made with a
 // time to live goes when it is up, and a watcher on the bucket hears it go. The controller may write it, and it is
 // no undeclared state.
-func TestTheMergeWindowsKeysExpireAndAWatcherHearsIt(t *testing.T) {
-	controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !slices.Contains(controller.Publish, "$KV."+MergeWindowBucket+".>") {
-		t.Error("the controller may not write the merge window's bucket")
-	}
-	if !IsControllerBucket(MergeWindowBucket) || !slices.Contains(ControllerBuckets(), MergeWindowBucket) {
-		t.Error("the merge window's bucket is not asserted as the controller's own")
-	}
-	js, err := Dial(testbus.URL(t))
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer js.Close()
-	if err := js.EnsureControllerBuckets(); err != nil {
-		t.Fatal(err)
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
-	defer cancel()
-	api, err := jetstream.New(js.Conn())
-	if err != nil {
-		t.Fatal(err)
-	}
-	kv, err := api.KeyValue(ctx, MergeWindowBucket)
-	if err != nil {
-		t.Fatal(err)
-	}
-	w, err := kv.WatchAll(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer w.Stop()
-	if _, err := kv.Create(ctx, "plan-1.quiet", []byte("x"), jetstream.KeyTTL(time.Second)); err != nil {
-		t.Fatal(err)
-	}
-	for {
-		select {
-		case e := <-w.Updates():
-			if e != nil && e.Key() == "plan-1.quiet" && e.Operation() != jetstream.KeyValuePut {
-				return
-			}
-		case <-ctx.Done():
-			t.Fatal("no watcher heard the key expire")
-		}
-	}
-}
 
 // **A rollout's wait for a machine is a bucket whose keys expire** (novox/hq ADR 0406 §5, issue 974): a key made with a
 // time to live goes when it is up, and a watcher hears it go as a purge; a key a report deleted is a delete. The
 // controller may write it, and it is no undeclared state.
-func TestTheRolloutsSilentKeysExpireAsAPurgeAndAReportsDeleteIsNot(t *testing.T) {
-	controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !slices.Contains(controller.Publish, "$KV."+RolloutSilentBucket+".>") {
-		t.Error("the controller may not write the rollout's silent keys' bucket")
-	}
-	if !IsControllerBucket(RolloutSilentBucket) || !slices.Contains(ControllerBuckets(), RolloutSilentBucket) {
-		t.Error("the rollout's silent keys' bucket is not asserted as the controller's own")
-	}
-	js, err := Dial(testbus.URL(t))
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer js.Close()
-	if err := js.EnsureControllerBuckets(); err != nil {
-		t.Fatal(err)
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
-	defer cancel()
-	api, err := jetstream.New(js.Conn())
-	if err != nil {
-		t.Fatal(err)
-	}
-	kv, err := api.KeyValue(ctx, RolloutSilentBucket)
-	if err != nil {
-		t.Fatal(err)
-	}
-	w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer w.Stop()
-	if _, err := kv.Create(ctx, "plan-1.0.ace", []byte("x"), jetstream.KeyTTL(time.Hour)); err != nil {
-		t.Fatal(err)
-	}
-	if err := kv.Delete(ctx, "plan-1.0.ace"); err != nil {
-		t.Fatal(err)
-	}
-	if _, err := kv.Create(ctx, "plan-1.0.g14", []byte("x"), jetstream.KeyTTL(time.Second)); err != nil {
-		t.Fatal(err)
-	}
-	for {
-		select {
-		case e := <-w.Updates():
-			if e == nil || e.Operation() == jetstream.KeyValuePut {
-				continue
-			}
-			if e.Key() == "plan-1.0.ace" && e.Operation() != jetstream.KeyValueDelete {
-				t.Fatalf("a report's delete was heard as %v", e.Operation())
-			}
-			if e.Key() == "plan-1.0.g14" {
-				if e.Operation() != jetstream.KeyValuePurge {
-					t.Fatalf("an expiry was heard as %v, not a purge", e.Operation())
-				}
-				return
-			}
-		case <-ctx.Done():
-			t.Fatal("no watcher heard the key expire")
-		}
-	}
-}
 
 // **The silent keys' bucket ages**, or its delete markers would fill it (issue 974, review 1): every report deletes a key,
 // the delete leaves a marker, and with one value per key and no age the markers stay for ever. A bucket made before the
 // age was set is brought to it by the next start, which the server accepts as an update of an existing bucket.
-func TestTheSilentKeysBucketAgesAndAnOlderOneIsBroughtToIt(t *testing.T) {
-	js, err := Dial(testbus.URL(t))
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer js.Close()
-	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
-	defer cancel()
-	api, err := jetstream.New(js.Conn())
-	if err != nil {
-		t.Fatal(err)
-	}
-	// As the first build made it: no age.
-	if _, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: RolloutSilentBucket, History: 1,
-		MaxValueSize: 4 << 10, MaxBytes: 8 << 20, Storage: jetstream.FileStorage, LimitMarkerTTL: RolloutSilentMarkerTTL}); err != nil {
-		t.Fatal(err)
-	}
-	if err := js.EnsureControllerBuckets(); err != nil {
-		t.Fatalf("a bucket made without the age is not brought to it: %v", err)
-	}
-	kv, err := api.KeyValue(ctx, RolloutSilentBucket)
-	if err != nil {
-		t.Fatal(err)
-	}
-	status, err := kv.Status(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if status.TTL() != RolloutSilentKeptFor {
-		t.Errorf("the bucket ages at %v; want %v", status.TTL(), RolloutSilentKeptFor)
-	}
-}
 
 // **The merges have a consumer of their own, which a reset of the events consumer leaves alone** (issue 973, novox/hq
 // ADR 0407 decision 3): healer H4 re-makes the events consumer from now, dropping what it holds pending; a merge
 // waiting in the merges consumer is not that consumer's, and nothing would catch it up.
-func TestAResetOfTheEventsConsumerLeavesTheMergesAndTheirConsumerAlone(t *testing.T) {
-	js, err := Dial(testbus.URL(t))
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer js.Close()
-	if err := AssertMeshStreams(js); err != nil {
-		t.Fatal(err)
-	}
-	if err := AssertMeshConsumers(js); err != nil {
-		t.Fatal(err)
-	}
-	for _, c := range MeshConsumers() {
-		if c.Stream == EventsStream && c.Name == MergesConsumerName {
-			if c.Resettable != "" || c.MaxDeliver != 0 || len(c.Filters) != 1 || c.Filters[0] != ControllerFollows[3] {
-				t.Errorf("the merges consumer is %+v; want one filter, no max-deliver, not resettable", c)
-			}
-		}
-		if c.Stream == EventsStream && c.Name == ControllerName {
-			for _, f := range c.Filters {
-				if f == ControllerFollows[3] {
-					t.Errorf("the events consumer still follows the merges, which the merges consumer takes")
-				}
-			}
-		}
-	}
-	if _, err := js.Context().Publish(ControllerFollows[3], []byte(`{"commit":"c1"}`)); err != nil {
-		t.Fatal(err)
-	}
-	if _, err := js.Context().Publish(ControllerFollows[0], []byte(`{}`)); err != nil {
-		t.Fatal(err)
-	}
-	merges, err := js.Context().ConsumerInfo(EventsStream, MergesConsumerName)
-	if err != nil || merges.NumPending != 1 {
-		t.Fatalf("the merges consumer holds %v (%v); want the merge", merges, err)
-	}
-	if _, _, err := js.ResetConsumer(EventsStream, ControllerName); err != nil {
-		t.Fatal(err)
-	}
-	merges, err = js.Context().ConsumerInfo(EventsStream, MergesConsumerName)
-	if err != nil || merges.NumPending != 1 || merges.Delivered.Stream != 0 {
-		t.Fatalf("after the reset the merges consumer holds %+v (%v); want its merge left alone", merges, err)
-	}
-}
 
 // **Delivering this change loses no merge** (review of issue 973): a merge published under the old filter of the
 // `controller` consumer and never acknowledged by it is delivered by the merges consumer once the assertion has
 // made it, because it starts where `controller` has acknowledged up to, not from now.
-func TestAMergeTheOldConsumerHadNotAcknowledgedIsTheMergesConsumers(t *testing.T) {
-	js, err := Dial(testbus.URL(t))
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer js.Close()
-	if err := AssertMeshStreams(js); err != nil {
-		t.Fatal(err)
-	}
-	old := &nats.ConsumerConfig{Durable: ControllerName, AckPolicy: nats.AckExplicitPolicy, AckWait: 30 * time.Second,
-		MaxDeliver: 5, MaxAckPending: 1, DeliverSubject: "_DELIVER.controller.EVENTS", DeliverPolicy: nats.DeliverNewPolicy,
-		FilterSubjects: append(append([]string(nil), ControllerFollows...), AgainFilter(ControllerName))}
-	if _, err := js.Context().AddConsumer(EventsStream, old); err != nil {
-		t.Fatal(err)
-	}
-	if _, err := js.Context().Publish(ControllerFollows[3], []byte(`{"commit":"c1"}`)); err != nil {
-		t.Fatal(err)
-	}
-	if err := AssertMeshConsumers(js); err != nil {
-		t.Fatal(err)
-	}
-	merges, err := js.Context().ConsumerInfo(EventsStream, MergesConsumerName)
-	if err != nil || merges.NumPending != 1 {
-		t.Fatalf("the merges consumer holds %v (%v); want the merge the old consumer never acknowledged", merges, err)
-	}
-}

internal/broker/jetstream_test.go

@@ -1,3 +1,5 @@
+//go:build integration
+
 package broker
 
 import (

internal/broker/log_integration_test.go

@@ -0,0 +1,69 @@
+//go:build integration
+
+package broker
+
+import (
+	"context"
+	"slices"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+func TestALogIsAssertedInPlaceAndNeverRemoved(t *testing.T) {
+	bus := aLiveBus(t)
+	l := Log{Module: "logtest", Name: "changes", MaxMiB: 1}
+	if _, err := RaiseLogs(bus, []Log{l}); err != nil {
+		t.Fatalf("a real server refused a module's log: %v", err)
+	}
+	js, err := jetstream.New(bus.Conn())
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	ack, err := js.Publish(ctx, l.Subject()+".7", []byte(`{"op":"opened"}`))
+	if err != nil {
+		t.Fatalf("an append to the log was refused: %v", err)
+	}
+	stream, err := js.Stream(ctx, l.Stream())
+	if err != nil {
+		t.Fatal(err)
+	}
+	have := stream.CachedInfo().Config
+	want := l.Config()
+	if have.Storage != want.Storage || have.Retention != want.Retention || have.MaxAge != 0 ||
+		have.MaxMsgsPerSubject != -1 || have.MaxBytes != want.MaxBytes || have.MaxMsgSize != want.MaxMsgSize ||
+		have.Discard != jetstream.DiscardNew || !have.AllowDirect || !have.DenyDelete || !have.DenyPurge ||
+		have.Replicas != 1 || !slices.Equal(have.Subjects, want.Subjects) {
+		t.Fatalf("the server holds the log as %+v", have)
+	}
+	if err := stream.DeleteMsg(ctx, ack.Sequence); err == nil {
+		t.Fatal("an entry was deleted from a log")
+	}
+	l.MaxMiB = 2
+	if _, err := RaiseLogs(bus, []Log{l}); err != nil {
+		t.Fatalf("asserting the log again failed, so a restart would: %v", err)
+	}
+	info, err := stream.Info(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if info.Config.MaxBytes != 2*1024*1024 {
+		t.Fatalf("the changed cap was not brought to match: %d", info.Config.MaxBytes)
+	}
+	if info.State.Msgs < 1 {
+		t.Fatal("asserting the log again lost what it held")
+	}
+	undeclared, err := RaiseLogs(bus, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !slices.Contains(undeclared, l.Stream()) {
+		t.Fatalf("a log nothing declares was not reported: %v", undeclared)
+	}
+	if _, err := js.Stream(ctx, l.Stream()); err != nil {
+		t.Fatalf("a log nothing declares is gone: %v", err)
+	}
+}

internal/broker/log_test.go

@@ -1,7 +1,6 @@
 package broker
 
 import (
-	"context"
 	"encoding/json"
 	"go/ast"
 	"go/parser"
@@ -11,7 +10,6 @@ import (
 	"slices"
 	"strings"
 	"testing"
-	"time"
 
 	"github.com/nats-io/nats.go/jetstream"
 )
@@ -240,59 +238,3 @@ func stringsIn(f *ast.File) []string {
 // Against a real server: a log is created as its configuration says, asserting it again keeps what
 // it holds, a changed cap is brought to match in place, an entry cannot be deleted from it, and one
 // nothing declares any more is reported and stays.
-func TestALogIsAssertedInPlaceAndNeverRemoved(t *testing.T) {
-	bus := aLiveBus(t)
-	l := Log{Module: "logtest", Name: "changes", MaxMiB: 1}
-	if _, err := RaiseLogs(bus, []Log{l}); err != nil {
-		t.Fatalf("a real server refused a module's log: %v", err)
-	}
-	js, err := jetstream.New(bus.Conn())
-	if err != nil {
-		t.Fatal(err)
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	ack, err := js.Publish(ctx, l.Subject()+".7", []byte(`{"op":"opened"}`))
-	if err != nil {
-		t.Fatalf("an append to the log was refused: %v", err)
-	}
-	stream, err := js.Stream(ctx, l.Stream())
-	if err != nil {
-		t.Fatal(err)
-	}
-	have := stream.CachedInfo().Config
-	want := l.Config()
-	if have.Storage != want.Storage || have.Retention != want.Retention || have.MaxAge != 0 ||
-		have.MaxMsgsPerSubject != -1 || have.MaxBytes != want.MaxBytes || have.MaxMsgSize != want.MaxMsgSize ||
-		have.Discard != jetstream.DiscardNew || !have.AllowDirect || !have.DenyDelete || !have.DenyPurge ||
-		have.Replicas != 1 || !slices.Equal(have.Subjects, want.Subjects) {
-		t.Fatalf("the server holds the log as %+v", have)
-	}
-	if err := stream.DeleteMsg(ctx, ack.Sequence); err == nil {
-		t.Fatal("an entry was deleted from a log")
-	}
-	l.MaxMiB = 2
-	if _, err := RaiseLogs(bus, []Log{l}); err != nil {
-		t.Fatalf("asserting the log again failed, so a restart would: %v", err)
-	}
-	info, err := stream.Info(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if info.Config.MaxBytes != 2*1024*1024 {
-		t.Fatalf("the changed cap was not brought to match: %d", info.Config.MaxBytes)
-	}
-	if info.State.Msgs < 1 {
-		t.Fatal("asserting the log again lost what it held")
-	}
-	undeclared, err := RaiseLogs(bus, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !slices.Contains(undeclared, l.Stream()) {
-		t.Fatalf("a log nothing declares was not reported: %v", undeclared)
-	}
-	if _, err := js.Stream(ctx, l.Stream()); err != nil {
-		t.Fatalf("a log nothing declares is gone: %v", err)
-	}
-}

internal/broker/modstreams_live_test.go

@@ -1,3 +1,5 @@
+//go:build integration
+
 package broker
 
 import (

internal/broker/msgttl_live_test.go

@@ -1,3 +1,5 @@
+//go:build integration
+
 package broker
 
 import (

internal/broker/prov_applied_live_test.go

@@ -1,3 +1,5 @@
+//go:build integration
+
 package broker
 
 import (

internal/broker/provisioner_live_test.go

@@ -1,3 +1,5 @@
+//go:build integration
+
 package broker
 
 import (

internal/broker/provisioner_server_live_test.go

@@ -1,3 +1,5 @@
+//go:build integration
+
 package broker
 
 import (

internal/broker/raise_live_test.go

@@ -1,3 +1,5 @@
+//go:build integration
+
 package broker
 
 import (

internal/broker/snapshot_live_test.go

@@ -1,3 +1,5 @@
+//go:build integration
+
 package broker
 
 import (

internal/broker/state_integration_test.go

@@ -0,0 +1,55 @@
+//go:build integration
+
+package broker
+
+import (
+	"testing"
+
+	"github.com/nats-io/nats.go"
+)
+
+func TestABucketIsAssertedInPlace(t *testing.T) {
+	js := aLiveBus(t)
+	b := Bucket{Module: "statetest", Name: "servers"}
+	if _, err := RaiseBuckets(js, []Bucket{b}); err != nil {
+		t.Fatalf("a real server refused a module's bucket: %v", err)
+	}
+	kv, err := js.Context().KeyValue(b.Bucket())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := kv.Put("all.one", []byte(`{"kept":true}`)); err != nil {
+		t.Fatal(err)
+	}
+	b.History = 3
+	if _, err := RaiseBuckets(js, []Bucket{b}); err != nil {
+		t.Fatalf("asserting the bucket again failed, so a restart would: %v", err)
+	}
+	got, err := kv.Get("all.one")
+	if err != nil || string(got.Value()) != `{"kept":true}` {
+		t.Fatalf("asserting again lost what the bucket held: %v %v", got, err)
+	}
+	status, err := kv.Status()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if status.History() != 3 {
+		t.Fatalf("history is %d, the owner declared 3", status.History())
+	}
+	if s, ok := status.(*nats.KeyValueBucketStatus); ok {
+		if c := s.StreamInfo().Config; c.MaxMsgSize != StateMaxValueBytes || c.MaxBytes != StateMaxBytes {
+			t.Fatalf("the mesh's caps are not on the bucket: value %d, bucket %d", c.MaxMsgSize, c.MaxBytes)
+		}
+	}
+}
+
+func TestABucketIsAssertedOverAHeldConnection(t *testing.T) {
+	js := aLiveBus(t)
+	held := OnConn(js.Conn())
+	if _, err := RaiseBuckets(held, []Bucket{{Module: "statetest", Name: "held"}}); err != nil {
+		t.Fatalf("asserting over a held connection failed: %v", err)
+	}
+	if _, err := js.Context().KeyValue("statetest_held"); err != nil {
+		t.Fatalf("the bucket is not there: %v", err)
+	}
+}

internal/broker/state_test.go

@@ -4,8 +4,6 @@ import (
 	"slices"
 	"strings"
 	"testing"
-
-	"github.com/nats-io/nats.go"
 )
 
 // The grants measured against a running server (novox/hq research 024): an owner reads and writes
@@ -131,54 +129,10 @@ func TestRaisingStateReportsWhatNothingDeclares(t *testing.T) {
 // Against a real server: a bucket is created with the owner's options and the mesh's caps,
 // asserting it again changes nothing and keeps what it holds, and a changed option is brought to
 // match in place.
-func TestABucketIsAssertedInPlace(t *testing.T) {
-	js := aLiveBus(t)
-	b := Bucket{Module: "statetest", Name: "servers"}
-	if _, err := RaiseBuckets(js, []Bucket{b}); err != nil {
-		t.Fatalf("a real server refused a module's bucket: %v", err)
-	}
-	kv, err := js.Context().KeyValue(b.Bucket())
-	if err != nil {
-		t.Fatal(err)
-	}
-	if _, err := kv.Put("all.one", []byte(`{"kept":true}`)); err != nil {
-		t.Fatal(err)
-	}
-	b.History = 3
-	if _, err := RaiseBuckets(js, []Bucket{b}); err != nil {
-		t.Fatalf("asserting the bucket again failed, so a restart would: %v", err)
-	}
-	got, err := kv.Get("all.one")
-	if err != nil || string(got.Value()) != `{"kept":true}` {
-		t.Fatalf("asserting again lost what the bucket held: %v %v", got, err)
-	}
-	status, err := kv.Status()
-	if err != nil {
-		t.Fatal(err)
-	}
-	if status.History() != 3 {
-		t.Fatalf("history is %d, the owner declared 3", status.History())
-	}
-	if s, ok := status.(*nats.KeyValueBucketStatus); ok {
-		if c := s.StreamInfo().Config; c.MaxMsgSize != StateMaxValueBytes || c.MaxBytes != StateMaxBytes {
-			t.Fatalf("the mesh's caps are not on the bucket: value %d, bucket %d", c.MaxMsgSize, c.MaxBytes)
-		}
-	}
-}
 
 // Against a real server: the handle over a connection the control plane already holds asserts a
 // bucket as Dial's does — what a push uses, so a module registered since the last start has its
 // bucket before its membership names it.
-func TestABucketIsAssertedOverAHeldConnection(t *testing.T) {
-	js := aLiveBus(t)
-	held := OnConn(js.Conn())
-	if _, err := RaiseBuckets(held, []Bucket{{Module: "statetest", Name: "held"}}); err != nil {
-		t.Fatalf("asserting over a held connection failed: %v", err)
-	}
-	if _, err := js.Context().KeyValue("statetest_held"); err != nil {
-		t.Fatalf("the bucket is not there: %v", err)
-	}
-}
 
 // **State keyed by machine reaches its machine's key alone** (novox/hq ADR 0260): the owner's instance on
 // a machine writes and reads that key, and a bar granted a key reads that key — a direct get of its

internal/broker/vault_calls_live_integration_test.go

@@ -0,0 +1,130 @@
+//go:build integration
+
+package broker
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats-server/v2/server"
+	"github.com/nats-io/nats.go"
+	"golang.org/x/crypto/bcrypt"
+)
+
+func TestOnARealBusTheVaultsAccountIsAnsweredByTheDeskAndTheProviderAndNothingElse(t *testing.T) {
+	users, err := Users(vaultRecords())
+	if err != nil {
+		t.Fatal(err)
+	}
+	hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i := range users {
+		users[i].PasswordHash = string(hash)
+	}
+	accounts, err := ComposeAccounts(users)
+	if err != nil {
+		t.Fatal(err)
+	}
+	conf := filepath.Join(t.TempDir(), "bus.conf")
+	if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+
+		`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	opts, err := server.ProcessConfigFile(conf)
+	if err != nil {
+		t.Fatalf("the composed accounts do not parse: %v", err)
+	}
+	opts.NoLog, opts.NoSigs = true, true
+	s, err := server.NewServer(opts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	go s.Start()
+	if !s.ReadyForConnections(10 * time.Second) {
+		t.Fatal("the bus did not come up")
+	}
+	defer s.Shutdown()
+
+	// anchor's runtime serves the desk's prompt and plex's issuing tool, on every subject it is granted, and says
+	// which subject it heard.
+	holder, err := nats.Connect(s.ClientURL(), nats.UserInfo("anchor.node-tools", "pw"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer holder.Close()
+	heard := make(chan string, 8)
+	answer := func(m *nats.Msg) {
+		heard <- m.Subject
+		_ = m.Respond([]byte(`{"result":{"sealed":"c2VhbGVkLXRvLXRoZS1jYWxs"}}`))
+	}
+	for _, subject := range []string{"mesh.seat.node-launcher.call.secret.anchor.*",
+		"mesh.seat.node-launcher.tool.secret.anchor", "mesh.mod.plex.call.plex_issued_credential.anchor.*",
+		"mesh.mod.other.call.tool.anchor.*", "mesh.seat.node-launcher.call.menu.anchor.*"} {
+		sub, err := holder.Subscribe(subject, answer)
+		if err != nil {
+			t.Fatal(err)
+		}
+		_ = holder.Flush()
+		if !sub.IsValid() {
+			t.Fatalf("the desk's runtime may not hear %s", subject)
+		}
+	}
+
+	refusals := make(chan error, 16)
+	vault, err := nats.Connect(s.ClientURL(), nats.UserInfo("novox.mesh-vault", "pw"),
+		nats.CustomInboxPrefix("_INBOX.novox.mesh-vault"),
+		nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refusals <- err }))
+	if err != nil {
+		t.Fatalf("the vault's own account cannot connect: %v", err)
+	}
+	defer vault.Close()
+	token := CallerToken("novox.mesh-vault")
+	for _, subject := range []string{"mesh.seat.node-launcher.call.secret.anchor." + token,
+		"mesh.mod.plex.call.plex_issued_credential.anchor." + token} {
+		if _, err := vault.Request(subject, []byte(`{"seal_to":"a2V5LW9mLXRoZS1jYWxs"}`), 3*time.Second); err != nil {
+			t.Fatalf("the vault's call %s was not answered: %v", subject, err)
+		}
+		if got := <-heard; got != subject {
+			t.Fatalf("the holder heard %s, want %s", got, subject)
+		}
+	}
+	for _, subject := range []string{"mesh.seat.node-launcher.tool.secret.anchor",
+		"mesh.seat.node-launcher.call.secret.anchor.controller",
+		"mesh.seat.node-launcher.call.menu.anchor." + token,
+		"mesh.mod.other.call.tool.anchor." + token} {
+		if err := vault.Publish(subject, []byte(`{}`)); err != nil {
+			t.Fatal(err)
+		}
+		_ = vault.Flush()
+		select {
+		case err := <-refusals:
+			if !errors.Is(err, nats.ErrPermissionViolation) {
+				t.Errorf("%s: the server said %v, want a permissions violation", subject, err)
+			}
+		case <-time.After(3 * time.Second):
+			t.Errorf("the server did not refuse the vault %s", subject)
+		}
+	}
+	select {
+	case got := <-heard:
+		t.Errorf("a call the vault may not make reached the holder: %s", got)
+	case <-time.After(200 * time.Millisecond):
+	}
+
+	// The controller is still answered on today's path, for the modules filled the old way (until R1).
+	controller, err := nats.Connect(s.ClientURL(), nats.UserInfo("controller", "pw"),
+		nats.CustomInboxPrefix("_INBOX.controller"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer controller.Close()
+	if _, err := controller.Request("mesh.seat.node-launcher.tool.secret.anchor", []byte(`{}`), 3*time.Second); err != nil {
+		t.Errorf("the controller's call to the desk on today's path: %v", err)
+	}
+	<-heard
+}

internal/broker/vault_calls_live_test.go

@@ -1,16 +1,8 @@
 package broker
 
 import (
-	"errors"
-	"os"
-	"path/filepath"
 	"strings"
 	"testing"
-	"time"
-
-	"github.com/nats-io/nats-server/v2/server"
-	"github.com/nats-io/nats.go"
-	"golang.org/x/crypto/bcrypt"
 )
 
 // vaultRecords is a mesh of two machines: anchor, a desk whose runtime carries the desk's prompt (rofi holding
@@ -111,117 +103,3 @@ func TestTheVaultAsksTheDeskAndTheIssuingToolsInItsOwnNameAlone(t *testing.T) {
 // **On a real server, as composed** (issue 723): the vault's account asks the desk's prompt and an issuing tool
 // and is answered, naming itself; the server refuses it any other call; and the controller is still answered on
 // today's path. That only sealed values cross is the vault's test (mesh-catalog, mesh-vault intake_test.go).
-func TestOnARealBusTheVaultsAccountIsAnsweredByTheDeskAndTheProviderAndNothingElse(t *testing.T) {
-	users, err := Users(vaultRecords())
-	if err != nil {
-		t.Fatal(err)
-	}
-	hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
-	if err != nil {
-		t.Fatal(err)
-	}
-	for i := range users {
-		users[i].PasswordHash = string(hash)
-	}
-	accounts, err := ComposeAccounts(users)
-	if err != nil {
-		t.Fatal(err)
-	}
-	conf := filepath.Join(t.TempDir(), "bus.conf")
-	if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+
-		`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
-		t.Fatal(err)
-	}
-	opts, err := server.ProcessConfigFile(conf)
-	if err != nil {
-		t.Fatalf("the composed accounts do not parse: %v", err)
-	}
-	opts.NoLog, opts.NoSigs = true, true
-	s, err := server.NewServer(opts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	go s.Start()
-	if !s.ReadyForConnections(10 * time.Second) {
-		t.Fatal("the bus did not come up")
-	}
-	defer s.Shutdown()
-
-	// anchor's runtime serves the desk's prompt and plex's issuing tool, on every subject it is granted, and says
-	// which subject it heard.
-	holder, err := nats.Connect(s.ClientURL(), nats.UserInfo("anchor.node-tools", "pw"))
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer holder.Close()
-	heard := make(chan string, 8)
-	answer := func(m *nats.Msg) {
-		heard <- m.Subject
-		_ = m.Respond([]byte(`{"result":{"sealed":"c2VhbGVkLXRvLXRoZS1jYWxs"}}`))
-	}
-	for _, subject := range []string{"mesh.seat.node-launcher.call.secret.anchor.*",
-		"mesh.seat.node-launcher.tool.secret.anchor", "mesh.mod.plex.call.plex_issued_credential.anchor.*",
-		"mesh.mod.other.call.tool.anchor.*", "mesh.seat.node-launcher.call.menu.anchor.*"} {
-		sub, err := holder.Subscribe(subject, answer)
-		if err != nil {
-			t.Fatal(err)
-		}
-		_ = holder.Flush()
-		if !sub.IsValid() {
-			t.Fatalf("the desk's runtime may not hear %s", subject)
-		}
-	}
-
-	refusals := make(chan error, 16)
-	vault, err := nats.Connect(s.ClientURL(), nats.UserInfo("novox.mesh-vault", "pw"),
-		nats.CustomInboxPrefix("_INBOX.novox.mesh-vault"),
-		nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refusals <- err }))
-	if err != nil {
-		t.Fatalf("the vault's own account cannot connect: %v", err)
-	}
-	defer vault.Close()
-	token := CallerToken("novox.mesh-vault")
-	for _, subject := range []string{"mesh.seat.node-launcher.call.secret.anchor." + token,
-		"mesh.mod.plex.call.plex_issued_credential.anchor." + token} {
-		if _, err := vault.Request(subject, []byte(`{"seal_to":"a2V5LW9mLXRoZS1jYWxs"}`), 3*time.Second); err != nil {
-			t.Fatalf("the vault's call %s was not answered: %v", subject, err)
-		}
-		if got := <-heard; got != subject {
-			t.Fatalf("the holder heard %s, want %s", got, subject)
-		}
-	}
-	for _, subject := range []string{"mesh.seat.node-launcher.tool.secret.anchor",
-		"mesh.seat.node-launcher.call.secret.anchor.controller",
-		"mesh.seat.node-launcher.call.menu.anchor." + token,
-		"mesh.mod.other.call.tool.anchor." + token} {
-		if err := vault.Publish(subject, []byte(`{}`)); err != nil {
-			t.Fatal(err)
-		}
-		_ = vault.Flush()
-		select {
-		case err := <-refusals:
-			if !errors.Is(err, nats.ErrPermissionViolation) {
-				t.Errorf("%s: the server said %v, want a permissions violation", subject, err)
-			}
-		case <-time.After(3 * time.Second):
-			t.Errorf("the server did not refuse the vault %s", subject)
-		}
-	}
-	select {
-	case got := <-heard:
-		t.Errorf("a call the vault may not make reached the holder: %s", got)
-	case <-time.After(200 * time.Millisecond):
-	}
-
-	// The controller is still answered on today's path, for the modules filled the old way (until R1).
-	controller, err := nats.Connect(s.ClientURL(), nats.UserInfo("controller", "pw"),
-		nats.CustomInboxPrefix("_INBOX.controller"))
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer controller.Close()
-	if _, err := controller.Request("mesh.seat.node-launcher.tool.secret.anchor", []byte(`{}`), 3*time.Second); err != nil {
-		t.Errorf("the controller's call to the desk on today's path: %v", err)
-	}
-	<-heard
-}

internal/broker/worker_type_change_test.go

@@ -1,3 +1,5 @@
+//go:build integration
+
 package broker
 
 import (