mesh-catalog/modules/nats/cmd/mesh-nats-snapshot/live_test.go

//go:build integration

package main

// The snapshot and the way back, against real servers: the bus's own release in throwaway containers.
//
//	go test -tags integration -race ./...   (run through mesh-controller.try, never by hand on a workstation)
//
// Every container and file it makes is removed when it ends. MESH_TEST_SNAPSHOT_MESSAGES sets how
// many events the source holds (default 6000), to time a snapshot at the live bus's size.

import (
	"bytes"
	"context"
	"crypto/ecdsa"
	"crypto/elliptic"
	"crypto/rand"
	"crypto/sha256"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/hex"
	"encoding/pem"
	"errors"
	"fmt"
	"math/big"
	mrand "math/rand"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/nats-io/nats.go"
	"nats-tools/internal/busarchive"
)

// image is the bus's release (the manifest's server artifact names the same release in its tag).
const image = "nats:2.11-alpine"

// snapshotUser is the bus's own module on a machine, as the controller names it.
const snapshotUser = "anchor.nats"

// snapshotGrants are what mesh-controller composes for the module holding mesh-broker
// (internal/broker BusSnapshotGrants), written out here because this repository does not import the
// controller. Its own test proves the same list against a real server; this one proves the program
// needs no more than it.
var snapshotGrants = struct{ publish, subscribe []string }{
	publish: []string{
		"$JS.API.STREAM.NAMES",
		"$JS.API.STREAM.INFO.*",
		"$JS.API.STREAM.SNAPSHOT.*",
		"$JS.SNAPSHOT.ACK.>",
	},
	subscribe: []string{"_INBOX." + snapshotUser + ".>"},
}

func needDocker(t *testing.T) {
	t.Helper()
	if _, err := exec.LookPath("docker"); err != nil {
		// The Go toolchain container of the merge check holds no docker. Where docker is expected
		// (MESH_REQUIRE_DOCKER=1, which merge-check.sh sets for its docker step) its absence is a failure.
		if os.Getenv("MESH_REQUIRE_DOCKER") == "1" {
			t.Fatalf("MESH_REQUIRE_DOCKER=1 and there is no docker here: %v", err)
		}
		t.Skipf("NOT RUN: this test starts a throwaway nats container and there is no docker in this toolchain (%v); "+
			"it is proof only where merge-check.sh's docker step runs it (MESH_CHECK_DOCKER=1)", err)
	}
}

func docker(t *testing.T, args ...string) string {
	t.Helper()
	out, err := exec.Command("docker", args...).CombinedOutput()
	if err != nil {
		t.Fatalf("docker %s: %v\n%s", strings.Join(args, " "), err, out)
	}
	return strings.TrimSpace(string(out))
}

func owner() string { return fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid()) }

// server is one throwaway container.
type server struct {
	name string
	port string
}

// startServer runs the bus's release with a configuration, as this user so what it writes into a
// mounted directory is removable by the test.
func startServer(t *testing.T, conf string, mounts ...string) server {
	t.Helper()
	dir := t.TempDir()
	if err := os.WriteFile(filepath.Join(dir, "nats.conf"), []byte(conf), 0o644); err != nil {
		t.Fatal(err)
	}
	name := fmt.Sprintf("mesh-snapshot-test-%d-%d", os.Getpid(), time.Now().UnixNano())
	args := []string{"run", "-d", "--rm", "--name", name, "--user", owner(), "-p", "127.0.0.1::4222",
		"-v", dir + ":/etc/nats:ro"}
	data := false
	for _, m := range mounts {
		args = append(args, "-v", m)
		data = data || strings.HasSuffix(m, ":/data")
	}
	if !data {
		// The store on a directory of the test's, so the server writes it as this user and the
		// test removes it.
		args = append(args, "-v", t.TempDir()+":/data")
	}
	args = append(args, image, "-c", "/etc/nats/nats.conf")
	docker(t, args...)
	t.Cleanup(func() { _ = exec.Command("docker", "rm", "-f", name).Run() })
	mapped := docker(t, "port", name, "4222/tcp")
	_, port, _ := strings.Cut(strings.Split(mapped, "\n")[0], ":")
	return server{name: name, port: port}
}

// selfSigned is a certificate with no names, the ordinary case for a mesh's bus, and its pin.
func selfSigned(t *testing.T, dir string) string {
	t.Helper()
	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
	if err != nil {
		t.Fatal(err)
	}
	tmpl := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "bus"},
		NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour)}
	der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
	if err != nil {
		t.Fatal(err)
	}
	keyDER, _ := x509.MarshalECPrivateKey(key)
	_ = os.WriteFile(filepath.Join(dir, "tls.crt"), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644)
	_ = os.WriteFile(filepath.Join(dir, "tls.key"), pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o644)
	sum := sha256.Sum256(der)
	return "sha256:" + hex.EncodeToString(sum[:])
}

func quotedList(subjects []string) string {
	q := make([]string, len(subjects))
	for i, s := range subjects {
		q[i] = strconv.Quote(s)
	}
	return strings.Join(q, ", ")
}

// liveBusConf is a server shaped like the mesh's: TLS, the one account, an administrator to fill it,
// and the bus's own module with exactly the grants the controller composes for it.
func liveBusConf() string {
	return fmt.Sprintf(`listen: "0.0.0.0:4222"
tls { cert_file: "/tls/tls.crt", key_file: "/tls/tls.key" }
jetstream { store_dir: "/data" }
accounts { MESH { jetstream: enabled, users: [
  { user: "admin", password: "admin" },
  { user: %q, password: "snap", permissions: {
      publish: { allow: [%s] },
      subscribe: { allow: [%s] } } }
] } }
`, snapshotUser, quotedList(snapshotGrants.publish), quotedList(snapshotGrants.subscribe))
}

// plainConf is a fresh server with the one account and an administrator, nothing in it.
func plainConf() string {
	return `listen: "0.0.0.0:4222"
jetstream { store_dir: "/data" }
accounts { MESH { jetstream: enabled, users: [ { user: "admin", password: "admin" } ] } }
`
}

func dial(t *testing.T, url string, c Credential, errs chan<- error) *nats.Conn {
	t.Helper()
	var nc *nats.Conn
	var err error
	for deadline := time.Now().Add(20 * time.Second); ; {
		nc, err = connect(url, c)
		if err == nil {
			break
		}
		if time.Now().After(deadline) {
			t.Fatal(err)
		}
		time.Sleep(200 * time.Millisecond)
	}
	if errs != nil {
		nc.SetErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {
			select {
			case errs <- err:
			default:
			}
		})
	}
	t.Cleanup(nc.Close)
	return nc
}

func messageCount() int {
	if n, err := strconv.Atoi(os.Getenv("MESH_TEST_SNAPSHOT_MESSAGES")); err == nil && n > 0 {
		return n
	}
	return 6000
}

// fill gives the source what the mesh's bus holds: events with headers and a gap deleted from the
// middle, a work queue partly worked, key-value buckets with history, deletes and a purge, an
// object in chunks, durable consumers part way through, and a memory stream.
func fill(t *testing.T, js nats.JetStreamContext) {
	t.Helper()
	must := func(err error) {
		t.Helper()
		if err != nil {
			t.Fatal(err)
		}
	}
	_, err := js.AddStream(&nats.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"},
		MaxMsgsPerSubject: 10000, MaxAge: 7 * 24 * time.Hour})
	must(err)
	r := mrand.New(mrand.NewSource(1))
	payload := make([]byte, 2048)
	total := messageCount()
	var acks []nats.PubAckFuture
	for i := 0; i < total; i++ {
		r.Read(payload)
		m := nats.NewMsg(fmt.Sprintf("mesh.mod.m%d.event.happened", i%30))
		m.Header.Set("Mesh-Seq", strconv.Itoa(i))
		m.Data = append([]byte(nil), payload[:200+r.Intn(1800)]...)
		f, err := js.PublishMsgAsync(m)
		must(err)
		acks = append(acks, f)
		if len(acks) == 2000 {
			<-js.PublishAsyncComplete()
			acks = acks[:0]
		}
	}
	<-js.PublishAsyncComplete()
	for seq := uint64(100); seq < 120; seq++ {
		must(js.DeleteMsg("EVENTS", seq))
	}
	_, err = js.AddConsumer("EVENTS", &nats.ConsumerConfig{Durable: "audit", AckPolicy: nats.AckExplicitPolicy})
	must(err)
	sub, err := js.PullSubscribe("mesh.mod.*.event.>", "audit", nats.Bind("EVENTS", "audit"))
	must(err)
	msgs, err := sub.Fetch(500, nats.MaxWait(5*time.Second))
	must(err)
	for _, m := range msgs {
		must(m.Ack())
	}

	_, err = js.AddStream(&nats.StreamConfig{Name: "CONTROL", Subjects: []string{"mesh.control.*.report"},
		Retention: nats.WorkQueuePolicy})
	must(err)
	for i := 0; i < 50; i++ {
		_, err := js.Publish(fmt.Sprintf("mesh.control.n%d.report", i%4), []byte(fmt.Sprintf("report %d", i)))
		must(err)
	}
	_, err = js.AddConsumer("CONTROL", &nats.ConsumerConfig{Durable: "controller", AckPolicy: nats.AckExplicitPolicy})
	must(err)
	wsub, err := js.PullSubscribe("mesh.control.*.report", "controller", nats.Bind("CONTROL", "controller"))
	must(err)
	worked, err := wsub.Fetch(20, nats.MaxWait(5*time.Second))
	must(err)
	for _, m := range worked {
		must(m.AckSync())
	}

	conditions, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "mesh-controller_conditions"})
	must(err)
	history, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "mesh-controller_condition-history", History: 5})
	must(err)
	for i := 0; i < 200; i++ {
		key := fmt.Sprintf("machine.n%d.silent", i%40)
		_, err := conditions.Put(key, []byte(fmt.Sprintf(`{"raised":%d}`, i)))
		must(err)
		_, err = history.Put(key, []byte(fmt.Sprintf(`{"at":%d}`, i)))
		must(err)
	}
	for i := 0; i < 5; i++ {
		must(conditions.Delete(fmt.Sprintf("machine.n%d.silent", i)))
	}
	must(history.Purge("machine.n7.silent"))
	_, err = js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "mesh-controller_lease", TTL: time.Hour})
	must(err)

	objects, err := js.CreateObjectStore(&nats.ObjectStoreConfig{Bucket: "artifacts"})
	must(err)
	big := make([]byte, 3<<20)
	r.Read(big)
	_, err = objects.PutBytes("facts/latest", big)
	must(err)

	_, err = js.AddStream(&nats.StreamConfig{Name: "SCRATCH", Subjects: []string{"scratch.>"}, Storage: nats.MemoryStorage})
	must(err)
	_, err = js.Publish("scratch.x", []byte("gone at a restart"))
	must(err)
}

// sameContent holds every stream the manifest names, in the restored server, to the source: the same
// state at the snapshot, every message by sequence (subject, headers, body, and a deleted one still
// deleted), and the durable consumers where they were.
func sameContent(t *testing.T, m *busarchive.Manifest, src, dst nats.JetStreamContext) {
	t.Helper()
	for _, s := range m.Streams {
		info, err := dst.StreamInfo(s.Name)
		if err != nil {
			t.Errorf("%s was not restored: %v", s.Name, err)
			continue
		}
		if err := heldTo(streamState{Messages: info.State.Msgs, LastSeq: info.State.LastSeq}, s); err != nil {
			t.Errorf("%s: %v", s.Name, err)
		}
		// **Up to the snapshot's last sequence, exactly the source**: every message by sequence —
		// subject, headers and body — and a message deleted before the snapshot still deleted. Past it,
		// what was written while the blocks were read out: a message there may or may not have made it
		// (the server's state says the stream reached it before the block holding it was read), and
		// one that did is the source's. The source only grows after the snapshot.
		for seq := info.State.FirstSeq; seq <= info.State.LastSeq && seq > 0; seq++ {
			want, werr := src.GetMsg(s.Name, seq)
			got, gerr := dst.GetMsg(s.Name, seq)
			switch {
			case errors.Is(werr, nats.ErrMsgNotFound):
				if gerr == nil {
					t.Errorf("%s %d was deleted and came back", s.Name, seq)
				}
				continue
			case errors.Is(gerr, nats.ErrMsgNotFound):
				if seq <= s.LastSeq {
					t.Errorf("%s %d was in the stream at its snapshot and is missing from the restore", s.Name, seq)
				}
				continue
			case werr != nil || gerr != nil:
				t.Fatalf("%s %d: source %v, restored %v", s.Name, seq, werr, gerr)
			}
			if got.Subject != want.Subject || !bytes.Equal(got.Data, want.Data) || fmt.Sprint(got.Header) != fmt.Sprint(want.Header) {
				t.Fatalf("%s %d differs: %s %q vs %s %q", s.Name, seq, got.Subject, truncate(got.Data), want.Subject, truncate(want.Data))
			}
		}
		if info.State.LastSeq > s.LastSeq {
			t.Logf("%s: restored to sequence %d, written while the snapshot read it; the manifest said %d", s.Name, info.State.LastSeq, s.LastSeq)
		}
		for _, consumer := range []string{"audit", "controller"} {
			wantC, err := src.ConsumerInfo(s.Name, consumer)
			if err != nil {
				continue
			}
			gotC, err := dst.ConsumerInfo(s.Name, consumer)
			if err != nil {
				t.Errorf("%s's consumer %s was not restored: %v", s.Name, consumer, err)
				continue
			}
			if gotC.AckFloor.Stream != wantC.AckFloor.Stream {
				t.Errorf("%s's consumer %s acknowledged up to %d, the source up to %d", s.Name, consumer,
					gotC.AckFloor.Stream, wantC.AckFloor.Stream)
			}
		}
	}
	for _, s := range m.Skipped {
		if _, err := dst.StreamInfo(s.Name); err == nil {
			t.Errorf("%s was skipped and yet exists", s.Name)
		}
	}
	// And the buckets work as buckets, not only as streams.
	kv, err := dst.KeyValue("mesh-controller_condition-history")
	if err != nil {
		t.Fatalf("the condition history is not a bucket after the restore: %v", err)
	}
	h, err := kv.History("machine.n3.silent")
	if err != nil || len(h) != 5 {
		t.Errorf("the history of a key came back as %d entries (%v), not 5", len(h), err)
	}
	if _, err := kv.Get("machine.n7.silent"); !errors.Is(err, nats.ErrKeyNotFound) {
		t.Errorf("a purged key came back: %v", err)
	}
	cond, _ := dst.KeyValue("mesh-controller_conditions")
	if _, err := cond.Get("machine.n1.silent"); !errors.Is(err, nats.ErrKeyNotFound) {
		t.Errorf("a deleted key came back: %v", err)
	}
	if e, err := cond.Get("machine.n10.silent"); err != nil || string(e.Value()) != `{"raised":170}` {
		t.Errorf("a condition came back wrong: %v", err)
	}
	srcObj, _ := src.ObjectStore("artifacts")
	dstObj, err := dst.ObjectStore("artifacts")
	if err != nil {
		t.Fatalf("the object store was not restored: %v", err)
	}
	want, _ := srcObj.GetBytes("facts/latest")
	got, err := dstObj.GetBytes("facts/latest")
	if err != nil || !bytes.Equal(got, want) {
		t.Errorf("the object came back different (%d bytes, %v)", len(got), err)
	}
}

func TestASnapshotOfALiveBusRestoresToTheSameContent(t *testing.T) {
	needDocker(t)
	tlsDir := t.TempDir()
	pin := selfSigned(t, tlsDir)
	source := startServer(t, liveBusConf(), tlsDir+":/tls:ro")
	url := "tls://127.0.0.1:" + source.port

	admin := dial(t, url, Credential{Fingerprint: pin, User: "admin", Password: "admin"}, nil)
	srcJS, _ := admin.JetStream()
	fill(t, srcJS)

	// The bus goes on taking writes while it is snapshotted, and every one is accepted.
	writer := dial(t, url, Credential{Fingerprint: pin, User: "admin", Password: "admin"}, nil)
	writerJS, _ := writer.JetStream()
	stop := make(chan struct{})
	var wrote, failed atomic.Int64
	var slowest atomic.Int64
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		for {
			select {
			case <-stop:
				return
			default:
			}
			began := time.Now()
			if _, err := writerJS.Publish("mesh.mod.writer.event.tick", []byte("during")); err != nil {
				failed.Add(1)
			} else {
				wrote.Add(1)
			}
			if d := time.Since(began).Milliseconds(); d > slowest.Load() {
				slowest.Store(d)
			}
			time.Sleep(2 * time.Millisecond)
		}
	}()

	snap := dial(t, url, Credential{Fingerprint: pin, User: snapshotUser, Password: "snap"}, nil)
	var archive bytes.Buffer
	var said []string
	var sayMu sync.Mutex
	m, err := Snapshot(context.Background(), snap, &archive, SnapshotOptions{StreamTimeout: time.Minute, Pause: 50 * time.Millisecond,
		Log: func(f string, a ...any) { sayMu.Lock(); said = append(said, fmt.Sprintf(f, a...)); sayMu.Unlock() }})
	time.Sleep(100 * time.Millisecond)
	close(stop)
	wg.Wait()
	if err != nil {
		t.Fatalf("the snapshot failed: %v\n%s", err, strings.Join(said, "\n"))
	}
	t.Logf("snapshot: %d streams, %d messages, %s of streams → %s archived in %.2fs; %d writes during it, %d refused, slowest %dms",
		len(m.Streams), m.Messages, human(m.StreamBytes), human(m.ArchiveBytes), m.Seconds, wrote.Load(), failed.Load(), slowest.Load())
	for _, line := range said {
		t.Log(line)
	}
	if failed.Load() > 0 {
		t.Errorf("%d writes to the bus failed while it was snapshotted", failed.Load())
	}
	if len(m.Skipped) != 1 || m.Skipped[0].Name != "SCRATCH" {
		t.Errorf("the memory stream was not said as skipped: %+v", m.Skipped)
	}
	var names []string
	for _, s := range m.Streams {
		names = append(names, s.Name)
	}
	wantNames := "CONTROL EVENTS KV_mesh-controller_condition-history KV_mesh-controller_conditions KV_mesh-controller_lease OBJ_artifacts"
	if strings.Join(names, " ") != wantNames {
		t.Errorf("snapshotted %v, want %s", names, wantNames)
	}

	tarPath := filepath.Join(t.TempDir(), "bus.tar")
	if err := os.WriteFile(tarPath, archive.Bytes(), 0o644); err != nil {
		t.Fatal(err)
	}
	a, err := busarchive.ReadArchive(bytes.NewReader(archive.Bytes()), "")
	if err != nil {
		t.Fatal(err)
	}
	defer a.Close()
	if err := a.Verify(); err != nil {
		t.Fatalf("a fresh snapshot does not verify: %v", err)
	}

	t.Run("nothing is restored over a live stream", func(t *testing.T) {
		err := RestoreAll(context.Background(), admin, a, t.Logf)
		if err == nil || !strings.Contains(err.Error(), "already holds") {
			t.Fatalf("restoring over the live streams was not refused: %v", err)
		}
	})

	t.Run("into a fresh server", func(t *testing.T) {
		fresh := startServer(t, plainConf())
		nc := dial(t, "nats://127.0.0.1:"+fresh.port, Credential{User: "admin", Password: "admin"}, nil)
		if err := RestoreAll(context.Background(), nc, a, t.Logf); err != nil {
			t.Fatal(err)
		}
		dstJS, _ := nc.JetStream()
		sameContent(t, m, srcJS, dstJS)
	})

	t.Run("into a new store, swapped in and served", func(t *testing.T) {
		binary := filepath.Join(t.TempDir(), "mesh-nats-snapshot")
		build := exec.Command("go", "build", "-o", binary, ".")
		build.Env = append(os.Environ(), "CGO_ENABLED=0")
		if out, err := build.CombinedOutput(); err != nil {
			t.Fatalf("building the program: %v\n%s", err, out)
		}
		out := t.TempDir()
		// The program in the bus's own image, as a person runs it: the archive in, a new store out.
		docker(t, "run", "--rm", "--user", owner(),
			"-v", binary+":/usr/local/bin/mesh-nats-snapshot:ro",
			"-v", tarPath+":/in/bus.tar:ro", "-v", out+":/out",
			"--entrypoint", "/usr/local/bin/mesh-nats-snapshot", image,
			"restore", "--into", "/out/store", "--from", "/in/bus.tar")
		if _, err := os.Stat(filepath.Join(out, "store", "jetstream", "MESH", "streams", "EVENTS")); err != nil {
			t.Fatalf("the new store is not laid out as the bus's: %v", err)
		}
		served := startServer(t, plainConf(), filepath.Join(out, "store")+":/data")
		nc := dial(t, "nats://127.0.0.1:"+served.port, Credential{User: "admin", Password: "admin"}, nil)
		dstJS, _ := nc.JetStream()
		sameContent(t, m, srcJS, dstJS)
	})

	t.Run("a damaged archive is refused", func(t *testing.T) {
		damaged := append([]byte(nil), archive.Bytes()...)
		// A byte in the last stream's archive, well past every header.
		damaged[len(damaged)-2048] ^= 0xff
		d, err := busarchive.ReadArchive(bytes.NewReader(damaged), "")
		if err == nil {
			defer d.Close()
			err = d.Verify()
		}
		if err == nil {
			t.Fatal("a damaged archive verified")
		}
	})
}

// The bus's own module may snapshot and do nothing else: every API that changes a stream, every
// publish into one, and every subscription beyond its own inbox is refused by the server.
func TestTheSnapshotUserCannotChangeTheBus(t *testing.T) {
	needDocker(t)
	tlsDir := t.TempDir()
	pin := selfSigned(t, tlsDir)
	source := startServer(t, liveBusConf(), tlsDir+":/tls:ro")
	url := "tls://127.0.0.1:" + source.port
	admin := dial(t, url, Credential{Fingerprint: pin, User: "admin", Password: "admin"}, nil)
	js, _ := admin.JetStream()
	if _, err := js.AddStream(&nats.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"}}); err != nil {
		t.Fatal(err)
	}
	if _, err := js.Publish("mesh.mod.a.event.b", []byte("x")); err != nil {
		t.Fatal(err)
	}
	if _, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "mesh-controller_conditions"}); err != nil {
		t.Fatal(err)
	}

	errs := make(chan error, 64)
	snap := dial(t, url, Credential{Fingerprint: pin, User: snapshotUser, Password: "snap"}, errs)
	for _, subject := range []string{
		"$JS.API.STREAM.CREATE.NEW", "$JS.API.STREAM.UPDATE.EVENTS", "$JS.API.STREAM.DELETE.EVENTS",
		"$JS.API.STREAM.PURGE.EVENTS", "$JS.API.STREAM.MSG.DELETE.EVENTS", "$JS.API.STREAM.RESTORE.NEW",
		"$JS.API.CONSUMER.CREATE.EVENTS", "$JS.API.STREAM.MSG.GET.EVENTS",
		"mesh.mod.a.event.b", "$KV.mesh-controller_conditions.k",
	} {
		_, err := snap.Request(subject, []byte(`{}`), 500*time.Millisecond)
		if err == nil {
			t.Errorf("%s was answered for the snapshot user", subject)
		}
		select {
		case e := <-errs:
			if !strings.Contains(strings.ToLower(e.Error()), "permissions violation") {
				t.Errorf("%s: %v", subject, e)
			}
		case <-time.After(2 * time.Second):
			t.Errorf("the server did not refuse %s", subject)
		}
	}
	for _, subject := range []string{"mesh.>", "_INBOX.other.>", "$JS.API.>"} {
		if _, err := snap.SubscribeSync(subject); err != nil {
			t.Fatal(err)
		}
		_ = snap.Flush()
		select {
		case e := <-errs:
			if !strings.Contains(strings.ToLower(e.Error()), "permissions violation") {
				t.Errorf("subscribing %s: %v", subject, e)
			}
		case <-time.After(2 * time.Second):
			t.Errorf("the server let the snapshot user subscribe %s", subject)
		}
	}
	info, err := js.StreamInfo("EVENTS")
	if err != nil || info.State.Msgs != 1 {
		t.Fatalf("the stream changed: %v %+v", err, info)
	}
	// And what it may do, it can.
	names, err := StreamNames(context.Background(), snap)
	if err != nil || len(names) != 2 {
		t.Fatalf("the snapshot user cannot list the streams: %v %v", names, err)
	}
}