mesh-controller/internal/broker/provisioner_server_live_test.go

package broker

import (
	"context"
	"errors"
	"os"
	"path/filepath"
	"testing"
	"time"

	"github.com/nats-io/nats-server/v2/server"
	"github.com/nats-io/nats.go"
	"github.com/nats-io/nats.go/jetstream"
	"golang.org/x/crypto/bcrypt"
)

// **On a real server, as composed** (design 57 package P4, issue 670): the provisioner server's buckets are
// written by the server's own account whole, and by each machine's provisioner client for its own machine's keys
// alone (ADR 0349 clause 3). The server refuses a client another machine's needs, applied or items, and the
// registry; and refuses every other account a write, a provider reading the needs included.
func TestOnlyTheServerAndEachClientForItsOwnMachineWriteTheProvisionersBuckets(t *testing.T) {
	hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
	if err != nil {
		t.Fatal(err)
	}
	pw := string(hash)
	accounts, err := ComposeAccounts([]Principal{
		{Kind: KindController, PasswordHash: pw},
		{Kind: KindModule, Node: "novox", Module: ProvisionerServerModule,
			State: []string{"registry", "needs", "applied", "collect"}, PasswordHash: pw},
		// A provider learns its consumers from the needs (ADR 0349 clause 12): it reads them, and writes none.
		{Kind: KindModule, Node: "novox", Module: "postgres", Reads: []string{"mesh-provisioner.needs"}, PasswordHash: pw},
		{Kind: KindModule, Node: "g14", Module: "dunst", PasswordHash: pw},
		{Kind: KindProvisioner, Node: "g14", Module: ProvisionerServerModule, PasswordHash: pw},
		{Kind: KindProvisioner, Node: "shanks", Module: ProvisionerServerModule, PasswordHash: pw},
	})
	if err != nil {
		t.Fatal(err)
	}
	conf := filepath.Join(t.TempDir(), "bus.conf")
	if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+
		`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
		t.Fatal(err)
	}
	opts, err := server.ProcessConfigFile(conf)
	if err != nil {
		t.Fatalf("the composed accounts do not parse: %v", err)
	}
	opts.NoLog, opts.NoSigs = true, true
	s, err := server.NewServer(opts)
	if err != nil {
		t.Fatal(err)
	}
	go s.Start()
	if !s.ReadyForConnections(10 * time.Second) {
		t.Fatal("the bus did not come up")
	}
	defer s.Shutdown()
	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
	defer cancel()

	type account struct {
		js       jetstream.JetStream
		refusals chan error
	}
	connect := func(user string) account {
		t.Helper()
		refusals := make(chan error, 16)
		nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, "pw"), nats.CustomInboxPrefix("_INBOX."+user),
			nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refusals <- err }))
		if err != nil {
			t.Fatal(err)
		}
		t.Cleanup(nc.Close)
		js, err := jetstream.New(nc)
		if err != nil {
			t.Fatal(err)
		}
		return account{js: js, refusals: refusals}
	}
	bucket := func(a account, name string) jetstream.KeyValue {
		t.Helper()
		kv, err := a.js.KeyValue(ctx, name)
		if err != nil {
			t.Fatalf("binding to %s: %v", name, err)
		}
		return kv
	}

	// The controller makes the buckets from the server's manifest, as it does for every module's state.
	control := connect("controller")
	for _, name := range append([]string{ProvisionerRegistryBucket}, ProvisionerWrites...) {
		if _, err := control.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name}); err != nil {
			t.Fatal(err)
		}
	}

	// The server writes all four, every machine's keys.
	srv := connect("novox." + ProvisionerServerModule)
	for name, key := range map[string]string{ProvisionerRegistryBucket: "config", ProvisionerNeedsBucket: "shanks.dunst.config.font-size",
		ProvisionerAppliedBucket: "shanks.dunst", ProvisionerCollectBucket: "shanks.zsh-snippets.git"} {
		if _, err := bucket(srv, name).PutString(ctx, key, `{}`); err != nil {
			t.Fatalf("the server writing %s in %s: %v", key, name, err)
		}
	}

	// A client writes, reads and watches its own machine's keys, and reads the registry whole.
	g14 := connect("provisioner.g14")
	own := map[string]string{ProvisionerNeedsBucket: "g14.dunst.config.font-size", ProvisionerAppliedBucket: "g14.dunst",
		ProvisionerCollectBucket: "g14.zsh-snippets.git"}
	for name, key := range own {
		kv := bucket(g14, name)
		if _, err := kv.PutString(ctx, key, `{}`); err != nil {
			t.Fatalf("the client writing its own machine's %s in %s: %v", key, name, err)
		}
		if e, err := kv.Get(ctx, key); err != nil || string(e.Value()) != `{}` {
			t.Fatalf("the client reading back its own %s in %s: %v %v", key, name, e, err)
		}
		w, err := kv.Watch(ctx, "g14.>")
		if err != nil {
			t.Fatalf("the client watching its own machine's keys in %s: %v", name, err)
		}
		select {
		case e := <-w.Updates():
			if e == nil || e.Key() != key {
				t.Fatalf("the watch of %s delivered %v first", name, e)
			}
		case <-time.After(5 * time.Second):
			t.Fatalf("the watch of its own machine's keys in %s delivered nothing", name)
		}
		_ = w.Stop()
		if err := kv.Delete(ctx, key); err != nil {
			t.Fatalf("the client deleting its own %s in %s (a module taken off its machine): %v", key, name, err)
		}
	}
	registry := bucket(g14, ProvisionerRegistryBucket)
	if e, err := registry.Get(ctx, "config"); err != nil || e == nil {
		t.Fatalf("the client reading the registry: %v %v", e, err)
	}

	// Each refusal is the server's word, not a missing key.
	serverRefuses := func(a account, what string, act func(context.Context) error) {
		t.Helper()
		for drained := false; !drained; {
			select {
			case <-a.refusals:
			default:
				drained = true
			}
		}
		short, done := context.WithTimeout(ctx, 2*time.Second)
		defer done()
		if err := act(short); err == nil {
			t.Errorf("%s was let through", what)
		}
		select {
		case err := <-a.refusals:
			if !errors.Is(err, nats.ErrPermissionViolation) {
				t.Errorf("%s: the server said %v, want a permissions violation", what, err)
			}
		case <-time.After(time.Second):
			t.Errorf("%s: the server did not refuse it", what)
		}
	}
	put := func(kv jetstream.KeyValue, key string) func(context.Context) error {
		return func(c context.Context) error { _, err := kv.PutString(c, key, `{}`); return err }
	}
	for name, key := range map[string]string{ProvisionerNeedsBucket: "shanks.dunst.config.font-size",
		ProvisionerAppliedBucket: "shanks.dunst", ProvisionerCollectBucket: "shanks.zsh-snippets.git"} {
		kv := bucket(g14, name)
		serverRefuses(g14, "g14's client writing shanks's "+key+" in "+name, put(kv, key))
		serverRefuses(g14, "g14's client reading shanks's "+key+" in "+name, func(c context.Context) error {
			_, err := kv.Get(c, key)
			return err
		})
	}
	serverRefuses(g14, "g14's client watching every machine's needs", func(c context.Context) error {
		w, err := bucket(g14, ProvisionerNeedsBucket).Watch(c, ">")
		if err != nil {
			return err
		}
		defer func() { _ = w.Stop() }()
		select {
		case e := <-w.Updates():
			if e != nil {
				return nil
			}
			return errors.New("nothing delivered")
		case <-c.Done():
			return c.Err()
		}
	})
	serverRefuses(g14, "a client writing the registry", put(registry, "config"))

	// Shanks's client is held to shanks's keys the same way: its write of g14's applied is refused.
	shanks := connect("provisioner.shanks")
	if _, err := bucket(shanks, ProvisionerAppliedBucket).PutString(ctx, "shanks.dunst", `{}`); err != nil {
		t.Fatalf("shanks's client writing its own applied: %v", err)
	}
	serverRefuses(shanks, "shanks's client writing g14's applied", put(bucket(shanks, ProvisionerAppliedBucket), "g14.dunst"))

	// A provider reads every machine's needs and writes none; a module on a client's machine writes none.
	provider := connect("novox.postgres")
	needs := bucket(provider, ProvisionerNeedsBucket)
	if e, err := needs.Get(ctx, "shanks.dunst.config.font-size"); err != nil || e == nil {
		t.Fatalf("the provider reading the needs: %v %v", e, err)
	}
	serverRefuses(provider, "a provider writing the needs", put(needs, "shanks.postgres.bound.db"))
	dunst := connect("g14.dunst")
	for _, name := range append([]string{ProvisionerRegistryBucket}, ProvisionerWrites...) {
		serverRefuses(dunst, "a module on g14 writing "+name, func(c context.Context) error {
			// Bound by the stream's own subject: the module is granted no bind, so its put is published raw.
			_, err := dunst.js.Publish(c, "$KV."+name+".g14.dunst", []byte(`{}`))
			return err
		})
	}
}