mesh-controller/cmd/mesh-controller/process_live_test.go

//go:build integration

package main

import (
	"errors"
	"os"
	"path/filepath"
	"testing"
	"time"

	"github.com/nats-io/nats-server/v2/server"
	"github.com/nats-io/nats.go"
	"github.com/nats-io/nats.go/jetstream"

	"github.com/novox/mesh-controller/internal/broker"
	"github.com/novox/mesh-controller/internal/catalogue"
	"github.com/novox/mesh-controller/internal/inventory"
)

// mailSenderLiveManifest is mesh-mail's future manifest shape whole (novox/hq ADR 0436 §7, §8): the
// process "send" declares every clause of bus (emits, its one work-queue stream, its state, asks, and its
// invoked seat verb), so the account module issue delivers for it can be put through every grant at once.
func mailSenderLiveManifest(t *testing.T) catalogue.Manifest {
	t.Helper()
	m, err := catalogue.ParseManifest([]byte(`{"module":"mesh-mail","version":"1",
		"emits":["mail.send.sent","mail.send.failed"],"state":["sends"],
		"streams":[{"name":"mail-sends","events":["mail.send.approved"],"max-age-days":1,"max-mib":8,
			"retention":"work-queue","max-deliveries":5}],
		"own-secrets":{"broker-send":{"path":"/var/lib/mesh-mail/broker-send","owner":"mesh-mail-send"}},
		"resources":[{"id":"account","type":"user","name":"mesh-mail-send"},
			{"id":"send","type":"process","run":["./mesh-mail","send"],"user":"mesh-mail-send",
			 "bus":{"emits":["mail.send.sent","mail.send.failed"],"queues":["mail-sends"],"state":["sends"],
				"asks":true,"invokes":["seat:operator-channel.tell"]}}]}`))
	if err != nil {
		t.Fatal(err)
	}
	return m
}

// operatorChannelWithTell is operator-channel as messenger declares it (broker.operatorChannel, novox/hq
// ADR 0259 §3), with the one verb this release adds: `tell` (novox/hq ADR 0436 §1's own fixture), so the
// process's invoked seat verb has a holder to answer it.
func operatorChannelWithTell() broker.Seat {
	return broker.Seat{Name: "operator-channel", Scope: "mesh", Accepts: []string{"ask", "cancel"},
		Emits: []string{"decided"}, Serves: []string{"open", "history", "notify", "tell"},
		ByCaller: []string{"ask", "cancel", "decided"}, Records: []string{"messenger_asks"}}
}

// review round 1, finding 5 (1181/AC3), the release's acceptance test, on a real bus and the store: the
// account module issue delivers for mesh-mail's process "send" — composed from the manifest the real way
// (inventory.ProcessesOf -> broker.Users -> broker.ComposeAccounts), minted with the password module issue
// mints (aMesh) — does exactly what its bus declares and nothing else.
func TestAProcessAccountOnARealBusAndStore(t *testing.T) {
	open := aMesh(t)
	ctx := t.Context()
	inv := open.inventory

	m := mailSenderLiveManifest(t)
	operatorChannel := operatorChannelWithTell()
	mailSends := broker.ModuleStream{Module: "mesh-mail", Name: "MAIL_SENDS", Events: []string{"mail.send.approved"},
		MaxAgeDays: 1, MaxMiB: 8, WorkQueue: true, Duplicates: 1}
	records := broker.Records{Nodes: []string{"laptop"}, Assigned: map[string][]broker.Declared{"laptop": {
		{Module: broker.RuntimeModule},
		{Module: "messenger", Holds: []broker.Seat{operatorChannel}, RunsAs: "messenger"},
		{Module: "mesh-mail", NoAccount: true, Uses: []broker.Seat{operatorChannel}, Streams: []broker.ModuleStream{mailSends},
			Processes: inventory.ProcessesOf(m)},
	}}}
	users, err := broker.Users(records)
	if err != nil {
		t.Fatal(err)
	}

	// The four principals this test drives, each minted the password module issue mints — the same
	// inv.MintBusPassword call issueProcess makes for the process account. mesh-mail publishes its own
	// stream's events as the tool runner carrying it (review round 2's finding): the controller's own
	// grants do not cover a module's stream, only the runtime that carries the module does.
	var admin, runtime, messenger, sender broker.Principal
	for _, p := range users {
		switch {
		case p.Kind == broker.KindController:
			admin = p
		case p.Kind == broker.KindNodeTools:
			runtime = p
		case p.Module == "messenger":
			messenger = p
		case p.Kind == broker.KindProcess:
			sender = p
		}
	}
	if admin.Kind == "" || runtime.Kind == "" || messenger.Kind == "" || sender.Kind == "" {
		t.Fatalf("not every principal was composed: admin=%+v runtime=%+v messenger=%+v sender=%+v",
			admin, runtime, messenger, sender)
	}
	if sender.Username() != "laptop.mesh-mail.send" {
		t.Fatalf("the process's username: %q", sender.Username())
	}

	passwords := map[string]string{}
	hashes := map[string]string{}
	for _, p := range []struct {
		principal broker.Principal
		kind      string
	}{{admin, inventory.BusController}, {runtime, inventory.BusNodeTools}, {messenger, inventory.BusModule},
		{sender, inventory.BusProcess}} {
		pw, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.principal.Username(), Kind: p.kind,
			Node: p.principal.Node, Module: p.principal.Module})
		if err != nil {
			t.Fatal(err)
		}
		passwords[p.principal.Username()] = pw
		hash, held, err := inv.BusUserHash(ctx, p.principal.Username())
		if err != nil || !held {
			t.Fatalf("the minted password of %s was not recorded: %v, %v", p.principal.Username(), held, err)
		}
		hashes[p.principal.Username()] = hash
	}
	filled, missing := broker.WithPasswords([]broker.Principal{admin, runtime, messenger, sender}, hashes)
	if len(missing) > 0 {
		t.Fatalf("no password for %v", missing)
	}

	accounts, err := broker.ComposeAccounts(filled)
	if err != nil {
		t.Fatal(err)
	}
	conf := filepath.Join(t.TempDir(), "bus.conf")
	if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+
		`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
		t.Fatal(err)
	}
	opts, err := server.ProcessConfigFile(conf)
	if err != nil {
		t.Fatalf("the composed accounts do not parse: %v", err)
	}
	opts.NoLog, opts.NoSigs = true, true
	s, err := server.NewServer(opts)
	if err != nil {
		t.Fatal(err)
	}
	go s.Start()
	if !s.ReadyForConnections(10 * time.Second) {
		t.Fatal("the bus did not come up")
	}
	defer s.Shutdown()

	// Every connection names its own inbox prefix: the grant is `_INBOX.<username>.>`, and the client
	// library's own default random inbox matches nothing granted, so replies are refused without it.
	adminConn, err := broker.Dial(s.ClientURL(), nats.UserInfo(admin.Username(), passwords[admin.Username()]),
		nats.CustomInboxPrefix("_INBOX."+admin.Username()))
	if err != nil {
		t.Fatal(err)
	}
	defer adminConn.Close()
	if err := adminConn.EnsureStream(mailSends.Stream()); err != nil {
		t.Fatal(err)
	}
	if err := adminConn.EnsureConsumer(broker.QueueConsumer(sender, sender.Queues[0])); err != nil {
		t.Fatal(err)
	}
	if err := adminConn.EnsureBucket(broker.Bucket{Module: "mesh-mail", Name: "sends"}); err != nil {
		t.Fatal(err)
	}
	js, err := jetstream.New(adminConn.Conn())
	if err != nil {
		t.Fatal(err)
	}
	if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "messenger_asks"}); err != nil {
		t.Fatal(err)
	}

	// The machine's tool runner, which carries mesh-mail (it has no account of its own) and so is the
	// principal that publishes its own stream's events on its behalf.
	runtimeConn, err := nats.Connect(s.ClientURL(), nats.UserInfo(runtime.Username(), passwords[runtime.Username()]),
		nats.CustomInboxPrefix("_INBOX."+runtime.Username()))
	if err != nil {
		t.Fatal(err)
	}
	defer runtimeConn.Close()

	// messenger, the router, answers exactly its granted verbs — never a wildcard the grant does not give it.
	holder, err := nats.Connect(s.ClientURL(), nats.UserInfo(messenger.Username(), passwords[messenger.Username()]),
		nats.CustomInboxPrefix("_INBOX."+messenger.Username()))
	if err != nil {
		t.Fatal(err)
	}
	defer holder.Close()
	heard := make(chan string, 1)
	for _, verb := range []string{"open", "history", "notify", "tell"} {
		if _, err := holder.Subscribe("mesh.seat.operator-channel.tool."+verb, func(m *nats.Msg) {
			heard <- m.Subject
			_ = m.Respond([]byte(`{"result":{}}`))
		}); err != nil {
			t.Fatal(err)
		}
	}
	_ = holder.Flush()

	refusals := make(chan error, 8)
	sender2, err := nats.Connect(s.ClientURL(), nats.UserInfo(sender.Username(), passwords[sender.Username()]),
		nats.CustomInboxPrefix("_INBOX."+sender.Username()),
		nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refusals <- err }))
	if err != nil {
		t.Fatal(err)
	}
	defer sender2.Close()

	// 1. Its event.
	if err := sender2.Publish("mesh.mod.mesh-mail.event.mail.send.sent", []byte("{}")); err != nil {
		t.Fatal(err)
	}

	// 2. Its state: writes and reads its own bucket.
	senderKV, err := jetstream.New(sender2)
	if err != nil {
		t.Fatal(err)
	}
	bucket, err := senderKV.KeyValue(ctx, "mesh-mail_sends")
	if err != nil {
		t.Fatal(err)
	}
	if _, err := bucket.Put(ctx, "key", []byte("value")); err != nil {
		t.Fatalf("the process may not write its own state: %v", err)
	}
	if got, err := bucket.Get(ctx, "key"); err != nil || string(got.Value()) != "value" {
		t.Fatalf("the process may not read its own state back: %v, %v", got, err)
	}

	// 3. Its queue: pulls and acknowledges MAIL_SENDS through its own consumer. Published as the tool
	// runner carrying mesh-mail — the live path a module's own stream's event takes — never as the
	// controller, whose own grants do not cover a module's stream at all.
	runtimeJS, err := jetstream.New(runtimeConn)
	if err != nil {
		t.Fatal(err)
	}
	if _, err := runtimeJS.Publish(ctx, broker.ModuleStreamSubject("mesh-mail", "mail.send.approved"), []byte("{}")); err != nil {
		t.Fatal(err)
	}
	reading, err := jetstream.New(sender2)
	if err != nil {
		t.Fatal(err)
	}
	consumer, err := reading.Consumer(ctx, "MAIL_SENDS", broker.QueueConsumer(sender, sender.Queues[0]).Name)
	if err != nil {
		t.Fatalf("the process may not bind its own queue consumer: %v", err)
	}
	batch, err := consumer.Fetch(1, jetstream.FetchMaxWait(3*time.Second))
	if err != nil {
		t.Fatal(err)
	}
	got := false
	for msg := range batch.Messages() {
		got = true
		if err := msg.Ack(); err != nil {
			t.Fatalf("the process may not acknowledge its own queue: %v", err)
		}
	}
	if !got {
		t.Fatal("the process did not pull the message its own queue holds")
	}

	// 4. Its invoked seat verb, in its own name.
	if _, err := sender2.Request("mesh.seat.operator-channel.tool.tell", []byte(`{}`), 3*time.Second); err != nil {
		t.Fatalf("the process may not call operator-channel.tell: %v", err)
	}
	select {
	case s := <-heard:
		if s != "mesh.seat.operator-channel.tool.tell" {
			t.Fatalf("messenger heard %s", s)
		}
	case <-time.After(time.Second):
		t.Fatal("messenger never heard the call")
	}

	// 5. Refused, one at a time, so each refusal is counted against the attempt that caused it rather than
	// any other: answering a tool call it holds no seat and serves no tool for.
	if _, err := sender2.Subscribe("mesh.mod.mesh-mail.tool.>", func(*nats.Msg) {}); err != nil {
		t.Fatal(err)
	}
	_ = sender2.Flush()
	select {
	case err := <-refusals:
		if !errors.Is(err, nats.ErrPermissionViolation) {
			t.Errorf("subscribing a tool call was not refused as a permissions violation: %v", err)
		}
	case <-time.After(3 * time.Second):
		t.Error("subscribing a tool call the process does not serve was not refused")
	}
	// 6. Refused: publishing a warrant — only the holder, messenger, may say operator-channel decided.
	if err := sender2.Publish("mesh.seat.operator-channel.event.decided.mesh-mail", []byte("{}")); err != nil {
		t.Fatal(err)
	}
	_ = sender2.Flush()
	select {
	case err := <-refusals:
		if !errors.Is(err, nats.ErrPermissionViolation) {
			t.Errorf("publishing a warrant was not refused as a permissions violation: %v", err)
		}
	case <-time.After(3 * time.Second):
		t.Error("publishing a warrant the process does not hold the seat for was not refused")
	}
}