mesh-controller/internal/recordmirror/provisioning_live_test.go

package recordmirror

import (
	"context"
	"encoding/json"
	"errors"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/nats-io/nats-server/v2/server"
	"github.com/nats-io/nats.go"
	"github.com/nats-io/nats.go/jetstream"
	"golang.org/x/crypto/bcrypt"

	"github.com/novox/mesh-controller/internal/broker"
	"github.com/novox/mesh-controller/internal/inventory"
)

type silent struct{}

func (silent) PublishSeatEvent(context.Context, string, string, []byte) error { return nil }

// **The provisioner's plan reaches its machine's client, and no other** (design 57 P5, issue 671): on a real
// server composed from the grants, the controller writes each assignment with the plan of a module handed to the
// provisioner into the assignments mirror; g14's client reads and watches its own machine's plan, reads shanks's
// (who binds a provider, ADR 0373), and the server refuses it any write of the mirror.
func TestNatsTheProvisionerClientReadsEveryMachinesPlanAndWritesNone(t *testing.T) {
	hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
	if err != nil {
		t.Fatal(err)
	}
	accounts, err := broker.ComposeAccounts([]broker.Principal{
		{Kind: broker.KindController, PasswordHash: string(hash)},
		{Kind: broker.KindProvisioner, Node: "g14", Module: "mesh-provisioner-client", PasswordHash: string(hash)},
	})
	if err != nil {
		t.Fatal(err)
	}
	conf := filepath.Join(t.TempDir(), "bus.conf")
	if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+
		`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
		t.Fatal(err)
	}
	opts, err := server.ProcessConfigFile(conf)
	if err != nil {
		t.Fatalf("the composed accounts do not parse: %v", err)
	}
	opts.NoLog, opts.NoSigs = true, true
	s, err := server.NewServer(opts)
	if err != nil {
		t.Fatal(err)
	}
	go s.Start()
	if !s.ReadyForConnections(10 * time.Second) {
		t.Fatal("the bus did not come up")
	}
	defer s.Shutdown()
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()

	refusals := make(chan error, 16)
	connect := func(user string) *nats.Conn {
		t.Helper()
		nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, "pw"), nats.CustomInboxPrefix("_INBOX."+user),
			nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refusals <- err }))
		if err != nil {
			t.Fatal(err)
		}
		t.Cleanup(nc.Close)
		return nc
	}

	// The controller writes the mirror from what the store says, the plan among it.
	control, err := jetstream.New(connect("controller"))
	if err != nil {
		t.Fatal(err)
	}
	kv, err := control.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: broker.AssignmentsBucket, History: 1})
	if err != nil {
		t.Fatal(err)
	}
	plan := inventory.Provisioning{
		Templates: inventory.TemplatesArchive{Source: "artifacts/dunst/files.tar.gz",
			Digest: "sha256:" + strings.Repeat("cd", 32)},
		Files: []inventory.ProvisionedFile{{ID: "dunstrc", Path: "/home/nox/.config/dunst/dunstrc",
			Template: "dunstrc", OwnerAtRuntime: "owned", Takes: "reload-on"}},
		Placement: map[string]string{"machine:account-home": "/home/nox"},
	}
	at := time.Date(2026, 10, 14, 20, 0, 0, 0, time.UTC)
	values, err := inventory.AssignmentValues([]inventory.AssignmentMirrored{
		{Node: "g14", Module: "dunst", Assigned: at}, {Node: "shanks", Module: "dunst", Assigned: at}},
		map[string]inventory.Provisioning{"g14.dunst": plan, "shanks.dunst": plan})
	if err != nil {
		t.Fatal(err)
	}
	w := &Writer{Mirror: Assignments, KV: kv, Announce: silent{}, Holds: func() bool { return true }}
	if r, err := w.Sync(ctx, values); err != nil || r.Written != 2 {
		t.Fatalf("the controller wrote %+v (%v)", r, err)
	}

	client, err := jetstream.New(connect("provisioner.g14"))
	if err != nil {
		t.Fatal(err)
	}
	mirror, err := client.KeyValue(ctx, broker.AssignmentsBucket)
	if err != nil {
		t.Fatalf("the client cannot bind to the assignments mirror: %v", err)
	}
	watch, err := mirror.Watch(ctx, "g14.>")
	if err != nil {
		t.Fatal(err)
	}
	defer func() { _ = watch.Stop() }()
	select {
	case e := <-watch.Updates():
		var got inventory.AssignmentMirrored
		if e == nil || json.Unmarshal(e.Value(), &got) != nil || got.Provisioning == nil ||
			got.Provisioning.Files[0].Template != "dunstrc" ||
			got.Provisioning.Placement["machine:account-home"] != "/home/nox" {
			t.Fatalf("the client's watch delivered %v first", e)
		}
	case <-time.After(5 * time.Second):
		t.Fatal("the client's watch of its own machine's plans delivered nothing")
	}

	refused := func(what string, act func(context.Context) error) {
		t.Helper()
		for len(refusals) > 0 {
			<-refusals
		}
		short, done := context.WithTimeout(ctx, 2*time.Second)
		defer done()
		if err := act(short); err == nil {
			t.Errorf("the client may %s", what)
		}
		select {
		case err := <-refusals:
			if !errors.Is(err, nats.ErrPermissionViolation) {
				t.Errorf("%s: the server said %v, want a permissions violation", what, err)
			}
		case <-time.After(time.Second):
			t.Errorf("%s: the server did not refuse it", what)
		}
	}
	// Another machine's plan is read, never written: a provider here is bound by consumers on any machine, in their
	// plans, which its machine's client reads to fill ${consumers:<provision>} (novox/hq ADR 0373, issue 679).
	if _, err := mirror.Get(ctx, "shanks.dunst"); err != nil {
		t.Errorf("another machine's plan, which binds its consumers to a provider here: %v", err)
	}
	refused("write another machine's plan", func(c context.Context) error {
		_, err := mirror.PutString(c, "shanks.dunst", "{}")
		return err
	})
	refused("write its own machine's plan", func(c context.Context) error {
		_, err := mirror.PutString(c, "g14.dunst", "{}")
		return err
	})
}