mesh-controller/internal/broker/prov_applied_live_test.go

package broker

import (
	"context"
	"errors"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/nats-io/nats-server/v2/server"
	"github.com/nats-io/nats.go"
	"golang.org/x/crypto/bcrypt"
)

// **On a real server, as composed** (design 57 P6, issue 672, ADR 0362): a machine's provisioner client's word on a
// module's files is kept by the mesh's own stream, newest per module, and the controller reads it by subject; the
// client cannot say it of another machine, and nothing but a machine's own client is granted to say it.
func TestTheProvisionersReportIsKeptAndReadByTheController(t *testing.T) {
	hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
	if err != nil {
		t.Fatal(err)
	}
	accounts, err := ComposeAccounts([]Principal{
		{Kind: KindController, PasswordHash: string(hash)},
		{Kind: KindProvisioner, Node: "g14", Module: "mesh-provisioner-client", PasswordHash: string(hash)},
		{Kind: KindProvisioner, Node: "shanks", Module: "mesh-provisioner-client", PasswordHash: string(hash)},
	})
	if err != nil {
		t.Fatal(err)
	}
	conf := filepath.Join(t.TempDir(), "bus.conf")
	if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+
		`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
		t.Fatal(err)
	}
	opts, err := server.ProcessConfigFile(conf)
	if err != nil {
		t.Fatalf("the composed accounts do not parse: %v", err)
	}
	opts.NoLog, opts.NoSigs = true, true
	s, err := server.NewServer(opts)
	if err != nil {
		t.Fatal(err)
	}
	go s.Start()
	if !s.ReadyForConnections(10 * time.Second) {
		t.Fatal("the bus did not come up")
	}
	defer s.Shutdown()

	refusals := make(chan error, 16)
	connect := func(user string) *nats.Conn {
		t.Helper()
		nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, "pw"), nats.CustomInboxPrefix("_INBOX."+user),
			nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refusals <- err }))
		if err != nil {
			t.Fatal(err)
		}
		t.Cleanup(nc.Close)
		return nc
	}
	control := OnConn(connect("controller"))
	if err := AssertMeshStreams(control); err != nil {
		t.Fatalf("the mesh's streams cannot be asserted: %v", err)
	}

	g14 := connect("provisioner.g14")
	for _, said := range []string{`{"status":"waiting"}`, `{"status":"applied","templates":"sha256:new"}`} {
		if err := g14.Publish(ProvisionerAppliedSubject("g14", "dunst"), []byte(said)); err != nil {
			t.Fatal(err)
		}
	}
	if err := g14.Flush(); err != nil {
		t.Fatal(err)
	}
	// Another machine's client says nothing of g14: refused by the server, and nothing kept.
	shanks := connect("provisioner.shanks")
	if err := shanks.Publish(ProvisionerAppliedSubject("g14", "dunst"), []byte(`{"status":"failed"}`)); err != nil {
		t.Fatal(err)
	}
	_ = shanks.Flush()
	select {
	case err := <-refusals:
		if !errors.Is(err, nats.ErrPermissionViolation) {
			t.Fatalf("the server said %v, want a permissions violation", err)
		}
	case <-time.After(2 * time.Second):
		t.Fatal("the server did not refuse shanks's client a report on g14")
	}

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	js, err := control.Conn().JetStream(nats.Context(ctx))
	if err != nil {
		t.Fatal(err)
	}
	// A core publish is kept by the stream once the server has stored it: read until it has, a few seconds at most.
	var got []byte
	var said bool
	for deadline := time.Now().Add(5 * time.Second); ; {
		got, said, err = LastProvApplied(js, "g14", "dunst")
		if (err == nil && said && strings.Contains(string(got), "applied")) || time.Now().After(deadline) {
			break
		}
		time.Sleep(20 * time.Millisecond)
	}
	if err != nil || !said || string(got) != `{"status":"applied","templates":"sha256:new"}` {
		t.Fatalf("the controller read %q %v %v, want g14's newest report", got, said, err)
	}
	if _, said, err := LastProvApplied(js, "shanks", "dunst"); err != nil || said {
		t.Fatalf("a module nothing was said of reads as said: %v %v", said, err)
	}
	all, err := ProvAppliedReports(js)
	if err != nil || len(all) != 1 || string(all["prov.applied.g14.dunst"]) != `{"status":"applied","templates":"sha256:new"}` {
		t.Fatalf("every report, newest per module: %q %v", all, err)
	}
	info, err := js.StreamInfo(ProvAppliedStream)
	if err != nil || info.State.Msgs != 1 {
		t.Fatalf("the stream keeps the newest report per module only: %+v %v", info, err)
	}

	// The writers table: a machine's own client alone is granted to say it.
	if err := CheckWriters(Principal{Kind: KindProvisioner, Node: "g14"}, []string{"prov.applied.g14.>"}); err != nil {
		t.Fatalf("g14's client may not say its own: %v", err)
	}
	for _, p := range []Principal{{Kind: KindProvisioner, Node: "shanks"}, {Kind: KindNode, Node: "g14"},
		{Kind: KindModule, Node: "g14", Module: "dunst"}} {
		err := CheckWriters(p, []string{"prov.applied.g14.>"})
		if err == nil || !strings.Contains(err.Error(), "a module's files placed on a machine") {
			t.Errorf("%s may say g14's provisioner's report: %v", p.Username(), err)
		}
	}
}