mesh-catalog/modules/mesh-vault/cmd/mesh-vault/rotation_integration_test.go

//go:build integration

package main

// The vault whole against a real bus, rotating a node's login with a stand-in engine (novox/hq ADR 0583 §1 to §7, issue
// 774): the built binary played over its stdin and stdout as the node's tool runtime plays it, as in the mint test. The
// test is the engine and the bus node: it states its sealing key, takes the new login through `own`, and "reloads the
// list" by handing the bus-accounts hashes in the bus module's have, as the bus node's node-engine does. At every step at
// least one of the engine's logins authenticates against the list loaded last, and with the list never reloaded the vault
// never says node-account-ready.

import (
	"context"
	"crypto/rand"
	"encoding/base64"
	"encoding/json"
	"errors"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	"testing"
	"time"

	"github.com/nats-io/nats.go"
	"github.com/nats-io/nats.go/jetstream"
	"golang.org/x/crypto/bcrypt"
	"golang.org/x/crypto/nacl/box"

	"git.novox.be/novox/mesh-sdk/go/delivery"
	"git.novox.be/novox/mesh-sdk/go/seal"
	"git.novox.be/novox/mesh-sdk/go/testbus"
)

// 774/S50/AC1.
func TestTheVaultWholeRotatesANodesLoginWithAStandInEngine(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	t.Cleanup(cancel)
	dir := t.TempDir()
	bin := filepath.Join(dir, "mesh-vault")
	build := exec.Command("go", "build", "-o", bin, ".")
	build.Stderr = os.Stderr
	if err := build.Run(); err != nil {
		t.Fatalf("the vault does not build: %v", err)
	}
	nc, err := nats.Connect(testbus.URL(t))
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(nc.Close)
	js, err := jetstream.New(nc)
	if err != nil {
		t.Fatal(err)
	}
	for _, b := range []string{"mesh-vault_bus-accounts", "mesh-vault_node-logins", "mesh-grants_accounts",
		"mesh-provisioner_plans", "node-engine_have", "mesh-delivery-assigner_placement"} {
		if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 2}); err != nil {
			t.Fatal(err)
		}
	}
	opPub, _, err := box.GenerateKey(rand.Reader)
	if err != nil {
		t.Fatal(err)
	}
	root := filepath.Join(dir, "root")
	_ = os.MkdirAll(root, 0o700)
	export, _ := json.Marshal(map[string]string{"operator-key": base64.StdEncoding.EncodeToString(opPub[:])})
	if err := os.WriteFile(filepath.Join(root, "export.json"), export, 0o600); err != nil {
		t.Fatal(err)
	}
	cmd := exec.CommandContext(ctx, bin)
	cmd.Env = append(os.Environ(), "MESH_VAULT_STORE="+filepath.Join(dir, "store"), "MESH_VAULT_STATE="+filepath.Join(dir, "state"),
		"MESH_VAULT_ROOT="+root, "MESH_VAULT_LEDGER="+filepath.Join(dir, "ledger"), "MESH_RECEIVES="+filepath.Join(dir, "mesh.json"),
		"MESH_NODE=novox", "MESH_MODULE=mesh-vault")
	stdin, _ := cmd.StdinPipe()
	stdout, _ := cmd.StdoutPipe()
	cmd.Stderr = os.Stderr
	if err := cmd.Start(); err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() })
	p := &player{t: t, ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}}
	// mesh-hand-acts answers the one hand act the test recorded.
	p.ask = func(key string, _ json.RawMessage) (any, error) {
		if key != "mesh-hand-acts.hand-acts" {
			return nil, errors.New("nats: no responders available for request")
		}
		return map[string]any{"acts": []HandActSeen{{ID: "act-1", By: "person.jochen", At: time.Now(), Verb: "rotate-node", Args: []string{"g14"}}}}, nil
	}
	p.followPlacement()
	go p.serve(stdout)

	oldPassword := "the-old-password-of-g14"
	oldHash, _ := bcrypt.GenerateFromPassword([]byte(oldPassword), bcrypt.MinCost)
	p.put(t, "mesh-vault.bus-accounts", "node.g14", AccountHash{User: "node.g14", Hash: string(oldHash), Adopted: true})
	// The vault's public key, which the bus node's engine seals its handing to.
	var vaultKey seal.KeyPair
	eventually(t, "the vault's key", func() bool {
		raw, err := os.ReadFile(filepath.Join(dir, "state", OwnKeyFile))
		return err == nil && json.Unmarshal(raw, &vaultKey) == nil && vaultKey.PublicKey != ""
	})

	// The bus node reloads the list from bus-accounts and hands the hashes of what it loaded in its have; the engine's logins
	// are its passwords. An authentication is a bcrypt compare against the list loaded last.
	passwords := map[string]string{"node.g14": oldPassword}
	var loaded map[string]string
	revision := uint64(0)
	reload := func() {
		kv, _ := p.kv("mesh-vault.bus-accounts")
		keys, err := kv.Keys(ctx)
		if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
			t.Fatal(err)
		}
		sort.Strings(keys)
		var accounts []delivery.HandedAccount
		loaded = map[string]string{}
		for _, k := range keys {
			a := p.hashOf(t, k)
			if a == nil || !delivery.IsBcrypt(a.Hash) {
				continue
			}
			accounts = append(accounts, delivery.HandedAccount{User: k, Hash: a.Hash})
			loaded[k] = a.Hash
		}
		revision++
		list, _ := json.Marshal(accounts)
		handed, err := delivery.HandAccounts(accounts, vaultKey.PublicKey, Fingerprint(string(list)), time.Now())
		if err != nil {
			t.Fatal(err)
		}
		raw, err := delivery.EncodeHave(delivery.Have{Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
			UserList: &delivery.UserListWritten{Source: "k", Revision: revision, Users: len(accounts), Handed: &handed}})
		if err != nil {
			t.Fatal(err)
		}
		hk, _ := p.kv("node-engine.have")
		if _, err := hk.Put(ctx, "novox.nats", raw); err != nil {
			t.Fatal(err)
		}
	}
	authenticates := func(step string) {
		t.Helper()
		for user, password := range passwords {
			if h, ok := loaded[user]; ok && bcrypt.CompareHashAndPassword([]byte(h), []byte(password)) == nil {
				return
			}
		}
		t.Fatalf("%s: none of the engine's logins %v authenticates against the loaded list %v", step, passwords, loaded)
	}
	said := func(event string) bool {
		p.mu.Lock()
		defer p.mu.Unlock()
		return strings.Contains(strings.Join(p.said, "\n"), event+" ")
	}
	reload()
	// The bus node (novox, running nats) and the node that rotates (g14); no lease key anywhere (issue 1126: the controller
	// ended by its unassignment); g14's login adopted from
	// the loaded list, its password known to the engine. Written after the vault's first view of the placement, so they
	// reach it only as the owner's views events, and after the bus node's have: the vault heard that have while no node
	// held the bus, and adopts from it (learning the loaded list's revision) when the placement names novox.
	p.firstViewAnswered()
	for node, modules := range map[string][]string{"novox": {"nats"}, "g14": {"web"}} {
		pl := delivery.Placement{V: delivery.Version, Node: node}
		for _, m := range modules {
			pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: m})
		}
		values, err := delivery.PlacementValues(pl)
		if err != nil {
			t.Fatal(err)
		}
		for k, raw := range values {
			kv, _ := p.kv("mesh-delivery-assigner.placement")
			if _, err := kv.Put(ctx, k, raw); err != nil {
				t.Fatal(err)
			}
		}
	}
	authenticates("before the rotation")

	enginePub, enginePriv := aSealingKey(t)
	if _, err := p.call(Seat+".node-key", "node.g14", map[string]any{"key": enginePub}); err != nil {
		t.Fatal(err)
	}
	// The rotation begins once the vault has heard the loaded list's revision.
	var begun RotateNodeAnswer
	eventually(t, "the rotation begins", func() bool {
		raw, err := p.call(Seat+".rotate-node", "person.jochen", map[string]any{"node": "g14", "why": "the weekly rotation", "act": "act-1"})
		return err == nil && json.Unmarshal(raw, &begun) == nil && begun.Rotating == "node.g14--b"
	})
	authenticates("rotation begun")
	// The list is never reloaded: node-account-ready is never said.
	time.Sleep(time.Second)
	if said(NodeAccountReadyEvent) {
		t.Fatal("node-account-ready was said while the loaded list lacks the new login")
	}
	// The engine takes the new login through own, sealed to its key.
	raw, err := p.call(Seat+".own", "node.g14", map[string]any{})
	if err != nil {
		t.Fatal(err)
	}
	var own OwnAnswer
	if err := json.Unmarshal(raw, &own); err != nil || len(own.Own) != 1 || own.Own[0].Module != MeshHostModule || own.Own[0].Name != "node.g14--b" {
		t.Fatalf("own answers %+v, %v", own.Own, err)
	}
	value, err := seal.Open(own.Own[0].Sealed, enginePriv)
	if err != nil {
		t.Fatal(err)
	}
	var account struct{ User, Password string }
	if err := json.Unmarshal([]byte(value), &account); err != nil || account.User != "node.g14--b" {
		t.Fatalf("the new login is %s, %v", value, err)
	}
	passwords["node.g14--b"] = account.Password
	// The bus node reloads the list: the new login is in it, both authenticate, node-account-ready is said.
	reload()
	eventually(t, "node-account-ready", func() bool { return said(NodeAccountReadyEvent) })
	authenticates("the list holds both logins")
	if bcrypt.CompareHashAndPassword([]byte(loaded["node.g14--b"]), []byte(account.Password)) != nil {
		t.Fatal("the loaded hash of the new login is not of the value the engine took")
	}
	// The engine, dialled as the new login, confirms; the vault removes the old hash, and the next reload drops it.
	if _, err := p.call(Seat+".node-account-confirm", "node.g14--b", map[string]any{}); err != nil {
		t.Fatal(err)
	}
	if h := p.hashOf(t, "node.g14"); h != nil {
		t.Fatalf("the old login's hash remains after the confirm: %+v", h)
	}
	authenticates("the confirm, before the list reloads")
	reload()
	authenticates("the list without the old login")
	eventually(t, "rotation-done", func() bool { return said(RotationDoneEvent) })
	kv, _ := p.kv("mesh-vault.node-logins")
	e, err := kv.Get(ctx, "g14")
	if err != nil {
		t.Fatal(err)
	}
	var nl NodeLogin
	_ = json.Unmarshal(e.Value(), &nl)
	if nl.Login != "node.g14--b" || nl.Rotating != "" || nl.Stage != "" {
		t.Errorf("node-logins g14 is %+v", nl)
	}
	if h := p.hashOf(t, "node.g14--b"); h == nil || h.Rotating != nil {
		t.Errorf("the new login's key is %+v", h)
	}
}