The engine's rotation against a real bus whose user list is reloaded step by step

A stand-in vault reloads a nats-server's users as the grants part's list is loaded, and the engine's own rotation code runs against it with real test-dials and a real confirmation request: at every step at least one login authenticates; the engine switches only once the list holds both logins and confirms in the new login's name; at rotation-done the old login is retired and its value dropped; with the list never loaded the test-dial is refused and the engine never switches; and an abandon after the switch brings the engine back to the old login before the new value goes (ADR 0583 §2-§6). One integration test; mesh-host holds seven of fifty. The real vault and the real engine together are proven live on g14 first. (issue 774, 774/S65) Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
2026-11-02 01:49:01 +01:00
parent 9ae30a7bbb
commit 72aca552e1
1 changed files with 170 additions and 0 deletions
@@ -0,0 +1,170 @@
+//go:build integration
+
+package link
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	natsserver "github.com/nats-io/nats-server/v2/server"
+	"github.com/nats-io/nats.go"
+
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+)
+
+// 774/S65/AC1 (ADR 0583 §2-§6): the engine's rotation against a real bus whose list of users a stand-in vault reloads step by
+// step. At every step at least one login authenticates; the engine switches only once the list holds both logins; the
+// confirmation arrives over the new connection, in the new login's name; at rotation-done the old login is retired and its
+// value dropped; with the list never loaded the engine never switches; and an abandon after the switch restores the old login.
+func TestARotationAgainstARealBusWhoseListIsReloadedStepByStep(t *testing.T) {
+	const first, second = "node.ace", "node.ace--b"
+	users := func(logins ...string) []*natsserver.User {
+		out := []*natsserver.User{{Username: "vault", Password: "vault-pw"}}
+		for _, l := range logins {
+			pw := map[string]string{first: "pw-first", second: "pw-second"}[l]
+			out = append(out, &natsserver.User{Username: l, Password: pw, Permissions: &natsserver.Permissions{
+				Publish:   &natsserver.SubjectPermission{Allow: []string{"mesh.seat.mesh-vault.call.>", "_INBOX.node.ace.>"}},
+				Subscribe: &natsserver.SubjectPermission{Allow: []string{"_INBOX.node.ace.>"}}}})
+		}
+		return out
+	}
+	start := func(t *testing.T) (*natsserver.Server, func(logins ...string)) {
+		opts := &natsserver.Options{Host: "127.0.0.1", Port: natsserver.RANDOM_PORT, NoLog: true, NoSigs: true, Users: users(first)}
+		srv, err := natsserver.NewServer(opts)
+		if err != nil {
+			t.Fatal(err)
+		}
+		go srv.Start()
+		if !srv.ReadyForConnections(20 * time.Second) {
+			t.Fatal("the bus did not come up")
+		}
+		t.Cleanup(srv.Shutdown)
+		return srv, func(logins ...string) { // the vault's step: the grants part's list, written and loaded by the bus node
+			next := opts.Clone()
+			next.Users = users(logins...)
+			if err := srv.ReloadOptions(next); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+	dial := func(srv *natsserver.Server, login string) (*nats.Conn, error) {
+		pw := map[string]string{first: "pw-first", second: "pw-second", "vault": "vault-pw"}[login]
+		return nats.Connect(srv.ClientURL(), nats.UserInfo(login, pw), nats.CustomInboxPrefix("_INBOX.node.ace"), nats.NoReconnect(), nats.Timeout(3*time.Second))
+	}
+	authenticates := func(srv *natsserver.Server, login string) bool {
+		c, err := dial(srv, login)
+		if err == nil {
+			c.Close()
+		}
+		return err == nil
+	}
+	oneAuthenticates := func(t *testing.T, srv *natsserver.Server, step string) {
+		t.Helper()
+		if !authenticates(srv, first) && !authenticates(srv, second) {
+			t.Fatalf("%s: no login of the node authenticates", step)
+		}
+	}
+	// The rotator over the real bus: the test-dial and the confirmation are real connections, the confirmation a request to the
+	// stand-in vault, which names the caller it saw in the subject it was called on.
+	build := func(t *testing.T, srv *natsserver.Server) (*Rotator, *Logins, chan string) {
+		confirmed := make(chan string, 4)
+		vault, err := dial(srv, "vault")
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Cleanup(vault.Close)
+		if _, err := vault.Subscribe("mesh.seat.mesh-vault.call.node-account-confirm.*", func(m *nats.Msg) {
+			confirmed <- m.Subject[strings.LastIndex(m.Subject, ".")+1:]
+			_ = m.Respond([]byte(`{"result":{}}`))
+		}); err != nil {
+			t.Fatal(err)
+		}
+		_ = vault.Flush()
+		l := NewLogins("ace", "pw-first", LoginsState{}, nil)
+		r := &Rotator{Logins: l, FailBound: 2,
+			Fetch: func(context.Context, string) (string, error) { return "pw-second", nil },
+			TestDial: func(_ context.Context, login, pw string) error {
+				c, err := nats.Connect(srv.ClientURL(), nats.UserInfo(login, pw), nats.CustomInboxPrefix("_INBOX.node.ace"), nats.NoReconnect(), nats.Timeout(3*time.Second))
+				if err == nil {
+					c.Close()
+				}
+				return err
+			},
+			Confirm: func(ctx context.Context) error {
+				login, pw := l.Current()
+				c, err := nats.Connect(srv.ClientURL(), nats.UserInfo(login, pw), nats.CustomInboxPrefix("_INBOX.node.ace"), nats.NoReconnect(), nats.Timeout(3*time.Second))
+				if err != nil {
+					return err
+				}
+				defer c.Close()
+				reply, err := c.RequestWithContext(ctx, CalledVaultSubjectAs("node-account-confirm", login), []byte(`{}`))
+				if err != nil {
+					return err
+				}
+				if _, err := seatAnswer(reply.Data); err != nil {
+					return errors.New("refused: " + err.Error())
+				}
+				return nil
+			}}
+		r.Watch()
+		return r, l, confirmed
+	}
+	ctx := context.Background()
+	since := time.Date(2026, 11, 2, 12, 0, 0, 0, time.UTC)
+	key := func(login, rotating, stage string) delivery.NodeLogin {
+		return delivery.NodeLogin{Node: "ace", Login: login, Rotating: rotating, Stage: stage, Since: since}
+	}
+
+	// A whole rotation: begun, the list holds both logins, ready, the switch and the confirmation, the old login retired, done.
+	srv, reload := start(t)
+	r, l, confirmed := build(t, srv)
+	oneAuthenticates(t, srv, "before the rotation")
+	r.Heard(ctx, key(first, second, delivery.RotationBegun))
+	oneAuthenticates(t, srv, "begun")
+	reload(first, second)
+	oneAuthenticates(t, srv, "the list holds both")
+	r.Heard(ctx, key(first, second, delivery.RotationReady))
+	if cur, _ := l.Current(); cur != second {
+		t.Fatalf("the engine did not switch once the list held both logins: %s", cur)
+	}
+	select {
+	case caller := <-confirmed:
+		if caller != "node~ace--b" {
+			t.Errorf("the confirmation came in the name of %s, not the new login's", caller)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("no confirmation reached the vault")
+	}
+	oneAuthenticates(t, srv, "switched")
+	reload(second) // the vault removes the old login's hash after the confirmation
+	oneAuthenticates(t, srv, "the old login retired")
+	r.Heard(ctx, key(second, "", ""))
+	if _, held := l.Value(first); held || authenticates(srv, first) || !authenticates(srv, second) {
+		t.Errorf("after rotation-done: old value held %v, old authenticates %v, new authenticates %v", held, authenticates(srv, first), authenticates(srv, second))
+	}
+
+	// With the list never loaded the engine never switches: the test-dial is refused.
+	srv, _ = start(t)
+	r, l, _ = build(t, srv)
+	r.Heard(ctx, key(first, second, delivery.RotationReady))
+	if cur, _ := l.Current(); cur != first || !authenticates(srv, first) {
+		t.Errorf("the list was never loaded and the engine is on %s", cur)
+	}
+
+	// An abandon after the switch restores the old login: the vault drops the new one, the engine switches back first.
+	srv, reload = start(t)
+	r, l, _ = build(t, srv)
+	reload(first, second)
+	r.Heard(ctx, key(first, second, delivery.RotationReady))
+	reload(first)
+	r.Heard(ctx, key(first, "", ""))
+	if cur, _ := l.Current(); cur != first || !authenticates(srv, first) {
+		t.Errorf("after an abandon the engine is on %s", cur)
+	}
+	if _, held := l.Value(second); held {
+		t.Error("the abandoned login's value is kept")
+	}
+}