mesh-host/internal/link/hearing_nats.go

package link

import (
	"context"
	"crypto/ed25519"
	"errors"
	"fmt"
	"os"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/nats-io/nats.go"
	"github.com/nats-io/nats.go/micro"
)

// The host's link on the bus: one connection, and what is asked of this node over it. The node hears no declaration
// (novox/hq ADR 0539): it watches its wants and its compositions, and says its facts.

// EnrolSubject is where a joining machine asks. One subject for every node, because a machine
// enrolling has no name the mesh has agreed to yet — which is why its authority to publish here is
// the whole of what its enrolment user may do.
const EnrolSubject = "mesh.control.enrol"

// AskReportSubject is where the mesh asks this node to say its facts again: its own, on core NATS and off any stream.
// The answer is the facts event on its report subject — nobody's inbox is answered.
func AskReportSubject(node string) string { return "mesh.node." + node + ".ask.report" }

// natsURL is a bus address as the client wants it. A membership records host and port, because that
// is what genesis sealed into it and what the other transport takes; the scheme is this transport's
// own business.
func natsURL(address string) string {
	if strings.Contains(address, "://") {
		return address
	}
	return "nats://" + address
}

// natsLink is this node's connection as a JetStream subscription.
type natsLink struct {
	// mu guards signerNow (FollowSigner).
	mu        sync.Mutex
	signerNow func() ed25519.PublicKey
	conn      *nats.Conn
	js        nats.JetStreamContext
	asking    *nats.Subscription
	asked     chan struct{}
	handing   *nats.Subscription
	// searching hears the controller's terminal asking for a fresh setuid search (novox/hq issue 361).
	searching *nats.Subscription
	// judging hears the controller asking for a read-only judgement of this node's directories (issue 358).
	judging *nats.Subscription
	// acting hears the controller asking for the take act on a library (novox/hq ADR 0328 §7).
	acting *nats.Subscription
	// moving hears the controller asking to move a value sealed to this machine into the vault (issue 676).
	moving *nats.Subscription
	// engine serves the node seat node-engine, and warrants hears the operator's answers to its asks (ADR 0563).
	engine   micro.Service
	warrants *nats.Subscription
	node     string
	// refused carries each subject the bus refused this link a subscription to (RefusedSubscription), which serve drops
	// the link for after a pause, so every subscription is asked again on the next.
	refused chan string
	// signer is the mesh's key, which a hand-over ask is verified with as a declaration is (issue 356).
	signer ed25519.PublicKey
	lost   chan error
}

func dialNats(ctx context.Context, m Membership, timeout time.Duration) (Link, error) {
	// Pinned exactly as the other transport is, and for once the Go client makes that easy: it
	// takes a *tls.Config, so the same PinnedConfig with the same VerifyPeerCertificate does the
	// work. **The constraint recorded against the tool runtime does not apply here** — that client
	// takes PEM strings with no verify hook, which is why the bus's certificate must carry a name
	// matching the address *modules* dial it by. A host checks the fingerprint and nothing else.
	config, err := PinnedConfig(m.Fingerprint)
	if err != nil {
		return nil, err
	}
	opts := []nats.Option{
		nats.Secure(config),
		// The mesh names a machine's bus user "node.<name>" (the controller's principal scheme), and
		// the server refused the bare name the first time a machine dialled it: "authentication
		// error - User". The same string the mesh composed into the user list, or nothing connects.
		nats.UserInfo("node."+m.Node, m.Password),
		// Replies to what this client asks the server arrive on its inbox, and the mesh grants a
		// machine exactly its own: the same prefix the user list was composed with.
		nats.CustomInboxPrefix("_INBOX.node." + m.Node),
		nats.Name("mesh-host/" + m.Node),
		nats.Timeout(timeout),
		// A node that has silently lost its route notices, rather than holding a connection the
		// server forgot about and believing it is still in the mesh.
		nats.PingInterval(10 * time.Second),
		nats.MaxPingsOutstanding(2),
		// Reconnection is the caller's: Hold already decides when to try again and how long to
		// wait, and a client quietly reconnecting underneath it would make that reasoning a
		// duplicate of the library's.
		nats.NoReconnect(),
	}

	conn, err := nats.Connect(natsURL(m.Broker), opts...)
	if err != nil {
		if errors.Is(err, ErrWrongCertificate) {
			return nil, err
		}
		return nil, fmt.Errorf("cannot reach the bus at %s: %w", m.Broker, err)
	}
	js, err := conn.JetStream()
	if err != nil {
		conn.Close()
		return nil, fmt.Errorf("the bus at %s has no JetStream: %w", m.Broker, err)
	}

	l := &natsLink{
		conn: conn, js: js, node: m.Node, signer: m.Signer,
		lost:    make(chan error, 1),
		asked:   make(chan struct{}, 1),
		refused: make(chan string, 64),
	}
	// Set before anything subscribes, so the node seat's service (micro), which wraps the handler it finds, hands every
	// refusal of its own subjects on to it too.
	conn.SetErrorHandler(l.asyncError)

	l.hearAsks()

	conn.SetDisconnectErrHandler(func(_ *nats.Conn, err error) {
		select {
		case l.lost <- fmt.Errorf("the link dropped: %w", err):
		default:
		}
	})
	conn.SetClosedHandler(func(*nats.Conn) {
		select {
		case l.lost <- errors.New("the link closed"):
		default:
		}
	})

	return l, nil
}

// hearAsks listens for the mesh asking what this node last applied (the `report` verb). One question
// waiting is enough: a second asked before the first is taken is the same question. A bus whose user
// list is older than the grant refuses the subscription asynchronously — said by the client, and the
// mesh then sends again instead of asking; nothing here depends on being asked.
func (l *natsLink) hearAsks() {
	if l.asked == nil {
		l.asked = make(chan struct{}, 1)
	}
	asking, err := l.conn.Subscribe(AskReportSubject(l.node), func(*nats.Msg) {
		select {
		case l.asked <- struct{}{}:
		default:
		}
	})
	if err == nil {
		l.asking = asking
	}
}

// FollowSigner has every signed ask heard on this link verified with the key the engine trusts at that ask, the
// controller's while it holds its lease and none from its hand-over (novox/hq ADR 0563 §7, ADR 0550 §2).
func (l *natsLink) FollowSigner(f func() ed25519.PublicKey) {
	l.mu.Lock()
	l.signerNow = f
	l.mu.Unlock()
}

// check is how a signed ask is believed on this link: the static membership signer until FollowSigner names the trust.
func (l *natsLink) check() HandOverCheck {
	l.mu.Lock()
	defer l.mu.Unlock()
	return HandOverCheck{Signer: l.signer, SignerNow: l.signerNow, Node: l.node}
}

// HearHandOvers listens for the controller asking this node to hand a directory used as found to the mesh
// (novox/hq issue 356): a request on this machine's own subject, answered once on the reply it carried. The
// ask is verified with the mesh's key before anything is read out of it (AnswerHandOver), because a responder
// can reach this subject through a reply; the engine runs nothing it is sent. A bus whose user list is older
// than the grant refuses the subscription asynchronously, said by the client; the operator's line then says
// that nothing on this node answers.
func (l *natsLink) HearHandOvers(handle HandOverFunc) {
	if handle == nil {
		return
	}
	check := l.check()
	handing, err := l.conn.Subscribe(AskHandOverSubject(l.node), func(msg *nats.Msg) {
		if body, answer := HandOverReply(check, handle, msg.Reply, msg.Data); answer {
			_ = msg.Respond(body)
		}
	})
	if err == nil {
		l.handing = handing
	}
}

// HearSetuidSearches listens for the controller asking this node for a fresh search for setuid programs
// (novox/hq issue 361), on this machine's own subject, verified as a hand-over is (AnswerSetuidSearch). A bus
// whose user list is older than the grant refuses the subscription, said by the client; the operator's line
// then says that nothing on this node answers.
func (l *natsLink) HearSetuidSearches(handle SetuidSearchFunc) {
	if handle == nil {
		return
	}
	check := l.check()
	searching, err := l.conn.Subscribe(AskSetuidSearchSubject(l.node), func(msg *nats.Msg) {
		if body, answer := SetuidSearchReply(check, handle, msg.Reply, msg.Data); answer {
			_ = msg.Respond(body)
		}
	})
	if err == nil {
		l.searching = searching
	}
}

// HearDirectoryJudgements listens for the controller asking this node to judge its declared directories,
// read-only (novox/hq issue 358), on this machine's own subject, verified as a hand-over is
// (AnswerJudgeDirectories). A bus whose user list is older than the grant refuses the subscription, said by the
// client; the operator's line then says that nothing on this node answers.
func (l *natsLink) HearDirectoryJudgements(handle JudgeDirectoriesFunc) {
	if handle == nil {
		return
	}
	check := l.check()
	judging, err := l.conn.Subscribe(AskJudgeDirectoriesSubject(l.node), func(msg *nats.Msg) {
		if body, answer := JudgeDirectoriesReply(check, handle, msg.Reply, msg.Data); answer {
			_ = msg.Respond(body)
		}
	})
	if err == nil {
		l.judging = judging
	}
}

// HearAccessActs listens for the controller asking this node to run the take act on an access's folder (novox/hq ADR
// 0328 §7), on this machine's own subject, verified as a hand-over is (AnswerAccessAct). A bus whose user list is older
// than the grant refuses the subscription, said by the client; the controller then hears no answer.
func (l *natsLink) HearAccessActs(handle AccessActFunc) {
	if handle == nil {
		return
	}
	check := l.check()
	acting, err := l.conn.Subscribe(AskAccessActSubject(l.node), func(msg *nats.Msg) {
		if body, answer := AccessActReply(check, handle, msg.Reply, msg.Data); answer {
			_ = msg.Respond(body)
		}
	})
	if err == nil {
		l.acting = acting
	}
}

// HearMoveSecrets listens for the controller asking this node to move one value sealed to it into the vault (design
// 57's V3, issue 676), on this machine's own subject, verified as a hand-over is (AnswerMoveSecret). A bus whose
// user list is older than the grant refuses the subscription; the controller then hears no answer.
func (l *natsLink) HearMoveSecrets(handle MoveSecretFunc) {
	if handle == nil {
		return
	}
	check := l.check()
	moving, err := l.conn.Subscribe(AskMoveSecretSubject(l.node), func(msg *nats.Msg) {
		if body, answer := MoveSecretReply(check, handle, msg.Reply, msg.Data); answer {
			_ = msg.Respond(body)
		}
	})
	if err == nil {
		l.moving = moving
	}
}

// asyncError is said by the client as before, and a subscription the bus refused is handed to serve (Refused).
func (l *natsLink) asyncError(_ *nats.Conn, _ *nats.Subscription, err error) {
	if err == nil {
		return
	}
	fmt.Fprintf(os.Stderr, "%v\n", err)
	if subject, ok := RefusedSubscription(err); ok && l.refused != nil {
		select {
		case l.refused <- subject:
		default:
		}
	}
}

// Refused says each subject the bus refused this link a subscription to.
func (l *natsLink) Refused() <-chan string { return l.refused }

func (l *natsLink) AskedToReport() <-chan struct{} { return l.asked }
func (l *natsLink) Lost() <-chan error             { return l.lost }

func (l *natsLink) Close() {
	if l.asking != nil {
		_ = l.asking.Unsubscribe()
	}
	if l.handing != nil {
		_ = l.handing.Unsubscribe()
	}
	if l.searching != nil {
		_ = l.searching.Unsubscribe()
	}
	if l.acting != nil {
		_ = l.acting.Unsubscribe()
	}
	if l.moving != nil {
		_ = l.moving.Unsubscribe()
	}
	if l.judging != nil {
		_ = l.judging.Unsubscribe()
	}
	if l.engine != nil {
		_ = l.engine.Stop()
	}
	if l.warrants != nil {
		_ = l.warrants.Unsubscribe()
	}
	if l.conn != nil {
		l.conn.Close()
	}
}

func (l *natsLink) Report(ctx context.Context, node string, body []byte) error {
	return OverNATS{Conn: l.conn, JS: l.js}.Report(ctx, node, body)
}

func (l *natsLink) Health(ctx context.Context, node string, body []byte) error {
	return OverNATS{Conn: l.conn, JS: l.js}.Health(ctx, node, body)
}

func (l *natsLink) HaveBind(ctx context.Context) (HaveBucket, error) {
	return OverNATS{Conn: l.conn, JS: l.js}.HaveBind(ctx)
}

// Ask asks a module's tool for a declared tool check (novox/hq ADR 0240, to-be 48 §3). The link the
// node-engine holds is this one, not OverNATS: without it here the queue's ToolBus assertion failed on
// every live link and each tool check read "no link to the bus is open" while the link was up.
func (l *natsLink) Ask(ctx context.Context, subject string, body []byte) ([]byte, error) {
	before := l.conn.LastError()
	reply, err := OverNATS{Conn: l.conn, JS: l.js}.Ask(ctx, subject, body)
	switch {
	case err == nil:
		return reply, nil
	case errors.Is(err, nats.ErrNoResponders):
		// The server says nobody subscribes the subject: delivered to no one.
		return nil, fmt.Errorf("%w: nothing on this machine answers %s", ErrNotAsked, subject)
	}
	if refused := l.refusal(subject, before); refused != "" {
		return nil, fmt.Errorf("%s: %v%s", subject, err, refused)
	}
	if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, nats.ErrTimeout) {
		// Said as the deadline, which the caller words with the time it gave the question.
		return nil, fmt.Errorf("no answer from %s: %w", subject, context.DeadlineExceeded)
	}
	return nil, fmt.Errorf("%s: %w", subject, err)
}

// What the queue asks of a live link beside Bus, each by a type assertion that fails quietly: said here
// so a link that drops one does not build.
var (
	_ Bus          = (*natsLink)(nil)
	_ HealthBus    = (*natsLink)(nil)
	_ HaveBus      = (*natsLink)(nil)
	_ ToolBus      = (*natsLink)(nil)
	_ Asker        = (*natsLink)(nil)
	_ Asked        = (*natsLink)(nil)
	_ EngineSeats  = (*natsLink)(nil)
	_ OperatorAsks = (*natsLink)(nil)
)

// refusedSubscription is how the server words a subscription its user list does not grant.
var refusedSubscription = regexp.MustCompile(`Permissions Violation for Subscription to "([^"]+)"`)

// RefusedSubscription is the subject a permissions violation names when it refused a subscription; false for any other
// error, a refused publish among them (a publish is refused each time it is tried, and is said where it is tried).
func RefusedSubscription(err error) (string, bool) {
	if err == nil || !errors.Is(err, nats.ErrPermissionViolation) && !strings.Contains(err.Error(), "permissions violation") {
		return "", false
	}
	m := refusedSubscription.FindStringSubmatch(err.Error())
	if m == nil {
		return "", false
	}
	return m[1], true
}