mesh-controller/internal/link/enrolment.go

package link

import (
	"context"
	"crypto/ed25519"
	"crypto/rand"
	"crypto/sha256"
	"encoding/base64"
	"encoding/hex"
	"errors"
	"fmt"
	"log"
	"sort"

	"github.com/novox/mesh-controller/internal/broker"
	"github.com/novox/mesh-controller/internal/identity"
	"github.com/novox/mesh-controller/internal/inventory"
)

// Enrolment is what actually happens when a node presents a token: the token is spent, the key is
// recorded, and the node gets its own queue.
//
// It reaches across two contexts and reads neither one's store from the other (novox/hq ADR 0008)
// — it holds both grants and asks each for its part, which is what the process running them is
// for.
type Enrolment struct {
	Inventory *inventory.Inventory
	Identity  *identity.Identity
	Broker    broker.Broker

	// OnNATS says the mesh's own traffic is on the bus being built, so a node's credential is
	// minted into the mesh's records and composed into the bus's user list rather than pushed
	// through a management call (novox/hq design 25 §4).
	//
	// **One bus, and a node gets a credential for exactly one** — refused at start if the
	// controller is told about both (broker.MustBeOneBus), because a node holding a credential for
	// each is one that could be half-moved, and nothing would say which half.
	OnNATS bool
}

// Enrol records what the node presented and spends the token.
//
// Order matters and it is the order things become irreversible (novox/hq issue 083). The token is
// claimed first, in a single statement that both finds it and holds it for this presenter's key, so
// two machines racing on one secret produce one holder. Then everything the node presented is
// written — each write an overwrite, so an attempt interrupted by the store going away can be made
// again by the same presenter. Then the token is spent. Last, the token's secret stops being the
// node's broker password: done after the spend, because a password replaced by an attempt that
// then failed would be one nobody holds, and the node could not even log in to ask again.
func (e Enrolment) Enrol(ctx context.Context, request EnrolRequest) (reply EnrolReply, err error) {
	secret, public, profile := request.Secret, ed25519.PublicKey(request.PublicKey), request.Profile

	// A store that could not be asked right now, or a token another presenter holds for the
	// moment, is "not now": the node asks again with the same request (novox/hq issue 083).
	defer func() {
		if inventory.Unreachable(err) || errors.Is(err, inventory.ErrTokenInUse) ||
			errors.Is(err, context.Canceled) {
			err = fmt.Errorf("%w: %w", ErrTryAgain, err)
		}
	}()

	if len(public) != ed25519.PublicKeySize {
		return EnrolReply{}, fmt.Errorf("a node presented a %d-byte key, and an identity is %d",
			len(public), ed25519.PublicKeySize)
	}

	// Claimed, not spent: the token is held for this presenter while the node is written, and
	// spent only as the last write. The node's identity lives in another database than the
	// token, so the two cannot be one transaction; a failure between them used to leave a spent
	// token and a node with no key, which the host — making new keys on every attempt — could not
	// recover from. Every write below overwrites, so an attempt made again is safe.
	// A proof that does not verify is refused outright: it was made with another key, or for
	// another request. One that verifies lets this presenter finish an enrolment whose token it
	// already spent — never a request the broker handed over a second time, which may already
	// have been answered.
	proven := false
	if len(request.Proof) > 0 {
		if !ed25519.Verify(public, EnrolProof(secret, public, request.OverlayKey, request.SealingKey,
			request.ServingKey), request.Proof) {
			return EnrolReply{}, errors.New("the enrolment's proof does not match the key it presents")
		}
		proven = true
	}
	by := claimant(public)
	node, err := e.Inventory.Claim(ctx, secret, by, proven && !request.Redelivered)
	if err != nil {
		return EnrolReply{}, err
	}

	if _, err := e.Identity.RecordNodeKey(ctx, node.ID, public); err != nil {
		return EnrolReply{}, fmt.Errorf("%s's key could not be recorded: %w", node.Name, err)
	}

	key, err := e.Identity.Active(ctx)
	if err != nil {
		return EnrolReply{}, err
	}

	reply = EnrolReply{
		Accepted:    true,
		Node:        node.Name,
		Queue:       QueueFor(node.Name),
		Broker:      e.Broker.Address,
		Fingerprint: e.Broker.Fingerprint,
		Signer:      key.Public,
	}

	// Recorded before the profile because the overlay is the first declaration this node will
	// receive, and without this key the mesh cannot compose one. A node enrolled with no overlay
	// key is a node the graph skips — an ordinary in-between state, and one worth leaving as
	// briefly as possible.
	// And the key its secrets are sealed to. Same reasoning as the overlay key below and one step
	// stronger: without it the mesh cannot send this node a credential at all, and a node that
	// enrolled without one will be refused a sealed file rather than quietly given none.
	// And the key it serves TLS with, so the mesh can certify its internal name. Public, so it is
	// recorded rather than sealed — the node keeps the half that matters.
	if request.ServingKey != "" {
		if err := e.Identity.RecordServingKey(ctx, node.ID, request.ServingKey); err != nil {
			return EnrolReply{}, fmt.Errorf(
				"%s's serving key could not be recorded: %w", node.Name, err)
		}
	}
	if request.SealingKey != "" {
		if err := e.Inventory.RecordSealingKey(ctx, node.ID, request.SealingKey); err != nil {
			return EnrolReply{}, fmt.Errorf(
				"%s's sealing key could not be recorded: %w", node.Name, err)
		}
	}
	if request.OverlayKey != "" {
		if err := e.Inventory.RecordOverlayKey(ctx, node.ID, request.OverlayKey); err != nil {
			return EnrolReply{}, fmt.Errorf(
				"%s's overlay key could not be recorded: %w", node.Name, err)
		}
	}
	// And the tunnel it found, whose key is the overlay key above (novox/hq ADR 0105). Recorded
	// before the token is spent for the same reason as the keys: the first declaration this node
	// receives is composed from it, and a hub enrolled without its tunnel would be placed at an
	// address of the mesh's choosing rather than the tunnel's.
	if request.Tunnel != nil {
		if request.Tunnel.PublicKey != request.OverlayKey {
			return EnrolReply{}, fmt.Errorf("%s presented a tunnel under key %s and an overlay key "+
				"that is not it; a tunnel is taken over with its own key or not at all", node.Name,
				request.Tunnel.PublicKey)
		}
		peers := make([]inventory.TunnelPeer, 0, len(request.Tunnel.Peers))
		for _, p := range request.Tunnel.Peers {
			peers = append(peers, inventory.TunnelPeer{PublicKey: p.PublicKey, Address: p.Address})
		}
		if err := e.Inventory.RecordTunnel(ctx, node.ID, inventory.Tunnel{
			Interface: request.Tunnel.Interface, Unit: request.Tunnel.Unit,
			Config: request.Tunnel.Config, Port: request.Tunnel.Port, MTU: request.Tunnel.MTU,
			Address: request.Tunnel.Address, Range: request.Tunnel.Range,
			PublicKey: request.Tunnel.PublicKey, Peers: peers,
		}); err != nil {
			return EnrolReply{}, fmt.Errorf("%s's found tunnel could not be recorded: %w", node.Name, err)
		}
	}

	// Spent once the node is complete in the store.
	if err := e.Inventory.Spend(ctx, secret, by); err != nil {
		return EnrolReply{}, fmt.Errorf("%s was written and its token could not be spent: %w", node.Name, err)
	}

	// The token's secret was the broker password up to this moment, which is what let this
	// connection exist at all. It is replaced now, so the one-time thing stays one-time and the
	// credential the node keeps for years is not the one that was pasted into a terminal. After
	// the spend and not before: a replaced password on an attempt that failed would be held by
	// nobody. If the broker will not take it now, the enrolment still stands — the node keeps
	// the token's secret as its password, which it is told, and which is said here.
	switch {
	case e.OnNATS:
		// **Minted into the mesh's records, not pushed to a server.** The bus's users are a file
		// the controller composes, so a credential becomes usable at the next composition rather
		// than at the moment it is made — and the plaintext is returned once, here, and then exists
		// only on the machine it was sealed to.
		//
		// The node reconnects as itself and may be refused until that composition reaches the
		// machine running the bus. That is what the host's reconnect backoff is for and it is
		// survivable by design (ADR 0004: disconnection is an ordinary situation); waiting for the
		// push here would hold an enrolment open for as long as a declaration takes to apply.
		password, err := e.Inventory.MintBusPassword(ctx, inventory.BusUser{
			Username: broker.Principal{Kind: broker.KindNode, Node: node.Name}.Username(),
			Kind:     inventory.BusNode,
			Node:     node.Name,
		})
		if err != nil {
			// Not fatal to the enrolment: the node is recorded and the token is spent, and a node
			// that keeps the token's secret is told so. Said loudly, because until this is minted
			// the machine has no credential of its own.
			log.Printf("%s is enrolled and the mesh could not mint its bus credential, so it keeps "+
				"the token's secret as its password: %v", node.Name, err)
		} else {
			reply.Password = password
		}

	}

	if profile != nil {
		// Not fatal if it fails. The profile is what the control plane needs in order to decide
		// what this machine should run, and it is reported again on every connection — so losing
		// it here costs a decision that can be made later, not the enrolment.
		_ = e.Inventory.RecordProfile(ctx, node.ID, profile)
	}
	return reply, nil
}

// rekey applies a verified rekey: the node's overlay key and tunnel are recorded as enrolment
// would have recorded them, and a hub moves to the tunnel's address.
func (e Enrolment) rekey(ctx context.Context, node inventory.Node, r Rekey) error {
	if r.Tunnel == nil || r.OverlayKey == "" {
		return fmt.Errorf("%s sent a rekey naming no tunnel or no key; refused", node.Name)
	}
	if e.Identity == nil {
		return fmt.Errorf("%s sent a rekey and this mesh has no identity store to verify it against", node.Name)
	}
	if err := e.Identity.VerifyNode(ctx, node.ID,
		RekeyProof(node.Name, r.Previous, r.OverlayKey, r.Tunnel), r.Proof); err != nil {
		return fmt.Errorf("%s's rekey is not signed by %s's identity key; refused: %w", node.Name, node.Name, err)
	}
	peers := make([]inventory.TunnelPeer, 0, len(r.Tunnel.Peers))
	for _, p := range r.Tunnel.Peers {
		peers = append(peers, inventory.TunnelPeer{PublicKey: p.PublicKey, Address: p.Address})
	}
	err := e.Inventory.Rekey(ctx, node.ID, r.Previous, r.OverlayKey, inventory.Tunnel{
		Interface: r.Tunnel.Interface, Unit: r.Tunnel.Unit, Config: r.Tunnel.Config, Port: r.Tunnel.Port, MTU: r.Tunnel.MTU,
		Address: r.Tunnel.Address, Range: r.Tunnel.Range, PublicKey: r.Tunnel.PublicKey, Peers: peers,
	})
	if err != nil {
		return fmt.Errorf("%s's rekey was not recorded: %w", node.Name, err)
	}
	log.Printf("%s took over the tunnel on %s: its overlay key is the tunnel's now", node.Name, r.Tunnel.Interface)
	return nil
}

// claimant names the key presenting a token, so a claim can be held for it alone.
func claimant(public ed25519.PublicKey) string {
	sum := sha256.Sum256(public)
	return hex.EncodeToString(sum[:])
}

// freshPassword is the node's own broker credential from enrolment onward.
func freshPassword() (string, error) {
	raw := make([]byte, 32)
	if _, err := rand.Read(raw); err != nil {
		return "", fmt.Errorf("cannot generate a broker password: %w", err)
	}
	return base64.RawURLEncoding.EncodeToString(raw), nil
}

var _ Enroller = Enrolment{}

// Heard records what a node reported about itself.
//
// A node states; the owning context writes (novox/hq ADR 0006). What a node says it applied is
// its own account of its own machine, kept as a copy for recovery — so this writes it down and
// decides nothing from it.
// Outstanding is the declaration the mesh last sent a node, so a report about an older one is not
// acted on (design 25 §3, window.go).
//
// **Here rather than on Listener.** A report is recorded by whatever keeps records, and a great
// many things that record reports have no idea what was sent — every test in this package among
// them. So the serving loop asks for this when the listener happens to be able to answer, and
// where it cannot, a report has nothing to be stale against and is simply acted on.
func (e Enrolment) Outstanding(ctx context.Context, node string) (string, error) {
	return e.Inventory.Outstanding(ctx, node)
}

func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err error) {
	// A store that could not be asked right now is said as such, so the report is kept for
	// another attempt rather than acknowledged and lost (novox/hq issue 082).
	defer func() {
		if inventory.Unreachable(err) {
			err = fmt.Errorf("%w: %w", ErrTryAgain, err)
		}
	}()
	if report.Node == "" {
		return false, errors.New("a report named no node")
	}
	node, err := e.Inventory.NodeByName(ctx, report.Node)
	if err != nil {
		return false, err
	}

	// What an adopted node holds, which firewall it found, and what is reachable on it (novox/hq
	// ADR 0100). Recorded whenever a report carries any of it — a node reports these on its own
	// schedule, when what it holds changes, not only after an apply — and never cleared by a
	// report that carries none, which is every bare word that the node is there. An adopted node
	// always names its firewall, so a report from one replaces all three, emptied held included.
	if len(report.Held) > 0 || report.Firewall != "" || len(report.Reachable) > 0 || len(report.Strays) > 0 {
		held := make([]inventory.Held, 0, len(report.Held))
		for _, h := range report.Held {
			held = append(held, inventory.Held{ID: h.ID, Module: h.Module, Kind: h.Kind,
				Target: h.Target, Since: h.Since, Changed: h.Changed, Kept: h.Kept, Facts: h.Facts})
		}
		strays := make([]inventory.Stray, 0, len(report.Strays))
		for _, s := range report.Strays {
			strays = append(strays, inventory.Stray{Kind: s.Kind, Name: s.Name, Detail: s.Detail})
		}
		reachable := make([]inventory.Reach, 0, len(report.Reachable))
		for _, r := range report.Reachable {
			reachable = append(reachable, inventory.Reach{Protocol: r.Protocol, Address: r.Address,
				Port: r.Port, By: r.By, Published: r.Published, ContainerPort: r.ContainerPort})
		}
		if err := e.Inventory.RecordAdoptionWithStrays(ctx, node.ID, held, report.Firewall, reachable, strays); err != nil {
			return false, err
		}
	}
	// What filters the machine, and the state of its found firewall (novox/hq ADR 0168), whenever
	// it says — every apply of a host that knows how, adopted or converged; never cleared by a
	// report that carries none, which is every bare word that the node is there.
	if len(report.Filters) > 0 || report.FoundFirewall != nil {
		filters := make([]inventory.Filter, 0, len(report.Filters))
		for _, f := range report.Filters {
			filters = append(filters, inventory.Filter{Where: f.Where, Owner: f.Owner, Refuses: f.Refuses})
		}
		var found *inventory.FoundFirewall
		if report.FoundFirewall != nil {
			found = &inventory.FoundFirewall{Kind: report.FoundFirewall.Kind, Active: report.FoundFirewall.Active,
				RetiredBy: report.FoundFirewall.RetiredBy}
		}
		if err := e.Inventory.RecordFiltering(ctx, node.ID, filters, found); err != nil {
			return false, err
		}
	}
	// Which of its links face outside (novox/hq ADR 0140), whenever it says so. Recorded on every
	// report that carries it, adopted or converged, because the filter the mesh composes is written
	// around it — and never cleared by a report that carries none, which is every bare word that the
	// node is there. A machine whose routing table it could not read reports nothing rather than
	// guessing, and keeps whatever it last said; a machine with genuinely no route off itself is one
	// the mesh composes no filter for at all.
	if len(report.Outward) > 0 {
		if err := e.Inventory.RecordOutwardLinks(ctx, node.ID, report.Outward); err != nil {
			return false, err
		}
	}
	// Which host produced this report (novox/hq 04-ISSUES/087), whenever it says. Recorded on every
	// report that carries it and never cleared by one that does not — a bare word that the node is
	// there says nothing about its host, and a machine whose host predates this reports none.
	if report.Host != "" {
		if err := e.Inventory.RecordHostVersion(ctx, node.ID, report.Host); err != nil {
			return false, err
		}
	}
	if len(report.Profile) > 0 {
		// The latest wins, as at enrolment: a capability the machine lost is one the plan must
		// stop counting on (novox/hq ADR 0161).
		if err := e.Inventory.RecordProfile(ctx, node.ID, report.Profile); err != nil {
			return false, err
		}
	}
	// What it says about the tunnel it carried (novox/hq ADR 0105), whenever it says it.
	if report.Tunnel != nil {
		if err := e.Inventory.RecordCarriedTunnel(ctx, node.ID, inventory.Carried{
			Interface: report.Tunnel.Interface, Port: report.Tunnel.Port, Range: report.Tunnel.Range,
			Peers: report.Tunnel.Peers, State: report.Tunnel.State, Note: report.Tunnel.Note,
			Kept: report.Tunnel.Kept,
		}); err != nil {
			return false, err
		}
	}
	// A node taking a found tunnel's key after enrolment (novox/hq ADR 0105). Verified against the
	// node's live identity key before anything is written: the broker account authenticates the
	// connection, the signature proves the node itself said it. Refused outright when the proof
	// does not verify or is stale — a refusal, not "not now", so the node hears why.
	if report.Rekey != nil {
		if err := e.rekey(ctx, node, *report.Rekey); err != nil {
			return false, err
		}
		return false, e.Inventory.Seen(ctx, node.ID)
	}

	// A bare word that a node is there is not an account of what the machine did or holds: it
	// moves last_seen and touches nothing else. This arrives every minute (link.AliveEvery),
	// while a real report is rare, so recording it as one would overwrite the node's last real
	// apply with an empty one — wiping the declaration digest that decides whether the node is
	// current, the carried ports a push assigns around, and the clean-or-failed outcome — and a
	// node that had just caught up would read as behind within the minute. The alive path calls
	// this with only a node name; a real report always carries an account (something applied, or
	// a refusal, or a failure), so those are the reports that get written down.
	if report.Applied == nil && report.Refused == "" && len(report.Failed) == 0 {
		if report.Superseded != "" {
			log.Printf("%s set aside declaration %s for the newer %s", report.Node, report.Declared, report.Superseded)
		} else if err := e.Inventory.RecordHeartbeatGap(ctx, report.Node, node.LastSeen); err != nil {
			// The silence before this word, which the machine-silent bound will be set from (novox/hq
			// to-be 45 Phase 0). A measurement lost is said and costs the report nothing.
			log.Printf("the silence before %s's word could not be recorded: %v", report.Node, err)
		}
		return false, e.Inventory.Seen(ctx, node.ID)
	}
	// What it did is kept whichever way it went. Until this, a refusal or a failure moved
	// last_seen and the reason went to a log line, so "which machine is not doing what it was
	// told" had no answer the next morning — which is the question a mesh exists to answer.
	doing := inventory.Doing{
		Outcome:  inventory.OutcomeApplied,
		Refused:  report.Refused,
		Applied:  len(report.Applied),
		Declared: report.Declared,
	}
	switch {
	case report.Refused != "":
		doing.Outcome = inventory.OutcomeRefused
	case len(report.Failed) > 0:
		doing.Outcome = inventory.OutcomeFailed
	}
	for id, why := range report.Failed {
		doing.Failed = append(doing.Failed, inventory.FailedResource{ID: id, Error: why})
	}
	// Ordered, so two readings of one failure are the same reading.
	sort.Slice(doing.Failed, func(i, j int) bool { return doing.Failed[i].ID < doing.Failed[j].ID })

	// And what that machine says it already holds, so a port is assigned around it rather than
	// on top of it (novox/hq ADR 0038). Kept even when the declaration was refused: what the
	// machine carries is true regardless of what it thought of the last thing it was sent.
	if err := e.Inventory.RecordCarried(ctx, report.Node, report.Carried); err != nil {
		return false, err
	}
	// **An account of a declaration the mesh has moved past never replaces the account it keeps**
	// (novox/hq issue 267). The machine-facts half of such a report is kept above, whenever it
	// arrives; this half is the machine's word on what it did with what it was sent, and the release
	// plan reads it as such. A reconcile that held a machine to the older declaration a moment before
	// the newer one arrived reported *after* the newer apply's report, and stored last, it read as the
	// machine never having applied what it was sent — the plan waited until somebody pushed by hand.
	// One row per node means the last write wins, so the order of arrival must not decide it.
	if report.Declared != "" {
		sent, err := e.Inventory.Outstanding(ctx, report.Node)
		if err != nil {
			return false, err
		}
		if Superseded(report.Declared, sent) {
			log.Printf("kept what %s says about the machine, and not its account of declaration %s: "+
				"the mesh has sent it %s since", report.Node, short(report.Declared), short(sent))
			return false, e.Inventory.Seen(ctx, node.ID)
		}
	}

	// **Whether this is news** is the store's answer: it holds the previous report, and a machine
	// that reconciles every minute says the same thing until something changes (novox/hq ADR 0134).
	news, err = e.Inventory.RecordDoing(ctx, node.ID, doing)
	if err != nil {
		return false, err
	}
	// And how long the machine took, from the send to this first account of it (novox/hq to-be 45
	// Phase 0): what the sent-not-reported bound will be set from. Said if lost, never a failure.
	if err := e.Inventory.RecordApplyDuration(ctx, report.Node, report.Declared, doing.Outcome); err != nil {
		log.Printf("how long %s took to apply could not be recorded: %v", report.Node, err)
	}

	// A refusal, a failure, or a bare word that the node is there — none of them is an account of
	// what the machine holds, so each moves last_seen and nothing else. Recording a partial list
	// as though it were the whole would tell a rebuilding node to remove what it still has.
	if report.Refused != "" || len(report.Failed) > 0 || report.Applied == nil {
		return news, e.Inventory.Seen(ctx, node.ID)
	}
	return news, e.Inventory.RecordOwned(ctx, node.ID, report.Applied)
}