Nothing the control queue carries is lost while the store restarts: an enrolment claims its token and spends it last, and is asked to try again; build results, upgrades and catch-ups are handed back, bounded (novox/hq issue 083)

internal/link/enrolment.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"crypto/ed25519"
 	"crypto/rand"
+	"crypto/sha256"
 	"encoding/base64"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"log"
@@ -34,26 +36,35 @@ type Enrolment struct {
 // single statement that both finds and marks it, so two machines racing on one secret produce one
 // winner. Only then is a key recorded — because recording a key for a node whose token turned out
 // to be spent would leave the mesh believing a machine that never had the right to join.
-func (e Enrolment) Enrol(ctx context.Context, request EnrolRequest) (EnrolReply, error) {
+func (e Enrolment) Enrol(ctx context.Context, request EnrolRequest) (reply EnrolReply, err error) {
 	secret, public, profile := request.Secret, ed25519.PublicKey(request.PublicKey), request.Profile
 
+	// A store that could not be asked right now, or a token another presenter holds for the
+	// moment, is "not now": the node asks again with the same request (novox/hq issue 083).
+	defer func() {
+		if inventory.Unreachable(err) || errors.Is(err, inventory.ErrTokenInUse) {
+			err = fmt.Errorf("%w: %w", ErrTryAgain, err)
+		}
+	}()
+
 	if len(public) != ed25519.PublicKeySize {
 		return EnrolReply{}, fmt.Errorf("a node presented a %d-byte key, and an identity is %d",
 			len(public), ed25519.PublicKeySize)
 	}
 
-	node, err := e.Inventory.Redeem(ctx, secret)
+	// Claimed, not spent: the token is held for this presenter while the node is written, and
+	// spent only as the last write. The node's identity lives in another database than the
+	// token, so the two cannot be one transaction; a failure between them used to leave a spent
+	// token and a node with no key, which the host — making new keys on every attempt — could not
+	// recover from. Every write below overwrites, so an attempt made again is safe.
+	by := claimant(public)
+	node, err := e.Inventory.Claim(ctx, secret, by)
 	if err != nil {
 		return EnrolReply{}, err
 	}
 
-	// From here the token is gone whatever happens next, so anything that fails leaves a node
-	// record with no live key — which is visible and fixable with a new token, where a spent
-	// token believed to be unspent is neither.
 	if _, err := e.Identity.RecordNodeKey(ctx, node.ID, public); err != nil {
-		return EnrolReply{}, fmt.Errorf(
-			"the token was spent and the key could not be recorded, so %s has no identity and "+
-				"needs a new token: %w", node.Name, err)
+		return EnrolReply{}, fmt.Errorf("%s's key could not be recorded: %w", node.Name, err)
 	}
 
 	key, err := e.Identity.Active(ctx)
@@ -61,7 +72,7 @@ func (e Enrolment) Enrol(ctx context.Context, request EnrolRequest) (EnrolReply,
 		return EnrolReply{}, err
 	}
 
-	reply := EnrolReply{
+	reply = EnrolReply{
 		Accepted:    true,
 		Node:        node.Name,
 		Queue:       QueueFor(node.Name),
@@ -80,8 +91,7 @@ func (e Enrolment) Enrol(ctx context.Context, request EnrolRequest) (EnrolReply,
 		}
 		if err := e.Management.CreateNodeAccount(ctx, node.Name, password); err != nil {
 			return EnrolReply{}, fmt.Errorf(
-				"the token was spent and %s's broker password could not be replaced: %w",
-				node.Name, err)
+				"%s's broker password could not be replaced: %w", node.Name, err)
 		}
 		reply.Password = password
 	}
@@ -98,24 +108,27 @@ func (e Enrolment) Enrol(ctx context.Context, request EnrolRequest) (EnrolReply,
 	if request.ServingKey != "" {
 		if err := e.Identity.RecordServingKey(ctx, node.ID, request.ServingKey); err != nil {
 			return EnrolReply{}, fmt.Errorf(
-				"the token was spent and %s's serving key could not be recorded: %w",
-				node.Name, err)
+				"%s's serving key could not be recorded: %w", node.Name, err)
 		}
 	}
 	if request.SealingKey != "" {
 		if err := e.Inventory.RecordSealingKey(ctx, node.ID, request.SealingKey); err != nil {
 			return EnrolReply{}, fmt.Errorf(
-				"the token was spent and %s's sealing key could not be recorded: %w",
-				node.Name, err)
+				"%s's sealing key could not be recorded: %w", node.Name, err)
 		}
 	}
 	if request.OverlayKey != "" {
 		if err := e.Inventory.RecordOverlayKey(ctx, node.ID, request.OverlayKey); err != nil {
 			return EnrolReply{}, fmt.Errorf(
-				"the token was spent and %s's overlay key could not be recorded: %w", node.Name, err)
+				"%s's overlay key could not be recorded: %w", node.Name, err)
 		}
 	}
 
+	// Spent last, so a token is used exactly when the node it enrolled is complete.
+	if err := e.Inventory.Spend(ctx, secret, by); err != nil {
+		return EnrolReply{}, fmt.Errorf("%s was written and its token could not be spent: %w", node.Name, err)
+	}
+
 	if profile != nil {
 		// Not fatal if it fails. The profile is what the control plane needs in order to decide
 		// what this machine should run, and it is reported again on every connection — so losing
@@ -125,6 +138,12 @@ func (e Enrolment) Enrol(ctx context.Context, request EnrolRequest) (EnrolReply,
 	return reply, nil
 }
 
+// claimant names the key presenting a token, so a claim can be held for it alone.
+func claimant(public ed25519.PublicKey) string {
+	sum := sha256.Sum256(public)
+	return hex.EncodeToString(sum[:])
+}
+
 // freshPassword is the node's own broker credential from enrolment onward.
 func freshPassword() (string, error) {
 	raw := make([]byte, 32)