Review of 083: a message the store cannot take is held and retried on a ticker, not slept on, so enrolments are answered meanwhile; a newer one per subject supersedes; the password is replaced after the spend; the same presenter may finish after a lost answer; upgrades retry only on the store

internal/link/enrolment.go

@@ -30,12 +30,15 @@ type Enrolment struct {
 	Broker     broker.Broker
 }
 
-// Enrol spends the token and records what the node presented.
+// Enrol records what the node presented and spends the token.
 //
-// Order matters and it is the order things become irreversible. The token is spent first, in a
-// single statement that both finds and marks it, so two machines racing on one secret produce one
-// winner. Only then is a key recorded — because recording a key for a node whose token turned out
-// to be spent would leave the mesh believing a machine that never had the right to join.
+// Order matters and it is the order things become irreversible (novox/hq issue 083). The token is
+// claimed first, in a single statement that both finds it and holds it for this presenter's key, so
+// two machines racing on one secret produce one holder. Then everything the node presented is
+// written — each write an overwrite, so an attempt interrupted by the store going away can be made
+// again by the same presenter. Then the token is spent. Last, the token's secret stops being the
+// node's broker password: done after the spend, because a password replaced by an attempt that
+// then failed would be one nobody holds, and the node could not even log in to ask again.
 func (e Enrolment) Enrol(ctx context.Context, request EnrolRequest) (reply EnrolReply, err error) {
 	secret, public, profile := request.Secret, ed25519.PublicKey(request.PublicKey), request.Profile
 
@@ -81,21 +84,6 @@ func (e Enrolment) Enrol(ctx context.Context, request EnrolRequest) (reply Enrol
 		Signer:      key.Public,
 	}
 
-	// The token's secret was the broker password up to this moment, which is what let this
-	// connection exist at all. It is replaced now, so the one-time thing stays one-time and the
-	// credential the node keeps for years is not the one that was pasted into a terminal.
-	if e.Management != nil {
-		password, err := freshPassword()
-		if err != nil {
-			return EnrolReply{}, err
-		}
-		if err := e.Management.CreateNodeAccount(ctx, node.Name, password); err != nil {
-			return EnrolReply{}, fmt.Errorf(
-				"%s's broker password could not be replaced: %w", node.Name, err)
-		}
-		reply.Password = password
-	}
-
 	// Recorded before the profile because the overlay is the first declaration this node will
 	// receive, and without this key the mesh cannot compose one. A node enrolled with no overlay
 	// key is a node the graph skips — an ordinary in-between state, and one worth leaving as
@@ -124,11 +112,30 @@ func (e Enrolment) Enrol(ctx context.Context, request EnrolRequest) (reply Enrol
 		}
 	}
 
-	// Spent last, so a token is used exactly when the node it enrolled is complete.
+	// Spent once the node is complete in the store.
 	if err := e.Inventory.Spend(ctx, secret, by); err != nil {
 		return EnrolReply{}, fmt.Errorf("%s was written and its token could not be spent: %w", node.Name, err)
 	}
 
+	// The token's secret was the broker password up to this moment, which is what let this
+	// connection exist at all. It is replaced now, so the one-time thing stays one-time and the
+	// credential the node keeps for years is not the one that was pasted into a terminal. After
+	// the spend and not before: a replaced password on an attempt that failed would be held by
+	// nobody. If the broker will not take it now, the enrolment still stands — the node keeps
+	// the token's secret as its password, which it is told, and which is said here.
+	if e.Management != nil {
+		password, err := freshPassword()
+		if err != nil {
+			return EnrolReply{}, err
+		}
+		if err := e.Management.CreateNodeAccount(ctx, node.Name, password); err != nil {
+			log.Printf("%s is enrolled and its broker password could not be replaced, so it keeps "+
+				"the token's secret as its password: %v", node.Name, err)
+		} else {
+			reply.Password = password
+		}
+	}
+
 	if profile != nil {
 		// Not fatal if it fails. The profile is what the control plane needs in order to decide
 		// what this machine should run, and it is reported again on every connection — so losing