Nothing the control queue carries is lost while the store restarts: an enrolment claims its token and spends it last, and is asked to try again; build results, upgrades and catch-ups are handed back, bounded (novox/hq issue 083)

2026-09-22 14:06:28 +02:00
parent b9cdb96a90
commit 1a41b88ed3
8 changed files with 433 additions and 59 deletions
@@ -383,3 +383,67 @@ func TestBeingHeardFromDoesNotChangeWhatANodeOwns(t *testing.T) {
 		t.Errorf("after a bare word that the node is here, the mesh believes it owns %v", owned)
 	}
 }
+
+// **A token is claimed, then spent** (novox/hq issue 083). A presenter may claim it again — an
+// enrolment interrupted by a restarting store asks again with the same key — while another is held
+// off until the lease lapses; and it is spent only by the one holding the claim.
+func TestATokenIsClaimedByOnePresenterAndSpentOnlyByIt(t *testing.T) {
+	inv := fresh(t)
+	ctx := t.Context()
+	if _, err := inv.AddNode(ctx, "laptop"); err != nil {
+		t.Fatal(err)
+	}
+	issued, err := inv.IssueToken(ctx, "laptop", time.Hour)
+	if err != nil {
+		t.Fatal(err)
+	}
+	node, err := inv.Claim(ctx, issued.Secret, "key-a")
+	if err != nil || node.Name != "laptop" {
+		t.Fatalf("a fresh token was not claimed for its node: %v %q", err, node.Name)
+	}
+	if _, err := inv.Claim(ctx, issued.Secret, "key-a"); err != nil {
+		t.Fatalf("the presenter holding the claim could not claim again after an interruption: %v", err)
+	}
+	if _, err := inv.Claim(ctx, issued.Secret, "key-b"); !errors.Is(err, ErrTokenInUse) {
+		t.Fatalf("a second presenter was not held off while the claim is live: %v", err)
+	}
+	if err := inv.Spend(ctx, issued.Secret, "key-b"); !errors.Is(err, ErrTokenRefused) {
+		t.Fatalf("a presenter not holding the claim spent the token: %v", err)
+	}
+	if err := inv.Spend(ctx, issued.Secret, "key-a"); err != nil {
+		t.Fatalf("the presenter holding the claim could not spend it: %v", err)
+	}
+	for _, by := range []string{"key-a", "key-b"} {
+		if _, err := inv.Claim(ctx, issued.Secret, by); !errors.Is(err, ErrTokenRefused) {
+			t.Fatalf("a spent token was claimed again by %s: %v", by, err)
+		}
+	}
+}
+
+// A claim lapses: a host that gave up and was started over, with keys of its own, is not held off
+// for longer than the lease.
+func TestAClaimThatLapsedCanBeTakenByAnotherPresenter(t *testing.T) {
+	inv := fresh(t)
+	ctx := t.Context()
+	if _, err := inv.AddNode(ctx, "laptop"); err != nil {
+		t.Fatal(err)
+	}
+	issued, err := inv.IssueToken(ctx, "laptop", time.Hour)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.Claim(ctx, issued.Secret, "key-a"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.store.Pool().Exec(ctx,
+		`update enrolment_token set claimed_until = now() - interval '1 second' where secret = $1`,
+		hashSecret(issued.Secret)); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.Claim(ctx, issued.Secret, "key-b"); err != nil {
+		t.Fatalf("a lapsed claim held off a new presenter: %v", err)
+	}
+	if err := inv.Spend(ctx, issued.Secret, "key-a"); !errors.Is(err, ErrTokenRefused) {
+		t.Fatalf("the presenter whose claim lapsed could still spend the token: %v", err)
+	}
+}
@@ -0,0 +1,11 @@
+-- A token is claimed by the enrolment presenting it, and spent only when that enrolment has
+-- written everything it needs (novox/hq 04-ISSUES/083).
+--
+-- Spending came first and the node's keys after, in another database: a store that went away
+-- between the two left a spent token and a node with no key, and the host — which makes new keys
+-- on every attempt — could not try again. The claim holds the token for one presenter for a short
+-- lease, so an attempt that failed part-way can be made again by the same presenter, and a second
+-- presenter cannot interleave with the first.
+
+alter table enrolment_token add column claimed_by text;
+alter table enrolment_token add column claimed_until timestamptz;
@@ -212,6 +212,60 @@ var ErrTokenRefused = errors.New("that token cannot be used")
 // The update is the check: one statement that both finds a live token and marks it used, so two
 // simultaneous redemptions of one secret cannot both succeed. Reading first and writing second
 // would leave exactly that gap.
+// ClaimLease is how long a claimed token is held for the one presenter that claimed it. Long
+// enough for an enrolment to be tried again through a store restart; short enough that a host
+// which gave up and was started over, with keys of its own, is not kept waiting long.
+const ClaimLease = 2 * time.Minute
+
+// ErrTokenInUse is a token another presenter holds a claim on right now. Not a refusal: the claim
+// lapses, and asking again after it is the answer.
+var ErrTokenInUse = errors.New("the token is being used by another enrolment")
+
+// Claim takes a token for one presenter — `by`, which names the key presenting it — for the length
+// of a lease, and says which node it enrols. The same presenter may claim it again, as may anyone
+// once the lease has lapsed; nothing is spent until Spend (novox/hq 04-ISSUES/083).
+func (i *Inventory) Claim(ctx context.Context, secret, by string) (Node, error) {
+	var id string
+	err := i.store.Pool().QueryRow(ctx,
+		`update enrolment_token set claimed_by = $2, claimed_until = now() + $3::interval
+		  where secret = $1 and redeemed is null and expires > now()
+		    and (claimed_by is null or claimed_by = $2 or claimed_until < now())
+		  returning node`, hashSecret(secret), by, ClaimLease.String()).Scan(&id)
+	if errors.Is(err, pgx.ErrNoRows) {
+		// Unusable, or held by someone else — told apart, because the second passes.
+		var held bool
+		probe := i.store.Pool().QueryRow(ctx,
+			`select true from enrolment_token
+			  where secret = $1 and redeemed is null and expires > now()`, hashSecret(secret)).Scan(&held)
+		if probe == nil && held {
+			return Node{}, ErrTokenInUse
+		}
+		return Node{}, ErrTokenRefused
+	}
+	if err != nil {
+		return Node{}, err
+	}
+	var n Node
+	err = i.store.Pool().QueryRow(ctx,
+		`select id, name, created from node where id = $1`, id).Scan(&n.ID, &n.Name, &n.Created)
+	return n, err
+}
+
+// Spend makes a claimed token used, only for the presenter holding the claim. The last write of an
+// enrolment, so a token is spent exactly when the node it enrolled is complete.
+func (i *Inventory) Spend(ctx context.Context, secret, by string) error {
+	tag, err := i.store.Pool().Exec(ctx,
+		`update enrolment_token set redeemed = now()
+		  where secret = $1 and redeemed is null and claimed_by = $2`, hashSecret(secret), by)
+	if err != nil {
+		return err
+	}
+	if tag.RowsAffected() == 0 {
+		return ErrTokenRefused
+	}
+	return nil
+}
+
 func (i *Inventory) Redeem(ctx context.Context, secret string) (Node, error) {
 	var id string
 	err := i.store.Pool().QueryRow(ctx,