mesh-controller/internal/inventory/inventory_test.go

package inventory

import (
	"context"
	"errors"
	"strings"
	"sync"
	"testing"
	"time"
)

// Against a real PostgreSQL, for the reason novox/hq ADR 0017 gives: what is being tested here is
// that the database enforces what this code relies on it enforcing — a unique name, a token that
// two racing redemptions cannot both spend, a cascade that leaves no token behind. A fake would
// assert that the fake enforces them.

// fresh is a database of this test's own, made and dropped around it.
func fresh(t *testing.T) *Inventory {
	t.Helper()
	return ForTest(t)
}

func TestANodeRecordRoundTrips(t *testing.T) {
	inv := fresh(t)
	made, err := inv.AddNode(t.Context(), "workstation")
	if err != nil {
		t.Fatal(err)
	}

	found, err := inv.NodeByName(t.Context(), "workstation")
	if err != nil {
		t.Fatal(err)
	}
	if found.ID != made.ID {
		t.Errorf("added %s and found %s", made.ID, found.ID)
	}
}

func TestTwoNodesCannotShareAName(t *testing.T) {
	// A name is how a token is issued for a node. Two records with one name makes that command
	// ambiguous at the moment it grants access to the mesh.
	inv := fresh(t)
	if _, err := inv.AddNode(t.Context(), "workstation"); err != nil {
		t.Fatal(err)
	}
	_, err := inv.AddNode(t.Context(), "workstation")
	if !errors.Is(err, ErrNameTaken) {
		t.Fatalf("a duplicate name gave %v; it must be a plain answer a person can act on", err)
	}
}

func TestAnUnknownNodeIsNotAnEmptyRecord(t *testing.T) {
	inv := fresh(t)
	_, err := inv.NodeByName(t.Context(), "never-existed")
	if !errors.Is(err, ErrNoSuchNode) {
		t.Fatalf("expected ErrNoSuchNode, got %v", err)
	}
}

func TestATokenIsRedeemableExactlyOnce(t *testing.T) {
	// "Useless once used" (novox/hq ADR 0004). Without it a token that leaked after a successful
	// join is a second machine's way in, and nothing would have noticed the first.
	inv := fresh(t)
	if _, err := inv.AddNode(t.Context(), "laptop"); err != nil {
		t.Fatal(err)
	}
	issued, err := inv.IssueToken(t.Context(), "laptop", time.Hour)
	if err != nil {
		t.Fatal(err)
	}

	node, err := inv.Redeem(t.Context(), issued.Secret)
	if err != nil {
		t.Fatalf("a fresh token was refused: %v", err)
	}
	if node.Name != "laptop" {
		t.Errorf("redeemed a token for %q", node.Name)
	}

	if _, err := inv.Redeem(t.Context(), issued.Secret); !errors.Is(err, ErrTokenRefused) {
		t.Fatal("the same token was redeemed twice")
	}
}

func TestAnExpiredTokenIsRefused(t *testing.T) {
	// "Useless after it expires" — the other half, and the one nothing notices, because a token
	// ages out with nobody watching. It has to be read from the row rather than from a status
	// something would have had to write.
	inv := fresh(t)
	if _, err := inv.AddNode(t.Context(), "laptop"); err != nil {
		t.Fatal(err)
	}
	issued, err := inv.IssueToken(t.Context(), "laptop", 40*time.Millisecond)
	if err != nil {
		t.Fatal(err)
	}
	time.Sleep(120 * time.Millisecond)

	if _, err := inv.Redeem(t.Context(), issued.Secret); !errors.Is(err, ErrTokenRefused) {
		t.Fatal("an expired token was accepted")
	}
}

func TestATokenWithNoLifetimeIsRefused(t *testing.T) {
	inv := fresh(t)
	if _, err := inv.AddNode(t.Context(), "laptop"); err != nil {
		t.Fatal(err)
	}
	if _, err := inv.IssueToken(t.Context(), "laptop", 0); err == nil {
		t.Fatal("a token that never expires was issued")
	}
}

func TestIssuingAgainInvalidatesTheOutstandingToken(t *testing.T) {
	// Two live tokens for one node record are two machines able to join as the same node, with
	// nothing downstream able to tell which was meant.
	inv := fresh(t)
	if _, err := inv.AddNode(t.Context(), "laptop"); err != nil {
		t.Fatal(err)
	}
	first, err := inv.IssueToken(t.Context(), "laptop", time.Hour)
	if err != nil {
		t.Fatal(err)
	}
	second, err := inv.IssueToken(t.Context(), "laptop", time.Hour)
	if err != nil {
		t.Fatal(err)
	}

	if _, err := inv.Redeem(t.Context(), first.Secret); !errors.Is(err, ErrTokenRefused) {
		t.Error("the first token still worked after a second was issued")
	}
	if _, err := inv.Redeem(t.Context(), second.Secret); err != nil {
		t.Errorf("the newest token was refused: %v", err)
	}
}

func TestTheSecretIsNotStored(t *testing.T) {
	// A copy of this database must not be a set of working credentials.
	inv := fresh(t)
	if _, err := inv.AddNode(t.Context(), "laptop"); err != nil {
		t.Fatal(err)
	}
	issued, err := inv.IssueToken(t.Context(), "laptop", time.Hour)
	if err != nil {
		t.Fatal(err)
	}

	var stored string
	if err := inv.store.Pool().QueryRow(t.Context(),
		`select secret from enrolment_token limit 1`).Scan(&stored); err != nil {
		t.Fatal(err)
	}
	if stored == issued.Secret {
		t.Fatal("the token secret is stored verbatim; this table would be a set of live credentials")
	}
	if strings.Contains(stored, issued.Secret) {
		t.Fatal("the stored value contains the secret")
	}
}

func TestTwoRedemptionsOfOneSecretCannotBothWin(t *testing.T) {
	// The check and the spend are one statement for this reason. Reading first and writing second
	// leaves a window where two machines both pass the check and both join as the same node.
	inv := fresh(t)
	if _, err := inv.AddNode(t.Context(), "laptop"); err != nil {
		t.Fatal(err)
	}
	issued, err := inv.IssueToken(t.Context(), "laptop", time.Hour)
	if err != nil {
		t.Fatal(err)
	}

	var wg sync.WaitGroup
	results := make([]error, 8)
	for i := range results {
		wg.Add(1)
		go func(i int) {
			defer wg.Done()
			_, results[i] = inv.Redeem(context.Background(), issued.Secret)
		}(i)
	}
	wg.Wait()

	won := 0
	for _, err := range results {
		if err == nil {
			won++
		}
	}
	if won != 1 {
		t.Errorf("%d of 8 concurrent redemptions succeeded; exactly one may", won)
	}
}

func TestRemovingANodeTakesItsTokensWithIt(t *testing.T) {
	// A token outliving the record it was issued for is a right to join as nobody.
	inv := fresh(t)
	node, err := inv.AddNode(t.Context(), "laptop")
	if err != nil {
		t.Fatal(err)
	}
	if _, err := inv.IssueToken(t.Context(), "laptop", time.Hour); err != nil {
		t.Fatal(err)
	}
	if _, err := inv.store.Pool().Exec(t.Context(), `delete from node where id = $1`, node.ID); err != nil {
		t.Fatal(err)
	}

	var left int
	if err := inv.store.Pool().QueryRow(t.Context(),
		`select count(*) from enrolment_token`).Scan(&left); err != nil {
		t.Fatal(err)
	}
	if left != 0 {
		t.Errorf("%d token(s) outlived the node record they were issued for", left)
	}
}

func TestAnUnknownSecretIsRefusedTheSameWayAsAnExpiredOne(t *testing.T) {
	// One error for every reason. Somebody guessing must not learn which of their guesses was a
	// real token that had merely expired.
	inv := fresh(t)
	if _, err := inv.AddNode(t.Context(), "laptop"); err != nil {
		t.Fatal(err)
	}
	expired, err := inv.IssueToken(t.Context(), "laptop", 30*time.Millisecond)
	if err != nil {
		t.Fatal(err)
	}
	time.Sleep(100 * time.Millisecond)

	_, unknownErr := inv.Redeem(t.Context(), "not-a-token-at-all")
	_, expiredErr := inv.Redeem(t.Context(), expired.Secret)

	if unknownErr == nil || expiredErr == nil {
		t.Fatal("one of them was accepted")
	}
	if unknownErr.Error() != expiredErr.Error() {
		t.Errorf("the two are distinguishable:\n  unknown: %v\n  expired: %v", unknownErr, expiredErr)
	}
}

func TestNeverReportedIsNotTheSameAsReportedNothing(t *testing.T) {
	// The distinction that makes this safe to hand back. A node that applied nothing holds
	// nothing; a node that has never spoken is unknown — and returning an empty list for the
	// second would tell a rebuilding node it owns nothing, and have it remove whatever it found
	// on the machine.
	inv := fresh(t)
	node, err := inv.AddNode(t.Context(), "laptop")
	if err != nil {
		t.Fatal(err)
	}

	owned, reported, err := inv.Owned(t.Context(), node.ID)
	if err != nil {
		t.Fatal(err)
	}
	if owned != nil {
		t.Errorf("a node that never reported came back owning %v", owned)
	}
	if !reported.IsZero() {
		t.Error("a node that never reported has a report time")
	}

	if err := inv.RecordOwned(t.Context(), node.ID, []string{}); err != nil {
		t.Fatal(err)
	}
	owned, reported, err = inv.Owned(t.Context(), node.ID)
	if err != nil {
		t.Fatal(err)
	}
	if owned == nil {
		t.Error("a node that reported holding nothing is indistinguishable from one that never spoke")
	}
	if reported.IsZero() {
		t.Error("a report that happened has no time on it")
	}
}

func TestWhatANodeOwnsIsReplacedNotAccumulated(t *testing.T) {
	// The question this answers is what is on that machine now. A node that stopped owning
	// something and had it remembered would be handed it back on a rebuild and put it there again.
	inv := fresh(t)
	node, err := inv.AddNode(t.Context(), "laptop")
	if err != nil {
		t.Fatal(err)
	}
	if err := inv.RecordOwned(t.Context(), node.ID, []string{"a", "b", "c"}); err != nil {
		t.Fatal(err)
	}
	if err := inv.RecordOwned(t.Context(), node.ID, []string{"a"}); err != nil {
		t.Fatal(err)
	}

	owned, _, err := inv.Owned(t.Context(), node.ID)
	if err != nil {
		t.Fatal(err)
	}
	if len(owned) != 1 || owned[0] != "a" {
		t.Errorf("after reporting a, the mesh believes the node owns %v", owned)
	}
}

func TestAnAnswerAboutAMachineCarriesItsAge(t *testing.T) {
	// This repository has already been bitten by a cache with no age on it: a node running from
	// one looked identical to a node running from the database. An answer about a machine is
	// worth much less without knowing how old it is, so the age comes back with it.
	inv := fresh(t)
	node, err := inv.AddNode(t.Context(), "laptop")
	if err != nil {
		t.Fatal(err)
	}
	before := time.Now().Add(-time.Second)
	if err := inv.RecordOwned(t.Context(), node.ID, []string{"a"}); err != nil {
		t.Fatal(err)
	}

	_, reported, err := inv.Owned(t.Context(), node.ID)
	if err != nil {
		t.Fatal(err)
	}
	if reported.Before(before) {
		t.Errorf("the report time is %s, which is before the report", reported)
	}
}

func TestNeverHeardFromIsNotTheSameAsLongAgo(t *testing.T) {
	// The distinction the whole thing rests on. A node that has never spoken did not finish
	// joining; a node last heard from a month ago is running a month-old picture of the mesh.
	// Until this existed both looked exactly like a node that is current.
	inv := fresh(t)
	node, err := inv.AddNode(t.Context(), "laptop")
	if err != nil {
		t.Fatal(err)
	}

	nodes, err := inv.Nodes(t.Context())
	if err != nil {
		t.Fatal(err)
	}
	if _, ever := nodes[0].Silent(); ever {
		t.Error("a node that has never spoken reports a time since it last did")
	}

	if err := inv.Seen(t.Context(), node.ID); err != nil {
		t.Fatal(err)
	}
	nodes, err = inv.Nodes(t.Context())
	if err != nil {
		t.Fatal(err)
	}
	silent, ever := nodes[0].Silent()
	if !ever {
		t.Fatal("a node that has spoken still reports never having done so")
	}
	if silent > time.Minute {
		t.Errorf("a node heard from just now has been silent for %s", silent)
	}
}

func TestBeingHeardFromDoesNotChangeWhatANodeOwns(t *testing.T) {
	// A node saying it is there is not an account of what it holds. Treating one as the other
	// would replace the recovery copy with an empty list every minute, and a rebuilding node
	// would then be told it owns nothing.
	inv := fresh(t)
	node, err := inv.AddNode(t.Context(), "laptop")
	if err != nil {
		t.Fatal(err)
	}
	if err := inv.RecordOwned(t.Context(), node.ID, []string{"a", "b"}); err != nil {
		t.Fatal(err)
	}
	if err := inv.Seen(t.Context(), node.ID); err != nil {
		t.Fatal(err)
	}

	owned, _, err := inv.Owned(t.Context(), node.ID)
	if err != nil {
		t.Fatal(err)
	}
	if len(owned) != 2 {
		t.Errorf("after a bare word that the node is here, the mesh believes it owns %v", owned)
	}
}

// **A token is claimed, then spent** (novox/hq issue 083). A presenter may claim it again — an
// enrolment interrupted by a restarting store asks again with the same key — while another is held
// off until the lease lapses; and it is spent only by the one holding the claim.
func TestATokenIsClaimedByOnePresenterAndSpentOnlyByIt(t *testing.T) {
	inv := fresh(t)
	ctx := t.Context()
	if _, err := inv.AddNode(ctx, "laptop"); err != nil {
		t.Fatal(err)
	}
	issued, err := inv.IssueToken(ctx, "laptop", time.Hour)
	if err != nil {
		t.Fatal(err)
	}
	node, err := inv.Claim(ctx, issued.Secret, "key-a")
	if err != nil || node.Name != "laptop" {
		t.Fatalf("a fresh token was not claimed for its node: %v %q", err, node.Name)
	}
	if _, err := inv.Claim(ctx, issued.Secret, "key-a"); err != nil {
		t.Fatalf("the presenter holding the claim could not claim again after an interruption: %v", err)
	}
	if _, err := inv.Claim(ctx, issued.Secret, "key-b"); !errors.Is(err, ErrTokenInUse) {
		t.Fatalf("a second presenter was not held off while the claim is live: %v", err)
	}
	if err := inv.Spend(ctx, issued.Secret, "key-b"); !errors.Is(err, ErrTokenRefused) {
		t.Fatalf("a presenter not holding the claim spent the token: %v", err)
	}
	if err := inv.Spend(ctx, issued.Secret, "key-a"); err != nil {
		t.Fatalf("the presenter holding the claim could not spend it: %v", err)
	}
	for _, by := range []string{"key-a", "key-b"} {
		if _, err := inv.Claim(ctx, issued.Secret, by); !errors.Is(err, ErrTokenRefused) {
			t.Fatalf("a spent token was claimed again by %s: %v", by, err)
		}
	}
}

// A claim lapses: a host that gave up and was started over, with keys of its own, is not held off
// for longer than the lease.
func TestAClaimThatLapsedCanBeTakenByAnotherPresenter(t *testing.T) {
	inv := fresh(t)
	ctx := t.Context()
	if _, err := inv.AddNode(ctx, "laptop"); err != nil {
		t.Fatal(err)
	}
	issued, err := inv.IssueToken(ctx, "laptop", time.Hour)
	if err != nil {
		t.Fatal(err)
	}
	if _, err := inv.Claim(ctx, issued.Secret, "key-a"); err != nil {
		t.Fatal(err)
	}
	if _, err := inv.store.Pool().Exec(ctx,
		`update enrolment_token set claimed_until = now() - interval '1 second' where secret = $1`,
		hashSecret(issued.Secret)); err != nil {
		t.Fatal(err)
	}
	if _, err := inv.Claim(ctx, issued.Secret, "key-b"); err != nil {
		t.Fatalf("a lapsed claim held off a new presenter: %v", err)
	}
	if err := inv.Spend(ctx, issued.Secret, "key-a"); !errors.Is(err, ErrTokenRefused) {
		t.Fatalf("the presenter whose claim lapsed could still spend the token: %v", err)
	}
}