package inventory import ( "context" "errors" "strings" "sync" "testing" "time" ) // Against a real PostgreSQL, for the reason novox/hq ADR 0017 gives: what is being tested here is // that the database enforces what this code relies on it enforcing — a unique name, a token that // two racing redemptions cannot both spend, a cascade that leaves no token behind. A fake would // assert that the fake enforces them. // fresh is a database of this test's own, made and dropped around it. func fresh(t *testing.T) *Inventory { t.Helper() return ForTest(t) } func TestANodeRecordRoundTrips(t *testing.T) { inv := fresh(t) made, err := inv.AddNode(t.Context(), "workstation") if err != nil { t.Fatal(err) } found, err := inv.NodeByName(t.Context(), "workstation") if err != nil { t.Fatal(err) } if found.ID != made.ID { t.Errorf("added %s and found %s", made.ID, found.ID) } } func TestTwoNodesCannotShareAName(t *testing.T) { // A name is how a token is issued for a node. Two records with one name makes that command // ambiguous at the moment it grants access to the mesh. inv := fresh(t) if _, err := inv.AddNode(t.Context(), "workstation"); err != nil { t.Fatal(err) } _, err := inv.AddNode(t.Context(), "workstation") if !errors.Is(err, ErrNameTaken) { t.Fatalf("a duplicate name gave %v; it must be a plain answer a person can act on", err) } } func TestAnUnknownNodeIsNotAnEmptyRecord(t *testing.T) { inv := fresh(t) _, err := inv.NodeByName(t.Context(), "never-existed") if !errors.Is(err, ErrNoSuchNode) { t.Fatalf("expected ErrNoSuchNode, got %v", err) } } func TestATokenIsRedeemableExactlyOnce(t *testing.T) { // "Useless once used" (novox/hq ADR 0004). Without it a token that leaked after a successful // join is a second machine's way in, and nothing would have noticed the first. inv := fresh(t) if _, err := inv.AddNode(t.Context(), "laptop"); err != nil { t.Fatal(err) } issued, err := inv.IssueToken(t.Context(), "laptop", time.Hour) if err != nil { t.Fatal(err) } node, err := inv.Redeem(t.Context(), issued.Secret) if err != nil { t.Fatalf("a fresh token was refused: %v", err) } if node.Name != "laptop" { t.Errorf("redeemed a token for %q", node.Name) } if _, err := inv.Redeem(t.Context(), issued.Secret); !errors.Is(err, ErrTokenRefused) { t.Fatal("the same token was redeemed twice") } } func TestAnExpiredTokenIsRefused(t *testing.T) { // "Useless after it expires" — the other half, and the one nothing notices, because a token // ages out with nobody watching. It has to be read from the row rather than from a status // something would have had to write. inv := fresh(t) if _, err := inv.AddNode(t.Context(), "laptop"); err != nil { t.Fatal(err) } issued, err := inv.IssueToken(t.Context(), "laptop", 40*time.Millisecond) if err != nil { t.Fatal(err) } time.Sleep(120 * time.Millisecond) if _, err := inv.Redeem(t.Context(), issued.Secret); !errors.Is(err, ErrTokenRefused) { t.Fatal("an expired token was accepted") } } func TestATokenWithNoLifetimeIsRefused(t *testing.T) { inv := fresh(t) if _, err := inv.AddNode(t.Context(), "laptop"); err != nil { t.Fatal(err) } if _, err := inv.IssueToken(t.Context(), "laptop", 0); err == nil { t.Fatal("a token that never expires was issued") } } func TestIssuingAgainInvalidatesTheOutstandingToken(t *testing.T) { // Two live tokens for one node record are two machines able to join as the same node, with // nothing downstream able to tell which was meant. inv := fresh(t) if _, err := inv.AddNode(t.Context(), "laptop"); err != nil { t.Fatal(err) } first, err := inv.IssueToken(t.Context(), "laptop", time.Hour) if err != nil { t.Fatal(err) } second, err := inv.IssueToken(t.Context(), "laptop", time.Hour) if err != nil { t.Fatal(err) } if _, err := inv.Redeem(t.Context(), first.Secret); !errors.Is(err, ErrTokenRefused) { t.Error("the first token still worked after a second was issued") } if _, err := inv.Redeem(t.Context(), second.Secret); err != nil { t.Errorf("the newest token was refused: %v", err) } } func TestTheSecretIsNotStored(t *testing.T) { // A copy of this database must not be a set of working credentials. inv := fresh(t) if _, err := inv.AddNode(t.Context(), "laptop"); err != nil { t.Fatal(err) } issued, err := inv.IssueToken(t.Context(), "laptop", time.Hour) if err != nil { t.Fatal(err) } var stored string if err := inv.store.Pool().QueryRow(t.Context(), `select secret from enrolment_token limit 1`).Scan(&stored); err != nil { t.Fatal(err) } if stored == issued.Secret { t.Fatal("the token secret is stored verbatim; this table would be a set of live credentials") } if strings.Contains(stored, issued.Secret) { t.Fatal("the stored value contains the secret") } } func TestTwoRedemptionsOfOneSecretCannotBothWin(t *testing.T) { // The check and the spend are one statement for this reason. Reading first and writing second // leaves a window where two machines both pass the check and both join as the same node. inv := fresh(t) if _, err := inv.AddNode(t.Context(), "laptop"); err != nil { t.Fatal(err) } issued, err := inv.IssueToken(t.Context(), "laptop", time.Hour) if err != nil { t.Fatal(err) } var wg sync.WaitGroup results := make([]error, 8) for i := range results { wg.Add(1) go func(i int) { defer wg.Done() _, results[i] = inv.Redeem(context.Background(), issued.Secret) }(i) } wg.Wait() won := 0 for _, err := range results { if err == nil { won++ } } if won != 1 { t.Errorf("%d of 8 concurrent redemptions succeeded; exactly one may", won) } } func TestRemovingANodeTakesItsTokensWithIt(t *testing.T) { // A token outliving the record it was issued for is a right to join as nobody. inv := fresh(t) node, err := inv.AddNode(t.Context(), "laptop") if err != nil { t.Fatal(err) } if _, err := inv.IssueToken(t.Context(), "laptop", time.Hour); err != nil { t.Fatal(err) } if _, err := inv.store.Pool().Exec(t.Context(), `delete from node where id = $1`, node.ID); err != nil { t.Fatal(err) } var left int if err := inv.store.Pool().QueryRow(t.Context(), `select count(*) from enrolment_token`).Scan(&left); err != nil { t.Fatal(err) } if left != 0 { t.Errorf("%d token(s) outlived the node record they were issued for", left) } } func TestAnUnknownSecretIsRefusedTheSameWayAsAnExpiredOne(t *testing.T) { // One error for every reason. Somebody guessing must not learn which of their guesses was a // real token that had merely expired. inv := fresh(t) if _, err := inv.AddNode(t.Context(), "laptop"); err != nil { t.Fatal(err) } expired, err := inv.IssueToken(t.Context(), "laptop", 30*time.Millisecond) if err != nil { t.Fatal(err) } time.Sleep(100 * time.Millisecond) _, unknownErr := inv.Redeem(t.Context(), "not-a-token-at-all") _, expiredErr := inv.Redeem(t.Context(), expired.Secret) if unknownErr == nil || expiredErr == nil { t.Fatal("one of them was accepted") } if unknownErr.Error() != expiredErr.Error() { t.Errorf("the two are distinguishable:\n unknown: %v\n expired: %v", unknownErr, expiredErr) } } func TestNeverReportedIsNotTheSameAsReportedNothing(t *testing.T) { // The distinction that makes this safe to hand back. A node that applied nothing holds // nothing; a node that has never spoken is unknown — and returning an empty list for the // second would tell a rebuilding node it owns nothing, and have it remove whatever it found // on the machine. inv := fresh(t) node, err := inv.AddNode(t.Context(), "laptop") if err != nil { t.Fatal(err) } owned, reported, err := inv.Owned(t.Context(), node.ID) if err != nil { t.Fatal(err) } if owned != nil { t.Errorf("a node that never reported came back owning %v", owned) } if !reported.IsZero() { t.Error("a node that never reported has a report time") } if err := inv.RecordOwned(t.Context(), node.ID, []string{}); err != nil { t.Fatal(err) } owned, reported, err = inv.Owned(t.Context(), node.ID) if err != nil { t.Fatal(err) } if owned == nil { t.Error("a node that reported holding nothing is indistinguishable from one that never spoke") } if reported.IsZero() { t.Error("a report that happened has no time on it") } } func TestWhatANodeOwnsIsReplacedNotAccumulated(t *testing.T) { // The question this answers is what is on that machine now. A node that stopped owning // something and had it remembered would be handed it back on a rebuild and put it there again. inv := fresh(t) node, err := inv.AddNode(t.Context(), "laptop") if err != nil { t.Fatal(err) } if err := inv.RecordOwned(t.Context(), node.ID, []string{"a", "b", "c"}); err != nil { t.Fatal(err) } if err := inv.RecordOwned(t.Context(), node.ID, []string{"a"}); err != nil { t.Fatal(err) } owned, _, err := inv.Owned(t.Context(), node.ID) if err != nil { t.Fatal(err) } if len(owned) != 1 || owned[0] != "a" { t.Errorf("after reporting a, the mesh believes the node owns %v", owned) } } func TestAnAnswerAboutAMachineCarriesItsAge(t *testing.T) { // This repository has already been bitten by a cache with no age on it: a node running from // one looked identical to a node running from the database. An answer about a machine is // worth much less without knowing how old it is, so the age comes back with it. inv := fresh(t) node, err := inv.AddNode(t.Context(), "laptop") if err != nil { t.Fatal(err) } before := time.Now().Add(-time.Second) if err := inv.RecordOwned(t.Context(), node.ID, []string{"a"}); err != nil { t.Fatal(err) } _, reported, err := inv.Owned(t.Context(), node.ID) if err != nil { t.Fatal(err) } if reported.Before(before) { t.Errorf("the report time is %s, which is before the report", reported) } } func TestNeverHeardFromIsNotTheSameAsLongAgo(t *testing.T) { // The distinction the whole thing rests on. A node that has never spoken did not finish // joining; a node last heard from a month ago is running a month-old picture of the mesh. // Until this existed both looked exactly like a node that is current. inv := fresh(t) node, err := inv.AddNode(t.Context(), "laptop") if err != nil { t.Fatal(err) } nodes, err := inv.Nodes(t.Context()) if err != nil { t.Fatal(err) } if _, ever := nodes[0].Silent(); ever { t.Error("a node that has never spoken reports a time since it last did") } if err := inv.Seen(t.Context(), node.ID); err != nil { t.Fatal(err) } nodes, err = inv.Nodes(t.Context()) if err != nil { t.Fatal(err) } silent, ever := nodes[0].Silent() if !ever { t.Fatal("a node that has spoken still reports never having done so") } if silent > time.Minute { t.Errorf("a node heard from just now has been silent for %s", silent) } } func TestBeingHeardFromDoesNotChangeWhatANodeOwns(t *testing.T) { // A node saying it is there is not an account of what it holds. Treating one as the other // would replace the recovery copy with an empty list every minute, and a rebuilding node // would then be told it owns nothing. inv := fresh(t) node, err := inv.AddNode(t.Context(), "laptop") if err != nil { t.Fatal(err) } if err := inv.RecordOwned(t.Context(), node.ID, []string{"a", "b"}); err != nil { t.Fatal(err) } if err := inv.Seen(t.Context(), node.ID); err != nil { t.Fatal(err) } owned, _, err := inv.Owned(t.Context(), node.ID) if err != nil { t.Fatal(err) } if len(owned) != 2 { t.Errorf("after a bare word that the node is here, the mesh believes it owns %v", owned) } } // **A token is claimed, then spent** (novox/hq issue 083). A presenter may claim it again — an // enrolment interrupted by a restarting store asks again with the same key — while another is held // off until the lease lapses; and it is spent only by the one holding the claim. func TestATokenIsClaimedByOnePresenterAndSpentOnlyByIt(t *testing.T) { inv := fresh(t) ctx := t.Context() if _, err := inv.AddNode(ctx, "laptop"); err != nil { t.Fatal(err) } issued, err := inv.IssueToken(ctx, "laptop", time.Hour) if err != nil { t.Fatal(err) } node, err := inv.Claim(ctx, issued.Secret, "key-a", false) if err != nil || node.Name != "laptop" { t.Fatalf("a fresh token was not claimed for its node: %v %q", err, node.Name) } if _, err := inv.Claim(ctx, issued.Secret, "key-a", false); err != nil { t.Fatalf("the presenter holding the claim could not claim again after an interruption: %v", err) } if _, err := inv.Claim(ctx, issued.Secret, "key-b", false); !errors.Is(err, ErrTokenInUse) { t.Fatalf("a second presenter was not held off while the claim is live: %v", err) } if err := inv.Spend(ctx, issued.Secret, "key-b"); !errors.Is(err, ErrTokenRefused) { t.Fatalf("a presenter not holding the claim spent the token: %v", err) } if err := inv.Spend(ctx, issued.Secret, "key-a"); err != nil { t.Fatalf("the presenter holding the claim could not spend it: %v", err) } // Spent: nobody else may claim it, ever. if _, err := inv.Claim(ctx, issued.Secret, "key-b", false); !errors.Is(err, ErrTokenRefused) { t.Fatalf("a spent token was claimed by another presenter: %v", err) } // Nor the presenter that spent it, without proof it holds the key: a public key is no secret. if _, err := inv.Claim(ctx, issued.Secret, "key-a", false); !errors.Is(err, ErrTokenRefused) { t.Fatalf("a spent token was claimed again with no proof of the key: %v", err) } // With it, and inside the lease, it may, and spend it again: its spend reached the store and // the answer did not reach the node, which asked again — refusing it would lock out a machine // the mesh holds as enrolled. if _, err := inv.Claim(ctx, issued.Secret, "key-a", true); err != nil { t.Fatalf("the proven presenter whose answer was lost after its spend was refused: %v", err) } if err := inv.Spend(ctx, issued.Secret, "key-a"); err != nil { t.Fatalf("spending again by the same presenter failed: %v", err) } // And not once the lease is over: then a spent token is spent to everyone, proof or not. if _, err := inv.store.Pool().Exec(ctx, `update enrolment_token set claimed_until = now() - interval '1 second' where secret = $1`, hashSecret(issued.Secret)); err != nil { t.Fatal(err) } if _, err := inv.Claim(ctx, issued.Secret, "key-a", true); !errors.Is(err, ErrTokenRefused) { t.Fatalf("a spent token was claimed again after its lease: %v", err) } } // A claim lapses: a host that gave up and was started over, with keys of its own, is not held off // for longer than the lease. func TestAClaimThatLapsedCanBeTakenByAnotherPresenter(t *testing.T) { inv := fresh(t) ctx := t.Context() if _, err := inv.AddNode(ctx, "laptop"); err != nil { t.Fatal(err) } issued, err := inv.IssueToken(ctx, "laptop", time.Hour) if err != nil { t.Fatal(err) } if _, err := inv.Claim(ctx, issued.Secret, "key-a", false); err != nil { t.Fatal(err) } if _, err := inv.store.Pool().Exec(ctx, `update enrolment_token set claimed_until = now() - interval '1 second' where secret = $1`, hashSecret(issued.Secret)); err != nil { t.Fatal(err) } if _, err := inv.Claim(ctx, issued.Secret, "key-b", false); err != nil { t.Fatalf("a lapsed claim held off a new presenter: %v", err) } if err := inv.Spend(ctx, issued.Secret, "key-a"); !errors.Is(err, ErrTokenRefused) { t.Fatalf("the presenter whose claim lapsed could still spend the token: %v", err) } }