1.7 finished: minting, the file delivered, and a test flake I caused
**First, a correction: the previous commit went in on a false check.** Its message says the suite passed; it did not. The check piped `go test` through a filter that swallowed the failures and then printed "green" regardless. Two tests were failing when4de10e3landed. What was failing was my own doing. Purging the streams instead of deleting them (4de10e3) left the *consumers* behind, because deleting a stream takes its consumers with it and purging does not. A durable push consumer surviving between tests keeps pushing to a delivery subject the previous test's subscription has gone from: the messages count as delivered, go nowhere, and the next test waits out its timeout for an announcement the server believes it already sent. Consumers are now removed with the purge. Five consecutive clean runs. `-p 1` stays, because two packages asserting and deleting the same fixed-name objects on one bus is a real race — but its comment said the cause I had guessed and not the one I found, so it now says the right thing. **And delivery was not finished when I said it was.** Nothing filled `Rendering.BusUsers`, so the composed file would never have reached a node. `composeBusUsers` closes it: composed per push for the machine holding `mesh-broker`, never kept, because the list is a function of the mesh's records and a stored copy could disagree with them while both looked consistent. A user with no credential is left out and named rather than written as a user without a password — an ordinary situation with an obvious remedy — but a file with no users at all is refused, because that bus would refuse every connection in the mesh. **Minting, on both halves.** A node at enrolment and a module at `module issue`. Three things differ from a management call and each is the point of the move: the credential is minted into the mesh's records and becomes usable at the next composition, so no server need be reachable; the password travels beside the address rather than inside it, because a credential embedded in a URL leaks into every log line that prints a connection; and a module's durable consumer is derived from what it declared rather than named, so it cannot ask for delivery of something it did not say it consumes. A node reconnecting may be refused until that composition reaches the machine running the bus. That is what the host's reconnect backoff is for and it is survivable by design; waiting for the push would hold an enrolment open for as long as a declaration takes to apply. Tested that the switch is a switch: a node enrolling on one bus comes away with a credential for that bus and none for the other, because one that held both could be half-moved and nothing would say which half.
This commit is contained in:
@@ -0,0 +1,107 @@
|
||||
package link_test
|
||||
|
||||
import (
|
||||
"crypto/ed25519"
|
||||
"crypto/rand"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/identity"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// What a node is given to come back with, on the bus being built.
|
||||
//
|
||||
// **The credential becomes usable at the next composition, not when it is made**, which is the one
|
||||
// real difference from the bus the mesh runs on today: there a management call makes it live at once.
|
||||
// So what has to be true here is that the mesh recorded it and told the node, and the rest is a push.
|
||||
|
||||
// aMeshReadyToEnrol is both stores with a signing key established, which a control plane does at
|
||||
// start: one that cannot sign is one whose declarations every node correctly refuses.
|
||||
func aMeshReadyToEnrol(t *testing.T) (*inventory.Inventory, *identity.Identity) {
|
||||
t.Helper()
|
||||
inv := inventory.ForTest(t)
|
||||
ident := identity.ForTest(t)
|
||||
if _, err := ident.Establish(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return inv, ident
|
||||
}
|
||||
|
||||
func aTokenFor(t *testing.T, inv *inventory.Inventory, node string) (string, ed25519.PublicKey) {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
if _, err := inv.AddNode(ctx, node); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
issued, err := inv.IssueToken(ctx, node, time.Hour)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
public, _, err := ed25519.GenerateKey(rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return issued.Secret, public
|
||||
}
|
||||
|
||||
// A node enrolling onto the bus being built is told a password of its own, and the mesh keeps only
|
||||
// its hash — which is what the next composition writes into the bus's user list.
|
||||
func TestANodeEnrollingOnTheNewBusIsMintedACredentialTheMeshOnlyHashes(t *testing.T) {
|
||||
inv, ident := aMeshReadyToEnrol(t)
|
||||
ctx := t.Context()
|
||||
secret, public := aTokenFor(t, inv, "anchor")
|
||||
|
||||
reply, err := link.Enrolment{Inventory: inv, Identity: ident, OnNATS: true}.Enrol(ctx, link.EnrolRequest{
|
||||
Node: "anchor", Secret: secret, PublicKey: public})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if reply.Password == "" {
|
||||
t.Fatal("the node was told no password, so it keeps a one-time secret as a credential")
|
||||
}
|
||||
if reply.Password == secret {
|
||||
t.Fatal("the node was handed the token's own secret back: a credential that lives for years " +
|
||||
"must not be the string that was pasted into a terminal")
|
||||
}
|
||||
|
||||
// Recorded under the name the composed file will use, and as a hash: a credential recoverable
|
||||
// from the mesh's store is one whose blast radius is the store's.
|
||||
hash, known, err := inv.BusUserHash(ctx, "node.anchor")
|
||||
if err != nil || !known {
|
||||
t.Fatalf("the mesh kept no credential for the node it enrolled: %v %v", known, err)
|
||||
}
|
||||
if hash == reply.Password {
|
||||
t.Fatal("the store holds the password itself")
|
||||
}
|
||||
if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(reply.Password)); err != nil {
|
||||
t.Fatalf("what the mesh kept does not verify what it told the node: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// On the bus the mesh runs on today, with no management configured, nothing is minted and the node is
|
||||
// told so by being given no password — it keeps the token's secret, which it says out loud.
|
||||
//
|
||||
// **This is the check that the switch is a switch.** A node enrolling on one bus must not come away
|
||||
// with a credential for the other: it would be half-moved, and nothing anywhere would say which half.
|
||||
func TestANodeEnrollingOnTheOldBusIsMintedNoCredentialForTheNewOne(t *testing.T) {
|
||||
inv, ident := aMeshReadyToEnrol(t)
|
||||
ctx := t.Context()
|
||||
secret, public := aTokenFor(t, inv, "anchor")
|
||||
|
||||
reply, err := link.Enrolment{Inventory: inv, Identity: ident}.Enrol(ctx, link.EnrolRequest{
|
||||
Node: "anchor", Secret: secret, PublicKey: public})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if reply.Password != "" {
|
||||
t.Fatalf("a node on the old bus was given a password from nowhere: %q", reply.Password)
|
||||
}
|
||||
if _, known, err := inv.BusUserHash(ctx, "node.anchor"); err != nil || known {
|
||||
t.Fatalf("a node enrolling on the old bus was given a credential for the new one: %v %v",
|
||||
known, err)
|
||||
}
|
||||
}
|
||||
@@ -28,6 +28,15 @@ type Enrolment struct {
|
||||
Identity *identity.Identity
|
||||
Management *broker.Management
|
||||
Broker broker.Broker
|
||||
|
||||
// OnNATS says the mesh's own traffic is on the bus being built, so a node's credential is
|
||||
// minted into the mesh's records and composed into the bus's user list rather than pushed
|
||||
// through a management call (novox/hq design 25 §4).
|
||||
//
|
||||
// **One bus, and a node gets a credential for exactly one** — refused at start if the
|
||||
// controller is told about both (broker.MustBeOneBus), because a node holding a credential for
|
||||
// each is one that could be half-moved, and nothing would say which half.
|
||||
OnNATS bool
|
||||
}
|
||||
|
||||
// Enrol records what the node presented and spends the token.
|
||||
@@ -159,7 +168,33 @@ func (e Enrolment) Enrol(ctx context.Context, request EnrolRequest) (reply Enrol
|
||||
// the spend and not before: a replaced password on an attempt that failed would be held by
|
||||
// nobody. If the broker will not take it now, the enrolment still stands — the node keeps
|
||||
// the token's secret as its password, which it is told, and which is said here.
|
||||
if e.Management != nil {
|
||||
switch {
|
||||
case e.OnNATS:
|
||||
// **Minted into the mesh's records, not pushed to a server.** The bus's users are a file
|
||||
// the controller composes, so a credential becomes usable at the next composition rather
|
||||
// than at the moment it is made — and the plaintext is returned once, here, and then exists
|
||||
// only on the machine it was sealed to.
|
||||
//
|
||||
// The node reconnects as itself and may be refused until that composition reaches the
|
||||
// machine running the bus. That is what the host's reconnect backoff is for and it is
|
||||
// survivable by design (ADR 0004: disconnection is an ordinary situation); waiting for the
|
||||
// push here would hold an enrolment open for as long as a declaration takes to apply.
|
||||
password, err := e.Inventory.MintBusPassword(ctx, inventory.BusUser{
|
||||
Username: broker.Principal{Kind: broker.KindNode, Node: node.Name}.Username(),
|
||||
Kind: inventory.BusNode,
|
||||
Node: node.Name,
|
||||
})
|
||||
if err != nil {
|
||||
// Not fatal to the enrolment: the node is recorded and the token is spent, and a node
|
||||
// that keeps the token's secret is told so. Said loudly, because until this is minted
|
||||
// the machine has no credential of its own.
|
||||
log.Printf("%s is enrolled and the mesh could not mint its bus credential, so it keeps "+
|
||||
"the token's secret as its password: %v", node.Name, err)
|
||||
} else {
|
||||
reply.Password = password
|
||||
}
|
||||
|
||||
case e.Management != nil:
|
||||
password, err := freshPassword()
|
||||
if err != nil {
|
||||
return EnrolReply{}, err
|
||||
|
||||
@@ -36,19 +36,30 @@ func aBus(t *testing.T) *broker.JetStream {
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
|
||||
// The mesh's own streams and consumers, asserted the way the controller asserts them — and
|
||||
// torn down after, so one test's held message is never another's surprise.
|
||||
for _, s := range broker.MeshStreams() {
|
||||
_ = js.Context().DeleteStream(s.Name)
|
||||
}
|
||||
// **Streams purged, consumers removed.** Both halves, and each was learned by getting it wrong.
|
||||
//
|
||||
// The streams are emptied rather than deleted and recreated, because delete-then-add is not a
|
||||
// reset: the server's teardown races the creation, and a test then inherits the previous one's
|
||||
// messages — which reads as a redelivery bug in the code under test.
|
||||
//
|
||||
// The consumers are removed, because deleting a stream used to take them with it and purging
|
||||
// does not. A durable *push* consumer that survives between tests keeps pushing to a delivery
|
||||
// subject the previous test's subscription has gone from: the messages count as delivered, go
|
||||
// nowhere, and the next test waits out its timeout for an announcement the server believes it
|
||||
// already sent. The controller recreates what it needs on start, so leaving none is correct.
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
for _, s := range broker.MeshStreams() {
|
||||
_ = js.Context().DeleteStream(s.Name)
|
||||
clean := func() {
|
||||
for _, c := range broker.MeshConsumers() {
|
||||
_ = js.Context().DeleteConsumer(c.Stream, c.Name)
|
||||
}
|
||||
})
|
||||
for _, s := range broker.MeshStreams() {
|
||||
_ = js.Context().PurgeStream(s.Name)
|
||||
}
|
||||
}
|
||||
clean()
|
||||
t.Cleanup(clean)
|
||||
return js
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user