1.7 finished: minting, the file delivered, and a test flake I caused

**First, a correction: the previous commit went in on a false check.** Its message
says the suite passed; it did not. The check piped `go test` through a filter that
swallowed the failures and then printed "green" regardless. Two tests were failing
when 4de10e3 landed.

What was failing was my own doing. Purging the streams instead of deleting them
(4de10e3) left the *consumers* behind, because deleting a stream takes its
consumers with it and purging does not. A durable push consumer surviving between
tests keeps pushing to a delivery subject the previous test's subscription has gone
from: the messages count as delivered, go nowhere, and the next test waits out its
timeout for an announcement the server believes it already sent. Consumers are now
removed with the purge. Five consecutive clean runs.

`-p 1` stays, because two packages asserting and deleting the same fixed-name
objects on one bus is a real race — but its comment said the cause I had guessed
and not the one I found, so it now says the right thing.

**And delivery was not finished when I said it was.** Nothing filled
`Rendering.BusUsers`, so the composed file would never have reached a node.
`composeBusUsers` closes it: composed per push for the machine holding
`mesh-broker`, never kept, because the list is a function of the mesh's records and
a stored copy could disagree with them while both looked consistent. A user with no
credential is left out and named rather than written as a user without a password —
an ordinary situation with an obvious remedy — but a file with no users at all is
refused, because that bus would refuse every connection in the mesh.

**Minting, on both halves.** A node at enrolment and a module at `module issue`.
Three things differ from a management call and each is the point of the move: the
credential is minted into the mesh's records and becomes usable at the next
composition, so no server need be reachable; the password travels beside the address
rather than inside it, because a credential embedded in a URL leaks into every log
line that prints a connection; and a module's durable consumer is derived from what
it declared rather than named, so it cannot ask for delivery of something it did not
say it consumes.

A node reconnecting may be refused until that composition reaches the machine
running the bus. That is what the host's reconnect backoff is for and it is
survivable by design; waiting for the push would hold an enrolment open for as long
as a declaration takes to apply.

Tested that the switch is a switch: a node enrolling on one bus comes away with a
credential for that bus and none for the other, because one that held both could be
half-moved and nothing would say which half.
This commit is contained in:
2026-09-27 03:19:41 +02:00
parent 4de10e32e3
commit eb72ec36ba
8 changed files with 340 additions and 23 deletions
+107
View File
@@ -0,0 +1,107 @@
package link_test
import (
"crypto/ed25519"
"crypto/rand"
"testing"
"time"
"golang.org/x/crypto/bcrypt"
"github.com/novox/mesh-controller/internal/identity"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// What a node is given to come back with, on the bus being built.
//
// **The credential becomes usable at the next composition, not when it is made**, which is the one
// real difference from the bus the mesh runs on today: there a management call makes it live at once.
// So what has to be true here is that the mesh recorded it and told the node, and the rest is a push.
// aMeshReadyToEnrol is both stores with a signing key established, which a control plane does at
// start: one that cannot sign is one whose declarations every node correctly refuses.
func aMeshReadyToEnrol(t *testing.T) (*inventory.Inventory, *identity.Identity) {
t.Helper()
inv := inventory.ForTest(t)
ident := identity.ForTest(t)
if _, err := ident.Establish(t.Context()); err != nil {
t.Fatal(err)
}
return inv, ident
}
func aTokenFor(t *testing.T, inv *inventory.Inventory, node string) (string, ed25519.PublicKey) {
t.Helper()
ctx := t.Context()
if _, err := inv.AddNode(ctx, node); err != nil {
t.Fatal(err)
}
issued, err := inv.IssueToken(ctx, node, time.Hour)
if err != nil {
t.Fatal(err)
}
public, _, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
t.Fatal(err)
}
return issued.Secret, public
}
// A node enrolling onto the bus being built is told a password of its own, and the mesh keeps only
// its hash — which is what the next composition writes into the bus's user list.
func TestANodeEnrollingOnTheNewBusIsMintedACredentialTheMeshOnlyHashes(t *testing.T) {
inv, ident := aMeshReadyToEnrol(t)
ctx := t.Context()
secret, public := aTokenFor(t, inv, "anchor")
reply, err := link.Enrolment{Inventory: inv, Identity: ident, OnNATS: true}.Enrol(ctx, link.EnrolRequest{
Node: "anchor", Secret: secret, PublicKey: public})
if err != nil {
t.Fatal(err)
}
if reply.Password == "" {
t.Fatal("the node was told no password, so it keeps a one-time secret as a credential")
}
if reply.Password == secret {
t.Fatal("the node was handed the token's own secret back: a credential that lives for years " +
"must not be the string that was pasted into a terminal")
}
// Recorded under the name the composed file will use, and as a hash: a credential recoverable
// from the mesh's store is one whose blast radius is the store's.
hash, known, err := inv.BusUserHash(ctx, "node.anchor")
if err != nil || !known {
t.Fatalf("the mesh kept no credential for the node it enrolled: %v %v", known, err)
}
if hash == reply.Password {
t.Fatal("the store holds the password itself")
}
if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(reply.Password)); err != nil {
t.Fatalf("what the mesh kept does not verify what it told the node: %v", err)
}
}
// On the bus the mesh runs on today, with no management configured, nothing is minted and the node is
// told so by being given no password — it keeps the token's secret, which it says out loud.
//
// **This is the check that the switch is a switch.** A node enrolling on one bus must not come away
// with a credential for the other: it would be half-moved, and nothing anywhere would say which half.
func TestANodeEnrollingOnTheOldBusIsMintedNoCredentialForTheNewOne(t *testing.T) {
inv, ident := aMeshReadyToEnrol(t)
ctx := t.Context()
secret, public := aTokenFor(t, inv, "anchor")
reply, err := link.Enrolment{Inventory: inv, Identity: ident}.Enrol(ctx, link.EnrolRequest{
Node: "anchor", Secret: secret, PublicKey: public})
if err != nil {
t.Fatal(err)
}
if reply.Password != "" {
t.Fatalf("a node on the old bus was given a password from nowhere: %q", reply.Password)
}
if _, known, err := inv.BusUserHash(ctx, "node.anchor"); err != nil || known {
t.Fatalf("a node enrolling on the old bus was given a credential for the new one: %v %v",
known, err)
}
}
+36 -1
View File
@@ -28,6 +28,15 @@ type Enrolment struct {
Identity *identity.Identity
Management *broker.Management
Broker broker.Broker
// OnNATS says the mesh's own traffic is on the bus being built, so a node's credential is
// minted into the mesh's records and composed into the bus's user list rather than pushed
// through a management call (novox/hq design 25 §4).
//
// **One bus, and a node gets a credential for exactly one** — refused at start if the
// controller is told about both (broker.MustBeOneBus), because a node holding a credential for
// each is one that could be half-moved, and nothing would say which half.
OnNATS bool
}
// Enrol records what the node presented and spends the token.
@@ -159,7 +168,33 @@ func (e Enrolment) Enrol(ctx context.Context, request EnrolRequest) (reply Enrol
// the spend and not before: a replaced password on an attempt that failed would be held by
// nobody. If the broker will not take it now, the enrolment still stands — the node keeps
// the token's secret as its password, which it is told, and which is said here.
if e.Management != nil {
switch {
case e.OnNATS:
// **Minted into the mesh's records, not pushed to a server.** The bus's users are a file
// the controller composes, so a credential becomes usable at the next composition rather
// than at the moment it is made — and the plaintext is returned once, here, and then exists
// only on the machine it was sealed to.
//
// The node reconnects as itself and may be refused until that composition reaches the
// machine running the bus. That is what the host's reconnect backoff is for and it is
// survivable by design (ADR 0004: disconnection is an ordinary situation); waiting for the
// push here would hold an enrolment open for as long as a declaration takes to apply.
password, err := e.Inventory.MintBusPassword(ctx, inventory.BusUser{
Username: broker.Principal{Kind: broker.KindNode, Node: node.Name}.Username(),
Kind: inventory.BusNode,
Node: node.Name,
})
if err != nil {
// Not fatal to the enrolment: the node is recorded and the token is spent, and a node
// that keeps the token's secret is told so. Said loudly, because until this is minted
// the machine has no credential of its own.
log.Printf("%s is enrolled and the mesh could not mint its bus credential, so it keeps "+
"the token's secret as its password: %v", node.Name, err)
} else {
reply.Password = password
}
case e.Management != nil:
password, err := freshPassword()
if err != nil {
return EnrolReply{}, err
+20 -9
View File
@@ -36,19 +36,30 @@ func aBus(t *testing.T) *broker.JetStream {
}
t.Cleanup(js.Close)
// The mesh's own streams and consumers, asserted the way the controller asserts them — and
// torn down after, so one test's held message is never another's surprise.
for _, s := range broker.MeshStreams() {
_ = js.Context().DeleteStream(s.Name)
}
// **Streams purged, consumers removed.** Both halves, and each was learned by getting it wrong.
//
// The streams are emptied rather than deleted and recreated, because delete-then-add is not a
// reset: the server's teardown races the creation, and a test then inherits the previous one's
// messages — which reads as a redelivery bug in the code under test.
//
// The consumers are removed, because deleting a stream used to take them with it and purging
// does not. A durable *push* consumer that survives between tests keeps pushing to a delivery
// subject the previous test's subscription has gone from: the messages count as delivered, go
// nowhere, and the next test waits out its timeout for an announcement the server believes it
// already sent. The controller recreates what it needs on start, so leaving none is correct.
if err := broker.AssertMeshStreams(js); err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
for _, s := range broker.MeshStreams() {
_ = js.Context().DeleteStream(s.Name)
clean := func() {
for _, c := range broker.MeshConsumers() {
_ = js.Context().DeleteConsumer(c.Stream, c.Name)
}
})
for _, s := range broker.MeshStreams() {
_ = js.Context().PurgeStream(s.Name)
}
}
clean()
t.Cleanup(clean)
return js
}