novox/mesh-host
18
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Say the node's facts in its haves alone, and enrol through the seat alone

Browse Source 
The CONTROL stream the facts, the enrolment and the overlay rekey were published into was the controller's, which ended
by its unassignment (issue 1126) and whose stream is removed after group B's delivery: a publish there would fail every
facts event before its haves are written. The facts go into the haves as they did after the report, the enrolment goes
through mesh-enrolment alone, and overlay take is refused because no seat takes a rekey now.

Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
This commit is contained in:
jschoubben
2026-11-02 11:27:28 +01:00
parent f5ccbb2700
commit a9a0f7b4e0
21 changed files with 55 additions and 1273 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-host/enrol_seat.go
+8 -15
View File
@@ -16,17 +16,17 @@ import (
// enrolThroughSeat joins this machine through the seat mesh-enrolment (novox/hq ADR 0587 §6, issue 774): it redeems the token
// with the keys it made, verifies the vault's signed answer against the key the token carries before it keeps anything, keeps
// its membership and its login, dials as the login node-logins names and confirms. handled is false when the bus does not serve
// the seat to this token (the controller still holds its lease, so its enrolment subject is the way in): nothing was sent or
// kept then, and the caller enrols as before.
// its membership and its login, dials as the login node-logins names and confirms. A bus that does not serve the seat to this
// token (link.ErrNoSeat) is said as the error: nothing was sent or kept.
func enrolThroughSeat(ctx context.Context, opts options, token identity.Token, node string, mine identity.Identity,
sealing identity.SealingKey, serving identity.ServingKey, reported map[string]any, found *link.Tunnel, identityPath string) (handled bool, err error) {
sealing identity.SealingKey, serving identity.ServingKey, reported map[string]any, found *link.Tunnel, identityPath string) error {
req := delivery.Redeem{Node: node, PublicKey: base64.StdEncoding.EncodeToString(mine.Public), OverlayKey: mine.Overlay.Public,
SealingKey: sealing.Public, ServingKey: serving.Public, Profile: reported}
if found != nil {
var err error
if req.Tunnel, err = json.Marshal(found); err != nil {
return true, err
return err
}
}
req.Proof = base64.StdEncoding.EncodeToString(mine.Sign(delivery.RedeemProof(node, req.PublicKey, req.OverlayKey, req.SealingKey, req.ServingKey)))
@@ -61,20 +61,13 @@ func enrolThroughSeat(ctx context.Context, opts options, token identity.Token, n
},
}
answer, err := j.Run(ctx)
switch {
case errors.Is(err, link.ErrNoSeat):
// The path before the lease write: the controller's enrolment subject is granted to the token only while the
// controller holds its lease, so after the write this fallback is refused too, and that refusal is what is said.
fmt.Printf("the seat mesh-enrolment is not served to this token (%v); enrolling through the controller's enrolment "+
"subject, the path before the lease write, which the bus refuses once the lease is written\n", err)
return false, nil
case err != nil:
return true, err
if err != nil {
return err
}
fmt.Printf("\nenrolled as %s through the seat mesh-enrolment, dialled as %s and confirmed\n", answer.Node, answer.Login)
fmt.Printf(" identity %s\n", identityPath)
fmt.Println(" nothing is applied until the placement of this node names a module")
return true, nil
return nil
}
// keepJoinedKeys writes the private halves of the keys this machine made, after its identity: an identity naming a key that is
cmd/mesh-host/healthowed_test.go
-1
View File
@@ -21,7 +21,6 @@ type owedBus struct {
rv map[string]uint64
}
func (b *owedBus) Report(context.Context, string, []byte) error { return nil }
func (b *owedBus) Health(context.Context, string, []byte) error { return nil }
func (b *owedBus) HaveBind(context.Context) (link.HaveBucket, error) { return b, nil }
cmd/mesh-host/main.go
+10 -156
View File
@@ -809,167 +809,21 @@ func enrol(ctx context.Context, opts options) error {
detected := profile.Detect(ctx, profile.Default(nil), opts.timeout)
reported := profileAsReported(detected)
// Signed with the identity just generated, so the mesh can tell this machine from anyone else
// who knows its public key (novox/hq issue 083).
proof := mine.Sign(link.EnrolProof(token.Secret, mine.Public, mine.Overlay.Public,
sealing.Public, serving.Public))
// The token says where to go and which certificate that address must present. It says nothing
// about which bus is there, and does not need to: there is one, and this host knows which
// (novox/hq ADR 0131 — the mesh speaks to one seat and the old transport is gone).
//
// **It used to leave this empty** and mean "whatever the mesh runs today", which was true
// while two buses existed and became a refusal the moment one did: an empty transport is not
// the bus's name, so every enrolment ended at "this token is for the \"\" bus"
// (novox/hq 04-ISSUES/146). Nothing caught it because nothing had enrolled since the bus
// changed.
// Through the seat mesh-enrolment when the bus serves it to this token (novox/hq ADR 0587 §6): the controller's enrolment
// subject below stays for as long as the controller holds its lease.
if handled, err := enrolThroughSeat(ctx, opts, token, *name, mine, sealing, serving, reported, found, identityPath); handled || err != nil {
return err
}
reply, err := link.Enrol(ctx,
link.Approach{Address: token.Broker, Fingerprint: token.Fingerprint, Transport: link.OnNATS},
*name, token.Secret,
mine.Public, mine.Overlay.Public, sealing.Public, serving.Public, reported, proof, found,
opts.timeout)
if err != nil {
return err
}
// The mesh's name for this node wins over what the machine called itself: the token was
// issued for a node record, and that record is what the identity binds to.
mine.Node = reply.Node
mine.Membership = identity.Membership{
Broker: firstNonEmpty(reply.Broker, token.Broker),
Fingerprint: firstNonEmpty(reply.Fingerprint, token.Fingerprint),
Signer: firstNonEmpty2(reply.Signer, token.Signer),
Password: reply.Password,
// **Which bus this membership is for, said rather than left empty** (novox/hq
// 04-ISSUES/146). The link refuses a membership that names another bus, and an empty name
// is not this one's — so a node enrolled without it came up and reconnected for ever
// against its own record: "this membership is for \"\", and the mesh's bus is nats". The
// reply does not carry it because there is one bus and the host knows which (ADR 0131);
// what was missing was writing that down where the link reads it.
Transport: link.OnNATS,
}
if mine.Membership.Password == "" {
// The mesh did not replace the token's secret, so it is still this node's broker
// password. Said rather than silently kept: a one-time secret living on as a credential
// is worth knowing about.
mine.Membership.Password = token.Secret
fmt.Println("\nnote: the mesh issued no separate broker password, so the token's secret " +
"remains this node's credential")
}
// Saved only now, and only once the mesh has said it knows this node. A node holding an
// identity the mesh has never recorded would believe it had joined and be believed by
// nobody — worse than not having joined, because nothing would look wrong.
if err := identity.Save(identityPath, mine); err != nil {
return fmt.Errorf(
"the mesh accepted this node as %q and its identity could not be saved: %w\n"+
"That token is spent, so getting back needs a new one", reply.Node, err)
}
if !mine.Membership.Joined() {
return fmt.Errorf(
"the mesh accepted this node as %q but did not say how to reach it again, so this "+
"identity could not be used after a restart. Nothing was saved", reply.Node)
}
// Written before the identity, so a node that dies between the two has a key file with no
// identity — which enrols again cleanly — rather than an identity naming a key that is not
// there, which looks joined and cannot come up.
if err := os.WriteFile(identity.OverlayKeyPath(opts.state),
[]byte(mine.Overlay.Private+"\n"), 0o600); err != nil {
return fmt.Errorf("cannot write this node's overlay key: %w", err)
}
if err := os.WriteFile(identity.SealingKeyPath(opts.state),
[]byte(sealing.Private+"\n"), 0o600); err != nil {
return fmt.Errorf("cannot write this node's sealing key: %w", err)
}
if err := identity.WriteServingKey(identity.ServingKeyPath(opts.state), serving); err != nil {
return fmt.Errorf("cannot write this node's serving key: %w", err)
}
fmt.Printf("\nenrolled as %s\n", reply.Node)
fmt.Printf(" identity %s\n", identityPath)
fmt.Printf(" queue %s\n", reply.Queue)
return nil
// Through the seat mesh-enrolment (novox/hq ADR 0587 §6), the one way in: the controller's enrolment subject ended
// with the controller (issue 1126).
return enrolThroughSeat(ctx, opts, token, *name, mine, sealing, serving, reported, found, identityPath)
}
// overlayCommand is `mesh-host overlay take`: this node takes the tunnel found on its machine over
// after it enrolled (novox/hq ADR 0105). For a node that enrolled before the mesh knew to take a
// tunnel over — re-enrolling would rotate its identity, sealing and serving keys, and with them
// every credential the mesh sealed to it.
func overlayCommand(ctx context.Context, opts options) error {
// overlayCommand is `mesh-host overlay take`: a node taking the tunnel found on its machine over after it enrolled
// (novox/hq ADR 0105). Refused, and nothing is changed: the rekey it sent was taken by the controller alone, which
// ended by its unassignment (issue 1126), and no seat takes one now; a key written here that no part of the mesh knows
// would raise the mesh's interface on a key its hub does not peer.
func overlayCommand(_ context.Context, opts options) error {
if opts.file != "take" {
return errors.New("overlay take [--tunnel <iface>] — the one thing `overlay` does here")
}
identityPath := identity.Path(opts.state)
mine, err := identity.Load(identityPath)
if err != nil {
return err
}
found, err := tunnel.Find(ctx, apply.ExecRunner, opts.tunnel)
if err != nil {
return fmt.Errorf("%w. Nothing was changed", err)
}
taken, rekey, err := rekeyOnto(mine, found)
if err != nil {
return err
}
fmt.Printf("taking over %s: this node's overlay key becomes the tunnel's, %s\n", found, taken.Overlay.Public)
fmt.Printf(" identity, sealing and serving keys are untouched\n")
// Told first, then written: a mesh told and a machine not yet written is put right by running
// this again (the mesh refuses the stale second rekey and changes nothing; the files are
// rewritten the same). A machine written and a mesh not told would raise the mesh's interface
// on a key the mesh does not know at the next restart.
if err := link.Publish(ctx, link.Membership{
Node: mine.Node, Broker: mine.Membership.Broker, Fingerprint: mine.Membership.Fingerprint,
Password: mine.Membership.Password, Signer: mine.Membership.Signer,
Transport: mine.Membership.Transport,
}, link.Report{Node: mine.Node, Rekey: &rekey}, opts.timeout); err != nil {
return fmt.Errorf("the mesh could not be told; nothing was written here: %w", err)
}
fmt.Printf(" told the mesh signed rekey sent; `node show %s` on the controller says whether it took\n", mine.Node)
if err := os.WriteFile(identity.OverlayKeyPath(opts.state),
[]byte(taken.Overlay.Private+"\n"), 0o600); err != nil {
return fmt.Errorf("the mesh was told and this node's overlay key could not be written: %w — run this again", err)
}
if err := identity.Save(identityPath, taken); err != nil {
return fmt.Errorf("the mesh was told and this node's identity could not be saved: %w — run this again", err)
}
fmt.Printf(" written %s and the identity; the mesh's interface reads the key when the next push restarts it\n",
identity.OverlayKeyPath(opts.state))
fmt.Printf(" next on the controller: `overlay place %s --hub --endpoint <host>:%d …`, `plan %s --json`, then push\n",
mine.Node, found.Port, mine.Node)
return nil
}
// rekeyOnto is the identity with the found tunnel's key as its overlay key, and the signed rekey
// that tells the mesh. Pure, so it can be held to: node, sealing and serving keys are the same
// bytes in and out; only the overlay key moves. Run again after a take, the previous key it names
// is the one before the take, so the mesh can tell a repeat from a replay.
func rekeyOnto(mine identity.Identity, found tunnel.Found) (identity.Identity, link.Rekey, error) {
overlay, err := identity.OverlayKeyFrom(found.PrivateKey())
if err != nil {
return identity.Identity{}, link.Rekey{}, err
}
previous := mine.Overlay.Public
if previous == overlay.Public && mine.OverlayBefore != "" {
previous = mine.OverlayBefore
}
taken := mine
taken.Overlay = overlay
if previous != overlay.Public {
taken.OverlayBefore = previous
}
presented := carried(found)
rekey := link.Rekey{Previous: previous, OverlayKey: overlay.Public, Tunnel: presented}
rekey.Proof = mine.Sign(link.RekeyProof(mine.Node, previous, overlay.Public, presented))
return taken, rekey, nil
return errors.New("overlay take is refused: the controller that took a rekey ended (issue 1126) and no seat " +
"takes one now. Nothing was changed")
}
// carried is a found tunnel as it is presented to the mesh: everything but its private key.
cmd/mesh-host/nodeagent_integration_test.go
-4
View File
@@ -440,10 +440,6 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) {
builds := kvOf(delivery.BuilderBuildsBucket)
candidates := kvOf(delivery.PlacementCandidatesBucket)
packages := kvOf(link.PackagesBucket)
// The controller's stream the node's facts go into, so the node's key carries the facts it says.
if _, err := b.js.AddStream(&nats.StreamConfig{Name: "CONTROL", Subjects: []string{link.ReportSubject(itNode)}}); err != nil {
t.Fatal(err)
}
path := filepath.Join(b.dir, "web.conf")
man := `{"module":"web","version":"1","config":{"greeting":{"type":"string","description":"what the file says","default":"hello"}},` +
`"resources":[{"id":"conf","type":"file","path":"` + path + `","mode":"0644","content":"greeting=${config:greeting}\n"},` +
cmd/mesh-host/rekey_test.go
-94
View File
@@ -1,94 +0,0 @@
package main
import (
"crypto/ed25519"
"strings"
"testing"
"github.com/novox/mesh-host/internal/identity"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/tunnel"
)
// novox/hq ADR 0105: `overlay take` moves this node's overlay key onto the found tunnel's and
// nothing else — identity, sealing and serving keys stay as they were — and tells the mesh with a
// proof signed by the identity key, over the previous key, the new one and the tunnel.
func anEnrolledNode(t *testing.T) identity.Identity {
t.Helper()
mine, err := identity.Generate("anchor")
if err != nil {
t.Fatal(err)
}
mine.Overlay, err = identity.GenerateOverlayKey()
if err != nil {
t.Fatal(err)
}
mine.Membership = identity.Membership{Broker: "198.51.100.1:5671", Fingerprint: "sha256:aa",
Signer: make([]byte, ed25519.PublicKeySize), Password: "p"}
return mine
}
func aFoundTunnel(t *testing.T) tunnel.Found {
t.Helper()
private, err := identity.GenerateOverlayKey()
if err != nil {
t.Fatal(err)
}
found, err := tunnel.Parse([]byte("[Interface]\nPrivateKey = " + private.Private + "\nListenPort = 51900\n" +
"Address = 192.0.2.1/24\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n"))
if err != nil {
t.Fatal(err)
}
found.Interface, found.Unit, found.Config = "wg0", "wg-quick@wg0", "/etc/wireguard/wg0.conf"
return found
}
func TestTakingATunnelMovesOnlyTheOverlayKeyAndSignsForIt(t *testing.T) {
mine := anEnrolledNode(t)
found := aFoundTunnel(t)
before := mine.Overlay.Public
taken, rekey, err := rekeyOnto(mine, found)
if err != nil {
t.Fatal(err)
}
if taken.Overlay.Public != found.PublicKey || taken.Overlay.Private != found.PrivateKey() {
t.Fatal("the overlay key is not the tunnel's")
}
if string(taken.Public) != string(mine.Public) || string(taken.Private) != string(mine.Private) ||
taken.Node != mine.Node || taken.Membership.Password != mine.Membership.Password ||
taken.Membership.Broker != mine.Membership.Broker {
t.Fatal("something other than the overlay key moved")
}
if taken.OverlayBefore != before {
t.Errorf("the key before the take was not kept: %q", taken.OverlayBefore)
}
if rekey.Previous != before || rekey.OverlayKey != found.PublicKey || rekey.Tunnel == nil ||
rekey.Tunnel.PublicKey != found.PublicKey || len(rekey.Tunnel.Peers) != 1 {
t.Fatalf("the rekey does not say what moved: %+v", rekey)
}
if !ed25519.Verify(ed25519.PublicKey(mine.Public),
link.RekeyProof("anchor", before, found.PublicKey, rekey.Tunnel), rekey.Proof) {
t.Fatal("the rekey is not signed by this node's identity key over what it says")
}
if ed25519.Verify(ed25519.PublicKey(mine.Public),
link.RekeyProof("laptop", before, found.PublicKey, rekey.Tunnel), rekey.Proof) {
t.Fatal("the proof is not bound to the node")
}
for _, said := range []string{rekey.Previous, rekey.OverlayKey, rekey.Tunnel.Interface} {
if strings.Contains(said, found.PrivateKey()) {
t.Fatal("the private key travels")
}
}
// Run again after the take — the mesh not yet told, or told and refused — the previous key it
// names is still the one before the take, so the mesh can tell a repeat from a replay.
again, second, err := rekeyOnto(taken, found)
if err != nil {
t.Fatal(err)
}
if second.Previous != before || again.OverlayBefore != before || again.Overlay.Public != found.PublicKey {
t.Fatalf("a take run again does not name the key before the first: %+v", second)
}
}
internal/link/asking.go
-53
View File
@@ -1,53 +0,0 @@
package link
import (
"context"
"fmt"
"time"
)
// The enrolment conversation, as the host's own words for it.
//
// **Its own seam rather than part of Link**, because almost nothing about it is the same. The
// credential is a one-time secret rather than this node's own; there is no declaration to hear; the
// whole exchange is a single question asked and possibly asked again. And the stakes differ: a node
// that fails here is not in the mesh at all, where a node that fails in Link has merely lost touch
// with one it belongs to.
// Approach is how a node reaches a mesh it does not yet belong to.
//
// The three things a token carries about where to go, and nothing about who is asking: the address,
// the certificate that address must present, and which bus is at the other end — the mesh's own
// (hearing.go), and a token naming any other is refused before anything is sent.
type Approach struct {
Address string
Fingerprint string
Transport string
}
// Asking is one open enrolment conversation.
type Asking interface {
// Ask puts the request to the mesh and waits for one answer, or says why none came.
//
// Called again, with the same bytes, while the mesh says "try again": the keys this node
// generated are the ones it keeps, so the same request is the same enrolment and the mesh holds
// the token for it (novox/hq issue 083).
Ask(ctx context.Context, request []byte, wait time.Duration) ([]byte, error)
// Close lets go of the connection made with the token.
Close()
}
// Present opens an enrolment conversation with the mesh.
//
// The connection is made before anything is sent, and the certificate is checked while it is being
// made — so a node pointed at the wrong bus finds out before its token has left the machine (ADR
// 0004).
func Present(ctx context.Context, to Approach, node, secret string,
timeout time.Duration) (Asking, error) {
if to.Transport != OnNATS {
return nil, fmt.Errorf("this token is for the %q bus, and the mesh's bus is %s", to.Transport, OnNATS)
}
return presentNats(ctx, to, node, secret, timeout)
}
internal/link/asking_nats.go
-186
View File
@@ -1,186 +0,0 @@
package link
import (
"context"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"time"
"github.com/nats-io/nats.go"
)
// The enrolment conversation on the bus being built.
//
// **The reply address is the whole of what changes**, and it changes for a reason the transport
// forces rather than a preference. Core NATS request/reply puts the caller's inbox in the message's
// reply field and a plain responder answers it — but this request goes into a stream, and a message
// a JetStream consumer delivers has had that field claimed for the consumer's own ack address. So by
// the time the controller reads the request, the transport's reply field names where the
// *controller* must acknowledge. Verified against a running server (design 25 §2).
//
// The address therefore travels as a field of the request, and this subscribes it before publishing:
// a node that published first could miss an answer to a question nobody was listening for.
// enrolInbox is where a node enrolling waits.
//
// Under `_INBOX.enrol.<node>.`, which is exactly what its enrolment user may subscribe and no
// wider — so an answer sealed to one machine cannot be read by another enrolling beside it. The
// random tail is this attempt's own: a reply left over from an attempt that timed out is not the
// answer to this question, which is what the correlation id does on the other transport.
func enrolInbox(node string) (string, error) {
tail := make([]byte, 8)
if _, err := rand.Read(tail); err != nil {
return "", fmt.Errorf("cannot make a reply address: %w", err)
}
return "_INBOX.enrol." + node + "." + hex.EncodeToString(tail), nil
}
type natsAsking struct {
conn *nats.Conn
js nats.JetStreamContext
inbox string
answers *nats.Subscription
lost chan error
}
func presentNats(_ context.Context, to Approach, node, secret string,
timeout time.Duration) (Asking, error) {
config, err := PinnedConfig(to.Fingerprint)
if err != nil {
return nil, err
}
inbox, err := enrolInbox(node)
if err != nil {
return nil, err
}
lost := make(chan error, 1)
// The user is this token's own — `enrol.<node>`, which may publish the enrolment subject and
// subscribe its own inbox and nothing else (design 25 §6). The secret is its password, the same
// string the request claims, so the server proves somebody holds the token and the request
// proves the same thing to the controller without it having to ask the server who connected.
// **Its own inbox space, because that is the only one it may listen in** (novox/hq
// 04-ISSUES/146). A JetStream publish waits for the stream's acknowledgement on an inbox the
// client picks, and the client's default is `_INBOX.<random>` — which this user may not
// subscribe to, so the enrolment failed with a permissions violation on a subject nobody had
// chosen. The permission is `_INBOX.enrol.<node>.>` (design 25 §6), so the client is told to
// pick its inboxes there; the reply address below is in the same space for the same reason.
conn, err := nats.Connect(natsURL(to.Address),
nats.Secure(config),
nats.CustomInboxPrefix("_INBOX.enrol."+node),
nats.UserInfo("enrol."+node, secret),
nats.Name("mesh-host/enrol/"+node),
nats.Timeout(timeout),
nats.NoReconnect(),
nats.DisconnectErrHandler(func(_ *nats.Conn, err error) {
select {
case lost <- fmt.Errorf("the bus closed the connection: %w", err):
default:
}
}),
)
if err != nil {
if errors.Is(err, ErrWrongCertificate) {
return nil, err
}
// Not quoted back with the credential: the secret is one-time and still a secret.
return nil, fmt.Errorf("cannot reach the bus at %s as %s: %w", to.Address, node, err)
}
js, err := conn.JetStream()
if err != nil {
conn.Close()
return nil, fmt.Errorf("the bus at %s has no JetStream: %w", to.Address, err)
}
// Subscribed before anything is published, so an answer cannot arrive before there is anywhere
// for it to land.
answers, err := conn.SubscribeSync(inbox)
if err != nil {
conn.Close()
return nil, fmt.Errorf("this node cannot listen for the mesh's answer: %w", err)
}
if err := conn.Flush(); err != nil {
conn.Close()
return nil, fmt.Errorf("this node's reply address did not reach the bus: %w", err)
}
return &natsAsking{conn: conn, js: js, inbox: inbox, answers: answers, lost: lost}, nil
}
func (a *natsAsking) Close() {
if a.answers != nil {
_ = a.answers.Unsubscribe()
}
if a.conn != nil {
a.conn.Close()
}
}
// Ask publishes the request with this attempt's reply address written into it, and waits there.
func (a *natsAsking) Ask(ctx context.Context, request []byte, wait time.Duration) ([]byte, error) {
addressed, err := withReplyTo(request, a.inbox)
if err != nil {
return nil, err
}
publish, cancel := context.WithTimeout(ctx, wait)
defer cancel()
// Into the stream and awaited: an enrolment the bus never accepted must fail here rather than be
// assumed, because the node has nothing else to go on.
//
// **Once, however many times it is sent** (novox/hq 04-ISSUES/146). The client re-publishes when
// an acknowledgement is slow, and the mesh enrolled the machine on each copy — minting a second
// credential, which replaced the first, which is the one the node had already been given. The
// machine then reconnected for ever as a user whose password the mesh had rotated out from under
// it, and the controller's log said "enrolled anchor" twice in the same second.
//
// The id is the message: the same bytes carry the same id, so the stream discards the client's
// own retry, and a genuine second attempt — which carries a new reply address — is a different
// message and is let through.
sum := sha256.Sum256(addressed)
if _, err := a.js.Publish(EnrolSubject, addressed,
nats.MsgId(hex.EncodeToString(sum[:])), nats.Context(publish)); err != nil {
return nil, fmt.Errorf("cannot ask the mesh to enrol this node: %w", err)
}
// Waited for rather than assumed. A published message that nothing answers means the controller
// is not running, and a node that carried on regardless would believe it had joined a mesh that
// has never heard of it.
//
// **The wait may legitimately be several store-window cycles long**: the controller naks the
// request with a delay while its store is restarting, and the node is waiting on the other side
// of that — which is exactly the combination that would have delivered the answer to a caller
// who had given up, had the address travelled in the transport's field.
answered, cancelAnswer := context.WithTimeout(ctx, wait)
defer cancelAnswer()
for {
msg, err := a.answers.NextMsgWithContext(answered)
switch {
case err == nil:
return msg.Data, nil
case errors.Is(err, context.DeadlineExceeded):
select {
case reason := <-a.lost:
return nil, reason
default:
}
return nil, fmt.Errorf(
"the bus accepted this node's connection and nothing answered within %s. The mesh's "+
"bus is running and its controller is not", wait)
case errors.Is(err, context.Canceled):
return nil, ctx.Err()
default:
select {
case reason := <-a.lost:
return nil, reason
default:
}
return nil, fmt.Errorf("waiting for the mesh's answer: %w", err)
}
}
}
internal/link/asking_nats_test.go
-192
View File
@@ -1,192 +0,0 @@
package link
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
)
// The enrolment round trip against a real server.
//
// **This is the test that keeps a reason from becoming folklore.** The reply address travels in the
// request's payload because a JetStream consumer's delivery has had the transport's reply field
// claimed for its own ack address — which is a fact about a server, not a rule anybody can check by
// reading. Both halves are asserted here: that the field really is eaten, and that the answer
// reaches the node anyway.
//
// docker run -d --rm --name t -p 14223:4222 nats:2.10-alpine -js
// MESH_TEST_NATS=nats://127.0.0.1:14223 go test ./internal/link/ -run TestNatsAnEnrolment
// asking is a conversation on a bus with no TLS. Built directly rather than through Present because
// the pin is what Present adds and PinnedConfig's own tests cover it; what is under test here is the
// address the answer comes back on.
func asking(t *testing.T, conn *nats.Conn, js nats.JetStreamContext, node string) *natsAsking {
t.Helper()
inbox, err := enrolInbox(node)
if err != nil {
t.Fatal(err)
}
answers, err := conn.SubscribeSync(inbox)
if err != nil {
t.Fatal(err)
}
if err := conn.Flush(); err != nil {
t.Fatal(err)
}
a := &natsAsking{conn: conn, js: js, inbox: inbox, answers: answers, lost: make(chan error, 1)}
// **Not a.Close.** Close closes the connection, and this connection is the test's own, shared with
// the stand-in. Cleanups run last-in first-out and the stand-in was made first, so its cleanup
// would run on a closed connection: Unsubscribe returns ErrConnectionClosed without deleting the
// consumer it made, and every request after that fails. The consumer then outlives the test on a
// workqueue stream, which allows one consumer per filter, and the next test cannot subscribe
// (issue 604). The connection is closed by aBus, last of all.
t.Cleanup(func() { _ = answers.Unsubscribe() })
return a
}
// theMeshAnswers stands in for the controller: it consumes the enrolment off the stream, reads the
// reply address out of the payload — never from the transport field — and answers there. It reports
// what the transport field actually held, which is the claim design 25 §2 rests on.
func theMeshAnswers(t *testing.T, conn *nats.Conn, js nats.JetStreamContext,
reply EnrolReply) <-chan string {
t.Helper()
sawReplyField := make(chan string, 1)
// **A consumer of this test's own, gone before it starts and gone when it ends.** The stand-in
// used to be one durable name shared by every test, released by Unsubscribe alone. Unsubscribe
// only asks the server to let go: it deletes the consumer only when this subscription made it,
// and the server drops the binding on its own time. A test that started in that window (or one
// that found the consumer left by an earlier run on a long-lived bus) was refused with "consumer
// is already bound to a subscription". Per-test names remove the sharing; the delete-and-wait
// before and after removes the dependence on the server's timing.
durable := standinName(t)
stream, err := js.StreamNameBySubject(EnrolSubject)
if err != nil {
t.Fatal(err)
}
standinGone(t, js, stream, durable)
sub, err := js.Subscribe(EnrolSubject, func(msg *nats.Msg) {
select {
case sawReplyField <- msg.Reply:
default:
}
var addressed struct {
ReplyTo string `json:"reply_to"`
}
if err := json.Unmarshal(msg.Data, &addressed); err != nil || addressed.ReplyTo == "" {
_ = msg.Ack()
return
}
body, _ := json.Marshal(reply)
// Published explicitly to the address the payload named, never msg.Respond — which would
// send it to whatever the transport's reply field holds, and that is the point.
_ = conn.Publish(addressed.ReplyTo, body)
_ = msg.Ack()
}, nats.Durable(durable), nats.ManualAck())
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_ = sub.Unsubscribe()
standinGone(t, js, stream, durable)
})
return sawReplyField
}
// standinName is the stand-in's durable name for one test: the test's name with every character a
// consumer name may not hold replaced.
func standinName(t *testing.T) string {
t.Helper()
return "controller-standin-" + strings.Map(func(r rune) rune {
if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' {
return r
}
return '_'
}, t.Name())
}
// standinGone deletes the durable consumer if it exists and waits until the server no longer reports
// it, so that nothing after this call can meet its binding.
func standinGone(t *testing.T, js nats.JetStreamContext, stream, durable string) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for {
_, err := js.ConsumerInfo(stream, durable)
if errors.Is(err, nats.ErrConsumerNotFound) {
return
}
if errors.Is(err, nats.ErrConnectionClosed) {
t.Fatalf("the connection closed before consumer %s was deleted: %v", durable, err)
}
_ = js.DeleteConsumer(stream, durable)
if time.Now().After(deadline) {
t.Fatalf("consumer %s on %s still exists 5s after it was deleted", durable, stream)
}
time.Sleep(20 * time.Millisecond)
}
}
// An enrolment is answered on the address the request carried, and the transport's own reply field
// held something else entirely.
func TestNatsAnEnrolmentIsAnsweredOnTheAddressInItsPayload(t *testing.T) {
conn, js := aBus(t)
const node = "joining"
sawReplyField := theMeshAnswers(t, conn, js, EnrolReply{Accepted: true, Node: node, Password: "p"})
a := asking(t, conn, js, node)
request, _ := json.Marshal(EnrolRequest{Node: node, Secret: "t"})
answer, err := a.Ask(context.Background(), request, 8*time.Second)
if err != nil {
t.Fatalf("no answer reached the node: %v", err)
}
var reply EnrolReply
if err := json.Unmarshal(answer, &reply); err != nil {
t.Fatal(err)
}
if !reply.Accepted || reply.Node != node {
t.Fatalf("the answer was not the mesh's: %+v", reply)
}
// And the field the answer would have gone to, had it used the transport's: the consumer's own
// ack address. If a future server stopped doing this, the payload-borne address would still
// work and this line is what would say the reason had changed.
select {
case field := <-sawReplyField:
if field == a.inbox {
t.Fatalf("the transport's reply field held this node's inbox (%s), so the payload "+
"address is no longer load-bearing — check design 25 §2 before relying on it", field)
}
if field == "" {
t.Fatal("the transport's reply field was empty rather than claimed, which is a third " +
"behaviour from the two design 25 §2 describes")
}
case <-time.After(2 * time.Second):
t.Fatal("the stand-in never saw the request")
}
}
// A request that names no reply address is not answered, and the node says so as a mesh that is not
// running rather than hanging. The controller has nowhere to send an answer, which is the failure
// the payload field exists to make impossible — asserted so that a request built without it fails
// loudly here rather than quietly on a machine.
func TestNatsAnEnrolmentWithNoReplyAddressIsNotAnswered(t *testing.T) {
conn, js := aBus(t)
const node = "silent"
theMeshAnswers(t, conn, js, EnrolReply{Accepted: true, Node: node})
a := asking(t, conn, js, node)
// Published without going through Ask, so the reply address is genuinely absent.
request, _ := json.Marshal(EnrolRequest{Node: node, Secret: "t"})
if _, err := js.Publish(EnrolSubject, request); err != nil {
t.Fatal(err)
}
if _, err := a.Ask(context.Background(), []byte(`{"node":"`+node+`"}`), 0); err == nil {
t.Fatal("a node with no answer coming was told it had one")
}
}
internal/link/bus.go
+5 -35
View File
@@ -10,28 +10,14 @@ import (
"github.com/nats-io/nats.go"
)
// Bus is what a host needs of the mesh's bus, in the mesh's own words.
//
// A host says exactly two things unprompted: what it applied, and that it is here. They are not
// the same kind of statement and the difference is the whole of this interface — one must arrive
// and one must not be insisted on.
//
// **The host still imports nothing of the mesh's own** (novox/hq ADR 0005): this is its own
// interface over its own client libraries, not a contract shared with the controller. The two
// agree because a conformance fixture holds them to one envelope, which is the only kind of
// agreement that survives being in different repositories.
type Bus interface {
// Report says what this node applied. **It must arrive.** A report that fails leaves the
// mesh believing the node never answered while the node believes it did, and the two go on
// disagreeing with nothing anywhere saying so — the shape of fault this project keeps
// finding. Returns false when it could not be delivered, so the caller can say so.
Report(ctx context.Context, node string, body []byte) error
}
// Bus is the link open now as the session holds it. What it can do is asked of it by what it implements (HaveBus,
// HealthBus, ToolBus, FilesBus): the report it once published into the controller's CONTROL stream went with the
// controller (issue 1126), and what the node applied is said in its haves.
type Bus interface{}
// --- The bus the mesh runs on today -----------------------------------------------------------
// OverNATS is the bus as a connection. A report goes through JetStream because it must survive
// the controller's store restarting; a heartbeat does not, because it must not.
// OverNATS is the bus as a connection.
type OverNATS struct {
Conn *nats.Conn
JS nats.JetStreamContext
@@ -39,11 +25,6 @@ type OverNATS struct {
Node string
}
// ReportSubject is this node's own, and no other node's: a host's account may
// publish `mesh.control.<its own node>.>` and nothing wider, so the subject is the authority on
// which node a report is about.
func ReportSubject(node string) string { return "mesh.control." + node + ".report" }
// HealthSubject is where this node says its long-running resources' health between reports (novox/hq
// ADR 0240): its own, inside the grant every host already has, and on core NATS like the heartbeat — a
// statement lost is said again within a minute while anything is not healthy.
@@ -90,17 +71,6 @@ func (b OverNATS) Ask(ctx context.Context, subject string, body []byte) ([]byte,
return msg.Data, nil
}
func (b OverNATS) Report(ctx context.Context, node string, body []byte) error {
// Into the CONTROL stream and awaited: this is the message the store-window guarantee is
// about (novox/hq ADR 0083). The controller naks with a delay while its store is away and
// the message is redelivered; a publish the bus never accepted must fail here rather than
// be assumed.
if _, err := b.JS.Publish(ReportSubject(node), body, nats.Context(ctx)); err != nil {
return fmt.Errorf("reporting: %w", err)
}
return nil
}
// HaveBind binds the have bucket the controller made (novox/hq ADR 0429 §4, issue 1132), once for a pass of writes.
// **Only binds**: a node-engine's account reaches no part of the JetStream API beyond what it is granted, and the bucket
// is the controller's to create and to configure, so a bucket that is not there is an error said as what it is. Every
internal/link/enrol.go
+10 -180
View File
@@ -1,70 +1,21 @@
package link
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"time"
)
// The wire format shared with the control plane, which defines it separately because this binary
// requires nothing present and does not import it. A test on each side asserts the field names.
const (
Exchange = "mesh"
KeyEnrol = "enrol"
)
// What a joining machine keeps of the enrolment's words: the tunnel it found, as it travels, and how long it asks
// while the seat mesh-enrolment says to ask again (Redeem). The enrolment through the controller's subject on the
// CONTROL stream ended with the controller (issue 1126).
// QueueFor is the queue this node consumes from — the only one its account may read.
func QueueFor(node string) string { return "node." + node }
// EnrolRequest is what this node says when joining.
type EnrolRequest struct {
Node string `json:"node"`
Secret string `json:"secret"`
PublicKey []byte `json:"public_key"`
// OverlayKey is the public half of this node's key on the private network — a different key
// from PublicKey above, generated at the same moment and for a different purpose.
//
// Sent with enrolment because the overlay is the first declaration a node receives, and the
// mesh cannot compose it without this. Asking for it afterwards would mean a node is enrolled
// and unreachable for a round trip, which is the state everything else here works to avoid.
OverlayKey string `json:"overlay_key,omitempty"`
// SealingKey is the public half of the key secrets are sealed to. A third key, and the
// reasoning is the same one twice over: the mesh must be able to send this node something
// nothing else can read, and it must never be able to read it either.
SealingKey string `json:"sealing_key,omitempty"`
// ServingKey is the public half of the key this node serves TLS with on its internal name.
// The mesh signs a certificate binding it; the private half never leaves the machine, so
// there is nothing to seal and nothing that could be stolen from the mesh's copy.
ServingKey string `json:"serving_key,omitempty"`
Profile map[string]any `json:"profile,omitempty"`
// Proof is this node's identity key signing EnrolProof over this request: that the presenter
// holds the private half of PublicKey. The mesh asks for it before letting an enrolment finish
// on a token this key already spent (novox/hq issue 083).
Proof []byte `json:"proof,omitempty"`
// ReplyTo is where the mesh's answer goes, as a field of the request rather than the
// transport's own reply address (design 25 §2). Written by the transport that needs it —
// withReplyTo, once per attempt — because a request going into a stream has had the transport's
// reply field claimed for the consumer's ack address before the controller ever reads it.
//
// Empty on the bus the mesh runs on today, where the delivery carries the reply queue and the
// field means what it has always meant. Named here so both sides of the wire hold the same
// field name, which is what the shape test on each side is for.
ReplyTo string `json:"reply_to,omitempty"`
// Tunnel is the tunnel this node found and whose key it took as its overlay key (novox/hq ADR
// 0105): everything about it but that key. Sent with the keys because it is one of them —
// OverlayKey above IS this tunnel's public key when this is set — and the mesh composes the
// hub's address, the range and the carried peers from it before the first declaration.
Tunnel *Tunnel `json:"tunnel,omitempty"`
// Approach is how a node reaches a mesh it does not yet belong to: the address, the certificate that address must
// present, and which bus is at the other end, the mesh's own (hearing.go); a token naming any other is refused before
// anything is sent.
type Approach struct {
Address string
Fingerprint string
Transport string
}
// Tunnel is a found tunnel as it travels: no private key.
@@ -86,36 +37,6 @@ type TunnelPeer struct {
Address string `json:"address"`
}
// EnrolReply is what the mesh says back.
type EnrolReply struct {
Accepted bool `json:"accepted"`
// TryAgain is the mesh saying it cannot answer right now — its store is restarting, or the
// token is held for a moment by another enrolment — and that nothing was spent. The same
// request is asked again (novox/hq issue 083).
TryAgain bool `json:"try_again,omitempty"`
Node string `json:"node,omitempty"`
Queue string `json:"queue,omitempty"`
// What this node keeps so it can come back on its own. Without these a restart would need a
// person with a new token, which would make disconnection a crisis rather than the ordinary
// situation novox/hq ADR 0004 says it is.
Password string `json:"password,omitempty"`
Broker string `json:"broker,omitempty"`
Fingerprint string `json:"fingerprint,omitempty"`
Signer []byte `json:"signer,omitempty"`
Refusal string `json:"refusal,omitempty"`
}
// EnrolProof is what a node signs with its identity key when it enrols: the token and every key it
// presents, so a proof cannot be moved to another request. The mesh builds the same bytes.
func EnrolProof(secret string, public []byte, overlay, sealing, serving string) []byte {
return []byte("novox-mesh-enrol\x00" + secret + "\x00" + base64.StdEncoding.EncodeToString(public) +
"\x00" + overlay + "\x00" + sealing + "\x00" + serving)
}
// ErrRefused is what a node gets when the mesh will not have it.
var ErrRefused = errors.New("the mesh refused this enrolment")
@@ -126,94 +47,3 @@ const EnrolPatience = 2 * time.Minute
// AskAgainAfter is the pause between asks while the mesh says "try again".
const AskAgainAfter = 3 * time.Second
// ErrNotNow is a mesh that said "try again" for longer than this node would keep asking.
var ErrNotNow = errors.New("the mesh could not answer this enrolment")
// answered decides what one reply means: done, ask again, or stop with an error. Separate from the
// broker so it can be held to that by a test.
func answered(reply EnrolReply, asking time.Duration) (again bool, err error) {
switch {
case reply.Accepted:
return false, nil
case reply.TryAgain && asking < EnrolPatience:
return true, nil
case reply.TryAgain:
// Said with what to do. The mesh holds the token for this attempt's keys for as long as
// this node kept asking, so a new attempt — with keys of its own — waits that out first.
return false, fmt.Errorf("%w for %s: %s. The token was not spent: wait about %s and run "+
"enrol again with it; if it is then refused, issue a new one",
ErrNotNow, EnrolPatience, reply.Refusal, EnrolPatience)
default:
return false, fmt.Errorf("%w: %s", ErrRefused, reply.Refusal)
}
}
// Enrol presents this node's key and its one-time secret, and waits to be told it is known.
//
// The broker has already authenticated this connection: the account was created when the token
// was issued and the secret is its password. So this is not how the node gets in — it is what it
// says once it is in, and the secret travels again because the control plane must not have to ask
// the broker who connected.
func Enrol(ctx context.Context, to Approach, node, secret string, public []byte,
overlayKey, sealingKey, servingKey string, profile map[string]any, proof []byte,
tunnel *Tunnel, timeout time.Duration) (EnrolReply, error) {
asking, err := Present(ctx, to, node, secret, timeout)
if err != nil {
return EnrolReply{}, err
}
defer asking.Close()
request := EnrolRequest{Node: node, Secret: secret, PublicKey: public,
OverlayKey: overlayKey, SealingKey: sealingKey, ServingKey: servingKey, Profile: profile,
Proof: proof, Tunnel: tunnel}
body, err := json.Marshal(request)
if err != nil {
return EnrolReply{}, err
}
// Asked, and asked again with the same request while the mesh says "try again": the keys this
// node generated are the ones it keeps, so the same request is the same enrolment, and the mesh
// holds the token for it (novox/hq issue 083).
began := time.Now()
for {
answer, err := asking.Ask(ctx, body, timeout)
if err != nil {
return EnrolReply{}, err
}
var reply EnrolReply
if err := json.Unmarshal(answer, &reply); err != nil {
return EnrolReply{}, fmt.Errorf("the mesh's answer could not be read: %w", err)
}
again, err := answered(reply, time.Since(began))
if err != nil {
return reply, err
}
if !again {
return reply, nil
}
select {
case <-ctx.Done():
return EnrolReply{}, ctx.Err()
case <-time.After(AskAgainAfter):
}
}
}
// withReplyTo writes this attempt's reply address into the request, as a field of its own.
//
// **Written into the bytes rather than carried beside them**, because the whole point is that the
// address survives a stream: a JetStream consumer's delivery has had the transport's reply field
// claimed for its own ack address, so a reply address that is not in the payload is one the
// controller cannot read (design 25 §2). Done by decoding and re-encoding rather than by setting the
// field before marshalling, so one request can be asked again with a fresh address each time without
// the caller knowing that is what happens.
func withReplyTo(request []byte, inbox string) ([]byte, error) {
var fields map[string]any
if err := json.Unmarshal(request, &fields); err != nil {
return nil, fmt.Errorf("this node's own enrolment request cannot be read back: %w", err)
}
fields["reply_to"] = inbox
return json.Marshal(fields)
}
internal/link/enrol_again_test.go
-34
View File
@@ -1,34 +0,0 @@
package link
import (
"errors"
"strings"
"testing"
"time"
)
// What one reply means to a node enrolling (novox/hq issue 083): accepted is done; "try again" is
// asked again, until the node's patience runs out and it says the token was not spent; anything
// else is a refusal.
func TestAnEnrollingNodeAsksAgainWhileTheMeshSaysNotNow(t *testing.T) {
if again, err := answered(EnrolReply{Accepted: true}, 0); again || err != nil {
t.Fatalf("an accepted enrolment was not done: again=%v err=%v", again, err)
}
if again, err := answered(EnrolReply{TryAgain: true}, time.Second); !again || err != nil {
t.Fatalf("a mesh saying not now was not asked again: again=%v err=%v", again, err)
}
again, err := answered(EnrolReply{TryAgain: true, Refusal: "restarting"}, EnrolPatience)
if again || !errors.Is(err, ErrNotNow) {
t.Fatalf("a mesh saying not now past the node's patience did not stop it: again=%v err=%v", again, err)
}
if !errorsMention(err, "not spent") {
t.Errorf("giving up did not say the token can be used again: %v", err)
}
if again, err := answered(EnrolReply{Refusal: "that token cannot be used"}, 0); again || !errors.Is(err, ErrRefused) {
t.Fatalf("a refusal was not a refusal: again=%v err=%v", again, err)
}
}
func errorsMention(err error, what string) bool {
return err != nil && strings.Contains(err.Error(), what)
}
internal/link/enrol_proof_test.go
-13
View File
@@ -1,13 +0,0 @@
package link
import "testing"
// The bytes a node signs when it enrols. The mesh builds the same bytes to check the signature, in
// another repository; this known answer is repeated in its test, so the two cannot drift apart
// without one of them failing (novox/hq issue 083).
func TestWhatAnEnrollingNodeSignsIsFixed(t *testing.T) {
got := string(EnrolProof("s", []byte{1, 2, 3}, "o", "e", "v"))
if want := "novox-mesh-enrol\x00s\x00AQID\x00o\x00e\x00v"; got != want {
t.Fatalf("the enrolment proof's message changed: %q, want %q", got, want)
}
}
internal/link/enrol_seat.go
+4 -4
View File
@@ -33,8 +33,8 @@ func CalledSeatSubject(seat, verb, user string) string {
return "mesh.seat." + seat + ".call." + verb + "." + strings.ReplaceAll(user, ".", "~")
}
// ErrNoSeat is the bus saying nothing serves the seat for this caller: the enrolment user may not publish the seat's call (the
// controller still holds its lease) or no holder answers it.
// ErrNoSeat is the bus saying nothing serves the seat for this caller: the enrolment user may not publish the seat's call, or
// no holder answers it.
var ErrNoSeat = errors.New("the seat mesh-enrolment is not served to this machine's token")
// VerifyRedeemAnswer holds an answer to what it must be before anything is kept (ADR 0587 §6): it names the node asked for,
@@ -80,8 +80,8 @@ func seatAnswer(reply []byte) (json.RawMessage, error) {
}
// Redeem calls the seat's redeem as enrol.<node>, with the token's secret as the bus password, on the pinned certificate. It
// asks again with the same keys while the holder says to (a stopped redeem finishes; the bus node loads the account), for
// as long as a machine asking the controller would (EnrolPatience). An answer is returned as the holder gave it: the caller
// asks again with the same keys while the holder says to (a stopped redeem finishes; the bus node loads the account), , for
// EnrolPatience. An answer is returned as the holder gave it: the caller
// verifies it (VerifyRedeemAnswer) before it keeps anything.
func Redeem(ctx context.Context, to Approach, node, secret string, req delivery.Redeem, timeout time.Duration) (delivery.RedeemAnswer, error) {
if to.Transport != OnNATS {
internal/link/enrol_shape_test.go
-67
View File
@@ -1,67 +0,0 @@
package link
import (
"encoding/json"
"os"
"testing"
"github.com/novox/mesh-host/internal/identity"
)
// What this node says when it joins, written out so the mesh's own suite can accept it.
//
// The two ends are separate structs in separate repositories. Every field here is one somebody
// could rename on one side, and the failure would be silent: enrolment succeeds, a key is simply
// absent, and the node looks joined until the first thing sealed to it cannot be opened. That is
// exactly the shape of fault this project keeps finding late.
//
// Skipped unless MESH_ENROL_OUT names a file, so this is a check somebody runs deliberately
// rather than a dependency between two repositories.
func TestWhatThisNodeSaysWhenItJoins(t *testing.T) {
path := os.Getenv("MESH_ENROL_OUT")
if path == "" {
t.Skip("set MESH_ENROL_OUT to write the enrolment request the mesh's suite reads")
}
// A real one. Generated the way enrolment generates them rather than typed as literals, so a
// key that stopped being a key would be caught here rather than travelling.
mine, err := identity.Generate("workstation")
if err != nil {
t.Fatal(err)
}
overlay, err := identity.GenerateOverlayKey()
if err != nil {
t.Fatal(err)
}
sealing, err := identity.GenerateSealingKey()
if err != nil {
t.Fatal(err)
}
serving, err := identity.GenerateServingKey()
if err != nil {
t.Fatal(err)
}
request := EnrolRequest{
Node: "workstation",
Secret: "a-one-time-secret",
PublicKey: mine.Public,
OverlayKey: overlay.Public,
SealingKey: sealing.Public,
ServingKey: serving.Public,
Profile: map[string]any{"seat": true},
}
body, err := json.MarshalIndent(request, "", " ")
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, append(body, '\n'), 0o644); err != nil {
t.Fatal(err)
}
// The private half of the sealing key goes beside it, so the mesh's suite can prove what it
// sealed is openable rather than merely present.
if err := os.WriteFile(path+".sealing-private", []byte(sealing.Private), 0o600); err != nil {
t.Fatal(err)
}
t.Logf("wrote %s", path)
}
internal/link/have_test.go
+6 -27
View File
@@ -24,7 +24,7 @@ type haveLink struct {
hmu sync.Mutex
kv map[string]haveEntry
sets []string // keys written, in order
events []string // "report" and "have <key>", in the order they happened
events []string // "have <key>", in the order they happened
failing bool
binds int
setTries int
@@ -40,16 +40,6 @@ type haveEntry struct {
func newHaveLink() *haveLink { return &haveLink{quietLink: newQuietLink(), kv: map[string]haveEntry{}} }
func (l *haveLink) Report(ctx context.Context, node string, body []byte) error {
err := l.quietLink.Report(ctx, node, body)
if err == nil {
l.hmu.Lock()
l.events = append(l.events, "report")
l.hmu.Unlock()
}
return err
}
func (l *haveLink) Health(context.Context, string, []byte) error { return nil }
func (l *haveLink) HaveBind(ctx context.Context) (HaveBucket, error) {
@@ -241,9 +231,8 @@ func TestAModulesHealthFollowsTheLook(t *testing.T) {
}
}
// 1132/AC6: the report is sent first and carries no new key; the node's key holds its windows, busy, strays, filters
// and profile.
func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
// 1132/AC6: the node's key holds its windows, busy, strays, filters and profile.
func TestTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
l := newHaveLink()
q, _ := aHaveQueue(l)
rep := appliedModules()
@@ -253,13 +242,6 @@ func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
rep.Filters = []Filter{{Where: "inet mesh", Owner: "mesh"}}
rep.Profile = map[string]any{"cores": 8}
q.SayFacts(context.Background(), rep)
if len(l.events) < 2 || l.events[0] != "report" {
t.Fatalf("the report was not first: %v", l.events)
}
raw, _ := json.Marshal(l.said()[0])
if strings.Contains(string(raw), "Modules") || strings.Contains(string(raw), "\"modules\"") {
t.Errorf("the report carries the module facts: %s", raw)
}
got, err := delivery.DecodeHaveNode(l.kv["n1.node"].value)
if err != nil {
t.Fatal(err)
@@ -310,16 +292,13 @@ func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
}
}
// 1132/AC7: a write the bucket refuses is said in the log; the report went out, and the next apply writes again.
func TestAFailedHaveWriteIsSaidAndNeverBlocksTheReport(t *testing.T) {
// 1132/AC7: a write the bucket refuses is said in the log, and the next apply writes again.
func TestAFailedHaveWriteIsSaidAndWrittenAgain(t *testing.T) {
l := newHaveLink()
l.failing = true
q, said := aHaveQueue(l)
if !q.SayFacts(context.Background(), appliedModules()) {
t.Fatal("the report did not go out")
}
if len(l.said()) != 1 {
t.Fatalf("reports sent: %d", len(l.said()))
t.Fatal("the facts were not said with a link open")
}
if !strings.Contains(strings.Join(*said, "\n"), "n1.a") || !strings.Contains(strings.Join(*said, "\n"), "refused") {
t.Errorf("the failure is not in the log: %v", *said)
internal/link/hearing_nats.go
-9
View File
@@ -17,11 +17,6 @@ import (
// The host's link on the bus: one connection, and what is asked of this node over it. The node hears no declaration
// (novox/hq ADR 0539): it watches its wants and its compositions, and says its facts.
// EnrolSubject is where a joining machine asks. One subject for every node, because a machine
// enrolling has no name the mesh has agreed to yet — which is why its authority to publish here is
// the whole of what its enrolment user may do.
const EnrolSubject = "mesh.control.enrol"
// AskReportSubject is where the mesh asks this node to say its facts again: its own, on core NATS and off any stream.
// The answer is the facts event on its report subject — nobody's inbox is answered.
func AskReportSubject(node string) string { return "mesh.node." + node + ".ask.report" }
@@ -311,10 +306,6 @@ func (l *natsLink) Close() {
}
}
func (l *natsLink) Report(ctx context.Context, node string, body []byte) error {
return OverNATS{Conn: l.conn, JS: l.js}.Report(ctx, node, body)
}
func (l *natsLink) Health(ctx context.Context, node string, body []byte) error {
return OverNATS{Conn: l.conn, JS: l.js}.Health(ctx, node, body)
}
internal/link/messages.go
-63
View File
@@ -1,8 +1,6 @@
package link
import (
"strconv"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -47,13 +45,6 @@ type Report struct {
// machine saying what is true of it, not asking for anything.
Carried []int `json:"carried,omitempty"`
// Writer, and Order — the node-engine's own epoch and sequence — say who wrote these facts and where they stand
// among everything it wrote (novox/hq ADR 0227 §2, ADR 0539 §9): the epoch is one per start of the node-engine,
// the sequence one higher for every facts event, so the controller refuses older facts by this order. On the
// wire as three top-level keys.
Writer string `json:"writer,omitempty"`
Order
// Held is what this adopted node found and is keeping as it was until its module is taken
// (novox/hq ADR 0100). Without it an adopted node reads as converged.
Held []Held `json:"held,omitempty"`
@@ -142,10 +133,6 @@ type Report struct {
// that does not load is a machine filtering nothing while its unit reports success.
Outward []string `json:"outward,omitempty"`
// Rekey is this node taking a found tunnel's key as its overlay key after enrolment (novox/hq
// ADR 0105). Not an account of the machine: a report carrying one says nothing else.
Rekey *Rekey `json:"rekey,omitempty"`
// Rollbacks is what this machine's witnesses decided about a core build that was not healthy
// in bound (novox/hq to-be 45 §8, ADR 0227 rule 8): the node-engine's launcher about the
// engine, the engine about the controller and the node tools. **Said on every report while it
@@ -530,26 +517,6 @@ type Rollback struct {
At time.Time `json:"at"`
}
// Order is where a writer's saying stands among everything it said (novox/hq ADR 0227 §2): its **epoch**, and its
// **sequence** within it. A node-engine's epoch is the time it started, so a later start orders after an earlier one
// with nothing kept on disk.
type Order struct {
Epoch int64 `json:"epoch,omitempty"`
Sequence int64 `json:"sequence,omitempty"`
}
// Words is an order as a person reads it in a log line. Not String: Report embeds Order, and a
// Stringer promoted onto every report would print each one as its order alone.
func (o Order) Words() string {
switch {
case o.Epoch > 0:
return "epoch " + strconv.FormatInt(o.Epoch, 10) + ", sequence " + strconv.FormatInt(o.Sequence, 10)
case o.Sequence > 0:
return "sequence " + strconv.FormatInt(o.Sequence, 10) + ", no epoch"
}
return "no order"
}
// CarriedTunnel is this node's account of the tunnel it took over. State is one of the Carried
// states below; Note is what the host did about it, when it did something.
type CarriedTunnel struct {
@@ -572,36 +539,6 @@ const (
CarriedDown = "down"
)
// Rekey is this node saying it took a found tunnel's key as its overlay key after enrolling
// (novox/hq ADR 0105): the path for a node that enrolled before the mesh knew to take a tunnel
// over, since re-enrolling would rotate every key it holds. Signed with the identity key over
// RekeyProof, so a report forged on a stolen broker account cannot move this node's overlay key.
type Rekey struct {
// Previous is the overlay key this node held until now, as the mesh records it; the mesh
// refuses a rekey naming another, which is how a replayed one is refused.
Previous string `json:"previous"`
OverlayKey string `json:"overlay_key"`
Tunnel *Tunnel `json:"tunnel"`
Proof []byte `json:"proof"`
}
// RekeyProof is what a node signs when it rekeys — the node, the key it leaves, the key it takes
// and the tunnel it took it from — so a proof cannot be moved to another node or another tunnel.
// Byte for byte the mesh's own (mesh-controller internal/link RekeyProof).
func RekeyProof(node, previous, key string, tunnel *Tunnel) []byte {
var t Tunnel
if tunnel != nil {
t = *tunnel
}
peers := make([]string, 0, len(t.Peers))
for _, p := range t.Peers {
peers = append(peers, p.PublicKey+"@"+p.Address)
}
return []byte("novox-mesh-rekey\x00" + node + "\x00" + previous + "\x00" + key + "\x00" +
t.Interface + "\x00" + t.Unit + "\x00" + t.Config + "\x00" + strconv.Itoa(t.Port) + "\x00" +
t.Address + "\x00" + t.Range + "\x00" + t.PublicKey + "\x00" + strings.Join(peers, ","))
}
// Held is one file or container found on an adopted node and kept as it was.
type Held struct {
ID string `json:"id"`
internal/link/messages_test.go
-3
View File
@@ -12,9 +12,6 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) {
value any
expect []string
}{
// novox/hq ADR 0539 §9: the node's facts carry the node-engine's own writer, epoch and sequence.
{Report{Node: "n", Writer: Writer, Order: Order{Epoch: 1, Sequence: 2}},
[]string{"node", "writer", "epoch", "sequence"}},
{Report{Node: "n", Applied: []string{"a"}, Failed: map[string]string{"k": "v"}, Refused: "r"},
[]string{"node", "applied", "failed", "refused"}},
// novox/hq ADR 0100: what an adopted node holds, the firewall it was found with, and what
internal/link/run.go
-39
View File
@@ -3,7 +3,6 @@ package link
import (
"context"
"crypto/ed25519"
"encoding/json"
"errors"
"fmt"
"time"
@@ -284,41 +283,3 @@ func serve(ctx context.Context, link Link, m Membership, queue *Session, say Ann
}
}
}
// publishReport tells the mesh what this node did, and says whether the broker took it.
// Publish sends one report on this node's own connection and returns: the one-shot path for a
// report a command makes rather than the running host — a rekey (novox/hq ADR 0105). The same
// account, the same pinned certificate and the same exchange as the running host's reports.
func Publish(ctx context.Context, m Membership, report Report, timeout time.Duration) error {
link, err := Open(ctx, m, timeout)
if err != nil {
return err
}
defer link.Close()
var said string
if !publishReport(ctx, link, m, report, func(s string) { said = s }, timeout) {
return errors.New(said)
}
return nil
}
func publishReport(ctx context.Context, bus Bus, m Membership, report Report,
say Announce, timeout time.Duration) bool {
report.Node = m.Node
body, err := json.Marshal(report)
if err != nil {
say("cannot encode this node's own report: " + err.Error())
return false
}
publish, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
// Said rather than swallowed. A report that fails to publish leaves the mesh believing this
// node never answered, while the node believes it did — and the two would go on disagreeing
// with nothing anywhere saying so. That shape of fault is the one this project keeps finding.
if err := bus.Report(publish, m.Node, body); err != nil {
say(fmt.Sprintf("applied, and could not tell the mesh: %v", err))
return false
}
return true
}
internal/link/session.go
+9 -24
View File
@@ -13,16 +13,11 @@ import (
"git.novox.be/novox/mesh-sdk/go/health"
)
// Writer is what the node-engine calls itself on the facts it says (novox/hq ADR 0227 §2, ADR 0539 §9): the program
// that wrote them, beside its epoch and sequence.
const Writer = "node-engine"
// Session is the node-engine's hold on the bus (novox/hq ADR 0539 §9): the link open now, what the controller asks
// of this machine over it, and what the machine says on it unasked: its health, its haves and its facts.
// Session is the node-engine's hold on the bus (novox/hq ADR 0539 §9): the link open now, what is asked of this
// machine over it, and what the machine says on it unasked: its health and its haves.
//
// **It applies nothing and orders nothing.** A module is applied by the node agent from its want, and the facts are an
// event of the node, ordered by the node-engine's own epoch and sequence. Nothing here receives, orders or reconciles
// a whole declaration.
// **It applies nothing and orders nothing.** A module is applied by the node agent from its want. Nothing here
// receives, orders or reconciles a whole declaration.
type Session struct {
// Fallback is the engine's fall back to its old login its health report carries now (Rotator.Fallback); nil says none.
Fallback func() *delivery.LoginFallback
@@ -84,12 +79,8 @@ type Session struct {
haveAgainPending bool
haveAgainTries int
// saying is held while a facts event is made and said, so they leave in the order they are made.
// saying is held while a facts event is noted, so the haves are written in the order the facts are made.
saying sync.Mutex
// epoch is this process's own: the time it started, in milliseconds, so a later process orders after an earlier one
// without anything kept on disk; sequence is one higher for every facts event it says (ADR 0227 §2).
epoch int64
sequence int64
// haves is what this node-engine writes beside its facts (novox/hq ADR 0429 §4, issue 1132).
haves haveState
@@ -133,7 +124,6 @@ func (q *Session) init() {
if q.Say == nil {
q.Say = func(string) {}
}
q.epoch = time.Now().UnixMilli()
q.changed = make(chan struct{})
q.owed = make(chan struct{}, 1)
})
@@ -156,8 +146,10 @@ func (q *Session) oweHealth() {
}
}
// SayFacts says this node's facts on the link open now, in the node-engine's own order, and whether the bus took them.
// A facts event made with no link open is not kept: the next link says the facts as they are then.
// SayFacts says this node's facts in its haves on the link open now, in the order they are made, and whether a link
// was open to say them on. A facts event made with no link open is not kept: the next link says the facts as they are
// then. They are said in the haves alone: the CONTROL stream they were also published on was the controller's, which
// ended by its unassignment (issue 1126). A write the bucket refuses is said and written again (issue 1132).
func (q *Session) SayFacts(ctx context.Context, r Report) bool {
q.init()
q.saying.Lock()
@@ -168,14 +160,7 @@ func (q *Session) SayFacts(ctx context.Context, r Report) bool {
if bus == nil {
return false
}
q.sequence++
r.Node = q.Membership.Node
r.Writer = Writer
r.Order = Order{Epoch: q.epoch, Sequence: q.sequence}
if !publishReport(context.WithoutCancel(ctx), bus, q.Membership, r, q.Say, q.Timeout) {
return false
}
// After the facts, which are sent first and unchanged; a write the bucket refuses never undoes them (issue 1132).
if q.noteApply(r) {
q.writeHaves(ctx, bus)
}
internal/link/session_test.go
+3 -74
View File
@@ -3,7 +3,6 @@ package link
import (
"context"
"crypto/ed25519"
"encoding/json"
"errors"
"fmt"
"os"
@@ -15,42 +14,16 @@ import (
"github.com/nats-io/nats.go"
)
// quietLink is a link with no broker behind it. A report is refused once the context it is published on is cancelled,
// as the bus's is ("reporting: context canceled"), or while refusing.
// quietLink is a link with no broker behind it.
type quietLink struct {
mu sync.Mutex
reports []Report
refusing bool
lost chan error
lost chan error
}
func newQuietLink() *quietLink { return &quietLink{lost: make(chan error, 1)} }
func (l *quietLink) Report(ctx context.Context, _ string, body []byte) error {
if err := ctx.Err(); err != nil {
return err
}
l.mu.Lock()
defer l.mu.Unlock()
if l.refusing {
return errors.New("refused")
}
var r Report
if err := json.Unmarshal(body, &r); err != nil {
return err
}
l.reports = append(l.reports, r)
return nil
}
func (l *quietLink) Lost() <-chan error { return l.lost }
func (l *quietLink) Close() {}
func (l *quietLink) said() []Report {
l.mu.Lock()
defer l.mu.Unlock()
return append([]Report(nil), l.reports...)
}
func aMember(t *testing.T) (Membership, ed25519.PrivateKey) {
t.Helper()
public, private, err := ed25519.GenerateKey(nil)
@@ -60,40 +33,6 @@ func aMember(t *testing.T) (Membership, ed25519.PrivateKey) {
return Membership{Node: "n1", Signer: public}, private
}
// R6-A AC7: the facts event carries the node-engine's own writer, epoch and sequence (novox/hq ADR 0227 §2, ADR 0539
// §9): one epoch per start, a sequence one higher for every event, so the controller can refuse older facts.
func TestTheFactsEventCarriesTheNodeEnginesWriterEpochAndSequence(t *testing.T) {
m, _ := aMember(t)
l := newQuietLink()
s := &Session{Membership: m, Timeout: time.Second}
s.Attach(context.Background(), l)
if !s.SayFacts(context.Background(), Report{Firewall: "none"}) || !s.SayFacts(context.Background(), Report{}) {
t.Fatal("the facts were not taken")
}
got := l.said()
if len(got) != 2 {
t.Fatalf("%d facts events were said", len(got))
}
for _, r := range got {
if r.Writer != Writer || r.Node != "n1" || r.Epoch <= 0 {
t.Fatalf("the facts do not say who wrote them: %+v", r)
}
}
if got[0].Epoch != got[1].Epoch || got[1].Sequence != got[0].Sequence+1 || got[0].Sequence <= 0 {
t.Fatalf("the order is not one epoch with a rising sequence: %+v then %+v", got[0].Order, got[1].Order)
}
// A later start orders after this one with nothing kept on disk.
time.Sleep(2 * time.Millisecond)
later := &Session{Membership: m, Timeout: time.Second}
later.Attach(context.Background(), l)
later.SayFacts(context.Background(), Report{})
if last := l.said()[2]; last.Epoch <= got[1].Epoch {
t.Fatalf("a later start has epoch %d, not above %d", last.Epoch, got[1].Epoch)
}
}
// Facts made with no link open are not kept, and nothing is said: the next link says them as they are then.
func TestFactsWithNoLinkOpenAreNotSaid(t *testing.T) {
m, _ := aMember(t)
@@ -104,7 +43,7 @@ func TestFactsWithNoLinkOpenAreNotSaid(t *testing.T) {
}
s.Attach(context.Background(), l)
s.Detach(l)
if s.Linked() || s.SayFacts(context.Background(), Report{}) || len(l.said()) != 0 {
if s.Linked() || s.SayFacts(context.Background(), Report{}) {
t.Fatal("facts were said on a link that was let go")
}
}
@@ -140,16 +79,6 @@ func aBus(t *testing.T) (*nats.Conn, nats.JetStreamContext) {
if err != nil {
t.Fatal(err)
}
want := &nats.StreamConfig{Name: "CONTROL", Subjects: []string{"mesh.control.*.report", "mesh.control.enrol"},
Retention: nats.WorkQueuePolicy}
if _, err := js.StreamInfo(want.Name); err != nil {
if _, err := js.AddStream(want); err != nil {
t.Fatal(err)
}
}
if err := js.PurgeStream(want.Name); err != nil {
t.Fatal(err)
}
return conn, js
}