diff --git a/cmd/mesh-host/enrol_seat.go b/cmd/mesh-host/enrol_seat.go index 4d253a8..94cb23b 100644 --- a/cmd/mesh-host/enrol_seat.go +++ b/cmd/mesh-host/enrol_seat.go @@ -16,17 +16,17 @@ import ( // enrolThroughSeat joins this machine through the seat mesh-enrolment (novox/hq ADR 0587 §6, issue 774): it redeems the token // with the keys it made, verifies the vault's signed answer against the key the token carries before it keeps anything, keeps -// its membership and its login, dials as the login node-logins names and confirms. handled is false when the bus does not serve -// the seat to this token (the controller still holds its lease, so its enrolment subject is the way in): nothing was sent or -// kept then, and the caller enrols as before. +// its membership and its login, dials as the login node-logins names and confirms. A bus that does not serve the seat to this +// token (link.ErrNoSeat) is said as the error: nothing was sent or kept. func enrolThroughSeat(ctx context.Context, opts options, token identity.Token, node string, mine identity.Identity, - sealing identity.SealingKey, serving identity.ServingKey, reported map[string]any, found *link.Tunnel, identityPath string) (handled bool, err error) { + sealing identity.SealingKey, serving identity.ServingKey, reported map[string]any, found *link.Tunnel, identityPath string) error { req := delivery.Redeem{Node: node, PublicKey: base64.StdEncoding.EncodeToString(mine.Public), OverlayKey: mine.Overlay.Public, SealingKey: sealing.Public, ServingKey: serving.Public, Profile: reported} if found != nil { + var err error if req.Tunnel, err = json.Marshal(found); err != nil { - return true, err + return err } } req.Proof = base64.StdEncoding.EncodeToString(mine.Sign(delivery.RedeemProof(node, req.PublicKey, req.OverlayKey, req.SealingKey, req.ServingKey))) @@ -61,20 +61,13 @@ func enrolThroughSeat(ctx context.Context, opts options, token identity.Token, n }, } answer, err := j.Run(ctx) - switch { - case errors.Is(err, link.ErrNoSeat): - // The path before the lease write: the controller's enrolment subject is granted to the token only while the - // controller holds its lease, so after the write this fallback is refused too, and that refusal is what is said. - fmt.Printf("the seat mesh-enrolment is not served to this token (%v); enrolling through the controller's enrolment "+ - "subject, the path before the lease write, which the bus refuses once the lease is written\n", err) - return false, nil - case err != nil: - return true, err + if err != nil { + return err } fmt.Printf("\nenrolled as %s through the seat mesh-enrolment, dialled as %s and confirmed\n", answer.Node, answer.Login) fmt.Printf(" identity %s\n", identityPath) fmt.Println(" nothing is applied until the placement of this node names a module") - return true, nil + return nil } // keepJoinedKeys writes the private halves of the keys this machine made, after its identity: an identity naming a key that is diff --git a/cmd/mesh-host/healthowed_test.go b/cmd/mesh-host/healthowed_test.go index 08c5d96..a46871c 100644 --- a/cmd/mesh-host/healthowed_test.go +++ b/cmd/mesh-host/healthowed_test.go @@ -21,7 +21,6 @@ type owedBus struct { rv map[string]uint64 } -func (b *owedBus) Report(context.Context, string, []byte) error { return nil } func (b *owedBus) Health(context.Context, string, []byte) error { return nil } func (b *owedBus) HaveBind(context.Context) (link.HaveBucket, error) { return b, nil } diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 697b692..1dcf431 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -809,167 +809,21 @@ func enrol(ctx context.Context, opts options) error { detected := profile.Detect(ctx, profile.Default(nil), opts.timeout) reported := profileAsReported(detected) - // Signed with the identity just generated, so the mesh can tell this machine from anyone else - // who knows its public key (novox/hq issue 083). - proof := mine.Sign(link.EnrolProof(token.Secret, mine.Public, mine.Overlay.Public, - sealing.Public, serving.Public)) - // The token says where to go and which certificate that address must present. It says nothing - // about which bus is there, and does not need to: there is one, and this host knows which - // (novox/hq ADR 0131 — the mesh speaks to one seat and the old transport is gone). - // - // **It used to leave this empty** and mean "whatever the mesh runs today", which was true - // while two buses existed and became a refusal the moment one did: an empty transport is not - // the bus's name, so every enrolment ended at "this token is for the \"\" bus" - // (novox/hq 04-ISSUES/146). Nothing caught it because nothing had enrolled since the bus - // changed. - // Through the seat mesh-enrolment when the bus serves it to this token (novox/hq ADR 0587 §6): the controller's enrolment - // subject below stays for as long as the controller holds its lease. - if handled, err := enrolThroughSeat(ctx, opts, token, *name, mine, sealing, serving, reported, found, identityPath); handled || err != nil { - return err - } - reply, err := link.Enrol(ctx, - link.Approach{Address: token.Broker, Fingerprint: token.Fingerprint, Transport: link.OnNATS}, - *name, token.Secret, - mine.Public, mine.Overlay.Public, sealing.Public, serving.Public, reported, proof, found, - opts.timeout) - if err != nil { - return err - } - - // The mesh's name for this node wins over what the machine called itself: the token was - // issued for a node record, and that record is what the identity binds to. - mine.Node = reply.Node - mine.Membership = identity.Membership{ - Broker: firstNonEmpty(reply.Broker, token.Broker), - Fingerprint: firstNonEmpty(reply.Fingerprint, token.Fingerprint), - Signer: firstNonEmpty2(reply.Signer, token.Signer), - Password: reply.Password, - // **Which bus this membership is for, said rather than left empty** (novox/hq - // 04-ISSUES/146). The link refuses a membership that names another bus, and an empty name - // is not this one's — so a node enrolled without it came up and reconnected for ever - // against its own record: "this membership is for \"\", and the mesh's bus is nats". The - // reply does not carry it because there is one bus and the host knows which (ADR 0131); - // what was missing was writing that down where the link reads it. - Transport: link.OnNATS, - } - if mine.Membership.Password == "" { - // The mesh did not replace the token's secret, so it is still this node's broker - // password. Said rather than silently kept: a one-time secret living on as a credential - // is worth knowing about. - mine.Membership.Password = token.Secret - fmt.Println("\nnote: the mesh issued no separate broker password, so the token's secret " + - "remains this node's credential") - } - - // Saved only now, and only once the mesh has said it knows this node. A node holding an - // identity the mesh has never recorded would believe it had joined and be believed by - // nobody — worse than not having joined, because nothing would look wrong. - if err := identity.Save(identityPath, mine); err != nil { - return fmt.Errorf( - "the mesh accepted this node as %q and its identity could not be saved: %w\n"+ - "That token is spent, so getting back needs a new one", reply.Node, err) - } - - if !mine.Membership.Joined() { - return fmt.Errorf( - "the mesh accepted this node as %q but did not say how to reach it again, so this "+ - "identity could not be used after a restart. Nothing was saved", reply.Node) - } - - // Written before the identity, so a node that dies between the two has a key file with no - // identity — which enrols again cleanly — rather than an identity naming a key that is not - // there, which looks joined and cannot come up. - if err := os.WriteFile(identity.OverlayKeyPath(opts.state), - []byte(mine.Overlay.Private+"\n"), 0o600); err != nil { - return fmt.Errorf("cannot write this node's overlay key: %w", err) - } - if err := os.WriteFile(identity.SealingKeyPath(opts.state), - []byte(sealing.Private+"\n"), 0o600); err != nil { - return fmt.Errorf("cannot write this node's sealing key: %w", err) - } - if err := identity.WriteServingKey(identity.ServingKeyPath(opts.state), serving); err != nil { - return fmt.Errorf("cannot write this node's serving key: %w", err) - } - - fmt.Printf("\nenrolled as %s\n", reply.Node) - fmt.Printf(" identity %s\n", identityPath) - fmt.Printf(" queue %s\n", reply.Queue) - return nil + // Through the seat mesh-enrolment (novox/hq ADR 0587 §6), the one way in: the controller's enrolment subject ended + // with the controller (issue 1126). + return enrolThroughSeat(ctx, opts, token, *name, mine, sealing, serving, reported, found, identityPath) } -// overlayCommand is `mesh-host overlay take`: this node takes the tunnel found on its machine over -// after it enrolled (novox/hq ADR 0105). For a node that enrolled before the mesh knew to take a -// tunnel over — re-enrolling would rotate its identity, sealing and serving keys, and with them -// every credential the mesh sealed to it. -func overlayCommand(ctx context.Context, opts options) error { +// overlayCommand is `mesh-host overlay take`: a node taking the tunnel found on its machine over after it enrolled +// (novox/hq ADR 0105). Refused, and nothing is changed: the rekey it sent was taken by the controller alone, which +// ended by its unassignment (issue 1126), and no seat takes one now; a key written here that no part of the mesh knows +// would raise the mesh's interface on a key its hub does not peer. +func overlayCommand(_ context.Context, opts options) error { if opts.file != "take" { return errors.New("overlay take [--tunnel ] — the one thing `overlay` does here") } - identityPath := identity.Path(opts.state) - mine, err := identity.Load(identityPath) - if err != nil { - return err - } - found, err := tunnel.Find(ctx, apply.ExecRunner, opts.tunnel) - if err != nil { - return fmt.Errorf("%w. Nothing was changed", err) - } - taken, rekey, err := rekeyOnto(mine, found) - if err != nil { - return err - } - fmt.Printf("taking over %s: this node's overlay key becomes the tunnel's, %s\n", found, taken.Overlay.Public) - fmt.Printf(" identity, sealing and serving keys are untouched\n") - - // Told first, then written: a mesh told and a machine not yet written is put right by running - // this again (the mesh refuses the stale second rekey and changes nothing; the files are - // rewritten the same). A machine written and a mesh not told would raise the mesh's interface - // on a key the mesh does not know at the next restart. - if err := link.Publish(ctx, link.Membership{ - Node: mine.Node, Broker: mine.Membership.Broker, Fingerprint: mine.Membership.Fingerprint, - Password: mine.Membership.Password, Signer: mine.Membership.Signer, - Transport: mine.Membership.Transport, - }, link.Report{Node: mine.Node, Rekey: &rekey}, opts.timeout); err != nil { - return fmt.Errorf("the mesh could not be told; nothing was written here: %w", err) - } - fmt.Printf(" told the mesh signed rekey sent; `node show %s` on the controller says whether it took\n", mine.Node) - - if err := os.WriteFile(identity.OverlayKeyPath(opts.state), - []byte(taken.Overlay.Private+"\n"), 0o600); err != nil { - return fmt.Errorf("the mesh was told and this node's overlay key could not be written: %w — run this again", err) - } - if err := identity.Save(identityPath, taken); err != nil { - return fmt.Errorf("the mesh was told and this node's identity could not be saved: %w — run this again", err) - } - fmt.Printf(" written %s and the identity; the mesh's interface reads the key when the next push restarts it\n", - identity.OverlayKeyPath(opts.state)) - fmt.Printf(" next on the controller: `overlay place %s --hub --endpoint :%d …`, `plan %s --json`, then push\n", - mine.Node, found.Port, mine.Node) - return nil -} - -// rekeyOnto is the identity with the found tunnel's key as its overlay key, and the signed rekey -// that tells the mesh. Pure, so it can be held to: node, sealing and serving keys are the same -// bytes in and out; only the overlay key moves. Run again after a take, the previous key it names -// is the one before the take, so the mesh can tell a repeat from a replay. -func rekeyOnto(mine identity.Identity, found tunnel.Found) (identity.Identity, link.Rekey, error) { - overlay, err := identity.OverlayKeyFrom(found.PrivateKey()) - if err != nil { - return identity.Identity{}, link.Rekey{}, err - } - previous := mine.Overlay.Public - if previous == overlay.Public && mine.OverlayBefore != "" { - previous = mine.OverlayBefore - } - taken := mine - taken.Overlay = overlay - if previous != overlay.Public { - taken.OverlayBefore = previous - } - presented := carried(found) - rekey := link.Rekey{Previous: previous, OverlayKey: overlay.Public, Tunnel: presented} - rekey.Proof = mine.Sign(link.RekeyProof(mine.Node, previous, overlay.Public, presented)) - return taken, rekey, nil + return errors.New("overlay take is refused: the controller that took a rekey ended (issue 1126) and no seat " + + "takes one now. Nothing was changed") } // carried is a found tunnel as it is presented to the mesh: everything but its private key. diff --git a/cmd/mesh-host/nodeagent_integration_test.go b/cmd/mesh-host/nodeagent_integration_test.go index 8a958f3..cf471e4 100644 --- a/cmd/mesh-host/nodeagent_integration_test.go +++ b/cmd/mesh-host/nodeagent_integration_test.go @@ -440,10 +440,6 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) { builds := kvOf(delivery.BuilderBuildsBucket) candidates := kvOf(delivery.PlacementCandidatesBucket) packages := kvOf(link.PackagesBucket) - // The controller's stream the node's facts go into, so the node's key carries the facts it says. - if _, err := b.js.AddStream(&nats.StreamConfig{Name: "CONTROL", Subjects: []string{link.ReportSubject(itNode)}}); err != nil { - t.Fatal(err) - } path := filepath.Join(b.dir, "web.conf") man := `{"module":"web","version":"1","config":{"greeting":{"type":"string","description":"what the file says","default":"hello"}},` + `"resources":[{"id":"conf","type":"file","path":"` + path + `","mode":"0644","content":"greeting=${config:greeting}\n"},` + diff --git a/cmd/mesh-host/rekey_test.go b/cmd/mesh-host/rekey_test.go deleted file mode 100644 index 7b19edd..0000000 --- a/cmd/mesh-host/rekey_test.go +++ /dev/null @@ -1,94 +0,0 @@ -package main - -import ( - "crypto/ed25519" - "strings" - "testing" - - "github.com/novox/mesh-host/internal/identity" - "github.com/novox/mesh-host/internal/link" - "github.com/novox/mesh-host/internal/tunnel" -) - -// novox/hq ADR 0105: `overlay take` moves this node's overlay key onto the found tunnel's and -// nothing else — identity, sealing and serving keys stay as they were — and tells the mesh with a -// proof signed by the identity key, over the previous key, the new one and the tunnel. - -func anEnrolledNode(t *testing.T) identity.Identity { - t.Helper() - mine, err := identity.Generate("anchor") - if err != nil { - t.Fatal(err) - } - mine.Overlay, err = identity.GenerateOverlayKey() - if err != nil { - t.Fatal(err) - } - mine.Membership = identity.Membership{Broker: "198.51.100.1:5671", Fingerprint: "sha256:aa", - Signer: make([]byte, ed25519.PublicKeySize), Password: "p"} - return mine -} - -func aFoundTunnel(t *testing.T) tunnel.Found { - t.Helper() - private, err := identity.GenerateOverlayKey() - if err != nil { - t.Fatal(err) - } - found, err := tunnel.Parse([]byte("[Interface]\nPrivateKey = " + private.Private + "\nListenPort = 51900\n" + - "Address = 192.0.2.1/24\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n")) - if err != nil { - t.Fatal(err) - } - found.Interface, found.Unit, found.Config = "wg0", "wg-quick@wg0", "/etc/wireguard/wg0.conf" - return found -} - -func TestTakingATunnelMovesOnlyTheOverlayKeyAndSignsForIt(t *testing.T) { - mine := anEnrolledNode(t) - found := aFoundTunnel(t) - before := mine.Overlay.Public - - taken, rekey, err := rekeyOnto(mine, found) - if err != nil { - t.Fatal(err) - } - if taken.Overlay.Public != found.PublicKey || taken.Overlay.Private != found.PrivateKey() { - t.Fatal("the overlay key is not the tunnel's") - } - if string(taken.Public) != string(mine.Public) || string(taken.Private) != string(mine.Private) || - taken.Node != mine.Node || taken.Membership.Password != mine.Membership.Password || - taken.Membership.Broker != mine.Membership.Broker { - t.Fatal("something other than the overlay key moved") - } - if taken.OverlayBefore != before { - t.Errorf("the key before the take was not kept: %q", taken.OverlayBefore) - } - if rekey.Previous != before || rekey.OverlayKey != found.PublicKey || rekey.Tunnel == nil || - rekey.Tunnel.PublicKey != found.PublicKey || len(rekey.Tunnel.Peers) != 1 { - t.Fatalf("the rekey does not say what moved: %+v", rekey) - } - if !ed25519.Verify(ed25519.PublicKey(mine.Public), - link.RekeyProof("anchor", before, found.PublicKey, rekey.Tunnel), rekey.Proof) { - t.Fatal("the rekey is not signed by this node's identity key over what it says") - } - if ed25519.Verify(ed25519.PublicKey(mine.Public), - link.RekeyProof("laptop", before, found.PublicKey, rekey.Tunnel), rekey.Proof) { - t.Fatal("the proof is not bound to the node") - } - for _, said := range []string{rekey.Previous, rekey.OverlayKey, rekey.Tunnel.Interface} { - if strings.Contains(said, found.PrivateKey()) { - t.Fatal("the private key travels") - } - } - - // Run again after the take — the mesh not yet told, or told and refused — the previous key it - // names is still the one before the take, so the mesh can tell a repeat from a replay. - again, second, err := rekeyOnto(taken, found) - if err != nil { - t.Fatal(err) - } - if second.Previous != before || again.OverlayBefore != before || again.Overlay.Public != found.PublicKey { - t.Fatalf("a take run again does not name the key before the first: %+v", second) - } -} diff --git a/internal/link/asking.go b/internal/link/asking.go deleted file mode 100644 index 2a53261..0000000 --- a/internal/link/asking.go +++ /dev/null @@ -1,53 +0,0 @@ -package link - -import ( - "context" - "fmt" - "time" -) - -// The enrolment conversation, as the host's own words for it. -// -// **Its own seam rather than part of Link**, because almost nothing about it is the same. The -// credential is a one-time secret rather than this node's own; there is no declaration to hear; the -// whole exchange is a single question asked and possibly asked again. And the stakes differ: a node -// that fails here is not in the mesh at all, where a node that fails in Link has merely lost touch -// with one it belongs to. - -// Approach is how a node reaches a mesh it does not yet belong to. -// -// The three things a token carries about where to go, and nothing about who is asking: the address, -// the certificate that address must present, and which bus is at the other end — the mesh's own -// (hearing.go), and a token naming any other is refused before anything is sent. -type Approach struct { - Address string - Fingerprint string - Transport string -} - -// Asking is one open enrolment conversation. -type Asking interface { - // Ask puts the request to the mesh and waits for one answer, or says why none came. - // - // Called again, with the same bytes, while the mesh says "try again": the keys this node - // generated are the ones it keeps, so the same request is the same enrolment and the mesh holds - // the token for it (novox/hq issue 083). - Ask(ctx context.Context, request []byte, wait time.Duration) ([]byte, error) - - // Close lets go of the connection made with the token. - Close() -} - -// Present opens an enrolment conversation with the mesh. -// -// The connection is made before anything is sent, and the certificate is checked while it is being -// made — so a node pointed at the wrong bus finds out before its token has left the machine (ADR -// 0004). -func Present(ctx context.Context, to Approach, node, secret string, - timeout time.Duration) (Asking, error) { - - if to.Transport != OnNATS { - return nil, fmt.Errorf("this token is for the %q bus, and the mesh's bus is %s", to.Transport, OnNATS) - } - return presentNats(ctx, to, node, secret, timeout) -} diff --git a/internal/link/asking_nats.go b/internal/link/asking_nats.go deleted file mode 100644 index ed8bebd..0000000 --- a/internal/link/asking_nats.go +++ /dev/null @@ -1,186 +0,0 @@ -package link - -import ( - "context" - "crypto/rand" - "crypto/sha256" - "encoding/hex" - "errors" - "fmt" - "time" - - "github.com/nats-io/nats.go" -) - -// The enrolment conversation on the bus being built. -// -// **The reply address is the whole of what changes**, and it changes for a reason the transport -// forces rather than a preference. Core NATS request/reply puts the caller's inbox in the message's -// reply field and a plain responder answers it — but this request goes into a stream, and a message -// a JetStream consumer delivers has had that field claimed for the consumer's own ack address. So by -// the time the controller reads the request, the transport's reply field names where the -// *controller* must acknowledge. Verified against a running server (design 25 §2). -// -// The address therefore travels as a field of the request, and this subscribes it before publishing: -// a node that published first could miss an answer to a question nobody was listening for. - -// enrolInbox is where a node enrolling waits. -// -// Under `_INBOX.enrol..`, which is exactly what its enrolment user may subscribe and no -// wider — so an answer sealed to one machine cannot be read by another enrolling beside it. The -// random tail is this attempt's own: a reply left over from an attempt that timed out is not the -// answer to this question, which is what the correlation id does on the other transport. -func enrolInbox(node string) (string, error) { - tail := make([]byte, 8) - if _, err := rand.Read(tail); err != nil { - return "", fmt.Errorf("cannot make a reply address: %w", err) - } - return "_INBOX.enrol." + node + "." + hex.EncodeToString(tail), nil -} - -type natsAsking struct { - conn *nats.Conn - js nats.JetStreamContext - inbox string - answers *nats.Subscription - lost chan error -} - -func presentNats(_ context.Context, to Approach, node, secret string, - timeout time.Duration) (Asking, error) { - - config, err := PinnedConfig(to.Fingerprint) - if err != nil { - return nil, err - } - - inbox, err := enrolInbox(node) - if err != nil { - return nil, err - } - - lost := make(chan error, 1) - // The user is this token's own — `enrol.`, which may publish the enrolment subject and - // subscribe its own inbox and nothing else (design 25 §6). The secret is its password, the same - // string the request claims, so the server proves somebody holds the token and the request - // proves the same thing to the controller without it having to ask the server who connected. - // **Its own inbox space, because that is the only one it may listen in** (novox/hq - // 04-ISSUES/146). A JetStream publish waits for the stream's acknowledgement on an inbox the - // client picks, and the client's default is `_INBOX.` — which this user may not - // subscribe to, so the enrolment failed with a permissions violation on a subject nobody had - // chosen. The permission is `_INBOX.enrol..>` (design 25 §6), so the client is told to - // pick its inboxes there; the reply address below is in the same space for the same reason. - conn, err := nats.Connect(natsURL(to.Address), - nats.Secure(config), - nats.CustomInboxPrefix("_INBOX.enrol."+node), - nats.UserInfo("enrol."+node, secret), - nats.Name("mesh-host/enrol/"+node), - nats.Timeout(timeout), - nats.NoReconnect(), - nats.DisconnectErrHandler(func(_ *nats.Conn, err error) { - select { - case lost <- fmt.Errorf("the bus closed the connection: %w", err): - default: - } - }), - ) - if err != nil { - if errors.Is(err, ErrWrongCertificate) { - return nil, err - } - // Not quoted back with the credential: the secret is one-time and still a secret. - return nil, fmt.Errorf("cannot reach the bus at %s as %s: %w", to.Address, node, err) - } - js, err := conn.JetStream() - if err != nil { - conn.Close() - return nil, fmt.Errorf("the bus at %s has no JetStream: %w", to.Address, err) - } - - // Subscribed before anything is published, so an answer cannot arrive before there is anywhere - // for it to land. - answers, err := conn.SubscribeSync(inbox) - if err != nil { - conn.Close() - return nil, fmt.Errorf("this node cannot listen for the mesh's answer: %w", err) - } - if err := conn.Flush(); err != nil { - conn.Close() - return nil, fmt.Errorf("this node's reply address did not reach the bus: %w", err) - } - - return &natsAsking{conn: conn, js: js, inbox: inbox, answers: answers, lost: lost}, nil -} - -func (a *natsAsking) Close() { - if a.answers != nil { - _ = a.answers.Unsubscribe() - } - if a.conn != nil { - a.conn.Close() - } -} - -// Ask publishes the request with this attempt's reply address written into it, and waits there. -func (a *natsAsking) Ask(ctx context.Context, request []byte, wait time.Duration) ([]byte, error) { - addressed, err := withReplyTo(request, a.inbox) - if err != nil { - return nil, err - } - - publish, cancel := context.WithTimeout(ctx, wait) - defer cancel() - // Into the stream and awaited: an enrolment the bus never accepted must fail here rather than be - // assumed, because the node has nothing else to go on. - // - // **Once, however many times it is sent** (novox/hq 04-ISSUES/146). The client re-publishes when - // an acknowledgement is slow, and the mesh enrolled the machine on each copy — minting a second - // credential, which replaced the first, which is the one the node had already been given. The - // machine then reconnected for ever as a user whose password the mesh had rotated out from under - // it, and the controller's log said "enrolled anchor" twice in the same second. - // - // The id is the message: the same bytes carry the same id, so the stream discards the client's - // own retry, and a genuine second attempt — which carries a new reply address — is a different - // message and is let through. - sum := sha256.Sum256(addressed) - if _, err := a.js.Publish(EnrolSubject, addressed, - nats.MsgId(hex.EncodeToString(sum[:])), nats.Context(publish)); err != nil { - return nil, fmt.Errorf("cannot ask the mesh to enrol this node: %w", err) - } - - // Waited for rather than assumed. A published message that nothing answers means the controller - // is not running, and a node that carried on regardless would believe it had joined a mesh that - // has never heard of it. - // - // **The wait may legitimately be several store-window cycles long**: the controller naks the - // request with a delay while its store is restarting, and the node is waiting on the other side - // of that — which is exactly the combination that would have delivered the answer to a caller - // who had given up, had the address travelled in the transport's field. - answered, cancelAnswer := context.WithTimeout(ctx, wait) - defer cancelAnswer() - for { - msg, err := a.answers.NextMsgWithContext(answered) - switch { - case err == nil: - return msg.Data, nil - case errors.Is(err, context.DeadlineExceeded): - select { - case reason := <-a.lost: - return nil, reason - default: - } - return nil, fmt.Errorf( - "the bus accepted this node's connection and nothing answered within %s. The mesh's "+ - "bus is running and its controller is not", wait) - case errors.Is(err, context.Canceled): - return nil, ctx.Err() - default: - select { - case reason := <-a.lost: - return nil, reason - default: - } - return nil, fmt.Errorf("waiting for the mesh's answer: %w", err) - } - } -} diff --git a/internal/link/asking_nats_test.go b/internal/link/asking_nats_test.go deleted file mode 100644 index 919b9d9..0000000 --- a/internal/link/asking_nats_test.go +++ /dev/null @@ -1,192 +0,0 @@ -package link - -import ( - "context" - "encoding/json" - "errors" - "strings" - "testing" - "time" - - "github.com/nats-io/nats.go" -) - -// The enrolment round trip against a real server. -// -// **This is the test that keeps a reason from becoming folklore.** The reply address travels in the -// request's payload because a JetStream consumer's delivery has had the transport's reply field -// claimed for its own ack address — which is a fact about a server, not a rule anybody can check by -// reading. Both halves are asserted here: that the field really is eaten, and that the answer -// reaches the node anyway. -// -// docker run -d --rm --name t -p 14223:4222 nats:2.10-alpine -js -// MESH_TEST_NATS=nats://127.0.0.1:14223 go test ./internal/link/ -run TestNatsAnEnrolment - -// asking is a conversation on a bus with no TLS. Built directly rather than through Present because -// the pin is what Present adds and PinnedConfig's own tests cover it; what is under test here is the -// address the answer comes back on. -func asking(t *testing.T, conn *nats.Conn, js nats.JetStreamContext, node string) *natsAsking { - t.Helper() - inbox, err := enrolInbox(node) - if err != nil { - t.Fatal(err) - } - answers, err := conn.SubscribeSync(inbox) - if err != nil { - t.Fatal(err) - } - if err := conn.Flush(); err != nil { - t.Fatal(err) - } - a := &natsAsking{conn: conn, js: js, inbox: inbox, answers: answers, lost: make(chan error, 1)} - // **Not a.Close.** Close closes the connection, and this connection is the test's own, shared with - // the stand-in. Cleanups run last-in first-out and the stand-in was made first, so its cleanup - // would run on a closed connection: Unsubscribe returns ErrConnectionClosed without deleting the - // consumer it made, and every request after that fails. The consumer then outlives the test on a - // workqueue stream, which allows one consumer per filter, and the next test cannot subscribe - // (issue 604). The connection is closed by aBus, last of all. - t.Cleanup(func() { _ = answers.Unsubscribe() }) - return a -} - -// theMeshAnswers stands in for the controller: it consumes the enrolment off the stream, reads the -// reply address out of the payload — never from the transport field — and answers there. It reports -// what the transport field actually held, which is the claim design 25 §2 rests on. -func theMeshAnswers(t *testing.T, conn *nats.Conn, js nats.JetStreamContext, - reply EnrolReply) <-chan string { - t.Helper() - sawReplyField := make(chan string, 1) - // **A consumer of this test's own, gone before it starts and gone when it ends.** The stand-in - // used to be one durable name shared by every test, released by Unsubscribe alone. Unsubscribe - // only asks the server to let go: it deletes the consumer only when this subscription made it, - // and the server drops the binding on its own time. A test that started in that window (or one - // that found the consumer left by an earlier run on a long-lived bus) was refused with "consumer - // is already bound to a subscription". Per-test names remove the sharing; the delete-and-wait - // before and after removes the dependence on the server's timing. - durable := standinName(t) - stream, err := js.StreamNameBySubject(EnrolSubject) - if err != nil { - t.Fatal(err) - } - standinGone(t, js, stream, durable) - sub, err := js.Subscribe(EnrolSubject, func(msg *nats.Msg) { - select { - case sawReplyField <- msg.Reply: - default: - } - var addressed struct { - ReplyTo string `json:"reply_to"` - } - if err := json.Unmarshal(msg.Data, &addressed); err != nil || addressed.ReplyTo == "" { - _ = msg.Ack() - return - } - body, _ := json.Marshal(reply) - // Published explicitly to the address the payload named, never msg.Respond — which would - // send it to whatever the transport's reply field holds, and that is the point. - _ = conn.Publish(addressed.ReplyTo, body) - _ = msg.Ack() - }, nats.Durable(durable), nats.ManualAck()) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { - _ = sub.Unsubscribe() - standinGone(t, js, stream, durable) - }) - return sawReplyField -} - -// standinName is the stand-in's durable name for one test: the test's name with every character a -// consumer name may not hold replaced. -func standinName(t *testing.T) string { - t.Helper() - return "controller-standin-" + strings.Map(func(r rune) rune { - if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' { - return r - } - return '_' - }, t.Name()) -} - -// standinGone deletes the durable consumer if it exists and waits until the server no longer reports -// it, so that nothing after this call can meet its binding. -func standinGone(t *testing.T, js nats.JetStreamContext, stream, durable string) { - t.Helper() - deadline := time.Now().Add(5 * time.Second) - for { - _, err := js.ConsumerInfo(stream, durable) - if errors.Is(err, nats.ErrConsumerNotFound) { - return - } - if errors.Is(err, nats.ErrConnectionClosed) { - t.Fatalf("the connection closed before consumer %s was deleted: %v", durable, err) - } - _ = js.DeleteConsumer(stream, durable) - if time.Now().After(deadline) { - t.Fatalf("consumer %s on %s still exists 5s after it was deleted", durable, stream) - } - time.Sleep(20 * time.Millisecond) - } -} - -// An enrolment is answered on the address the request carried, and the transport's own reply field -// held something else entirely. -func TestNatsAnEnrolmentIsAnsweredOnTheAddressInItsPayload(t *testing.T) { - conn, js := aBus(t) - const node = "joining" - - sawReplyField := theMeshAnswers(t, conn, js, EnrolReply{Accepted: true, Node: node, Password: "p"}) - a := asking(t, conn, js, node) - - request, _ := json.Marshal(EnrolRequest{Node: node, Secret: "t"}) - answer, err := a.Ask(context.Background(), request, 8*time.Second) - if err != nil { - t.Fatalf("no answer reached the node: %v", err) - } - var reply EnrolReply - if err := json.Unmarshal(answer, &reply); err != nil { - t.Fatal(err) - } - if !reply.Accepted || reply.Node != node { - t.Fatalf("the answer was not the mesh's: %+v", reply) - } - - // And the field the answer would have gone to, had it used the transport's: the consumer's own - // ack address. If a future server stopped doing this, the payload-borne address would still - // work and this line is what would say the reason had changed. - select { - case field := <-sawReplyField: - if field == a.inbox { - t.Fatalf("the transport's reply field held this node's inbox (%s), so the payload "+ - "address is no longer load-bearing — check design 25 §2 before relying on it", field) - } - if field == "" { - t.Fatal("the transport's reply field was empty rather than claimed, which is a third " + - "behaviour from the two design 25 §2 describes") - } - case <-time.After(2 * time.Second): - t.Fatal("the stand-in never saw the request") - } -} - -// A request that names no reply address is not answered, and the node says so as a mesh that is not -// running rather than hanging. The controller has nowhere to send an answer, which is the failure -// the payload field exists to make impossible — asserted so that a request built without it fails -// loudly here rather than quietly on a machine. -func TestNatsAnEnrolmentWithNoReplyAddressIsNotAnswered(t *testing.T) { - conn, js := aBus(t) - const node = "silent" - - theMeshAnswers(t, conn, js, EnrolReply{Accepted: true, Node: node}) - a := asking(t, conn, js, node) - - // Published without going through Ask, so the reply address is genuinely absent. - request, _ := json.Marshal(EnrolRequest{Node: node, Secret: "t"}) - if _, err := js.Publish(EnrolSubject, request); err != nil { - t.Fatal(err) - } - if _, err := a.Ask(context.Background(), []byte(`{"node":"`+node+`"}`), 0); err == nil { - t.Fatal("a node with no answer coming was told it had one") - } -} diff --git a/internal/link/bus.go b/internal/link/bus.go index dda5a4d..3ebc8fa 100644 --- a/internal/link/bus.go +++ b/internal/link/bus.go @@ -10,28 +10,14 @@ import ( "github.com/nats-io/nats.go" ) -// Bus is what a host needs of the mesh's bus, in the mesh's own words. -// -// A host says exactly two things unprompted: what it applied, and that it is here. They are not -// the same kind of statement and the difference is the whole of this interface — one must arrive -// and one must not be insisted on. -// -// **The host still imports nothing of the mesh's own** (novox/hq ADR 0005): this is its own -// interface over its own client libraries, not a contract shared with the controller. The two -// agree because a conformance fixture holds them to one envelope, which is the only kind of -// agreement that survives being in different repositories. -type Bus interface { - // Report says what this node applied. **It must arrive.** A report that fails leaves the - // mesh believing the node never answered while the node believes it did, and the two go on - // disagreeing with nothing anywhere saying so — the shape of fault this project keeps - // finding. Returns false when it could not be delivered, so the caller can say so. - Report(ctx context.Context, node string, body []byte) error -} +// Bus is the link open now as the session holds it. What it can do is asked of it by what it implements (HaveBus, +// HealthBus, ToolBus, FilesBus): the report it once published into the controller's CONTROL stream went with the +// controller (issue 1126), and what the node applied is said in its haves. +type Bus interface{} // --- The bus the mesh runs on today ----------------------------------------------------------- -// OverNATS is the bus as a connection. A report goes through JetStream because it must survive -// the controller's store restarting; a heartbeat does not, because it must not. +// OverNATS is the bus as a connection. type OverNATS struct { Conn *nats.Conn JS nats.JetStreamContext @@ -39,11 +25,6 @@ type OverNATS struct { Node string } -// ReportSubject is this node's own, and no other node's: a host's account may -// publish `mesh.control..>` and nothing wider, so the subject is the authority on -// which node a report is about. -func ReportSubject(node string) string { return "mesh.control." + node + ".report" } - // HealthSubject is where this node says its long-running resources' health between reports (novox/hq // ADR 0240): its own, inside the grant every host already has, and on core NATS like the heartbeat — a // statement lost is said again within a minute while anything is not healthy. @@ -90,17 +71,6 @@ func (b OverNATS) Ask(ctx context.Context, subject string, body []byte) ([]byte, return msg.Data, nil } -func (b OverNATS) Report(ctx context.Context, node string, body []byte) error { - // Into the CONTROL stream and awaited: this is the message the store-window guarantee is - // about (novox/hq ADR 0083). The controller naks with a delay while its store is away and - // the message is redelivered; a publish the bus never accepted must fail here rather than - // be assumed. - if _, err := b.JS.Publish(ReportSubject(node), body, nats.Context(ctx)); err != nil { - return fmt.Errorf("reporting: %w", err) - } - return nil -} - // HaveBind binds the have bucket the controller made (novox/hq ADR 0429 §4, issue 1132), once for a pass of writes. // **Only binds**: a node-engine's account reaches no part of the JetStream API beyond what it is granted, and the bucket // is the controller's to create and to configure, so a bucket that is not there is an error said as what it is. Every diff --git a/internal/link/enrol.go b/internal/link/enrol.go index f0a9d5c..da23fe2 100644 --- a/internal/link/enrol.go +++ b/internal/link/enrol.go @@ -1,70 +1,21 @@ package link import ( - "context" - "encoding/base64" - "encoding/json" "errors" - "fmt" "time" ) -// The wire format shared with the control plane, which defines it separately because this binary -// requires nothing present and does not import it. A test on each side asserts the field names. -const ( - Exchange = "mesh" - KeyEnrol = "enrol" -) +// What a joining machine keeps of the enrolment's words: the tunnel it found, as it travels, and how long it asks +// while the seat mesh-enrolment says to ask again (Redeem). The enrolment through the controller's subject on the +// CONTROL stream ended with the controller (issue 1126). -// QueueFor is the queue this node consumes from — the only one its account may read. -func QueueFor(node string) string { return "node." + node } - -// EnrolRequest is what this node says when joining. -type EnrolRequest struct { - Node string `json:"node"` - Secret string `json:"secret"` - PublicKey []byte `json:"public_key"` - - // OverlayKey is the public half of this node's key on the private network — a different key - // from PublicKey above, generated at the same moment and for a different purpose. - // - // Sent with enrolment because the overlay is the first declaration a node receives, and the - // mesh cannot compose it without this. Asking for it afterwards would mean a node is enrolled - // and unreachable for a round trip, which is the state everything else here works to avoid. - OverlayKey string `json:"overlay_key,omitempty"` - - // SealingKey is the public half of the key secrets are sealed to. A third key, and the - // reasoning is the same one twice over: the mesh must be able to send this node something - // nothing else can read, and it must never be able to read it either. - SealingKey string `json:"sealing_key,omitempty"` - - // ServingKey is the public half of the key this node serves TLS with on its internal name. - // The mesh signs a certificate binding it; the private half never leaves the machine, so - // there is nothing to seal and nothing that could be stolen from the mesh's copy. - ServingKey string `json:"serving_key,omitempty"` - - Profile map[string]any `json:"profile,omitempty"` - - // Proof is this node's identity key signing EnrolProof over this request: that the presenter - // holds the private half of PublicKey. The mesh asks for it before letting an enrolment finish - // on a token this key already spent (novox/hq issue 083). - Proof []byte `json:"proof,omitempty"` - - // ReplyTo is where the mesh's answer goes, as a field of the request rather than the - // transport's own reply address (design 25 §2). Written by the transport that needs it — - // withReplyTo, once per attempt — because a request going into a stream has had the transport's - // reply field claimed for the consumer's ack address before the controller ever reads it. - // - // Empty on the bus the mesh runs on today, where the delivery carries the reply queue and the - // field means what it has always meant. Named here so both sides of the wire hold the same - // field name, which is what the shape test on each side is for. - ReplyTo string `json:"reply_to,omitempty"` - - // Tunnel is the tunnel this node found and whose key it took as its overlay key (novox/hq ADR - // 0105): everything about it but that key. Sent with the keys because it is one of them — - // OverlayKey above IS this tunnel's public key when this is set — and the mesh composes the - // hub's address, the range and the carried peers from it before the first declaration. - Tunnel *Tunnel `json:"tunnel,omitempty"` +// Approach is how a node reaches a mesh it does not yet belong to: the address, the certificate that address must +// present, and which bus is at the other end, the mesh's own (hearing.go); a token naming any other is refused before +// anything is sent. +type Approach struct { + Address string + Fingerprint string + Transport string } // Tunnel is a found tunnel as it travels: no private key. @@ -86,36 +37,6 @@ type TunnelPeer struct { Address string `json:"address"` } -// EnrolReply is what the mesh says back. -type EnrolReply struct { - Accepted bool `json:"accepted"` - - // TryAgain is the mesh saying it cannot answer right now — its store is restarting, or the - // token is held for a moment by another enrolment — and that nothing was spent. The same - // request is asked again (novox/hq issue 083). - TryAgain bool `json:"try_again,omitempty"` - - Node string `json:"node,omitempty"` - Queue string `json:"queue,omitempty"` - - // What this node keeps so it can come back on its own. Without these a restart would need a - // person with a new token, which would make disconnection a crisis rather than the ordinary - // situation novox/hq ADR 0004 says it is. - Password string `json:"password,omitempty"` - Broker string `json:"broker,omitempty"` - Fingerprint string `json:"fingerprint,omitempty"` - Signer []byte `json:"signer,omitempty"` - - Refusal string `json:"refusal,omitempty"` -} - -// EnrolProof is what a node signs with its identity key when it enrols: the token and every key it -// presents, so a proof cannot be moved to another request. The mesh builds the same bytes. -func EnrolProof(secret string, public []byte, overlay, sealing, serving string) []byte { - return []byte("novox-mesh-enrol\x00" + secret + "\x00" + base64.StdEncoding.EncodeToString(public) + - "\x00" + overlay + "\x00" + sealing + "\x00" + serving) -} - // ErrRefused is what a node gets when the mesh will not have it. var ErrRefused = errors.New("the mesh refused this enrolment") @@ -126,94 +47,3 @@ const EnrolPatience = 2 * time.Minute // AskAgainAfter is the pause between asks while the mesh says "try again". const AskAgainAfter = 3 * time.Second - -// ErrNotNow is a mesh that said "try again" for longer than this node would keep asking. -var ErrNotNow = errors.New("the mesh could not answer this enrolment") - -// answered decides what one reply means: done, ask again, or stop with an error. Separate from the -// broker so it can be held to that by a test. -func answered(reply EnrolReply, asking time.Duration) (again bool, err error) { - switch { - case reply.Accepted: - return false, nil - case reply.TryAgain && asking < EnrolPatience: - return true, nil - case reply.TryAgain: - // Said with what to do. The mesh holds the token for this attempt's keys for as long as - // this node kept asking, so a new attempt — with keys of its own — waits that out first. - return false, fmt.Errorf("%w for %s: %s. The token was not spent: wait about %s and run "+ - "enrol again with it; if it is then refused, issue a new one", - ErrNotNow, EnrolPatience, reply.Refusal, EnrolPatience) - default: - return false, fmt.Errorf("%w: %s", ErrRefused, reply.Refusal) - } -} - -// Enrol presents this node's key and its one-time secret, and waits to be told it is known. -// -// The broker has already authenticated this connection: the account was created when the token -// was issued and the secret is its password. So this is not how the node gets in — it is what it -// says once it is in, and the secret travels again because the control plane must not have to ask -// the broker who connected. -func Enrol(ctx context.Context, to Approach, node, secret string, public []byte, - overlayKey, sealingKey, servingKey string, profile map[string]any, proof []byte, - tunnel *Tunnel, timeout time.Duration) (EnrolReply, error) { - - asking, err := Present(ctx, to, node, secret, timeout) - if err != nil { - return EnrolReply{}, err - } - defer asking.Close() - - request := EnrolRequest{Node: node, Secret: secret, PublicKey: public, - OverlayKey: overlayKey, SealingKey: sealingKey, ServingKey: servingKey, Profile: profile, - Proof: proof, Tunnel: tunnel} - body, err := json.Marshal(request) - if err != nil { - return EnrolReply{}, err - } - - // Asked, and asked again with the same request while the mesh says "try again": the keys this - // node generated are the ones it keeps, so the same request is the same enrolment, and the mesh - // holds the token for it (novox/hq issue 083). - began := time.Now() - for { - answer, err := asking.Ask(ctx, body, timeout) - if err != nil { - return EnrolReply{}, err - } - var reply EnrolReply - if err := json.Unmarshal(answer, &reply); err != nil { - return EnrolReply{}, fmt.Errorf("the mesh's answer could not be read: %w", err) - } - again, err := answered(reply, time.Since(began)) - if err != nil { - return reply, err - } - if !again { - return reply, nil - } - select { - case <-ctx.Done(): - return EnrolReply{}, ctx.Err() - case <-time.After(AskAgainAfter): - } - } -} - -// withReplyTo writes this attempt's reply address into the request, as a field of its own. -// -// **Written into the bytes rather than carried beside them**, because the whole point is that the -// address survives a stream: a JetStream consumer's delivery has had the transport's reply field -// claimed for its own ack address, so a reply address that is not in the payload is one the -// controller cannot read (design 25 §2). Done by decoding and re-encoding rather than by setting the -// field before marshalling, so one request can be asked again with a fresh address each time without -// the caller knowing that is what happens. -func withReplyTo(request []byte, inbox string) ([]byte, error) { - var fields map[string]any - if err := json.Unmarshal(request, &fields); err != nil { - return nil, fmt.Errorf("this node's own enrolment request cannot be read back: %w", err) - } - fields["reply_to"] = inbox - return json.Marshal(fields) -} diff --git a/internal/link/enrol_again_test.go b/internal/link/enrol_again_test.go deleted file mode 100644 index 7ce0b6f..0000000 --- a/internal/link/enrol_again_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package link - -import ( - "errors" - "strings" - "testing" - "time" -) - -// What one reply means to a node enrolling (novox/hq issue 083): accepted is done; "try again" is -// asked again, until the node's patience runs out and it says the token was not spent; anything -// else is a refusal. -func TestAnEnrollingNodeAsksAgainWhileTheMeshSaysNotNow(t *testing.T) { - if again, err := answered(EnrolReply{Accepted: true}, 0); again || err != nil { - t.Fatalf("an accepted enrolment was not done: again=%v err=%v", again, err) - } - if again, err := answered(EnrolReply{TryAgain: true}, time.Second); !again || err != nil { - t.Fatalf("a mesh saying not now was not asked again: again=%v err=%v", again, err) - } - again, err := answered(EnrolReply{TryAgain: true, Refusal: "restarting"}, EnrolPatience) - if again || !errors.Is(err, ErrNotNow) { - t.Fatalf("a mesh saying not now past the node's patience did not stop it: again=%v err=%v", again, err) - } - if !errorsMention(err, "not spent") { - t.Errorf("giving up did not say the token can be used again: %v", err) - } - if again, err := answered(EnrolReply{Refusal: "that token cannot be used"}, 0); again || !errors.Is(err, ErrRefused) { - t.Fatalf("a refusal was not a refusal: again=%v err=%v", again, err) - } -} - -func errorsMention(err error, what string) bool { - return err != nil && strings.Contains(err.Error(), what) -} diff --git a/internal/link/enrol_proof_test.go b/internal/link/enrol_proof_test.go deleted file mode 100644 index 6d6d47b..0000000 --- a/internal/link/enrol_proof_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package link - -import "testing" - -// The bytes a node signs when it enrols. The mesh builds the same bytes to check the signature, in -// another repository; this known answer is repeated in its test, so the two cannot drift apart -// without one of them failing (novox/hq issue 083). -func TestWhatAnEnrollingNodeSignsIsFixed(t *testing.T) { - got := string(EnrolProof("s", []byte{1, 2, 3}, "o", "e", "v")) - if want := "novox-mesh-enrol\x00s\x00AQID\x00o\x00e\x00v"; got != want { - t.Fatalf("the enrolment proof's message changed: %q, want %q", got, want) - } -} diff --git a/internal/link/enrol_seat.go b/internal/link/enrol_seat.go index c5d7eb7..7c4d6c4 100644 --- a/internal/link/enrol_seat.go +++ b/internal/link/enrol_seat.go @@ -33,8 +33,8 @@ func CalledSeatSubject(seat, verb, user string) string { return "mesh.seat." + seat + ".call." + verb + "." + strings.ReplaceAll(user, ".", "~") } -// ErrNoSeat is the bus saying nothing serves the seat for this caller: the enrolment user may not publish the seat's call (the -// controller still holds its lease) or no holder answers it. +// ErrNoSeat is the bus saying nothing serves the seat for this caller: the enrolment user may not publish the seat's call, or +// no holder answers it. var ErrNoSeat = errors.New("the seat mesh-enrolment is not served to this machine's token") // VerifyRedeemAnswer holds an answer to what it must be before anything is kept (ADR 0587 §6): it names the node asked for, @@ -80,8 +80,8 @@ func seatAnswer(reply []byte) (json.RawMessage, error) { } // Redeem calls the seat's redeem as enrol., with the token's secret as the bus password, on the pinned certificate. It -// asks again with the same keys while the holder says to (a stopped redeem finishes; the bus node loads the account), for -// as long as a machine asking the controller would (EnrolPatience). An answer is returned as the holder gave it: the caller +// asks again with the same keys while the holder says to (a stopped redeem finishes; the bus node loads the account), , for +// EnrolPatience. An answer is returned as the holder gave it: the caller // verifies it (VerifyRedeemAnswer) before it keeps anything. func Redeem(ctx context.Context, to Approach, node, secret string, req delivery.Redeem, timeout time.Duration) (delivery.RedeemAnswer, error) { if to.Transport != OnNATS { diff --git a/internal/link/enrol_shape_test.go b/internal/link/enrol_shape_test.go deleted file mode 100644 index 7b53f95..0000000 --- a/internal/link/enrol_shape_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package link - -import ( - "encoding/json" - "os" - "testing" - - "github.com/novox/mesh-host/internal/identity" -) - -// What this node says when it joins, written out so the mesh's own suite can accept it. -// -// The two ends are separate structs in separate repositories. Every field here is one somebody -// could rename on one side, and the failure would be silent: enrolment succeeds, a key is simply -// absent, and the node looks joined until the first thing sealed to it cannot be opened. That is -// exactly the shape of fault this project keeps finding late. -// -// Skipped unless MESH_ENROL_OUT names a file, so this is a check somebody runs deliberately -// rather than a dependency between two repositories. -func TestWhatThisNodeSaysWhenItJoins(t *testing.T) { - path := os.Getenv("MESH_ENROL_OUT") - if path == "" { - t.Skip("set MESH_ENROL_OUT to write the enrolment request the mesh's suite reads") - } - - // A real one. Generated the way enrolment generates them rather than typed as literals, so a - // key that stopped being a key would be caught here rather than travelling. - mine, err := identity.Generate("workstation") - if err != nil { - t.Fatal(err) - } - overlay, err := identity.GenerateOverlayKey() - if err != nil { - t.Fatal(err) - } - sealing, err := identity.GenerateSealingKey() - if err != nil { - t.Fatal(err) - } - serving, err := identity.GenerateServingKey() - if err != nil { - t.Fatal(err) - } - - request := EnrolRequest{ - Node: "workstation", - Secret: "a-one-time-secret", - PublicKey: mine.Public, - OverlayKey: overlay.Public, - SealingKey: sealing.Public, - ServingKey: serving.Public, - Profile: map[string]any{"seat": true}, - } - body, err := json.MarshalIndent(request, "", " ") - if err != nil { - t.Fatal(err) - } - if err := os.WriteFile(path, append(body, '\n'), 0o644); err != nil { - t.Fatal(err) - } - // The private half of the sealing key goes beside it, so the mesh's suite can prove what it - // sealed is openable rather than merely present. - if err := os.WriteFile(path+".sealing-private", []byte(sealing.Private), 0o600); err != nil { - t.Fatal(err) - } - t.Logf("wrote %s", path) -} diff --git a/internal/link/have_test.go b/internal/link/have_test.go index 86f5a95..f092ed2 100644 --- a/internal/link/have_test.go +++ b/internal/link/have_test.go @@ -24,7 +24,7 @@ type haveLink struct { hmu sync.Mutex kv map[string]haveEntry sets []string // keys written, in order - events []string // "report" and "have ", in the order they happened + events []string // "have ", in the order they happened failing bool binds int setTries int @@ -40,16 +40,6 @@ type haveEntry struct { func newHaveLink() *haveLink { return &haveLink{quietLink: newQuietLink(), kv: map[string]haveEntry{}} } -func (l *haveLink) Report(ctx context.Context, node string, body []byte) error { - err := l.quietLink.Report(ctx, node, body) - if err == nil { - l.hmu.Lock() - l.events = append(l.events, "report") - l.hmu.Unlock() - } - return err -} - func (l *haveLink) Health(context.Context, string, []byte) error { return nil } func (l *haveLink) HaveBind(ctx context.Context) (HaveBucket, error) { @@ -241,9 +231,8 @@ func TestAModulesHealthFollowsTheLook(t *testing.T) { } } -// 1132/AC6: the report is sent first and carries no new key; the node's key holds its windows, busy, strays, filters -// and profile. -func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) { +// 1132/AC6: the node's key holds its windows, busy, strays, filters and profile. +func TestTheNodeKeyCarriesTheNodesFacts(t *testing.T) { l := newHaveLink() q, _ := aHaveQueue(l) rep := appliedModules() @@ -253,13 +242,6 @@ func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) { rep.Filters = []Filter{{Where: "inet mesh", Owner: "mesh"}} rep.Profile = map[string]any{"cores": 8} q.SayFacts(context.Background(), rep) - if len(l.events) < 2 || l.events[0] != "report" { - t.Fatalf("the report was not first: %v", l.events) - } - raw, _ := json.Marshal(l.said()[0]) - if strings.Contains(string(raw), "Modules") || strings.Contains(string(raw), "\"modules\"") { - t.Errorf("the report carries the module facts: %s", raw) - } got, err := delivery.DecodeHaveNode(l.kv["n1.node"].value) if err != nil { t.Fatal(err) @@ -310,16 +292,13 @@ func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) { } } -// 1132/AC7: a write the bucket refuses is said in the log; the report went out, and the next apply writes again. -func TestAFailedHaveWriteIsSaidAndNeverBlocksTheReport(t *testing.T) { +// 1132/AC7: a write the bucket refuses is said in the log, and the next apply writes again. +func TestAFailedHaveWriteIsSaidAndWrittenAgain(t *testing.T) { l := newHaveLink() l.failing = true q, said := aHaveQueue(l) if !q.SayFacts(context.Background(), appliedModules()) { - t.Fatal("the report did not go out") - } - if len(l.said()) != 1 { - t.Fatalf("reports sent: %d", len(l.said())) + t.Fatal("the facts were not said with a link open") } if !strings.Contains(strings.Join(*said, "\n"), "n1.a") || !strings.Contains(strings.Join(*said, "\n"), "refused") { t.Errorf("the failure is not in the log: %v", *said) diff --git a/internal/link/hearing_nats.go b/internal/link/hearing_nats.go index 960fd20..93209a9 100644 --- a/internal/link/hearing_nats.go +++ b/internal/link/hearing_nats.go @@ -17,11 +17,6 @@ import ( // The host's link on the bus: one connection, and what is asked of this node over it. The node hears no declaration // (novox/hq ADR 0539): it watches its wants and its compositions, and says its facts. -// EnrolSubject is where a joining machine asks. One subject for every node, because a machine -// enrolling has no name the mesh has agreed to yet — which is why its authority to publish here is -// the whole of what its enrolment user may do. -const EnrolSubject = "mesh.control.enrol" - // AskReportSubject is where the mesh asks this node to say its facts again: its own, on core NATS and off any stream. // The answer is the facts event on its report subject — nobody's inbox is answered. func AskReportSubject(node string) string { return "mesh.node." + node + ".ask.report" } @@ -311,10 +306,6 @@ func (l *natsLink) Close() { } } -func (l *natsLink) Report(ctx context.Context, node string, body []byte) error { - return OverNATS{Conn: l.conn, JS: l.js}.Report(ctx, node, body) -} - func (l *natsLink) Health(ctx context.Context, node string, body []byte) error { return OverNATS{Conn: l.conn, JS: l.js}.Health(ctx, node, body) } diff --git a/internal/link/messages.go b/internal/link/messages.go index f0dd7a2..3f86def 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -1,8 +1,6 @@ package link import ( - "strconv" - "strings" "time" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -47,13 +45,6 @@ type Report struct { // machine saying what is true of it, not asking for anything. Carried []int `json:"carried,omitempty"` - // Writer, and Order — the node-engine's own epoch and sequence — say who wrote these facts and where they stand - // among everything it wrote (novox/hq ADR 0227 §2, ADR 0539 §9): the epoch is one per start of the node-engine, - // the sequence one higher for every facts event, so the controller refuses older facts by this order. On the - // wire as three top-level keys. - Writer string `json:"writer,omitempty"` - Order - // Held is what this adopted node found and is keeping as it was until its module is taken // (novox/hq ADR 0100). Without it an adopted node reads as converged. Held []Held `json:"held,omitempty"` @@ -142,10 +133,6 @@ type Report struct { // that does not load is a machine filtering nothing while its unit reports success. Outward []string `json:"outward,omitempty"` - // Rekey is this node taking a found tunnel's key as its overlay key after enrolment (novox/hq - // ADR 0105). Not an account of the machine: a report carrying one says nothing else. - Rekey *Rekey `json:"rekey,omitempty"` - // Rollbacks is what this machine's witnesses decided about a core build that was not healthy // in bound (novox/hq to-be 45 §8, ADR 0227 rule 8): the node-engine's launcher about the // engine, the engine about the controller and the node tools. **Said on every report while it @@ -530,26 +517,6 @@ type Rollback struct { At time.Time `json:"at"` } -// Order is where a writer's saying stands among everything it said (novox/hq ADR 0227 §2): its **epoch**, and its -// **sequence** within it. A node-engine's epoch is the time it started, so a later start orders after an earlier one -// with nothing kept on disk. -type Order struct { - Epoch int64 `json:"epoch,omitempty"` - Sequence int64 `json:"sequence,omitempty"` -} - -// Words is an order as a person reads it in a log line. Not String: Report embeds Order, and a -// Stringer promoted onto every report would print each one as its order alone. -func (o Order) Words() string { - switch { - case o.Epoch > 0: - return "epoch " + strconv.FormatInt(o.Epoch, 10) + ", sequence " + strconv.FormatInt(o.Sequence, 10) - case o.Sequence > 0: - return "sequence " + strconv.FormatInt(o.Sequence, 10) + ", no epoch" - } - return "no order" -} - // CarriedTunnel is this node's account of the tunnel it took over. State is one of the Carried // states below; Note is what the host did about it, when it did something. type CarriedTunnel struct { @@ -572,36 +539,6 @@ const ( CarriedDown = "down" ) -// Rekey is this node saying it took a found tunnel's key as its overlay key after enrolling -// (novox/hq ADR 0105): the path for a node that enrolled before the mesh knew to take a tunnel -// over, since re-enrolling would rotate every key it holds. Signed with the identity key over -// RekeyProof, so a report forged on a stolen broker account cannot move this node's overlay key. -type Rekey struct { - // Previous is the overlay key this node held until now, as the mesh records it; the mesh - // refuses a rekey naming another, which is how a replayed one is refused. - Previous string `json:"previous"` - OverlayKey string `json:"overlay_key"` - Tunnel *Tunnel `json:"tunnel"` - Proof []byte `json:"proof"` -} - -// RekeyProof is what a node signs when it rekeys — the node, the key it leaves, the key it takes -// and the tunnel it took it from — so a proof cannot be moved to another node or another tunnel. -// Byte for byte the mesh's own (mesh-controller internal/link RekeyProof). -func RekeyProof(node, previous, key string, tunnel *Tunnel) []byte { - var t Tunnel - if tunnel != nil { - t = *tunnel - } - peers := make([]string, 0, len(t.Peers)) - for _, p := range t.Peers { - peers = append(peers, p.PublicKey+"@"+p.Address) - } - return []byte("novox-mesh-rekey\x00" + node + "\x00" + previous + "\x00" + key + "\x00" + - t.Interface + "\x00" + t.Unit + "\x00" + t.Config + "\x00" + strconv.Itoa(t.Port) + "\x00" + - t.Address + "\x00" + t.Range + "\x00" + t.PublicKey + "\x00" + strings.Join(peers, ",")) -} - // Held is one file or container found on an adopted node and kept as it was. type Held struct { ID string `json:"id"` diff --git a/internal/link/messages_test.go b/internal/link/messages_test.go index 5bc7c29..7ea51d2 100644 --- a/internal/link/messages_test.go +++ b/internal/link/messages_test.go @@ -12,9 +12,6 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) { value any expect []string }{ - // novox/hq ADR 0539 §9: the node's facts carry the node-engine's own writer, epoch and sequence. - {Report{Node: "n", Writer: Writer, Order: Order{Epoch: 1, Sequence: 2}}, - []string{"node", "writer", "epoch", "sequence"}}, {Report{Node: "n", Applied: []string{"a"}, Failed: map[string]string{"k": "v"}, Refused: "r"}, []string{"node", "applied", "failed", "refused"}}, // novox/hq ADR 0100: what an adopted node holds, the firewall it was found with, and what diff --git a/internal/link/run.go b/internal/link/run.go index 825373d..2e8b08d 100644 --- a/internal/link/run.go +++ b/internal/link/run.go @@ -3,7 +3,6 @@ package link import ( "context" "crypto/ed25519" - "encoding/json" "errors" "fmt" "time" @@ -284,41 +283,3 @@ func serve(ctx context.Context, link Link, m Membership, queue *Session, say Ann } } } - -// publishReport tells the mesh what this node did, and says whether the broker took it. -// Publish sends one report on this node's own connection and returns: the one-shot path for a -// report a command makes rather than the running host — a rekey (novox/hq ADR 0105). The same -// account, the same pinned certificate and the same exchange as the running host's reports. -func Publish(ctx context.Context, m Membership, report Report, timeout time.Duration) error { - link, err := Open(ctx, m, timeout) - if err != nil { - return err - } - defer link.Close() - var said string - if !publishReport(ctx, link, m, report, func(s string) { said = s }, timeout) { - return errors.New(said) - } - return nil -} - -func publishReport(ctx context.Context, bus Bus, m Membership, report Report, - say Announce, timeout time.Duration) bool { - report.Node = m.Node - body, err := json.Marshal(report) - if err != nil { - say("cannot encode this node's own report: " + err.Error()) - return false - } - publish, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - // Said rather than swallowed. A report that fails to publish leaves the mesh believing this - // node never answered, while the node believes it did — and the two would go on disagreeing - // with nothing anywhere saying so. That shape of fault is the one this project keeps finding. - if err := bus.Report(publish, m.Node, body); err != nil { - say(fmt.Sprintf("applied, and could not tell the mesh: %v", err)) - return false - } - return true -} diff --git a/internal/link/session.go b/internal/link/session.go index 882e03d..471f6a6 100644 --- a/internal/link/session.go +++ b/internal/link/session.go @@ -13,16 +13,11 @@ import ( "git.novox.be/novox/mesh-sdk/go/health" ) -// Writer is what the node-engine calls itself on the facts it says (novox/hq ADR 0227 §2, ADR 0539 §9): the program -// that wrote them, beside its epoch and sequence. -const Writer = "node-engine" - -// Session is the node-engine's hold on the bus (novox/hq ADR 0539 §9): the link open now, what the controller asks -// of this machine over it, and what the machine says on it unasked: its health, its haves and its facts. +// Session is the node-engine's hold on the bus (novox/hq ADR 0539 §9): the link open now, what is asked of this +// machine over it, and what the machine says on it unasked: its health and its haves. // -// **It applies nothing and orders nothing.** A module is applied by the node agent from its want, and the facts are an -// event of the node, ordered by the node-engine's own epoch and sequence. Nothing here receives, orders or reconciles -// a whole declaration. +// **It applies nothing and orders nothing.** A module is applied by the node agent from its want. Nothing here +// receives, orders or reconciles a whole declaration. type Session struct { // Fallback is the engine's fall back to its old login its health report carries now (Rotator.Fallback); nil says none. Fallback func() *delivery.LoginFallback @@ -84,12 +79,8 @@ type Session struct { haveAgainPending bool haveAgainTries int - // saying is held while a facts event is made and said, so they leave in the order they are made. + // saying is held while a facts event is noted, so the haves are written in the order the facts are made. saying sync.Mutex - // epoch is this process's own: the time it started, in milliseconds, so a later process orders after an earlier one - // without anything kept on disk; sequence is one higher for every facts event it says (ADR 0227 §2). - epoch int64 - sequence int64 // haves is what this node-engine writes beside its facts (novox/hq ADR 0429 §4, issue 1132). haves haveState @@ -133,7 +124,6 @@ func (q *Session) init() { if q.Say == nil { q.Say = func(string) {} } - q.epoch = time.Now().UnixMilli() q.changed = make(chan struct{}) q.owed = make(chan struct{}, 1) }) @@ -156,8 +146,10 @@ func (q *Session) oweHealth() { } } -// SayFacts says this node's facts on the link open now, in the node-engine's own order, and whether the bus took them. -// A facts event made with no link open is not kept: the next link says the facts as they are then. +// SayFacts says this node's facts in its haves on the link open now, in the order they are made, and whether a link +// was open to say them on. A facts event made with no link open is not kept: the next link says the facts as they are +// then. They are said in the haves alone: the CONTROL stream they were also published on was the controller's, which +// ended by its unassignment (issue 1126). A write the bucket refuses is said and written again (issue 1132). func (q *Session) SayFacts(ctx context.Context, r Report) bool { q.init() q.saying.Lock() @@ -168,14 +160,7 @@ func (q *Session) SayFacts(ctx context.Context, r Report) bool { if bus == nil { return false } - q.sequence++ r.Node = q.Membership.Node - r.Writer = Writer - r.Order = Order{Epoch: q.epoch, Sequence: q.sequence} - if !publishReport(context.WithoutCancel(ctx), bus, q.Membership, r, q.Say, q.Timeout) { - return false - } - // After the facts, which are sent first and unchanged; a write the bucket refuses never undoes them (issue 1132). if q.noteApply(r) { q.writeHaves(ctx, bus) } diff --git a/internal/link/session_test.go b/internal/link/session_test.go index 331efb0..7fc1fdd 100644 --- a/internal/link/session_test.go +++ b/internal/link/session_test.go @@ -3,7 +3,6 @@ package link import ( "context" "crypto/ed25519" - "encoding/json" "errors" "fmt" "os" @@ -15,42 +14,16 @@ import ( "github.com/nats-io/nats.go" ) -// quietLink is a link with no broker behind it. A report is refused once the context it is published on is cancelled, -// as the bus's is ("reporting: context canceled"), or while refusing. +// quietLink is a link with no broker behind it. type quietLink struct { - mu sync.Mutex - reports []Report - refusing bool - lost chan error + lost chan error } func newQuietLink() *quietLink { return &quietLink{lost: make(chan error, 1)} } -func (l *quietLink) Report(ctx context.Context, _ string, body []byte) error { - if err := ctx.Err(); err != nil { - return err - } - l.mu.Lock() - defer l.mu.Unlock() - if l.refusing { - return errors.New("refused") - } - var r Report - if err := json.Unmarshal(body, &r); err != nil { - return err - } - l.reports = append(l.reports, r) - return nil -} func (l *quietLink) Lost() <-chan error { return l.lost } func (l *quietLink) Close() {} -func (l *quietLink) said() []Report { - l.mu.Lock() - defer l.mu.Unlock() - return append([]Report(nil), l.reports...) -} - func aMember(t *testing.T) (Membership, ed25519.PrivateKey) { t.Helper() public, private, err := ed25519.GenerateKey(nil) @@ -60,40 +33,6 @@ func aMember(t *testing.T) (Membership, ed25519.PrivateKey) { return Membership{Node: "n1", Signer: public}, private } -// R6-A AC7: the facts event carries the node-engine's own writer, epoch and sequence (novox/hq ADR 0227 §2, ADR 0539 -// §9): one epoch per start, a sequence one higher for every event, so the controller can refuse older facts. -func TestTheFactsEventCarriesTheNodeEnginesWriterEpochAndSequence(t *testing.T) { - m, _ := aMember(t) - l := newQuietLink() - s := &Session{Membership: m, Timeout: time.Second} - s.Attach(context.Background(), l) - - if !s.SayFacts(context.Background(), Report{Firewall: "none"}) || !s.SayFacts(context.Background(), Report{}) { - t.Fatal("the facts were not taken") - } - got := l.said() - if len(got) != 2 { - t.Fatalf("%d facts events were said", len(got)) - } - for _, r := range got { - if r.Writer != Writer || r.Node != "n1" || r.Epoch <= 0 { - t.Fatalf("the facts do not say who wrote them: %+v", r) - } - } - if got[0].Epoch != got[1].Epoch || got[1].Sequence != got[0].Sequence+1 || got[0].Sequence <= 0 { - t.Fatalf("the order is not one epoch with a rising sequence: %+v then %+v", got[0].Order, got[1].Order) - } - - // A later start orders after this one with nothing kept on disk. - time.Sleep(2 * time.Millisecond) - later := &Session{Membership: m, Timeout: time.Second} - later.Attach(context.Background(), l) - later.SayFacts(context.Background(), Report{}) - if last := l.said()[2]; last.Epoch <= got[1].Epoch { - t.Fatalf("a later start has epoch %d, not above %d", last.Epoch, got[1].Epoch) - } -} - // Facts made with no link open are not kept, and nothing is said: the next link says them as they are then. func TestFactsWithNoLinkOpenAreNotSaid(t *testing.T) { m, _ := aMember(t) @@ -104,7 +43,7 @@ func TestFactsWithNoLinkOpenAreNotSaid(t *testing.T) { } s.Attach(context.Background(), l) s.Detach(l) - if s.Linked() || s.SayFacts(context.Background(), Report{}) || len(l.said()) != 0 { + if s.Linked() || s.SayFacts(context.Background(), Report{}) { t.Fatal("facts were said on a link that was let go") } } @@ -140,16 +79,6 @@ func aBus(t *testing.T) (*nats.Conn, nats.JetStreamContext) { if err != nil { t.Fatal(err) } - want := &nats.StreamConfig{Name: "CONTROL", Subjects: []string{"mesh.control.*.report", "mesh.control.enrol"}, - Retention: nats.WorkQueuePolicy} - if _, err := js.StreamInfo(want.Name); err != nil { - if _, err := js.AddStream(want); err != nil { - t.Fatal(err) - } - } - if err := js.PurgeStream(want.Name); err != nil { - t.Fatal(err) - } return conn, js }