The mesh certifies names inside it

08-connectivity keeps two authorities apart on purpose: a public one for
names the outside world reaches, and the mesh's own for names only the
mesh knows. Nothing implemented the second, so anything between machines
was plaintext or trust-on-first-use — which the design refuses everywhere
else.

A node now generates a fourth key at enrolment and reports the public
half. A fourth, because a key used for two purposes is one rotation away
from breaking the other: the identity key signs messages to the mesh and
would do for TLS, and reusing it would mean rotating a node's identity
every time its certificate is replaced.

**Nothing secret travels and nothing is sealed.** A certificate authority
says "this name belongs to the holder of this key", so the mesh signs a
public half it cannot use, and the certificate it issues is public. A
module asks for one and is given the certificate and, if it wants,
the mesh's own — the private key is a path to a file the machine already
has, the same arrangement the private network's key uses.

Asserted by verifying rather than inspecting, because a certificate that
parses and does not chain fails at the moment something connects:

- what the mesh issues verifies against the mesh, for the name asked for
- the name is in the subject alternative names, since a certificate
  carrying it only in the common name is refused by every modern client
- it certifies the key the node generated and no other
- another mesh's certificate does not verify, which is the whole point of
  two authorities being separate
- the authority cannot sign another authority — one that could is one
  that can be delegated without anybody deciding to
- two control planes starting together agree on one authority, or a mesh
  has certificates half its machines refuse

Certificates last ten years, which is a choice: a short life needs
something to renew it, and a renewal that fails silently is a mesh that
stops trusting itself on a date nobody wrote down. What makes one
replaceable is that the mesh reissues on demand, not that it expires.

internal/catalogue/declaration.go

@@ -44,6 +44,11 @@ type Grant struct {
 
 // Rendering is everything needed to turn a resolution into the declaration a node is sent.
 type Rendering struct {
+	// Certificate is what the mesh issued for this machine's internal name, and the mesh's own
+	// certificate. Both public — the key they belong to never left the machine.
+	Certificate string
+	Authority   string
+
 	// Needed is each module's own secrets, sealed to this node, keyed by module and then by the
 	// name the module gave it.
 	Needed map[string]map[string]string
@@ -78,6 +83,26 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) {
 	var out []map[string]any
 	for _, m := range r.Modules {
 		resources := m.Resources
+		if c := m.Certificate; c != nil {
+			if with.Certificate == "" {
+				// Asked for and not issued. Refused rather than skipped: a module that serves TLS
+				// with no certificate does not start, and the reason is somewhere else entirely.
+				return nil, fmt.Errorf(
+					"%s wants a certificate for this machine and none was issued", m.Module)
+			}
+			resources = append(append([]map[string]any{}, resources...), map[string]any{
+				"id": CertificateID(), "type": "file", "path": c.Into,
+				// Public. It travels in the open like any other file, because it is a statement
+				// about a key rather than the key.
+				"content": with.Certificate, "mode": "0644",
+			})
+			if c.Authority != "" {
+				resources = append(resources, map[string]any{
+					"id": AuthorityID(), "type": "file", "path": c.Authority,
+					"content": with.Authority, "mode": "0644",
+				})
+			}
+		}
 		for _, name := range sortedKeys(m.Needs) {
 			sealed := with.Needed[m.Module][name]
 			if sealed == "" {

internal/catalogue/manifest.go

@@ -212,6 +212,18 @@ type Manifest struct {
 	// module, in a file anybody can read, for ever.
 	Needs map[string]string `json:"needs,omitempty"`
 
+	// Certificate is where this module wants a certificate for its machine's name inside the
+	// mesh, and where the key that goes with it can be found.
+	//
+	// **The key is named, not delivered.** The node generated it at enrolment and keeps it; the
+	// mesh only ever signs the public half. So what arrives is a certificate, which is public,
+	// and a path to a file the machine already has.
+	//
+	// Two authorities are kept apart on purpose (novox/hq 08-connectivity): this is the mesh's,
+	// for names only the mesh knows. A name the outside world reaches is a different authority
+	// and a different problem.
+	Certificate *Certificate `json:"certificate,omitempty"`
+
 	// Grants is a directory this module wants the credentials of its consumers written into, per
 	// provision it offers — one file per consumer, named for it, holding the value alone.
 	//
@@ -262,6 +274,19 @@ const (
 	ArtifactUpstream = "upstream"
 )
 
+// Certificate says where a module wants what the mesh issued for its machine.
+type Certificate struct {
+	// Into is where the certificate is written.
+	Into string `json:"into"`
+	// Authority is where the mesh's own certificate is written, so something connecting to this
+	// machine can be told what to believe. Optional: a module that only serves does not need it.
+	Authority string `json:"authority,omitempty"`
+}
+
+// CertificateID and AuthorityID are the resource identities of what the mesh issued.
+func CertificateID() string { return "certificate" }
+func AuthorityID() string   { return "certificate-authority" }
+
 // NeedID is the resource identity of the file a module's own secret lands in.
 func NeedID(name string) string { return "needs-" + name }
 
@@ -404,6 +429,17 @@ func ParseManifest(raw []byte) (Manifest, error) {
 				"%s binds %q and does not require it", m.Module, to))
 		}
 	}
+	if c := m.Certificate; c != nil {
+		if !strings.HasPrefix(c.Into, "/") {
+			problems = append(problems, fmt.Sprintf(
+				"%s wants its certificate at %q, which is not an absolute path", m.Module, c.Into))
+		}
+		if c.Authority != "" && !strings.HasPrefix(c.Authority, "/") {
+			problems = append(problems, fmt.Sprintf(
+				"%s wants the authority at %q, which is not an absolute path",
+				m.Module, c.Authority))
+		}
+	}
 	for name, where := range m.Needs {
 		if !strings.HasPrefix(where, "/") {
 			problems = append(problems, fmt.Sprintf(

internal/identity/authority.go

@@ -0,0 +1,186 @@
+package identity
+
+import (
+	"context"
+	"crypto/ed25519"
+	"crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/base64"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"math/big"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+)
+
+// The authority that certifies names inside the mesh.
+//
+// novox/hq 08-connectivity keeps two authorities apart on purpose: a public one issues for names
+// the outside world reaches, and this one for names only the mesh knows. **It certifies a public
+// key a node generated**, which is the whole of what a certificate authority does — so nothing
+// secret travels, nothing is sealed, and a copy of this context's store certifies nothing it did
+// not already certify.
+//
+// It is not a bootstrap concern. A joining node verifies the control plane against the fingerprint
+// in its token (ADR 0004), so nothing needs this before membership.
+
+// forever is how long an internal certificate lasts.
+//
+// Long, and that is a choice rather than laziness. A short life needs something that renews it,
+// and a renewal that fails silently is a mesh that stops trusting itself on a date nobody wrote
+// down. What makes an internal certificate replaceable is that the mesh can reissue it on demand
+// and the node is told in the ordinary way — not that it expires.
+const forever = 10 * 365 * 24 * time.Hour
+
+// Authority is the mesh's own certificate authority.
+type Authority struct {
+	Certificate string
+	private     ed25519.PrivateKey
+}
+
+// EstablishAuthority makes the mesh's authority if it has none, and returns it either way.
+//
+// Idempotent like the signing key beside it: two authorities and nothing says which certificate to
+// believe, so the row is written once and read forever after.
+func (i *Identity) EstablishAuthority(ctx context.Context) (Authority, error) {
+	held, err := i.authority(ctx)
+	if err == nil {
+		return held, nil
+	}
+	if !errors.Is(err, pgx.ErrNoRows) {
+		return Authority{}, err
+	}
+
+	public, private, err := ed25519.GenerateKey(rand.Reader)
+	if err != nil {
+		return Authority{}, err
+	}
+	serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
+	if err != nil {
+		return Authority{}, err
+	}
+	template := &x509.Certificate{
+		SerialNumber: serial,
+		Subject:      pkix.Name{CommonName: "the mesh"},
+		NotBefore:    time.Now().Add(-time.Hour),
+		NotAfter:     time.Now().Add(forever),
+		IsCA:         true,
+		KeyUsage:     x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
+		// No BasicConstraintsValid path length: this signs leaves and nothing else, and an
+		// authority that could sign another authority is one that can be delegated without
+		// anybody deciding to.
+		BasicConstraintsValid: true,
+		MaxPathLen:            0,
+		MaxPathLenZero:        true,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, template, template, public, private)
+	if err != nil {
+		return Authority{}, err
+	}
+	certificate := string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}))
+
+	// Written once. A second insert loses to the first, and both callers then read the same
+	// authority — which is what must happen when two control planes start together.
+	if _, err := i.store.Pool().Exec(ctx,
+		`insert into authority (singleton, certificate, private) values (true, $1, $2)
+		 on conflict (singleton) do nothing`,
+		certificate, base64.StdEncoding.EncodeToString(private)); err != nil {
+		return Authority{}, err
+	}
+	return i.authority(ctx)
+}
+
+func (i *Identity) authority(ctx context.Context) (Authority, error) {
+	var certificate, private string
+	if err := i.store.Pool().QueryRow(ctx,
+		`select certificate, private from authority where singleton`).Scan(&certificate, &private); err != nil {
+		return Authority{}, err
+	}
+	raw, err := base64.StdEncoding.DecodeString(private)
+	if err != nil || len(raw) != ed25519.PrivateKeySize {
+		return Authority{}, fmt.Errorf("the mesh's authority key is unusable")
+	}
+	return Authority{Certificate: certificate, private: ed25519.PrivateKey(raw)}, nil
+}
+
+// Certify issues a certificate for a node's internal name, binding the key that node generated.
+//
+// **The public key is given, never made here.** A certificate authority's whole job is to say
+// *this name belongs to the holder of this key*, and an authority that made the key would be
+// saying something about a key it also holds.
+func (i *Identity) Certify(ctx context.Context, node, name, servingKey string) (string, error) {
+	public, err := base64.StdEncoding.DecodeString(servingKey)
+	if err != nil || len(public) != ed25519.PublicKeySize {
+		return "", fmt.Errorf("%s presented something that is not a serving key", node)
+	}
+
+	authority, err := i.EstablishAuthority(ctx)
+	if err != nil {
+		return "", err
+	}
+	parent, err := parse(authority.Certificate)
+	if err != nil {
+		return "", err
+	}
+
+	serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
+	if err != nil {
+		return "", err
+	}
+	template := &x509.Certificate{
+		SerialNumber: serial,
+		Subject:      pkix.Name{CommonName: name},
+		// The name is in the subject alternative names, which is the only place anything has
+		// looked for a decade — a certificate carrying it only in the common name is a
+		// certificate every modern client refuses.
+		DNSNames:    []string{name},
+		NotBefore:   time.Now().Add(-time.Hour),
+		NotAfter:    time.Now().Add(forever),
+		KeyUsage:    x509.KeyUsageDigitalSignature,
+		ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
+	}
+	der, err := x509.CreateCertificate(rand.Reader, template, parent,
+		ed25519.PublicKey(public), authority.private)
+	if err != nil {
+		return "", err
+	}
+	return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})), nil
+}
+
+func parse(certificate string) (*x509.Certificate, error) {
+	block, _ := pem.Decode([]byte(certificate))
+	if block == nil {
+		return nil, fmt.Errorf("the mesh's authority is not a certificate")
+	}
+	return x509.ParseCertificate(block.Bytes)
+}
+
+// RecordServingKey keeps the public half a node generated for serving TLS.
+func (i *Identity) RecordServingKey(ctx context.Context, node, key string) error {
+	if key == "" {
+		return nil
+	}
+	_, err := i.store.Pool().Exec(ctx,
+		`update node_key set serving_key = $2 where node = $1 and revoked is null`, node, key)
+	return err
+}
+
+// ServingKeyOf is what a node serves TLS with, empty if it has said nothing.
+func (i *Identity) ServingKeyOf(ctx context.Context, node string) (string, error) {
+	var key *string
+	err := i.store.Pool().QueryRow(ctx,
+		`select serving_key from node_key where node = $1 and revoked is null`, node).Scan(&key)
+	if errors.Is(err, pgx.ErrNoRows) {
+		return "", nil
+	}
+	if err != nil {
+		return "", err
+	}
+	if key == nil {
+		return "", nil
+	}
+	return *key, nil
+}

internal/identity/authority_test.go

@@ -0,0 +1,206 @@
+package identity
+
+import (
+	"context"
+	"crypto/ed25519"
+	"crypto/rand"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/pem"
+	"strings"
+	"sync"
+	"testing"
+)
+
+// The authority that certifies names inside the mesh.
+//
+// Asserted by verifying, not by inspecting: a certificate that parses and does not chain is a
+// certificate that fails at the moment something connects, which is the worst place to find out.
+
+func aServingKey(t *testing.T) (public string, private ed25519.PrivateKey) {
+	t.Helper()
+	pub, priv, err := ed25519.GenerateKey(rand.Reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return base64.StdEncoding.EncodeToString(pub), priv
+}
+
+func parsed(t *testing.T, certificate string) *x509.Certificate {
+	t.Helper()
+	block, _ := pem.Decode([]byte(certificate))
+	if block == nil {
+		t.Fatal("not a certificate")
+	}
+	got, err := x509.ParseCertificate(block.Bytes)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return got
+}
+
+func TestACertificateChainsToTheMeshsOwnAuthority(t *testing.T) {
+	ident := fresh(t)
+	ctx := context.Background()
+	public, _ := aServingKey(t)
+
+	certificate, err := ident.Certify(ctx, "workstation", "workstation.internal", public)
+	if err != nil {
+		t.Fatal(err)
+	}
+	authority, err := ident.EstablishAuthority(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	roots := x509.NewCertPool()
+	if !roots.AppendCertsFromPEM([]byte(authority.Certificate)) {
+		t.Fatal("the mesh's authority is not usable as a root")
+	}
+	if _, err := parsed(t, certificate).Verify(x509.VerifyOptions{
+		Roots: roots, DNSName: "workstation.internal",
+	}); err != nil {
+		t.Fatalf("what the mesh issued does not verify against the mesh: %v", err)
+	}
+}
+
+func TestTheNameIsWhereEverythingLooksForIt(t *testing.T) {
+	// A certificate carrying the name only in its common name is one every modern client refuses.
+	ident := fresh(t)
+	public, _ := aServingKey(t)
+	certificate, err := ident.Certify(context.Background(), "a", "a.internal", public)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := parsed(t, certificate)
+	if len(got.DNSNames) != 1 || got.DNSNames[0] != "a.internal" {
+		t.Fatalf("the name is not in the subject alternative names: %v", got.DNSNames)
+	}
+}
+
+func TestItCertifiesTheKeyTheNodeGeneratedAndNoOther(t *testing.T) {
+	// A certificate authority's whole job is to say "this name belongs to the holder of this
+	// key". One that made the key would be saying something about a key it also holds.
+	ident := fresh(t)
+	public, private := aServingKey(t)
+	certificate, err := ident.Certify(context.Background(), "a", "a.internal", public)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	inside, ok := parsed(t, certificate).PublicKey.(ed25519.PublicKey)
+	if !ok {
+		t.Fatalf("the certificate carries a %T", parsed(t, certificate).PublicKey)
+	}
+	if !inside.Equal(private.Public()) {
+		t.Fatal("the certificate is for a key the node does not hold")
+	}
+}
+
+func TestAnAuthorityIsEstablishedOnceAndKept(t *testing.T) {
+	// Two authorities and nothing says which certificate to believe.
+	ident := fresh(t)
+	ctx := context.Background()
+	first, err := ident.EstablishAuthority(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	second, err := ident.EstablishAuthority(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if first.Certificate != second.Certificate {
+		t.Fatal("asking twice made a second authority")
+	}
+}
+
+func TestSomethingThatIsNotAServingKeyIsRefused(t *testing.T) {
+	ident := fresh(t)
+	for _, bad := range []string{"", "not-base64!", base64.StdEncoding.EncodeToString([]byte("short"))} {
+		if _, err := ident.Certify(context.Background(), "a", "a.internal", bad); err == nil {
+			t.Fatalf("%q was certified", bad)
+		}
+	}
+}
+
+func TestTheAuthorityCannotBeUsedToMakeAnotherAuthority(t *testing.T) {
+	// An authority that could sign another is one that can be delegated without anybody deciding
+	// to. The path length says it cannot.
+	ident := fresh(t)
+	authority, err := ident.EstablishAuthority(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := parsed(t, authority.Certificate)
+	if !got.IsCA {
+		t.Fatal("the authority is not an authority")
+	}
+	if got.MaxPathLen != 0 || !got.MaxPathLenZero {
+		t.Fatalf("the authority may sign another authority: path length %d", got.MaxPathLen)
+	}
+}
+
+func TestACertificateFromAnotherMeshDoesNotVerify(t *testing.T) {
+	// The whole point of two authorities being separate: one mesh's certificate means nothing to
+	// another, and the check that says so is the one that must not be skipped.
+	one, two := fresh(t), fresh(t)
+	public, _ := aServingKey(t)
+	certificate, err := one.Certify(context.Background(), "a", "a.internal", public)
+	if err != nil {
+		t.Fatal(err)
+	}
+	other, err := two.EstablishAuthority(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	roots := x509.NewCertPool()
+	roots.AppendCertsFromPEM([]byte(other.Certificate))
+	if _, err := parsed(t, certificate).Verify(x509.VerifyOptions{
+		Roots: roots, DNSName: "a.internal",
+	}); err == nil {
+		t.Fatal("another mesh's certificate verified")
+	} else if !strings.Contains(err.Error(), "authority") && !strings.Contains(err.Error(), "signed") {
+		t.Fatalf("refused for the wrong reason: %v", err)
+	}
+}
+
+func TestTwoProcessesStartingTogetherAgreeOnOneAuthority(t *testing.T) {
+	// A restart while another copy is coming up. Both find nothing and both generate; only one
+	// insert may survive, and the loser must read back the winner rather than return the
+	// authority it generated and did not store — a mesh with two authorities has certificates
+	// half its machines refuse.
+	ident := fresh(t)
+
+	var wg sync.WaitGroup
+	authorities := make([]Authority, 6)
+	errs := make([]error, 6)
+	for i := range authorities {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			authorities[i], errs[i] = ident.EstablishAuthority(context.Background())
+		}(i)
+	}
+	wg.Wait()
+
+	for i, err := range errs {
+		if err != nil {
+			t.Fatalf("establish %d failed: %v", i, err)
+		}
+	}
+	for i, a := range authorities {
+		if a.Certificate != authorities[0].Certificate {
+			t.Errorf("establish %d has a different authority from establish 0", i)
+		}
+	}
+
+	var count int
+	if err := ident.store.Pool().QueryRow(t.Context(),
+		`select count(*) from authority`).Scan(&count); err != nil {
+		t.Fatal(err)
+	}
+	if count != 1 {
+		t.Errorf("%d authorities exist; exactly one may", count)
+	}
+}

internal/identity/migrations/0003-the-mesh-authority.sql

@@ -0,0 +1,32 @@
+-- The authority that certifies names inside the mesh.
+--
+-- novox/hq 08-connectivity keeps two authorities apart on purpose: a public one issues for names
+-- the outside world reaches, and this one issues for names only the mesh knows. Collapsing them
+-- would mean a public authority being asked to certify a name it cannot verify, and a mesh
+-- authority being trusted by things outside it.
+--
+-- **It is not a bootstrap concern.** A joining node verifies the control plane against the
+-- fingerprint in its token, so nothing needs this before membership. It certifies internal names
+-- afterwards, and that is all it does.
+
+create table authority (
+    -- One row, like the signing key beside it. Two authorities and nothing says which certificate
+    -- to believe.
+    singleton   boolean     primary key default true check (singleton),
+
+    certificate text        not null,
+    -- The private half. Held here because signing is what this context is for -- the same
+    -- reasoning as the signing key, which is also held and also never leaves.
+    private     text        not null,
+
+    made_at     timestamptz not null default now()
+);
+
+-- What a node serves TLS with, and what was issued for it.
+--
+-- The public half only. The node generated the pair and keeps the private one, so a copy of this
+-- table certifies nothing and impersonates nobody -- which is the same property the node keys
+-- table has, for the same reason.
+alter table node_key add column serving_key text;
+alter table node_key add column certificate text;
+alter table node_key add column certified_at timestamptz;

internal/link/enrolment.go

@@ -92,6 +92,15 @@ func (e Enrolment) Enrol(ctx context.Context, request EnrolRequest) (EnrolReply,
 	// And the key its secrets are sealed to. Same reasoning as the overlay key below and one step
 	// stronger: without it the mesh cannot send this node a credential at all, and a node that
 	// enrolled without one will be refused a sealed file rather than quietly given none.
+	// And the key it serves TLS with, so the mesh can certify its internal name. Public, so it is
+	// recorded rather than sealed — the node keeps the half that matters.
+	if request.ServingKey != "" {
+		if err := e.Identity.RecordServingKey(ctx, node.ID, request.ServingKey); err != nil {
+			return EnrolReply{}, fmt.Errorf(
+				"the token was spent and %s's serving key could not be recorded: %w",
+				node.Name, err)
+		}
+	}
 	if request.SealingKey != "" {
 		if err := e.Inventory.RecordSealingKey(ctx, node.ID, request.SealingKey); err != nil {
 			return EnrolReply{}, fmt.Errorf(

internal/link/protocol.go

@@ -50,6 +50,11 @@ type EnrolRequest struct {
 	// something nothing else can read, and it must never be able to read it either.
 	SealingKey string `json:"sealing_key,omitempty"`
 
+	// ServingKey is the public half of the key this node serves TLS with on its internal name.
+	// The mesh signs a certificate binding it; the private half never leaves the machine, so
+	// there is nothing to seal and a copy of what the mesh holds certifies nothing new.
+	ServingKey string `json:"serving_key,omitempty"`
+
 	// Profile is what this machine can be asked to do. The control plane cannot decide what a
 	// node should run without it, so it arrives with enrolment rather than being asked for after.
 	Profile map[string]any `json:"profile,omitempty"`