The bus on NATS: both transports behind seams, and the rollout switch #87
+42
-2
@@ -365,20 +365,60 @@ func Compose(s Server, principals []Principal) (string, error) {
|
|||||||
fmt.Fprintf(&b, "port: %d\n", s.ClientPort)
|
fmt.Fprintf(&b, "port: %d\n", s.ClientPort)
|
||||||
fmt.Fprintf(&b, "http: 127.0.0.1:%d\n\n", s.MonitoringPort)
|
fmt.Fprintf(&b, "http: 127.0.0.1:%d\n\n", s.MonitoringPort)
|
||||||
|
|
||||||
|
// **No `verify`, and it said `verify: true` until this configuration was run.** That setting
|
||||||
|
// makes the server demand a *client* certificate, and nothing in the mesh presents one: a host
|
||||||
|
// pins this server's exact certificate and authenticates with the password the mesh minted
|
||||||
|
// (ADR 0004, design 25 §4), and so does a module's runtime. With it on, every connection in the
|
||||||
|
// mesh is refused at the TLS handshake before any password is looked at, and the error —
|
||||||
|
// "client didn't provide a certificate" — reads as a fault in the client.
|
||||||
|
//
|
||||||
|
// TLS is still required: the block is what requires it, and verify only decides whether client
|
||||||
|
// certificates are checked.
|
||||||
b.WriteString("tls {\n")
|
b.WriteString("tls {\n")
|
||||||
fmt.Fprintf(&b, " cert_file: %q\n", s.TLSCert)
|
fmt.Fprintf(&b, " cert_file: %q\n", s.TLSCert)
|
||||||
fmt.Fprintf(&b, " key_file: %q\n", s.TLSKey)
|
fmt.Fprintf(&b, " key_file: %q\n", s.TLSKey)
|
||||||
fmt.Fprintf(&b, " ca_file: %q\n", s.TLSCA)
|
fmt.Fprintf(&b, " ca_file: %q\n", s.TLSCA)
|
||||||
b.WriteString(" verify: true\n")
|
|
||||||
b.WriteString("}\n\n")
|
b.WriteString("}\n\n")
|
||||||
|
|
||||||
b.WriteString("jetstream {\n")
|
b.WriteString("jetstream {\n")
|
||||||
fmt.Fprintf(&b, " store_dir: %q\n", s.StoreDir)
|
fmt.Fprintf(&b, " store_dir: %q\n", s.StoreDir)
|
||||||
b.WriteString("}\n\n")
|
b.WriteString("}\n\n")
|
||||||
|
|
||||||
|
accounts, err := ComposeAccounts(sorted)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
b.WriteString(accounts)
|
||||||
|
return b.String(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ComposeAccounts is the accounts block alone — every user, and nothing about the server.
|
||||||
|
//
|
||||||
|
// **This is the only part of the configuration the mesh writes, and the split is deliberate.** A
|
||||||
|
// server's ports, its TLS paths and its store directory are properties of the container the module
|
||||||
|
// raises: they live in its image and its mounts, and they change when it does. The controller has no
|
||||||
|
// business knowing them, and a controller that did would have to be kept in step with a Dockerfile
|
||||||
|
// it never sees. What only the mesh knows is *who may connect*, so that is what it writes, and the
|
||||||
|
// module's own configuration includes it.
|
||||||
|
//
|
||||||
|
// Four things checked against a running server before this shape was committed to: a user in an
|
||||||
|
// included file authenticates; an unknown user is refused, so the include is the whole authority
|
||||||
|
// rather than an addition to something; a publish outside a user's grant is refused; and rewriting
|
||||||
|
// this file alone and signalling a reload makes a new user appear **without dropping the connection
|
||||||
|
// the mesh already has** — which is what makes every later account, permission or person change cost
|
||||||
|
// nothing (task 1.2's payoff).
|
||||||
|
func ComposeAccounts(principals []Principal) (string, error) {
|
||||||
|
sorted := append([]Principal(nil), principals...)
|
||||||
|
sort.Slice(sorted, func(i, j int) bool { return sorted[i].Username() < sorted[j].Username() })
|
||||||
|
|
||||||
|
var b strings.Builder
|
||||||
|
b.WriteString("# The mesh's users, composed by the controller. Do not edit: the next\n")
|
||||||
|
b.WriteString("# composition overwrites it. Permissions are derived from what each module\n")
|
||||||
|
b.WriteString("# declares and nothing else (novox/hq ADR 0043, design 29 §2).\n\n")
|
||||||
|
|
||||||
// One account for the mesh: accounts in NATS isolate subject spaces entirely, and the mesh is
|
// One account for the mesh: accounts in NATS isolate subject spaces entirely, and the mesh is
|
||||||
// one space (design 25 §4). The cost of that — that permissions are the only isolation — is
|
// one space (design 25 §4). The cost of that — that permissions are the only isolation — is
|
||||||
// paid above, in the scoping of every inbox and every ack subject.
|
// paid in the scoping of every inbox and every ack subject.
|
||||||
b.WriteString("accounts {\n MESH {\n users = [\n")
|
b.WriteString("accounts {\n MESH {\n users = [\n")
|
||||||
for _, p := range sorted {
|
for _, p := range sorted {
|
||||||
perms, err := PermissionsFor(p)
|
perms, err := PermissionsFor(p)
|
||||||
|
|||||||
+4
-1
@@ -9,13 +9,16 @@ tls {
|
|||||||
cert_file: "/tls/tls.crt"
|
cert_file: "/tls/tls.crt"
|
||||||
key_file: "/tls/tls.key"
|
key_file: "/tls/tls.key"
|
||||||
ca_file: "/tls/ca.crt"
|
ca_file: "/tls/ca.crt"
|
||||||
verify: true
|
|
||||||
}
|
}
|
||||||
|
|
||||||
jetstream {
|
jetstream {
|
||||||
store_dir: "/data"
|
store_dir: "/data"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# The mesh's users, composed by the controller. Do not edit: the next
|
||||||
|
# composition overwrites it. Permissions are derived from what each module
|
||||||
|
# declares and nothing else (novox/hq ADR 0043, design 29 §2).
|
||||||
|
|
||||||
accounts {
|
accounts {
|
||||||
MESH {
|
MESH {
|
||||||
users = [
|
users = [
|
||||||
|
|||||||
@@ -154,3 +154,50 @@ func TestRecordsComposeIntoAFile(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The accounts block alone is what the mesh writes, and it holds nothing about the server.
|
||||||
|
//
|
||||||
|
// **The split is the whole design decision** (ComposeAccounts): ports, TLS paths and a store
|
||||||
|
// directory are properties of the container the module raises, and a controller that wrote them
|
||||||
|
// would have to be kept in step with a Dockerfile it never sees. So this test says what must not be
|
||||||
|
// in the file as plainly as what must.
|
||||||
|
func TestWhatTheMeshWritesIsUsersAndNothingAboutTheServer(t *testing.T) {
|
||||||
|
users, err := Users(someRecords())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
hashes := map[string]string{}
|
||||||
|
for _, u := range users {
|
||||||
|
hashes[u.Username()] = "$2a$11$" + strings.Repeat("x", 22)
|
||||||
|
}
|
||||||
|
filled, missing := WithPasswords(users, hashes)
|
||||||
|
if len(missing) != 0 {
|
||||||
|
t.Fatalf("users with no password: %v", missing)
|
||||||
|
}
|
||||||
|
got, err := ComposeAccounts(filled)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, want := range []string{"accounts {", `user: "controller"`, `user: "one.telegram"`} {
|
||||||
|
if !strings.Contains(got, want) {
|
||||||
|
t.Errorf("the accounts file does not contain %s", want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// None of the server's own settings. Each of these in the mesh's file is a value the controller
|
||||||
|
// would then own, and the module could no longer change its own image without the mesh agreeing.
|
||||||
|
for _, absent := range []string{"port:", "http:", "jetstream", "tls {", "store_dir", "cert_file"} {
|
||||||
|
if strings.Contains(got, absent) {
|
||||||
|
t.Errorf("the accounts file contains %q, which belongs to the module that raises the "+
|
||||||
|
"server, not to the mesh", absent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A user with no password is refused here too, not only by the whole-file composition: this is the
|
||||||
|
// function the controller actually calls, and a user without a password is a user anybody is.
|
||||||
|
func TestTheAccountsFileRefusesAUserWithNoPassword(t *testing.T) {
|
||||||
|
if _, err := ComposeAccounts([]Principal{{Kind: KindController}}); err == nil {
|
||||||
|
t.Fatal("a user with no password hash was written")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -97,6 +97,13 @@ type Rendering struct {
|
|||||||
// compose it a second time.
|
// compose it a second time.
|
||||||
Suffix string
|
Suffix string
|
||||||
|
|
||||||
|
// BusUsers is the mesh's composed user list, for the module holding `mesh-broker`. Empty on
|
||||||
|
// every other node, and on this one until the controller has composed it.
|
||||||
|
//
|
||||||
|
// **Only the users, never the server's own settings**: those are the module's, in its image and
|
||||||
|
// its mounts (Manifest.BusUsers).
|
||||||
|
BusUsers string
|
||||||
|
|
||||||
// Kept is every operator-sealed secret in the mesh, for a module that `keeps` them. Nil when
|
// Kept is every operator-sealed secret in the mesh, for a module that `keeps` them. Nil when
|
||||||
// nothing on this node keeps them, or the mesh has no operator key.
|
// nothing on this node keeps them, or the mesh has no operator key.
|
||||||
Kept *KeptExport
|
Kept *KeptExport
|
||||||
@@ -359,6 +366,35 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if m.BusUsers != "" {
|
||||||
|
// **The claim authorises it, not the field.** This file holds every user's password
|
||||||
|
// hash, so a module that could ask for it could read every credential on the bus.
|
||||||
|
// Checked from this manifest alone, which is the cheapest check there is: whether some
|
||||||
|
// other module also claims the seat is resolution's business elsewhere, and one holder
|
||||||
|
// mesh-wide is already guaranteed.
|
||||||
|
if !m.ClaimsSeat("mesh-broker") {
|
||||||
|
return nil, fmt.Errorf(
|
||||||
|
"%s asks for the mesh's user list and does not claim mesh-broker. That file "+
|
||||||
|
"holds every user's password hash, so the seat is what authorises it",
|
||||||
|
m.Module)
|
||||||
|
}
|
||||||
|
if with.BusUsers == "" {
|
||||||
|
// Asked for and not composed. Refused rather than skipped, for the reason a
|
||||||
|
// certificate is: a bus with no user list refuses every connection in the mesh, and
|
||||||
|
// an empty file would look like a configuration problem on the machine.
|
||||||
|
return nil, fmt.Errorf(
|
||||||
|
"%s holds mesh-broker and the mesh composed no user list, so the bus would "+
|
||||||
|
"refuse every connection", m.Module)
|
||||||
|
}
|
||||||
|
first = append(first, map[string]any{
|
||||||
|
"id": BusUsersID(), "type": "file", "path": m.BusUsers,
|
||||||
|
"content": with.BusUsers,
|
||||||
|
// Readable by the server and nothing else. Hashes rather than passwords, so this is
|
||||||
|
// not a set of working credentials — but a list of every user in the mesh is worth
|
||||||
|
// keeping to the one process that needs it.
|
||||||
|
"mode": "0600",
|
||||||
|
})
|
||||||
|
}
|
||||||
for _, name := range sortedKeys(m.OwnSecrets) {
|
for _, name := range sortedKeys(m.OwnSecrets) {
|
||||||
sealed := with.Needed[m.Module][name]
|
sealed := with.Needed[m.Module][name]
|
||||||
if sealed == "" {
|
if sealed == "" {
|
||||||
|
|||||||
@@ -0,0 +1,80 @@
|
|||||||
|
package catalogue
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
// The mesh's user list reaches the module holding the bus, and nothing else.
|
||||||
|
//
|
||||||
|
// Three refusals and one delivery, because each of the refusals would be silent in a different way:
|
||||||
|
// a module that asked and was given it could read every credential on the bus; a bus given an empty
|
||||||
|
// file refuses every connection in the mesh and looks like a machine problem; and a bus that never
|
||||||
|
// asked gets nothing rather than a file it does not read.
|
||||||
|
func TestTheMeshsUserListGoesOnlyToTheModuleHoldingTheBus(t *testing.T) {
|
||||||
|
theBus := func() Manifest {
|
||||||
|
return Manifest{
|
||||||
|
Module: "nats", Version: "1",
|
||||||
|
Claims: []Claim{{Name: "mesh-broker", Scope: ScopeMesh}},
|
||||||
|
BusUsers: "/var/lib/nats-module/conf/accounts.conf",
|
||||||
|
Resources: []map[string]any{},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
on := func(t *testing.T, m Manifest, with Rendering) ([]map[string]any, error) {
|
||||||
|
t.Helper()
|
||||||
|
return Resolution{Node: "anchor", Modules: []Manifest{m}}.Declaration(with)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("the holder is given it", func(t *testing.T) {
|
||||||
|
resources, err := on(t, theBus(), Rendering{BusUsers: "accounts { MESH { users = [] } }"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Prefixed with the module it came from, like every resource: two modules may reasonably
|
||||||
|
// both call something "config", and without the prefix the second would silently replace
|
||||||
|
// the first.
|
||||||
|
var found map[string]any
|
||||||
|
for _, r := range resources {
|
||||||
|
if r["id"] == "nats."+BusUsersID() {
|
||||||
|
found = r
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if found == nil {
|
||||||
|
t.Fatalf("the bus was given no user list: %+v", resources)
|
||||||
|
}
|
||||||
|
if found["path"] != "/var/lib/nats-module/conf/accounts.conf" {
|
||||||
|
t.Errorf("written to %v rather than where the module asked", found["path"])
|
||||||
|
}
|
||||||
|
if found["mode"] != "0600" {
|
||||||
|
t.Errorf("mode %v: a list of every user in the mesh belongs to the one process that "+
|
||||||
|
"needs it", found["mode"])
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("a module that does not claim the seat is refused", func(t *testing.T) {
|
||||||
|
m := theBus()
|
||||||
|
m.Claims = nil
|
||||||
|
if _, err := on(t, m, Rendering{BusUsers: "accounts {}"}); err == nil {
|
||||||
|
t.Fatal("a module that claims nothing was handed every user's password hash")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("the holder with nothing composed is refused", func(t *testing.T) {
|
||||||
|
if _, err := on(t, theBus(), Rendering{}); err == nil {
|
||||||
|
t.Fatal("the bus was given an empty user list, so it would refuse every connection in " +
|
||||||
|
"the mesh and look like a machine problem")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("a module that did not ask gets nothing", func(t *testing.T) {
|
||||||
|
m := theBus()
|
||||||
|
m.BusUsers = ""
|
||||||
|
resources, err := on(t, m, Rendering{BusUsers: "accounts {}"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, r := range resources {
|
||||||
|
if r["id"] == "nats."+BusUsersID() {
|
||||||
|
t.Fatal("a module that asked for no user list was given one")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -423,6 +423,20 @@ type Manifest struct {
|
|||||||
// A directory rather than one document for the same reason as above: each value is sealed
|
// A directory rather than one document for the same reason as above: each value is sealed
|
||||||
// separately and the mesh cannot open any of them to build a list.
|
// separately and the mesh cannot open any of them to build a list.
|
||||||
Grants map[string]string `json:"grants,omitempty"`
|
Grants map[string]string `json:"grants,omitempty"`
|
||||||
|
|
||||||
|
// BusUsers is where this module wants the mesh's user list written, and it is only ever
|
||||||
|
// answered for the module holding `mesh-broker`.
|
||||||
|
//
|
||||||
|
// **The mesh writes who may connect; the module owns everything else about its server**
|
||||||
|
// (novox/hq design 25 §4, task 1.7). Ports, TLS paths and a store directory live in this
|
||||||
|
// module's image and its mounts and change when it does, so the module's own configuration
|
||||||
|
// carries them and includes this file. A controller that wrote the whole configuration would
|
||||||
|
// have to be kept in step with a Dockerfile it never sees.
|
||||||
|
//
|
||||||
|
// **Asking for it is not enough to receive it.** This file holds every user's password hash, so
|
||||||
|
// a module that could ask for it could read every credential on the bus — and the claim on
|
||||||
|
// `mesh-broker` is what authorises it, checked from this manifest alone.
|
||||||
|
BusUsers string `json:"bus-users,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Build says how to produce this module's artifacts from its source.
|
// Build says how to produce this module's artifacts from its source.
|
||||||
@@ -641,7 +655,22 @@ type Certificate struct {
|
|||||||
|
|
||||||
// CertificateID and AuthorityID are the resource identities of what the mesh issued.
|
// CertificateID and AuthorityID are the resource identities of what the mesh issued.
|
||||||
func CertificateID() string { return "certificate" }
|
func CertificateID() string { return "certificate" }
|
||||||
func AuthorityID() string { return "certificate-authority" }
|
|
||||||
|
// ClaimsSeat says whether this manifest claims one named seat.
|
||||||
|
func (m Manifest) ClaimsSeat(seat string) bool {
|
||||||
|
for _, c := range m.Claims {
|
||||||
|
if c.Name == seat {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// BusUsersID names the mesh's composed user list, so it is the same resource across every
|
||||||
|
// declaration and a change to it is an update rather than a second file beside the old one — which
|
||||||
|
// on a bus reading a directory would be two account lists, and the server would take both.
|
||||||
|
func BusUsersID() string { return "bus-users" }
|
||||||
|
func AuthorityID() string { return "certificate-authority" }
|
||||||
|
|
||||||
// FilteringID names the computed rule set, so it is the same resource across every declaration
|
// FilteringID names the computed rule set, so it is the same resource across every declaration
|
||||||
// and a change to it is an update rather than an addition beside the old one.
|
// and a change to it is an update rather than an addition beside the old one.
|
||||||
|
|||||||
Reference in New Issue
Block a user