Compare commits

..
1 Commits
Author SHA1 Message Date
jschoubben ab74988f1c The filter constrains what arrives from outside, and names no network
The forward chain blocked everything passing through the machine and then allowed
the machine's own containers back by naming their address ranges: 172.16.0.0/12 and
192.168.128.0/17 fixed here, the rest recorded per machine by 0043. Every way of
keeping that list correct fails — a constant describes one machine, a recorded range
goes stale in silence and cannot tell a network the mesh made from one a predecessor
left behind, and generating it from the modules would put half the rule set on the
machine.

The mesh has no position on a container reaching outward: that is not a port opened
to anybody. So both chains are written around the links traffic arrives on. What did
not arrive from outside is accepted in one line; what did meets the declared rules.
The tunnel is named beside the outward links rather than treated as inside, or a port
nothing declares would be reachable from every machine in the mesh.

A machine that has not reported an outward link is sent no filter and keeps the one
it has, refused where a person reads it rather than as a rule set that will not load.

Removes the two constants, `node networks`, and the column behind it. novox/hq ADR
0140, superseding 0137 and 0139.
2026-09-28 23:49:14 +02:00
31 changed files with 42 additions and 2508 deletions
+2 -13
View File
@@ -27,18 +27,8 @@ build:
IMAGE ?= mesh-controller:$(VERSION)
DEV_TAG ?= mesh-controller:development
# The base the module declares, read from the manifest rather than written here twice.
#
# **`make image` was broken and stayed broken**, because the Dockerfile's fallback base was a Go
# older than go.mod asks for: every build died at `go mod download` with "go.mod requires go >=
# 1.26.0", and the pipeline never saw it because the pipeline passes the declared base in. Anybody
# building the image by hand hit it and had to find the digest themselves (novox/hq 04-ISSUES/146,
# what it cost).
GO_BASE ?= $(shell python3 -c "import json;print(next(o['image'] for o in json.load(open('module.json'))['build']['on'] if o['arg']=='GO_BASE'))" 2>/dev/null)
image:
@test -n "$(GO_BASE)" || { echo "module.json declares no GO_BASE; pass GO_BASE=<image> or fix the manifest"; exit 1; }
docker build --build-arg GO_BASE=$(GO_BASE) --build-arg VERSION=$(VERSION) -t $(IMAGE) -t $(DEV_TAG) .
docker build --build-arg VERSION=$(VERSION) -t $(IMAGE) -t $(DEV_TAG) .
@echo
@docker image inspect $(IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
@@ -48,8 +38,7 @@ BUILDER_IMAGE ?= mesh-builder:$(VERSION)
BUILDER_DEV_TAG ?= mesh-builder:development
builder-image:
@test -n "$(GO_BASE)" || { echo "module.json declares no GO_BASE; pass GO_BASE=<image> or fix the manifest"; exit 1; }
docker build --build-arg GO_BASE=$(GO_BASE) -f cmd/mesh-builder/Dockerfile -t $(BUILDER_IMAGE) -t $(BUILDER_DEV_TAG) .
docker build -f cmd/mesh-builder/Dockerfile -t $(BUILDER_IMAGE) -t $(BUILDER_DEV_TAG) .
@echo
@docker image inspect $(BUILDER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
-258
View File
@@ -1,258 +0,0 @@
package main
import (
"context"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"errors"
"fmt"
"math/big"
"net"
"os"
"path/filepath"
"strings"
"time"
"github.com/novox/mesh-controller/internal/broker"
)
// The bus's own certificate, made by the mesh rather than borrowed from an image.
//
// **The foundation asked a third-party image for a tool it never said must be there** (novox/hq
// 04-ISSUES/146). The bootstrap made this certificate by running `openssl` inside the broker's
// image, which worked while the broker was one that happened to carry it and stopped the day the
// bus changed: the new one has a shell and no openssl, so the step exited 127 and no mesh could be
// raised. Substituting another image the bundle names does not help — none of them carry it
// either.
//
// So the program that needs a certificate makes one. It is the mesh's own binary, already on the
// machine at this point in the bootstrap (the schema step ran it), and it needs nothing from the
// image it writes into but a mounted directory.
//
// **Self-signed, and that is the design** — a host pins this server's exact certificate and
// authenticates with a password (novox/hq ADR 0004). There is no authority above it to ask, and at
// this moment in a bootstrap there is no mesh to ask one of.
//
// Idempotent, because the step is applied again on every reconcile and a second certificate would
// be one the hosts that pinned the first no longer believe.
// busCertificateNames is what the bus is reached by: the container name on a mesh network, and the
// loopback address the machine's own foundation dials.
var busCertificateNames = []string{"mesh-broker"}
const busCertificateLife = 10 * 365 * 24 * time.Hour
// busCertificate makes the bus's certificate in a directory, or says whether one is there.
//
// broker certificate --into /tls make it if it is not there
// broker certificate --check --into /tls exit non-zero unless a usable pair is
func busCertificate(args []string) error {
into, check := "", false
for i := 0; i < len(args); i++ {
switch args[i] {
case "--check":
check = true
case "--into":
if i+1 >= len(args) {
return errors.New("--into needs a directory")
}
into = args[i+1]
i++
default:
return fmt.Errorf("broker certificate [--check] --into <directory>: %q", args[i])
}
}
if into == "" {
return errors.New("broker certificate [--check] --into <directory>")
}
crt, key := filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key")
if usable, err := busCertificateUsable(crt, key); err != nil {
return err
} else if usable {
fmt.Printf("the bus already has a certificate at %s, and it was left alone\n", crt)
return nil
}
if check {
// Said as a failure, because that is what the caller asked: a bootstrap's verify runs
// this and a false answer is what makes the step run.
return fmt.Errorf("no usable certificate and key at %s", into)
}
return writeBusCertificate(crt, key)
}
// busCertificateUsable says whether a certificate and its key are both there and parse.
//
// Both, and parsed rather than stat'ed: a half-written pair is the state a bootstrap interrupted
// between the two files leaves behind, and a step that treated it as done would hand the server a
// certificate with no key and report success.
func busCertificateUsable(crt, key string) (bool, error) {
certPEM, err := os.ReadFile(crt)
if errors.Is(err, os.ErrNotExist) {
return false, nil
}
if err != nil {
return false, err
}
keyPEM, err := os.ReadFile(key)
if errors.Is(err, os.ErrNotExist) {
return false, nil
}
if err != nil {
return false, err
}
if _, err := tlsPairParses(certPEM, keyPEM); err != nil {
return false, nil
}
return true, nil
}
func tlsPairParses(certPEM, keyPEM []byte) (*x509.Certificate, error) {
block, _ := pem.Decode(certPEM)
if block == nil || block.Type != "CERTIFICATE" {
return nil, errors.New("not a certificate")
}
certificate, err := x509.ParseCertificate(block.Bytes)
if err != nil {
return nil, err
}
keyBlock, _ := pem.Decode(keyPEM)
if keyBlock == nil {
return nil, errors.New("not a key")
}
if _, err := x509.ParsePKCS8PrivateKey(keyBlock.Bytes); err != nil {
if _, err := x509.ParsePKCS1PrivateKey(keyBlock.Bytes); err != nil {
return nil, err
}
}
return certificate, nil
}
func writeBusCertificate(crt, key string) error {
private, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
return err
}
serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
if err != nil {
return err
}
template := &x509.Certificate{
SerialNumber: serial,
Subject: pkix.Name{CommonName: busCertificateNames[0]},
DNSNames: busCertificateNames,
IPAddresses: []net.IP{net.ParseIP("127.0.0.1")},
NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().Add(busCertificateLife),
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
BasicConstraintsValid: true,
}
der, err := x509.CreateCertificate(rand.Reader, template, template, &private.PublicKey, private)
if err != nil {
return err
}
pkcs8, err := x509.MarshalPKCS8PrivateKey(private)
if err != nil {
return err
}
// **The key first, and only then the certificate**, so the pair a reader finds is never a
// certificate whose key has not been written yet — the one order in which an interruption
// leaves something that looks finished (novox/hq 04-ISSUES/014, a key present and unusable).
if err := os.WriteFile(key, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8}), 0o600); err != nil {
return err
}
if err := os.WriteFile(crt, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644); err != nil {
return err
}
fmt.Printf("made the bus a certificate for %v, valid until %s\n %s\n %s\n",
busCertificateNames, template.NotAfter.Format(time.RFC3339), crt, key)
return nil
}
// busAccounts writes the mesh's composed user list to a file.
//
// **For genesis, where no declaration can deliver it** (novox/hq 04-ISSUES/146). Everywhere else
// the list reaches the machine running the bus as a resource of the module that holds it — which
// requires that machine to be an enrolled node, and at genesis it is not: the first node cannot
// enrol because the account it would enrol with cannot be composed onto a bus it has no declaration
// for. The installer breaks that circle by placing the file itself, once, and the module takes the
// file over from its first push.
//
// The same composition, not a second one: this asks the store for the same records and renders them
// with the same composer the declaration uses. A genesis that hand-wrote an account would be a
// second statement of who may say what, able to disagree with the first.
//
// **It writes to standard output unless told a file**, and that is the point: the control plane
// composes and says what it composed, and whoever is raising the machine puts it where that
// machine's bus reads it. A control plane that wrote into the bus's own directory would have to
// know where that is and how to make the server re-read it — which is the module's knowledge, and
// the module is what takes this over on the first push.
//
// broker accounts > /var/lib/mesh-bus-conf/accounts.conf
func busAccounts(ctx context.Context, args []string) error {
into := ""
for i := 0; i < len(args); i++ {
switch args[i] {
case "--into":
if i+1 >= len(args) {
return errors.New("--into needs a file")
}
into = args[i+1]
i++
default:
return fmt.Errorf("broker accounts --into <file>: %q", args[i])
}
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
records, err := open.inventory.BusRecords(ctx)
if err != nil {
return err
}
users, err := broker.Users(records)
if err != nil {
return err
}
kept, err := open.inventory.BusUsers(ctx)
if err != nil {
return err
}
hashes := make(map[string]string, len(kept))
for name, u := range kept {
hashes[name] = u.PasswordHash
}
filled, missing := broker.WithPasswords(users, hashes)
if len(missing) > 0 {
// To standard error, always: the composed file may be going to standard output, and a
// remark in the middle of it is a configuration the server refuses to parse.
fmt.Fprintf(os.Stderr, "leaving out %d user(s) the mesh has minted no credential for: %s\n",
len(missing), strings.Join(missing, ", "))
}
if len(filled) == 0 {
return errors.New("not one user has a credential, so this list would refuse every " +
"connection in the mesh")
}
accounts, err := broker.ComposeAccounts(filled)
if err != nil {
return err
}
if into == "" {
fmt.Print(accounts)
return nil
}
if err := os.WriteFile(into, []byte(accounts), 0o600); err != nil {
return err
}
fmt.Printf("wrote %d user(s) to %s\n", len(filled), into)
return nil
}
-121
View File
@@ -1,121 +0,0 @@
package main
import (
"crypto/tls"
"crypto/x509"
"os"
"path/filepath"
"strings"
"testing"
)
// novox/hq 04-ISSUES/146. The bootstrap could not make the bus a certificate: it asked an image for
// `openssl` and the image it asks has none. What replaces it is this command, so what is checked is
// what the bootstrap needs from it — a pair a TLS server can actually load, made once and only once.
func TestTheBusCertificateLoadsAsAServersWould(t *testing.T) {
into := t.TempDir()
if err := busCertificate([]string{"--into", into}); err != nil {
t.Fatalf("the bus could not be given a certificate: %v", err)
}
// The check a cheaper test would not make. The key was present and valid and the server could
// not start, once, because nothing loaded the pair the way a server loads it
// (novox/hq 04-ISSUES/014).
pair, err := tls.LoadX509KeyPair(filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key"))
if err != nil {
t.Fatalf("a TLS server cannot load what was written: %v", err)
}
leaf := pair.Leaf
if leaf == nil {
if leaf, err = x509.ParseCertificate(pair.Certificate[0]); err != nil {
t.Fatal(err)
}
}
if err := leaf.VerifyHostname("mesh-broker"); err != nil {
t.Errorf("the certificate is not for the name the bus is reached by: %v", err)
}
if len(leaf.IPAddresses) == 0 || leaf.IPAddresses[0].String() != "127.0.0.1" {
t.Errorf("the certificate does not cover the loopback address the foundation dials: %v", leaf.IPAddresses)
}
// The key is not readable by anything else on the machine; the certificate is public and is.
key, err := os.Stat(filepath.Join(into, "tls.key"))
if err != nil {
t.Fatal(err)
}
if key.Mode().Perm() != 0o600 {
t.Errorf("the key is %v", key.Mode().Perm())
}
crt, err := os.Stat(filepath.Join(into, "tls.crt"))
if err != nil {
t.Fatal(err)
}
if crt.Mode().Perm() != 0o644 {
t.Errorf("the certificate is %v, which the server runs as another user cannot read", crt.Mode().Perm())
}
}
// **Made once.** The step is applied again on every reconcile, and a second certificate is one the
// hosts that pinned the first no longer believe (novox/hq ADR 0004).
func TestTheBusCertificateIsMadeOnce(t *testing.T) {
into := t.TempDir()
if err := busCertificate([]string{"--into", into}); err != nil {
t.Fatal(err)
}
first, err := os.ReadFile(filepath.Join(into, "tls.crt"))
if err != nil {
t.Fatal(err)
}
if err := busCertificate([]string{"--into", into}); err != nil {
t.Fatal(err)
}
again, err := os.ReadFile(filepath.Join(into, "tls.crt"))
if err != nil {
t.Fatal(err)
}
if string(first) != string(again) {
t.Fatal("running it twice replaced the certificate every host had pinned")
}
}
// The verify half: false before, true after, which is what makes the bootstrap run the step at all.
func TestTheCheckIsFalseUntilThereIsAPair(t *testing.T) {
into := t.TempDir()
if err := busCertificate([]string{"--check", "--into", into}); err == nil {
t.Fatal("an empty directory reported a usable certificate")
}
if err := busCertificate([]string{"--into", into}); err != nil {
t.Fatal(err)
}
if err := busCertificate([]string{"--check", "--into", into}); err != nil {
t.Fatalf("the certificate it just made does not satisfy its own check: %v", err)
}
}
// A half-written pair is not a pair. An interrupted bootstrap leaves exactly this, and a step that
// called it done would hand the server a certificate with no key and report success.
func TestACertificateWithoutItsKeyIsNotUsable(t *testing.T) {
into := t.TempDir()
if err := busCertificate([]string{"--into", into}); err != nil {
t.Fatal(err)
}
if err := os.Remove(filepath.Join(into, "tls.key")); err != nil {
t.Fatal(err)
}
if err := busCertificate([]string{"--check", "--into", into}); err == nil {
t.Fatal("a certificate with no key passed the check")
}
if err := busCertificate([]string{"--into", into}); err != nil {
t.Fatal(err)
}
if _, err := tls.LoadX509KeyPair(filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key")); err != nil {
t.Fatalf("it did not replace the unusable pair: %v", err)
}
}
func TestWhereToWriteIsRequired(t *testing.T) {
if err := busCertificate(nil); err == nil || !strings.Contains(err.Error(), "--into") {
t.Fatalf("it did not ask where to write: %v", err)
}
}
+1 -1
View File
@@ -88,7 +88,7 @@ func run() error {
case "identity":
return identityCommand(ctx, args[1:])
case "broker":
return brokerCommand(ctx, args[1:])
return brokerCommand(args[1:])
case "serve":
return serve(ctx)
case "upgrade":
+1 -8
View File
@@ -346,16 +346,9 @@ func whoResolves(ctx context.Context, open *stores, requirement string) (
refused := map[string]string{}
for _, n := range nodes {
plan, _, err := planFor(ctx, open, n.Name)
switch {
case unresolvable(err):
if err != nil {
refused[n.Name] = err.Error()
continue
case err != nil:
// Not a node that does not resolve — a question that went unanswered. Recording it as a
// refusal would take the machine off the private network, and the generator that reads
// this would then write a roster and a filter without it (novox/hq 04-ISSUES/152).
return nil, nil, fmt.Errorf("whether %s answers %q cannot be read: %w",
n.Name, requirement, err)
}
for _, m := range plan.Modules {
for _, offered := range m.Offers() {
+2 -8
View File
@@ -363,15 +363,9 @@ func identityCommand(ctx context.Context, args []string) error {
return nil
}
func brokerCommand(ctx context.Context, args []string) error {
if len(args) > 0 && args[0] == "certificate" {
return busCertificate(args[1:])
}
if len(args) > 0 && args[0] == "accounts" {
return busAccounts(ctx, args[1:])
}
func brokerCommand(args []string) error {
if len(args) == 0 || args[0] != "show" {
return errors.New("broker show | broker certificate [--check] --into <directory> | broker accounts --into <file>")
return errors.New("broker show")
}
known, err := broker.FromEnvironment()
if errors.Is(err, broker.ErrNotConfigured) {
+7 -63
View File
@@ -25,36 +25,7 @@ import (
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
// nothing in it was wrong, and no one edit was the one that should have been a new file.
// notResolvable marks the one failure in planFor that is a statement about the node: its assigned
// modules do not compose. Every other failure means the mesh could not be *asked* — the store was
// unreachable, a key could not be read — and says nothing about the node at all.
//
// The distinction exists because three callers gather something across every machine and must carry
// on when one machine's set is broken. Each of them read a plain error as "their set does not
// resolve", and so read a store that was briefly unreachable as a machine that runs nothing. On the
// roster of routed names that is not a degraded answer but a false one: it states, to every machine
// at once, that another machine's names do not exist. A control node spent hours replacing every
// container it ran, on a six-minute cycle, because each pass restarted the store this is read from,
// the read failed, one name left the roster, and the roster is part of every container's identity
// (novox/hq 04-ISSUES/152, and 04-ISSUES/151 for why a changed roster is a changed container).
//
// So: skip a node that cannot resolve, and never a node that could not be read.
type notResolvable struct{ err error }
func (n notResolvable) Error() string { return n.err.Error() }
func (n notResolvable) Unwrap() error { return n.err }
// unresolvable reports whether err is a node's own set failing to compose, rather than the mesh
// being unable to answer.
func unresolvable(err error) bool {
var n notResolvable
return errors.As(err, &n)
}
// planFor works out everything a node should run, from what was assigned to it.
//
// A failure to compose the node's own modules is wrapped as notResolvable; every other failure is
// returned as it is. Callers gathering across the mesh must tell them apart — see notResolvable.
func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Resolution, catalogue.SettingsBy, error) {
inv := open.inventory
shelf, err := inv.Catalogue(ctx)
@@ -124,9 +95,7 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
At: onNetwork[nodeName], PublicDomain: publicDomain,
Account: who.Account, AccountHome: who.AccountHome}, world)
if err != nil {
// The node's own set does not compose. Marked, because this is the only failure here that
// a mesh-wide gatherer may pass over — see notResolvable.
return catalogue.Resolution{}, nil, notResolvable{err}
return catalogue.Resolution{}, nil, err
}
// The credential for each thing this node takes from elsewhere. Made once and kept, so the
@@ -194,13 +163,8 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
if len(stray) > 0 {
// Somebody set something that reaches no file. Said here rather than discovered by the
// machine not behaving differently, which is the slowest way there is.
//
// Marked like a set that will not compose, and for the same reason: it is a standing fact
// about this node's own configuration, not a question the mesh could not answer. A gatherer
// passes over it as it always did — one node's stray setting must not stop every other node
// being described (novox/hq 04-ISSUES/152).
return catalogue.Resolution{}, nil, notResolvable{fmt.Errorf(
"these settings reach nothing:\n - %s", strings.Join(stray, "\n - "))}
return catalogue.Resolution{}, nil, fmt.Errorf(
"these settings reach nothing:\n - %s", strings.Join(stray, "\n - "))
}
return resolved, settings, nil
}
@@ -707,17 +671,11 @@ func renderingFor(ctx context.Context, open *stores, node string,
// routed name only because it carried a label the mesh composed, never because the mesh knows what
// "route" means. A node that does not resolve is skipped, so one machine's broken set does not cost
// the rest their names.
//
// **A node that could not be READ is a different matter and is raised.** Skipping one states, to
// every machine at once, that its names do not exist — and since the roster is part of every
// container's identity, that withdraws them and replaces every container (novox/hq 04-ISSUES/152,
// 151). So every failure here says which machine and which read, because the alternative is a
// mesh-wide refusal with nothing named in it.
func routeNamesInTheMesh(ctx context.Context, open *stores) (map[string]string, error) {
inv := open.inventory
places, err := inv.Overlays(ctx)
if err != nil {
return nil, fmt.Errorf("where the machines are cannot be read: %w", err)
return nil, err
}
address := map[string]string{}
for _, p := range places {
@@ -728,22 +686,14 @@ func routeNamesInTheMesh(ctx context.Context, open *stores) (map[string]string,
nodes, err := inv.Nodes(ctx)
if err != nil {
return nil, fmt.Errorf("which machines the mesh has cannot be read: %w", err)
return nil, err
}
out := map[string]string{}
for _, n := range nodes {
plan, settings, err := planFor(ctx, open, n.Name)
switch {
case unresolvable(err):
// Their set does not compose, so they serve no names. Passed over, so one machine's
// broken set does not cost the rest theirs.
if err != nil {
continue
case err != nil:
// The mesh could not be asked. Returning the roster without this machine's names would
// state that they do not exist — to every machine, and indistinguishably from the
// operator having withdrawn them (novox/hq 04-ISSUES/152).
return nil, fmt.Errorf("the names %s serves cannot be read: %w", n.Name, err)
}
for _, m := range plan.Modules {
for to := range m.Contributes {
@@ -873,17 +823,11 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
out := make([]catalogue.Grant, 0, len(issued))
for _, s := range issued {
plan, settings, err := planFor(ctx, open, s.Consumer)
switch {
case unresolvable(err):
if err != nil {
// Their set does not resolve. Skipped rather than fatal: this node is not the place
// to report another machine's problem, and a grant for something that is not going to
// run would have the provider create a user nothing uses.
continue
case err != nil:
// The mesh could not be asked what they wanted, which is not the same as their wanting
// nothing — and withholding a grant on that reading takes a consumer's access away
// (novox/hq 04-ISSUES/152).
return nil, fmt.Errorf("what %s asked of %s cannot be read: %w", s.Consumer, s.Name, err)
}
values, asks, err := plan.ContributionsFrom(s.Name, s.ConsumerModule, settings)
if err != nil {
-6
View File
@@ -717,12 +717,6 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string)
broker.BareAddress(address), err)
}
defer js.Close()
// What the raise decided not to fail over. Said, for the reason everything else here is said:
// a consumer kept as it was is a difference between what the mesh asked for and what the bus
// holds, and one nobody would find by reading either (novox/hq 04-ISSUES/156).
js.Note = func(format string, args ...any) {
fmt.Printf(" "+format+"\n", args...)
}
// **Its own user, before anything else.** The controller's account is created by the installer at
// a bootstrap password, before there is a controller to mint one — so nothing recorded a hash for
@@ -1,99 +0,0 @@
package main
import (
"context"
"strings"
"testing"
)
// A node's own set failing to compose, and the mesh being unable to answer at all, are different
// things, and only the first may be passed over when something is gathered across every machine
// (novox/hq 04-ISSUES/152). These pin that distinction where the three gatherers rely on it.
func TestASetThatDoesNotComposeIsMarkedAsTheNodesOwnProblem(t *testing.T) {
open := aMesh(t)
one, two := rivals()
register(t, open, one)
register(t, open, two)
for _, m := range []string{one.Module, two.Module} {
if _, err := open.inventory.Assign(t.Context(), "laptop", m); err != nil {
t.Fatal(err)
}
}
_, _, err := planFor(t.Context(), open, "laptop")
if err == nil {
t.Fatal("two modules claiming one seat composed anyway")
}
if !unresolvable(err) {
t.Fatalf("a set that cannot compose was not marked as the node's own problem: %v", err)
}
}
func TestAStoreThatCannotBeReadIsNotANodeThatDoesNotCompose(t *testing.T) {
open := aMesh(t)
// Nothing is wrong with anchor. The question simply cannot be asked.
stopped, cancel := context.WithCancel(t.Context())
cancel()
_, _, err := planFor(stopped, open, "anchor")
if err == nil {
t.Fatal("a plan composed against a store that could not be read")
}
if unresolvable(err) {
t.Fatalf("a question the mesh could not answer was read as a node that runs nothing: %v", err)
}
}
func TestOneIncoherentNodeDoesNotCostTheRestTheirNames(t *testing.T) {
open := aMesh(t)
one, two := rivals()
register(t, open, one)
register(t, open, two)
for _, m := range []string{one.Module, two.Module} {
if _, err := open.inventory.Assign(t.Context(), "laptop", m); err != nil {
t.Fatal(err)
}
}
// laptop cannot compose. That is laptop's problem and nobody else's: the roster is still
// answerable, and anchor keeps whatever it serves.
if _, err := routeNamesInTheMesh(t.Context(), open); err != nil {
t.Fatalf("one node's broken set cost the whole mesh its roster: %v", err)
}
}
func TestARosterIsNeverReturnedWithNamesItCouldNotRead(t *testing.T) {
open := aMesh(t)
stopped, cancel := context.WithCancel(t.Context())
cancel()
names, err := routeNamesInTheMesh(stopped, open)
if err == nil {
t.Fatalf("a roster was composed from a store that could not be read: %v", names)
}
// The failure must be raised, not turned into an absence. A roster missing a machine's names
// is indistinguishable, on every machine that receives it, from the operator withdrawing them —
// and because the roster is part of every container's identity, it replaces all of them.
if names != nil {
t.Fatalf("a partial roster was returned beside the error: %v", names)
}
}
// Kept so the reason survives the next person reading it: the message the gatherer raises must say
// which machine could not be read, or the operator is left with a mesh-wide failure and no name.
func TestTheRaisedFailureNamesTheMachineItCouldNotRead(t *testing.T) {
open := aMesh(t)
stopped, cancel := context.WithCancel(t.Context())
cancel()
_, err := routeNamesInTheMesh(stopped, open)
if err == nil {
t.Fatal("no failure was raised")
}
if !strings.Contains(err.Error(), "cannot be read") {
t.Fatalf("the failure does not say the mesh could not be read: %v", err)
}
}
-178
View File
@@ -1,178 +0,0 @@
package broker
import (
"fmt"
"os"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
)
const twoSeconds = 2 * time.Second
// A running mesh already holds consumers made before the delivery subject carried the stream
// (novox/hq 04-ISSUES/146). The server will not change a push consumer's delivery subject in place,
// so bringing one to match must replace it — and must not replay what it already acknowledged
// (novox/hq 04-ISSUES/156).
//
// docker run -d --rm --name t -p 14231:4222 nats:2.10-alpine -js
// MESH_TEST_NATS=nats://127.0.0.1:14231 go test ./internal/broker/ -run TestUpgrading
func TestUpgradingAConsumerWhoseDeliverySubjectMoved(t *testing.T) {
url := os.Getenv("MESH_TEST_NATS")
if url == "" {
t.Skip("MESH_TEST_NATS unset")
}
js, err := Dial(url)
if err != nil {
t.Fatal(err)
}
defer js.Close()
// The stream exactly as the mesh's own is — one declaration per node, always the newest.
// Reproduced rather than approximated: the first version of this test used a plain stream and
// a plain consumer, and the server accepted the update it refuses in a running mesh, so the
// test passed against the very code that was crash-looping on the control node.
const stream, name = "NODES", "novox"
subject := "mesh.node." + name + ".declare"
_ = js.js.DeleteStream(stream)
if _, err := js.js.AddStream(&nats.StreamConfig{
Name: stream, Subjects: []string{"mesh.node.*.declare"},
MaxMsgsPerSubject: 1, Storage: nats.MemoryStorage,
}); err != nil {
t.Fatal(err)
}
defer func() { _ = js.js.DeleteStream(stream) }()
for i := 0; i < 6; i++ {
if _, err := js.js.Publish(subject, []byte(fmt.Sprint(i))); err != nil {
t.Fatal(err)
}
}
// The consumer as a running mesh holds it: made before the subject carried the stream, and
// otherwise exactly what NodeConsumer asks for.
if _, err := js.js.AddConsumer(stream, &nats.ConsumerConfig{
Durable: name, AckPolicy: nats.AckExplicitPolicy,
AckWait: 300 * time.Second, MaxDeliver: -1,
FilterSubject: subject,
DeliverSubject: "_DELIVER." + name,
}); err != nil {
t.Fatal(err)
}
// It acknowledged the first four. Those must not come back.
sub, err := js.js.SubscribeSync(subject, nats.Bind(stream, name))
if err != nil {
t.Fatal(err)
}
for i := 0; i < 1; i++ {
m, err := sub.NextMsg(twoSeconds)
if err != nil {
t.Fatalf("message %d never arrived: %v", i, err)
}
if err := m.AckSync(); err != nil {
t.Fatal(err)
}
}
// **The subscription stays up.** In a running mesh the machine is attached to this consumer
// the whole time — that is what a node listening for its declaration IS. The first version of
// this test unsubscribed first, and the server then accepted an update it refuses while a
// subscriber is bound, so the test passed against the code that was crash-looping.
defer func() { _ = sub.Unsubscribe() }()
// Now the upgrade: the consumer the controller asserts on every start, with the subject that
// carries the stream.
want := NodeConsumer(name)
var notes []string
js.Note = func(f string, a ...any) { notes = append(notes, fmt.Sprintf(f, a...)) }
if err := js.EnsureConsumer(want); err != nil {
t.Fatalf("a consumer the mesh already held could not be brought to match, which is the "+
"control plane failing to start: %v", err)
}
info, err := js.js.ConsumerInfo(stream, name)
if err != nil {
t.Fatal(err)
}
// It KEEPS the subject it had. Moving it would need the holder's grant to have widened first,
// and that grant travels in the bus's user list, which a machine applies minutes later.
if got := info.Config.DeliverSubject; got != "_DELIVER."+name {
t.Fatalf("the consumer a machine is bound to was moved to %q; a machine not yet allowed "+
"to subscribe there is a machine that hears nothing", got)
}
if len(notes) != 1 {
t.Fatalf("keeping it was not reported, so it would be invisible: %v", notes)
}
if !strings.Contains(notes[0], "keeps working") {
t.Fatalf("the note does not say the consumer still works: %q", notes[0])
}
// And the machine bound to it is still being delivered to — the point of keeping it.
if _, err := js.js.Publish(subject, []byte("after the assertion")); err != nil {
t.Fatal(err)
}
m, err := sub.NextMsg(twoSeconds)
if err != nil {
t.Fatalf("the machine stopped hearing its declarations after the assertion: %v", err)
}
if string(m.Data) != "after the assertion" {
t.Fatalf("delivered %q", m.Data)
}
// Asserting again is a no-op, or the controller crash-loops on its own restart.
if err := js.EnsureConsumer(want); err != nil {
t.Fatalf("the second assertion failed: %v", err)
}
}
// And where nothing is bound, the subject DOES move — that is 04-ISSUES/146's fix, which this must
// not undo. The controller's own two consumers are in exactly this position: it asserts them before
// it subscribes.
func TestAConsumerNothingIsBoundToDoesMove(t *testing.T) {
url := os.Getenv("MESH_TEST_NATS")
if url == "" {
t.Skip("MESH_TEST_NATS unset")
}
js, err := Dial(url)
if err != nil {
t.Fatal(err)
}
defer js.Close()
const stream, name = "NODES", "shanks"
subject := "mesh.node." + name + ".declare"
_ = js.js.DeleteStream(stream)
if _, err := js.js.AddStream(&nats.StreamConfig{
Name: stream, Subjects: []string{"mesh.node.*.declare"},
MaxMsgsPerSubject: 1, Storage: nats.MemoryStorage,
}); err != nil {
t.Fatal(err)
}
defer func() { _ = js.js.DeleteStream(stream) }()
if _, err := js.js.AddConsumer(stream, &nats.ConsumerConfig{
Durable: name, AckPolicy: nats.AckExplicitPolicy,
AckWait: 300 * time.Second, MaxDeliver: -1,
FilterSubject: subject,
DeliverSubject: "_DELIVER." + name,
}); err != nil {
t.Fatal(err)
}
want := NodeConsumer(name)
if err := js.EnsureConsumer(want); err != nil {
t.Fatal(err)
}
info, err := js.js.ConsumerInfo(stream, name)
if err != nil {
t.Fatal(err)
}
if got := info.Config.DeliverSubject; got != DeliverSubjectFor(want) {
t.Fatalf("delivery subject is %q, wanted %q -- issue 146's fix no longer applies to a "+
"consumer nothing is holding", got, DeliverSubjectFor(want))
}
}
+11 -17
View File
@@ -62,22 +62,6 @@ func TestTheInstallersFirstUserListIsWhatTheControllerWouldCompose(t *testing.T)
// theCarriedAccounts is the accounts file the installer's template writes at genesis.
func theCarriedAccounts(t *testing.T) string {
t.Helper()
for _, r := range theTemplate(t) {
if r["id"] == "bus-accounts" {
content, _ := r["content"].(string)
if content == "" {
t.Fatal("the template's accounts file is empty, so the bus would refuse every connection")
}
return content
}
}
t.Fatal("the template carries no accounts file, so a mesh raised from it has a bus nobody may use")
return ""
}
// theTemplate is the installer's bundle, as resources.
func theTemplate(t *testing.T) []map[string]any {
t.Helper()
path := filepath.Join("..", "..", "..", "mesh-host", "examples", "foundation-first-node-nats.lock")
raw, err := os.ReadFile(path)
@@ -97,7 +81,17 @@ func theTemplate(t *testing.T) []map[string]any {
if err := json.Unmarshal([]byte(strings.Join(lines, "\n")), &bundle); err != nil {
t.Fatalf("the template is not readable: %v", err)
}
return bundle.Resources
for _, r := range bundle.Resources {
if r["id"] == "bus-accounts" {
content, _ := r["content"].(string)
if content == "" {
t.Fatal("the template's accounts file is empty, so the bus would refuse every connection")
}
return content
}
}
t.Fatal("the template carries no accounts file, so a mesh raised from it has a bus nobody may use")
return ""
}
// subjectsIn reads one allow-list out of a composed accounts file.
+2 -60
View File
@@ -26,16 +26,6 @@ import (
type JetStream struct {
conn *nats.Conn
js nats.JetStreamContext
// Note is how this says something it decided not to fail over. Nil is silent, which is only
// right for a caller that has no way to report; the controller sets it.
Note func(string, ...any)
}
// note reports without requiring a caller to have set one.
func (j *JetStream) note(format string, args ...any) {
if j.Note != nil {
j.Note(format, args...)
}
}
// Dial connects and returns the controller's JetStream handle.
@@ -194,60 +184,12 @@ func (j *JetStream) EnsureConsumer(c Consumer) error {
// without the other is refused by the server with a message that does not say which half is
// missing.
if c.Queue != "" || c.Push {
// **Per consumer, which means per stream as well as per name** (novox/hq 04-ISSUES/146).
// A push consumer delivers onto an ordinary subject, and everything subscribed to that
// subject gets a copy. The controller holds a consumer called `controller` on CONTROL and
// another called `controller` on EVENTS, and both were given `_DELIVER.controller` — so the
// one process, holding both subscriptions, acted on every message twice. It enrolled a
// joining machine twice from one request, minting a second credential that replaced the one
// the machine had just been given; the same doubling applied to every report and every
// event the controller follows.
//
// The stream is in the name because the pair is what identifies a consumer — the server
// scopes a durable's name to its stream, and this subject is the only place that scoping
// was dropped. Already within what the controller may subscribe (`_DELIVER.controller.>`),
// so no permission moves.
want.DeliverSubject = DeliverSubjectFor(c)
want.DeliverSubject = "_DELIVER." + c.Name
}
switch have, err := j.js.ConsumerInfo(c.Stream, c.Name); {
switch _, err := j.js.ConsumerInfo(c.Stream, c.Name); {
case err == nil:
// Where an existing consumer starts is its history, not something an assertion may move:
// the server refuses a changed deliver policy outright. Carried across, so asserting twice
// is the no-op a restart depends on.
want.DeliverPolicy = have.Config.DeliverPolicy
want.OptStartSeq = have.Config.OptStartSeq
want.OptStartTime = have.Config.OptStartTime
if _, err := j.js.UpdateConsumer(c.Stream, want); err != nil {
// **A consumer that works is not replaced to make its name tidier**
// (novox/hq 04-ISSUES/156).
//
// The server will not move a push consumer's delivery subject while a subscriber is
// bound to it, and answers `consumer name already in use` — a message about the name,
// for a conflict about the subject. A node is bound to its declaration consumer the
// whole time it is up; that IS a node listening. So when 04-ISSUES/146 put the stream
// into the subject, every node consumer in a running mesh became one this could not
// bring to match, and the control plane crash-looped on the assertion it makes before
// it serves. A fresh mesh showed nothing: nothing was bound.
//
// Kept rather than deleted and re-made. Re-making moves the subject, and a holder may
// not be allowed to subscribe to the new one yet — the wider grant travels in the bus's
// user list, which this same control plane composes and a machine applies minutes
// later. Re-making here would have silenced every machine in the mesh, which is worse
// than the collision it was fixing and harder to undo.
//
// Kept rather than fatal, which is what 146's change intended and did not do: the bare
// subject it replaces still delivers, and it collides only where one holder has two
// consumers of one name. That is the controller's own pair, and the controller is not
// bound to them while it asserts, so those do move. A node has one consumer and nothing
// to collide with.
if have.Config.DeliverSubject != want.DeliverSubject {
j.note("consumer %s on %s still delivers to %q and not %q: %v. It keeps working; "+
"the subject moves on an assertion made while nothing is bound to it",
c.Name, c.Stream, have.Config.DeliverSubject, want.DeliverSubject, err)
return nil
}
return fmt.Errorf("bringing consumer %s on %s to match: %w", c.Name, c.Stream, err)
}
return nil
+2 -8
View File
@@ -269,13 +269,7 @@ func PermissionsFor(p Principal) (Permissions, error) {
"mesh.control." + p.Node + ".>",
"$JS.API.CONSUMER.INFO.NODES." + p.Node,
}
// The deliver subject carries the stream as well as the consumer's name, so what a
// subscriber is permitted has to carry it too (novox/hq 04-ISSUES/146). The bare name
// stays: an existing consumer keeps delivering where it always did until the controller's
// next assertion moves it, and a permission that only allowed the new shape would refuse
// every node in the mesh for exactly as long as that took.
sub = []string{"mesh.node." + p.Node + ".declare",
"_DELIVER." + p.Node, "_DELIVER." + p.Node + ".>"}
sub = []string{"mesh.node." + p.Node + ".declare", "_DELIVER." + p.Node}
case KindModule:
// 1. Its own namespace: it publishes its events there and serves its tools there. Nothing
@@ -329,7 +323,7 @@ func PermissionsFor(p Principal) (Permissions, error) {
// take work over the new bus was refused the asking (2026-09-28).
worker := "SEAT_" + upperSnake(s.Name) + "_worker"
stream := seatStreamName(s.Name)
sub = append(sub, "_DELIVER."+worker, "_DELIVER."+worker+".>")
sub = append(sub, "_DELIVER."+worker)
pub = append(pub, "$JS.API.CONSUMER.INFO."+stream+"."+worker, "$JS.ACK."+stream+"."+worker+".>")
for _, a := range s.Accepts {
sub = append(sub, seatSubject(s, "accept", a))
-18
View File
@@ -94,24 +94,6 @@ func MeshStreams() []Stream {
}
}
// DeliverSubjectFor is where a push consumer's messages land.
//
// **Per consumer, which means per stream as well as per name** (novox/hq 04-ISSUES/146). A push
// consumer delivers onto an ordinary subject, and everything subscribed to that subject gets a
// copy. The controller holds a consumer called `controller` on CONTROL and another called
// `controller` on EVENTS; while both were given `_DELIVER.controller`, the one process holding
// both subscriptions acted on every message twice — a joining machine was enrolled twice from one
// request, and the second enrolment minted a credential that replaced the one the machine had just
// been handed. Every report and every followed event doubled the same way, silently: nothing is
// redelivered, no count is wrong, the work simply happens twice.
//
// The stream belongs in it because the pair is what identifies a consumer — the server scopes a
// durable's name to its stream, and this subject was the one place that scoping was dropped. It
// stays inside what a controller may already subscribe (`_DELIVER.controller.>`).
func DeliverSubjectFor(c Consumer) string {
return "_DELIVER." + c.Name + "." + c.Stream
}
// An Asserter is the part of a JetStream connection stream assertion needs. Narrow on purpose: it
// keeps this testable without a server, and keeps the client library out of everything that only
// wants to know what the streams are.
-27
View File
@@ -233,30 +233,3 @@ func containsStep(steps []string, want string) bool {
}
return false
}
// **Two consumers may share a name, and must not share a delivery subject** (novox/hq
// 04-ISSUES/146).
//
// A push consumer delivers onto an ordinary subject and everything subscribed to it gets a copy.
// The controller holds a consumer called `controller` on CONTROL and another called `controller` on
// EVENTS; while both were given `_DELIVER.controller`, the one process holding both subscriptions
// acted on every message twice — a joining machine enrolled twice from one request, with the second
// enrolment minting a credential that replaced the one the machine had just been handed.
//
// Checked here rather than against a server because it is a property of what the mesh asks for, and
// because the failure it produces is silent: every count is right, nothing is redelivered, and the
// work simply happens twice.
func TestNoTwoConsumersDeliverOntoTheSameSubject(t *testing.T) {
seen := map[string]string{}
for _, c := range MeshConsumers() {
if !c.Push && c.Queue == "" {
continue
}
subject := DeliverSubjectFor(c)
if other, taken := seen[subject]; taken {
t.Errorf("%s on %s and %s deliver onto %s, so whoever holds both acts on every "+
"message twice", c.Name, c.Stream, other, subject)
}
seen[subject] = c.Name + " on " + c.Stream
}
}
+2 -2
View File
@@ -34,11 +34,11 @@ accounts {
} }
{ user: "node.one", password: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn", permissions: {
publish: { allow: ["$JS.ACK.NODES.one.>", "$JS.API.CONSUMER.INFO.NODES.one", "mesh.control.one.>"] }
subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.node.one.declare"] }
subscribe: { allow: ["_DELIVER.one", "_INBOX.node.one.>", "mesh.node.one.declare"] }
} }
{ user: "one.telegram", password: "$2a$11$tttttttttttttttttttttt", permissions: {
publish: { allow: ["$JS.ACK.EVENTS.one_telegram.>", "$JS.ACK.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker.>", "$JS.API.CONSUMER.INFO.EVENTS.one_telegram", "$JS.API.CONSUMER.INFO.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_telegram", "mesh.seat.telegram-sender.event.delivered", "mesh.seat.telegram-sender.event.failed"] }
subscribe: { allow: ["_DELIVER.SEAT_TELEGRAM_SENDER_worker", "_DELIVER.SEAT_TELEGRAM_SENDER_worker.>", "_INBOX.one.telegram.>", "mesh.mod.telegram.tool.>", "mesh.seat.telegram-sender.accept.send"] }
subscribe: { allow: ["_DELIVER.SEAT_TELEGRAM_SENDER_worker", "_INBOX.one.telegram.>", "mesh.mod.telegram.tool.>", "mesh.seat.telegram-sender.accept.send"] }
allow_responses: { max: 1, ttl: "1m" }
} }
{ user: "two.audit", password: "$2a$11$aaaaaaaaaaaaaaaaaaaaaa", permissions: {
-59
View File
@@ -153,26 +153,6 @@ func (b *Build) problems(module string) []string {
"%s: %q is a bundle and says no language, so nothing can choose a compiler "+
"for it", module, a.Name))
}
// **A system, for a language that compiles to a binary** (novox/hq ADR 0142). A binary
// is pinned to one operating system at link time so a host refuses to touch a machine
// it was not built for (novox/hq ADR 0005); an artifact that says nothing would be
// compiled for whatever the build machine happened to be, which reads as portable and
// is not.
if compiled := compilesToABinary(a.Language); compiled && strings.TrimSpace(a.System) == "" {
problems = append(problems, fmt.Sprintf(
"%s: %q is compiled to a binary and says no system, so it would be built for "+
"whatever the build machine happens to be. Declare one artifact per "+
"system: %s", module, a.Name, spokenSystems()))
} else if !compiled && strings.TrimSpace(a.System) != "" {
problems = append(problems, fmt.Sprintf(
"%s: %q names the system %q and is written in %q, which compiles to code that "+
"runs anywhere — a system that decides nothing reads as though it did",
module, a.Name, a.System, a.Language))
} else if compiled && !knownSystem(a.System) {
problems = append(problems, fmt.Sprintf(
"%s: %q is built for %q, and a system is %s",
module, a.Name, a.System, spokenSystems()))
}
} else {
if a.From == "" {
problems = append(problems, fmt.Sprintf(
@@ -213,42 +193,3 @@ func oneOrOther(n int) string {
}
return "them"
}
// Systems the mesh builds binaries for, which is the set a host may be pinned to (novox/hq ADR 0005).
//
// **A closed list, and the host's own, not the compiler's.** These are not the values a Go toolchain
// would call an operating system — the difference between two of them is a C library, not a kernel.
// They are what a machine reports itself to be and what a host is linked to refuse, so the list that
// matters is the one the host understands.
var systems = []string{"alpine", "android", "arch"}
// knownSystem is whether the mesh builds for it.
func knownSystem(system string) bool {
want := strings.ToLower(strings.TrimSpace(system))
for _, s := range systems {
if s == want {
return true
}
}
return false
}
// spokenSystems is the list as a refusal says it, so a reader is one edit from right.
func spokenSystems() string {
return strings.Join(systems, ", ")
}
// compilesToABinary is whether this language's bundle is a binary for one operating system rather
// than code that runs wherever its interpreter does.
//
// **Asked of the language, not of the artifact.** A module says what it is written in; what that
// implies is the mesh's to know, exactly as the compiler is (novox/hq ADR 0142). Asking the artifact
// would let two artifacts in one language disagree about whether they are portable.
func compilesToABinary(language string) bool {
switch strings.ToLower(strings.TrimSpace(language)) {
case "go":
return true
default:
return false
}
}
-82
View File
@@ -1,82 +0,0 @@
package catalogue
import (
"strings"
"testing"
)
// bundleFor is a manifest whose one artifact is a bundle in the given language and system.
func bundleFor(language, system string) Manifest {
return Manifest{Module: "a-component", Build: &Build{Artifacts: []Artifact{
{Name: "binary", Kind: ArtifactBundle, Language: language, System: system},
}}}
}
func problemsOf(t *testing.T, m Manifest) string {
t.Helper()
return strings.Join(m.Build.problems(m.Module), "\n")
}
// **A language that compiles to a binary must say which system.**
//
// A binary is pinned to one operating system at link time, so a host refuses to touch a machine it
// was not built for. An artifact that says nothing would be compiled for whatever the build machine
// happened to be — which reads as portable and is not, and is the fault this check exists for.
func TestABinaryMustSayWhichSystemItIsFor(t *testing.T) {
got := problemsOf(t, bundleFor("go", ""))
if !strings.Contains(got, "says no system") {
t.Fatalf("a compiled bundle with no system was accepted:\n%s", got)
}
// And the refusal names what it could have said, so a reader is one edit from right.
for _, system := range []string{"alpine", "android", "arch"} {
if !strings.Contains(got, system) {
t.Fatalf("the refusal does not name %q as a choice:\n%s", system, got)
}
}
}
func TestABinaryThatNamesASystemIsAccepted(t *testing.T) {
if got := problemsOf(t, bundleFor("go", "arch")); got != "" {
t.Fatalf("a compiled bundle naming a system was refused:\n%s", got)
}
}
// A system the mesh does not build for is refused where it is written. These are the host's own
// names, not a compiler's: the difference between two of them is a C library rather than a kernel,
// so a value that looks like an operating system to a toolchain is still wrong here.
func TestASystemTheMeshDoesNotBuildForIsRefused(t *testing.T) {
for _, wrong := range []string{"linux", "debian", "darwin"} {
got := problemsOf(t, bundleFor("go", wrong))
if !strings.Contains(got, "and a system is") {
t.Fatalf("%q was accepted as a system:\n%s", wrong, got)
}
}
}
// **And a language that runs anywhere must not name one.** A system that decides nothing reads as
// though it did, which is the same fault as a restriction that restricts nothing (novox/hq ADR 0045).
func TestAPortableBundleMayNotNameASystem(t *testing.T) {
got := problemsOf(t, bundleFor("typescript", "arch"))
if !strings.Contains(got, "runs anywhere") {
t.Fatalf("a portable bundle was allowed to name a system:\n%s", got)
}
}
func TestAPortableBundleNamingNoSystemIsAccepted(t *testing.T) {
if got := problemsOf(t, bundleFor("typescript", "")); got != "" {
t.Fatalf("an ordinary bundle was refused:\n%s", got)
}
}
// One component, one artifact per system: the shape the mesh's own binaries are declared in, and the
// reason the target is the artifact's rather than the recipe's.
func TestOneArtifactPerSystemIsAccepted(t *testing.T) {
m := Manifest{Module: "the-host", Build: &Build{Artifacts: []Artifact{
{Name: "arch", Kind: ArtifactBundle, Language: "go", System: "arch"},
{Name: "alpine", Kind: ArtifactBundle, Language: "go", System: "alpine"},
{Name: "android", Kind: ArtifactBundle, Language: "go", System: "android"},
}}}
if got := problemsOf(t, m); got != "" {
t.Fatalf("one artifact per system was refused:\n%s", got)
}
}
@@ -1,47 +0,0 @@
package catalogue
import (
"os"
"path/filepath"
"testing"
)
// TestEveryCatalogueManifestParses runs the real catalogue through the real gate.
//
// Not a fixture: the point is whether the manifests as written are accepted by the control plane that
// will read them, and a copy of one manifest proves nothing about the other seventy-one.
func TestEveryCatalogueManifestParses(t *testing.T) {
root := os.Getenv("MESH_CATALOGUE")
if root == "" {
t.Skip("set MESH_CATALOGUE to a catalogue checkout to run this")
}
found, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
if err != nil || len(found) == 0 {
t.Fatalf("no manifests under %s: %v", root, err)
}
named, routed := 0, 0
for _, p := range found {
raw, err := os.ReadFile(p)
if err != nil {
t.Fatalf("%s: %v", p, err)
}
m, err := ParseManifest(raw)
if err != nil {
t.Errorf("%s: %v", filepath.Base(filepath.Dir(p)), err)
continue
}
for _, l := range m.Listens {
if l.Name != "" {
named++
}
}
for port := range RoutedPorts(m) {
_ = port
routed++
}
}
t.Logf("%d manifests, %d named endpoints, %d routed endpoints resolved", len(found), named, routed)
if named == 0 {
t.Fatal("no endpoint in the catalogue is named, so this proved nothing")
}
}
@@ -1,71 +0,0 @@
package catalogue
import (
"os"
"strings"
"testing"
)
// **A machine trusts the mesh's authority because a module put its root there** (novox/hq ADR
// 0147, issue 129). The module carries a shell script and a unit, and both are worthless unless
// the mesh fills in where the authority is — which is the one thing about it the manifest cannot
// state, because the authority's address is a fact about the mesh and not about the module.
//
// So what is checked here is the rendering, not the parsing: the script the machine will run
// names the authority it was bound to, and the unit runs that script both ways. The verification
// itself — a plain client trusting an internal name on a machine holding this, and failing on one
// that does not — is the lab's, and cannot be had here.
func TestCaTrustRendersTheAuthorityItWasBoundTo(t *testing.T) {
raw, err := os.ReadFile("../../../mesh-catalog/modules/ca-trust/module.json")
if err != nil {
t.Skipf("the catalogue is not beside this checkout: %v", err)
}
m, err := ParseManifest(raw)
if err != nil {
t.Fatalf("the trust module does not parse:\n%v", err)
}
r := Resolution{
Node: "workstation",
Modules: []Manifest{m},
Needs: []Needed{{
Name: "internal-acme-ca", From: "anchor", At: "anchor.internal", For: "ca-trust",
Serves: map[string]any{
"port": float64(9000), "path": "/acme/acme/directory", "roots": "/roots.pem",
},
}},
}
out, err := r.Declaration(Rendering{})
if err != nil {
t.Fatalf("the trust module could not be composed for a machine: %v", err)
}
script := fileNamed(out, "ca-trust.anchor")
if script == nil {
t.Fatalf("nothing writes the script the unit runs: %v", out)
}
body, _ := script["content"].(string)
if !strings.Contains(body, "https://anchor.internal:9000/roots.pem") {
t.Errorf("the script does not fetch from the authority it was bound to:\n%s", body)
}
if script["mode"] != "0755" {
t.Errorf("the script is written %v, which systemd cannot execute", script["mode"])
}
unit := fileNamed(out, "ca-trust.unit")
if unit == nil {
t.Fatalf("no unit: %v", out)
}
text, _ := unit["content"].(string)
// Both halves. A unit that only installs the anchor leaves a machine trusting an authority
// nobody assigned it to any more, which is the half issue 129 asked for by name.
for _, want := range []string{
"ExecStart=" + script["path"].(string) + " install",
"ExecStop=" + script["path"].(string) + " remove",
"RemainAfterExit=yes",
} {
if !strings.Contains(text, want) {
t.Errorf("the unit does not say %q:\n%s", want, text)
}
}
}
+3 -138
View File
@@ -897,35 +897,6 @@ func (r Resolution) Rules(with Rendering) ([]Rule, error) {
if err != nil {
return nil, err
}
// And how far each endpoint reaches, which says the same thing to the filter and more
// besides (novox/hq ADR 0138). Folded in here rather than beside: the filter has one
// question — from where — and a reach answers it, so giving it two inputs would let them
// disagree. Reaches refuses a port that both name, so this cannot silently prefer one.
reaches, err := Reaches(m, with.Settings[m.Module])
if err != nil {
return nil, err
}
// **Only for an endpoint the proxy does not serve.** A routed endpoint's port is how the
// proxy reaches it and nothing else (ADR 0045), so `public` there asks for a public name and
// says nothing about the port — opening it to the world as well would undo the arrangement
// the proxy exists for, and would silently reopen a port an operator had narrowed.
//
// Found by trying to express a real module: one whose routed name must be public and whose
// machine-side port must not be. Under one value for both, there was no way to say it.
routed := RoutedPorts(m)
for port, reach := range reaches {
if routed[port] {
continue
}
source, ok := FilterSource(reach)
if !ok {
return nil, fmt.Errorf("%s: %q is not a reach the filter can read", m.Module, reach)
}
if e == nil {
e = map[int]string{}
}
e[port] = source
}
if e != nil {
exposure[m.Module] = e
}
@@ -1094,16 +1065,7 @@ func (r Resolution) contributions(settings SettingsBy, grants []Grant,
if err != nil {
return nil, fmt.Errorf("%s contributing to %s: %w", m.Module, to, err)
}
reaches, err := Reaches(m, settings[m.Module])
if err != nil {
return nil, fmt.Errorf("%s contributing to %s: %w", m.Module, to, err)
}
blocks, err := Endpoints(m, settings[m.Module])
if err != nil {
return nil, fmt.Errorf("%s contributing to %s: %w", m.Module, to, err)
}
portOfEndpoint(values, endpointPorts(m))
composeName(values, r.PublicDomain, r.At, reaches, endpointPorts(m), blocks)
composeName(values, r.PublicDomain, r.At)
out[to] = append(out[to], Contribution{From: m.Module, Values: values})
}
// Several contributions to one requirement (ADR 0094's sibling for `contributes`): an
@@ -1117,16 +1079,7 @@ func (r Resolution) contributions(settings SettingsBy, grants []Grant,
if err != nil {
return nil, fmt.Errorf("%s contributing %s to %s: %w", m.Module, local, to, err)
}
reaches, err := Reaches(m, settings[m.Module])
if err != nil {
return nil, fmt.Errorf("%s contributing %s to %s: %w", m.Module, local, to, err)
}
blocks, err := Endpoints(m, settings[m.Module])
if err != nil {
return nil, fmt.Errorf("%s contributing %s to %s: %w", m.Module, local, to, err)
}
portOfEndpoint(values, endpointPorts(m))
composeName(values, r.PublicDomain, r.At, reaches, endpointPorts(m), blocks)
composeName(values, r.PublicDomain, r.At)
out[to] = append(out[to], Contribution{From: m.Module, Values: values})
}
}
@@ -1157,44 +1110,10 @@ func (r Resolution) contributions(settings SettingsBy, grants []Grant,
// the running mesh keeps serving the full names it has. And a labelled contribution on a node with
// no public domain composes nothing — there is nothing to join it to — which reads downstream as a
// route that named no host, the same as it would have before this existed.
func composeName(values map[string]any, publicDomain, internalDomain string, reaches map[int]string,
ports map[string]int, blocks map[string]Endpoint) {
func composeName(values map[string]any, publicDomain, internalDomain string) {
if values == nil {
return
}
// **The subdomain an assignment gave this endpoint**, before the name is joined (novox/hq ADR
// 0138). The module contributes a label because it names its own parts; an assignment may say a
// different one, because where a thing lives under a domain is the operator's to choose and used
// to require editing the module to change.
if name, ok := values[RouteEndpoint].(string); ok {
if ep, said := blocks[strings.TrimSpace(name)]; said && ep.Label != "" {
values["label"] = ep.Label
}
}
// **How far the endpoint this route serves reaches decides which names exist** (novox/hq ADR
// 0138). Both were composed whenever the node had both domains, so every routed module got a
// public name and an internal one whether anybody wanted them or not — and a certificate for
// each, because the proxy certifies the names it is given.
//
// Joined by the port: a route entry names the port it serves and the module declares a listen on
// it. An entry with no port is not an endpoint's route but a rule about a name — a path-level
// refusal shadowing another route — and it inherits whatever that route's names turned out to
// be, which is why it is left alone here.
//
// Nothing said is both names, as before. That is what keeps every mesh already running identical
// until an assignment speaks.
wantPublic, wantInternal := true, true
if port, ok := endpointPortOf(values, ports); ok {
if reach, said := reaches[port]; said {
wantPublic, wantInternal = WantsPublicName(reach), WantsInternalName(reach)
}
}
if !wantPublic {
publicDomain = ""
}
if !wantInternal {
internalDomain = ""
}
if _, already := values["name"]; already {
// A full name was given rather than a label. Left as-is: this is the legacy shape, and the
// point of the label is to not have to write the full name — a contribution that wrote both
@@ -1814,57 +1733,3 @@ func prepared(from map[string]any) map[string]any {
delete(step, "reload-on")
return step
}
// endpointPortOf is the port the endpoint a route serves listens on: looked up by the name the route
// gives, or read from the port it repeats (novox/hq ADR 0138).
//
// `ports` maps this module's endpoint names to their ports, computed once per module rather than
// re-scanned per contribution.
func endpointPortOf(values map[string]any, ports map[string]int) (int, bool) {
if name, ok := values[RouteEndpoint].(string); ok {
if port, found := ports[strings.TrimSpace(name)]; found {
return port, true
}
}
return asPort(values["port"])
}
// endpointPorts is a module's endpoint names against the ports they listen on.
func endpointPorts(m Manifest) map[string]int {
out := map[string]int{}
for _, l := range m.Listens {
if name := strings.TrimSpace(l.Name); name != "" {
out[name] = l.Port
}
}
return out
}
// portOfEndpoint fills in the port of the endpoint a contribution names, in place.
//
// **A contribution that names an endpoint must still carry that endpoint's port**, because everything
// downstream reads the port: the provider is told where to reach the consumer, and the machine-side
// redirection that turns a declared port into the number the machine published is keyed on it
// (atMachinePort). A route that named only its endpoint left the proxy with no port at all, and a
// proxy with no port has nothing to dial.
//
// Found before it shipped and after the catalogue had already been changed to name endpoints — the
// manifests were merged and the mesh had not yet picked them up, so nothing was broken yet. The
// declared port, not the machine one: the redirection happens later and is keyed on the declared
// number, so filling in the machine port here would be redirected a second time or not at all.
func portOfEndpoint(values map[string]any, ports map[string]int) {
if values == nil {
return
}
if _, already := values["port"]; already {
// A route that says both is its own answer; the older shape repeated the port and is still read.
return
}
name, ok := values[RouteEndpoint].(string)
if !ok {
return
}
if port, found := ports[strings.TrimSpace(name)]; found {
values["port"] = port
}
}
-205
View File
@@ -1,205 +0,0 @@
package catalogue
import (
"strings"
"testing"
)
// aMediaServer is the shape one port number per key cannot express: two endpoints of different kinds.
// A web surface a proxy serves under a subdomain, and a protocol port clients dial directly because
// the client expects that number.
func aMediaServer() Manifest {
return Manifest{
Module: "media",
Listens: []Listening{
{Name: "web", Port: 80, From: FromMesh, Why: "the app, behind the proxy"},
{Name: "stream", Port: 32400, From: FromEverywhere, Fixed: true,
Why: "the client dials this number; the protocol chose it"},
},
Contributes: map[string]map[string]any{
"route": {"label": "media", RouteEndpoint: "web"},
},
// Both endpoints are published by its container, which is what lets a machine port be given
// for either: the mesh moves a port the module publishes, never one it merely listens on.
Resources: []map[string]any{
{"id": "server", "type": "container", "name": "media",
"ports": []any{"80", "32400"}},
},
}
}
// **A route names the endpoint it serves.** A route and a listen both carried a port and nothing said
// they were the same thing; now one of them says so.
func TestARouteNamesTheEndpointItServes(t *testing.T) {
m := aMediaServer()
if port, ok := EndpointPort(m, "web"); !ok || port != 80 {
t.Fatalf("the web endpoint resolves to %d (%v), want 80", port, ok)
}
if port, ok := EndpointPort(m, "stream"); !ok || port != 32400 {
t.Fatalf("the stream endpoint resolves to %d (%v), want 32400", port, ok)
}
if _, ok := EndpointPort(m, "absent"); ok {
t.Fatal("an endpoint the module does not declare resolved to a port")
}
}
// And the routed set is read through the name, so the endpoint the proxy serves is known without a
// reader joining two numbers.
func TestTheRoutedEndpointIsFoundByName(t *testing.T) {
routed := RoutedPorts(aMediaServer())
if !routed[80] {
t.Fatalf("the routed endpoint was not found by name: %v", routed)
}
// And the directly-dialled one is not routed, which is what lets its reach govern its port.
if routed[32400] {
t.Fatalf("the endpoint clients dial directly reads as routed: %v", routed)
}
}
// **Two endpoints of different shapes, configured as themselves.** The web endpoint's reach asks for
// names and leaves its port to the proxy; the stream endpoint's reach governs its port, because
// clients dial it and there is no name.
func TestTwoEndpointsOfDifferentShapesAreConfiguredSeparately(t *testing.T) {
m := aMediaServer()
settings := SettingsBy{"media": {{From: "node anchor", Values: map[string]any{
ReachSetting: map[string]any{"80": ReachBoth, "32400": ReachPublic},
}}}}
r := Resolution{Node: "anchor", Modules: []Manifest{m},
PublicDomain: "example.test", At: "anchor.internal"}
rules, err := r.Rules(Rendering{Settings: settings})
if err != nil {
t.Fatal(err)
}
for _, rule := range rules {
switch rule.Port {
case 80:
if rule.From != FromMesh {
t.Fatalf("the routed endpoint's port opened to %q; the proxy is how it is reached",
rule.From)
}
case 32400:
if rule.From != FromEverywhere {
t.Fatalf("the directly-dialled endpoint's port is %q, want anywhere", rule.From)
}
}
}
// And the routed one carries both names, asked for by the same statement.
given, err := r.contributions(settings, nil, nil)
if err != nil {
t.Fatal(err)
}
var public, internal string
for _, c := range given["route"] {
public, _ = c.Values["name"].(string)
internal, _ = c.Values["internal-name"].(string)
}
if public != "media.example.test" || internal != "media.anchor.internal" {
t.Fatalf("names are %q and %q, want both", public, internal)
}
}
// A route naming an endpoint the module does not declare reaches nothing, and is refused where it is
// written rather than resolving to no port and serving nothing.
func TestARouteNamingAnEndpointTheModuleLacksIsRefused(t *testing.T) {
m := aMediaServer()
m.Contributes["route"][RouteEndpoint] = "absent"
got := strings.Join(RouteProblems(m), "\n")
if !strings.Contains(got, "does not declare") {
t.Fatalf("a route naming an absent endpoint was accepted:\n%s", got)
}
}
// **Two endpoints called the same would make an assignment configure whichever was read last.** The
// point of a name is that it identifies one thing.
func TestTwoEndpointsWithOneNameAreRefused(t *testing.T) {
m := Manifest{Module: "twice", Listens: []Listening{
{Name: "web", Port: 80, From: FromMesh},
{Name: "web", Port: 8080, From: FromMesh},
}}
got := strings.Join(endpointNameProblems(m), "\n")
if !strings.Contains(got, "could mean either") {
t.Fatalf("two endpoints with one name were accepted:\n%s", got)
}
}
// A name that is not a name is refused where it is written: it ends up in something a person types.
func TestAnEndpointNameIsHeldToItsShape(t *testing.T) {
for _, wrong := range []string{"Web", "web port", "3000", "-web", "web_surface"} {
m := Manifest{Module: "odd", Listens: []Listening{{Name: wrong, Port: 80, From: FromMesh}}}
if got := strings.Join(endpointNameProblems(m), "\n"); !strings.Contains(got, "a name is lowercase") {
t.Fatalf("%q was accepted as an endpoint name:\n%s", wrong, got)
}
}
}
// **Every endpoint in the catalogue is unnamed today, and must stay valid.** The word ships one
// release before anything uses it.
func TestAnUnnamedEndpointIsStillValid(t *testing.T) {
m := Manifest{Module: "ordinary", Listens: []Listening{{Port: 443, From: FromEverywhere}}}
if got := endpointNameProblems(m); len(got) != 0 {
t.Fatalf("an unnamed endpoint was refused: %v", got)
}
if got := RouteProblems(m); len(got) != 0 {
t.Fatalf("a module with no route was refused: %v", got)
}
}
// **A route that names an endpoint still carries that endpoint's port.**
//
// Everything downstream reads the port: the provider is told where to reach the consumer, and the
// redirection that turns a declared port into the number the machine published is keyed on it. A route
// naming only its endpoint left the proxy with no port, and a proxy with no port has nothing to dial.
//
// Caught after the catalogue had already been changed to name endpoints, and before the mesh picked
// those manifests up — which is the only reason nothing broke.
func TestARouteNamingAnEndpointStillCarriesItsPort(t *testing.T) {
m := aMediaServer()
r := Resolution{Node: "anchor", Modules: []Manifest{m},
PublicDomain: "example.test", At: "anchor.internal"}
given, err := r.contributions(nil, nil, nil)
if err != nil {
t.Fatal(err)
}
var saw bool
for _, c := range given["route"] {
saw = true
port, ok := asPort(c.Values["port"])
if !ok {
t.Fatalf("the route carries no port, so the proxy has nothing to dial: %v", c.Values)
}
if port != 80 {
t.Fatalf("the route carries port %d, want the web endpoint's 80", port)
}
}
if !saw {
t.Fatal("the module contributed no route")
}
}
// And the declared port, not the machine one: the redirection to where the machine published it
// happens later and is keyed on the declared number, so filling the machine port in here would be
// redirected twice or not at all.
func TestTheEndpointsDeclaredPortIsFilledInNotTheMachineOne(t *testing.T) {
m := aMediaServer()
values := map[string]any{RouteEndpoint: "web", "label": "media"}
portOfEndpoint(values, endpointPorts(m))
if got, _ := asPort(values["port"]); got != 80 {
t.Fatalf("filled in port %d, want the declared 80", got)
}
// Then the ordinary redirection puts it where the machine published it.
moved := atMachinePort(values, m.Module, map[string]map[int]int{"media": {80: 20009}})
if got, _ := asPort(moved["port"]); got != 20009 {
t.Fatalf("after redirection the port is %d, want the machine's 20009", got)
}
}
// A route that repeats a port keeps it, because that is the older shape and still read.
func TestARouteThatRepeatsItsPortKeepsIt(t *testing.T) {
values := map[string]any{RouteEndpoint: "web", "port": 8080}
portOfEndpoint(values, map[string]int{"web": 80})
if got, _ := asPort(values["port"]); got != 8080 {
t.Fatalf("the port it stated was overwritten with %d", got)
}
}
@@ -1,140 +0,0 @@
package catalogue
import (
"strings"
"testing"
)
func configured(block map[string]any) SettingsBy {
return SettingsBy{"media": {{From: "node anchor",
Values: map[string]any{EndpointsSetting: block}}}}
}
// **One block per endpoint, saying all three things.** The machine port, the subdomain and the reach
// were `ports`, the route's label and `reach`, each keyed by a port number, so configuring a module
// with two endpoints of different shapes meant knowing which number was which.
func TestAnEndpointsBlockSaysPortLabelAndReach(t *testing.T) {
m := aMediaServer()
// stream's reach NARROWS what the manifest says — the manifest has it from anywhere, the
// assignment says internal. Chosen deliberately: a reach that agrees with the manifest proves
// nothing about whether the block was read at all.
settings := configured(map[string]any{
"web": map[string]any{"port": 20009, "label": "cinema", "reach": ReachBoth},
"stream": map[string]any{"reach": ReachInternal},
})
// The machine port, where the mapping is read.
given, err := GivenPorts(m, settings["media"])
if err != nil {
t.Fatal(err)
}
if given[80] != 20009 {
t.Fatalf("the web endpoint is on machine port %d, want 20009: %v", given[80], given)
}
// The reach, where the filter reads it.
r := Resolution{Node: "anchor", Modules: []Manifest{m},
PublicDomain: "example.test", At: "anchor.internal"}
rules, err := r.Rules(Rendering{Settings: settings})
if err != nil {
t.Fatal(err)
}
for _, rule := range rules {
if rule.Port == 32400 && rule.From != FromMesh {
t.Fatalf("the directly-dialled endpoint is %q; the assignment narrowed it to the private "+
"network and the manifest's 'anywhere' should not win", rule.From)
}
if rule.Port == 80 && rule.From != FromMesh {
t.Fatalf("the routed endpoint's port opened to %q; the proxy is how it is reached", rule.From)
}
}
// And the subdomain, where the name is composed — the assignment's, not the module's.
nodes, err := r.contributions(settings, nil, nil)
if err != nil {
t.Fatal(err)
}
for _, c := range nodes["route"] {
if got, _ := c.Values["name"].(string); got != "cinema.example.test" {
t.Fatalf("the public name is %q, want the label the assignment gave", got)
}
if got, _ := c.Values["internal-name"].(string); got != "cinema.anchor.internal" {
t.Fatalf("the internal name is %q, want the label the assignment gave", got)
}
}
}
// A block that says only a reach leaves the port to the mesh and the label to the module, which is the
// ordinary case and must not require writing the other two.
func TestABlockMaySayOnlyAReach(t *testing.T) {
m := aMediaServer()
settings := configured(map[string]any{"web": map[string]any{"reach": ReachInternal}})
r := Resolution{Node: "anchor", Modules: []Manifest{m},
PublicDomain: "example.test", At: "anchor.internal"}
nodes, err := r.contributions(settings, nil, nil)
if err != nil {
t.Fatal(err)
}
for _, c := range nodes["route"] {
if got, _ := c.Values["name"].(string); got != "" {
t.Fatalf("an internal endpoint composed the public name %q", got)
}
// The module's own label, untouched.
if got, _ := c.Values["internal-name"].(string); got != "media.anchor.internal" {
t.Fatalf("the internal name is %q, want the module's own label", got)
}
}
}
// An endpoint the module does not declare reaches nothing, and the refusal says what it does declare.
func TestConfiguringAnEndpointTheModuleLacksIsRefused(t *testing.T) {
_, err := Endpoints(aMediaServer(), configured(map[string]any{
"admin": map[string]any{"reach": ReachInternal}})["media"])
if err == nil || !strings.Contains(err.Error(), "does not declare") {
t.Fatalf("configuring an absent endpoint was accepted: %v", err)
}
if err != nil && !strings.Contains(err.Error(), "stream") {
t.Fatalf("the refusal does not name what the module declares: %v", err)
}
}
func TestAReachInABlockIsHeldToTheFourValues(t *testing.T) {
_, err := Endpoints(aMediaServer(), configured(map[string]any{
"web": map[string]any{"reach": "mesh"}})["media"])
if err == nil || !strings.Contains(err.Error(), "a reach is") {
t.Fatalf("a filter word was accepted as a reach: %v", err)
}
}
// **Two places giving one endpoint a machine port is the confusion this key exists to end.**
func TestAnEndpointGivenAPortTwiceIsRefused(t *testing.T) {
m := aMediaServer()
_, err := GivenPorts(m, []Layer{{From: "node anchor", Values: map[string]any{
EndpointsSetting: map[string]any{"web": map[string]any{"port": 20009}},
PortsSetting: map[string]any{"80": 30000},
}}})
if err == nil || !strings.Contains(err.Error(), "published once") {
t.Fatalf("an endpoint given two machine ports was accepted: %v", err)
}
}
// And the same for its reach, said once here and once through the older key.
func TestAnEndpointWhoseReachIsAlsoExposedIsRefused(t *testing.T) {
_, err := Endpoints(aMediaServer(), []Layer{{From: "node anchor", Values: map[string]any{
EndpointsSetting: map[string]any{"web": map[string]any{"reach": ReachInternal}},
ExposeSetting: map[string]any{"80": FromEverywhere},
}}})
if err == nil || !strings.Contains(err.Error(), "same thing in different words") {
t.Fatalf("a reach said two ways was accepted: %v", err)
}
}
// A module whose endpoints are unnamed cannot be configured this way, and is told so rather than
// having a block silently reach nothing — which is every module in the catalogue today.
func TestAModuleWithNoNamedEndpointsIsToldSo(t *testing.T) {
m := Manifest{Module: "media", Listens: []Listening{{Port: 80, From: FromMesh}}}
_, err := Endpoints(m, configured(map[string]any{"web": map[string]any{"reach": ReachBoth}})["media"])
if err == nil || !strings.Contains(err.Error(), "no endpoints by name") {
t.Fatalf("a module with no named endpoints accepted a block: %v", err)
}
}
-375
View File
@@ -265,26 +265,6 @@ func AsNftables(rules []Rule, mesh []string, outward bool, foundation []int,
b.WriteString("\t\tct state established,related accept\n")
b.WriteString("\t\tct state invalid drop\n")
b.WriteString("\t\tiif lo accept\n")
// **Anything on this machine may call anything on this machine.**
//
// Local is not a boundary this mesh draws. A service running here is callable by everything else
// running here, whatever form either takes — a package with a unit, a binary, a container. Whether
// a caller sits in a container was never meant to change the answer, and the only reason it did was
// that this chain asked about addresses: a caller on the machine carries the machine's address, a
// caller in one of its containers carries a bridge address, and a rule naming the former silently
// refused the latter.
//
// Measured: a module reaching its database on this machine's own name timed out for eleven hours
// while the machine itself could reach it, and the mesh called the machine healthy throughout
// (novox/hq 04-ISSUES/145).
//
// Asked by the link it arrives on rather than the address it comes from: anything that did not
// arrive from outside this machine, and did not arrive over the private network, is this machine's
// own. One rule for every service here, in place of a line per port that only ever covered the
// ports somebody remembered to think about.
if inward != "" {
b.WriteString(fmt.Sprintf("\t\tiifname != { %s } accept\n", inward))
}
b.WriteString("\t\ticmp type echo-request accept\n")
b.WriteString("\t\ticmpv6 type { echo-request, nd-neighbor-solicit, nd-neighbor-advert, nd-router-advert } accept\n")
@@ -556,21 +536,6 @@ const MeshWideLayer = "the mesh"
// two mappings share a number, it is an entry one of them writes over the other's, and the reader
// that finds the survivor disagrees with the reader that recomputes it.
func GivenPorts(m Manifest, layers []Layer) (map[int]int, error) {
// An endpoint's own block may put it on a machine port, which is the same thing `ports` says about
// the number rather than about the endpoint (novox/hq ADR 0138). Collected first and then let the
// older key be read, which refuses a port said twice.
byName, err := Endpoints(m, layers)
if err != nil {
return nil, err
}
named := map[int]int{}
for name, ep := range byName {
if ep.Port == 0 {
continue
}
named[endpointPorts(m)[name]] = ep.Port
}
// Every name a setting may use, and the mapping it names.
names := map[int][]publishing{}
for _, p := range publishedPorts(m) {
@@ -669,17 +634,6 @@ func GivenPorts(m Manifest, layers []Layer) (map[int]int, error) {
out[key], by[key] = at, port
}
}
// And what the endpoints' own blocks put them on. Refused rather than merged where both keys name
// one endpoint: two places giving a port is the confusion this key exists to end.
for wanted, at := range named {
if was, twice := out[wanted]; twice && was != at {
return nil, fmt.Errorf(
"%s puts its port %d on %d through %s and on %d through %s — one endpoint, two "+
"machine ports, and it is published once. Keep the endpoint's own block",
m.Module, wanted, at, EndpointsSetting, was, PortsSetting)
}
out[wanted] = at
}
if len(out) == 0 {
return nil, nil
}
@@ -747,332 +701,3 @@ func sortedPorts(of map[int]int) []int {
sort.Ints(out)
return out
}
// ReachSetting is the settings key that says how far one of a module's endpoints reaches, per node
// (novox/hq ADR 0138):
//
// {"reach": {"3000": "internal"}}
//
// **One value, three readers.** Reachability used to be settled three times over: the filter read a
// listen's source, which `expose` could override; the proxy composed a public name and an internal
// name for every route it was given, because it could; and the certificate authority followed from
// which names existed. Each was defensible and the combination was unstated, so "this endpoint must
// not be public" could not be written and was therefore enforced by nothing — while a public
// certificate for that very name was obtained anyway.
//
// It keys on the port the module declares, the same key `ports` and `expose` use. A route names that
// port too, which is what lets one statement reach the names as well as the filter: of the 36 route
// entries in the catalogue, 35 name a port that the same module declares a listen on, and the one
// that does not is a path-level refusal — a rule about a name rather than an endpoint.
const ReachSetting = "reach"
// How far an endpoint reaches. Four values, because they have to cover everything `expose` could say
// as well as the two names.
const (
// ReachMachine is this machine only: not the private network, not the world, and no name.
ReachMachine = "machine"
// ReachInternal is the private network, under the internal name and not the public one.
ReachInternal = "internal"
// ReachPublic is the world, under the public name and not the internal one.
ReachPublic = "public"
// ReachBoth is the world, under both names — each certified by its own authority.
//
// The filter cannot distinguish this from ReachPublic, and should not try: the mesh's addresses
// are a subset of anywhere. What differs is the names, which is the whole reason reach is not
// simply the filter's vocabulary with nicer words.
ReachBoth = "both"
)
// reaches is every value, in the order a refusal lists them.
var reaches = []string{ReachMachine, ReachInternal, ReachPublic, ReachBoth}
// RoutedPorts are the ports a module serves through a proxy, taken from its route contributions.
//
// **A routed endpoint's port is how the proxy reaches it, and nothing else.** That is ADR 0045's
// decision and it is older than reach: a public service listens `from: mesh`, only the proxy reaches
// it, and it is exposed by name. So `public` on a routed endpoint asks for a public *name*; opening
// that port to the world as well would undo the arrangement the proxy exists for.
//
// Measured before this was written, not reasoned: a module's routed name answered from the internet
// over TLS while its machine-side port was refused from the same place. The port is not the path.
func RoutedPorts(m Manifest) map[int]bool {
out := map[int]bool{}
note := func(values map[string]any) {
// **The endpoint it serves, by name where it says one.** A route repeating a port number is
// the older shape and still read: 35 of the catalogue's 36 route entries name a port their
// module declares a listen on (novox/hq ADR 0138).
if name, ok := values[RouteEndpoint].(string); ok {
if port, found := EndpointPort(m, name); found {
out[port] = true
return
}
}
if port, ok := asPort(values["port"]); ok {
out[port] = true
}
}
if values, ok := m.Contributes["route"]; ok {
note(values)
}
for _, values := range m.ContributesMany["route"] {
note(values)
}
return out
}
// FilterSource is the source a reach means to the packet filter.
//
// `public` and `both` are the same here. A reach that opened a port to the mesh and not to the world
// would be `internal`; there is no reach that opens it to the world and *not* to the mesh, because a
// filter cannot express "everyone except these" and nobody has asked for it.
func FilterSource(reach string) (string, bool) {
switch reach {
case ReachMachine:
return FromMachine, true
case ReachInternal:
return FromMesh, true
case ReachPublic, ReachBoth:
return FromEverywhere, true
default:
return "", false
}
}
// WantsPublicName is whether a reach asks for the route's public name to be composed.
func WantsPublicName(reach string) bool { return reach == ReachPublic || reach == ReachBoth }
// WantsInternalName is whether a reach asks for the route's internal name to be composed.
func WantsInternalName(reach string) bool { return reach == ReachInternal || reach == ReachBoth }
// Reaches reads a module's per-node reach settings: declared port → how far it reaches.
//
// It refuses a reach for a port the module does not listen on, or a value that is not one of the
// four — the "reads as a restriction and is none" fault this whole mechanism exists to prevent
// (novox/hq ADR 0043/0045). It also refuses a port that `expose` names as well: the two say the same
// thing in different words, and a module whose reach and exposure disagree would have the filter
// following one and the names following the other, which is the very confusion ADR 0138 removes.
//
// A module with no `reach` setting yields nothing, and everything behaves exactly as before: the
// filter follows the manifest's `from`, and both names are composed. That is what keeps every machine
// already running unchanged until an assignment says otherwise.
func Reaches(m Manifest, layers []Layer) (map[int]string, error) {
listened := make(map[int]bool, len(m.Listens))
for _, l := range m.Listens {
listened[l.Port] = true
}
exposed, err := Exposure(m, layers)
if err != nil {
return nil, err
}
out := map[int]string{}
// What an endpoint's own block says, which is the same statement in the shape that names the
// endpoint rather than its port (novox/hq ADR 0138). Read first so the older key, which says less,
// cannot quietly win over the newer one that says more.
blocks, err := Endpoints(m, layers)
if err != nil {
return nil, err
}
declared := endpointPorts(m)
for name, ep := range blocks {
if ep.Reach == "" {
continue
}
out[declared[name]] = ep.Reach
}
for _, layer := range layers {
raw, ok := layer.Values[ReachSetting]
if !ok {
continue
}
entries, ok := raw.(map[string]any)
if !ok {
return nil, fmt.Errorf("%s: %s is a { port: reach } map, and %q set it to something else",
m.Module, ReachSetting, layer.From)
}
for portText, value := range entries {
port, err := strconv.Atoi(portText)
if err != nil {
return nil, fmt.Errorf("%s says how far %q reaches, which is not a port", m.Module, portText)
}
if !listened[port] {
return nil, fmt.Errorf(
"%s says how far port %d reaches, which it does not listen on — the setting "+
"reaches nothing", m.Module, port)
}
reach, ok := value.(string)
if !ok || !slices.Contains(reaches, reach) {
return nil, fmt.Errorf("%s says port %d reaches %v; a reach is %s",
m.Module, port, value, strings.Join(reaches, ", "))
}
if _, both := exposed[port]; both {
return nil, fmt.Errorf(
"%s sets both %s and %s for port %d. They say the same thing in different "+
"words, and the filter would follow one while its names followed the other "+
"— which is what %s exists to stop. Keep %s",
m.Module, ReachSetting, ExposeSetting, port, ReachSetting, ReachSetting)
}
out[port] = reach
}
}
if len(out) == 0 {
return nil, nil
}
return out, nil
}
// RouteEndpoint is the key a route contribution names the endpoint it serves with, instead of
// repeating that endpoint's port (novox/hq ADR 0138).
//
// **A route and a listen both carried a port, and nothing said they were the same thing.** They
// always were — a route serves one of the module's own endpoints — but a reader had to join two
// numbers, and an assignment configuring "the web endpoint" had to know which number that was. A
// route that names the endpoint says what it means, and the mesh looks the port up.
const RouteEndpoint = "endpoint"
// RouteProblems holds a module's route contributions to naming an endpoint it actually has.
//
// A route naming an endpoint the module does not declare reaches nothing, and is refused where it is
// written rather than resolving to no port and serving nothing — the fault this repository names most
// often, a declaration that reads as though it did something.
func RouteProblems(m Manifest) []string {
var problems []string
check := func(where string, values map[string]any) {
name, ok := values[RouteEndpoint].(string)
if !ok || strings.TrimSpace(name) == "" {
return
}
if _, found := EndpointPort(m, name); !found {
problems = append(problems, fmt.Sprintf(
"%s routes %s to the endpoint %q, which it does not declare", m.Module, where, name))
}
}
if values, ok := m.Contributes["route"]; ok {
check("a name", values)
}
for local, values := range m.ContributesMany["route"] {
check(local, values)
}
return problems
}
// EndpointsSetting is the settings key that configures a module's endpoints by name, per node
// (novox/hq ADR 0138):
//
// {"endpoints": {"web": {"port": 20009, "label": "media", "reach": "both"},
// "stream": {"reach": "public"}}}
//
// **One block per endpoint, instead of three keys joined by a number.** Which machine port it lands
// on, the subdomain a proxy serves it under, and how far it reaches are the three things an operator
// says when a module is assigned, and they were said in `ports`, in the route's label and in `reach`,
// each keyed by the port. A module with two endpoints of different shapes — a web surface behind the
// proxy and a protocol port clients dial directly — could only be configured by a reader who knew
// which number was which.
//
// Every field is optional. A block that says only a reach leaves the port to the mesh and the label to
// the module, which is the ordinary case.
const EndpointsSetting = "endpoints"
// Endpoint is what an assignment says about one of a module's endpoints.
type Endpoint struct {
// Port is the machine-side port it is published on. Zero means the mesh assigns one, which it
// does anyway — a fixed port is the module's claim and is honoured without being said here.
Port int
// Label is the subdomain a proxy serves it under, overriding the one the module contributes.
Label string
// Reach is how far it reaches: machine, internal, public or both.
Reach string
}
// Endpoints reads a module's per-node endpoint configuration, by endpoint name.
//
// It refuses a name the module does not declare — the setting would reach nothing — and a reach that
// is not one of the four. It also refuses an endpoint whose port or reach is said twice, once here and
// once through the older key: two places saying the same thing is what this key exists to end, and
// letting both stand would mean the mesh followed whichever it read last.
func Endpoints(m Manifest, layers []Layer) (map[string]Endpoint, error) {
declared := endpointPorts(m)
exposed, err := Exposure(m, layers)
if err != nil {
return nil, err
}
out := map[string]Endpoint{}
for _, layer := range layers {
raw, ok := layer.Values[EndpointsSetting]
if !ok {
continue
}
blocks, ok := raw.(map[string]any)
if !ok {
return nil, fmt.Errorf("%s: %s is a { endpoint: { … } } map, and %q set it to something else",
m.Module, EndpointsSetting, layer.From)
}
for name, body := range blocks {
port, known := declared[name]
if !known {
return nil, fmt.Errorf(
"%s configures the endpoint %q, which it does not declare — the setting reaches "+
"nothing. It declares %s", m.Module, name, spokenEndpoints(m))
}
values, ok := body.(map[string]any)
if !ok {
return nil, fmt.Errorf("%s: the endpoint %q is configured with something that is not a "+
"block of settings", m.Module, name)
}
ep := out[name]
if reach, said := values["reach"]; said {
text, ok := reach.(string)
if !ok || !slices.Contains(reaches, text) {
return nil, fmt.Errorf("%s says the endpoint %q reaches %v; a reach is %s",
m.Module, name, reach, strings.Join(reaches, ", "))
}
if _, also := exposed[port]; also {
return nil, fmt.Errorf(
"%s says how far %q reaches and also exposes port %d. They say the same thing "+
"in different words; keep the endpoint's own block",
m.Module, name, port)
}
ep.Reach = text
}
if at, said := values["port"]; said {
machine, ok := asPort(at)
if !ok {
return nil, fmt.Errorf("%s puts the endpoint %q on %v, which is not a port",
m.Module, name, at)
}
ep.Port = machine
}
if label, said := values["label"]; said {
text, ok := label.(string)
if !ok || strings.TrimSpace(text) == "" {
return nil, fmt.Errorf("%s gives the endpoint %q a label that is not a name: %v",
m.Module, name, label)
}
ep.Label = strings.TrimSpace(text)
}
out[name] = ep
}
}
if len(out) == 0 {
return nil, nil
}
return out, nil
}
// spokenEndpoints is what a module's endpoints are called, as a refusal lists them — so a reader who
// named one wrongly is one edit from right, and a module that has named none is told so.
func spokenEndpoints(m Manifest) string {
names := make([]string, 0, len(m.Listens))
for _, l := range m.Listens {
if name := strings.TrimSpace(l.Name); name != "" {
names = append(names, name)
}
}
if len(names) == 0 {
return "no endpoints by name"
}
sort.Strings(names)
return strings.Join(names, ", ")
}
-83
View File
@@ -804,86 +804,3 @@ func TestSSHIsNeverLeftWithoutARule(t *testing.T) {
t.Fatalf("a machine with no mesh addresses has no ssh rule, so adopting it locks it:\n%s", nft)
}
}
// chainBody is one chain's own lines, so an assertion cannot be satisfied by an identical line in
// another chain.
//
// **Written because that happened.** The rule letting this machine's own callers through appears in the
// input chain and, in the same words, in the forward chain. A test asserting on the whole rendered file
// passed with the input chain's copy deleted — it was reading the forward chain's. ADR 0137's own tests
// say to assert per chain body for exactly this reason, and this file was not doing it.
func chainBody(t *testing.T, nft, chain string) string {
t.Helper()
open := "\tchain " + chain + " {"
i := strings.Index(nft, open)
if i < 0 {
t.Fatalf("no chain %q in:\n%s", chain, nft)
}
rest := nft[i+len(open):]
j := strings.Index(rest, "\n\t}")
if j < 0 {
t.Fatalf("chain %q does not close in:\n%s", chain, nft)
}
return rest[:j]
}
// **Anything on this machine may call anything on this machine.**
//
// Local is not a boundary this mesh draws, and whether a caller sits in a container was never meant to
// change the answer. It did, because the chain asked about addresses: a caller on the machine carries
// the machine's address and a caller in one of its containers carries a bridge address, so a rule
// naming the machines' own addresses silently refused every container on them.
//
// Measured: a module reaching its database on its own machine's name timed out for eleven hours while
// the machine itself could reach it (novox/hq 04-ISSUES/145).
func TestAnythingOnThisMachineMayCallAnythingOnIt(t *testing.T) {
nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{
{Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}},
{Module: "private", Listens: []Listening{{Port: 9999, From: FromMachine}}},
}}, nil), []string{"10.10.0.1", "10.10.0.2"}, false, nil, []string{"eth0"}, "mesh0")
// In the INPUT chain, which is where a call to a service on this machine arrives. The forward
// chain carries the same line in the same words, so asserting on the whole file proves nothing.
input := chainBody(t, nft, "input")
if !strings.Contains(input, `iifname != { "eth0", "mesh0" } accept`) {
t.Fatalf("a caller on this machine cannot reach a service on it:\n%s", input)
}
// One rule, for every service here — not a line per port that only covers the ports somebody
// remembered to think about.
if strings.Contains(input, `iifname != { "eth0", "mesh0" } tcp dport 5432`) {
t.Fatalf("the local allowance is still written per port:\n%s", input)
}
// And the private network still reaches what is exposed to it, which is a different question.
if !strings.Contains(input, "ip saddr { 10.10.0.1, 10.10.0.2 } tcp dport 5432 accept") {
t.Fatalf("the private network no longer reaches a service exposed to it:\n%s", input)
}
}
// The three reaches, as three lines. This is the whole of what the filter says about who may call what.
func TestTheThreeReachesAreThreeLines(t *testing.T) {
nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{
{Module: "internal-only", Listens: []Listening{{Port: 5432, From: FromMesh}}},
{Module: "public", Listens: []Listening{{Port: 443, From: FromEverywhere}}},
}}, nil), []string{"10.10.0.1"}, false, nil, []string{"eth0"}, "mesh0")
input := chainBody(t, nft, "input")
for what, want := range map[string]string{
"on this machine": `iifname != { "eth0", "mesh0" } accept`,
"over the private network": "ip saddr { 10.10.0.1 } tcp dport 5432 accept",
"from anywhere": "tcp dport 443 accept",
} {
if !strings.Contains(input, want) {
t.Fatalf("a caller %s cannot reach what is exposed to it (%q):\n%s", what, want, input)
}
}
}
// A port open to everything needs no such line — it is already open to a guest.
func TestAPublicPortNeedsNoGuestLine(t *testing.T) {
nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{
{Module: "web", Listens: []Listening{{Port: 443, From: FromEverywhere}}},
}}, nil), []string{"10.10.0.1"}, false, nil, []string{"eth0"}, "mesh0")
if strings.Count(nft, `iifname != { "eth0", "mesh0" } tcp dport 443`) != 0 {
t.Fatalf("a public port was given a guest line it does not need:\n%s", nft)
}
}
+1 -83
View File
@@ -571,21 +571,6 @@ type Artifact struct {
// image built from this same module's own repository, the same as every other artifact.
Context *ArtifactContext `json:"context,omitempty"`
// System is the operating system this artifact is compiled for, for a bundle whose output is a
// binary rather than portable code (novox/hq ADR 0142).
//
// **Named by the artifact, not by the recipe.** A toolchain deliberately accepts nothing from
// the module — anything a module could override there it would be writing a Dockerfile to
// override — and yet a compiled binary is per operating system, pinned at link time so a host
// refuses to touch a machine it was not built for (novox/hq ADR 0005). The way out is that the
// target is a property of the artifact: one artifact declared per system, one build each, and
// the recipe stays the mesh's.
//
// Empty for a bundle whose output runs anywhere, which is every interpreted language, and for
// every other kind. A bundle in a language that compiles to a binary must say one, because
// "compiled for whatever the build machine happened to be" is the fault this exists to prevent.
System string `json:"system,omitempty"`
// Language is what this module's code is written in, for a bundle.
//
// **Declared, never guessed.** Inferring it from what files happen to be present makes a
@@ -657,23 +642,8 @@ const (
// on is a fact, and it should be written once.
const ArtifactStoreProvision = "artifact-store"
// Listening is one endpoint a module serves: a port it accepts connections on, and what may be said
// about that port from outside the module.
// Listening is one port a module accepts connections on.
type Listening struct {
// Name is what this endpoint is called, so an assignment and a route can refer to it as one thing
// (novox/hq ADR 0138).
//
// **Because a port number is not a name.** Three facts have to be said about an endpoint when a
// module is assigned — which machine port it lands on, the subdomain a proxy serves it under, and
// how far it reaches — and they were said in three places keyed by the port. A module with two
// endpoints of different shapes, a web surface behind a proxy and a protocol port clients dial
// directly, cannot be configured that way without a reader joining numbers by hand.
//
// The module's to choose, like the route's label: it names its own parts. Lowercase, and unique
// within the module, so a reference to it is unambiguous. Empty is allowed and means an endpoint
// nothing refers to by name, which is every endpoint in the catalogue until they are named.
Name string `json:"name,omitempty"`
Port int `json:"port"`
// Protocol is "tcp" or "udp". Absent means tcp, which is what almost everything is — and a
// field that had to be written every time would be written wrongly some of the time.
@@ -1280,8 +1250,6 @@ func ParseManifest(raw []byte) (Manifest, error) {
"%s listens on %d over %q, which is tcp or udp", m.Module, l.Port, p))
}
}
problems = append(problems, endpointNameProblems(m)...)
problems = append(problems, RouteProblems(m)...)
for _, port := range m.Guards {
if port < 1 || port > 65535 {
problems = append(problems, fmt.Sprintf(
@@ -1706,53 +1674,3 @@ func (m Manifest) undeclaredMounts() []string {
}
return problems
}
// endpointName is what an endpoint may be called: lowercase letters, digits and dashes, starting
// with a letter. The same shape a label has, because both end up in something a person types.
var endpointName = regexp.MustCompile(`^[a-z][a-z0-9-]*$`)
// endpointNameProblems holds a module's endpoint names to being usable as references (novox/hq ADR
// 0138).
//
// **Unique, because the point of a name is that it identifies one thing.** Two endpoints called the
// same would make an assignment that configures one silently configure whichever the mesh read last
// — the shape of fault this repository keeps finding, where a declaration appears to say something
// and says something else.
func endpointNameProblems(m Manifest) []string {
var problems []string
seen := map[string]int{}
for _, l := range m.Listens {
name := strings.TrimSpace(l.Name)
if name == "" {
continue
}
if !endpointName.MatchString(name) {
problems = append(problems, fmt.Sprintf(
"%s calls the endpoint on port %d %q; a name is lowercase letters, digits and "+
"dashes, starting with a letter", m.Module, l.Port, l.Name))
continue
}
if before, already := seen[name]; already {
problems = append(problems, fmt.Sprintf(
"%s calls both port %d and port %d %q, so anything naming that endpoint could mean "+
"either", m.Module, before, l.Port, name))
continue
}
seen[name] = l.Port
}
return problems
}
// EndpointPort is the port of the endpoint a module calls this, and whether it has one.
func EndpointPort(m Manifest, name string) (int, bool) {
want := strings.TrimSpace(name)
if want == "" {
return 0, false
}
for _, l := range m.Listens {
if strings.TrimSpace(l.Name) == want {
return l.Port, true
}
}
return 0, false
}
-206
View File
@@ -1,206 +0,0 @@
package catalogue
import (
"strings"
"testing"
)
// a web module with one routed endpoint, the shape almost every routed module in the catalogue has.
func aRoutedWeb() Manifest {
return Manifest{
Module: "web",
Listens: []Listening{{Port: 3000, From: FromMesh}},
Contributes: map[string]map[string]any{
"route": {"label": "app", "port": 3000},
},
}
}
func reachSet(reach string) SettingsBy {
return SettingsBy{"web": {{From: "node anchor",
Values: map[string]any{ReachSetting: map[string]any{"3000": reach}}}}}
}
// namesFor renders the contribution a routed module makes and returns the two names it carries.
func namesFor(t *testing.T, m Manifest, settings SettingsBy) (public, internal string) {
t.Helper()
r := Resolution{Node: "anchor", Modules: []Manifest{m},
PublicDomain: "example.test", At: "anchor.internal"}
given, err := r.contributions(settings, nil, nil)
if err != nil {
t.Fatalf("contributions: %v", err)
}
for _, c := range given["route"] {
p, _ := c.Values["name"].(string)
i, _ := c.Values["internal-name"].(string)
return p, i
}
t.Fatal("the module contributed no route")
return "", ""
}
// **Nothing said composes both names, exactly as before.** This is the assertion that keeps every
// mesh already running identical until an assignment speaks, and it is the one that would break first
// if reach were read where it should not be.
func TestAnEndpointWithNoReachKeepsBothNames(t *testing.T) {
public, internal := namesFor(t, aRoutedWeb(), nil)
if public != "app.example.test" || internal != "app.anchor.internal" {
t.Fatalf("names are %q and %q, want both composed as before", public, internal)
}
}
// An internal endpoint has an internal name and no public one — so the proxy serves it inside, and
// the public authority is never asked for a name nobody wanted. This is what "must not be public"
// could not say before.
func TestAnInternalEndpointHasNoPublicName(t *testing.T) {
public, internal := namesFor(t, aRoutedWeb(), reachSet(ReachInternal))
if public != "" {
t.Fatalf("an internal endpoint composed the public name %q", public)
}
if internal != "app.anchor.internal" {
t.Fatalf("internal name is %q, want app.anchor.internal", internal)
}
}
// And the mirror: a public endpoint gets the public name and not the internal one, so the mesh's own
// authority is not asked to certify a name the service is not reached by.
func TestAPublicEndpointHasNoInternalName(t *testing.T) {
public, internal := namesFor(t, aRoutedWeb(), reachSet(ReachPublic))
if internal != "" {
t.Fatalf("a public endpoint composed the internal name %q", internal)
}
if public != "app.example.test" {
t.Fatalf("public name is %q, want app.example.test", public)
}
}
func TestBothComposesBothNames(t *testing.T) {
public, internal := namesFor(t, aRoutedWeb(), reachSet(ReachBoth))
if public == "" || internal == "" {
t.Fatalf("both should compose both names, got %q and %q", public, internal)
}
}
// **The filter reads the same value — for an endpoint the proxy does not serve.**
//
// A routed endpoint's port is how the proxy reaches it and nothing else (ADR 0045): a public service
// listens from the mesh, only the proxy reaches it, and it is exposed by name. So on a routed
// endpoint the reach asks for a name and the port keeps what the manifest said.
func TestAnUnroutedEndpointsPortFollowsItsReach(t *testing.T) {
// The same module with its route taken away: now the port is the only way in, so reach governs it.
bare := aRoutedWeb()
bare.Contributes = nil
for _, c := range []struct{ reach, want string }{
{ReachInternal, FromMesh},
{ReachPublic, FromEverywhere},
{ReachBoth, FromEverywhere},
{ReachMachine, FromMachine},
} {
r := Resolution{Node: "anchor", Modules: []Manifest{bare}}
rules, err := r.Rules(Rendering{Settings: reachSet(c.reach)})
if err != nil {
t.Fatalf("%s: rules: %v", c.reach, err)
}
found := false
for _, rule := range rules {
if rule.Port == 3000 {
found = true
if rule.From != c.want {
t.Fatalf("reach %q made the filter say %q, want %q", c.reach, rule.From, c.want)
}
}
}
if !found {
t.Fatalf("reach %q produced no rule for the port", c.reach)
}
}
}
// **A public name does not open the machine's port**, which is the case that found this.
//
// A module whose routed name must be public and whose machine-side port must not be had no way to say
// so while one value drove both. Under one value it could not be expressed; the port would reopen.
func TestAPublicNameLeavesARoutedPortAsTheManifestSaid(t *testing.T) {
r := Resolution{Node: "anchor", Modules: []Manifest{aRoutedWeb()},
PublicDomain: "example.test", At: "anchor.internal"}
rules, err := r.Rules(Rendering{Settings: reachSet(ReachPublic)})
if err != nil {
t.Fatal(err)
}
for _, rule := range rules {
if rule.Port == 3000 && rule.From != FromMesh {
t.Fatalf("a public reach opened a routed port to %q; the proxy is how it is reached",
rule.From)
}
}
// And the name it asked for is there, so the reach was not simply ignored.
public, internal := namesFor(t, aRoutedWeb(), reachSet(ReachPublic))
if public != "app.example.test" || internal != "" {
t.Fatalf("names are %q and %q, want the public one only", public, internal)
}
}
// A reach for a port the module does not listen on reaches nothing, and is refused where it is
// written rather than accepted and ignored.
func TestAReachForAPortTheModuleDoesNotListenOnIsRefused(t *testing.T) {
_, err := Reaches(aRoutedWeb(), []Layer{{From: "node anchor",
Values: map[string]any{ReachSetting: map[string]any{"9999": ReachInternal}}}})
if err == nil || !strings.Contains(err.Error(), "reaches nothing") {
t.Fatalf("a reach naming an undeclared port was accepted: %v", err)
}
}
// A value that is not a reach is refused, and the refusal names the four so a reader is one edit from
// right. "mesh" is the tempting wrong answer, because that is the filter's word for nearly the same
// thing.
func TestAValueThatIsNotAReachIsRefused(t *testing.T) {
for _, wrong := range []string{"mesh", "anywhere", "private", "true"} {
_, err := Reaches(aRoutedWeb(), []Layer{{From: "node anchor",
Values: map[string]any{ReachSetting: map[string]any{"3000": wrong}}}})
if err == nil || !strings.Contains(err.Error(), "a reach is") {
t.Fatalf("%q was accepted as a reach: %v", wrong, err)
}
}
}
// **A port that says both reach and expose is refused.** They say the same thing in different words,
// and accepting both would have the filter follow one while the names followed the other — the
// disagreement ADR 0138 exists to remove, reintroduced by the migration away from the older word.
func TestReachAndExposeForOnePortAreRefused(t *testing.T) {
_, err := Reaches(aRoutedWeb(), []Layer{{From: "node anchor", Values: map[string]any{
ReachSetting: map[string]any{"3000": ReachInternal},
ExposeSetting: map[string]any{"3000": FromEverywhere},
}}})
if err == nil || !strings.Contains(err.Error(), "same thing in different") {
t.Fatalf("a port set both ways was accepted: %v", err)
}
}
// A path-level refusal carries no port: it is a rule about a name, not an endpoint, and it inherits
// whatever that name turned out to be. Narrowing the endpoint must not silently drop it.
func TestARuleWithNoPortIsLeftAlone(t *testing.T) {
m := aRoutedWeb()
m.ContributesMany = map[string]map[string]map[string]any{
"route": {"refused": {"label": "app", "path": "/internal", "deny": true}},
}
r := Resolution{Node: "anchor", Modules: []Manifest{m},
PublicDomain: "example.test", At: "anchor.internal"}
given, err := r.contributions(reachSet(ReachInternal), nil, nil)
if err != nil {
t.Fatal(err)
}
var sawDeny bool
for _, c := range given["route"] {
if deny, _ := c.Values["deny"].(bool); deny {
sawDeny = true
// It keeps both, because it named no endpoint to be narrowed by.
if c.Values["name"] == nil || c.Values["internal-name"] == nil {
t.Fatalf("the path rule lost a name it shadows: %v", c.Values)
}
}
}
if !sawDeny {
t.Fatal("the path rule was dropped")
}
}
-11
View File
@@ -186,17 +186,6 @@ func UnusedSettings(m Manifest, layers []Layer) []string {
if key == PortsSetting {
continue
}
// `reach` says how far one of this module's endpoints reaches (novox/hq ADR 0138) — the
// filter's source, which names are composed, and therefore which authority certifies
// them. Validated in Reaches, so not stray.
if key == ReachSetting && len(m.Listens) > 0 {
continue
}
// `endpoints` configures a module's endpoints by name — the machine port, the subdomain and
// the reach as one block each (novox/hq ADR 0138). Validated in Endpoints, so not stray.
if key == EndpointsSetting && len(m.Listens) > 0 {
continue
}
unused = append(unused, fmt.Sprintf(
"%s sets %q, and %s has no file or contribution to merge it into",
layer.From, key, m.Module))
-61
View File
@@ -8,7 +8,6 @@ import (
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"golang.org/x/crypto/bcrypt"
)
// Reading the bus's user list out of the mesh's records, against a real store.
@@ -165,63 +164,3 @@ func granted(all []string, one string) bool {
}
return false
}
// **A token is an account on the bus, or it is a string nothing accepts** (novox/hq 04-ISSUES/146).
//
// The composed list names an enrolment user for every machine with a live token, and nothing minted
// a credential for it — so the composer left it out as a user with no password, and every enrolment
// since the mesh moved to this bus was refused by the server before the mesh heard of it. Nothing
// caught it because nothing had enrolled since.
//
// The password cannot be minted, because it is the token's own secret: the machine will present
// exactly that string. So this checks the two halves that make the account usable — that a row
// exists under the name the composer asks for, and that the secret handed out is what that row
// accepts.
func TestIssuingATokenRecordsTheAccountItIsThePasswordOf(t *testing.T) {
inv, ctx := aMeshWith(t)
if _, err := inv.AddNode(ctx, "joiner"); err != nil {
t.Fatal(err)
}
issued, err := inv.IssueToken(ctx, "joiner", time.Hour)
if err != nil {
t.Fatal(err)
}
name := broker.Principal{Kind: broker.KindEnrolment, Node: "joiner"}.Username()
users, err := inv.BusUsers(ctx)
if err != nil {
t.Fatal(err)
}
user, has := users[name]
if !has {
t.Fatalf("no bus account for %q; the composer would leave the enrolment out and the "+
"machine would be refused before the mesh heard of it: %v", name, users)
}
if user.Kind != BusEnrolment || user.Node != "joiner" {
t.Errorf("the account is %+v, not this node's enrolment", user)
}
if err := bcrypt.CompareHashAndPassword([]byte(user.PasswordHash), []byte(issued.Secret)); err != nil {
t.Error("the account does not accept the secret the token carries, so presenting the " +
"token would be refused by the server")
}
// And the composition contains it, which is the thing the server reads.
records, err := inv.BusRecords(ctx)
if err != nil {
t.Fatal(err)
}
derived, err := broker.Users(records)
if err != nil {
t.Fatal(err)
}
hashes := map[string]string{}
for n, u := range users {
hashes[n] = u.PasswordHash
}
_, missing := broker.WithPasswords(derived, hashes)
for _, m := range missing {
if m == name {
t.Fatal("the enrolment user is composed without a password, which is a user nobody can be")
}
}
}
+8 -36
View File
@@ -51,40 +51,6 @@ const (
// reply, into a module's sealed environment — and the mesh keeps only the hash, so a credential is
// never recoverable from the store. A caller that loses it must mint again, which is a rotation and
// is meant to feel like one.
// RecordBusPassword records a hash for a password the caller already holds.
//
// **For the one credential the mesh does not choose**: an enrolment token's secret is the password
// of the user that presents it (novox/hq ADR 0004, design 25 §6), so the token cannot be given a
// minted password — it already has one, and the machine will connect with exactly that string.
// Everything else goes through Mint, which chooses and returns the plaintext once.
func (i *Inventory) RecordBusPassword(ctx context.Context, u BusUser, password string) error {
if u.Username == "" || u.Kind == "" {
return errors.New("a bus user needs a username and a kind")
}
if password == "" {
return errors.New("a bus user needs a password")
}
hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
if err != nil {
return fmt.Errorf("cannot hash a bus password: %w", err)
}
return i.writeBusUser(ctx, u, string(hash))
}
// writeBusUser is the row, whoever chose the password.
func (i *Inventory) writeBusUser(ctx context.Context, u BusUser, hash string) error {
if _, err := i.store.Pool().Exec(ctx,
`insert into bus_user (username, kind, node, module, password_hash)
values ($1, $2, $3, $4, $5)
on conflict (username) do update
set kind = excluded.kind, node = excluded.node, module = excluded.module,
password_hash = excluded.password_hash, minted_at = now()`,
u.Username, u.Kind, u.Node, u.Module, hash); err != nil {
return fmt.Errorf("cannot record the bus user %s: %w", u.Username, err)
}
return nil
}
func (i *Inventory) MintBusPassword(ctx context.Context, u BusUser) (string, error) {
if u.Username == "" || u.Kind == "" {
return "", errors.New("a bus user needs a username and a kind")
@@ -103,8 +69,14 @@ func (i *Inventory) MintBusPassword(ctx context.Context, u BusUser) (string, err
return "", fmt.Errorf("cannot hash a bus password: %w", err)
}
if err := i.writeBusUser(ctx, u, string(hash)); err != nil {
return "", err
if _, err := i.store.Pool().Exec(ctx,
`insert into bus_user (username, kind, node, module, password_hash)
values ($1, $2, $3, $4, $5)
on conflict (username) do update
set kind = excluded.kind, node = excluded.node, module = excluded.module,
password_hash = excluded.password_hash, minted_at = now()`,
u.Username, u.Kind, u.Node, u.Module, string(hash)); err != nil {
return "", fmt.Errorf("cannot record the bus user %s: %w", u.Username, err)
}
return password, nil
}
-24
View File
@@ -14,7 +14,6 @@ import (
"time"
"github.com/jackc/pgx/v5"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/store"
)
@@ -264,29 +263,6 @@ func (i *Inventory) IssueToken(ctx context.Context, nodeName string, validFor ti
return Issued{}, err
}
// **And the account that secret is the password of** (novox/hq 04-ISSUES/146). The composed
// user list names an enrolment user for every node with a live token, and nothing minted a
// credential for it — so the composer left it out as a user with no password and every
// enrolment was refused by the server before the mesh heard of it.
//
// Recorded rather than minted: the token's secret IS the password, which is what lets a
// machine's first connection be authenticated by the thing it is enrolling with. It cannot be
// chosen here, because it has already been handed to whoever will present it.
//
// Outside the transaction on purpose. The token is what the mesh promised; a credential that
// the next composition rewrites anyway is not worth failing an issue over, and a token with no
// account is recoverable by issuing another, while an account with no token is a user nobody
// can be.
if err := i.RecordBusPassword(ctx, BusUser{
Username: broker.Principal{Kind: broker.KindEnrolment, Node: node.Name}.Username(),
Kind: BusEnrolment,
Node: node.Name,
}, secret); err != nil {
return Issued{}, fmt.Errorf(
"the token for %s was issued and the bus account it is the password of was not "+
"recorded, so this token cannot connect: %w", node.Name, err)
}
return Issued{Node: node, Secret: secret, Expires: expires}, nil
}