Compare commits

..
Author SHA1 Message Date
jschoubben b7da02e370 An address is read from the node's settings where it is used, never recorded with a port
Three readers did not follow a moved foundation port (novox/hq 04-ISSUES/102),
and each took the control-node down in its own way: the control plane's own
store and broker connections, sealed at genesis with the port inside; and every
build the mesh ever recorded, kept as `<registry>:<port>/<module>/<artifact>@…`.

The control plane cannot open its own sealed connections to move a port, and it
cannot bind the store as a consumer would — a binding mints a credential. So its
settings get a third twin: `NAME_PORT`, composed into its container from the
node's settings by a placeholder that names a seat, `${seat:mesh-store:5432}`,
and read on top of the sealed value by the store, the broker, the management API
and the bus connection. The answer is empty when the mesh has nothing to add,
so what genesis wrote stands until the node says otherwise.

A build is now recorded by digest and path — `artifact-store://<module>/<artifact>@…`
— and the store's address is composed in where a reference is used: the
declaration, the trust file, the bases a build is handed, a replay to the
catalogue. A reference recorded before this, with an address, is re-routed the
same way when the mesh built it. The trust file and every provider's address
now come from one derivation, with the node's given port over the mesh's
assignment over the manifest's number.

novox/hq 04-ISSUES/102
2026-09-23 23:26:43 +02:00
183 changed files with 2575 additions and 20385 deletions
+3 -9
View File
@@ -86,19 +86,13 @@ proxy-image:
# The whole gate. Raises a database, runs everything against it, and takes it down again --
# including when the tests fail, which is why the teardown is not conditional.
#
# **One package at a time (-p 1), and it is not about speed.** The live tests reach one bus, and on
# it they assert, read and remove the mesh's own objects -- streams and consumers with fixed names,
# because those names are the mesh's and a test cannot choose others. Two packages doing that at once
# is one deleting a consumer the other is reading through, and the failure lands in whichever test
# was reading, as "no response from stream". That reads as a bug in the code under test.
check: fmt vet postgres
@go test -p 1 ./... ; status=$$? ; $(MAKE) postgres-stop ; exit $$status
@go test ./... ; status=$$? ; $(MAKE) postgres-stop ; exit $$status
# Without a database the live tests skip rather than fail, so this is the honest subset and not
# the gate. Serialised for the same reason check is: a bus may be configured even when a store is not.
# the gate.
test:
go test -p 1 ./...
go test ./...
vet:
go vet ./...
+168 -111
View File
@@ -17,15 +17,21 @@ package main
import (
"context"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"net/url"
"os"
"os/signal"
"strings"
"syscall"
"time"
amqp "github.com/rabbitmq/amqp091-go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/builder"
"github.com/novox/mesh-controller/internal/link"
)
@@ -45,6 +51,7 @@ const usage = `mesh-builder — builds modules for the mesh
It consumes build requests and answers with what it made. Nothing is listened on and nothing
is dialled except the broker.
MESH_BROKER_AMQP where the broker is, with this builder's own credential
MESH_BROKER_FILE a file the mesh sealed to this machine holding the same
MESH_REGISTRY host:port to publish artifacts to, when the mesh has not said
MESH_BINDING a file the mesh wrote saying where the artifact store is
@@ -107,48 +114,72 @@ func run() error {
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
machine, err := takeWorkFrom(credential, on)
conn, err := dial(credential)
if err != nil {
// Not quoted back: the URL carries this builder's broker password.
return fmt.Errorf("cannot reach the broker: %w", err)
}
defer conn.Close()
channel, err := conn.Channel()
if err != nil {
return err
}
defer channel.Close()
if _, err := channel.QueueDeclare(link.BuildQueue, true, false, false, false, nil); err != nil {
return err
}
// One at a time. A build machine that took five requests at once would run five container
// builds against one runtime and finish all of them slower than it would have finished the
// first — and the queue is what shares work between machines, so nothing is lost by it.
if err := channel.Qos(1, 0, false); err != nil {
return err
}
// Not auto-acknowledged. A request acknowledged on arrival is a build that vanishes if this
// process dies mid-way, with nobody waiting on it ever hearing why.
requests, err := channel.ConsumeWithContext(ctx, link.BuildQueue, "mesh-builder",
false, false, false, false, nil)
if err != nil {
return err
}
defer machine.Close()
fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry)
publisher := builder.Registry{Address: registry, Run: builder.Command}
return machine.Take(ctx, func(ctx context.Context, work link.Build) {
answer(ctx, publisher, on, workspace, work)
})
}
// takeWorkFrom opens this machine's link to whichever bus the mesh is on.
//
// **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5): a build
// machine told about both would take work from one and answer on the other, and every log line would
// say it was fine.
func takeWorkFrom(credential Credential, on string) (link.BuildMachine, error) {
// **The credential names the bus, and there is one** (novox/hq ADR 0131, design 28 task 5.5).
// A credential for the mesh's bus carries user, password and fingerprint beside the address,
// and that is enough to dial it, pinned.
if !credential.onTheNewBus() {
return nil, fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL)
for {
select {
case <-ctx.Done():
fmt.Println("stopping")
return nil
case delivery, ok := <-requests:
if !ok {
return fmt.Errorf("the broker closed the connection")
}
answer(ctx, channel, publisher, on, workspace, delivery)
}
}
js, err := broker.DialPinned(credential.natsURL(), credential.Fingerprint)
if err != nil {
return nil, err
}
return link.MachineOverNATS(js, on), nil
}
// answer does one build and says what happened, whichever way it went.
func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build) {
request := work.Request()
func answer(ctx context.Context, channel *amqp.Channel, publisher builder.Publisher,
on, workspace string, delivery amqp.Delivery) {
// **First thing, and to stdout.** A build request that arrives and produces no visible line until
// it either finishes or fails is indistinguishable from one that never arrived — which cost a long
// diagnosis against a running mesh, chasing "the handler never fired" when the truth was only that
// the handler said nothing until the end.
fmt.Fprintf(os.Stderr, "a build request arrived for %s\n", request.Repository)
// **First thing, and to stdout.** A build request that arrives and produces no visible line
// until it either finishes or fails is indistinguishable from one that never arrived — which
// cost a long diagnosis against a running mesh, chasing "the handler never fired" when the
// truth was only that the handler said nothing until the end.
fmt.Fprintf(os.Stderr, "a build request arrived (%d bytes)\n", len(delivery.Body))
var request link.BuildRequest
if err := json.Unmarshal(delivery.Body, &request); err != nil {
// Unreadable. Acknowledged and dropped rather than requeued: a message this builder
// cannot parse will not become parseable by being delivered again, and requeueing it
// would put it in front of every real request for ever.
fmt.Fprintf(os.Stderr, "a request could not be read and was dropped: %v\n", err)
_ = delivery.Ack(false)
return
}
result := link.BuildResult{
ID: request.ID, Repository: request.Repository, Path: request.Path,
@@ -167,11 +198,10 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
var built builder.Result
if err == nil {
// The package-registry credential is a build input, so it is resolved before the clone: a
// build that could not have resolved its dependencies is refused in front of the reason, not
// after a clone that then fails at npm ci.
// build that could not have resolved its dependencies is refused in front of the reason,
// not after a clone that then fails at npm ci.
built, err = builder.Build(ctx, builder.Command, publisher,
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
forgeFrom(),
func(step, message string) {
fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message)
})
@@ -194,26 +224,71 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
})
}
result.Against = built.Against
for _, r := range built.Read {
result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref})
}
fmt.Fprintf(os.Stderr, " built %s from %s\n", built.Manifest.Module, short(built.Commit))
}
}
if err := work.Announce(ctx, result); err != nil {
// Said, not fatal: the build happened. A build reported as failed because announcing it
// failed is a lie about work that was done — and the request stays unsettled below only if
// nothing was said at all, so another machine can try.
fmt.Fprintf(os.Stderr, "cannot say what came of a build: %v\n", err)
body, err := json.Marshal(result)
if err != nil {
fmt.Fprintf(os.Stderr, "cannot report a build: %v\n", err)
_ = delivery.Ack(false)
return
}
// Settled only once the outcome is away, so a machine that dies before answering leaves the work
// for another rather than losing it.
if err := work.Done(); err != nil {
fmt.Fprintf(os.Stderr, "the outcome is away and the request could not be settled: %v\n", err)
// Always through the exchange, whether or not somebody is waiting.
//
// **Never the default exchange.** Permission there is granted per exchange rather than per
// queue, so a builder allowed to use it could publish into any node's queue — the privilege a
// build machine most obviously should not have. An asker binds its own reply queue to this
// key and filters by correlation; a control plane that records builds is bound to it too, so
// a result nobody asked for is still kept rather than reported into the void.
publishCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
if err := channel.PublishWithContext(publishCtx, link.Exchange, link.KeyBuilt, false, false,
amqp.Publishing{
ContentType: "application/json",
CorrelationId: result.ID,
Body: body,
}); err != nil {
fmt.Fprintf(os.Stderr, "cannot answer a build request: %v\n", err)
}
// **And announced, which is a different act from answering.** The reply goes to whoever asked
// and is correlated to their request; this says to the whole mesh that a module now exists at
// a commit, and the catalogue places it in the module graph (novox/hq ADR 0072). A build
// nobody asked for still has to be announced, or the graph knows less than the registry does.
//
// Only on success: a failed build produced no module-version, and announcing one would put
// something in the graph that was never made.
if result.Failed == "" && result.Commit != "" {
announced := map[string]any{
"module": moduleOf(result.Manifest), "commit": result.Commit,
"repository": result.Repository, "path": result.Path, "ref": result.Ref,
"manifest": json.RawMessage(result.Manifest), "against": result.Against,
"made": result.Made,
}
if err := link.EmitEvent(publishCtx, channel, link.KeyModuleBuilt, "builder", on, announced); err != nil {
// Said, not fatal: the build happened and was answered. A module the catalogue has not
// heard of is a gap somebody can close; a build reported as failed because announcing
// it failed is a lie about work that was done.
fmt.Fprintf(os.Stderr, " built, but could not announce it: %v\n", err)
}
}
// Acknowledged only once the answer is away, so a builder that dies before answering leaves
// the request for another machine rather than losing it.
_ = delivery.Ack(false)
}
// moduleOf reads the module's name out of the manifest it just built, which is the only place it is
// authoritative — the request named a repository and a path, not a module.
func moduleOf(manifest json.RawMessage) string {
var named struct {
Module string `json:"module"`
}
if err := json.Unmarshal(manifest, &named); err != nil {
return ""
}
return named.Module
}
// packagesFrom is where a build resolves the mesh's own published packages — the SDK above all
@@ -292,53 +367,6 @@ func packagesFrom() (builder.Npmrc, error) {
return builder.Npmrc{Scope: scope, Registry: registry, Token: secret}, nil
}
// forgeFrom is the git credential this builder may offer a clone, composed from the same binding
// and sealed secret its package-registry half already reads: the forge that answers npm is the
// forge that hosts the repositories, and its provisioner applies one password to one user for
// both. Anything missing means no credential, and every clone stays anonymous — which is all a
// mesh of public repositories ever needs.
//
// The URL names the binding's own address — the machine the mesh says the forge is on — so a
// private repository is registered and built by that address, and a clone of anything else is
// never shown this credential (git's credential store matches the whole origin).
func forgeFrom() builder.GitCredential {
path := strings.TrimSpace(os.Getenv("MESH_PACKAGE_BINDING"))
if path == "" {
return builder.GitCredential{}
}
raw, err := os.ReadFile(path)
if err != nil {
return builder.GitCredential{}
}
var told struct {
At string `json:"at"`
As string `json:"as"`
Serves map[string]any `json:"serves"`
}
if err := json.Unmarshal(raw, &told); err != nil || told.At == "" || told.As == "" {
return builder.GitCredential{}
}
secret := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN"))
if file := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN_FILE")); file != "" {
if raw, err := os.ReadFile(file); err == nil {
secret = strings.TrimSpace(string(raw))
}
}
if secret == "" {
return builder.GitCredential{}
}
scheme := "https"
if s, ok := told.Serves["scheme"]; ok {
scheme = fmt.Sprintf("%v", s)
}
host := told.At
if port, ok := told.Serves["port"]; ok {
host = fmt.Sprintf("%s:%v", told.At, port)
}
made := url.URL{Scheme: scheme, User: url.UserPassword(told.As, secret), Host: host}
return builder.GitCredential{URL: made.String()}
}
func short(commit string) string {
if len(commit) > 8 {
return commit[:8]
@@ -422,8 +450,13 @@ func brokerFrom() (Credential, error) {
// broker is then verified against whatever this machine already trusts.
return Credential{URL: said}, nil
}
return Credential{}, fmt.Errorf(
"no MESH_BROKER_FILE: a build machine with no credential for the bus has nothing to build")
url := strings.TrimSpace(os.Getenv("MESH_BROKER_AMQP"))
if url == "" {
return Credential{}, fmt.Errorf(
"neither MESH_BROKER_FILE nor MESH_BROKER_AMQP: a builder with no broker has " +
"nothing to build")
}
return Credential{URL: url}, nil
}
// Credential is what a build machine is given so it can reach the broker.
@@ -435,24 +468,48 @@ func brokerFrom() (Credential, error) {
// **The same shape a node gets, for the same reason** (novox/hq ADR 0004): the fingerprint travels
// out of band — here, sealed with the credential — and the endpoint is verified once at connect.
type Credential struct {
URL string `json:"url"`
URL string `json:"url"`
// Fingerprint is SHA-256 over the broker certificate's DER bytes, or empty to verify the
// ordinary way.
Fingerprint string `json:"fingerprint,omitempty"`
// User and Password ride beside the address on the bus being built (design 25): a credential
// embedded in a URL leaks into every log line that prints a connection, so the mesh seals them
// as two fields and this machine joins them once, here, to dial.
User string `json:"user,omitempty"`
Password string `json:"password,omitempty"`
}
// onTheNewBus is whether a credential is for the bus being built: its address says so, and the
// mesh only ever seals such a credential with the user and password beside it.
func (c Credential) onTheNewBus() bool { return strings.HasPrefix(strings.TrimSpace(c.URL), "nats://") }
// dial opens the connection, pinning the broker's certificate when there is one to pin.
func dial(held Credential) (*amqp.Connection, error) {
if held.Fingerprint == "" {
return amqp.Dial(held.URL)
}
return amqp.DialTLS(held.URL, pinning(held.Fingerprint))
}
// natsURL is the address with this machine's credential in it, for the one dial that needs it.
func (c Credential) natsURL() string {
rest := strings.TrimPrefix(strings.TrimSpace(c.URL), "nats://")
if c.User == "" {
return "nats://" + rest
// pinning is a TLS configuration that trusts exactly one certificate.
//
// InsecureSkipVerify with a VerifyPeerCertificate is **pinning, not skipping**: the standard chain
// check is replaced, not removed, and what replaces it is stricter — one certificate is accepted
// rather than every certificate a public authority would sign.
//
// Its own function so a test can drive it against a real handshake. A pin check that is only ever
// exercised through a broker is a pin check nothing tests.
func pinning(fingerprint string) *tls.Config {
return &tls.Config{
InsecureSkipVerify: true,
VerifyPeerCertificate: func(raw [][]byte, _ [][]*x509.Certificate) error {
if len(raw) == 0 {
return errors.New("the broker presented no certificate")
}
// The leaf, and in the same spelling the mesh writes it — `sha256:` and 64 hex
// characters. Comparing a bare digest against a written fingerprint never matches,
// and the failure is indistinguishable from being pointed at the wrong broker.
sum := sha256.Sum256(raw[0])
got := "sha256:" + hex.EncodeToString(sum[:])
if got != fingerprint {
return fmt.Errorf(
"this is not the broker this builder was told about\n expected %s\n "+
"got %s\nEither this mesh's broker was replaced, or this builder is "+
"being pointed at something else. Retrying will not help",
fingerprint, got)
}
return nil
},
}
return "nats://" + c.User + ":" + c.Password + "@" + rest
}
+8 -19
View File
@@ -89,7 +89,6 @@ func buildOnce(ctx context.Context, args []string) error {
return err
}
built, buildErr := builder.Build(ctx, builder.Command, publisher, repository, *path, *ref, where, bases, npmrc,
forgeFrom(),
func(step, message string) { fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message) })
if buildErr != nil {
return buildErr
@@ -106,9 +105,6 @@ func buildOnce(ctx context.Context, args []string) error {
Manifest: built.Manifest,
Against: built.Against,
}
for _, r := range built.Read {
out.Read = append(out.Read, readRepository{Repository: r.Repository, Ref: r.Ref})
}
for _, made := range built.Built {
out.Made = append(out.Made, madeArtifact{Name: made.Name, Kind: made.Kind, Reference: made.Reference})
}
@@ -126,21 +122,14 @@ func buildOnce(ctx context.Context, args []string) error {
// The same fields the mesh records for a build, so a reader comparing a genesis build against an
// ordinary one is comparing the same thing said the same way.
type onceResult struct {
Module string `json:"module"`
Commit string `json:"commit"`
Repository string `json:"repository"`
Path string `json:"path,omitempty"`
Ref string `json:"ref,omitempty"`
Manifest any `json:"manifest"`
Made []madeArtifact `json:"made"`
Against []string `json:"against,omitempty"`
Read []readRepository `json:"read,omitempty"`
}
// readRepository is a repository this build read source from besides the module's own.
type readRepository struct {
Repository string `json:"repository"`
Ref string `json:"ref,omitempty"`
Module string `json:"module"`
Commit string `json:"commit"`
Repository string `json:"repository"`
Path string `json:"path,omitempty"`
Ref string `json:"ref,omitempty"`
Manifest any `json:"manifest"`
Made []madeArtifact `json:"made"`
Against []string `json:"against,omitempty"`
}
type madeArtifact struct {
+2 -2
View File
@@ -14,8 +14,8 @@ func TestBuilderDiagnosticsStayOffStdout(t *testing.T) {
allowed := map[string]bool{
"string(body)": true, // once.go: the result JSON, which IS stdout
"version)": true, // --version
`"stopping")`: true, // the loop.s shutdown line
"usage)": true, // --help text, for a human
`"stopping")`: true, // the loop.s shutdown line
"usage)": true, // --help text, for a human
}
for _, file := range []string{"once.go", "main.go"} {
src, err := os.ReadFile(file)
+2 -4
View File
@@ -15,8 +15,6 @@ import (
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/broker"
)
// Where a builder publishes.
@@ -195,9 +193,9 @@ func TestThePinIsComparedInTheSpellingTheMeshWritesIt(t *testing.T) {
}
}
// handshakeWith runs the pin check the builder dials with against an address.
// handshakeWith runs the builder's own pin check against an address.
func handshakeWith(address, pin string) error {
conn, err := tls.Dial("tcp", address, broker.PinnedToFingerprint(pin))
conn, err := tls.Dial("tcp", address, pinning(pin))
if err != nil {
return err
}
+1 -8
View File
@@ -46,16 +46,9 @@ func assign(ctx context.Context, open *stores, node, module string) (string, err
return "", err
}
defer release()
fresh, err := open.inventory.Assign(ctx, node, module)
if err != nil {
if err := open.inventory.Assign(ctx, node, module); err != nil {
return "", err
}
if !fresh {
// Nothing changed, and saying "is assigned" would read as an action. One node runs one
// of each — the module's name is the assignment's identity (novox/hq ADR 0115).
return fmt.Sprintf("%s already runs %s — one node runs one of each (ADR 0115); nothing changed",
node, module), nil
}
said := fmt.Sprintf("%s is assigned %s", node, module)
plan, _, err := planFor(ctx, open, node)
if err != nil {
+11 -137
View File
@@ -8,7 +8,6 @@ import (
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/licences"
"github.com/novox/mesh-controller/internal/link"
)
@@ -17,8 +16,8 @@ import (
var aDigest = "sha256:" + strings.Repeat("e", 64)
// **A build is recorded by digest and path**, whatever address the builder pushed to — what it
// made and what it stood on both; an image the module runs from elsewhere is left where it says.
// **A build is recorded by digest and path**, whatever address the builder pushed to — and only
// what the build made is rewritten: an image the module runs from elsewhere is left where it says.
func TestABuildIsRecordedWithoutTheStoresAddress(t *testing.T) {
manifest, _ := json.Marshal(map[string]any{
"module": "gitea", "version": "1",
@@ -65,11 +64,8 @@ func TestABuildIsRecordedWithoutTheStoresAddress(t *testing.T) {
if strings.Contains(string(kept.Manifest), "anchor.internal:5100") {
t.Errorf("the recorded manifest still carries the store's address:\n%s", kept.Manifest)
}
// What the build stood on is an edge to another module's artifact, and it is recorded the way
// that artifact is: by path in the store, so the edge still names the same thing when the
// store answers at another address.
if kept.Against[0] != catalogue.ArtifactStoreScheme+"mesh-tools/runtime@"+aDigest {
t.Errorf("what the build stood on was recorded by address: %v", kept.Against)
if kept.Against[0] != "anchor.internal:5100/mesh-tools/runtime@"+aDigest {
t.Errorf("what the build stood on was rewritten: %v", kept.Against)
}
}
@@ -157,7 +153,7 @@ func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T)
if err != nil {
t.Fatal(err)
}
control, err := withSeatPorts(m).Resolve([]catalogue.Built{{Name: "server", Kind: catalogue.ArtifactImage,
control, err := m.Resolve([]catalogue.Built{{Name: "server", Kind: catalogue.ArtifactImage,
Reference: "registry.example/control@" + aDigest}})
if err != nil {
t.Fatal(err)
@@ -202,137 +198,15 @@ func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T)
t.Fatal("the control plane's container is not in its own node's declaration")
}
for key, want := range map[string]string{
"MESH_STORE_INVENTORY_PORT": "6852",
"MESH_STORE_IDENTITY_PORT": "6852",
"MESH_STORE_LICENCES_PORT": "6852",
"MESH_BROKER_AMQP_PORT": "5679",
// Neither given nor assigned by the mesh: the manifest's own number is NOT the answer,
// because the sealed value beside it carries the port genesis wrote (finding F3).
"MESH_BROKER_ADDRESS_PORT": "",
"MESH_BROKER_MANAGEMENT_PORT": "",
"MESH_STORE_INVENTORY_PORT": "6852",
"MESH_STORE_IDENTITY_PORT": "6852",
"MESH_STORE_LICENCES_PORT": "6852",
"MESH_BROKER_AMQP_PORT": "5679",
"MESH_BROKER_ADDRESS_PORT": "5671",
"MESH_BROKER_MANAGEMENT_PORT": "15672",
} {
if env[key] != want {
t.Errorf("the control plane is told %s=%v; the node says %q", key, env[key], want)
}
}
}
// withSeatPorts is the control plane's manifest with the seat placeholders in its environment —
// added here until module.json carries them (the manifest lands one commit after the code that
// fills it, so a control plane one build behind never sees a placeholder it cannot fill).
func withSeatPorts(m catalogue.Manifest) catalogue.Manifest {
seatPorts := map[string]string{
"MESH_STORE_INVENTORY_PORT": "${seat:mesh-store:5432}",
"MESH_STORE_IDENTITY_PORT": "${seat:mesh-store:5432}",
"MESH_STORE_LICENCES_PORT": "${seat:mesh-store:5432}",
"MESH_BROKER_AMQP_PORT": "${seat:mesh-broker:5672}",
"MESH_BROKER_MANAGEMENT_PORT": "${seat:mesh-broker:15672}",
"MESH_BROKER_ADDRESS_PORT": "${seat:mesh-broker:5671}",
}
out := m
out.Resources = nil
for _, r := range m.Resources {
if r["type"] != "container" {
out.Resources = append(out.Resources, r)
continue
}
copied := map[string]any{}
for k, v := range r {
copied[k] = v
}
env := map[string]any{}
if had, ok := r["env"].(map[string]any); ok {
for k, v := range had {
env[k] = v
}
}
for k, v := range seatPorts {
if _, said := env[k]; !said {
env[k] = v
}
}
copied["env"] = env
out.Resources = append(out.Resources, copied)
}
return out
}
// aLoneNode is one capable machine with nothing placed on any network — the control-node during
// genesis, before the "network" step, which is after the store, the broker, the vault and the
// catalogue have each been built and pushed (finding F2).
func aLoneNode(t *testing.T) *stores {
t.Helper()
inventory.ForTest(t)
licences.ForTest(t)
open, err := openStores(t.Context())
if err != nil {
t.Fatal(err)
}
t.Cleanup(open.Close)
for _, m := range provided {
if err := open.inventory.Provide(t.Context(), m); err != nil {
t.Fatal(err)
}
}
record, err := open.inventory.AddNode(t.Context(), "anchor")
if err != nil {
t.Fatal(err)
}
reported, _ := json.Marshal(map[string]any{"capabilities": []map[string]any{
{"name": "container-runtime", "present": true}}})
var profile map[string]any
_ = json.Unmarshal(reported, &profile)
if err := open.inventory.RecordProfile(t.Context(), record.ID, profile); err != nil {
t.Fatal(err)
}
if err := open.inventory.RecordSealingKey(t.Context(), record.ID, aPublicKey(t)); err != nil {
t.Fatal(err)
}
return open
}
// **Before the network exists, the store's own node reaches it by loopback** — never refused,
// never handed the scheme: a genesis pushes the store, the broker, the vault and the catalogue to
// a node on no network, and builds the catalogue on a base it must be able to pull.
func TestOnANodeWithNoNetworkTheStoreIsReachedByLoopback(t *testing.T) {
open := aLoneNode(t)
ctx := t.Context()
register(t, open, aStore())
register(t, open, catalogue.Manifest{Module: "builder", Version: "1",
Requires: []string{catalogue.ArtifactStoreProvision},
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-builder",
"image": "registry.example/mesh-builder@" + aDigest}}})
if err := open.inventory.RecordBuild(ctx, inventory.Build{
ID: "b1", Repository: "r", Module: "postgres", Commit: "abc",
Made: []inventory.Artifact{{Name: "runtime", Kind: "image",
Reference: catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}},
}); err != nil {
t.Fatal(err)
}
register(t, open, catalogue.Manifest{Module: "postgres", Version: "1",
Resources: []map[string]any{{"id": "runtime", "type": "container", "name": "mesh-postgres",
"image": catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}}})
for _, module := range []string{"distribution", "builder", "postgres"} {
if _, err := assign(ctx, open, "anchor", module); err != nil {
t.Fatal(err)
}
}
if err := open.inventory.SetSettings(ctx, "anchor", "distribution",
map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5100}}); err != nil {
t.Fatal(err)
}
var image any
for _, r := range composed(t, open, "anchor").Resources {
if r["id"] == "postgres.runtime" {
image = r["image"]
}
}
if image != "127.0.0.1:5100/postgres/runtime@"+aDigest {
t.Fatalf("on the store's own node, off any network, the image is fetched as %v", image)
}
held := heldBy(ctx)
if got := held["postgres/runtime"]; got != "127.0.0.1:5100/postgres/runtime@"+aDigest {
t.Fatalf("a builder beside the store is handed the base %q", got)
}
}
+1 -65
View File
@@ -14,7 +14,6 @@ import (
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/overlay"
)
// A node is adopted or converged (novox/hq ADR 0100), and it is said to be adopted wherever the
@@ -46,9 +45,6 @@ func showMode(ctx context.Context, inv *inventory.Inventory, node inventory.Node
return nil
}
fmt.Printf(" firewall found %s\n", orNone(said.Firewall))
if err := showTunnel(ctx, inv, node.Name); err != nil {
return err
}
if len(said.Held) == 0 {
fmt.Printf(" holding nothing found\n")
}
@@ -67,44 +63,6 @@ func showMode(ctx context.Context, inv *inventory.Inventory, node inventory.Node
return nil
}
// showTunnel is the node show lines about the tunnel an adopted node found and carried (novox/hq
// ADR 0105): what it presented at enrolment, and what it last said about taking it over.
func showTunnel(ctx context.Context, inv *inventory.Inventory, name string) error {
tunnel, err := inv.TunnelOf(ctx, name)
if errors.Is(err, inventory.ErrNoTunnel) {
return nil
}
if err != nil {
return err
}
fmt.Printf(" tunnel found %s on port %d, %s in %s, %d peer(s)\n",
tunnel.Interface, tunnel.Port, tunnel.Address, tunnel.Range, len(tunnel.Peers))
carried, said, err := inv.CarriedTunnelOf(ctx, name)
if err != nil {
return err
}
switch {
case !said:
fmt.Printf(" %-17s not yet taken over — the node has not said so\n", "")
case carried.State == inventory.CarriedTaken:
fmt.Printf(" %-17s taken over: %s is down and disabled, never flushed; the mesh's interface "+
"runs with its key, port and %d peer(s)\n", "", carried.Interface, carried.Peers)
case carried.State == inventory.CarriedDown:
fmt.Printf(" %-17s TUNNEL DOWN: %s is stopped and the mesh's interface is not up — the peers "+
"reach nothing. On the machine: systemctl start %s\n", "", carried.Interface,
"wg-quick@"+carried.Interface)
default:
fmt.Printf(" %-17s NOT taken over: %s is still the interface the peers reach\n", "", carried.Interface)
}
if said && carried.Note != "" {
fmt.Printf(" %-17s %s\n", "", carried.Note)
}
if said && carried.Kept != "" {
fmt.Printf(" %-17s its configuration's original kept at %s\n", "", carried.Kept)
}
return nil
}
func orNone(s string) string {
if s == "" {
return "none reported"
@@ -255,28 +213,6 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s
return "", fmt.Errorf("%s still holds what it found, and a service is taken on its own, "+
"never by the flip:\n%s", node, strings.Join(holding, "\n"))
}
// And refused while a peer of the tunnel this hub took over has not enrolled (novox/hq ADR
// 0105): the flip loads the derived filter and retires the found firewall, and a machine the
// mesh has no record of is not one the filter admits — it would go dark.
if _, hubName, adopted, err := inv.AdoptedTunnel(ctx); err != nil {
return "", err
} else if adopted && hubName == node {
carried, err := inv.CarriedPeers(ctx)
if err != nil {
return "", err
}
var waiting []string
for _, c := range carried {
if c.EnrolledAs == "" {
waiting = append(waiting, fmt.Sprintf(" %s at %s", overlay.CarriedName(c.PublicKey), c.Address))
}
}
if len(waiting) > 0 {
return "", fmt.Errorf("%s carries peers of the tunnel it took over that have not enrolled, and "+
"converging would cut them off — enrol each first (`overlay show` says which are enrolled):\n%s",
node, strings.Join(waiting, "\n"))
}
}
shelf, err := inv.Catalogue(ctx)
if err != nil {
@@ -355,7 +291,7 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s
strings.Join(assigned, ", "))
}
if !slices.Contains(assigned, filter) {
if _, err := inv.Assign(ctx, node, filter); err != nil {
if err := inv.Assign(ctx, node, filter); err != nil {
return "", err
}
if _, _, err := planFor(ctx, open, node); err != nil {
+2 -2
View File
@@ -37,13 +37,13 @@ func askCommand(ctx context.Context, args []string) error {
arguments = json.RawMessage(positionals[2])
}
server, err := connectLink(ctx, nil, nil, nil)
server, err := link.Connect(nil, nil)
if err != nil {
return err
}
defer server.Close()
answer, err := link.Ask(ctx, server.Bus(), module, tool, arguments, *wait)
answer, err := link.Ask(ctx, server.Channel(), module, tool, arguments, *wait)
if err != nil {
return err
}
+88 -171
View File
@@ -2,6 +2,8 @@ package main
import (
"context"
"crypto/rand"
"encoding/base64"
"encoding/json"
"errors"
"flag"
@@ -29,51 +31,6 @@ import (
// control plane may send a machine is bounded by the declaration language. This is the shape the
// builder module will take when it is given work over the broker; today a person runs it, and the
// mesh records the result the same way either way.
// buildOn rebuilds every module the mesh holds that stands on the named module's artifacts — the
// rebuild a changed base needs, which nothing else asks for: their sources did not move, and
// "behind" does not see a base that did (novox/hq 04-ISSUES/131). Bases first among them too.
func buildOn(ctx context.Context, base string, wait time.Duration) error {
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
held, err := open.inventory.Catalogued(ctx)
if err != nil {
return err
}
against, err := open.inventory.BuiltAgainst(ctx)
if err != nil {
return err
}
var on []inventory.Entry
for _, e := range held {
if standsOnModule(e, base, against) {
on = append(on, e)
}
}
if len(on) == 0 {
fmt.Printf("nothing the mesh holds stands on %s\n", base)
return nil
}
on = orderByBases(on, against)
fmt.Printf("%d module(s) stand on %s:\n", len(on), base)
var failed []string
for _, e := range on {
fmt.Printf("--- %s\n", e.Manifest.Module)
source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
if err := buildOne(ctx, source, e.Source.Path, e.Source.Ref, wait); err != nil {
fmt.Printf(" %v\n", err)
failed = append(failed, e.Manifest.Module)
}
}
if len(failed) > 0 {
return fmt.Errorf("%d of %d could not be built: %s", len(failed), len(on), strings.Join(failed, ", "))
}
fmt.Printf("\n%d module(s) rebuilt on %s. `push --behind` sends them on\n", len(on), base)
return nil
}
func buildCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("build", flag.ContinueOnError)
ref := set.String("ref", "", "the branch, tag or commit to build")
@@ -89,52 +46,33 @@ func buildCommand(ctx context.Context, args []string) error {
// retype each repository is asking them to be the loop. Naming a repository and asking which
// ones need building are different requests, so they are not combined.
behind := set.Bool("behind", false, "every module the mesh holds older than its source has")
on := set.String("on", "", "rebuild every module that stands on this module's artifacts — the rebuild a changed base needs")
// A repository on the mesh's own forge, named by its path there (novox/hq ADR 0111). Without it
// the repository is external, cloned exactly as given — see source.go.
self := set.Bool("self", false, "the repository is a path on the forge holding the git seat")
positionals, err := parseAround(set, args)
if err != nil {
return err
}
if *on != "" {
if len(positionals) != 0 || *behind || *self {
return errors.New("build --on <module> names a base and nothing else")
}
return buildOn(ctx, *on, *wait)
}
if *behind {
if len(positionals) != 0 || *self {
if len(positionals) != 0 {
return errors.New("build <repository> or build --behind, not both: one names a " +
"repository and the other asks which need building")
}
return buildBehind(ctx, *wait)
}
if len(positionals) != 1 {
return errors.New("build <repository> [--self] [--path P] [--ref R] [--wait D] [--dry-run] | build --behind | build --on <module>")
}
source := buildSource{Repository: positionals[0]}
if *self {
if err := onASeat(source.Repository); err != nil {
return err
}
source.Seat = gitSeat
return errors.New("build <repository> [--ref R] [--wait D] [--dry-run]")
}
if *dryRun {
return buildAndShow(ctx, source, *path, *ref, *wait)
return buildAndShow(ctx, positionals[0], *path, *ref, *wait)
}
return buildOne(ctx, source, *path, *ref, *wait)
return buildOne(ctx, positionals[0], *path, *ref, *wait)
}
// buildFrom turns what a builder said into what the mesh keeps.
//
// **By digest and path, never by where it was pushed** (novox/hq 04-ISSUES/102). The builder
// says `<registry>:<port>/<module>/<artifact>@sha256:…`; the mesh records the artifact-store
// reference and composes the store's address back in where a reference is used. `against` — what
// the build stood on, the catalogue's edge — is recorded the same way, so an edge names a module's
// artifact and not the machine it was pulled from.
// reference and composes the store's address back in where a reference is used. `against` is kept
// as announced: it is what the build stood on as the builder saw it, and the catalogue's edge.
func buildFrom(result link.BuildResult) inventory.Build {
kept := inventory.Build{
ID: result.ID, Repository: result.Repository, Ref: result.Ref,
@@ -144,13 +82,7 @@ func buildFrom(result link.BuildResult) inventory.Build {
// edges, and it is not always listening when a build happens — on a fresh mesh it cannot
// be, for exactly the modules it needs most. Keeping them is what makes a replay able to
// rebuild the graph rather than a list of names.
Path: result.Path,
}
for _, ref := range result.Against {
kept.Against = append(kept.Against, catalogue.Recorded(ref))
}
for _, r := range result.Read {
kept.Read = append(kept.Read, inventory.ReadRepository{Repository: r.Repository, Ref: r.Ref})
Path: result.Path, Against: result.Against,
}
var announced []inventory.Artifact
for _, made := range result.Made {
@@ -264,45 +196,85 @@ func builderCommand(ctx context.Context, args []string) error {
}
name := positionals[1]
// **The build machine's credential is a module's credential** (novox/hq ADR 0131, design 28
// task 5.5): minted into the mesh's records and sealed to the machine as the builder module's
// broker secret, usable at the next push — the same act `module issue` performs, and the same
// account the composed user list carries. Nothing is created on a server; the bus reads the list.
open, err := openStores(ctx)
management, err := broker.ManagementFromEnvironment()
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
shelf, err := inv.Catalogue(ctx)
if err != nil {
// The same shape of secret a token carries: enough entropy that guessing is not a strategy,
// and safe to put in a URL because that is where it goes.
raw := make([]byte, 32)
if _, err := rand.Read(raw); err != nil {
return err
}
m, known := shelf[*module]
if !known {
return fmt.Errorf("%s is not in the catalogue; `module add` it first", *module)
password := base64.RawURLEncoding.EncodeToString(raw)
if err := management.CreateBuilderAccount(ctx, name, password); err != nil {
return err
}
fmt.Printf("build machine %s: ", name)
node := *forNode
if node == "" {
entries, err := inv.Catalogued(ctx)
fmt.Printf("broker account %s created, scoped to the %s queue and the %s exchange\n\n",
name, link.BuildQueue, link.Exchange)
if *forNode != "" {
known, err := broker.FromEnvironment()
if err != nil {
return fmt.Errorf("cannot deliver a credential without knowing where the broker is: %w", err)
}
inv, err := openInventory(ctx)
if err != nil {
return err
}
for _, e := range entries {
if e.Manifest.Module == *module && len(e.On) > 0 {
node = e.On[0]
}
defer inv.Close()
brokerAddr, err := brokerReachableAt(ctx, inv, known, *forNode)
if err != nil {
return err
}
// The URL and what verifies the broker, together. A mesh's broker presents a certificate
// of the mesh's own, which is in no public trust store — so a URL on its own reaches only
// a broker somebody else vouches for, and the connection fails at TLS with an error about
// an unknown authority rather than about a missing pin.
//
// **The same two facts a node's token carries** (novox/hq ADR 0004), delivered the same
// way: out of band relative to the broker, so what is trusted does not come from the thing
// being trusted.
held, err := json.Marshal(struct {
URL string `json:"url"`
Fingerprint string `json:"fingerprint,omitempty"`
}{
URL: fmt.Sprintf("amqps://%s:%s@%s/", name, password, brokerAddr),
Fingerprint: known.Fingerprint,
})
if err != nil {
return err
}
if err := inv.AcceptSecretForModule(ctx, *forNode, *module, "broker", string(held)); err != nil {
return err
}
// Not printed. It is sealed to that machine and the mesh cannot read it back, which is
// the whole point — printing it here would put the one copy that matters on a terminal.
fmt.Printf(" sealed to %s, for the %s module. It arrives with the next push.\n",
*forNode, *module)
fmt.Printf(" run `push %s` to send it\n", *forNode)
return nil
}
if node == "" {
return fmt.Errorf("%s is assigned nowhere; `assign <machine> %s` first, or say --node", *module, *module)
// The whole line only when the address is known. A URL with a placeholder where the host
// should be is a URL somebody pastes and then debugs, and the placeholder is the last thing
// they look at.
if known, err := broker.FromEnvironment(); err == nil {
fmt.Printf(" MESH_BROKER_AMQP=amqps://%s:%s@%s/\n\n", name, password, known.Address)
} else {
fmt.Printf(" the password is %s\n\n", password)
fmt.Printf(" This control plane has no %s, so it cannot say where the broker is.\n"+
" Put the password in MESH_BROKER_AMQP on the build machine.\n\n",
broker.AddressVar)
}
address, err := broker.BusAddress()
if err != nil {
return err
}
return issueOnTheNewBus(ctx, inv, m, node, address)
// Shown once, like a token, and for the same reason: what is stored is the broker's own hash
// of it, and a control plane that could show it back would be a control plane that holds it.
fmt.Println("This is the only time it is shown.")
return nil
}
// buildBehind builds every module the mesh holds older than its source has.
@@ -347,22 +319,13 @@ func buildBehind(ctx context.Context, wait time.Duration) error {
}
fmt.Println()
// Bases first: a module built before the module it stands on is built against the old one
// and reports success (novox/hq 04-ISSUES/131).
against, err := inv.BuiltAgainst(ctx)
if err != nil {
return err
}
stale = orderByBases(stale, against)
var failed []string
for _, e := range stale {
fmt.Printf("--- %s\n", e.Manifest.Module)
// Its own recorded ref, not its head commit: a module tracking a branch should be built
// from that branch, and pinning to the commit the mesh happened to notice would quietly
// turn a tracked branch into a pin.
source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
if err := buildOne(ctx, source, e.Source.Path, e.Source.Ref, wait); err != nil {
if err := buildOne(ctx, e.Source.Repository, e.Source.Path, e.Source.Ref, wait); err != nil {
fmt.Printf(" %v\n", err)
failed = append(failed, e.Manifest.Module)
}
@@ -380,21 +343,14 @@ func buildBehind(ctx context.Context, wait time.Duration) error {
// buildOne asks a build machine for one repository and records everything that came back.
//
// Separated from the command so `--behind` can walk a list without a second path to the same act.
func buildOne(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error {
// Before anything is asked of a builder: a source on a seat nobody holds is refused here, with
// the reason, rather than sent to a machine to fail at `git clone`.
repository, err := cloneFrom(ctx, source)
if err != nil {
return err
}
func buildOne(ctx context.Context, repository, path, ref string, wait time.Duration) error {
ident, err := openIdentity(ctx)
if err != nil {
return err
}
defer ident.Close()
server, err := connectLink(ctx, nil, nil, nil)
server, err := link.Connect(nil, nil)
if err != nil {
return err
}
@@ -409,10 +365,7 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti
Ref: ref,
Held: heldBy(ctx),
}
fmt.Printf("asked for %s", source)
if source.Seat != "" {
fmt.Printf(" (%s)", repository)
}
fmt.Printf("asked for %s", request.Repository)
if path != "" {
fmt.Printf(" at %s", path)
}
@@ -421,13 +374,7 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti
}
fmt.Println()
ask, err := askOver(server)
if err != nil {
return err
}
defer ask.Close()
result, err := ask.Submit(ctx, request, wait)
result, err := link.RequestBuild(ctx, server.Channel(), request, wait)
if err != nil {
return err
}
@@ -467,18 +414,11 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti
}
// Recorded with where it came from, so "is this current?" is answerable without building it
// again (novox/hq ADR 0009). **For a source on a seat, as the path and the seat, never the URL
// just cloned** (ADR 0111): the URL is where the forge runs today, and recording it would put
// the forge's address back into every module built from it. The build log above keeps the URL,
// because that is what was cloned.
recorded := inventory.Source{
// again (novox/hq ADR 0009).
if err := inv.RegisterModule(ctx, manifest, inventory.Source{
Repository: result.Repository, Path: result.Path, Ref: result.Ref,
BuiltFrom: result.Commit, Head: result.Commit,
}
if source.Seat != "" {
recorded.Repository, recorded.Seat = source.Repository, source.Seat
}
if err := inv.RegisterModule(ctx, manifest, recorded); err != nil {
}); err != nil {
return err
}
fmt.Printf("\n%s %s, built on %s from %s\n",
@@ -488,29 +428,19 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti
}
// buildAndShow builds and prints the manifest without recording anything.
func buildAndShow(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error {
repository, err := cloneFrom(ctx, source)
if err != nil {
return err
}
func buildAndShow(ctx context.Context, repository, path, ref string, wait time.Duration) error {
ident, err := openIdentity(ctx)
if err != nil {
return err
}
defer ident.Close()
server, err := connectLink(ctx, nil, nil, nil)
server, err := link.Connect(nil, nil)
if err != nil {
return err
}
defer server.Close()
ask, err := askOver(server)
if err != nil {
return err
}
defer ask.Close()
result, err := ask.Submit(ctx, link.BuildRequest{
result, err := link.RequestBuild(ctx, server.Channel(), link.BuildRequest{
ID: fmt.Sprintf("%s-%d", "build", time.Now().UnixNano()),
Repository: repository, Path: path, Ref: ref,
Held: heldBy(ctx),
@@ -580,7 +510,7 @@ func heldBy(ctx context.Context) map[string]string {
fmt.Fprintf(os.Stderr, "could not read what this mesh has built: %v\n", err)
return nil
}
address, err := whereABuilderReachesTheStore(ctx, open.inventory)
address, err := whereTheStoreIs(ctx, open.inventory)
if err != nil {
fmt.Fprintf(os.Stderr, "could not find the artifact store on this mesh's network, so a "+
"module naming a base will be handed a reference nothing can fetch: %v\n", err)
@@ -595,16 +525,3 @@ func heldBy(ctx context.Context) map[string]string {
}
return routed
}
// askOver opens the way a build is asked for, on whichever bus the mesh is on.
//
// **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5). On the bus
// the mesh runs on today this needs the controller's own connection, so it is handed one; on the bus
// being built it dials, because a build request is a one-shot and holds nothing else.
func askOver(_ *link.Server) (link.Builders, error) {
address, err := broker.BusAddress()
if err != nil {
return nil, err
}
return link.BuildsOverNATS(address)
}
@@ -1,33 +0,0 @@
package main
// The broker opening belongs only on the node that listens on it (novox/hq: it leaked onto
// every enrolled node's declaration, opening a from-anywhere hole for a port nothing there
// serves). foundationPortsFor is the scope.
import (
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
func TestTheBrokerHostGetsTheFoundationOpening(t *testing.T) {
broker := catalogue.Manifest{Module: "lavinmq", Listens: []catalogue.Listening{
{Port: 5671, Protocol: "tcp", From: "mesh"},
{Port: 5672, Protocol: "tcp", From: "mesh"},
}}
got := foundationPortsFor(5671, []catalogue.Manifest{broker})
if len(got) != 1 || got[0] != 5671 {
t.Fatalf("the node that listens on the broker port keeps it; got %v", got)
}
}
func TestANodeThatOnlyDialsTheBrokerGetsNoOpening(t *testing.T) {
// ace's set: things that reach the broker as a client, none listening on 5671.
ace := []catalogue.Manifest{
{Module: "plex", Listens: []catalogue.Listening{{Port: 32400, Protocol: "tcp", From: "anywhere"}}},
{Module: "postgres", Listens: []catalogue.Listening{{Port: 5432, Protocol: "tcp", From: "mesh"}}},
}
if got := foundationPortsFor(5671, ace); got != nil {
t.Fatalf("a node that only dials out opens nothing for the broker; got %v", got)
}
}
+1 -13
View File
@@ -76,10 +76,7 @@ func run() error {
return pinCommand(ctx, args[1:], true)
case "unpin":
return pinCommand(ctx, args[1:], false)
// `prepare` is how the mesh asks any module to bring its state to the shape this version needs
// (novox/hq ADR 0135), and the control plane answers it the same way as everything else — its
// own schema is not a special case. `migrate` remains the word a person types.
case "prepare", "migrate":
case "migrate":
return migrate(ctx)
case "node":
return nodeCommand(ctx, args[1:])
@@ -117,12 +114,6 @@ func run() error {
return planCommand(ctx, args[1:])
case "push":
return pushCommand(ctx, args[1:])
case "rollout":
return rolloutCommand(ctx, args[1:])
case "seats":
return seatsCommand(ctx, args[1:])
case "seat":
return seatCommand(ctx, args[1:])
case "status":
return statusCommand(ctx, args[1:])
case "version":
@@ -141,7 +132,6 @@ func usage() {
fmt.Fprint(os.Stderr, `mesh-controller — the control plane
migrate bring each context's schema up to date
prepare the same, asked the way the mesh asks any module (ADR 0135)
node add <name> [--adopted] create a node record; --adopted: the machine is in use
node list the nodes this mesh knows about
node show <name> what one machine reported it can do, and why
@@ -167,7 +157,6 @@ func usage() {
upgrade <name> roll-out [--together] ...send it to the machines running it
upgrade <name> record ...record that they are behind, and send nothing
status [--json] what is wrong, what is quiet, and what is out of date
seats [--json] every seat this mesh defines, what it delivers, and who holds it
board [--listen ADDR] the same three questions, as a page that holds nothing
api --issuer URL [--listen A] assign and unassign over http, for a surface that is not here
assign <node> <module> put a module on a node
@@ -188,7 +177,6 @@ func usage() {
operator key show the operator key, and what it can recover
build <repository> [--ref R] have a build machine build it, and record what came out
build --behind build every module the mesh holds older than its source
build --on <module> rebuild every module that stands on this module's artifacts, bases first
builds [<module>] what has been built lately, and what came of it
builder issue <name> a broker account for a build machine, scoped to build work,
delivered as the builder module's broker secret (module add it first)
+1 -1
View File
@@ -73,7 +73,7 @@ func aMesh(t *testing.T) *stores {
if err := open.inventory.RecordOverlayKey(t.Context(), record.ID, aPublicKey(t)); err != nil {
t.Fatal(err)
}
if _, err := open.inventory.Assign(t.Context(), name, overlay.Name); err != nil {
if err := open.inventory.Assign(t.Context(), name, overlay.Name); err != nil {
t.Fatal(err)
}
}
+70 -148
View File
@@ -2,6 +2,8 @@ package main
import (
"context"
"crypto/rand"
"encoding/base64"
"encoding/json"
"errors"
"flag"
@@ -69,20 +71,12 @@ func moduleCommand(ctx context.Context, args []string) error {
repo := set.String("source", "", "where this module comes from")
ref := set.String("ref", "", "the branch followed there")
commit := set.String("commit", "", "the commit this manifest was read at")
// **Where inside the repository the module is** (novox/hq ADR 0069). A module is a
// repository *and* a directory, and a record that carries only the repository names a
// module.json at its root — so every later build of it looks in the wrong place and fails
// with "no module.json at its root". Nine modules on this mesh were registered that way
// and none of them could be rebuilt (2026-09-28).
path := set.String("path", "", "the module's directory inside that repository")
self := set.Bool("self", false, "the source is a path on the forge holding the git seat")
positionals, err := parseAround(set, args[1:])
if err != nil {
return err
}
if len(positionals) != 1 {
return errors.New("module add <manifest.json> [--source <repo> [--self] [--path P] " +
"--ref <branch> --commit <sha>]")
return errors.New("module add <manifest.json> [--source <repo> --ref <branch> --commit <sha>]")
}
raw, err := os.ReadFile(positionals[0])
if err != nil {
@@ -92,24 +86,23 @@ func moduleCommand(ctx context.Context, args []string) error {
if err != nil {
return err
}
from, err := whereItComesFrom(*repo, *ref, *commit, *path, *self)
if err != nil {
return err
// Provenance together or not at all. A source with no commit cannot be compared against
// anything, so it would record where the module came from and still never be able to say
// the mesh is behind it — which is the one thing recording it is for.
if (*repo == "") != (*commit == "") {
return errors.New("--source and --commit go together: a source with no commit " +
"cannot be compared against anything, and a commit with no source has nothing " +
"to be compared with")
}
if err := inv.RegisterModule(ctx, m, from); err != nil {
if err := inv.RegisterModule(ctx, m, inventory.Source{
Repository: *repo, Ref: *ref, BuiltFrom: *commit,
}); err != nil {
return err
}
fmt.Printf("%s registered", m.Module)
if *commit != "" {
fmt.Printf(" from %s", short(*commit))
}
if *repo != "" && *path == "" {
// Said, not refused: a module really at the root is the ordinary case for a repository
// of its own. But a repository holding many modules and a record naming none of them is
// a module nothing can rebuild, and the person adding it is the one who knows which.
fmt.Printf("\n no directory inside %s, so it is built from that repository's root — "+
"`--path` if the module lives in a directory there", *repo)
}
if len(m.Provides) > 0 {
fmt.Printf(", providing %s", describeOffers(m.Provides))
}
@@ -264,15 +257,66 @@ func moduleCommand(ctx context.Context, args []string) error {
return err
}
// Which bus this mesh is on. A module gets a credential for exactly one, and the two are
// made in entirely different ways: on the bus the mesh runs on today an account is a
// management call, and on the bus being built it is a row the next composition writes into
// the server's user list (novox/hq design 25 §4).
busAddress, err := broker.BusAddress()
management, err := broker.ManagementFromEnvironment()
if err != nil {
return err
}
return issueOnTheNewBus(ctx, inv, m, *forNode, busAddress)
// The foundation owns the bus; make sure it exists before a module binds onto it.
if err := management.EnsureEventExchanges(ctx); err != nil {
return err
}
secret := make([]byte, 32)
if _, err := rand.Read(secret); err != nil {
return err
}
password := base64.RawURLEncoding.EncodeToString(secret)
account, err := management.CreateModuleAccount(ctx, *forNode, module, password, m.Emits, m.Consumes)
if err != nil {
return err
}
// A consumer's queue, with its dead-letter, is the foundation's to declare — its own account
// may not (ADR 0043). Made now, so it exists before the module binds onto it.
if len(m.Consumes) > 0 {
if err := management.EnsureModuleQueue(ctx, *forNode, module); err != nil {
return err
}
}
known, err := broker.FromEnvironment()
if err != nil {
return fmt.Errorf("cannot deliver a credential without knowing where the broker is: %w", err)
}
brokerAddr, err := brokerReachableAt(ctx, inv, known, *forNode)
if err != nil {
return err
}
// The URL and what verifies the broker, together — a mesh's broker presents its own
// certificate, in no public trust store, so a URL alone fails at TLS (as `builder issue`).
held, err := json.Marshal(struct {
URL string `json:"url"`
Fingerprint string `json:"fingerprint,omitempty"`
Node string `json:"node"`
Module string `json:"module"`
}{
URL: fmt.Sprintf("amqps://%s:%s@%s/", account, password, brokerAddr),
Fingerprint: known.Fingerprint,
// The node and module the account is for, so the runtime names its queue as the mesh
// scoped it (<node>.<module>.events) without a manifest having to interpolate a node.
Node: *forNode,
Module: module,
})
if err != nil {
return err
}
if err := inv.AcceptSecretForModule(ctx, *forNode, module, "broker", string(held)); err != nil {
return err
}
fmt.Printf("broker account %s created for %s, scoped to what it emits and consumes\n",
account, module)
fmt.Printf(" sealed to %s. It arrives with the next push — `push %s` to send it\n",
*forNode, *forNode)
return nil
default:
return fmt.Errorf("module has no %q; it has add, list, moved, forget and issue", args[0])
@@ -523,125 +567,3 @@ func mayIssue(m catalogue.Manifest) error {
}
return nil
}
// issueOnTheNewBus gives an assigned module its credential on the bus being built.
//
// **Three things differ from a management call, and each is the point of the move.** The credential
// is minted into the mesh's records and becomes usable at the next composition, so there is no
// server to be reachable for this to work. The password travels beside the address rather than inside
// it, because the runtime's contract already separates them and a credential embedded in a URL is one
// that leaks into every log line that prints a connection. And the module's durable consumer is
// derived from what it declared rather than declared by name, so a module cannot ask for delivery of
// something it did not say it consumes.
func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest,
node, busAddress string) error {
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: m.Module}.Username()
password, err := inv.MintBusPassword(ctx, inventory.BusUser{
Username: user, Kind: inventory.BusModule, Node: node, Module: m.Module,
})
if err != nil {
return err
}
// Where the module is told to find the bus, and what certificate it must present. The same pair
// a node is told, for the same reason: a mesh's bus presents its own certificate, in no public
// trust store, so an address alone fails at TLS.
known, err := broker.FromEnvironment()
if err != nil {
return fmt.Errorf("cannot deliver a credential without knowing where the bus is: %w", err)
}
reachable, err := brokerReachableAt(ctx, inv, known, node)
if err != nil {
return err
}
return issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password)
}
// issueWith is the delivery half: the minted password sealed to the machine as the module's broker
// secret, and the module's consumer created where the bus can be reached. Split from the minting
// so the move can issue every module against a bus whose address it worked out itself
// (`rollout mint`, design 28 task 5.2) rather than the one in this process's environment.
func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest,
node, busAddress string, known broker.Broker, reachable, user, password string) error {
held, err := json.Marshal(struct {
URL string `json:"url"`
Fingerprint string `json:"fingerprint,omitempty"`
Node string `json:"node"`
Module string `json:"module"`
User string `json:"user"`
Password string `json:"password"`
}{
URL: "nats://" + reachable, Fingerprint: known.Fingerprint,
Node: node, Module: m.Module, User: user, Password: password,
})
if err != nil {
return err
}
if err := inv.AcceptSecretForModule(ctx, node, m.Module, "broker", string(held)); err != nil {
return err
}
// And how it hears what it consumes. Derived from its declaration, and only when it declared
// something: a module that consumes nothing needs no consumer, and creating one would be a
// durable subscription nobody reads.
if consumer, needed := broker.ConsumerFor(broker.Principal{
Kind: broker.KindModule, Node: node, Module: m.Module,
Emits: m.Emits, Consumes: m.Consumes, Serves: m.Tools,
}); needed {
if busAddress == "" {
fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push`, then "+
"`rollout mint` again is harmless)\n", m.Module)
} else {
js, err := broker.Dial(busAddress)
if err != nil {
return fmt.Errorf("the credential is minted and the mesh cannot reach the bus to create "+
"how %s hears what it consumes: %w", m.Module, err)
}
defer js.Close()
if err := js.EnsureConsumer(consumer); err != nil {
return err
}
}
}
fmt.Printf("bus user %s minted for %s, scoped to what it emits and consumes\n", user, m.Module)
fmt.Printf(" sealed to %s. It arrives with the next push — `push %s` to send it\n", node, node)
fmt.Printf(" and it works once the bus has been told: the user list is composed into the " +
"machine holding mesh-broker\n")
return nil
}
// whereItComesFrom is the provenance a module handed over by hand records, and what a record must
// say to be worth anything later.
//
// **A module is a repository and a directory inside it** (novox/hq ADR 0069). A record carrying only
// the repository names a module.json at its root, so every later build of it looks in the wrong
// place — nine modules on this mesh were registered that way and none of them could be rebuilt
// (2026-09-28). The directory cannot be checked from here, because the control plane does not clone;
// what can be checked is that the record is whole.
func whereItComesFrom(repository, ref, commit, path string, self bool) (inventory.Source, error) {
// Provenance together or not at all. A source with no commit cannot be compared against
// anything, so it would record where the module came from and still never be able to say the
// mesh is behind it — which is the one thing recording it is for.
if (repository == "") != (commit == "") {
return inventory.Source{}, errors.New("--source and --commit go together: a source with " +
"no commit cannot be compared against anything, and a commit with no source has " +
"nothing to be compared with")
}
// A directory or a forge with no repository is half a location, and the half it keeps is the
// half nothing can be found with.
if repository == "" && (path != "" || self) {
return inventory.Source{}, errors.New("--path and --self say where inside a source and " +
"which forge holds it, so they need --source: without one there is nothing for them " +
"to be part of")
}
from := inventory.Source{Repository: repository, Ref: ref, BuiltFrom: commit, Path: path}
if self {
if err := onASeat(repository); err != nil {
return inventory.Source{}, err
}
from.Seat = gitSeat
}
return from, nil
}
+18 -241
View File
@@ -7,7 +7,6 @@ import (
"fmt"
"os"
"sort"
"strconv"
"strings"
"time"
@@ -22,33 +21,16 @@ import (
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
// nothing in it was wrong, and no one edit was the one that should have been a new file.
// DefaultOverlayCIDR is the range the mesh allocates from when nothing says another.
const DefaultOverlayCIDR = "10.42.0.0/16"
// overlayRange is the range the mesh allocates node addresses from.
//
// **The adopted tunnel's range first** (novox/hq ADR 0105): a hub that took over the tunnel it
// found is at that tunnel's address, its peers are at theirs, and every node's address is
// composed from the same range — the hub's, and every binding, hosts entry and endpoint derived
// from it. Those are readers of this; none of them stores the range. Without an adopted tunnel,
// the range genesis was told, or the default.
func overlayRange(ctx context.Context, inv *inventory.Inventory) (string, error) {
tunnel, _, adopted, err := inv.AdoptedTunnel(ctx)
if err != nil {
return "", err
}
if adopted {
return tunnel.Range, nil
}
func overlayCIDR() string {
if v := strings.TrimSpace(os.Getenv(OverlayCIDRVar)); v != "" {
return v, nil
return v
}
return DefaultOverlayCIDR, nil
return "10.42.0.0/16"
}
func overlayCommand(ctx context.Context, args []string) error {
if len(args) == 0 {
return errors.New("overlay place <node> [flags], overlay name <address> <name>, or overlay show")
return errors.New("overlay place <node> [flags], or overlay show")
}
// Answered before anything is opened. A message about which command to use should not need a
// database to say so, and needing one turns a redirect into a connection error.
@@ -69,29 +51,12 @@ func overlayCommand(ctx context.Context, args []string) error {
return overlayPlace(ctx, inv, args[1:])
case "show":
return overlayShow(ctx, open)
case "name":
return overlayName(ctx, inv, args[1:])
default:
return fmt.Errorf("overlay has no %q; it has place, name and show", args[0])
return fmt.Errorf("overlay has no %q; it has place and show", args[0])
}
}
// overlayName is the operator saying which machine a carried address is (novox/hq issue 112),
// so the mesh answers for its name until the machine enrols and verifies it.
func overlayName(ctx context.Context, inv *inventory.Inventory, args []string) error {
if len(args) != 2 {
return errors.New("overlay name <carried-address> <node-name>")
}
address, name := args[0], args[1]
if err := inv.NamePeer(ctx, address, name); err != nil {
return err
}
fmt.Printf("the peer at %s is %s until it enrols — the mesh answers for %s.<suffix> from the "+
"operator's word, and enrolment under this key must use this name\n", address, name, name)
return nil
}
func overlayPlace(ctx context.Context, inv *inventory.Inventory, args []string) error {
if len(args) == 0 {
return errors.New(
@@ -145,46 +110,16 @@ func overlayPlace(ctx context.Context, inv *inventory.Inventory, args []string)
}
}
// A hub that took over a tunnel listens on that tunnel's port — it is what the peers dial, and
// the reason the port is worth having (novox/hq ADR 0105). An endpoint on another port would
// have the mesh's interface up where no peer is listening for it.
found, err := inv.NodeByName(ctx, node)
if err != nil {
return err
}
var tunnel inventory.Tunnel
adoptsTunnel := false
if *hub && found.Adopted {
if t, err := inv.TunnelOf(ctx, node); err == nil {
tunnel = t
placed, _ := inv.Overlays(ctx)
for _, o := range placed {
if o.Name == node && o.Key == t.PublicKey {
adoptsTunnel = true
}
}
} else if !errors.Is(err, inventory.ErrNoTunnel) {
return err
}
}
if adoptsTunnel {
if port := portOfEndpoint(*endpoint); port != strconv.Itoa(tunnel.Port) {
return fmt.Errorf("%s takes over the tunnel it found on %s, which listens on port %d, and "+
"its endpoint %q names another port: the peers dial the tunnel's port, so the hub's "+
"endpoint must be on it", node, tunnel.Interface, tunnel.Port, *endpoint)
}
}
// Declared, all three. The address is evidence of reachability and is not the fact, and hub
// election by address prefix fails silently (novox/hq ADR 0007).
if err := inv.SetPlace(ctx, node, *endpoint, *site, *hub, ""); err != nil {
return err
}
cidr, err := overlayRange(ctx, inv)
found, err := inv.NodeByName(ctx, node)
if err != nil {
return err
}
address, err := inv.AssignAddress(ctx, found.ID, cidr)
address, err := inv.AssignAddress(ctx, found.ID, overlayCIDR())
if err != nil {
return err
}
@@ -193,10 +128,6 @@ func overlayPlace(ctx context.Context, inv *inventory.Inventory, args []string)
fmt.Println(" credentials issued for it before this placement keep their old broker address —" +
" `module issue` them again and push (novox/hq issue 059)")
switch {
case adoptsTunnel:
fmt.Printf(" the hub — it takes over the tunnel it found on %s: range %s, port %d, "+
"%d peer(s) carried until they enrol\n", tunnel.Interface, tunnel.Range, tunnel.Port,
len(tunnel.Peers))
case *hub:
fmt.Println(" the hub — every node not sharing a site routes through it")
case *endpoint == "":
@@ -223,47 +154,15 @@ func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool,
if err != nil {
return nil, err
}
// The tunnels adopted nodes take over, and the peers the hub's carries (novox/hq ADR 0105).
tunnels, err := inv.Tunnels(ctx)
if err != nil {
return nil, err
}
carried, err := inv.CarriedPeers(ctx)
if err != nil {
return nil, err
}
nodes := make([]overlay.Node, 0, len(places))
for _, p := range places {
if !on[p.Name] {
continue
}
n := overlay.Node{
nodes = append(nodes, overlay.Node{
Name: p.Name, Key: p.Key, Endpoint: p.Endpoint,
Site: p.Site, Hub: p.Hub, Address: p.Address,
}
if t, takes := tunnels[p.Name]; takes && t.NodeAdopted {
// Only an adopted node is told to take the found unit over: on a converged one there
// is nothing found to keep, and the host refuses the field. The range and the carried
// peers do not depend on the mode; the takeover does.
//
// **Refused, not composed, when the hub's record disagrees with the tunnel.** A
// declaration that stopped the found unit and raised the mesh's interface on another
// port or address would leave every peer dark while reporting the tunnel taken — so a
// hub placed before it took the tunnel over (or at the wrong port) is named here, and
// nothing is sent until it is re-placed.
if wrong := disagrees(p, t.Tunnel); wrong != "" {
return nil, fmt.Errorf("%s takes over the tunnel on %s and its placement disagrees with it: %s. "+
"Re-place it — `overlay place %s --hub --endpoint <host>:%d …` — and push again; "+
"nothing was composed", p.Name, t.Interface, wrong, p.Name, t.Port)
}
n.TakesOver = &overlay.TakeOver{Interface: t.Interface, Unit: t.Unit, Config: t.Config, Port: t.Port, MTU: t.MTU}
}
if p.Hub {
for _, c := range carried {
n.Carried = append(n.Carried, overlay.Carried{Key: c.PublicKey, Address: c.Address})
}
}
nodes = append(nodes, n)
})
}
if len(nodes) == 0 {
// Nobody was given it. An empty network is a legitimate mesh, not a broken one, so this
@@ -271,11 +170,7 @@ func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool,
// "no hub" to somebody who never asked for a network would be a lie about the cause.
return overlay.Empty(), nil
}
cidr, err := overlayRange(ctx, inv)
if err != nil {
return nil, err
}
g, err := overlay.From(nodes, cidr, "")
g, err := overlay.From(nodes, overlayCIDR(), "")
if g != nil {
// The artifact store, as this network reaches it. Found rather than configured: the
// provider is whichever module offers it, on whichever machine holds that module — and if
@@ -397,17 +292,6 @@ func overlayShow(ctx context.Context, open *stores) error {
return nil
}
// The tunnel the hub took over, if any, and the peers carried from it (novox/hq ADR 0105):
// listed apart from the nodes, because they are peers of the tunnel and not nodes of the
// mesh until they enrol — and once one has, it is listed as the node it became.
tunnel, hubName, adopted, err := open.inventory.AdoptedTunnel(ctx)
if err != nil {
return err
}
carried, err := open.inventory.CarriedPeers(ctx)
if err != nil {
return err
}
for _, n := range nodes {
place := n.Address
if place == "" {
@@ -417,74 +301,22 @@ func overlayShow(ctx context.Context, open *stores) error {
}
fmt.Printf("%-16s %-14s", n.Name, place)
switch {
case n.Hub && adopted:
fmt.Printf(" hub — over the tunnel it took over on %s (range %s, port %d)",
tunnel.Interface, tunnel.Range, tunnel.Port)
case n.Hub && hubName == n.Name && tunnel.Interface != "":
fmt.Printf(" hub — found a tunnel on %s and did NOT take it over: its key is not the tunnel's; "+
"`mesh-host overlay take --tunnel %s` on the machine takes it", tunnel.Interface, tunnel.Interface)
case n.Hub:
fmt.Print(" hub — found no tunnel; if the machine runs the predecessor's, " +
"`mesh-host overlay take --tunnel <iface>` there adopts it (novox/hq ADR 0105)")
fmt.Print(" hub")
case !n.Reachable():
fmt.Print(" not dialable")
}
if n.Site != "" {
fmt.Printf(" at %s", n.Site)
}
if n.TakesOver != nil && !n.Hub {
fmt.Printf(" takes over %s", n.TakesOver.Interface)
}
fmt.Println()
for _, p := range computed[n.Name] {
fmt.Printf(" → %-14s %-18s %s\n", p.Name, p.Allowed, p.Why)
}
}
if len(carried) > 0 {
fmt.Printf("\npeers of the tunnel %s took over — not nodes of the mesh until they enrol:\n", hubName)
for _, c := range carried {
state := "not yet enrolled"
if c.EnrolledAs != "" {
state = "enrolled as " + c.EnrolledAs + ", which keeps this address"
}
fmt.Printf(" %-16s %-14s %s\n", overlay.CarriedName(c.PublicKey), c.Address, state)
}
}
return nil
}
// disagrees says how a node's placement differs from the tunnel it takes over — its address not
// the tunnel's, its endpoint not on the tunnel's port — or nothing when both agree.
func disagrees(p inventory.Overlay, t inventory.Tunnel) string {
var wrong []string
want := t.Address
if i := strings.Index(want, "/"); i >= 0 {
want = want[:i]
}
if p.Address != want {
wrong = append(wrong, fmt.Sprintf("its address is %s and the tunnel's is %s", orNothing(p.Address), want))
}
if p.Reachable() && portOfEndpoint(p.Endpoint) != strconv.Itoa(t.Port) {
wrong = append(wrong, fmt.Sprintf("its endpoint %s is not on the tunnel's port %d", p.Endpoint, t.Port))
}
return strings.Join(wrong, "; ")
}
func orNothing(s string) string {
if s == "" {
return "unset"
}
return s
}
// portOfEndpoint is the port in host:port, or empty.
func portOfEndpoint(endpoint string) string {
if i := strings.LastIndex(endpoint, ":"); i >= 0 {
return endpoint[i+1:]
}
return ""
}
// SilentFor is how long a node may be quiet before the mesh says so.
//
// A node speaks every minute, so three of them missed is a gap rather than a slow one. The number
@@ -537,15 +369,6 @@ func onTheNetwork(ctx context.Context, inv *inventory.Inventory,
if err != nil {
return nil, err
}
// **With the seat holders on record**, or a machine running the next holder of a seat beside
// the current one resolves as two holders, is refused, and drops out of the map — taking the
// address every other machine composes for what it offers (novox/hq ADR 0131). Found live:
// the control node vanished from the private network the moment the new bus was assigned
// beside the old one.
holdings, err := inv.Holdings(ctx)
if err != nil {
return nil, err
}
var out []inventory.Overlay
for _, p := range places {
if p.Address == "" {
@@ -558,7 +381,7 @@ func onTheNetwork(ctx context.Context, inv *inventory.Inventory,
caps, _ := inv.ProfileOf(ctx, p.Name)
got, err := catalogue.Resolve(shelf, assigned,
catalogue.Node{Name: p.Name, Site: p.Site, Capabilities: caps},
catalogue.World{Unchecked: true, Holdings: holdings})
catalogue.World{Unchecked: true})
if err != nil {
continue
}
@@ -614,24 +437,6 @@ func namesInTheMesh(ctx context.Context, inv *inventory.Inventory,
for _, p := range places {
out[overlay.InternalName(p.Name)] = p.Address
}
// And the carried peers the operator has named (novox/hq issue 112): machines the
// predecessor's resolver answers for and the mesh routes to, known by name on the operator's
// word until they enrol — at which point enrolment verifies the name and the node's own
// entry takes over above. A name the predecessor answers for must keep resolving until the
// machine behind it is a node; without these, taking the resolver silences three machines.
carried, err := inv.CarriedPeers(ctx)
if err != nil {
return nil, err
}
for _, p := range carried {
if p.Named == "" || p.EnrolledAs != "" {
continue
}
if _, taken := out[overlay.InternalName(p.Named)]; taken {
continue // a node of the mesh owns the name; the stale statement loses
}
out[overlay.InternalName(p.Named)] = p.Address
}
return out, nil
}
@@ -689,18 +494,11 @@ func artifactStoreOnNetwork(ctx context.Context, inv *inventory.Inventory,
return "", "", false, nil
}
// artifactStoreAddress is the artifact store as `forNode` reaches it: `<node>.internal:<port>`
// over the private network, or — when nothing is on the network yet — `127.0.0.1:<port>` for the
// node that holds the store itself, and "" for any other. The address composed into every
// reference the mesh built, at the moment it is used and never before (novox/hq 04-ISSUES/102).
//
// **Genesis places the network after the store, the broker, the vault and the catalogue.** Each
// of those is built and pushed to a node that is on no network, and the store is on that same
// node; an answer of "no store" there would refuse every one of those pushes and hand every one
// of those builds a base nothing can pull. Loopback is the truth on that machine, and it is the
// address genesis itself reaches the store by.
// artifactStoreAddress is the artifact store as this network reaches it, `<node>.internal:<port>`,
// or "" when the mesh has none on its network yet — the address composed into every reference
// the mesh built, at the moment it is used and never before (novox/hq 04-ISSUES/102).
func artifactStoreAddress(ctx context.Context, inv *inventory.Inventory,
shelf map[string]catalogue.Manifest, forNode string) (string, error) {
shelf map[string]catalogue.Manifest) (string, error) {
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
if err != nil {
return "", err
@@ -710,29 +508,8 @@ func artifactStoreAddress(ctx context.Context, inv *inventory.Inventory,
on[name] = true
}
node, port, found, err := artifactStoreOnNetwork(ctx, inv, on)
if err != nil {
if err != nil || !found {
return "", err
}
if found {
return overlay.InternalName(node) + ":" + port, nil
}
holder, port, found, err := artifactStoreHolder(ctx, inv)
if err != nil || !found || holder != forNode {
return "", err
}
return "127.0.0.1:" + port, nil
}
// artifactStoreHolder is whichever node is assigned a module offering the artifact store, on or
// off the network, and the port that node put it on.
func artifactStoreHolder(ctx context.Context, inv *inventory.Inventory) (node, port string, found bool, err error) {
nodes, err := inv.Nodes(ctx)
if err != nil {
return "", "", false, err
}
all := map[string]bool{}
for _, n := range nodes {
all[n.Name] = true
}
return artifactStoreOnNetwork(ctx, inv, all)
return overlay.InternalName(node) + ":" + port, nil
}
-195
View File
@@ -2,11 +2,9 @@ package main
import (
"context"
"os"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/overlay"
)
@@ -110,196 +108,3 @@ func TestOnlyAMachineOnThePrivateNetworkIsNamed(t *testing.T) {
t.Fatalf("a machine that left the network is still named, or the one that stayed is not: %v", names)
}
}
// novox/hq ADR 0105: the range every address is composed from is the adopted tunnel's, read from
// the tunnel the hub holds — never stored anywhere else.
func TestTheOverlaysRangeIsTheAdoptedTunnels(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
t.Setenv(OverlayCIDRVar, "10.99.0.0/16")
before, err := overlayRange(ctx, inv)
if err != nil || before != "10.99.0.0/16" {
t.Fatalf("without an adopted tunnel the range is not what genesis said: %q %v", before, err)
}
// The hub becomes what genesis makes of a machine in use: adopted, enrolled with the found
// tunnel's key, and presenting the tunnel.
if err := inv.SetAdopted(ctx, "anchor", true); err != nil {
t.Fatal(err)
}
hub, err := inv.NodeByName(ctx, "anchor")
if err != nil {
t.Fatal(err)
}
const key = "THE-TUNNELS-KEY========================="
if err := inv.RecordOverlayKey(ctx, hub.ID, key); err != nil {
t.Fatal(err)
}
if err := inv.RecordTunnel(ctx, hub.ID, inventory.Tunnel{
Interface: "wg0", Unit: "wg-quick@wg0", Config: "/etc/wireguard/wg0.conf", Port: 51900,
Address: "192.0.2.1/24", Range: "192.0.2.0/24", PublicKey: key,
Peers: []inventory.TunnelPeer{{PublicKey: "PEER-TWO", Address: "192.0.2.2"}},
}); err != nil {
t.Fatal(err)
}
after, err := overlayRange(ctx, inv)
if err != nil || after != "192.0.2.0/24" {
t.Fatalf("with an adopted tunnel the range is %q (%v), not the tunnel's", after, err)
}
// A placement whose endpoint is on another port than the tunnel's is refused: the peers dial
// the tunnel's port.
err = overlayPlace(ctx, inv, []string{"anchor", "--endpoint", "198.51.100.10:51820", "--site", "hosting", "--hub"})
if err == nil || !strings.Contains(err.Error(), "51900") {
t.Fatalf("an endpoint off the tunnel's port was accepted: %v", err)
}
// On the tunnel's port, the hub is placed at the tunnel's address — whatever it had before.
if err := inv.SetPlace(ctx, "anchor", "", "", false, ""); err != nil {
t.Fatal(err)
}
if err := overlayPlace(ctx, inv, []string{"anchor", "--endpoint", "198.51.100.10:51900", "--site", "hosting", "--hub"}); err != nil {
t.Fatal(err)
}
if placed := placementOf(t, ctx, inv, "anchor"); placed.Address != "192.0.2.1" {
t.Fatalf("the hub was placed at %s, not the tunnel's own address", placed.Address)
}
// And the hub's declaration carries the peer and the takeover.
nodes, computed, err := graph(ctx, open)
if err != nil {
t.Fatal(err)
}
var hubNode overlay.Node
for _, n := range nodes {
if n.Name == "anchor" {
hubNode = n
}
}
if hubNode.TakesOver == nil || hubNode.TakesOver.Unit != "wg-quick@wg0" {
t.Errorf("the hub is not told to take over the found tunnel: %+v", hubNode)
}
carried := false
for _, p := range computed["anchor"] {
if p.Key == "PEER-TWO" && p.Allowed == "192.0.2.2/32" {
carried = true
}
}
if !carried {
t.Errorf("the hub's peer list does not carry the tunnel's peer: %+v", computed["anchor"])
}
}
// A takeover is composed only for a hub whose placement agrees with the tunnel: an address or an
// endpoint port that differs would have the host stop the found interface and raise the mesh's
// where no peer is listening.
func TestATakeoverIsNotComposedForAHubPlacedOffItsTunnel(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
if err := inv.SetAdopted(ctx, "anchor", true); err != nil {
t.Fatal(err)
}
hub, err := inv.NodeByName(ctx, "anchor")
if err != nil {
t.Fatal(err)
}
const key = "THE-TUNNELS-KEY========================="
if err := inv.RecordOverlayKey(ctx, hub.ID, key); err != nil {
t.Fatal(err)
}
if err := inv.RecordTunnel(ctx, hub.ID, inventory.Tunnel{
Interface: "wg0", Unit: "wg-quick@wg0", Config: "/etc/wireguard/wg0.conf", Port: 51900,
Address: "192.0.2.1/24", Range: "192.0.2.0/24", PublicKey: key}); err != nil {
t.Fatal(err)
}
// aMesh placed anchor at 10.77.0.1 on :51820 — the record of a hub placed before it took the
// tunnel over.
_, _, err = graph(ctx, open)
if err == nil {
t.Fatal("a takeover was composed for a hub whose address and port are not the tunnel's")
}
for _, want := range []string{"10.77.0.1", "192.0.2.1", "51820", "51900", "overlay place anchor"} {
if !strings.Contains(err.Error(), want) {
t.Errorf("the refusal does not say %q: %v", want, err)
}
}
// Re-placed on the tunnel, it composes.
if err := inv.SetPlace(ctx, "anchor", "", "", false, ""); err != nil {
t.Fatal(err)
}
if err := overlayPlace(ctx, inv, []string{"anchor", "--endpoint", "198.51.100.10:51900", "--site", "here", "--hub"}); err != nil {
t.Fatal(err)
}
if _, _, err := graph(ctx, open); err != nil {
t.Fatalf("re-placed on the tunnel, the graph still refuses: %v", err)
}
}
// theResolver is the catalogue's dnsmasq module as it is, or the test is skipped where the
// catalogue is not beside this checkout.
func theResolver(t *testing.T) catalogue.Manifest {
t.Helper()
raw, err := os.ReadFile("../../../mesh-catalog/modules/dnsmasq/module.json")
if err != nil {
t.Skipf("the catalogue is not beside this checkout: %v", err)
}
m, err := catalogue.ParseManifest(raw)
if err != nil {
t.Fatalf("dnsmasq does not parse:\n%v", err)
}
return m
}
// The resolver is handed every machine on the private network as a wildcard, the same set and the
// same source as the hosts file, and is handed it again when a machine leaves — through the
// module's own manifest asking for the fact, with no module of the mesh's own in between (hal
// dnsmasq-app conversion, novox/hq 08-connectivity). The runtime on that machine is pointed at the
// machine's own address, where the resolver answers for its containers.
func TestTheResolverIsToldEveryMachineOnTheNetworkAndToldAgainWhenOneLeaves(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, theResolver(t))
if _, err := assign(ctx, open, "anchor", "dnsmasq"); err != nil {
t.Fatal(err)
}
zones := func() string {
t.Helper()
for _, r := range composed(t, open, "anchor").Resources {
if r["id"] == "dnsmasq.fact-node-zones" {
if r["path"] != "/etc/mesh-resolver/nodes.conf" {
t.Fatalf("the machines were written somewhere the resolver does not read: %v", r["path"])
}
return r["content"].(string)
}
}
t.Fatal("the resolver was not handed the machines")
return ""
}
first := zones()
for _, want := range []string{
"local=/internal/", "address=/anchor.internal/10.77.0.1\n", "address=/laptop.internal/10.77.0.2\n",
} {
if !strings.Contains(first, want) {
t.Errorf("the resolver's machines lack %q:\n%s", want, first)
}
}
for _, r := range composed(t, open, "anchor").Resources {
if r["id"] == "dnsmasq.runtime-dns" {
if !strings.Contains(r["content"].(string), `"10.77.0.1"`) || r["into"] != "json" {
t.Errorf("the runtime is not pointed at this machine's own address, written into its file: %v", r)
}
}
}
// The laptop keeps its place and its address, and stops running the network.
if err := open.inventory.Unassign(ctx, "laptop", overlay.Name); err != nil {
t.Fatal(err)
}
after := zones()
if strings.Contains(after, "laptop") || !strings.Contains(after, "address=/anchor.internal/10.77.0.1\n") {
t.Fatalf("a machine that left the network is still a wildcard, or the one that stayed is not:\n%s", after)
}
}
+11 -40
View File
@@ -10,6 +10,7 @@ import (
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/token"
)
@@ -66,14 +67,8 @@ func nodeCommand(ctx context.Context, args []string) error {
// because the damage is already done by the time it prints.
return publicDomain(ctx, inv, args[1:])
case "account":
// The operator's login on this machine (novox/hq to-be 29): what a home-scoped file is
// owned by and which account `ssh <node>` uses. Reports with no argument; sets with one;
// an optional second argument is the home when it is not /home/<account>.
return nodeAccount(ctx, inv, args[1:])
default:
return fmt.Errorf("node has no %q; it has add, list, show, public-domain and account", args[0])
return fmt.Errorf("node has no %q; it has add, list, show and public-domain", args[0])
}
}
@@ -111,39 +106,6 @@ func modeOf(n inventory.Node) string {
}
// publicDomainUsage is the one description of the three forms, so a refusal and the help agree.
// nodeAccount reports or sets a node's operator account (novox/hq to-be 29). Read-shaped with no
// argument, like public-domain: `node account novox` answers, it does not change anything.
func nodeAccount(ctx context.Context, inv *inventory.Inventory, positionals []string) error {
if len(positionals) == 0 || len(positionals) > 3 {
return errors.New("node account <name> — what it is now; " +
"node account <name> <account> [home] — set it (home defaults to /home/<account>)")
}
node := positionals[0]
if len(positionals) == 1 {
who, err := inv.NodeByName(ctx, node)
if err != nil {
return err
}
if who.Account == "" {
fmt.Printf("%s has no operator account known\n", node)
fmt.Printf(" `node account %s <account>` sets it\n", node)
return nil
}
fmt.Printf("%s logs a person in as %s (home %s)\n", node, who.Account, who.Home())
return nil
}
home := ""
if len(positionals) == 3 {
home = positionals[2]
}
if err := inv.SetAccount(ctx, node, positionals[1], home); err != nil {
return err
}
fmt.Printf("%s logs a person in as %s\n", node, positionals[1])
fmt.Printf(" run `push %s` once ssh-client is assigned, to send its operator config\n", node)
return nil
}
const publicDomainUsage = "node public-domain <name> — what it is now; " +
"<name> <domain> to set it; <name> --clear to take it away"
@@ -257,6 +219,15 @@ func tokenCommand(ctx context.Context, args []string) error {
// chicken-and-egg entirely: the mesh runs the broker, so a joining node's credentials can
// exist before it does. The one-time secret IS the password, so a node's first connection is
// already authenticated and enrolment is what happens over it.
if management, err := broker.ManagementFromEnvironment(); err == nil {
if err := management.CreateNodeAccount(ctx, issued.Node.Name, issued.Secret); err != nil {
return err
}
fmt.Printf("broker account %s created, scoped to %s and the %s exchange\n\n",
issued.Node.Name, link.QueueFor(issued.Node.Name), link.Exchange)
} else if !errors.Is(err, broker.ErrNotConfigured) {
return err
}
made := token.Token{Node: issued.Node.Name, Signer: key.Public, Secret: issued.Secret,
Adopted: issued.Node.Adopted}
+1 -146
View File
@@ -2,15 +2,12 @@ package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"os"
"strings"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/secrets"
)
@@ -29,25 +26,9 @@ import (
// operator key make [--out <file>] make a keypair: private half to the file, public half printed
// operator key set <public> tell the mesh which key to seal to
// operator key show the public key, its fingerprint, and what it can recover
const operatorUsage = "operator key make [--out <file>] | operator key set <public> [--replace] | " +
"operator key show | operator issue <name> --invokes <tool,tool|*> | operator revoke <name> | " +
"operator list"
const operatorUsage = "operator key make [--out <file>] | operator key set <public> [--replace] | operator key show"
func operatorCommand(ctx context.Context, args []string) error {
if len(args) == 0 {
return errors.New(operatorUsage)
}
// The people who may reach the mesh's tools (design 25 §7). Beside the operator's key because
// both answer "who, other than a machine, may do something here" — and a person reading this
// command's usage is asking exactly that.
switch args[0] {
case "issue":
return personIssue(ctx, args[1:])
case "revoke":
return personRevoke(ctx, args[1:])
case "list":
return personList(ctx)
}
if len(args) < 2 || args[0] != "key" {
return errors.New(operatorUsage)
}
@@ -179,129 +160,3 @@ func readPrivateKey(path string) (string, error) {
}
return strings.TrimSpace(string(raw)), nil
}
// personIssue gives somebody a credential for the mesh's tools, and prints it once.
//
// **Printed, not stored.** The mesh keeps a hash and nothing else, so this is the only moment the
// credential exists anywhere but on the workstation that will use it — the same contract a token has,
// and for the same reason: a credential recoverable from the mesh's store has the store's blast
// radius.
func personIssue(ctx context.Context, args []string) error {
set := flag.NewFlagSet("operator issue", flag.ContinueOnError)
invokes := set.String("invokes", "", "the tools this person may call, comma-separated, or * for every one")
// Flags on either side of the name, because the usage this command prints puts them after it —
// and the standard parser stops at the first thing that is not a flag, so the order the command
// documents was the one order it refused (2026-09-28).
positionals, err := parseAround(set, args)
if err != nil {
return err
}
if len(positionals) != 1 {
return errors.New("operator issue <name> --invokes <tool,tool|*>")
}
name := positionals[0]
if *invokes == "" {
return errors.New(
"say what this person may call: --invokes mesh-catalog.catalog_tools,gitea.repo_create, " +
"or --invokes '*' for an administrator")
}
var tools []string
for _, t := range strings.Split(*invokes, ",") {
if t = strings.TrimSpace(t); t != "" {
tools = append(tools, t)
}
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
if err := inv.RecordPerson(ctx, inventory.Person{Name: name, Invokes: tools}); err != nil {
return err
}
// Refused here rather than at the next composition, where it would stop the whole file being
// written for everybody. A name that cannot be part of a subject is one the server would read as
// a wider permission than anybody granted.
if _, err := broker.PermissionsFor(broker.Principal{
Kind: broker.KindPerson, Module: name, Invokes: tools, PasswordHash: "x",
}); err != nil {
return err
}
user := broker.Principal{Kind: broker.KindPerson, Module: name}.Username()
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: user, Kind: inventory.BusPerson})
if err != nil {
return err
}
where, err := broker.FromEnvironment()
if err != nil && !errors.Is(err, broker.ErrNotConfigured) {
return err
}
held, err := json.Marshal(struct {
URL string `json:"url"`
Fingerprint string `json:"fingerprint,omitempty"`
User string `json:"user"`
Password string `json:"password"`
Person string `json:"person"`
Invokes []string `json:"invokes"`
}{
URL: "nats://" + where.Address, Fingerprint: where.Fingerprint,
User: user, Password: password, Person: name, Invokes: tools,
})
if err != nil {
return err
}
fmt.Printf("issued %s, who may call %s\n", name, strings.Join(tools, ", "))
fmt.Println(" this is the only time the credential is printed; the mesh keeps a hash")
fmt.Println(" it works once the bus has been told, which is the next push to the machine holding mesh-broker")
fmt.Println()
fmt.Println(string(held))
return nil
}
func personRevoke(ctx context.Context, args []string) error {
if len(args) != 1 {
return errors.New("operator revoke <name>")
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
if err := open.inventory.ForgetPerson(ctx, args[0]); err != nil {
return err
}
// **Revoked at the next composition, not now.** The bus's users are a file, so a credential stops
// working when the file no longer names it. Said plainly, because "revoked" that still works for
// another minute is worth knowing about.
fmt.Printf("%s is forgotten, and their credential stops working at the next composition — "+
"push the machine holding mesh-broker to make it so\n", args[0])
return nil
}
func personList(ctx context.Context) error {
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
people, err := open.inventory.People(ctx)
if err != nil {
return err
}
if len(people) == 0 {
fmt.Println("nobody but machines reaches this mesh")
return nil
}
for _, p := range people {
fmt.Printf("%-20s %s\n", p.Name, strings.Join(p.Invokes, ", "))
}
return nil
}
-212
View File
@@ -1,212 +0,0 @@
package main
import (
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// entry is a module as the catalogue holds it: built, so its manifest carries no `build` any more.
func entry(module string, _ ...string) inventory.Entry {
return inventory.Entry{Manifest: catalogue.Manifest{Module: module}}
}
// stoodOn is what each module's newest build recorded it was handed.
func stoodOn(edges map[string][]string) map[string][]string {
out := map[string][]string{}
for module, bases := range edges {
for _, b := range bases {
out[module] = append(out[module], catalogue.ArtifactStoreScheme+b+"/runtime@sha256:"+strings.Repeat("0", 64))
}
}
return out
}
// A module built before the module it stands on is built against the old one and reports success
// (novox/hq 04-ISSUES/131). So bases come first, however the set arrived.
func TestBasesAreBuiltBeforeWhatStandsOnThem(t *testing.T) {
in := []inventory.Entry{entry("app"), entry("runtime"), entry("other"), entry("base")}
edges := stoodOn(map[string][]string{"app": {"runtime"}, "runtime": {"base"}})
got := orderByBases(in, edges)
pos := map[string]int{}
for i, e := range got {
pos[e.Manifest.Module] = i
}
if !(pos["base"] < pos["runtime"] && pos["runtime"] < pos["app"]) {
t.Fatalf("bases not first: %v", pos)
}
if len(got) != 4 {
t.Fatalf("an entry was lost or doubled: %d", len(got))
}
// A base outside the set is not waited for: it is not being rebuilt.
got = orderByBases([]inventory.Entry{entry("app")}, stoodOn(map[string][]string{"app": {"elsewhere"}}))
if len(got) != 1 {
t.Fatalf("a dependency outside the set changed the set: %v", got)
}
}
// A module registered from its manifest and never built still names its bases there; once built,
// the recorded edge is what says so. Both are read, and a module never stands on itself.
func TestWhatStandsOnAModuleIsReadFromItsBuildOrItsManifest(t *testing.T) {
built := entry("gitea")
edges := stoodOn(map[string][]string{"gitea": {"mesh-tools"}})
if !standsOnModule(built, "mesh-tools", edges) {
t.Fatal("a recorded edge was not read")
}
if standsOnModule(built, "gitea", edges) || standsOnModule(built, "postgres", edges) {
t.Fatal("an edge was invented")
}
fresh := inventory.Entry{Manifest: catalogue.Manifest{Module: "plex", Build: &catalogue.Build{
On: []catalogue.BuildsOn{{Arg: "RUNTIME_BASE", Module: "mesh-tools", Artifact: "runtime"}},
}}}
if !standsOnModule(fresh, "mesh-tools", nil) {
t.Fatal("a manifest's own base was not read")
}
}
// A merge names a repository the way the forge does; a source is recorded the way a build was
// asked for. The two meet on owner/repo and branch, whichever form the record took.
func TestAMergeMatchesTheSourcesBuiltFromIt(t *testing.T) {
m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main",
CloneURL: "http://forge.internal:20000/novox/mesh-controller.git"}
for _, s := range []inventory.Source{
{Repository: "http://forge.internal:20000/novox/mesh-controller.git", Ref: "main"},
{Repository: "novox/mesh-controller", Seat: "git", Ref: ""},
{Repository: "https://elsewhere.example/novox/mesh-controller", Ref: "main"},
} {
if !sourceIs(s, m) {
t.Errorf("%+v was not matched by the merge", s)
}
}
for _, s := range []inventory.Source{
{Repository: "novox/mesh-host", Seat: "git"},
{Repository: "http://forge.internal:20000/novox/mesh-controller.git", Ref: "release"},
} {
if sourceIs(s, m) {
t.Errorf("%+v was matched by a merge that is not its", s)
}
}
}
// A merge made before the source was last seen is history: it does not move the source, and a
// merge that says nothing about when it was made is taken as news.
func TestAMergeOlderThanTheLastLookIsHistory(t *testing.T) {
seen := time.Date(2026, 9, 28, 3, 0, 0, 0, time.UTC)
if !isHistory("2026-09-28T02:00:00Z", seen) {
t.Fatal("an older merge was taken as news")
}
if isHistory("2026-09-28T04:00:00Z", seen) {
t.Fatal("a newer merge was taken as history")
}
if isHistory("", seen) || isHistory("2026-09-28T02:00:00Z", time.Time{}) {
t.Fatal("a merge or a source with no time on it was refused")
}
}
// A module as the catalogue holds it: built from a repository, at a directory inside it.
func fromRepo(module, repository, path string) inventory.Entry {
return inventory.Entry{
Manifest: catalogue.Manifest{Module: module},
Source: inventory.Source{Repository: repository, Path: path, Ref: "main"},
}
}
// A merge rebuilds the modules whose own directories it changed, and everything when what it changed
// is shared. One repository holding many modules is the ordinary case here, and rebuilding all of
// them for a change to one is what exhausted a registry's pull limit the first night this ran.
func TestAMergeRebuildsTheModulesItChanged(t *testing.T) {
const repo = "http://forge.internal:20000/novox/mesh-catalog.git"
gitea := fromRepo("gitea", repo, "modules/gitea")
keycloak := fromRepo("keycloak", repo, "modules/keycloak")
known := []inventory.Entry{gitea, keycloak, fromRepo("plex", repo, "modules/plex")}
candidates := []inventory.Entry{gitea, keycloak}
merge := func(paths []string, truncated bool) link.SourceMoved {
return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main",
Paths: paths, PathsTruncated: truncated}
}
named := func(entries []inventory.Entry) string {
var names []string
for _, e := range entries {
names = append(names, e.Manifest.Module)
}
return strings.Join(names, ",")
}
for _, c := range []struct {
what string
m link.SourceMoved
want string
}{
{"one module's own files", merge([]string{"modules/gitea/index.ts", "modules/gitea/client.ts"}, false), "gitea"},
{"two modules' files", merge([]string{"modules/gitea/index.ts", "modules/keycloak/module.json"}, false), "gitea,keycloak"},
{"a file they share", merge([]string{"tsconfig.json"}, false), "gitea,keycloak"},
{"a module the mesh does not hold", merge([]string{"modules/plex/index.ts"}, false), ""},
{"nothing said about the files", merge(nil, false), "gitea,keycloak"},
{"more files than were listed", merge([]string{"modules/gitea/index.ts"}, true), "gitea,keycloak"},
} {
if got := named(whatTheMergeTouched(candidates, known, c.m)); got != c.want {
t.Errorf("%s: rebuilt %q, wanted %q", c.what, got, c.want)
}
}
}
// A module whose recipe packages source from another repository is affected when that repository
// moves — the manifest the mesh keeps says nothing about it, so the record of what the build read is
// the only thing that can say so.
func TestAModuleIsAffectedByTheRepositoryItPackages(t *testing.T) {
m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main",
CloneURL: "http://forge.internal:20000/novox/mesh-controller.git"}
for _, read := range [][]inventory.ReadRepository{
{{Repository: "http://forge.internal:20000/novox/mesh-controller.git", Ref: "main"}},
{{Repository: "novox/mesh-controller"}},
{{Repository: "https://elsewhere.example/novox/other"}, {Repository: "novox/mesh-controller.git", Ref: "main"}},
} {
if !readsFrom(read, m) {
t.Errorf("%+v was not matched by the merge", read)
}
}
for _, read := range [][]inventory.ReadRepository{
nil,
{{Repository: "novox/mesh-host", Ref: "main"}},
{{Repository: "novox/mesh-controller", Ref: "release"}},
} {
if readsFrom(read, m) {
t.Errorf("%+v was matched by a merge that is not its", read)
}
}
}
// What a module handed over by hand records about where it came from, and what is refused.
func TestWhatAHandedOverModuleRecordsAboutItsSource(t *testing.T) {
// The whole location: a repository on the mesh's own forge, the directory inside it, the branch
// and the commit the manifest was read at.
from, err := whereItComesFrom("novox/mesh-catalog", "main", "c0ffee", "modules/gitea", true)
if err != nil {
t.Fatal(err)
}
if from.Path != "modules/gitea" || from.Seat != "git" || from.Repository != "novox/mesh-catalog" {
t.Fatalf("the source records as %+v", from)
}
// A manifest with no provenance at all is legitimate: fixing something in a hurry.
if from, err := whereItComesFrom("", "", "", "", false); err != nil || from != (inventory.Source{}) {
t.Fatalf("a manifest handed over with no provenance was refused: %+v, %v", from, err)
}
for _, c := range []struct {
what string
repository, ref, commit, path string
self bool
}{
{what: "a source with no commit", repository: "novox/mesh-catalog", commit: ""},
{what: "a commit with no source", commit: "c0ffee"},
{what: "a directory inside nothing", path: "modules/gitea"},
{what: "a forge holding nothing", self: true},
{what: "an address given as a path on the forge", repository: "http://forge.internal:20000/novox/x.git", commit: "c0ffee", self: true},
} {
if _, err := whereItComesFrom(c.repository, c.ref, c.commit, c.path, c.self); err == nil {
t.Errorf("%s was recorded as a source", c.what)
}
}
}
+34 -227
View File
@@ -83,17 +83,9 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
return catalogue.Resolution{}, nil, err
}
// The operator account this node logs a person in as, and where its home is (novox/hq to-be
// 29) — carried so a home-scoped file's owner and path resolve for this machine.
who, err := inv.NodeByName(ctx, nodeName)
if err != nil {
return catalogue.Resolution{}, nil, err
}
resolved, err := catalogue.Resolve(shelf, assigned,
catalogue.Node{Name: nodeName, Site: site, Capabilities: capabilities,
At: onNetwork[nodeName], PublicDomain: publicDomain,
Account: who.Account, AccountHome: who.AccountHome}, world)
At: onNetwork[nodeName], PublicDomain: publicDomain}, world)
if err != nil {
return catalogue.Resolution{}, nil, err
}
@@ -185,15 +177,6 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
// Every node, not only the placed ones. A machine that was never put on the private network
// still runs modules, still holds claims, and still offers whatever it offers.
// **Who holds each seat on record, before anything is resolved** (novox/hq ADR 0131). Both
// passes below need it: without it, the assignment standing beside a seat's holder — the next
// holder, waiting for the handover — is refused as a second holder, and its node's whole set
// with it.
holdings, err := inv.Holdings(ctx)
if err != nil {
return catalogue.World{}, err
}
nodes, err := inv.Nodes(ctx)
if err != nil {
return catalogue.World{}, err
@@ -234,15 +217,13 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
}
offered := map[string][]catalogue.Provider{}
var firstHeld []catalogue.Held
for _, o := range others {
got, err := catalogue.Resolve(shelf, o.assigned, o.node, catalogue.World{Unchecked: true, Holdings: holdings})
got, err := catalogue.Resolve(shelf, o.assigned, o.node, catalogue.World{Unchecked: true})
if err != nil {
// Their set does not resolve for some other reason. Not this node's problem to
// report, and nothing of theirs is running, so it offers nothing.
continue
}
firstHeld = append(firstHeld, got.Claims...)
for _, m := range got.Modules {
for _, name := range m.OffersAt(catalogue.ScopeMesh) {
// What that module says a consumer needs to know, with that node's settings on
@@ -253,7 +234,7 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
return catalogue.World{}, err
}
offered[name] = append(offered[name], catalogue.Provider{
Node: o.node.Name, At: o.node.At, Serves: serves, Module: m.Module})
Node: o.node.Name, At: o.node.At, Serves: serves})
}
}
}
@@ -263,20 +244,14 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
})
}
// **The second pass is given the first pass's holdings.** A seat's holder answers a requirement
// with several providers (novox/hq ADR 0110), so a node consuming one resolves only once the
// holder is known. Without them its set is refused here, and a refused node's own claims drop
// out of what the mesh holds — so a second holder of one of its seats would pass unrefused.
world := catalogue.World{Offered: offered, Held: firstHeld, Holdings: holdings}
var held []catalogue.Held
world := catalogue.World{Offered: offered}
for _, o := range others {
got, err := catalogue.Resolve(shelf, o.assigned, o.node, world)
if err != nil {
continue
}
held = append(held, got.Claims...)
world.Held = append(world.Held, got.Claims...)
}
world.Held = held
return world, nil
}
@@ -505,18 +480,18 @@ func renderingFor(ctx context.Context, open *stores, node string,
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
// The private network's range, offered to a module as ${machine:mesh-range} — a module that must
// name the whole mesh (an intrusion filter that must never ban a tunnel peer) names it here
// rather than hardcoding a value it cannot know.
meshRange, err := overlayRange(ctx, inv)
// Where this node put the foundation's servers, for the control plane's own connections
// (novox/hq 04-ISSUES/102): read from the node's settings for whatever claims each seat,
// exactly as a consumer's binding is, never from what genesis wrote into a secret.
seats, err := seatsOn(ctx, inv, shelf, node, plan.Modules)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
// The artifact store as this node reaches it now — the address every image and archive the
// mesh built is fetched through, composed here and recorded nowhere — with what the mesh has
// built, so a reference recorded with an address before that is re-routed too.
artifactStore, err := artifactStoreAddress(ctx, inv, shelf, node)
// And the artifact store as this network reaches it now — the address every image and
// archive the mesh built is fetched through, composed here and recorded nowhere — with what
// the mesh has built, so a reference recorded with an address before that is re-routed too.
artifactStore, err := artifactStoreAddress(ctx, inv, shelf)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
@@ -537,29 +512,10 @@ func renderingFor(ctx context.Context, open *stores, node string,
return catalogue.Rendering{}, inventory.Node{}, err
}
// Each machine's operator account, so an ssh Host block can name the login for every node
// (novox/hq to-be 29). Keyed by the bare node name, which entriesFrom falls back to.
allNodes, err := inv.Nodes(ctx)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
accounts := map[string]string{}
for _, n := range allNodes {
if n.Account != "" {
accounts[n.Name] = n.Account
}
}
// And every routed name → the node that serves it (novox/hq ADR 0066). Alongside the
// `<node>.internal` names above, so a container — or an internal ACME validator — resolves a
// routed name to the proxy that serves it, mesh-wide. The mesh publishes the names it was told
// to serve and knows nothing about what they mean.
// Kept apart from the machines, because a fact about the machines must not be handed the names
// the mesh merely serves (novox/hq 04-ISSUES/111).
machines := make(map[string]string, len(names))
for name, at := range names {
machines[name] = at
}
routes, err := routeNamesInTheMesh(ctx, open)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
@@ -571,19 +527,11 @@ func renderingFor(ctx context.Context, open *stores, node string,
// The ports the mesh itself needs open, which no module declares. Read from the broker this
// control plane was told about rather than written down twice: the address a node is handed in
// its token and the port its machine must accept on are the same fact.
//
// **Only on the node that listens on it** (novox/hq issue: the broker opening leaked onto
// every node). The opening exists to WIDEN the broker's port to from-anywhere — a machine
// enrolling is not on the mesh yet, so the broker's own `from: mesh` listen would refuse its
// first dial. That widening belongs on the broker's host and nowhere else: a node that only
// dials out needs no incoming rule, and an opening for a port nothing here listens on is a
// from-anywhere hole for a dead port. So the foundation port is kept only when a module
// resolved onto THIS node actually listens on it.
var foundation []int
if b, err := broker.FromEnvironment(); err == nil {
if _, port, err := net.SplitHostPort(b.Address); err == nil {
if n, err := strconv.Atoi(port); err == nil {
foundation = foundationPortsFor(n, plan.Modules)
foundation = append(foundation, n)
}
}
}
@@ -623,32 +571,11 @@ func renderingFor(ctx context.Context, open *stores, node string,
taken[m] = true
}
}
// Where this node put the foundation's servers, for the control plane's own connections
// (novox/hq 04-ISSUES/102): read from the node's settings for whatever claims each seat,
// exactly as a consumer's binding is, never from what genesis wrote into a secret.
seats, err := seatsOn(ctx, inv, shelf, node, plan.Modules, record.Adopted, taken)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
// The bus's user list, for the machine that runs the bus. Composed per push rather than kept,
// because it is a function of the mesh's records and a kept copy could disagree with them.
busUsers, err := composeBusUsers(ctx, inv, plan.Modules)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
memberships, err := inv.BusMemberships(ctx)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
return catalogue.Rendering{
BusMembership: memberships[node],
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
Machines: machines,
Suffix: overlay.Suffix(), MeshRange: meshRange, Accounts: accounts, Foundation: foundation,
Kept: kept, Adopted: record.Adopted,
Suffix: overlay.Suffix(), Foundation: foundation, Kept: kept, Adopted: record.Adopted,
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, Built: built,
BusUsers: busUsers,
}, record, nil
}
@@ -866,22 +793,6 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
return out, nil
}
// listensLines is what a person is told about what this module would open, and why — the same
// `why` every listens entry already carries for the firewall it also feeds (novox/hq ADR 0007), so
// deciding whether to assign a module can see what it would open before it opens it, not only
// after. A module with nothing to listen on prints nothing extra, same as today.
func listensLines(m catalogue.Manifest) []string {
var out []string
for _, l := range m.Listens {
if l.Why == "" {
out = append(out, fmt.Sprintf(" listens %d/%s from %s", l.Port, l.At(), l.From))
continue
}
out = append(out, fmt.Sprintf(" listens %d/%s from %s — %s", l.Port, l.At(), l.From, l.Why))
}
return out
}
func planCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("plan", flag.ContinueOnError)
// Because "one resource" does not tell you whether the settings landed. Being able to read
@@ -935,9 +846,6 @@ func planCommand(ctx context.Context, args []string) error {
fmt.Printf("%s would run:\n", args[0])
for _, m := range plan.Modules {
fmt.Printf(" %-20s %s\n", m.Module, plan.Because[m.Module])
for _, line := range listensLines(m) {
fmt.Println(line)
}
}
// What was assigned here and cannot run here. Said with the rest rather than as a refusal: it is
// one module on the wrong machine, the others still run, and the remedy is to move this one.
@@ -973,26 +881,12 @@ func planCommand(ctx context.Context, args []string) error {
if !ok {
continue
}
fmt.Printf("\n--- %s ---\n%s", shownAs(r), content)
fmt.Printf("\n--- %v %v ---\n%s", r["id"], r["path"], content)
}
}
return nil
}
// shownAs is the heading `plan --show` puts over a resource's content.
//
// **A file written into says so.** Its content is the mesh's part of a file that is otherwise the
// machine's — the keys of a JSON document (novox/hq ADR 0102), the region of a hosts file (issue
// 128). Shown under a bare path it reads as the whole file, and a person checking what a take
// replaces would see a hosts file of a dozen lines where the machine keeps thirty.
func shownAs(r map[string]any) string {
heading := fmt.Sprintf("%v %v", r["id"], r["path"])
if into, ok := r["into"].(string); ok && into != "" {
heading += fmt.Sprintf(" (written into, %s)", into)
}
return heading
}
// licencesFor is what this node can be answered with by record, and what it was put on.
//
// A mesh with no licences at all is the ordinary case and must not be an error: every existing
@@ -1129,7 +1023,7 @@ func portsGivenOn(ctx context.Context, inv *inventory.Inventory, node string,
// and not yet assigned (04-ISSUES/085), and the control plane must follow that setting from the
// first declaration it composes for itself. A holder in this node's set wins over one that is not.
func seatsOn(ctx context.Context, inv *inventory.Inventory, shelf map[string]catalogue.Manifest,
node string, inSet []catalogue.Manifest, adopted bool, taken map[string]bool) (map[string]map[int]int, error) {
node string, inSet []catalogue.Manifest) (map[string]map[int]int, error) {
assigned := map[string]bool{}
for _, m := range inSet {
assigned[m.Module] = true
@@ -1151,26 +1045,26 @@ func seatsOn(ctx context.Context, inv *inventory.Inventory, shelf map[string]cat
if len(claims) == 0 {
continue
}
// **Only a port the node was given or the mesh assigned — never the manifest's own
// number.** The sealed value the answer sits beside carries the port genesis wrote, which
// on a given-port node is the predecessor's; a manifest's long-form mapping is the
// catalogue's default, and answering with it would override the right number with one
// the mesh never checked (the contract in seat_into.go). And on an adopted node a holder
// assigned but not yet taken is the found container, on the ports it was found with, not
// the declaration's — so its mesh-assigned ports do not count there either; a given port
// does, because a given port is the found one by construction (ADR 0100).
ports, _, err := portsGivenOn(ctx, inv, node, m)
if err != nil {
return nil, err
}
if adopted && !taken[name] {
layers, err := inv.SettingsFor(ctx, node, m.Module)
if err != nil {
return nil, err
if assigned[name] {
// Running here, so what its manifest publishes the long way is where this machine
// has it — the same reading the declaration itself makes (ADR 0038). Only for a
// holder in this node's set: a manifest's number says nothing about a machine the
// module does not run on.
var declared []int
for _, l := range m.Listens {
declared = append(declared, l.Port)
}
ports = map[int]int{}
if given, err := catalogue.GivenPorts(m, layers); err == nil {
ports = given
for _, wanted := range append(declared, m.Guards...) {
if _, said := ports[wanted]; said {
continue
}
if at, mayAssign := m.MachineSide(wanted); !mayAssign && at != 0 {
ports[wanted] = at
}
}
}
if len(ports) == 0 {
@@ -1207,90 +1101,3 @@ func portsOn(
}
return out, nil
}
// composeBusUsers is the bus's user list, for a push to the machine that runs the bus.
//
// Empty for every other machine, and for every machine while the mesh is on the bus it runs on
// today — where accounts are a management call and there is no file to write.
//
// **Composed on each push, never kept.** The list is a function of the mesh's records (who exists,
// what runs where, what each declares), and a stored copy would be a second account of who may reach
// the bus, able to disagree with the records while both looked internally consistent (ADR 0043).
//
// A user the mesh has never minted a password for is **left out and said**, not written as a user
// without one — the composer refuses that, because a user with no password is a user anybody is. That
// is an ordinary situation with an obvious remedy (`module issue`, or enrolling), so the push carries
// the rest rather than failing: a bus that is missing one module's user is a mesh where that module
// cannot connect, and a bus with no file at all is a mesh where nothing can.
func composeBusUsers(ctx context.Context, inv *inventory.Inventory,
onThisNode []catalogue.Manifest) (string, error) {
// **Not gated on which bus the controller is on, and that was a bug.** It read "compose this only
// once the mesh is on the new bus" — which cannot work, because the server needs its user list
// *before* anything moves onto it. Step 2 of the change is exactly that: the server stands in the
// mesh carrying nothing, on its own ports, while every node is still on the old bus (novox/hq
// ADR 0116). Under the old gating that step could not happen: the module would come up, find no
// accounts file, and its entrypoint would wait for one the controller had decided not to write.
//
// So the question is only whether this machine runs the module that asked for the file. A mesh
// that never moves has written a user list nothing reads, which costs a few hundred bytes on one
// node; the reverse cost a step that cannot be taken.
//
// Asked of what this push resolves to rather than of the seat's holder mesh-wide: the file is a
// resource of that module, so the question is whether it is here.
holdsTheBus := false
for _, m := range onThisNode {
if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") {
holdsTheBus = true
}
}
if !holdsTheBus {
return "", nil
}
records, err := inv.BusRecords(ctx)
if err != nil {
return "", err
}
users, err := broker.Users(records)
if err != nil {
return "", err
}
kept, err := inv.BusUsers(ctx)
if err != nil {
return "", err
}
hashes := make(map[string]string, len(kept))
for name, u := range kept {
hashes[name] = u.PasswordHash
}
filled, missing := broker.WithPasswords(users, hashes)
if len(missing) > 0 {
fmt.Printf("the bus's user list leaves out %d user(s) the mesh has minted no credential "+
"for: %s. Each is a user that cannot connect until one is issued\n",
len(missing), strings.Join(missing, ", "))
}
if len(filled) == 0 {
return "", fmt.Errorf(
"this machine runs the bus and not one user has a credential, so the composed list " +
"would refuse every connection in the mesh")
}
return broker.ComposeAccounts(filled)
}
// foundationPortsFor is the broker port, kept only when a module resolved onto this node listens
// on it (novox/hq issue: the broker opening leaked onto every node). The foundation opening
// exists to WIDEN the broker's `from: mesh` port to from-anywhere, because a machine enrolling is
// not on the mesh yet and its first dial would be refused. That widening belongs on the broker's
// host alone: a node that only dials out needs no incoming rule, and an opening for a port
// nothing here listens on is a from-anywhere hole for a dead port.
func foundationPortsFor(brokerPort int, modules []catalogue.Manifest) []int {
for _, m := range modules {
for _, l := range m.Listens {
if l.Port == brokerPort {
return []int{brokerPort}
}
}
}
return nil
}
-51
View File
@@ -1,51 +0,0 @@
package main
import (
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// `plan` tells a person what a module would open and why, from the same `why` every listens
// entry already carries for the firewall (novox/hq ADR 0007) — so deciding whether to assign a
// module does not need reading its manifest first.
func TestListensLinesShowWhatAModuleWouldOpenAndWhy(t *testing.T) {
m := catalogue.Manifest{Module: "minio", Listens: []catalogue.Listening{
{Port: 9000, From: catalogue.FromMesh, Why: "the S3 endpoint"},
{Port: 9001, From: catalogue.FromMesh},
}}
got := listensLines(m)
if len(got) != 2 {
t.Fatalf("two listens entries, got %d: %v", len(got), got)
}
if !strings.Contains(got[0], "9000/tcp") || !strings.Contains(got[0], "the S3 endpoint") {
t.Errorf("the port and its why did not both appear: %q", got[0])
}
if strings.Contains(got[1], "—") {
t.Errorf("a listens entry with no why should not print a dash: %q", got[1])
}
if !strings.Contains(got[1], "9001/tcp") {
t.Errorf("the port still appears without a why: %q", got[1])
}
}
func TestListensLinesAreEmptyForAModuleWithNothingToListenOn(t *testing.T) {
if got := listensLines(catalogue.Manifest{Module: "board"}); len(got) != 0 {
t.Errorf("a module with no listens should print nothing, got %v", got)
}
}
// `plan --show` says when a file is written into rather than over (novox/hq issue 128), or the
// mesh's region of a hosts file reads as the whole file.
func TestAFileWrittenIntoIsShownAsSuch(t *testing.T) {
region := shownAs(map[string]any{
"id": "mesh-wireguard.fact-node-names", "path": "/etc/hosts", "into": "block"})
if region != "mesh-wireguard.fact-node-names /etc/hosts (written into, block)" {
t.Errorf("the region is shown as %q", region)
}
whole := shownAs(map[string]any{"id": "dnsmasq.fact-node-zones", "path": "/etc/mesh-resolver/nodes.conf"})
if strings.Contains(whole, "written into") {
t.Errorf("a whole file is shown as written into: %q", whole)
}
}
+16 -167
View File
@@ -38,27 +38,6 @@ func reportUnhostable(node string, plan catalogue.Resolution) {
// nothing in it was wrong, and no one edit was the one that should have been a new file.
// serve is the control plane running: one connection to the broker, one queue, one consumer.
// connectLink opens the controller's link over whichever bus this process is on (design 25: one
// variable moves it). The streams and this controller's consumers are raised first on the new bus,
// so nothing served here finds them missing.
func connectLink(ctx context.Context, inv *inventory.Inventory, enroller link.Enroller, listener link.Listener) (*link.Server, error) {
busAddress, err := broker.BusAddress()
if err != nil {
return nil, err
}
if inv != nil {
if err := raiseTheBus(ctx, inv, busAddress); err != nil {
return nil, err
}
}
js, err := broker.Dial(busAddress)
if err != nil {
return nil, fmt.Errorf("the mesh is on the bus at %s and this control plane cannot reach it: %w",
broker.BareAddress(busAddress), err)
}
return link.ConnectNats(js, enroller, listener), nil
}
func serve(ctx context.Context) error {
open, err := openStores(ctx)
if err != nil {
@@ -82,6 +61,11 @@ func serve(ctx context.Context) error {
}
fmt.Printf("signing as %s\n", key.Fingerprint()[:16])
management, err := broker.ManagementFromEnvironment()
if err != nil && !errors.Is(err, broker.ErrNotConfigured) {
return err
}
// Where the broker is and what to expect there, so a node can be told how to come back
// without a person and a new token.
known, err := broker.FromEnvironment()
@@ -93,22 +77,12 @@ func serve(ctx context.Context) error {
"reconnect. Set %s and %s.\n", broker.AddressVar, broker.CertificateVar)
}
// **Which bus this mesh is on, read once** (novox/hq ADR 0116 step 5). Both clients ship; both
// being live is refused, because a mesh half on each is one where a declaration goes out on one
// and the report comes back on the other, and every component logs success while it happens.
work := link.Enrolment{Inventory: inv, Identity: ident, Broker: known,
OnNATS: true}
server, err := connectLink(ctx, inv, work, work)
work := link.Enrolment{Inventory: inv, Identity: ident, Management: management, Broker: known}
server, err := link.Connect(work, work)
if err != nil {
return err
}
defer server.Close()
// The bus's own objects, asserted on every start. **Not created once at genesis**: a stream
// somebody deleted, a mesh raised from a restored backup, or a bus whose data directory was
// replaced all have records and no objects — and a node whose consumer is missing hears nothing
// while everything else about it looks correct.
// And build results nobody was waiting for. A build triggered any other way than `build`
// would otherwise be reported into the void, which is the same as not reporting it.
server.Records(builds{inv})
@@ -162,13 +136,13 @@ func declare(ctx context.Context, args []string) error {
return err
}
server, err := connectLink(ctx, nil, nil, nil)
server, err := link.Connect(nil, nil)
if err != nil {
return err
}
defer server.Close()
if err := link.Declare(ctx, server.Bus(), ident, node, raw, 15*time.Second); err != nil {
if err := link.Declare(ctx, server.Channel(), ident, node, raw, 15*time.Second); err != nil {
return err
}
fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw))
@@ -275,7 +249,7 @@ func pushCommand(ctx context.Context, args []string) error {
return err
}
server, err := connectLink(ctx, nil, nil, nil)
server, err := link.Connect(nil, nil)
if err != nil {
return err
}
@@ -336,7 +310,7 @@ func pushCommand(ctx context.Context, args []string) error {
if err != nil {
return err
}
if err := link.Declare(ctx, server.Bus(), ident, s.node, body, 15*time.Second); err != nil {
if err := link.Declare(ctx, server.Channel(), ident, s.node, body, 15*time.Second); err != nil {
return err
}
// After it is away, not before. A digest recorded for something that failed to send would
@@ -419,7 +393,7 @@ func pushCommand(ctx context.Context, args []string) error {
return declarationWith(held, open, node, plan, settings, gens, Allocating)
},
func(s readyNode, body []byte) error {
if err := link.Declare(ctx, server.Bus(), ident, s.node, body,
if err := link.Declare(ctx, server.Channel(), ident, s.node, body,
15*time.Second); err != nil {
return err
}
@@ -528,14 +502,8 @@ func composeEach(names []string,
continue
}
if len(declared.Resources) == 0 {
// Sent, not skipped (novox/hq issue 127). A node whose declaration composes to
// nothing may have HELD something before — the broker opening a placement gave it,
// say — and skipping the empty declaration leaves that last resource in force
// forever, re-applied by the node's own heartbeat, with no way for the mesh to say
// it is gone. An empty declaration is the correction: the host drops what the mesh
// owned and keeps what it found (the adoption envelope still rides along). A node
// that never held anything applies it as the no-op it is.
fmt.Printf("%s owns nothing now — sent so it drops what it last held\n", name)
fmt.Printf("%s is assigned nothing — skipped\n", name)
continue
}
sending = append(sending, readyNode{name, declared})
}
@@ -632,7 +600,7 @@ func sendTo(ctx context.Context, open *stores, names []string) error {
len(refusals), strings.Join(refusals, "\n\n"))
}
server, err := connectLink(ctx, nil, nil, nil)
server, err := link.Connect(nil, nil)
if err != nil {
return err
}
@@ -643,7 +611,7 @@ func sendTo(ctx context.Context, open *stores, names []string) error {
if err != nil {
return err
}
if err := link.Declare(ctx, server.Bus(), ident, s.node, body, 15*time.Second); err != nil {
if err := link.Declare(ctx, server.Channel(), ident, s.node, body, 15*time.Second); err != nil {
return err
}
record, err := inv.NodeByName(ctx, s.node)
@@ -696,122 +664,3 @@ func wouldSend(ctx context.Context, open *stores,
}
return out, nil
}
// raiseTheBus asserts the streams and consumers the mesh's own traffic needs.
//
// **Every start, and it says what it did.** The objects are the mesh's, created by nothing else —
// the controller is their only writer (design 25 §3) — so a mesh that came up without them is one
// where nodes connect, authenticate, and hear nothing. Said rather than silent for the reason the
// first line of `serve` is said: a log that is quiet on success and loud on failure reads as broken
// when it is working.
func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string) error {
js, err := broker.Dial(address)
if err != nil {
return fmt.Errorf("the mesh is on the bus at %s and this control plane cannot reach it: %w",
broker.BareAddress(address), err)
}
defer js.Close()
// **Its own user, before anything else.** The controller's account is created by the installer at
// a bootstrap password, before there is a controller to mint one — so nothing recorded a hash for
// it, and the first composition would leave the writer out of the file it was writing. Recorded
// only if absent: a credential the mesh minted since is the one that counts.
// **Its own user, before anything else it does here.** The controller's account is created by the
// installer at a bootstrap password, before there is a controller to mint one — so nothing
// recorded a hash for it, and the first composition would leave the writer out of the file it was
// writing: a bus nothing can connect to, produced by the thing connected to it. Recorded only if
// absent, so a restart cannot put the bootstrap credential back over a rotated one.
if user, password, _ := broker.CredentialIn(address); user != "" && password != "" {
if err := inv.SeedBusUser(ctx, inventory.BusUser{
Username: user, Kind: inventory.BusController,
}, password); err != nil {
return fmt.Errorf("cannot record the credential this control plane is using: %w", err)
}
}
nodes, err := inv.Nodes(ctx)
if err != nil {
return err
}
names := make([]string, 0, len(nodes))
for _, n := range nodes {
names = append(names, n.Name)
}
if err := broker.Raise(js, names); err != nil {
return err
}
// The work queues of the mesh's own roles (novox/hq ADR 0121). The queue before the holder,
// deliberately: work queues until somebody arrives to do it, so assigning a build machine a week
// after something started asking for builds flushes the backlog instead of having lost it.
// With the seats' holders, so each role's work queue gets the consumer its holder takes
// work from. Passed as nil until the first live raise, which left the build machine bound to a
// consumer nothing had created (2026-09-28).
holders, err := seatHolders(ctx, inv)
if err != nil {
return err
}
if err := broker.RaiseSeats(js, inventory.MeshSeats(), holders); err != nil {
return err
}
// And how every module hears what it consumes. Derived from the same records the user list is
// composed from, so a module the mesh grants a consumer's subjects has that consumer waiting.
// Done on every raise, not only when a credential is issued: every module moved onto this bus
// by the rollout was issued on the old one, and came up with nothing to bind to (2026-09-28).
records, err := inv.BusRecords(ctx)
if err != nil {
return err
}
users, err := broker.Users(records)
if err != nil {
return err
}
hearing := 0
for _, p := range users {
consumer, needed := broker.ConsumerFor(p)
if !needed {
continue
}
if err := js.EnsureConsumer(consumer); err != nil {
return fmt.Errorf("how %s on %s hears what it consumes: %w", p.Module, p.Node, err)
}
hearing++
}
fmt.Printf("the bus at %s has its streams, %d machine(s) can hear a declaration, and %d module(s) "+
"can hear what they consume\n", broker.BareAddress(address), len(names), hearing)
return nil
}
// seatHolders is who holds each of the mesh's seats, by seat name: the record where a handover
// wrote one, and the assigned module claiming the seat otherwise — the same derivation the
// resolver makes, read from the catalogue rather than re-resolved.
func seatHolders(ctx context.Context, inv *inventory.Inventory) (map[string]broker.Holder, error) {
out := map[string]broker.Holder{}
entries, err := inv.Catalogued(ctx)
if err != nil {
return nil, err
}
for _, e := range entries {
if len(e.On) == 0 {
continue
}
for _, c := range e.Manifest.Claims {
seat, known := catalogue.SeatNamed(c.Name)
if !known {
continue
}
if _, taken := out[seat.Name]; !taken {
out[seat.Name] = broker.Holder{Node: e.On[0], Module: e.Manifest.Module}
}
}
}
recorded, err := inv.Holdings(ctx)
if err != nil {
return nil, err
}
for _, h := range recorded {
if seat, known := catalogue.SeatNamed(h.Claim); known {
out[seat.Name] = broker.Holder{Node: h.Node, Module: h.Module}
}
}
return out, nil
}
+4 -6
View File
@@ -39,14 +39,12 @@ func TestOneUnresolvableNodeStillLetsTheRestBeSent(t *testing.T) {
}
}
// A machine whose declaration composes to nothing is SENT the empty declaration, not skipped
// (novox/hq issue 127): it may have held something before, and only sending the empty
// declaration tells it to drop what the mesh owned. It is never a refusal.
func TestAnEmptyDeclarationIsSentSoTheNodeDropsWhatItHeld(t *testing.T) {
// And a machine assigned nothing is neither sent nor a refusal — it is nothing to say.
func TestAMachineAssignedNothingIsNotARefusal(t *testing.T) {
sending, refusals := composeEach([]string{"spare"},
func(string) (sendable, error) { return sendable{}, nil })
if len(sending) != 1 || len(refusals) != 0 {
t.Errorf("an empty declaration must be sent, not skipped or refused: %v / %v", sending, refusals)
if len(sending) != 0 || len(refusals) != 0 {
t.Errorf("a machine assigned nothing was treated as something: %v / %v", sending, refusals)
}
}
+4 -42
View File
@@ -101,50 +101,12 @@ func routedArtifacts(made []inventory.Artifact, address string) []inventory.Arti
return out
}
// whereTheStoreIs is the artifact store's address as something on `forNode` reaches it, or "" —
// read for a caller that has the inventory open and nothing else in hand. With no node named, the
// store's own node: loopback when nothing is on the network yet.
func whereTheStoreIs(ctx context.Context, inv *inventory.Inventory, forNode string) (string, error) {
// whereTheStoreIs is the artifact store's address as this network reaches it, or "" — read for a
// caller that has the inventory open and nothing else in hand.
func whereTheStoreIs(ctx context.Context, inv *inventory.Inventory) (string, error) {
shelf, err := inv.Catalogue(ctx)
if err != nil {
return "", err
}
if forNode == "" {
if holder, _, found, err := artifactStoreHolder(ctx, inv); err != nil {
return "", err
} else if found {
forNode = holder
}
}
return artifactStoreAddress(ctx, inv, shelf, forNode)
}
// whereABuilderReachesTheStore is the store's address for the machine that builds: the network's
// when there is one, else loopback on the store's own node — when that node also holds a module
// requiring the store, which is what a builder is (genesis: one node holds both).
func whereABuilderReachesTheStore(ctx context.Context, inv *inventory.Inventory) (string, error) {
shelf, err := inv.Catalogue(ctx)
if err != nil {
return "", err
}
holder, _, found, err := artifactStoreHolder(ctx, inv)
if err != nil || !found {
return "", err
}
assigned, err := inv.Assigned(ctx, holder)
if err != nil {
return "", err
}
besideIt := false
for _, a := range assigned {
for _, r := range shelf[a].Requires {
if r == catalogue.ArtifactStoreProvision {
besideIt = true
}
}
}
if !besideIt {
holder = ""
}
return artifactStoreAddress(ctx, inv, shelf, holder)
return artifactStoreAddress(ctx, inv, shelf)
}
-455
View File
@@ -1,455 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/secrets"
)
// Moving the mesh's own traffic to the bus being built (novox/hq ADR 0116 step 5).
//
// **The whole mesh moves at once, so there is nothing to inspect afterwards.** Every seam ships both
// transports and every one of them chooses by a single fact; this is the step that flips it. That
// shape is deliberate — steps 1 to 4 leave every node where it is, so the cost of being wrong stays
// bounded until here — and it means the useful work is almost all in the checking.
//
// So `rollout check` is the command that matters and the one that can be run any number of times
// against a mesh that is serving. It answers from records: what is missing, and what would happen.
// `rollout` itself refuses unless the check is clean.
//
// **The old broker goes with the move, and goes last** (novox/hq ADR 0131): AMQP is not a provision,
// so once every machine reports on the new bus its module is unassigned. Only the mesh's own traffic
// is what moves, which is why this is survivable at all: what breaks if it goes wrong is the mesh's
// ability to change things, not the services its modules are serving — measured on 2026-09-27, when
// a seat emptied mid-change and the control plane looped for two hours while every service stayed up.
const rolloutUsage = "rollout check | rollout mint [--again] | rollout hand <node> | rollout --confirm"
func rolloutCommand(ctx context.Context, args []string) error {
switch {
case len(args) == 1 && args[0] == "check":
return rolloutCheck(ctx)
case len(args) == 1 && args[0] == "mint":
return rolloutMint(ctx, false)
case len(args) == 2 && args[0] == "hand":
return rolloutHand(ctx, args[1])
case len(args) == 2 && args[0] == "mint" && args[1] == "--again":
// Every credential minted afresh, whether or not one exists — for a mint that was wrong
// before anything was pushed. Afterwards nothing that received the old one still works,
// which is fine exactly when nothing received it.
return rolloutMint(ctx, true)
case len(args) == 1 && args[0] == "--confirm":
return errors.New(
"the rollout itself is not built yet: `rollout check` answers whether it could run, and " +
"what is missing. Moving every node at once is the one step with nothing to inspect " +
"afterwards, so it is not being written before the check it depends on has been run " +
"against a real mesh")
default:
return errors.New(rolloutUsage)
}
}
// rolloutCheck says whether the mesh could move, and what would happen if it did.
func rolloutCheck(ctx context.Context) error {
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
state, err := readinessOf(ctx, inv)
if err != nil {
return err
}
fmt.Println("the bus this mesh would move to")
if state.TheBus == "" {
fmt.Printf(" nothing names one (%s is unset)\n", broker.NATSVar)
} else {
standing := "not answering"
if state.ServerStanding {
standing = "answering"
}
fmt.Printf(" %s — %s\n", state.TheBus, standing)
}
fmt.Println()
fmt.Println("what would move")
for _, step := range broker.WhatMoves(state) {
fmt.Printf(" %s\n", step)
}
fmt.Println()
why := notReadyOf(state)
if len(why) == 0 {
fmt.Println("nothing is missing: this mesh could move its bus.")
fmt.Println()
fmt.Println("Read `what would move` above once more before running it. Every node moves at the")
fmt.Println("same moment and there is no half-moved state to look at afterwards.")
return nil
}
fmt.Printf("not ready — %d thing(s) to do first:\n", len(why))
for i, w := range why {
fmt.Printf(" %d. %s\n", i+1, w)
}
return nil
}
// readinessOf gathers what the mesh knows about its own ability to move.
//
// Reads and one dial, and nothing is written. Safe to run on a mesh that is serving, which is the
// point: the answer is only useful if it can be had without committing to anything.
func readinessOf(ctx context.Context, inv *inventory.Inventory) (broker.Readiness, error) {
state := broker.Readiness{
Credentialled: map[string]bool{},
ModuleCredentialled: map[string]bool{},
// The old broker keeps its other clients on this installation, and saying so is how the plan
// stops reading as a retirement.
}
address, _, err := broker.OnNATS()
if err != nil {
return state, err
}
state.TheBus = address
if address != "" {
// One dial, briefly. "Is it answering" is the one fact records cannot hold, and a mesh about
// to move onto a server that is not there should hear it here rather than afterwards.
//
// **Dialled the way the mesh dials it** — credential and pin — because a bare connect to a
// bus that requires TLS and a user fails at the handshake, and the check then reported a
// standing server as absent (seen live, 2026-09-28).
if js, err := broker.Dial(address, nats.Timeout(5*time.Second)); err == nil {
state.ServerStanding = true
js.Close()
}
}
nodes, err := inv.Nodes(ctx)
if err != nil {
return state, err
}
kept, err := inv.BusUsers(ctx)
if err != nil {
return state, err
}
shelf, err := inv.Catalogue(ctx)
if err != nil {
return state, err
}
for _, n := range nodes {
state.Nodes = append(state.Nodes, n.Name)
_, has := kept[broker.Principal{Kind: broker.KindNode, Node: n.Name}.Username()]
state.Credentialled[n.Name] = has
assigned, err := inv.Assigned(ctx, n.Name)
if err != nil {
return state, err
}
for _, module := range assigned {
m, known := shelf[module]
if !known {
continue
}
// The machine that holds the bus seat is the one that would be sent the user list.
if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") {
state.Holder = n.Name
state.AccountsComposed = wasSentTheUserList(ctx, inv, n.Name)
}
// A module that never speaks needs no credential, so it is not counted as missing one.
if !speaksOnTheBus(m) {
continue
}
named := n.Name + "/" + module
state.Modules = append(state.Modules, named)
_, hasOne := kept[broker.Principal{
Kind: broker.KindModule, Node: n.Name, Module: module,
}.Username()]
state.ModuleCredentialled[named] = hasOne
}
}
return state, nil
}
// speaksOnTheBus says whether a module reaches the bus at all.
//
// A third of the catalogue never does (novox/hq ADR 0120), and counting those as missing a credential
// would bury the ones that matter under a list nobody can act on.
func speaksOnTheBus(m catalogue.Manifest) bool {
return len(m.Emits) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
len(m.DefinesSeats) > 0 || len(m.Uses) > 0 || len(m.Claims) > 0
}
// wasSentTheUserList says whether the machine holding the bus has had a declaration since the user
// list became part of one.
//
// Read from what the mesh recorded sending rather than asked of the machine: a machine that is away
// has still been sent it, and this question is about whether the mesh did its part.
func wasSentTheUserList(ctx context.Context, inv *inventory.Inventory, node string) bool {
digest, err := inv.Outstanding(ctx, node)
return err == nil && strings.TrimSpace(digest) != ""
}
// notReadyOf is the readiness reasoning, named here so a test can reach it without the command's
// printing. The reasoning itself is the broker package's, where it is pure.
func notReadyOf(state broker.Readiness) []string { return broker.NotReady(state) }
// rolloutMint gives every principal the new bus will have a credential it does not yet have, and
// puts each where its owner reads it (novox/hq design 28, task 5.2): a machine's as a membership
// sealed into its declaration, a module's as its broker secret, the control plane's own as its
// `bus` secret. Idempotent: what already has a hash is left alone, so running it again is harmless.
//
// **Before anything moves, and it is what makes moving possible.** A machine moved without a
// credential cannot come back, and afterwards there is no bus to tell it anything over — which is
// why `rollout check` refuses until this has run. The bus's address is worked out here, from where
// the module that provides it is assigned, rather than read from this process's environment: this
// process is still on the old bus when this runs, and must be.
func rolloutMint(ctx context.Context, again bool) error {
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
known, err := broker.FromEnvironment()
if err != nil {
return fmt.Errorf("the bus's certificate is not known to this process, and every membership "+
"must carry its fingerprint: %w", err)
}
shelf, err := inv.Catalogue(ctx)
if err != nil {
return err
}
entries, err := inv.Catalogued(ctx)
if err != nil {
return err
}
var busNode, controllerNode string
for _, e := range entries {
switch {
case e.Manifest.ClaimsSeat("mesh-broker") && providesBus(e.Manifest) && len(e.On) > 0:
busNode = e.On[0]
case e.Manifest.Module == "mesh-controller" && len(e.On) > 0:
controllerNode = e.On[0]
}
}
if busNode == "" {
return errors.New("no assigned module provides mesh-bus and claims mesh-broker, so there is no " +
"bus to mint credentials for — register and assign it first")
}
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
if err != nil {
return err
}
busHost := onNetwork[busNode]
if busHost == "" {
// **The hub is not in that map.** The machine that took over the tunnel is where the current
// bus already answers, and every machine dials it at the address the mesh handed them — so
// when the new bus runs on the same machine, that address is the one to tell them, with the
// new port. Found live: the control node is the hub, and the map lists the machines placed
// around it.
// The host alone: no scheme (BareAddress adds one where none was, which is the wrong
// direction here — every URL built below adds its own) and no port.
_, _, host := broker.CredentialIn(known.Address)
if host == "" {
host = known.Address
}
if _, after, hasScheme := strings.Cut(host, "://"); hasScheme {
host = after
}
host = strings.TrimSpace(host)
if i := strings.LastIndex(host, ":"); i > 0 && !strings.Contains(host[i:], "]") {
host = host[:i]
}
if host == "" {
return fmt.Errorf("%s runs the new bus and has no address on the private network, and the "+
"current bus's address is unknown too, so no machine could be told where it is", busNode)
}
busHost = host
}
busAddress := busHost + ":4222"
records, err := inv.BusRecords(ctx)
if err != nil {
return err
}
users, err := broker.Users(records)
if err != nil {
return err
}
kept, err := inv.BusUsers(ctx)
if err != nil {
return err
}
hashes := make(map[string]string, len(kept))
for name, u := range kept {
hashes[name] = u.PasswordHash
}
_, missing := broker.WithPasswords(users, hashes)
wanted := map[string]bool{}
for _, m := range missing {
wanted[m] = true
}
var machines, modules, skipped int
for _, p := range users {
if !again && !wanted[p.Username()] {
continue
}
switch p.Kind {
case broker.KindController:
if controllerNode == "" {
return errors.New("the control plane is not assigned anywhere, so its credential has nowhere to go")
}
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusController})
if err != nil {
return err
}
url := "nats://" + p.Username() + ":" + password + "@" + busAddress
if err := inv.AcceptSecretForModule(ctx, controllerNode, "mesh-controller", "bus", url); err != nil {
return fmt.Errorf("the control plane's credential is minted and could not be sealed to %s: %w", controllerNode, err)
}
fmt.Printf("control plane: credential minted, sealed to %s as its `bus` secret\n", controllerNode)
case broker.KindNode:
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusNode, Node: p.Node})
if err != nil {
return err
}
membership, _ := json.Marshal(map[string]string{
"broker": busAddress, "fingerprint": known.Fingerprint, "password": password, "transport": "nats",
})
key, err := inv.SealingKeyOf(ctx, p.Node)
if err != nil {
return fmt.Errorf("%s has no sealing key, so its membership cannot be sealed to it: %w", p.Node, err)
}
sealed, err := secrets.Seal(key, membership)
if err != nil {
return err
}
if err := inv.PutBusMembership(ctx, p.Node, sealed); err != nil {
return err
}
machines++
case broker.KindModule:
if p.Module == "mesh-controller" {
// The control plane is a module too, and its `broker` secret is the old bus's
// credential it is still using while this runs. Writing the new bus's blob there
// cut the mesh off from its own old bus mid-move (2026-09-28). Its new-bus credential
// is the controller principal's `bus` secret above; nothing else is needed here.
skipped++
continue
}
m, inShelf := shelf[p.Module]
if !inShelf {
skipped++
continue
}
if _, reads := m.OwnSecrets["broker"]; !reads {
fmt.Printf(" %s on %s speaks on the bus but declares no `broker` secret to receive a credential in; skipped\n", p.Module, p.Node)
skipped++
continue
}
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusModule, Node: p.Node, Module: p.Module})
if err != nil {
return err
}
if err := issueWith(ctx, inv, m, p.Node, "", known, busAddress, p.Username(), password); err != nil {
return err
}
modules++
default:
skipped++
}
}
fmt.Printf("minted for %d machine(s) and %d module runtime(s); %d skipped; the bus is at %s\n",
machines, modules, skipped, busAddress)
fmt.Println(" each machine's membership and each module's credential arrive with the next push of its machine;")
fmt.Println(" push the machine running the bus first, so the bus stands with its user list before anything dials it")
return nil
}
// providesBus is whether a manifest provides the mesh's bus.
func providesBus(m catalogue.Manifest) bool {
for _, o := range m.Provides {
if o.Name == "mesh-bus" {
return true
}
}
return false
}
// rolloutHand mints a machine its credential for the new bus afresh and prints its membership
// once, for an operator to carry by hand — the rescue for a machine that cannot be reached over
// any bus: rotated while it still held the old password, or reachable only by ssh. The plaintext
// exists on this terminal and then only where it is written; the store keeps the hash, and the
// sealed copy in the machine's declaration is replaced too, so the next push says the same.
func rolloutHand(ctx context.Context, node string) error {
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
known, err := broker.FromEnvironment()
if err != nil {
return fmt.Errorf("the bus's certificate is not known to this process: %w", err)
}
busAddress, _, err := broker.OnNATS()
if err != nil {
return err
}
if busAddress == "" {
return errors.New("this control plane is not on the new bus, so there is no membership to hand out")
}
_, _, bare := broker.CredentialIn(busAddress)
if _, after, has := strings.Cut(bare, "://"); has {
bare = after
}
if _, err := inv.NodeByName(ctx, node); err != nil {
return err
}
p := broker.Principal{Kind: broker.KindNode, Node: node}
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusNode, Node: node})
if err != nil {
return err
}
membership, _ := json.Marshal(map[string]string{
"broker": bare, "fingerprint": known.Fingerprint, "password": password, "transport": "nats",
})
key, err := inv.SealingKeyOf(ctx, node)
if err != nil {
return err
}
sealed, err := secrets.Seal(key, membership)
if err != nil {
return err
}
if err := inv.PutBusMembership(ctx, node, sealed); err != nil {
return err
}
// The one line of output is the membership itself, so it can be piped to the machine without
// being read on the way. Everything else goes to stderr.
fmt.Fprintf(os.Stderr, "%s's credential is minted afresh. Write this to %s on it and restart its host; "+
"then push the machine running the bus so the user list carries the new hash.\n",
node, catalogue.BusMembershipPath)
fmt.Println(string(membership))
return nil
}
-93
View File
@@ -1,93 +0,0 @@
package main
import (
"context"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
)
// Whether a mesh could move its bus, read from a real store.
//
// The readiness reasoning has its own tests; this is about the gathering — that the question is
// answered from what the mesh actually holds, on a store with machines and modules in it, without
// writing anything.
func TestReadinessIsGatheredFromWhatTheMeshHolds(t *testing.T) {
inv := inventory.ForTest(t)
ctx := context.Background()
// A mesh mid-change: two machines, the bus module on one of them, a module that speaks and a
// module that never does.
for _, m := range []catalogue.Manifest{
{Module: "nats", Version: "1", BusUsers: "/var/lib/nats-module/conf/accounts.conf",
Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}}},
{Module: "gitea", Version: "1", Tools: []string{"repo_create"}},
{Module: "wallpaper", Version: "1"},
} {
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "/r"}); err != nil {
t.Fatal(err)
}
}
for _, n := range []string{"anchor", "laptop"} {
if _, err := inv.AddNode(ctx, n); err != nil {
t.Fatal(err)
}
}
for _, a := range [][2]string{{"anchor", "nats"}, {"anchor", "gitea"}, {"laptop", "wallpaper"}} {
if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
t.Fatal(err)
}
}
state, err := readinessOf(ctx, inv)
if err != nil {
t.Fatal(err)
}
if state.Holder != "anchor" {
t.Errorf("the machine holding the bus reads as %q", state.Holder)
}
if len(state.Nodes) != 2 {
t.Errorf("machines read as %v", state.Nodes)
}
// **A module that never speaks is not counted as missing a credential.** A third of the catalogue
// never reaches the bus, and listing those would bury the ones that matter.
for _, m := range state.Modules {
if strings.HasSuffix(m, "/wallpaper") {
t.Errorf("a module that never speaks was counted: %v", state.Modules)
}
}
if len(state.Modules) != 2 {
t.Errorf("modules that speak read as %v; expected the bus module and the one with a tool",
state.Modules)
}
// Nothing has been minted, so it is not ready — and it says so about each machine by name.
why := notReadyOf(state)
if len(why) == 0 {
t.Fatal("a mesh where nothing has a credential was reported ready to move")
}
said := strings.Join(why, "\n")
for _, name := range []string{"anchor", "laptop"} {
if !strings.Contains(said, name) {
t.Errorf("the refusal does not name %s: %s", name, said)
}
}
// Mint for one machine and it drops out of the complaint, which is how somebody works through it.
if _, err := inv.MintBusPassword(ctx, inventory.BusUser{
Username: "node.laptop", Kind: inventory.BusNode, Node: "laptop",
}); err != nil {
t.Fatal(err)
}
state, err = readinessOf(ctx, inv)
if err != nil {
t.Fatal(err)
}
if !state.Credentialled["laptop"] {
t.Error("a machine that was minted a credential still reads as having none")
}
}
-267
View File
@@ -1,267 +0,0 @@
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"os"
"slices"
"sort"
"strings"
"text/tabwriter"
"github.com/novox/mesh-controller/internal/catalogue"
)
// What this mesh can have one of, and who fills each (novox/hq ADR 0110).
//
// **Derived every time, never stored.** A seat is held by a module assignment, so the answer is
// computed from assignments by the same resolution that decides what every machine runs. A table
// of holders kept beside the assignments would be a second copy of one fact, and the first thing
// to be wrong about it.
// seatHolder is one assignment holding a seat.
type seatHolder struct {
Node string `json:"node"`
Module string `json:"module"`
}
// seatRow is one seat and who holds it. Unheld is an answer — "this mesh has no X" — not a fault.
type seatRow struct {
Seat string `json:"seat"`
Scope string `json:"scope"`
Delivers string `json:"delivers,omitempty"`
Decision string `json:"decision"`
Holders []seatHolder `json:"holders"`
}
// seatsHeld is every seat the mesh defines with its holders, and every claim held that names no
// seat in the set.
//
// **The second list is not empty by construction.** Manifests are held to the set when they are
// registered, and a mesh can hold one registered before the set closed. Leaving its claim out of the
// overview would make the one thing the overview is for — what does this mesh have — quietly
// incomplete.
func seatsHeld(seats []catalogue.Seat, held []catalogue.Held) ([]seatRow, []catalogue.Held) {
rows := make([]seatRow, 0, len(seats))
for _, s := range seats {
row := seatRow{Seat: s.Name, Scope: s.Scope, Delivers: s.Delivers, Decision: s.Decision,
Holders: []seatHolder{}}
seen := map[seatHolder]bool{}
for _, h := range held {
// Resolve the held claim to a seat rather than comparing names, so a record naming a
// seat's former name groups under it after a rename (novox/hq ADR 0122).
hs, ok := catalogue.SeatNamed(h.Claim)
if !ok || hs.Name != s.Name || h.Scope != s.Scope {
continue
}
holder := seatHolder{Node: h.Node, Module: h.Module}
if !seen[holder] {
seen[holder] = true
row.Holders = append(row.Holders, holder)
}
}
sort.Slice(row.Holders, func(i, j int) bool {
if row.Holders[i].Node != row.Holders[j].Node {
return row.Holders[i].Node < row.Holders[j].Node
}
return row.Holders[i].Module < row.Holders[j].Module
})
rows = append(rows, row)
}
var outside []catalogue.Held
for _, h := range held {
// Outside the set only if it resolves to no seat at all — a former name still resolves.
if _, ok := catalogue.SeatNamed(h.Claim); !ok {
outside = append(outside, h)
}
}
sort.Slice(outside, func(i, j int) bool {
if outside[i].Claim != outside[j].Claim {
return outside[i].Claim < outside[j].Claim
}
return outside[i].Node < outside[j].Node
})
return rows, outside
}
// seatCommand changes the set — the whole point of it being data (novox/hq ADR 0122) — and, since
// ADR 0131, changes who holds a seat.
func seatCommand(ctx context.Context, args []string) error {
if len(args) == 3 && args[0] == "rename" {
from, to := args[1], args[2]
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
if err := open.inventory.RenameSeat(ctx, from, to); err != nil {
return err
}
fmt.Printf("%s is now %s — its former name still resolves, so nothing is rebuilt, "+
"re-registered or frozen (novox/hq ADR 0122)\n", from, to)
return nil
}
if len(args) == 3 && args[1] == "--to" {
return handOver(ctx, args[0], args[2])
}
return fmt.Errorf("seat rename <from> <to> | seat <name> --to <node>/<module>")
}
// handOver makes one assignment the holder of a seat, as one act, so the seat is never without a
// holder in between (novox/hq ADR 0131, design 28 task 5.3). The control plane finds its own bus
// through one of these seats; the day it was left empty mid-change is why this exists.
//
// Everything that could make the new holder wrong is refused here, before the row is written: the
// seat must exist, the assignment must exist, and the module must be able to hold the seat —
// claim it at its scope and provide what it delivers, judged against the store's row. What is
// **not** checked is whether the module is running yet: that is what `push` confirms afterwards,
// and refusing to record a handover to a module the node has not started would make the handover
// impossible to do before the switch instead of as the switch.
func handOver(ctx context.Context, seatName, to string) error {
nodeName, module, ok := strings.Cut(to, "/")
if !ok || nodeName == "" || module == "" {
return fmt.Errorf("the new holder is named <node>/<module>, not %q", to)
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
seat, known := catalogue.SeatNamed(seatName)
if !known {
return fmt.Errorf("%q is not a seat this mesh defines — `seats` lists them", seatName)
}
assigned, err := inv.Assigned(ctx, nodeName)
if err != nil {
return err
}
if !slices.Contains(assigned, module) {
return fmt.Errorf("%s is not assigned to %s, so it cannot hold anything there — "+
"`assign %s %s` first", module, nodeName, nodeName, module)
}
entries, err := inv.Catalogued(ctx)
if err != nil {
return err
}
var m *catalogue.Manifest
for i := range entries {
if entries[i].Manifest.Module == module {
m = &entries[i].Manifest
}
}
if m == nil {
return fmt.Errorf("%s is assigned but not in the catalogue, which should not happen", module)
}
var was string
holdings, err := inv.Holdings(ctx)
if err != nil {
return err
}
for _, h := range holdings {
if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name {
was = h.Node
}
}
// **Recording who already holds the seat is not making a new holder, and is not judged like
// one.** On a mesh that predates the record, the first handover has to begin by writing down
// the standing holder — otherwise the next holder cannot be assigned beside it, because two
// eligible claimants with nothing on record are refused. That standing holder may no longer
// satisfy what the seat delivers (the row moved under it, on purpose, as ADR 0131's first step),
// and it holds regardless: derivation never read that column. So when nothing is on record and
// the named assignment is the one holding by derivation, only the claim itself is checked here.
// Every *change* of holder is judged in full.
claimsIt := false
for _, c := range m.Claims {
if cs, ok := catalogue.SeatNamed(c.Name); ok && cs.Name == seat.Name && c.At() == seat.Scope {
claimsIt = true
}
}
if was == "" && claimsIt {
fmt.Printf("nothing was on record for %s; recording %s on %s as its standing holder\n",
seat.Name, module, nodeName)
} else if err := catalogue.CanHold(*m, seat); err != nil {
return fmt.Errorf("%s cannot hold %s: %w", module, seat.Name, err)
}
if err := inv.HoldSeat(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
return err
}
fmt.Printf("%s is held by %s on %s\n", seat.Name, module, nodeName)
if was != "" && was != nodeName {
fmt.Printf(" `push %s` and `push %s` send both machines what changed\n", was, nodeName)
} else {
fmt.Printf(" `push %s` sends the machine what changed; every other machine that reads the "+
"seat is re-declared by `push --behind`\n", nodeName)
}
return nil
}
func seatsCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("seats", flag.ContinueOnError)
asJSON := set.Bool("json", false, "the same, as JSON")
if err := set.Parse(args); err != nil {
return err
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
shelf, err := inv.Catalogue(ctx)
if err != nil {
return err
}
// Every node, none excluded: the same view of what each machine holds that planning uses.
world, err := theRestOfTheMesh(ctx, inv, shelf, "")
if err != nil {
return err
}
rows, outside := seatsHeld(catalogue.Seats(), world.Held)
if *asJSON {
out := struct {
Seats []seatRow `json:"seats"`
Outside []catalogue.Held `json:"outside,omitempty"`
}{rows, outside}
body, err := json.MarshalIndent(out, "", " ")
if err != nil {
return err
}
fmt.Println(string(body))
return nil
}
w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
fmt.Fprintln(w, "SEAT\tSCOPE\tDELIVERS\tHELD BY")
for _, r := range rows {
delivers := r.Delivers
if delivers == "" {
delivers = "—"
}
holders := "unheld"
if len(r.Holders) > 0 {
parts := make([]string, 0, len(r.Holders))
for _, h := range r.Holders {
parts = append(parts, h.Module+" on "+h.Node)
}
holders = strings.Join(parts, ", ")
}
fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", r.Seat, r.Scope, delivers, holders)
}
if err := w.Flush(); err != nil {
return err
}
if len(outside) > 0 {
fmt.Println("\nheld, and not a seat this mesh defines (registered before the set closed — novox/hq ADR 0110):")
for _, h := range outside {
fmt.Printf(" %s %s on %s\n", h.Claim, h.Module, h.Node)
}
}
return nil
}
-64
View File
@@ -1,64 +0,0 @@
package main
import (
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// The overview of what a mesh has (novox/hq ADR 0110).
func TestEverySeatIsListedIncludingTheOnesNobodyHolds(t *testing.T) {
// An unheld seat is an answer — "this mesh has no forge" — so it is listed rather than omitted.
rows, _ := seatsHeld(catalogue.Seats(), []catalogue.Held{
{Claim: "mesh-store", Scope: catalogue.ScopeMesh, Node: "anchor", Module: "postgres"},
})
if len(rows) != len(catalogue.Seats()) {
t.Fatalf("%d seats listed of %d", len(rows), len(catalogue.Seats()))
}
for _, r := range rows {
switch r.Seat {
case "mesh-store":
if len(r.Holders) != 1 || r.Holders[0].Module != "postgres" || r.Holders[0].Node != "anchor" {
t.Errorf("mesh-store is held by %+v", r.Holders)
}
if r.Delivers != "postgres-database" {
t.Errorf("mesh-store does not say what it delivers: %q", r.Delivers)
}
case "git":
if len(r.Holders) != 0 {
t.Errorf("git is held by %+v in a mesh with no forge", r.Holders)
}
}
}
}
func TestANodeSeatListsEveryMachineHoldingIt(t *testing.T) {
rows, _ := seatsHeld(catalogue.Seats(), []catalogue.Held{
{Claim: "node-packet-filter", Scope: catalogue.ScopeNode, Node: "node2", Module: "nftables"},
{Claim: "node-packet-filter", Scope: catalogue.ScopeNode, Node: "anchor", Module: "nftables"},
// Resolved twice, reported once: a machine is one holder however many passes saw it.
{Claim: "node-packet-filter", Scope: catalogue.ScopeNode, Node: "anchor", Module: "nftables"},
})
for _, r := range rows {
if r.Seat != "node-packet-filter" {
continue
}
if len(r.Holders) != 2 || r.Holders[0].Node != "anchor" || r.Holders[1].Node != "node2" {
t.Fatalf("the packet filter is held by %+v", r.Holders)
}
return
}
t.Fatal("the packet filter is not listed")
}
func TestAClaimOutsideTheSetIsShownNotHidden(t *testing.T) {
// A manifest registered before the set closed can still hold one. Leaving it out would make
// the overview quietly incomplete, which is the one thing it may not be.
_, outside := seatsHeld(catalogue.Seats(), []catalogue.Held{
{Claim: "the-controller", Scope: catalogue.ScopeMesh, Node: "anchor", Module: "mesh-controller"},
})
if len(outside) != 1 || outside[0].Claim != "the-controller" {
t.Fatalf("a claim outside the set was not shown: %+v", outside)
}
}
-6
View File
@@ -38,12 +38,6 @@ func (s sendable) Body() ([]byte, error) {
if s.Adoption != nil {
envelope["adoption"] = s.Adoption
}
// An empty declaration is deliberate here — the node owns nothing the mesh put there
// (novox/hq issue 127) — and the host refuses an empty body unless it is told the emptiness
// is meant, so a truncated or mis-composed body is never mistaken for "own nothing".
if len(s.Resources) == 0 {
envelope["owns_nothing"] = true
}
return json.Marshal(envelope)
}
-22
View File
@@ -355,25 +355,3 @@ func TestTheMachineSideOfAMappingIsMovedEverywhereTheNumberIsUsed(t *testing.T)
t.Fatalf("the consumer is told the forge answers on %v", told)
}
}
func TestAnEmptyDeclarationSaysOwnsNothing(t *testing.T) {
// The host refuses an empty body unless told the emptiness is meant (novox/hq issue 127).
body, err := sendable{}.Body()
if err != nil {
t.Fatal(err)
}
var env map[string]any
if err := json.Unmarshal(body, &env); err != nil {
t.Fatal(err)
}
if env["owns_nothing"] != true {
t.Fatalf("an empty declaration must mark owns_nothing; got %v", env)
}
// A declaration with resources does not carry the marker.
body, _ = sendable{Resources: []map[string]any{{"id": "x"}}}.Body()
var env2 map[string]any
_ = json.Unmarshal(body, &env2)
if _, present := env2["owns_nothing"]; present {
t.Fatalf("a non-empty declaration must not mark owns_nothing; got %v", env)
}
}
-136
View File
@@ -1,136 +0,0 @@
package main
import (
"context"
"fmt"
"strconv"
"strings"
"github.com/novox/mesh-controller/internal/catalogue"
)
// where a build's repository is (novox/hq ADR 0111).
//
// A repository is on the mesh's own forge, or it is anywhere else. The first is recorded as its path
// on the forge holding the git seat, and cloned from wherever that forge runs at the moment of
// building; the second is a URL, recorded and cloned exactly as given. The build machine is not told
// the difference — it is handed a URL either way — because only the control plane knows where the
// seat's holder runs.
// gitSeat is the seat a self-hosted repository lives on.
const gitSeat = "git"
// buildSource is where a build's repository is: a URL, or a path on a seat's holder.
type buildSource struct {
Repository string
Seat string
}
// String is the source as a person reads it, which for one on a seat is not the URL: the URL is a
// fact about where the forge happens to run today.
func (s buildSource) String() string {
if s.Seat == "" {
return s.Repository
}
return fmt.Sprintf("%s on the %s seat", s.Repository, s.Seat)
}
// onASeat refuses an address given as a path on the forge.
//
// **A URL here would be recorded as a path**, and then composed onto the forge's address as one —
// cloning `http://forge:3000/https://github.com/…`. Refused by what an address plainly looks like,
// not repaired: `--self` promises a path, and something that is not one is a mistake to name.
func onASeat(repository string) error {
if strings.Contains(repository, ":") || strings.HasPrefix(repository, "/") ||
strings.Trim(repository, "/") == "" {
return fmt.Errorf("--self takes the repository's path on the forge, such as novox/mesh-catalog, "+
"and %q is not one — without --self it is built from exactly what is given", repository)
}
return nil
}
// cloneFrom is the URL a build machine clones for a source.
//
// A URL is itself. A path on a seat is composed from the seat's holder as the mesh sees it now —
// the same view planning takes of every machine, so the forge a build clones from is the forge the
// mesh says holds the seat.
func cloneFrom(ctx context.Context, source buildSource) (string, error) {
if source.Seat == "" {
return source.Repository, nil
}
open, err := openStores(ctx)
if err != nil {
return "", err
}
defer open.Close()
shelf, err := open.inventory.Catalogue(ctx)
if err != nil {
return "", err
}
world, err := theRestOfTheMesh(ctx, open.inventory, shelf, "")
if err != nil {
return "", err
}
return clonedFromSeat(world, source.Seat, source.Repository)
}
// clonedFromSeat composes the clone URL for a repository on a seat's holder.
//
// **Refused, never defaulted, at every step that has no answer.** Nobody holding the seat is a mesh
// without a forge of its own: it builds from external repositories and must say so rather than fail
// to clone. A holder off the private network cannot be reached by any build machine. A holder that
// serves no scheme or port has nothing to compose from — a default port here would be the forge's
// address guessed, which is the thing this exists to stop.
func clonedFromSeat(world catalogue.World, seatName, repository string) (string, error) {
seat, known := catalogue.SeatNamed(seatName)
if !known || seat.Delivers == "" {
return "", fmt.Errorf("%q is not a seat a repository can live on", seatName)
}
var holder *catalogue.Held
for i, h := range world.Held {
if h.Claim == seat.Name && h.Scope == seat.Scope {
holder = &world.Held[i]
break
}
}
if holder == nil {
return "", fmt.Errorf("nobody holds the %s seat, so %s cannot be cloned from this mesh's "+
"forge — assign a module that claims it, or build from the repository's URL without --self",
seat.Name, repository)
}
var provider *catalogue.Provider
for i, p := range world.Offered[seat.Delivers] {
if p.Node == holder.Node && p.Module == holder.Module {
provider = &world.Offered[seat.Delivers][i]
}
}
if provider == nil {
return "", fmt.Errorf("%s on %s holds the %s seat and offers no %q to clone from",
holder.Module, holder.Node, seat.Name, seat.Delivers)
}
if provider.At == "" {
return "", fmt.Errorf("%s on %s holds the %s seat and is not on the private network, so no "+
"build machine can reach it", holder.Module, holder.Node, seat.Name)
}
scheme, _ := provider.Serves["scheme"].(string)
port := servedPort(provider.Serves["port"])
if scheme == "" || port == "" {
return "", fmt.Errorf("%s on %s holds the %s seat and does not serve a scheme and a port for %q",
holder.Module, holder.Node, seat.Name, seat.Delivers)
}
path := strings.TrimSuffix(strings.Trim(repository, "/"), ".git")
return fmt.Sprintf("%s://%s:%s/%s.git", scheme, provider.At, port, path), nil
}
// servedPort is a served port as text, however the manifest and the node's settings carried it.
func servedPort(v any) string {
switch p := v.(type) {
case float64:
return strconv.Itoa(int(p))
case int:
return strconv.Itoa(p)
case string:
return p
}
return ""
}
-108
View File
@@ -1,108 +0,0 @@
package main
import (
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// Defends novox/hq ADR 0111: a build source is on the git seat, or it is external.
func forgeHolding(port any) catalogue.World {
return catalogue.World{
Held: []catalogue.Held{{Claim: "git", Scope: catalogue.ScopeMesh, Node: "anchor", Module: "gitea"}},
Offered: map[string][]catalogue.Provider{"git": {
// A second forge that does not hold the seat, so taking the first one found would be wrong.
{Node: "archive", At: "archive.internal", Module: "gitea-mirror",
Serves: map[string]any{"scheme": "http", "port": float64(3000)}},
{Node: "anchor", At: "anchor.internal", Module: "gitea",
Serves: map[string]any{"scheme": "http", "port": port}},
}},
}
}
func TestARepositoryOnTheSeatIsClonedFromItsHolder(t *testing.T) {
got, err := clonedFromSeat(forgeHolding(float64(3000)), "git", "novox/mesh-catalog")
if err != nil {
t.Fatal(err)
}
if got != "http://anchor.internal:3000/novox/mesh-catalog.git" {
t.Fatalf("cloned from %s", got)
}
}
func TestAMovedForgeIsFollowedWithoutRewritingAnything(t *testing.T) {
// The whole point: the node gave the forge another port, and the same recorded path clones
// from the new one. Nothing recorded contained the old one to be wrong.
got, err := clonedFromSeat(forgeHolding(float64(3100)), "git", "novox/mesh-catalog")
if err != nil {
t.Fatal(err)
}
if !strings.Contains(got, ":3100/") {
t.Fatalf("the moved port was not followed: %s", got)
}
}
func TestWithNobodyHoldingTheSeatASelfHostedBuildIsRefusedAndSaysWhy(t *testing.T) {
_, err := clonedFromSeat(catalogue.World{}, "git", "novox/mesh-catalog")
if err == nil {
t.Fatal("a repository was cloned from a forge the mesh does not have")
}
for _, want := range []string{"nobody holds the git seat", "without --self"} {
if !strings.Contains(err.Error(), want) {
t.Fatalf("the refusal does not say %q: %v", want, err)
}
}
}
func TestAnExternalRepositoryIsClonedExactlyAsGiven(t *testing.T) {
// Unaffected by the seat, held or not: GitHub and GitLab are the ordinary cases.
given := "https://github.com/someone/something.git"
got, err := cloneFrom(t.Context(), buildSource{Repository: given})
if err != nil {
t.Fatal(err)
}
if got != given {
t.Fatalf("an external repository became %s", got)
}
}
func TestAHolderOffThePrivateNetworkIsRefused(t *testing.T) {
world := forgeHolding(float64(3000))
world.Offered["git"][1].At = ""
if _, err := clonedFromSeat(world, "git", "novox/mesh-catalog"); err == nil ||
!strings.Contains(err.Error(), "private network") {
t.Fatalf("a forge nothing can reach was cloned from: %v", err)
}
}
func TestAHolderServingNoPortIsRefusedRatherThanGuessed(t *testing.T) {
// A default port would be the forge's address guessed, which is what this exists to stop.
if _, err := clonedFromSeat(forgeHolding(nil), "git", "novox/mesh-catalog"); err == nil {
t.Fatal("a port was guessed for a forge that serves none")
}
}
func TestAnAddressGivenAsAPathOnTheForgeIsRefused(t *testing.T) {
for _, bad := range []string{
"https://github.com/someone/something.git",
"git@anchor:novox/mesh-catalog.git",
"/srv/git/mesh-catalog",
"",
} {
if err := onASeat(bad); err == nil {
t.Errorf("--self accepted %q as a path on the forge", bad)
}
}
if err := onASeat("novox/mesh-catalog"); err != nil {
t.Errorf("a path on the forge was refused: %v", err)
}
}
func TestASourceOnTheSeatReadsAsAPathNotAnAddress(t *testing.T) {
s := buildSource{Repository: "novox/mesh-catalog", Seat: "git"}
if got := s.String(); got != "novox/mesh-catalog on the git seat" {
t.Fatalf("read as %q", got)
}
}
-21
View File
@@ -5,7 +5,6 @@ import (
"fmt"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/identity"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/licences"
@@ -73,14 +72,6 @@ func migrate(ctx context.Context) error {
}
fmt.Printf("provided %s\n", m.Module)
}
// The seats the mesh ships with, into the table that now holds the set (novox/hq ADR 0122).
// Idempotent: fills an empty table on first boot, adds a seat a release ships, and leaves an
// operator's changes in the table as they are.
added, err := inv.SeedSeats(ctx, catalogue.DefaultSeats())
if err != nil {
return err
}
fmt.Printf("seeded %d seat(s)\n", added)
return nil
}
@@ -94,18 +85,6 @@ func openInventory(ctx context.Context) (*inventory.Inventory, error) {
inv.Close()
return nil, err
}
// Load the seat set from the store, so the control plane reads the set as data rather than as
// the slice it was compiled with (novox/hq ADR 0122). A store not yet seeded — or one whose
// seat table a migration has not reached — returns nothing, and UseSeats leaves the compiled
// defaults in force: the set is never emptied by a read that found nothing, which would refuse
// every claim. So this can only ever replace the defaults with what the mesh actually holds.
if seats, err := inv.Seats(ctx); err == nil {
catalogue.UseSeats(seats)
}
// And the former names, so a reference to a seat's old name resolves after a rename (ADR 0122).
if aliases, err := inv.Aliases(ctx); err == nil {
catalogue.UseAliases(aliases)
}
return inv, nil
}
+1 -305
View File
@@ -6,9 +6,7 @@ import (
"flag"
"fmt"
"strings"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
@@ -186,7 +184,7 @@ func (f following) Announceable(ctx context.Context) ([]link.Announcement, error
if err != nil {
return nil, err
}
address, err := whereTheStoreIs(ctx, f.open.inventory, "")
address, err := whereTheStoreIs(ctx, f.open.inventory)
if err != nil {
return nil, err
}
@@ -220,305 +218,3 @@ func notNow(err error) error {
}
return err
}
// SourceMoved is the forge announcing a merge: every module recorded as built from that
// repository and branch is marked as moved to the merge commit, and built — bases first, so a
// module that stands on another's artifact is built after it and not against the old one
// (novox/hq 04-ISSUES/131). Nothing is pushed here: what a finished build does to the machines
// running the module is the upgrade's decision, taken when the catalogue announces it.
func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
inv := f.open.inventory
entries, err := inv.Catalogued(ctx)
if err != nil {
return notNow(err)
}
read, err := inv.ReadRepositories(ctx)
if err != nil {
return notNow(err)
}
// Two kinds of module are affected by one merge, and they are affected differently.
//
// A module **built from** this repository and branch has moved: the mesh records the new commit
// as what its source now has, and only what the merge actually changed is rebuilt. A module that
// only **packages source from** it has not moved — its own source is somewhere else, at the
// commit it already records — so it is rebuilt and its record left alone. Writing this commit as
// its source would make it permanently behind a repository its manifest does not come from.
var from, packaging []inventory.Entry
already := 0
for _, e := range entries {
switch {
case sourceIs(e.Source, m):
if e.Source.BuiltFrom == m.Commit {
already++
continue
}
// **A merge older than the last look at the source is history, not a move.** The forge
// announces what it finds merged, and an old merge surfacing late would otherwise move
// the recorded head backwards and rebuild everything built from that repository, once
// per old merge (2026-09-28).
if isHistory(m.MergedAt, e.Source.Seen) {
continue
}
from = append(from, e)
case readsFrom(read[e.Manifest.Module], m):
packaging = append(packaging, e)
}
}
if len(from) == 0 && len(packaging) == 0 {
// "Already built from it" and "nothing reads it" are different facts, and reading the first
// as the second sends somebody looking for a broken trigger when the mesh is up to date.
if already > 0 {
fmt.Printf("%s/%s merged into %s (%.8s); %d module(s) the mesh holds are already built "+
"from it\n", m.Owner, m.Repo, m.Base, m.Commit, already)
return nil
}
fmt.Printf("%s/%s merged into %s (%.8s); nothing the mesh holds reads it\n",
m.Owner, m.Repo, m.Base, m.Commit)
return nil
}
// The same judgement for the packaging kind, against the newest look at that repository by
// anything built from it: they keep no record of it themselves, and a replayed old merge should
// not rebuild them either.
if isHistory(m.MergedAt, lastLookAt(entries, m)) {
packaging = nil
}
touched := whatTheMergeTouched(from, entries, m)
for _, e := range touched {
if err := inv.SourceMoved(ctx, e.Manifest.Module, m.Commit); err != nil {
return notNow(err)
}
}
moved := append(append([]inventory.Entry{}, touched...), packaging...)
if len(moved) == 0 {
fmt.Printf("%s/%s merged into %s (%.8s); it changed nothing any module the mesh holds is "+
"built from\n", m.Owner, m.Repo, m.Base, m.Commit)
return nil
}
against, err := inv.BuiltAgainst(ctx)
if err != nil {
return notNow(err)
}
ordered := orderByBases(moved, against)
names := make([]string, 0, len(ordered))
for _, e := range ordered {
names = append(names, e.Manifest.Module)
}
fmt.Printf("%s/%s merged into %s (%.8s); building %s\n",
m.Owner, m.Repo, m.Base, m.Commit, strings.Join(names, ", "))
if len(packaging) > 0 {
var also []string
for _, e := range packaging {
also = append(also, e.Manifest.Module)
}
fmt.Printf(" %s package source from it, so they are rebuilt and their own source record "+
"is left where it is\n", strings.Join(also, ", "))
}
var failed []string
for _, e := range ordered {
source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
if err := buildOne(ctx, source, e.Source.Path, e.Source.Ref, 20*time.Minute); err != nil {
fmt.Printf(" %s: %v\n", e.Manifest.Module, err)
failed = append(failed, e.Manifest.Module)
// A base that failed is a reason to stop: what stands on it would be built against
// the old one, and report success (novox/hq 04-ISSUES/131).
if standsOn(ordered, e.Manifest.Module, against) {
fmt.Printf(" stopping: %s is a base of what was still to build\n", e.Manifest.Module)
break
}
}
}
if len(failed) > 0 {
fmt.Printf("%d of %d not built: %s\n", len(failed), len(ordered), strings.Join(failed, ", "))
}
return nil
}
// sourceIs is whether a recorded source is the repository and branch a merge announced. A source on
// the git seat is recorded as its path on the forge; one elsewhere as the URL it was cloned from.
// An empty recorded ref is the repository's default branch, which is what a merge into the base
// branch of the forge's default means.
func sourceIs(s inventory.Source, m link.SourceMoved) bool {
if !sameRepository(s.Repository, m) {
return false
}
return s.Ref == "" || s.Ref == m.Base
}
// sameRepository is whether a recorded repository is the one a merge names, in either spelling it
// may have been recorded in: a path on the git seat, or the URL it was cloned from.
func sameRepository(repository string, m link.SourceMoved) bool {
want := strings.ToLower(m.Owner + "/" + m.Repo)
repo := strings.ToLower(strings.TrimSuffix(repository, ".git"))
return repo == want || strings.HasSuffix(repo, "/"+want) ||
(m.CloneURL != "" && repo == strings.ToLower(strings.TrimSuffix(m.CloneURL, ".git")))
}
// readsFrom is whether a module's build read the repository a merge names: the second repository its
// recipe packages source from. Its ref must be the branch that moved, or unset — the same rule a
// module's own source follows.
func readsFrom(read []inventory.ReadRepository, m link.SourceMoved) bool {
for _, r := range read {
if sameRepository(r.Repository, m) && (r.Ref == "" || r.Ref == m.Base) {
return true
}
}
return false
}
// lastLookAt is the most recent look at this repository by anything built from it.
func lastLookAt(entries []inventory.Entry, m link.SourceMoved) time.Time {
var newest time.Time
for _, e := range entries {
if sameRepository(e.Source.Repository, m) && e.Source.Seen.After(newest) {
newest = e.Source.Seen
}
}
return newest
}
// whatTheMergeTouched narrows the modules built from a repository to the ones the merge changed.
//
// **A change inside no module's own directory is a change to what they share.** The forge lists the
// files a merge changed; a module is affected when one of them is inside its own directory, when it
// is built from the repository's root — everything there is its source — or when some changed file
// belongs to no module's directory at all, which is how a shared file, a build recipe or a
// dependency at the root rebuilds everything built from that repository.
//
// A change inside *another* module's directory is that module's business and not this one's, even
// when the mesh does not hold that module: `known` is every module this repository is known to hold,
// whatever branch it was registered from. That is also the limit of this — a repository whose shared
// code sits inside a directory the mesh has never seen a module in reads as shared, and everything
// is rebuilt. Rebuilding too much is the safe direction: the fault this whole path exists for is a
// mesh that believes it is current and is not (novox/hq 04-ISSUES/131).
func whatTheMergeTouched(candidates, known []inventory.Entry, m link.SourceMoved) []inventory.Entry {
// Nothing said about the files, or not all of them said: everything built from it is affected.
if len(m.Paths) == 0 || m.PathsTruncated {
return candidates
}
var dirs []string
for _, e := range known {
if e.Source.Path != "" && sameRepository(e.Source.Repository, m) {
dirs = append(dirs, e.Source.Path)
}
}
for _, p := range m.Paths {
if !insideAny(p, dirs) {
return candidates
}
}
var out []inventory.Entry
for _, e := range candidates {
if e.Source.Path == "" || anyInside(m.Paths, e.Source.Path) {
out = append(out, e)
}
}
return out
}
// inside is whether a changed file is in a directory: that directory itself, or under it.
func inside(path, dir string) bool {
dir = strings.Trim(dir, "/")
path = strings.TrimPrefix(path, "/")
return path == dir || strings.HasPrefix(path, dir+"/")
}
// insideAny is whether a changed file is in any of these directories.
func insideAny(path string, dirs []string) bool {
for _, dir := range dirs {
if inside(path, dir) {
return true
}
}
return false
}
// anyInside is whether any of these changed files is in a directory.
func anyInside(paths []string, dir string) bool {
for _, p := range paths {
if inside(p, dir) {
return true
}
}
return false
}
// orderByBases is the entries with every base before what stands on it: a module whose build stood
// on another's artifact comes after that module. Entries outside the set are not waited for — they
// are not being rebuilt. Stable for what has no order between it.
//
// `against` is what each module's newest build stood on (inventory.BuiltAgainst): the edges are
// derived from builds, not declared, because a recorded manifest no longer carries `build.on`.
func orderByBases(entries []inventory.Entry, against map[string][]string) []inventory.Entry {
inSet := map[string]bool{}
for _, e := range entries {
inSet[e.Manifest.Module] = true
}
var out []inventory.Entry
placed := map[string]bool{}
var place func(e inventory.Entry, seen map[string]bool)
place = func(e inventory.Entry, seen map[string]bool) {
name := e.Manifest.Module
if placed[name] || seen[name] {
return
}
seen[name] = true
for _, base := range entries {
if base.Manifest.Module != name && inSet[base.Manifest.Module] && standsOnModule(e, base.Manifest.Module, against) {
place(base, seen)
}
}
placed[name] = true
out = append(out, e)
}
for _, e := range entries {
place(e, map[string]bool{})
}
return out
}
// standsOn is whether anything in the set is built on the named module's artifacts.
func standsOn(entries []inventory.Entry, module string, against map[string][]string) bool {
for _, e := range entries {
if standsOnModule(e, module, against) {
return true
}
}
return false
}
// standsOnModule is whether an entry's build stood on the named module: by what its newest build
// recorded it was handed (`artifact-store://<module>/<artifact>@…`, the module's own artifact), or
// — for a module registered from a manifest and not yet built — by the base its manifest names.
func standsOnModule(e inventory.Entry, module string, against map[string][]string) bool {
if e.Manifest.Module == module {
return false
}
if e.Manifest.Build != nil {
for _, on := range e.Manifest.Build.On {
if on.Module == module {
return true
}
}
}
prefix := catalogue.ArtifactStoreScheme + module + "/"
for _, ref := range against[e.Manifest.Module] {
if strings.HasPrefix(ref, prefix) {
return true
}
}
return false
}
// isHistory is whether a merge made at mergedAt predates the last time the source was seen. A merge
// with no time on it is taken as news: refusing it would silence a forge that says less.
func isHistory(mergedAt string, seen time.Time) bool {
if mergedAt == "" || seen.IsZero() {
return false
}
at, err := time.Parse(time.RFC3339, mergedAt)
if err != nil {
return false
}
return at.Before(seen)
}
-109
View File
@@ -1,109 +0,0 @@
package main
// The challenge path falls through for real. autocert's own HTTPHandler answers 404 itself for a
// token it does not hold and never consults its fallback on the challenge path — the
// predecessor's fault, the edge owning /.well-known/acme-challenge outright, rediscovered live
// when Mailu's renewal died behind this proxy on cutover day (2026-09-26). These tests pin the
// three behaviours tokenOrRoute exists for.
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"golang.org/x/crypto/acme/autocert"
)
func routedTo(t *testing.T, marker string) http.Handler {
t.Helper()
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
if _, err := w.Write([]byte(marker)); err != nil {
t.Fatal(err)
}
})
}
func TestATokenNoAuthorityHoldsIsRoutedNot404d(t *testing.T) {
m := &autocert.Manager{Prompt: autocert.AcceptTOS, Cache: autocert.DirCache(t.TempDir())}
h := tokenOrRoute(routedTo(t, "the workload answered"), m)
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest("GET", "http://mail.example/.well-known/acme-challenge/somebody-elses-token", nil))
if rec.Code != http.StatusOK || rec.Body.String() != "the workload answered" {
t.Fatalf("a token no authority holds must reach plain routing; got %d %q", rec.Code, rec.Body.String())
}
}
func TestATokenAManagerHoldsIsAnsweredByIt(t *testing.T) {
// autocert reads a token it does not have in memory from its cache, under "<token>+http-01" —
// which is also how a token would survive the manager restarting mid-issuance.
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "held-token+http-01"), []byte("the-key-authorization"), 0o600); err != nil {
t.Fatal(err)
}
m := &autocert.Manager{Prompt: autocert.AcceptTOS, Cache: autocert.DirCache(dir)}
h := tokenOrRoute(routedTo(t, "must not be reached"), m)
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest("GET", "http://mail.example/.well-known/acme-challenge/held-token", nil))
if rec.Code != http.StatusOK || rec.Body.String() != "the-key-authorization" {
t.Fatalf("the manager holding a token answers it; got %d %q", rec.Code, rec.Body.String())
}
}
func TestASecondAuthorityIsProbedBeforeRouting(t *testing.T) {
first := &autocert.Manager{Prompt: autocert.AcceptTOS, Cache: autocert.DirCache(t.TempDir())}
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "internal-token+http-01"), []byte("internal-key"), 0o600); err != nil {
t.Fatal(err)
}
second := &autocert.Manager{Prompt: autocert.AcceptTOS, Cache: autocert.DirCache(dir)}
h := tokenOrRoute(routedTo(t, "must not be reached"), first, second)
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest("GET", "http://git.internal/.well-known/acme-challenge/internal-token", nil))
if rec.Code != http.StatusOK || rec.Body.String() != "internal-key" {
t.Fatalf("the second authority's token is found by probing past the first; got %d %q", rec.Code, rec.Body.String())
}
}
func TestAnAuthorityWhosePolicyRefusesTheNameIsProbedPast(t *testing.T) {
// autocert checks the host policy before the token and answers 403 — the internal authority
// does this for every public name. A policy refusal is as much "not mine" as a missing token:
// the request must still reach plain routing, where the workload's own ACME client answers.
refusing := &autocert.Manager{
Prompt: autocert.AcceptTOS,
Cache: autocert.DirCache(t.TempDir()),
HostPolicy: func(ctx context.Context, host string) error {
return fmt.Errorf("no internal-only route for %q in this mesh", host)
},
}
h := tokenOrRoute(routedTo(t, "the workload answered"), refusing)
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest("GET", "http://mail.example/.well-known/acme-challenge/mailus-token", nil))
if rec.Code != http.StatusOK || rec.Body.String() != "the workload answered" {
t.Fatalf("a policy refusal must fall through to routing; got %d %q", rec.Code, rec.Body.String())
}
}
func TestAnOrdinaryPathNeverTouchesTheChallengeMachinery(t *testing.T) {
m := &autocert.Manager{Prompt: autocert.AcceptTOS, Cache: autocert.DirCache(t.TempDir())}
h := tokenOrRoute(routedTo(t, "routed"), m)
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest("GET", "http://site.example/index.html", nil))
if rec.Code != http.StatusOK || rec.Body.String() != "routed" {
t.Fatalf("an ordinary path goes straight to routing; got %d %q", rec.Code, rec.Body.String())
}
}
+73 -624
View File
@@ -10,31 +10,10 @@
// program. What lives here is that contract, written as something that runs so it can be read
// rather than described.
//
// **A route also carries what a request arriving at it may do** (novox/hq ADR 0108). The grant used
// to say only where to send traffic, so this proxy applied nothing; the four things the ingress it
// replaces actually relies on are now part of the contribution. The set is closed at four, because
// an open middleware surface recreates the thing being replaced and is far harder to narrow later
// than a closed one is to widen.
//
// What it is given, written by the host from an ordinary declaration:
//
// $ROUTES every consumer, the name it asked for, and where the mesh says that machine is
//
// Each contribution's values carry the name and port as before, and optionally:
//
// path the path prefix this rule is scoped to; absent means every path
// priority which rule wins where two match; higher first, and the order is total
// deny refuse the request outright — the shape an incident mitigation needs
// redirect answer with a permanent redirect to this name, keeping the path and query
// auth the *path of a secret* holding `user:hash` lines, never the credential itself
//
// A host may appear more than once, which is what path scoping means: one rule refusing a path
// while another serves everything else on the same name.
//
// **`auth` names a secret and never holds one.** A declaration carrying a credential is refused
// outright rather than served unprotected, and a secret that cannot be read makes the route refuse
// rather than open — a gate that cannot check is not a gate that opens.
//
// It re-reads on change rather than being restarted, for the same reason the provisioner does:
// a route arriving or leaving is an ordinary event and must not drop the connections of every
// other workload.
@@ -44,7 +23,6 @@ import (
"bytes"
"context"
"crypto/sha256"
"crypto/subtle"
"crypto/tls"
"crypto/x509"
"encoding/hex"
@@ -64,7 +42,6 @@ import (
"golang.org/x/crypto/acme"
"golang.org/x/crypto/acme/autocert"
"golang.org/x/crypto/bcrypt"
)
// Where public certificates come from when nothing says otherwise.
@@ -97,23 +74,10 @@ func issuer() string {
// to what it may serve.
func onlyWhatTheMeshSaid(held *table) autocert.HostPolicy {
return func(_ context.Context, host string) error {
if held.eligibleForACME(host) {
if _, known := held.find(host); known {
return nil
}
return fmt.Errorf("no public route for %q in this mesh, so no certificate is asked for", host)
}
}
// onlyInternalNamesTheMeshSaid is onlyWhatTheMeshSaid's mirror for the internal authority — the
// same quota-spending concern applies even to an authority with no rate limit of its own, because
// an order for a name this proxy does not actually route is a bug worth refusing rather than
// serving.
func onlyInternalNamesTheMeshSaid(held *table) autocert.HostPolicy {
return func(_ context.Context, host string) error {
if held.eligibleForInternalACME(host) {
return nil
}
return fmt.Errorf("no internal-only route for %q in this mesh, so no certificate is asked for", host)
return fmt.Errorf("no route for %q in this mesh, so no certificate is asked for", host)
}
}
@@ -131,194 +95,48 @@ type contribution struct {
Values map[string]any `json:"values"`
}
// policy is what a rule does with a request that matched it.
//
// **Decided by the mesh, not here** (novox/hq ADR 0108). A route grant used to hand back a name and
// say nothing about what the name admitted, so this proxy admitted everything. The set is closed at
// four — authentication, refusal, path scoping, redirect — because an open middleware surface
// recreates the thing being replaced and is far harder to narrow later than a closed one is to widen.
type policy struct {
// deny refuses the request outright, whatever it is.
deny bool
// redirectTo answers with a permanent redirect instead of proxying. The request's own path and
// query are carried across, which is what canonicalising one public name onto another means.
redirectTo string
// users is what a request must present, read at load time from the secret the declaration
// *named*. A declaration never carries the credential itself.
users map[string]string
// sealed is set when authentication was declared and the secret could not be read. The rule then
// refuses everything and says why.
//
// **Fail closed.** The alternative — serve the route unauthenticated because the gate is
// missing — turns an unreadable file into a silently public admin surface, which is the exact
// outcome ADR 0108 exists to prevent. A gate that cannot check is not a gate that opens.
sealed string
}
// rule is one way a host may be routed. A host may have several, which is what path scoping means.
type rule struct {
path string // "" matches every path
priority int
policy policy
to *httputil.ReverseProxy
target string
// insecure skips certificate verification when target is reached over https. For a backend
// that terminates TLS with its own certificate this proxy has no reason to trust — Mailu's
// webmail front is the first of these — never for anything reached over plain http, where
// there is nothing to verify in the first place.
insecure bool
// maxRequestBody is the largest body, in bytes, this route carries. Zero is no limit, which is
// what every route gets by saying nothing: this proxy has never limited a body, and a default
// arriving with the field would change every route that never asked for one.
//
// **Configuration, not a policy** (novox/hq ADR 0108 closed that set at four). It belongs beside
// `insecure` for the same reason `insecure` is not a policy: both tune how this proxy carries a
// request to a backend, rather than deciding what the name admits or who may reach it. A
// registry is the case that needs it — image layers arrive as single requests of gigabytes, and
// a proxy's own default refuses them long before the workload is reached.
maxRequestBody int64
}
// table is what the proxy is currently serving, replaced whole whenever the file changes.
//
// Replaced rather than merged: the file is the whole truth about who has a route, so merging
// would keep serving a name whose module was unassigned — which is the stale-route fault
// 08-connectivity lists as open, reintroduced one level down.
//
// Keyed by host to an *ordered* list rather than to one target, because two of the four policies
// need a single host routed more than one way: a refusal on a path the ordinary route also matches,
// and a certificate-challenge path on a host that otherwise serves a workload.
type table struct {
mu sync.RWMutex
to map[string][]rule
// public is which routed hosts are eligible for a real certificate — every host reached as a
// route's own `name`, never one reached only as its `internal-name`. A private alias can never
// pass ACME's own validation (it has no public DNS to prove it against), so asking for it is
// not merely pointless but the failing order onlyWhatTheMeshSaid exists to prevent.
public map[string]bool
mu sync.RWMutex
to map[string]*httputil.ReverseProxy
targets map[string]string
}
func (t *table) set(routes map[string][]rule, public map[string]bool) {
made := map[string][]rule{}
for host, rules := range routes {
kept := make([]rule, 0, len(rules))
for _, r := range rules {
// A rule that only refuses or only redirects has nowhere to send anything, and needs
// nowhere: it answers by itself.
if r.policy.deny || r.policy.redirectTo != "" {
kept = append(kept, r)
continue
}
where, err := url.Parse(r.target)
if err != nil {
log.Printf("route %s points at %q, which is not a URL: %v", host, r.target, err)
continue
}
r.to = httputil.NewSingleHostReverseProxy(where)
if r.insecure {
r.to.Transport = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
}
kept = append(kept, r)
}
if len(kept) == 0 {
func (t *table) set(routes map[string]string) {
made := map[string]*httputil.ReverseProxy{}
for name, target := range routes {
where, err := url.Parse(target)
if err != nil {
log.Printf("route %s points at %q, which is not a URL: %v", name, target, err)
continue
}
inOrder(kept)
made[host] = kept
made[name] = httputil.NewSingleHostReverseProxy(where)
}
t.mu.Lock()
t.to = made
t.public = public
t.to, t.targets = made, routes
t.mu.Unlock()
}
// inOrder puts the rules for one host into the order they are matched in, and does so totally.
//
// **Equal priorities must resolve identically every time** (ADR 0108). Sorting only by priority
// leaves rules that share one in whatever order the map produced, so the same declaration would
// serve differently between restarts — a proxy that is not reproducible. Longest path first within a
// priority is also the intuitive reading: the more specific rule wins. The last two keys exist only
// to make the order total.
func inOrder(rules []rule) {
sort.SliceStable(rules, func(i, j int) bool {
a, b := rules[i], rules[j]
if a.priority != b.priority {
return a.priority > b.priority
}
if len(a.path) != len(b.path) {
return len(a.path) > len(b.path)
}
if a.path != b.path {
return a.path < b.path
}
return a.target < b.target
})
}
// find is the rule that answers this request, or nothing if the host is not routed here at all.
func (t *table) find(host, path string) (rule, bool) {
t.mu.RLock()
defer t.mu.RUnlock()
for _, r := range t.to[bareHost(host)] {
if r.path == "" || strings.HasPrefix(path, r.path) {
return r, true
}
}
return rule{}, false
}
// routed says whether this proxy serves the name at all, whatever the path.
//
// Separate from find because certificate issuance is a question about the *name*: a host whose only
// rules are path-scoped is still a name this proxy answers to, and still needs a certificate.
// eligibleForACME says whether this proxy may ask a certificate authority for this name — every
// host reached as a route's own public `name`, never one reached only as its `internal-name`
// alias, which no public CA can ever validate.
func (t *table) eligibleForACME(host string) bool {
t.mu.RLock()
defer t.mu.RUnlock()
bare := bareHost(host)
return len(t.to[bare]) > 0 && t.public[bare]
}
func (t *table) routed(host string) bool {
t.mu.RLock()
defer t.mu.RUnlock()
return len(t.to[bareHost(host)]) > 0
}
// eligibleForInternalACME says whether this proxy may ask its *internal* authority for a
// certificate for this name — every host it routes that is not also a route's public `name`.
//
// **The mesh has two name spaces and two authorities** (novox/hq 03-DESIGN/01-to-be/08-connectivity
// §2): a public name is certified by a public CA, an internal one by the mesh's own. This is
// composed only from `to` and `public`, which routesFrom already builds correctly — a host never
// lands in both a route's own `name` and only its `internal-name`, so nothing new has to be
// tracked to tell the two apart.
func (t *table) eligibleForInternalACME(host string) bool {
t.mu.RLock()
defer t.mu.RUnlock()
bare := bareHost(host)
return len(t.to[bare]) > 0 && !t.public[bare]
}
// bareHost is the name without the port, lower-cased.
//
// The port is not part of the name: a request to app.example:8080 is for app.example. Lower-cased
// because a Host header is not case-sensitive, and a route that only answers the spelling in the
// manifest answers half the requests made to it.
func bareHost(host string) string {
func (t *table) find(host string) (*httputil.ReverseProxy, bool) {
// The port is not part of the name. A request to app.example:8080 is for app.example.
if h, _, err := net.SplitHostPort(host); err == nil {
host = h
}
return strings.ToLower(host)
t.mu.RLock()
defer t.mu.RUnlock()
p, ok := t.to[strings.ToLower(host)]
return p, ok
}
func (t *table) names() []string {
t.mu.RLock()
defer t.mu.RUnlock()
out := make([]string, 0, len(t.to))
for name := range t.to {
out := make([]string, 0, len(t.targets))
for name := range t.targets {
out = append(out, name)
}
sort.Strings(out)
@@ -344,7 +162,7 @@ func run() error {
held := newTable()
read := func() {
routes, public, err := routesFrom(path)
routes, err := routesFrom(path)
if err != nil {
// Kept serving what it had. A file being rewritten is momentarily unreadable, and
// dropping every route because one read landed mid-write would turn an ordinary
@@ -352,7 +170,7 @@ func run() error {
log.Printf("cannot read %s, keeping what is already served: %v", path, err)
return
}
held.set(routes, public)
held.set(routes)
log.Printf("serving %d route(s): %s", len(routes), strings.Join(held.names(), ", "))
}
read()
@@ -379,158 +197,16 @@ func run() error {
return fmt.Errorf("TLS_LISTEN is set and ACME_CACHE is not: certificates need somewhere " +
"to persist, or every restart orders them again")
}
publicManager, err := newManager(cache, issuer(), strings.TrimSpace(os.Getenv("ACME_CA_BUNDLE")),
onlyWhatTheMeshSaid(held))
if err != nil {
return err
}
log.Printf("issuing public certificates from %s, for whatever the mesh routes here", issuer())
// The internal authority is optional: unset means this proxy serves internal-only aliases over
// plain HTTP exactly as it always has, which is the standalone-binary default and a safe one —
// it asks nothing of an authority it was not told about.
var internalManager *autocert.Manager
if directory := strings.TrimSpace(os.Getenv("INTERNAL_ACME_DIRECTORY")); directory != "" {
internalManager, err = newManager(cache, directory, strings.TrimSpace(os.Getenv("INTERNAL_ACME_CA_BUNDLE")),
onlyInternalNamesTheMeshSaid(held))
if err != nil {
return fmt.Errorf("internal certificate authority: %w", err)
}
log.Printf("issuing internal certificates from %s, for every internal-only alias this routes",
directory)
}
// Port 80 answers the HTTP-01 challenge and goes on proxying everything else. The challenge
// must be answered *at the name being certified*, which is why issuance happens on the node
// that is publicly reachable rather than wherever the workload runs.
//
// **autocert's own HTTPHandler does not fall through on the challenge path.** For a token it
// does not hold it answers 404 itself; its fallback only ever sees non-challenge paths — which
// is exactly the predecessor's fault, the edge owning `/.well-known/acme-challenge` outright,
// rediscovered live when Mailu's renewal died behind this proxy on cutover day. tokenOrRoute
// probes each manager and hands a token neither authority recognises to plain routing, which
// is what lets a consumer's own ACME client — Mailu's, certifying its own name for a protocol
// this proxy never proxies — answer its own challenge through an ordinary path-scoped route.
port80 := tokenOrRoute(handler(held), publicManager)
if internalManager != nil {
port80 = tokenOrRoute(handler(held), publicManager, internalManager)
}
go func() {
if err := http.ListenAndServe(listen, port80); err != nil {
log.Printf("plain HTTP stopped: %v", err)
}
}()
tlsConfig := publicManager.TLSConfig()
if internalManager != nil {
// Dispatched by which authority may certify this name at all — the same question
// eligibleForInternalACME already answers, asked once more at handshake time rather than
// only when an order is placed, since a cached certificate is served here on every request
// and never goes through HostPolicy again.
fromPublic, fromInternal := tlsConfig.GetCertificate, internalManager.TLSConfig().GetCertificate
tlsConfig.GetCertificate = func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
if held.eligibleForInternalACME(hello.ServerName) {
return fromInternal(hello)
}
return fromPublic(hello)
}
}
server := &http.Server{
Addr: secure,
Handler: handler(held),
TLSConfig: tlsConfig,
}
return server.ListenAndServeTLS("", "")
}
// tokenOrRoute serves port 80: each manager answers the challenge tokens it is itself holding,
// and a token none of them holds is routed like any other request instead of being 404'd at the
// edge.
//
// autocert gives no way to ask "is this your token?" — its HTTPHandler both answers and refuses —
// so each manager is probed against a buffered writer and its refusal (404 on the challenge path)
// is discarded in favour of the next candidate. The probe is cheap: the handler answers from
// memory, and the path only carries traffic while an issuance is actually running.
func tokenOrRoute(routes http.Handler, managers ...*autocert.Manager) http.Handler {
const challengePrefix = "/.well-known/acme-challenge/"
// Non-challenge paths never reach a manager at all; autocert's tryHTTP01 switch still has to
// be armed, which HTTPHandler is the only exported way to do.
probes := make([]http.Handler, len(managers))
for i, m := range managers {
probes[i] = m.HTTPHandler(routes)
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !strings.HasPrefix(r.URL.Path, challengePrefix) {
routes.ServeHTTP(w, r)
return
}
for _, probe := range probes {
buffered := &probedResponse{header: make(http.Header)}
probe.ServeHTTP(buffered, r)
// Two shapes of "not mine": 404, a token this manager is not holding — and 403, a
// name its host policy would never certify at all (autocert checks the policy before
// the token, so the internal authority answers 403 for every public name).
if buffered.status == http.StatusNotFound || buffered.status == http.StatusForbidden {
continue
}
buffered.replayTo(w)
return
}
routes.ServeHTTP(w, r) // no authority holds it: the workload behind a routed path may
})
}
// probedResponse buffers one handler's answer so a refusal can be discarded unseen.
type probedResponse struct {
header http.Header
status int
body bytes.Buffer
}
func (p *probedResponse) Header() http.Header { return p.header }
func (p *probedResponse) WriteHeader(status int) {
if p.status == 0 {
p.status = status
}
}
func (p *probedResponse) Write(b []byte) (int, error) {
if p.status == 0 {
p.status = http.StatusOK
}
return p.body.Write(b)
}
func (p *probedResponse) replayTo(w http.ResponseWriter) {
for k, vs := range p.header {
for _, v := range vs {
w.Header().Add(k, v)
}
}
status := p.status
if status == 0 {
status = http.StatusOK
}
w.WriteHeader(status)
_, _ = w.Write(p.body.Bytes())
}
// newManager is one ACME authority's autocert manager: where to ask, what to trust it with, and
// which names it may be asked to certify.
//
// **Trusting an authority names a file rather than skipping verification.** An issuer that is not
// one of the public ones — the lab's, or the mesh's own step-ca — serves its own ACME API over TLS
// with a certificate nothing trusts yet. *Skip* would also apply the day this points at a public
// issuer, and nothing would say so; naming a bundle is a deliberate, visible act instead.
func newManager(cache, directory, bundle string, policy autocert.HostPolicy) (*autocert.Manager, error) {
client := &acme.Client{DirectoryURL: directory}
client := &acme.Client{DirectoryURL: issuer()}
// An issuer that is not one of the public ones serves its own API over TLS with a certificate
// nothing trusts yet — the lab's, or an internal step-ca. Trusting it is a deliberate act and
// names a file, rather than the client being told to skip verification: *skip* would also
// apply on the day this points at a public issuer, and nothing would say so.
var root []byte
if bundle != "" {
if bundle := strings.TrimSpace(os.Getenv("ACME_CA_BUNDLE")); bundle != "" {
read, err := os.ReadFile(bundle)
if err != nil {
return nil, fmt.Errorf("the CA bundle names %s and it cannot be read: %w", bundle, err)
return fmt.Errorf("ACME_CA_BUNDLE names %s and it cannot be read: %w", bundle, err)
}
root = read
// An empty bundle means the issuer's root is already in the system trust store — a public
@@ -542,7 +218,7 @@ func newManager(cache, directory, bundle string, policy autocert.HostPolicy) (*a
if strings.TrimSpace(string(root)) != "" {
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(root) {
return nil, fmt.Errorf("%s holds no certificate this can trust", bundle)
return fmt.Errorf("%s holds no certificate this can trust", bundle)
}
client.HTTPClient = &http.Client{
Timeout: 30 * time.Second,
@@ -551,16 +227,31 @@ func newManager(cache, directory, bundle string, policy autocert.HostPolicy) (*a
}
}
// Where this authority's account and certificates are kept. Per authority, not per proxy — see
// forThisAuthority, which is what makes a re-initialised CA heal itself, and what lets the
// public and internal authorities share one ACME_CACHE without colliding: they hash to
// different names because their directories differ.
mine := forThisAuthority(cache, directory, root)
return &autocert.Manager{
// forThisAuthority, which is what makes a re-initialised CA heal itself.
mine := forThisAuthority(cache, issuer(), root)
manager := &autocert.Manager{
Cache: autocert.DirCache(mine),
Prompt: autocert.AcceptTOS,
HostPolicy: policy,
HostPolicy: onlyWhatTheMeshSaid(held),
Client: client,
}, nil
}
log.Printf("issuing from %s into %s, for whatever the mesh routes here", issuer(), mine)
// Port 80 answers the HTTP-01 challenge and goes on proxying everything else. The challenge
// must be answered *at the name being certified*, which is why issuance happens on the node
// that is publicly reachable rather than wherever the workload runs.
go func() {
if err := http.ListenAndServe(listen, manager.HTTPHandler(handler(held))); err != nil {
log.Printf("plain HTTP stopped: %v", err)
}
}()
server := &http.Server{
Addr: secure,
Handler: handler(held),
TLSConfig: manager.TLSConfig(),
}
return server.ListenAndServeTLS("", "")
}
// forThisAuthority is where one ACME authority's account and certificates are kept.
@@ -594,303 +285,61 @@ func forThisAuthority(cache, directory string, root []byte) string {
// newTable is an empty routing table.
func newTable() *table {
return &table{to: map[string][]rule{}}
return &table{to: map[string]*httputil.ReverseProxy{}, targets: map[string]string{}}
}
// handler is the proxy itself, separated so it can be driven by a test without a listener.
func handler(held *table) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
matched, known := held.find(r.Host, r.URL.Path)
proxy, known := held.find(r.Host)
if !known {
// **Named, not a bare 404.** A route that was withdrawn and a name that never existed
// are different things, and a proxy that says only "not found" makes an operator go
// and read the mesh to tell them apart. What it is serving is the answer to both.
//
// And since a host may now be routed only on some paths, those are a third thing:
// saying "no route for this name" while listing that very name as served is a
// contradiction an operator would have to disbelieve the proxy to get past.
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.WriteHeader(http.StatusNotFound)
if held.routed(r.Host) {
fmt.Fprintf(w, "%s is served here, but no route covers %q.\n",
bareHost(r.Host), r.URL.Path)
return
}
fmt.Fprintf(w, "no route for %q in this mesh.\nserving: %s\n",
r.Host, strings.Join(held.names(), ", "))
return
}
switch {
case matched.policy.sealed != "":
// Declared a gate, cannot check it. Refused, and says why — an operator reading this
// learns the secret is missing, rather than wondering why a protected name is 503.
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.WriteHeader(http.StatusServiceUnavailable)
fmt.Fprintf(w, "this route requires authentication and its credentials cannot be read: %s\n",
matched.policy.sealed)
return
case matched.policy.deny:
http.Error(w, "this path is not served to you", http.StatusForbidden)
return
case matched.policy.redirectTo != "":
http.Redirect(w, r, canonical(matched.policy.redirectTo, r.URL), http.StatusMovedPermanently)
return
case len(matched.policy.users) > 0 && !allowed(matched.policy.users, r):
// The realm is the name asked for, so a browser's prompt says which route it is for.
w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q, charset=\"UTF-8\"", bareHost(r.Host)))
http.Error(w, "unauthorized", http.StatusUnauthorized)
return
}
if matched.maxRequestBody > 0 {
// Refused on the declared length where there is one, so an upload that cannot succeed
// is answered before it is carried; and capped while reading for a chunked body, which
// declares no length at all. Without the second, a limit is advice.
if r.ContentLength > matched.maxRequestBody {
http.Error(w, fmt.Sprintf("request body too large for this route: %d bytes is the most it carries",
matched.maxRequestBody), http.StatusRequestEntityTooLarge)
return
}
r.Body = http.MaxBytesReader(w, r.Body, matched.maxRequestBody)
}
matched.to.ServeHTTP(w, r)
proxy.ServeHTTP(w, r)
})
}
// canonical is where a redirect sends this request.
//
// The declaration names the destination *name*; the request keeps its own path and query. That is
// what canonicalising one public name onto another means — a link to a page under the old name has
// to arrive at the same page under the new one, or the redirect silently loses every deep link.
func canonical(to string, from *url.URL) string {
where, err := url.Parse(to)
if err != nil {
return to
}
if where.Path == "" || where.Path == "/" {
where.Path = from.Path
}
if where.RawQuery == "" {
where.RawQuery = from.RawQuery
}
return where.String()
}
// allowed says whether the request presented credentials this route accepts.
//
// **Every path costs one bcrypt comparison**, including an unknown user, which is why the miss
// compares against a fixed hash rather than returning early. Returning early would make an unknown
// user measurably faster than a known one with a wrong password, and that difference is a way to
// enumerate the users of a route from outside it.
func allowed(users map[string]string, r *http.Request) bool {
// A hash of nothing anybody knows. Its only job is to cost what a real comparison costs.
const absent = "$2a$10$N9qo8uLOickgx2ZMRZoMyeIjZAgcfl7p92ldGxad68LJZdL17lhWy"
user, password, ok := r.BasicAuth()
if !ok {
return false
}
want, known := users[user]
if !known {
want = absent
}
if err := bcrypt.CompareHashAndPassword([]byte(want), []byte(password)); err != nil {
return false
}
// `known` is checked after the comparison, not instead of it, so the timing is the same either
// way. subtle.ConstantTimeByteEq keeps the branch from being the thing that differs.
return subtle.ConstantTimeByteEq(boolByte(known), 1) == 1
}
func boolByte(b bool) byte {
if b {
return 1
}
return 0
}
// routesFrom reads what the mesh wrote and turns it into host → the rules for that host, and
// which of those hosts is a public name — the second is `name`, ACME-eligible; a host reached
// only through `internal-name` never appears there.
func routesFrom(path string) (map[string][]rule, map[string]bool, error) {
// routesFrom reads what the mesh wrote and turns it into name → target.
func routesFrom(path string) (map[string]string, error) {
raw, err := os.ReadFile(path)
if err != nil {
return nil, nil, err
return nil, err
}
var said given
if err := json.Unmarshal(raw, &said); err != nil {
return nil, nil, err
return nil, err
}
out := map[string][]rule{}
public := map[string]bool{}
out := map[string]string{}
for _, c := range said.Given {
name, _ := c.Values["name"].(string)
if name == "" {
log.Printf("%s on %s asked for a route and named nothing; skipped", c.From, c.Node)
continue
}
host := strings.ToLower(name)
public[host] = true
made := rule{path: asPath(c.Values["path"])}
if p, ok := asWhole(c.Values["priority"]); ok {
made.priority = p
}
made.policy.deny, _ = c.Values["deny"].(bool)
made.policy.redirectTo, _ = c.Values["redirect"].(string)
if named, carried := c.Values["auth"].(string); carried && strings.TrimSpace(named) != "" {
// **A declaration names a secret; it never holds one** (ADR 0108). Refused rather than
// tolerated, and the whole rule is dropped rather than served unprotected — the
// rejected option cannot come back by accident, which is the failure this check exists
// to make impossible.
if looksLikeACredential(named) {
log.Printf("%s on %s declared route %q with a credential in the declaration rather "+
"than the name of a secret; the whole route is refused (novox/hq ADR 0108)",
c.From, c.Node, name)
continue
}
users, err := usersFrom(named)
if err != nil {
// Fail closed: the rule is kept so the name stays routed and answers, and it
// answers by refusing. Dropping it instead would make the name 404 and read as a
// withdrawn route rather than an unreadable secret.
made.policy.sealed = err.Error()
}
made.policy.users = users
}
// Only a rule that actually proxies needs somewhere to send the request.
if !made.policy.deny && made.policy.redirectTo == "" {
port, ok := asPort(c.Values["port"])
if !ok {
log.Printf("%s on %s asked for route %q and gave no usable port; skipped",
c.From, c.Node, name)
continue
}
// Where the mesh says that machine is. Empty means it is this one — a workload beside
// the proxy is ordinary, and reaching it over loopback is both correct and the only
// thing that works when there is no private network.
at := c.At
if at == "" {
at = "127.0.0.1"
}
// http unless the contribution says otherwise. A backend that terminates its own TLS
// with a certificate this proxy has no reason to trust — Mailu's webmail front is the
// first of these — is the reason `insecure` exists, and it stays the exception: every
// other target the mesh hands this proxy is a plain workload on the private network.
scheme, _ := c.Values["scheme"].(string)
scheme = strings.ToLower(strings.TrimSpace(scheme))
if scheme == "" {
scheme = "http"
}
if scheme != "http" && scheme != "https" {
log.Printf("%s on %s asked for route %q with scheme %q, which is neither http "+
"nor https; skipped", c.From, c.Node, name, scheme)
continue
}
made.insecure, _ = c.Values["insecure"].(bool)
// A limit this proxy cannot read is a route it does not serve, named like a port that
// is not a port. Serving it without the limit would carry exactly what the module said
// not to carry, and report success doing it.
if asked, said := c.Values["max-request-body"]; said {
bytes, whole := asWhole(asked)
if !whole || bytes <= 0 {
log.Printf("%s on %s asked for route %q with a max-request-body of %v, which is "+
"not a whole positive number of bytes; skipped", c.From, c.Node, name, asked)
continue
}
made.maxRequestBody = int64(bytes)
}
made.target = fmt.Sprintf("%s://%s:%d", scheme, at, port)
}
out[host] = append(out[host], made)
// The internal-network alias, the same rule under a second host — a predecessor proxy
// answered both for one route, as a convenience (reaching a service over the VPN without a
// public TLS round trip), not as an access boundary; composing it here restores exactly
// that, nothing more. Absent whenever the node composed no internal name (novox/hq ADR
// 0056's internalDomain half) — the same "nothing to join a label to" case the public name
// already has.
if internal, _ := c.Values["internal-name"].(string); strings.TrimSpace(internal) != "" {
out[strings.ToLower(internal)] = append(out[strings.ToLower(internal)], made)
}
}
return out, public, nil
}
// asWhole is any whole number the mesh wrote, whatever its magnitude.
//
// **Not asPort.** Priority was read with the port reader first, which caps at 65535 — so a rule
// declared at a priority above that silently became priority 0 and stopped shadowing the route it
// exists to shadow. The one real rule this has to reproduce is declared at 100000, so the bug was
// exactly load-bearing. A priority is an ordering, not a port: it has no range.
func asWhole(v any) (int, bool) {
switch n := v.(type) {
case float64:
// JSON makes a float of every number, so a non-integral one was not meant as a priority.
if n != float64(int(n)) {
return 0, false
}
return int(n), true
case int:
return n, true
}
return 0, false
}
// asPath is the path prefix a rule is scoped to, or "" for every path.
func asPath(v any) string {
p, _ := v.(string)
p = strings.TrimSpace(p)
if p == "" {
return ""
}
if !strings.HasPrefix(p, "/") {
p = "/" + p
}
return p
}
// looksLikeACredential is the check that keeps a secret out of a declaration.
//
// It errs towards refusing: a value holding a `:` (the htpasswd separator) or opening with a bcrypt
// identifier is a credential, not a path, and no filesystem path the mesh writes needs either. A
// false refusal is a loud log and a route that does not serve; a false accept is a credential
// committed to a declaration, which is the thing being prevented.
func looksLikeACredential(v string) bool {
v = strings.TrimSpace(v)
return strings.Contains(v, ":") || strings.HasPrefix(v, "$2")
}
// usersFrom reads the credentials the mesh mounted, in the one format every htpasswd already is.
func usersFrom(path string) (map[string]string, error) {
raw, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("cannot read the secret named for this route: %w", err)
}
users := map[string]string{}
for _, line := range strings.Split(string(raw), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
port, ok := asPort(c.Values["port"])
if !ok {
log.Printf("%s on %s asked for route %q and gave no usable port; skipped",
c.From, c.Node, name)
continue
}
user, hash, ok := strings.Cut(line, ":")
if !ok || user == "" || hash == "" {
continue
// Where the mesh says that machine is. Empty means it is this one — a workload beside the
// proxy is ordinary, and reaching it over loopback is both correct and the only thing
// that works when there is no private network.
at := c.At
if at == "" {
at = "127.0.0.1"
}
users[user] = hash
out[strings.ToLower(name)] = fmt.Sprintf("http://%s:%d", at, port)
}
if len(users) == 0 {
return nil, fmt.Errorf("the secret named for this route holds no usable credentials")
}
return users, nil
return out, nil
}
// asPort accepts what JSON makes of a number, which is a float even when it was written 8080.
-273
View File
@@ -1,273 +0,0 @@
package main
import (
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"golang.org/x/crypto/bcrypt"
)
// What a route carries about the requests arriving at it — novox/hq ADR 0108.
//
// Each test here is one of the four capabilities that record closed the set at, plus the negative
// case it promised would be refused. The negative case is the one that rots quietly: nothing fails
// if it stops working, so nothing tells you it has.
// served starts a workload and gives back the host and port the mesh would have recorded for it.
func served(t *testing.T, body string) (string, int) {
t.Helper()
workload := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(body))
}))
t.Cleanup(workload.Close)
host, port, _ := strings.Cut(strings.TrimPrefix(workload.URL, "http://"), ":")
n, err := strconv.Atoi(port)
if err != nil {
t.Fatal(err)
}
return host, n
}
// ask makes one request through the proxy for a given name and path, without following redirects.
func ask(t *testing.T, proxy, name, path string, auth [2]string) *http.Response {
t.Helper()
req, err := http.NewRequest(http.MethodGet, proxy+path, nil)
if err != nil {
t.Fatal(err)
}
req.Host = name
if auth[0] != "" {
req.SetBasicAuth(auth[0], auth[1])
}
client := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse
}}
answer, err := client.Do(req)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = answer.Body.Close() })
return answer
}
func proxyFor(t *testing.T, routesJSON string) string {
t.Helper()
path := filepath.Join(t.TempDir(), "routes.json")
if err := os.WriteFile(path, []byte(routesJSON), 0o644); err != nil {
t.Fatal(err)
}
routes, public, err := routesFrom(path)
if err != nil {
t.Fatal(err)
}
held := newTable()
held.set(routes, public)
server := httptest.NewServer(handler(held))
t.Cleanup(server.Close)
return server.URL
}
// A refusal on a path shadows the ordinary route for that path and leaves every other path alone.
//
// **This is why path scoping is a prerequisite and not a sibling capability.** The rule being
// reproduced matches a path on a host that is already routed to a workload, so a table mapping a
// host to one target cannot express it at all — no amount of authentication or source filtering
// would have helped.
func TestARefusedPathShadowsTheRouteAndLeavesTheRestServed(t *testing.T) {
at, port := served(t, "the workload")
proxy := proxyFor(t, `{"given":[
{"from":"forge","node":"anchor","at":"`+at+`","values":{"name":"forge.example","port":`+strconv.Itoa(port)+`}},
{"from":"forge","node":"anchor","values":{"name":"forge.example","path":"/api/internal","priority":100,"deny":true}}
]}`)
if got := ask(t, proxy, "forge.example", "/api/internal/hook", [2]string{}).StatusCode; got != http.StatusForbidden {
t.Fatalf("the refused path answered %d, so the block that was put in front of it during an "+
"incident is not in front of it any more", got)
}
if got := ask(t, proxy, "forge.example", "/", [2]string{}).StatusCode; got != http.StatusOK {
t.Fatalf("refusing one path took the whole route with it: %d", got)
}
}
// A redirect answers with the redirect, and the request keeps its own path and query.
//
// Losing the path would turn canonicalising one name onto another into "every deep link now lands
// on the front page", which is the kind of breakage that produces no error anywhere.
func TestARedirectKeepsThePathAndQuery(t *testing.T) {
proxy := proxyFor(t, `{"given":[
{"from":"site","node":"anchor","values":{"name":"www.example","redirect":"https://example/"}}
]}`)
answer := ask(t, proxy, "www.example", "/deep/page?ref=1", [2]string{})
if answer.StatusCode != http.StatusMovedPermanently {
t.Fatalf("a declared redirect answered %d", answer.StatusCode)
}
where := answer.Header.Get("Location")
if !strings.Contains(where, "/deep/page") || !strings.Contains(where, "ref=1") {
t.Fatalf("the redirect dropped the path or the query: %q", where)
}
}
// Authentication refuses a request with no credentials, admits one with the right ones, and refuses
// the wrong ones — with the credentials read from the secret the declaration *named*.
func TestAuthenticationAdmitsOnlyWhatTheSecretSays(t *testing.T) {
at, port := served(t, "the console")
hash, err := bcrypt.GenerateFromPassword([]byte("correct horse"), bcrypt.MinCost)
if err != nil {
t.Fatal(err)
}
secret := filepath.Join(t.TempDir(), "console-auth")
if err := os.WriteFile(secret, []byte("# a comment\nadmin:"+string(hash)+"\n"), 0o600); err != nil {
t.Fatal(err)
}
proxy := proxyFor(t, `{"given":[
{"from":"console","node":"anchor","at":"`+at+`","values":{"name":"console.example","port":`+strconv.Itoa(port)+`,"auth":"`+secret+`"}}
]}`)
if got := ask(t, proxy, "console.example", "/", [2]string{}).StatusCode; got != http.StatusUnauthorized {
t.Fatalf("an admin surface with no login of its own answered %d without credentials", got)
}
if got := ask(t, proxy, "console.example", "/", [2]string{"admin", "wrong"}).StatusCode; got != http.StatusUnauthorized {
t.Fatalf("the wrong password answered %d", got)
}
if got := ask(t, proxy, "console.example", "/", [2]string{"admin", "correct horse"}).StatusCode; got != http.StatusOK {
t.Fatalf("the right password answered %d", got)
}
}
// The negative case ADR 0108 promised would be refused: a credential in the declaration.
//
// **Refused whole, not tolerated and not served unprotected.** A hash carried in a declaration was
// the rejected option; nothing in the running system should quietly accept it later, because the
// precedent is far easier to set than to withdraw. If this test is deleted the option returns and
// nothing else notices.
func TestACredentialInTheDeclarationIsRefusedRatherThanServed(t *testing.T) {
inline := []string{
`{"given":[{"from":"c","node":"n","at":"127.0.0.1","values":{"name":"c.example","port":8080,"auth":"admin:$2a$10$abcdefghijklmnopqrstuv"}}]}`,
`{"given":[{"from":"c","node":"n","at":"127.0.0.1","values":{"name":"c.example","port":8080,"auth":"$2a$10$abcdefghijklmnopqrstuv"}}]}`,
}
for _, body := range inline {
path := filepath.Join(t.TempDir(), "routes.json")
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
t.Fatal(err)
}
routes, _, err := routesFrom(path)
if err != nil {
t.Fatal(err)
}
if len(routes) != 0 {
t.Fatalf("a declaration carrying a credential was served anyway: %v", routes)
}
}
}
// Authentication declared, secret unreadable: the route refuses. It does not serve unprotected.
//
// **Fail closed.** The alternative turns a missing file into a silently public admin surface, which
// is the outcome the whole record exists to prevent. It answers rather than 404s, so an operator
// sees "cannot read the credentials" instead of concluding the route was withdrawn.
func TestAnUnreadableSecretFailsClosed(t *testing.T) {
at, port := served(t, "the console")
missing := filepath.Join(t.TempDir(), "not-mounted")
proxy := proxyFor(t, `{"given":[
{"from":"console","node":"anchor","at":"`+at+`","values":{"name":"console.example","port":`+strconv.Itoa(port)+`,"auth":"`+missing+`"}}
]}`)
answer := ask(t, proxy, "console.example", "/", [2]string{})
if answer.StatusCode == http.StatusOK {
t.Fatal("a route whose credentials could not be read served the workload unprotected")
}
if answer.StatusCode != http.StatusServiceUnavailable {
t.Fatalf("expected the route to say it cannot check, got %d", answer.StatusCode)
}
}
// Equal priorities resolve the same way every time, so the same declaration serves the same way
// after a restart.
//
// Sorting only by priority leaves rules that share one in whatever order the map produced. The
// proxy would still work, and would work differently between restarts — which is the hardest kind
// of fault to believe when it is reported.
func TestRulesThatShareAPriorityAreStillTotallyOrdered(t *testing.T) {
first := []rule{
{path: "/a", priority: 10, target: "http://x:1"},
{path: "/bb", priority: 10, target: "http://y:2"},
{path: "", priority: 10, target: "http://z:3"},
}
second := []rule{
{path: "", priority: 10, target: "http://z:3"},
{path: "/bb", priority: 10, target: "http://y:2"},
{path: "/a", priority: 10, target: "http://x:1"},
}
inOrder(first)
inOrder(second)
for i := range first {
if first[i].path != second[i].path || first[i].target != second[i].target {
t.Fatalf("two orderings of the same rules disagree at %d: %q vs %q",
i, first[i].path, second[i].path)
}
}
// And the more specific rule is matched first, which is the intuitive reading.
if first[0].path != "/bb" {
t.Fatalf("the longest path is not matched first: %q", first[0].path)
}
}
// Priority decides before path length does, so a rule can be made to win regardless of specificity.
func TestPriorityOutranksPathLength(t *testing.T) {
rules := []rule{
{path: "/very/long/path", priority: 1, target: "http://x:1"},
{path: "", priority: 100, target: "http://y:2"},
}
inOrder(rules)
if rules[0].priority != 100 {
t.Fatalf("a higher priority did not win: %+v", rules[0])
}
}
// A priority above a port number survives, because a priority is an ordering and not a port.
//
// **Found by review, and it was load-bearing.** Priority was first read with the port reader, which
// caps at 65535 — so a rule declared above that silently became priority 0 and stopped shadowing the
// route it exists to shadow. The one real rule this has to reproduce is declared at 100000, so the
// capability would have shipped looking complete and doing nothing.
func TestAPriorityAboveAPortNumberSurvives(t *testing.T) {
at, port := served(t, "the workload")
proxy := proxyFor(t, `{"given":[
{"from":"forge","node":"anchor","at":"`+at+`","values":{"name":"forge.example","port":`+strconv.Itoa(port)+`}},
{"from":"forge","node":"anchor","values":{"name":"forge.example","path":"/api/internal","priority":100000,"deny":true}}
]}`)
if got := ask(t, proxy, "forge.example", "/api/internal/hook", [2]string{}).StatusCode; got != http.StatusForbidden {
t.Fatalf("a rule declared at priority 100000 answered %d instead of refusing", got)
}
}
// A host routed only on some paths says so, rather than claiming the name is not served here.
//
// Saying "no route for this name" while listing that very name as served is a contradiction an
// operator has to disbelieve the proxy to get past — and path scoping makes it reachable, because a
// host can now have rules that none of this request's paths match.
func TestAHostRoutedOnlyOnSomePathsSaysSo(t *testing.T) {
proxy := proxyFor(t, `{"given":[
{"from":"forge","node":"anchor","values":{"name":"forge.example","path":"/api/internal","deny":true}}
]}`)
answer := ask(t, proxy, "forge.example", "/elsewhere", [2]string{})
if answer.StatusCode != http.StatusNotFound {
t.Fatalf("an uncovered path answered %d", answer.StatusCode)
}
body := make([]byte, 256)
n, _ := answer.Body.Read(body)
said := string(body[:n])
if !strings.Contains(said, "is served here") || !strings.Contains(said, "/elsewhere") {
t.Fatalf("the refusal does not distinguish an uncovered path from an unserved name: %q", said)
}
}
+16 -308
View File
@@ -2,8 +2,6 @@ package main
import (
"context"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
@@ -21,37 +19,10 @@ func write(t *testing.T, body string) string {
return path
}
// plain is the table an ordinary set of routes makes: one host, one target, no policy.
func plain(routes map[string]string) map[string][]rule {
out := map[string][]rule{}
for host, target := range routes {
out[host] = []rule{{target: target}}
}
return out
}
// allPublic is every host in a routes map, ACME-eligible — the ordinary case for a test with no
// internal-name alias of its own to distinguish.
func allPublic(routes map[string][]rule) map[string]bool {
out := map[string]bool{}
for host := range routes {
out[host] = true
}
return out
}
// targetOf is where a host's first matching rule sends a request.
func targetOf(routes map[string][]rule, host string) string {
if rules := routes[host]; len(rules) > 0 {
return rules[0].target
}
return ""
}
// A route is a grant: the consumer supplies a target, and where that machine is comes from the
// mesh rather than from a naming convention the proxy has to know.
func TestARouteGoesToWhereTheMeshSaysTheConsumerIs(t *testing.T) {
routes, _, err := routesFrom(write(t, `{"contributions":1,"requirement":"route","given":[
routes, err := routesFrom(write(t, `{"contributions":1,"requirement":"route","given":[
{"from":"app","node":"laptop","at":"laptop.internal","values":{"name":"App.Example","port":8080}}
]}`))
if err != nil {
@@ -59,67 +30,28 @@ func TestARouteGoesToWhereTheMeshSaysTheConsumerIs(t *testing.T) {
}
// Lower-cased, because a Host header is not case-sensitive and a route that only answers the
// spelling in the manifest answers half the requests made to it.
if targetOf(routes, "app.example") != "http://laptop.internal:8080" {
if routes["app.example"] != "http://laptop.internal:8080" {
t.Fatalf("the route does not point at the consumer: %v", routes)
}
}
// A route with an internal-name alias is reachable under both hostnames, pointed at the same
// target — the same convenience a predecessor proxy gave for reaching a service over the VPN
// without a public TLS round trip.
func TestARouteWithAnInternalNameIsReachableUnderBoth(t *testing.T) {
routes, public, err := routesFrom(write(t, `{"given":[
{"from":"app","node":"anchor","at":"anchor.internal",
"values":{"name":"app.example","internal-name":"app.anchor.internal","port":8080}}
]}`))
if err != nil {
t.Fatal(err)
}
if targetOf(routes, "app.example") != "http://anchor.internal:8080" {
t.Fatalf("the public name does not point at the consumer: %v", routes)
}
if targetOf(routes, "app.anchor.internal") != "http://anchor.internal:8080" {
t.Fatalf("the internal alias does not point at the same consumer: %v", routes)
}
if !public["app.example"] {
t.Errorf("the public name is not eligible for a certificate: %v", public)
}
if public["app.anchor.internal"] {
t.Errorf("the internal alias is eligible for a certificate no public CA could ever issue: %v",
public)
}
}
// A route with no internal-name composed gets no second host — the ordinary case, unchanged.
func TestARouteWithNoInternalNameGetsNoAlias(t *testing.T) {
routes, _, err := routesFrom(write(t, `{"given":[
{"from":"app","node":"anchor","values":{"name":"app.example","port":8080}}
]}`))
if err != nil {
t.Fatal(err)
}
if len(routes) != 1 {
t.Fatalf("a route with no internal-name grew a second host: %v", routes)
}
}
// A workload beside the proxy is ordinary, and reaching it over loopback is both correct and the
// only thing that works when there is no private network.
func TestAConsumerOnTheProxysOwnMachineIsReachedOverLoopback(t *testing.T) {
routes, _, err := routesFrom(write(t, `{"given":[
routes, err := routesFrom(write(t, `{"given":[
{"from":"app","node":"anchor","values":{"name":"app.example","port":9000}}
]}`))
if err != nil {
t.Fatal(err)
}
if targetOf(routes, "app.example") != "http://127.0.0.1:9000" {
if routes["app.example"] != "http://127.0.0.1:9000" {
t.Fatalf("a workload on this machine was not reachable: %v", routes)
}
}
// Skipped rather than served wrongly. A route with no port would proxy to :0.
func TestAContributionMissingWhatARouteNeedsIsSkipped(t *testing.T) {
routes, _, err := routesFrom(write(t, `{"given":[
routes, err := routesFrom(write(t, `{"given":[
{"from":"a","node":"n","at":"n.internal","values":{"name":"no-port.example"}},
{"from":"b","node":"n","at":"n.internal","values":{"port":8080}},
{"from":"c","node":"n","at":"n.internal","values":{"name":"fine.example","port":8080}}
@@ -127,73 +59,11 @@ func TestAContributionMissingWhatARouteNeedsIsSkipped(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if len(routes) != 1 || targetOf(routes, "fine.example") == "" {
if len(routes) != 1 || routes["fine.example"] == "" {
t.Fatalf("an unusable contribution was served: %v", routes)
}
}
// A route may name a target reached over https, for a backend that terminates its own TLS — the
// shape Mailu's webmail front needs, which this proxy reaches as a plain workload otherwise.
func TestARouteMayTargetHttps(t *testing.T) {
routes, _, err := routesFrom(write(t, `{"given":[
{"from":"mail","node":"anchor","at":"anchor.internal",
"values":{"name":"mail.example","port":7443,"scheme":"https","insecure":true}}
]}`))
if err != nil {
t.Fatal(err)
}
if targetOf(routes, "mail.example") != "https://anchor.internal:7443" {
t.Fatalf("an https target was not built as one: %v", routes)
}
if !routes["mail.example"][0].insecure {
t.Fatal("insecure was declared and not carried onto the rule")
}
}
// A scheme that is neither http nor https is refused rather than guessed at.
func TestARouteWithAnUnknownSchemeIsSkipped(t *testing.T) {
routes, _, err := routesFrom(write(t, `{"given":[
{"from":"a","node":"n","at":"n.internal","values":{"name":"bad.example","port":80,"scheme":"ftp"}}
]}`))
if err != nil {
t.Fatal(err)
}
if len(routes) != 0 {
t.Fatalf("a route with an unusable scheme was served: %v", routes)
}
}
// End to end: a backend terminating TLS with a certificate nothing would ordinarily trust is still
// reached when the route declared `insecure`, and the response comes back through unmodified.
func TestTheProxyReachesAnInsecureHttpsBackend(t *testing.T) {
workload := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte("the workload, over its own TLS"))
}))
defer workload.Close()
target := strings.TrimPrefix(workload.URL, "https://")
held := newTable()
routes := map[string][]rule{"mail.example": {{target: "https://" + target, insecure: true}}}
held.set(routes, allPublic(routes))
proxy := httptest.NewServer(handler(held))
defer proxy.Close()
asked, err := http.NewRequest(http.MethodGet, proxy.URL, nil)
if err != nil {
t.Fatal(err)
}
asked.Host = "mail.example"
answer, err := http.DefaultClient.Do(asked)
if err != nil {
t.Fatal(err)
}
defer answer.Body.Close()
if answer.StatusCode != http.StatusOK {
t.Fatalf("an insecure https backend was not reached: %d", answer.StatusCode)
}
}
// End to end through the proxy itself: a request for the name reaches the workload, and a name
// nobody asked for is refused in a way that says what IS served.
func TestTheProxyReachesTheWorkloadAndNamesWhatItServes(t *testing.T) {
@@ -205,7 +75,7 @@ func TestTheProxyReachesTheWorkloadAndNamesWhatItServes(t *testing.T) {
host, port, _ := strings.Cut(target, ":")
held := newTable()
held.set(plain(map[string]string{"app.example": "http://" + host + ":" + port}), allPublic(plain(map[string]string{"app.example": "http://" + host + ":" + port})))
held.set(map[string]string{"app.example": "http://" + host + ":" + port})
proxy := httptest.NewServer(handler(held))
defer proxy.Close()
@@ -250,17 +120,16 @@ func TestTheProxyReachesTheWorkloadAndNamesWhatItServes(t *testing.T) {
// nothing fails more visibly than a stale grant, which is exactly why it must not survive.
func TestWithdrawingARouteStopsServingIt(t *testing.T) {
held := newTable()
initial := plain(map[string]string{
held.set(map[string]string{
"going.example": "http://a.internal:80",
"staying.example": "http://b.internal:80",
})
held.set(initial, allPublic(initial))
held.set(plain(map[string]string{"staying.example": "http://b.internal:80"}), allPublic(plain(map[string]string{"staying.example": "http://b.internal:80"})))
held.set(map[string]string{"staying.example": "http://b.internal:80"})
if _, still := held.find("going.example", "/"); still {
if _, still := held.find("going.example"); still {
t.Fatal("a route whose module was unassigned is still served")
}
if _, kept := held.find("staying.example", "/"); !kept {
if _, kept := held.find("staying.example"); !kept {
t.Fatal("withdrawing one route took another with it")
}
}
@@ -268,8 +137,8 @@ func TestWithdrawingARouteStopsServingIt(t *testing.T) {
// A Host header carries a port and the name does not.
func TestARequestNamingAPortStillFindsItsRoute(t *testing.T) {
held := newTable()
held.set(plain(map[string]string{"app.example": "http://a.internal:8080"}), allPublic(plain(map[string]string{"app.example": "http://a.internal:8080"})))
if _, found := held.find("app.example:8080", "/"); !found {
held.set(map[string]string{"app.example": "http://a.internal:8080"})
if _, found := held.find("app.example:8080"); !found {
t.Fatal("a request to app.example:8080 did not find the route for app.example")
}
}
@@ -299,7 +168,7 @@ func TestTheIssuerIsStagingUnlessNamed(t *testing.T) {
// rate limit — and the proxy would look healthy throughout.
func TestNoCertificateIsAskedForOnAnUnroutedName(t *testing.T) {
held := newTable()
held.set(plain(map[string]string{"photos.example": "http://127.0.0.1:8080"}), allPublic(plain(map[string]string{"photos.example": "http://127.0.0.1:8080"})))
held.set(map[string]string{"photos.example": "http://127.0.0.1:8080"})
policy := onlyWhatTheMeshSaid(held)
if err := policy(context.Background(), "photos.example"); err != nil {
@@ -312,179 +181,18 @@ func TestNoCertificateIsAskedForOnAnUnroutedName(t *testing.T) {
}
}
// A certificate is asked for on a route's public name, never on its internal-network alias — no
// public CA can validate a private name, and asking anyway would only spend the account's rate
// limit on an order that can never succeed.
func TestNoCertificateIsAskedForOnAnInternalAlias(t *testing.T) {
routes, public, err := routesFrom(write(t, `{"given":[
{"from":"app","node":"anchor","at":"anchor.internal",
"values":{"name":"app.example","internal-name":"app.anchor.internal","port":8080}}
]}`))
if err != nil {
t.Fatal(err)
}
held := newTable()
held.set(routes, public)
policy := onlyWhatTheMeshSaid(held)
if err := policy(context.Background(), "app.example"); err != nil {
t.Errorf("the route's public name was refused a certificate: %v", err)
}
if err := policy(context.Background(), "app.anchor.internal"); err == nil {
t.Error("a certificate was ordered for the internal alias, which no public CA can validate")
}
}
// A certificate is asked of the *internal* authority only for a name that is routed here and is
// not a route's own public name — the internal-network alias, never the route it accompanies.
func TestTheInternalAuthorityOnlyCertifiesInternalOnlyAliases(t *testing.T) {
routes, public, err := routesFrom(write(t, `{"given":[
{"from":"app","node":"anchor","at":"anchor.internal",
"values":{"name":"app.example","internal-name":"app.anchor.internal","port":8080}}
]}`))
if err != nil {
t.Fatal(err)
}
held := newTable()
held.set(routes, public)
policy := onlyInternalNamesTheMeshSaid(held)
if err := policy(context.Background(), "app.anchor.internal"); err != nil {
t.Errorf("the internal alias was refused by its own authority: %v", err)
}
if err := policy(context.Background(), "app.example"); err == nil {
t.Error("the internal authority certified a route's public name, which the public authority already covers")
}
if err := policy(context.Background(), "unrouted.internal"); err == nil {
t.Error("the internal authority certified a name nobody routed here")
}
}
// A route withdrawn stops being certifiable, without the proxy restarting.
func TestWithdrawingARouteWithdrawsItsCertificate(t *testing.T) {
held := newTable()
held.set(plain(map[string]string{"photos.example": "http://127.0.0.1:8080"}), allPublic(plain(map[string]string{"photos.example": "http://127.0.0.1:8080"})))
held.set(map[string]string{"photos.example": "http://127.0.0.1:8080"})
policy := onlyWhatTheMeshSaid(held)
if err := policy(context.Background(), "photos.example"); err != nil {
t.Fatal(err)
}
held.set(nil, nil)
held.set(nil)
if err := policy(context.Background(), "photos.example"); err == nil {
t.Fatal("a withdrawn route can still order certificates, so the policy read a copy taken " +
"once rather than what is served now")
}
}
// A route may say the largest body it carries, and the proxy holds requests to it.
//
// The registry is why: image layers arrive as single requests of gigabytes, and a proxy's own
// default refuses them long before the workload is reached. It is configuration beside `insecure`,
// not a fifth policy — novox/hq ADR 0108 closed that set at four.
func TestARouteMayLimitTheBodyItCarries(t *testing.T) {
routes, _, err := routesFrom(write(t, `{"given":[
{"from":"registry","node":"anchor","at":"anchor.internal",
"values":{"name":"images.example","port":5000,"max-request-body":21474836480}}
]}`))
if err != nil {
t.Fatal(err)
}
rules := routes["images.example"]
if len(rules) != 1 {
t.Fatalf("the route is not served once: %v", routes)
}
if rules[0].maxRequestBody != 21474836480 {
t.Fatalf("the limit did not survive the contribution: %d", rules[0].maxRequestBody)
}
}
// Saying nothing leaves the route unlimited, which is what every route already got.
func TestARouteThatSaysNothingCarriesAnySize(t *testing.T) {
routes, _, err := routesFrom(write(t, `{"given":[
{"from":"app","node":"anchor","at":"anchor.internal","values":{"name":"app.example","port":8080}}
]}`))
if err != nil {
t.Fatal(err)
}
if got := routes["app.example"][0].maxRequestBody; got != 0 {
t.Fatalf("a route that asked for no limit got one: %d", got)
}
}
// A limit that is not a whole positive number of bytes takes the route with it. Serving it without
// the limit would carry exactly what the module said not to carry, and report success doing it.
func TestARouteWithAnUnusableLimitIsSkipped(t *testing.T) {
for _, asked := range []string{`"lots"`, `-1`, `0`, `1.5`} {
routes, _, err := routesFrom(write(t, `{"given":[
{"from":"registry","node":"anchor","at":"anchor.internal",
"values":{"name":"images.example","port":5000,"max-request-body":`+asked+`}}
]}`))
if err != nil {
t.Fatal(err)
}
if len(routes["images.example"]) != 0 {
t.Errorf("a route asking for a max-request-body of %s was served anyway: %v", asked, routes)
}
}
}
// A request larger than the route carries is refused by the proxy, with the limit named, and the
// workload never sees it. A request within it is proxied normally.
func TestABodyOverTheLimitIsRefusedAndOneUnderItIsCarried(t *testing.T) {
var reached int
workload := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
reached++
fmt.Fprintf(w, "carried %d bytes", len(body))
}))
defer workload.Close()
at := strings.TrimPrefix(workload.URL, "http://")
host, port, _ := strings.Cut(at, ":")
routes, _, err := routesFrom(write(t, `{"given":[
{"from":"registry","node":"anchor","at":"`+host+`",
"values":{"name":"images.example","port":`+port+`,"max-request-body":8}}
]}`))
if err != nil {
t.Fatal(err)
}
held := newTable()
held.set(routes, map[string]bool{})
proxy := httptest.NewServer(handler(held))
defer proxy.Close()
over, err := post(proxy.URL, "images.example", "123456789")
if err != nil {
t.Fatal(err)
}
defer over.Body.Close()
if over.StatusCode != http.StatusRequestEntityTooLarge {
t.Fatalf("a body over the limit answered %d, not 413", over.StatusCode)
}
if reached != 0 {
t.Fatalf("the workload was reached by a request the route said it would not carry")
}
under, err := post(proxy.URL, "images.example", "1234")
if err != nil {
t.Fatal(err)
}
defer under.Body.Close()
if under.StatusCode != http.StatusOK {
t.Fatalf("a body within the limit answered %d, not 200", under.StatusCode)
}
if reached != 1 {
t.Fatalf("the workload was not reached by a request within the limit")
}
}
// post sends a body to the proxy as the named route, since a route is found by the Host header.
func post(url, host, body string) (*http.Response, error) {
asked, err := http.NewRequest(http.MethodPost, url, strings.NewReader(body))
if err != nil {
return nil, err
}
asked.Host = host
asked.Header.Set("Content-Type", "application/octet-stream")
return http.DefaultClient.Do(asked)
}
+7 -10
View File
@@ -1,22 +1,19 @@
module github.com/novox/mesh-controller
go 1.26.0
go 1.25.0
require (
github.com/jackc/pgx/v5 v5.10.0
github.com/nats-io/nats.go v1.54.0
golang.org/x/crypto v0.57.0
github.com/rabbitmq/amqp091-go v1.14.0
golang.org/x/crypto v0.55.0
)
require (
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/klauspost/compress v1.20.0 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/net v0.58.0 // indirect
golang.org/x/sync v0.23.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/text v0.42.0 // indirect
golang.org/x/net v0.57.0 // indirect
golang.org/x/sync v0.22.0 // indirect
golang.org/x/sys v0.47.0 // indirect
golang.org/x/text v0.41.0 // indirect
)
+14 -18
View File
@@ -9,31 +9,27 @@ github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0=
github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rabbitmq/amqp091-go v1.14.0 h1:RSaT7aOKt/OrkVUyswPDW29lnRz9psuGmfZFBmLqLek=
github.com/rabbitmq/amqp091-go v1.14.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
-125
View File
@@ -1,125 +0,0 @@
package broker
import (
"fmt"
"sort"
"strings"
)
// Do the emitters and the consumers of a catalogue agree?
//
// **The check that was missing** (novox/hq 04-ISSUES/127). Every manifest was individually
// well-formed and every derivation individually correct, and no cross-module subscription in the
// mesh matched anything: a consumer's declaration derived into a namespace nobody publishes to.
// Nothing failed, because a subscription that matches nothing is not an error — it is silence.
//
// The comparison has to be over the whole catalogue, because the two halves live in different
// manifests, and it cannot simply demand that every consumed event have a live emitter: a module
// may be installed long before the one whose events it wants. So the rule is narrower and still
// catches this: **where the emitter is present, it must emit what the consumer asked for.**
// AConsumer is one module's interest in another's events, as this check needs it.
type AConsumer struct {
Module string
Consumes []string
}
// AnEmitter is one module's events.
type AnEmitter struct {
Module string
Emits []string
}
// Disagreements are the consumed events whose emitter is in the catalogue and does not emit them.
//
// Returned as sentences rather than as structs: every one of them is read by a person deciding
// whether a manifest or a catalogue is wrong, and a pair of names without the reason is a puzzle.
func Disagreements(emitters []AnEmitter, consumers []AConsumer, seats []DeclaredSeat) []string {
emits := map[string]map[string]bool{}
for _, e := range emitters {
if emits[e.Module] == nil {
emits[e.Module] = map[string]bool{}
}
for _, name := range e.Emits {
emits[e.Module][name] = true
}
}
// A seat's events are published by its holder under the seat's name, so a consumer naming the
// seat is naming something real even though no module declares it as its own.
for _, s := range seats {
if len(s.Emits) == 0 {
continue
}
if emits[s.Name] == nil {
emits[s.Name] = map[string]bool{}
}
for _, name := range s.Emits {
emits[s.Name][name] = true
}
}
var out []string
for _, c := range consumers {
for _, pattern := range c.Consumes {
emitter, event, named := strings.Cut(pattern, ".")
// Every event from everyone, or every event from one module: both are deliberate and
// neither names a particular event to check.
if !named || emitter == "*" || emitter == catalogueTheRest || event == catalogueTheRest {
continue
}
known, present := emits[emitter]
if !present {
// Not installed here, which is ordinary: a module lives in its own repository and
// may be registered later. Nothing to compare, so nothing to say.
continue
}
if matchesAny(event, known) {
continue
}
out = append(out, fmt.Sprintf(
"%s consumes %q and %s emits %s — so that subscription would match nothing, and "+
"nothing would report it",
c.Module, pattern, emitter, listOf(known)))
}
}
sort.Strings(out)
return out
}
// matchesAny says whether one of an emitter's event names satisfies a consumer's pattern.
func matchesAny(pattern string, emitted map[string]bool) bool {
want := strings.Split(pattern, ".")
for name := range emitted {
if matches(want, strings.Split(name, ".")) {
return true
}
}
return false
}
func matches(pattern, name []string) bool {
for i, part := range pattern {
if part == catalogueTheRest {
return i < len(name)
}
if i >= len(name) {
return false
}
if part != "*" && part != name[i] {
return false
}
}
return len(pattern) == len(name)
}
func listOf(names map[string]bool) string {
if len(names) == 0 {
return "nothing"
}
out := make([]string, 0, len(names))
for n := range names {
out = append(out, n)
}
sort.Strings(out)
return strings.Join(out, ", ")
}
-236
View File
@@ -1,236 +0,0 @@
package broker
import (
"encoding/json"
"os"
"path/filepath"
"sort"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// **Do the catalogue's emitters and consumers agree?**
//
// This is the check whose absence let issue 127 stand: every manifest was individually well-formed,
// every derivation individually correct, and no cross-module subscription in the mesh matched
// anything. A subscription that matches nothing is not an error — it is silence — so nothing
// anywhere reported it.
//
// It compares what one manifest asks to hear against what another says it emits. It cannot demand
// that every consumed event have a live emitter, because a module lives in its own repository and
// may be registered long before the one whose events it wants. Where the emitter *is* here, it must
// emit what the consumer asked for.
func TestTheCataloguesEmittersAndConsumersAgree(t *testing.T) {
emitters, consumers, seats := theCataloguesEvents(t)
if bad := Disagreements(emitters, consumers, seats); len(bad) > 0 {
t.Fatalf("%d subscription(s) in the catalogue would match nothing:\n %s",
len(bad), strings.Join(bad, "\n "))
}
}
// And the check itself catches the thing it exists for, so it cannot pass by doing nothing.
func TestTheAgreementCheckCatchesASubscriptionThatMatchesNothing(t *testing.T) {
bad := Disagreements(
[]AnEmitter{{Module: "builder", Emits: []string{"built"}}},
[]AConsumer{{Module: "mesh-catalog", Consumes: []string{"builder.finished"}}},
nil)
if len(bad) != 1 {
t.Fatalf("a consumer asking for an event its emitter does not emit was not caught: %v", bad)
}
if !strings.Contains(bad[0], "builder.finished") || !strings.Contains(bad[0], "built") {
t.Fatalf("the report names neither what was asked for nor what is emitted: %s", bad[0])
}
// A module that is not here is not a disagreement: it may be registered later.
if bad := Disagreements(nil,
[]AConsumer{{Module: "plex", Consumes: []string{"sonarr.download.completed"}}}, nil); len(bad) != 0 {
t.Fatalf("a consumer whose emitter is not installed was reported: %v", bad)
}
// A wildcard over emitters is deliberate and names no particular event to check.
if bad := Disagreements([]AnEmitter{{Module: "sonarr", Emits: []string{"download.completed"}}},
[]AConsumer{{Module: "plex", Consumes: []string{"*.download.completed"}}}, nil); len(bad) != 0 {
t.Fatalf("a wildcard over emitters was reported: %v", bad)
}
// A consumer of a role's event whose role does not emit it is caught, which is what stops the
// catalogue check above from passing by knowing nothing about roles.
if bad := Disagreements(nil,
[]AConsumer{{Module: "mesh-catalog", Consumes: []string{"mesh-build-machine.finished"}}},
[]DeclaredSeat{{Name: "mesh-build-machine", Emits: []string{"built"}}}); len(bad) != 1 {
t.Fatalf("a consumer of a role event the role does not emit was not caught: %v", bad)
}
// An event published under a seat's name is real even though no module declares it as its own.
if bad := Disagreements(nil,
[]AConsumer{{Module: "watcher", Consumes: []string{"mesh-artifact-store.image.pushed"}}},
[]DeclaredSeat{{Name: "the-artifact-store", Emits: []string{"image.pushed"}}}); len(bad) != 0 {
t.Fatalf("an event a seat emits was reported as matching nothing: %v", bad)
}
}
func theCataloguesEvents(t *testing.T) ([]AnEmitter, []AConsumer, []DeclaredSeat) {
t.Helper()
root := filepath.Join("..", "..", "..", "mesh-catalog", "modules")
entries, err := os.ReadDir(root)
if err != nil {
t.Skipf("catalogue sibling not present: %v", err)
}
var emitters []AnEmitter
var consumers []AConsumer
// The mesh's own roles, which emit under the seat's name rather than any module's (novox/hq
// ADR 0121). Without these the check skips every consumer of a role's event as "the emitter is
// not installed" — which is how it passed vacuously the first time one existed.
var seats []DeclaredSeat
for _, own := range catalogue.SeatsWithAProtocol() {
seats = append(seats, DeclaredSeat{Name: own.Name, Accepts: own.Accepts, Emits: own.Emits})
}
for _, e := range entries {
if !e.IsDir() {
continue
}
raw, err := os.ReadFile(filepath.Join(root, e.Name(), "module.json"))
if err != nil {
continue
}
var m struct {
Module string `json:"module"`
Emits []string `json:"emits"`
Consumes []string `json:"consumes"`
Seats []struct {
Name string `json:"name"`
Emits []string `json:"emits"`
} `json:"seats"`
}
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatalf("%s: %v", e.Name(), err)
}
if len(m.Emits) > 0 {
emitters = append(emitters, AnEmitter{Module: m.Module, Emits: m.Emits})
}
if len(m.Consumes) > 0 {
consumers = append(consumers, AConsumer{Module: m.Module, Consumes: m.Consumes})
}
for _, s := range m.Seats {
seats = append(seats, DeclaredSeat{Name: s.Name, Emits: s.Emits})
}
}
if len(emitters) == 0 {
t.Skip("no manifests found beside this checkout")
}
return emitters, consumers, seats
}
// **Do the derived subjects meet, not just the names?**
//
// The check above compares what a consumer asks for against what an emitter says it emits, by name. It
// passed while the catalogue's subscription pointed at `mesh.mod.mesh-build-machine.event.built` — a
// module namespace for a role's event, which no emitter owns. The names agreed; the subjects did not,
// and the graph stayed empty.
//
// So this compares the thing that actually has to match: the subject a consumer subscribes against the
// subject an emitter publishes. It is the last place the two halves can be held together, because
// after this the server is the only thing that knows and it says nothing — a subscription that matches
// nothing is silence.
func TestTheCataloguesDerivedSubjectsMeet(t *testing.T) {
emitters, consumers, seats := theCataloguesEvents(t)
// Every subject something publishes: a module's own events, and the events of every role.
published := map[string]bool{}
for _, e := range emitters {
for _, name := range e.Emits {
published["mesh.mod."+e.Module+".event."+name] = true
}
}
for _, s := range seats {
for _, name := range s.Emits {
published["mesh.seat."+s.Name+".event."+name] = true
}
}
byName := map[string]DeclaredSeat{}
for _, s := range seats {
byName[s.Name] = s
}
var lonely []string
for _, c := range consumers {
principal := Principal{Kind: KindModule, Node: "one", Module: c.Module, PasswordHash: "x"}
for _, want := range c.Consumes {
emitter, event, named := strings.Cut(want, ".")
if named {
if s, isASeat := byName[emitter]; isASeat {
principal.Watches = append(principal.Watches,
Seat{Name: s.Name, Emits: []string{event}})
continue
}
}
principal.Consumes = append(principal.Consumes, want)
}
perms, err := PermissionsFor(principal)
if err != nil {
t.Fatalf("%s: %v", c.Module, err)
}
for _, subject := range perms.Subscribe {
if !strings.Contains(subject, ".event.") {
continue
}
if reaches(subject, published) {
continue
}
// A wildcard over emitters reaches whatever arrives later, and an emitter that is not
// installed is ordinary — both are already excused by the check above, so only a subject
// that can never match anything gets here.
if strings.Contains(subject, "*") || strings.Contains(subject, ">") {
continue
}
lonely = append(lonely, c.Module+" subscribes "+subject+", which nothing publishes")
}
}
if len(lonely) > 0 {
sort.Strings(lonely)
t.Fatalf("%d subscription(s) derive to a subject no emitter owns:\n %s",
len(lonely), strings.Join(lonely, "\n "))
}
}
// And it catches the thing it exists for: a role's event read as a module's.
func TestTheDerivedSubjectCheckCatchesARolesEventReadAsAModules(t *testing.T) {
published := map[string]bool{"mesh.seat.mesh-build-machine.event.built": true}
// What the derivation produced before a consumed seat name was resolved as one.
if reaches("mesh.mod.mesh-build-machine.event.built", published) {
t.Fatal("a module namespace was treated as reaching a role's event, which is the bug")
}
// And the corrected one does reach it.
if !reaches("mesh.seat.mesh-build-machine.event.built", published) {
t.Fatal("the role's own subject does not reach the role's event")
}
}
// reaches says whether a subscribed subject admits any published one.
func reaches(subject string, published map[string]bool) bool {
for p := range published {
if admitsSubject(strings.Split(subject, "."), strings.Split(p, ".")) {
return true
}
}
return false
}
func admitsSubject(pattern, subject []string) bool {
for i, token := range pattern {
if token == ">" {
return i < len(subject)
}
if i >= len(subject) {
return false
}
if token != "*" && token != subject[i] {
return false
}
}
return len(pattern) == len(subject)
}
+67
View File
@@ -0,0 +1,67 @@
package broker_test
import (
"regexp"
"testing"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/link"
)
// The names are written twice, so a test keeps them agreeing.
//
// `link` imports `broker`, so `broker` cannot import `link` — the queue and exchange names
// therefore exist in both. A scoped account naming a queue nothing publishes to produces a
// builder that takes no work and says nothing about why, which is the worst kind of silence.
//
// An external test package, because it may import both without either importing the other.
func TestTheNamesTheBrokerScopesAreTheNamesTheLinkUses(t *testing.T) {
for _, agreed := range []struct {
what string
scoped string
actually string
}{
{"the build queue", broker.BuildQueueName, link.BuildQueue},
{"the exchange", broker.ExchangeName, link.Exchange},
{"a node's queue", broker.QueueFor("somewhere"), link.QueueFor("somewhere")},
} {
if agreed.scoped != agreed.actually {
t.Errorf("%s: the broker scopes %q and the link uses %q",
agreed.what, agreed.scoped, agreed.actually)
}
}
}
func TestABuilderMayWriteToAReplyQueueAndReadNoNodesDeclarations(t *testing.T) {
// The scoping, checked as patterns rather than by connecting: what it may write must include
// the queue an asker actually waits on, and what it may read must not include any node's.
//
// This exists because the first version scoped writes to `amq.gen-*` — the name one broker
// happens to generate — and the builder built, could not answer, and the connection simply
// closed saying only "not allowed to publish to exchange \'\'". Answering through the
// exchange is what removed the need for any of that.
write := regexp.MustCompile("^" + regexp.QuoteMeta(broker.ExchangeName) + "$")
if !write.MatchString(broker.ExchangeName) {
t.Error("a builder may not write to the exchange, so it can take work and never answer")
}
// And not the default exchange, where permission is per exchange rather than per queue — a
// builder allowed to use it could publish into any node's queue.
//
// Confirmed against a real broker as well, and worth recording how that nearly went wrong:
// an unconfirmed publish is asynchronous, so a refusal arrives as a channel close afterwards
// and a naive check reports success. With publisher confirms the broker's refusal is
// immediate. **A negative security assertion made against an asynchronous call is not an
// assertion.**
if write.MatchString("") {
t.Error("a builder may publish to the default exchange, and so into any node's queue")
}
read := regexp.MustCompile("^" + regexp.QuoteMeta(broker.BuildQueueName) + "$")
if !read.MatchString(broker.BuildQueueName) {
t.Error("a builder may not read the build queue")
}
if read.MatchString(link.QueueFor("someone-else")) {
// A build machine is not a node, and a node's queue carries its declarations.
t.Error("a builder may read another machine's declarations")
}
}
-13
View File
@@ -66,19 +66,6 @@ func FromEnvironment() (Broker, error) {
if err != nil {
return Broker{}, err
}
// **One bus, one address** (novox/hq ADR 0131). Everything the mesh hands out — a token, a
// machine's membership, a person's credential — must name the bus the control plane itself is
// connected to; the setting above predates the move and, on a mesh that has moved, still names
// the broker it moved from. The first person issued after the move was handed the retired
// broker's port and could not connect to anything (2026-09-28).
//
// Read from the credential rather than from a second setting somebody keeps in step: the
// control plane cannot be wrong about where it is connected.
if bus, on, err := OnNATS(); err == nil && on {
if where := strings.TrimPrefix(BareAddress(bus), "nats://"); where != "" {
address = where
}
}
return Broker{Address: address, Fingerprint: fingerprint}, nil
}
+13
View File
@@ -194,3 +194,16 @@ func TestTheAddressPortFollowsThePortTwin(t *testing.T) {
t.Fatalf("the address is %q; the node put the bus on 5679", b.Address)
}
}
func TestTheManagementPortFollowsThePortTwin(t *testing.T) {
t.Setenv(ManagementVar, "http://guest:guest@127.0.0.1:15672")
t.Setenv(ManagementVar+"_FILE", "")
t.Setenv(ManagementVar+"_PORT", "15673")
m, err := ManagementFromEnvironment()
if err != nil {
t.Fatal(err)
}
if m.base.Host != "127.0.0.1:15673" {
t.Fatalf("the management API is at %q; the node put it on 15673", m.base.Host)
}
}
-237
View File
@@ -1,237 +0,0 @@
package broker
import (
"fmt"
"sort"
"strings"
)
// Streams and consumers derived from what modules declare.
//
// The mesh's own four exist before any module does (streams.go). Everything here is the other
// half: a seat's stream comes into being when the module declaring it is **registered**, and a
// consumer when a module is **assigned** — which is why ADR 0116's task 1.4 had to be narrowed to
// the foundation set. Neither has happened at genesis.
//
// All of it is a pure function of declarations. The controller is still the only writer; this is
// only what it writes.
// A Consumer is a durable subscription the controller creates on a module's behalf. A module
// declares what it reacts to, never how delivery works, so it does not name these and cannot
// misconfigure them.
type Consumer struct {
Name string
Stream string
// Filters are the subjects this consumer receives. One consumer per module with several
// filters, rather than one per consumed event: its ack subject is derived from its name, and
// a module with five consumers would need five ack permissions to ack its own deliveries.
Filters []string
// Queue is the queue group, set for a seat's worker so that "exactly one holder" survives a
// seat later being relaxed to several. Authority and delivery are kept separate on purpose.
Queue string
// Push asks the server to deliver to a subject rather than wait to be pulled.
//
// For the mesh's own consumer, where the controller wants every message to arrive in the one
// loop it already runs: pulling would mean a second goroutine fetching batches and handing
// them over, and a loop that acts on one message at a time is the property the store window
// depends on. A queue group implies this, because a group has nothing to pull from.
Push bool
// AckWaitSeconds before an unacknowledged delivery is redelivered.
AckWaitSeconds int
// MaxDeliver before the message is dead-lettered; zero for the mesh's default.
MaxDeliver int
Why string
}
// seatStreamName is the stream holding a seat's inbound work. Named after the seat rather than
// the module holding it, because the holder can change and the queued work must not care — which
// is the whole reason a caller addresses a seat instead of a module.
func seatStreamName(seat string) string { return "SEAT_" + upperSnake(seat) }
// SeatStreams is one work queue per declared seat, created when the declaring module is
// registered rather than when it is assigned.
//
// **The stream exists before anyone holds the seat, and that is the point.** Work queues until a
// holder appears, so installing the telegram module a week after something started sending to it
// flushes the backlog instead of having lost it. A stream created at assignment would make "the
// holder is not here yet" mean "your messages are gone".
func SeatStreams(seats []DeclaredSeat) []Stream {
sorted := append([]DeclaredSeat(nil), seats...)
sort.Slice(sorted, func(i, j int) bool { return sorted[i].Name < sorted[j].Name })
var out []Stream
for _, s := range sorted {
if len(s.Accepts) == 0 {
// A seat that only emits and serves needs no stream: its events ride EVENTS and its
// tools are core request/reply, which is never persisted.
continue
}
retain := s.RetainSeconds
if retain == 0 {
retain = 7 * 24 * 60 * 60
}
out = append(out, Stream{
Name: seatStreamName(s.Name),
Subjects: []string{"mesh.seat." + s.Name + ".accept.>"},
Retention: RetentionWorkQueue,
MaxAge: retain,
Why: fmt.Sprintf("work submitted to the %s seat; one holder consumes it, and it "+
"queues while nobody does", s.Name),
})
}
return out
}
// A DeclaredSeat is a seat as the catalogue knows it. Mirrored here rather than imported so this
// package stays free of the catalogue's own types — the same reason the host mirrors the
// contracts instead of importing the sdk.
type DeclaredSeat struct {
Name string
Accepts []string
// Emits are the verbs the seat's holder publishes under the seat's own name. An event about a
// role belongs here rather than in the holder's namespace, because the name then outlives
// whoever fills it (novox/hq ADR 0121, 04-ISSUES/127).
Emits []string
// Serves are the verbs the holder answers, request and reply.
Serves []string
RetainSeconds int
}
// ConsumerFor is the durable consumer a module's declarations imply, or false when it subscribes
// to nothing and needs none.
//
// One per module, with every consumed subject as a filter, because its ack permission is derived
// from its name: a module with a consumer per event would need an ack permission per consumer,
// and the permission list would stop being derivable from the declaration.
func ConsumerFor(p Principal) (Consumer, bool) {
// A module that reacts to anything — a module's events or a role's (novox/hq ADR 0121). Watching
// a role was missing here, so the one module that does it got no consumer at all: it started,
// connected, and its graph stayed empty with nothing anywhere reporting why.
if p.Kind != KindModule || (len(p.Consumes) == 0 && len(p.Watches) == 0) {
return Consumer{}, false
}
perms, err := PermissionsFor(p)
if err != nil {
return Consumer{}, false
}
// Events, wherever they live: a module's own namespace, and the namespace of any role it watches
// (novox/hq ADR 0121). Tool subjects and inboxes are subscribed directly and are not a consumer's
// business, which is why this is a filter and not the whole list.
var filters []string
for _, s := range perms.Subscribe {
if strings.Contains(s, ".event.") {
filters = append(filters, s)
}
}
if len(filters) == 0 {
return Consumer{}, false
}
sort.Strings(filters)
return Consumer{
Name: consumerDurable(p),
Stream: consumerStream(p),
Filters: filters,
AckWaitSeconds: 30,
MaxDeliver: 5,
Why: "what " + p.Module + " declared it consumes; after max-deliver it dead-letters",
}, true
}
// HolderConsumerFor is the worker a seat's holder gets on that seat's work queue.
//
// **A queue group even though the seat guarantees one holder.** The seat is *authority* — who may
// be the telegram sender — and the queue group is *delivery*. Tie delivery to the seat and the
// day somebody allows two holders for throughput, every message is processed twice with nothing
// reporting it. Kept separate, relaxing one changes nothing about the other.
func HolderConsumerFor(node, module string, seat DeclaredSeat) (Consumer, bool) {
if len(seat.Accepts) == 0 {
return Consumer{}, false
}
return Consumer{
Name: "SEAT_" + upperSnake(seat.Name) + "_worker",
Stream: seatStreamName(seat.Name),
Filters: []string{"mesh.seat." + seat.Name + ".accept.>"},
Queue: "holders",
AckWaitSeconds: 60,
MaxDeliver: 5,
Why: fmt.Sprintf("%s on %s holds %s; it acknowledges after the work is done, so a "+
"crash mid-work redelivers rather than loses", module, node, seat.Name),
}, true
}
// NodeConsumer is the durable consumer a node reads its own declaration through.
//
// **Derived from a node existing, and created by the controller, because a host cannot create it.**
// A host's account may subscribe its own declaration subject and publish its own ack subject, and
// reaches no part of the JetStream API — which is correct (the controller is the only writer of
// consumer definitions, design 25 §3) and means the consumer must be waiting before the host binds
// to it. Named after the node, because the node's ack grant is `$JS.ACK.NODES.<node>.>` and a
// consumer named anything else is one the host cannot acknowledge a delivery from.
//
// **No max-deliver, and a long ack wait.** A declaration is settled only after the node has applied
// it and reported, which is minutes on a machine pulling images; and a declaration the mesh cannot
// get a node to accept is not one to dead-letter, because the stream keeps only the newest per node
// anyway — so there is exactly one message per node to redeliver, for as long as that node is away.
func NodeConsumer(node string) Consumer {
return Consumer{
Name: node,
Stream: "NODES",
Filters: []string{"mesh.node." + node + ".declare"},
Push: true,
AckWaitSeconds: 300,
Why: "how " + node + " hears what it should be; last-per-subject, so a node that was away " +
"gets exactly the current declaration and nothing older",
}
}
// AssertNodeConsumers brings every known node's declaration consumer into being.
//
// Asserted on start as well as created at enrolment, for the reason the streams are: a mesh raised
// from a restored backup, or one whose bus was recreated, has node records and no consumers, and a
// node whose consumer is missing hears nothing while everything else about it looks correct.
func AssertNodeConsumers(e Ensurer, nodes []string) error {
for _, n := range nodes {
if err := e.EnsureConsumer(NodeConsumer(n)); err != nil {
return fmt.Errorf("asserting how %s hears its declaration: %w", n, err)
}
}
return nil
}
// AllOverlaps reports subject filters claimed by more than one stream, across the mesh's own and
// every derived one.
//
// NATS refuses an overlapping stream rather than merging it (verified against nats-server 2.10:
// "subjects overlap with an existing stream"), so this is not a subtle divergence — it is a
// registration that fails. Catching it here names both streams, before a half-applied mesh does.
func AllOverlaps(seats []DeclaredSeat) []string {
all := append(MeshStreams(), SeatStreams(seats)...)
seen := map[string]string{}
var clashes []string
for _, s := range all {
for _, subject := range s.Subjects {
if first, ok := seen[subject]; ok {
clashes = append(clashes, fmt.Sprintf("%s and %s both claim %s", first, s.Name, subject))
continue
}
seen[subject] = s.Name
}
}
sort.Strings(clashes)
return clashes
}
// upperSnake makes a stream name from a seat name. NATS stream names may not contain a dot,
// a space or a wildcard, and a hyphen is legal but reads badly beside the mesh's own.
func upperSnake(s string) string {
out := []rune(s)
for i, r := range out {
switch {
case r >= 'a' && r <= 'z':
out[i] = r - 32
case r == '-' || r == '.':
out[i] = '_'
}
}
return string(out)
}
-155
View File
@@ -1,155 +0,0 @@
package broker
import (
"strings"
"testing"
)
func telegramSeat() DeclaredSeat {
return DeclaredSeat{Name: "telegram-sender", Accepts: []string{"send"}}
}
// The stream exists from registration, not assignment: work queues until a holder appears, so
// installing the module a week later flushes the backlog rather than having lost it.
func TestASeatGetsAWorkQueueOfItsOwn(t *testing.T) {
got := SeatStreams([]DeclaredSeat{telegramSeat()})
if len(got) != 1 {
t.Fatalf("expected one stream, got %d", len(got))
}
s := got[0]
if s.Retention != RetentionWorkQueue {
t.Fatalf("a seat's inbound queue retains as %q; one holder must take each message once", s.Retention)
}
if s.Subjects[0] != "mesh.seat.telegram-sender.accept.>" {
t.Fatalf("filters on %v", s.Subjects)
}
}
// A seat that only emits and serves needs no stream: its events ride EVENTS and its tools are
// core request/reply, which is never persisted.
func TestASeatThatAcceptsNothingGetsNoStream(t *testing.T) {
if got := SeatStreams([]DeclaredSeat{{Name: "announcer"}}); len(got) != 0 {
t.Fatalf("a seat with no inbound work got %d stream(s)", len(got))
}
}
// Retention belongs to whoever owns the namespace, and a seat owns its own.
func TestASeatsRetentionIsItsOwn(t *testing.T) {
s := SeatStreams([]DeclaredSeat{{Name: "slow", Accepts: []string{"work"}, RetainSeconds: 30 * 24 * 60 * 60}})
if s[0].MaxAge != 30*24*60*60 {
t.Fatalf("the seat's declared retention was not used: %d", s[0].MaxAge)
}
d := SeatStreams([]DeclaredSeat{telegramSeat()})
if d[0].MaxAge == 0 {
t.Fatal("a seat that declares no retention got an unbounded queue")
}
}
// NATS refuses an overlapping stream outright, so a clash here is a registration that fails.
func TestNoDerivedStreamOverlapsTheMeshsOwn(t *testing.T) {
seats := []DeclaredSeat{telegramSeat(), {Name: "licensing-master", Accepts: []string{"report"}}}
if c := AllOverlaps(seats); len(c) != 0 {
t.Fatalf("overlapping filters: %v", c)
}
}
// One consumer per module, with every consumed subject as a filter — because its ack permission
// is derived from its name, and a consumer per event would need an ack permission per consumer.
func TestAModuleGetsOneConsumerCarryingEveryFilter(t *testing.T) {
c, ok := ConsumerFor(Principal{Kind: KindModule, Node: "one", Module: "audit",
Consumes: []string{"shop.order.placed", "billing.invoice.sent"}, PasswordHash: "x"})
if !ok {
t.Fatal("a module that consumes got no consumer")
}
if len(c.Filters) != 2 {
t.Fatalf("expected both subjects as filters, got %v", c.Filters)
}
perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "audit",
Consumes: []string{"shop.order.placed"}, PasswordHash: "x"})
ack := "$JS.ACK." + c.Stream + "." + c.Name + ".>"
found := false
for _, p := range perms.Publish {
if p == ack {
found = true
}
}
if !found {
t.Fatalf("the consumer is named %q but the ack permission is %v; a module could not ack "+
"its own deliveries", c.Name, perms.Publish)
}
}
// A module that subscribes to nothing needs no consumer, and creating one would leave an object
// nothing reads and everything has to maintain.
func TestAModuleThatConsumesNothingGetsNoConsumer(t *testing.T) {
if _, ok := ConsumerFor(Principal{Kind: KindModule, Node: "one", Module: "shop",
Emits: []string{"order.placed"}, PasswordHash: "x"}); ok {
t.Fatal("a pure emitter got a consumer")
}
}
// The seat is authority and the queue group is delivery. Tie them together and the day somebody
// allows two holders, every message is processed twice with nothing reporting it.
func TestAHoldersWorkerUsesAQueueGroupAnyway(t *testing.T) {
c, ok := HolderConsumerFor("one", "telegram", telegramSeat())
if !ok {
t.Fatal("the holder of a seat with inbound work got no worker")
}
if c.Queue == "" {
t.Fatal("the worker is not in a queue group, so a second holder would double-process")
}
if c.Stream != "SEAT_TELEGRAM_SENDER" {
t.Fatalf("the worker reads %q, not the seat's own stream", c.Stream)
}
if c.MaxDeliver == 0 {
t.Fatal("a failing worker would redeliver forever rather than dead-letter")
}
}
// The stream is named after the seat, not its holder: the holder can change and the queued work
// must not care.
func TestASeatsStreamIsNamedAfterTheSeat(t *testing.T) {
name := seatStreamName("telegram-sender")
if strings.Contains(name, "telegram-sender") {
t.Fatalf("%q keeps characters a stream name may not hold", name)
}
if name != "SEAT_TELEGRAM_SENDER" {
t.Fatalf("unexpected stream name %q", name)
}
}
// A node hears its declaration through a consumer only the controller can make.
//
// The three things that would each break it silently: a name other than the node's is one the host
// cannot acknowledge a delivery from, because its ack grant is derived from the node's name; a
// filter other than its own declaration subject is a node reading another's; and a pull consumer is
// one the host cannot bind a channel to without creating something, which it has no authority for.
func TestANodesDeclarationConsumerIsWhatItsOwnGrantAllows(t *testing.T) {
c := NodeConsumer("anchor")
if c.Name != "anchor" {
t.Fatalf("named %q, so the node cannot ack from it: its grant is $JS.ACK.NODES.anchor.>", c.Name)
}
if c.Stream != "NODES" {
t.Fatalf("on stream %q rather than the one declarations live in", c.Stream)
}
if len(c.Filters) != 1 || c.Filters[0] != "mesh.node.anchor.declare" {
t.Fatalf("filters %v, which is not this node's own declaration and nothing else", c.Filters)
}
if !c.Push {
t.Fatal("pulled, which a host cannot do: pulling needs the JetStream API and a host reaches none of it")
}
if c.MaxDeliver != 0 {
t.Fatalf("max-deliver %d: a declaration a node has not taken yet is not one to dead-letter, "+
"because the stream holds exactly one per node", c.MaxDeliver)
}
// And the grant the node actually gets has to match, or none of the above matters.
perms, err := PermissionsFor(Principal{Kind: KindNode, Node: "anchor"})
if err != nil {
t.Fatal(err)
}
// Without the ack grant every declaration a node receives is redelivered for ever; without the
// subscribe grant its consumer delivers to nobody.
has(t, perms.Publish, "$JS.ACK.NODES."+c.Name+".>")
has(t, perms.Subscribe, c.Filters[0])
}
-126
View File
@@ -1,126 +0,0 @@
package broker
import (
"encoding/json"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"testing"
"golang.org/x/crypto/bcrypt"
)
// **The first user list the installer carries must be the one the controller would compose.**
//
// At genesis there is no mesh to write the bus's user list, so the installer carries one: the
// controller's own account, at a bootstrap password, the way the store is reached at
// `postgres:bootstrap` (novox/hq design 25 §4, task 1.7). It is written by hand in a template and
// derived in code here, which is two statements of one fact — so this compares them.
//
// Getting it wrong is the worst kind of silent: a controller whose carried permissions are narrower
// than the ones it derives comes up, connects, and is refused on the first thing it tries, with an
// authorisation error that names a subject and not the template that forgot it. And a mesh cannot be
// raised twice to find out.
func TestTheInstallersFirstUserListIsWhatTheControllerWouldCompose(t *testing.T) {
accounts := theCarriedAccounts(t)
want, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
if err != nil {
t.Fatal(err)
}
carriedPub := subjectsIn(accounts, "publish")
carriedSub := subjectsIn(accounts, "subscribe")
if diff := missing(want.Publish, carriedPub); len(diff) > 0 {
t.Errorf("the installer's user list does not let the controller publish %v — it would come up "+
"and be refused on the first thing it tried", diff)
}
if diff := missing(want.Subscribe, carriedSub); len(diff) > 0 {
t.Errorf("the installer's user list does not let the controller subscribe %v", diff)
}
// And nothing wider than what it derives, or genesis quietly grants a privilege the composition
// takes away again on the first push.
if diff := missing(carriedPub, want.Publish); len(diff) > 0 {
t.Errorf("the installer's user list lets the controller publish %v, which it does not derive", diff)
}
if diff := missing(carriedSub, want.Subscribe); len(diff) > 0 {
t.Errorf("the installer's user list lets the controller subscribe %v, which it does not derive", diff)
}
// The credential is the bootstrap one and the hash really is of it, because a hash of something
// else is a controller that cannot log in to the bus it was just given.
hash := regexp.MustCompile(`\$2[aby]?\$[0-9]+\$[A-Za-z0-9./]{53}`).FindString(accounts)
if hash == "" {
t.Fatal("the installer's user list carries no password hash")
}
if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte("bootstrap")); err != nil {
t.Fatalf("the carried hash does not verify the bootstrap credential the template also carries: %v", err)
}
}
// theCarriedAccounts is the accounts file the installer's template writes at genesis.
func theCarriedAccounts(t *testing.T) string {
t.Helper()
path := filepath.Join("..", "..", "..", "mesh-host", "examples", "foundation-first-node-nats.lock")
raw, err := os.ReadFile(path)
if err != nil {
t.Skipf("the host's checkout is not beside this one: %v", err)
}
// The template is JSON with line comments, which is how every one of them is written.
var lines []string
for _, l := range strings.Split(string(raw), "\n") {
if !strings.HasPrefix(strings.TrimSpace(l), "//") {
lines = append(lines, l)
}
}
var bundle struct {
Resources []map[string]any `json:"resources"`
}
if err := json.Unmarshal([]byte(strings.Join(lines, "\n")), &bundle); err != nil {
t.Fatalf("the template is not readable: %v", err)
}
for _, r := range bundle.Resources {
if r["id"] == "bus-accounts" {
content, _ := r["content"].(string)
if content == "" {
t.Fatal("the template's accounts file is empty, so the bus would refuse every connection")
}
return content
}
}
t.Fatal("the template carries no accounts file, so a mesh raised from it has a bus nobody may use")
return ""
}
// subjectsIn reads one allow-list out of a composed accounts file.
func subjectsIn(accounts, which string) []string {
found := regexp.MustCompile(which + `: \{ allow: \[([^\]]*)\]`).FindStringSubmatch(accounts)
if len(found) != 2 {
return nil
}
var out []string
for _, part := range strings.Split(found[1], ",") {
if s := strings.Trim(strings.TrimSpace(part), `"`); s != "" {
out = append(out, s)
}
}
sort.Strings(out)
return out
}
// missing is what is in want and not in got.
func missing(want, got []string) []string {
have := map[string]bool{}
for _, g := range got {
have[g] = true
}
var out []string
for _, w := range want {
if !have[w] {
out = append(out, w)
}
}
return out
}
-213
View File
@@ -1,213 +0,0 @@
package broker
import (
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/hex"
"errors"
"fmt"
"os"
"strings"
"time"
"github.com/nats-io/nats.go"
)
// The JetStream side of the controller: the one place the mesh's streams and consumers are
// actually created.
//
// Everything that decides *what* they are is pure and lives beside this (streams.go, derived.go).
// This is only the part that talks to a server, kept small on purpose: a bug in a subject filter
// should be findable in a unit test, and only a bug in "did the server accept it" should need one
// running.
// A JetStream is a connection to the bus, as the controller uses it.
type JetStream struct {
conn *nats.Conn
js nats.JetStreamContext
}
// Dial connects and returns the controller's JetStream handle.
func Dial(url string, opts ...nats.Option) (*JetStream, error) {
opts = append(opts, nats.Name("mesh-controller"), nats.Timeout(10*time.Second))
// **Pinned, not named.** The bus presents the mesh's own certificate, which names nothing a
// public verifier would accept (design 25 §4: a host pins the server's exact certificate and
// checks nothing else, and so does this). Without this, the first connection failed with
// "certificate is not valid for any names" against a bus that was answering (2026-09-28).
if path := strings.TrimSpace(os.Getenv(CertificateVar)); path != "" {
pinned, err := pinnedTo(path)
if err != nil {
return nil, err
}
opts = append(opts, nats.Secure(pinned))
}
// **Its own inbox, and nothing wider.** Every principal is granted `_INBOX.<its user>.>` and
// no other inbox; the client's default prefix is random, and the server refused the first
// subscription to it (2026-09-28). The user is in the URL, so the prefix follows from it.
if user, _, _ := CredentialIn(url); user != "" {
opts = append(opts, nats.CustomInboxPrefix("_INBOX."+user))
}
// The address in an error is the address alone. The URL carries this controller's password,
// and an error here is written on the assumption it will be logged.
where := BareAddress(url)
conn, err := nats.Connect(url, opts...)
if err != nil {
return nil, fmt.Errorf("connecting to the bus at %s: %w", where, err)
}
js, err := conn.JetStream()
if err != nil {
conn.Close()
return nil, fmt.Errorf("the bus at %s has no JetStream: %w", where, err)
}
return &JetStream{conn: conn, js: js}, nil
}
// pinnedTo is a TLS configuration that accepts exactly the certificate in the file and no other:
// the leaf's SHA-256, compared on every handshake, with the name and the chain deliberately not
// consulted — a self-signed certificate with no names is the ordinary case for a mesh's bus.
func pinnedTo(path string) (*tls.Config, error) {
want, err := FingerprintOf(path)
if err != nil {
return nil, err
}
return PinnedToFingerprint(want), nil
}
// DialPinned is Dial with the server's certificate pinned by a fingerprint the caller already holds
// — a module or a build machine that was handed one beside its credential, and has no file.
func DialPinned(url, fingerprint string, opts ...nats.Option) (*JetStream, error) {
if strings.TrimSpace(fingerprint) != "" {
opts = append(opts, nats.Secure(PinnedToFingerprint(fingerprint)))
}
return Dial(url, opts...)
}
// PinnedToFingerprint accepts exactly the certificate with this SHA-256 and no other.
func PinnedToFingerprint(want string) *tls.Config {
return &tls.Config{
InsecureSkipVerify: true, //nolint:gosec // replaced by the pin below, which is stricter
MinVersion: tls.VersionTLS12,
VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
if len(rawCerts) == 0 {
return errors.New("the bus presented no certificate")
}
sum := sha256.Sum256(rawCerts[0])
got := "sha256:" + hex.EncodeToString(sum[:])
if got != want {
return fmt.Errorf("the bus presented a certificate this mesh does not know (%s…), expected %s…", got[:23], want[:23])
}
return nil
},
}
}
// Conn is the connection itself, for what the mesh keeps off JetStream on purpose — a heartbeat,
// a tool call — where a lost message is answered by the next one or by a timeout the caller
// already handles (design 25 §3).
func (j *JetStream) Conn() *nats.Conn { return j.conn }
// Context is the JetStream handle, for subscribing to what the consumers above define.
func (j *JetStream) Context() nats.JetStreamContext { return j.js }
func (j *JetStream) Close() {
if j.conn != nil {
j.conn.Close()
}
}
// EnsureStream creates the stream if it is absent and brings it to match if it is present.
//
// **Idempotent, because the controller asserts on every start** rather than creating once at
// genesis: a stream somebody deleted, or a mesh raised from a restored backup, has to converge
// rather than run without the guarantee its messages assume.
//
// An update, not a delete and recreate. Recreating would discard every message the stream holds
// and every consumer's position in it — which for CONTROL means the pushes being held through a
// store restart, exactly the guarantee the stream exists for.
func (j *JetStream) EnsureStream(s Stream) error {
want := &nats.StreamConfig{
Name: s.Name,
Subjects: s.Subjects,
Retention: retentionOf(s.Retention),
MaxAge: time.Duration(s.MaxAge) * time.Second,
MaxMsgsPerSubject: int64(s.MaxMsgsPerSubject),
Description: s.Why,
}
if s.Retention == RetentionLastPerSubject {
// Last-per-subject is a limits stream with one message kept per subject, not a
// retention policy of its own — the state shape, spelled the way the server spells it.
want.Retention = nats.LimitsPolicy
want.MaxMsgsPerSubject = 1
want.MaxAge = 0
}
switch _, err := j.js.StreamInfo(s.Name); {
case err == nil:
if _, err := j.js.UpdateStream(want); err != nil {
return fmt.Errorf("bringing stream %s to match: %w", s.Name, err)
}
return nil
case errors.Is(err, nats.ErrStreamNotFound):
if _, err := j.js.AddStream(want); err != nil {
return fmt.Errorf("creating stream %s: %w", s.Name, err)
}
return nil
default:
return fmt.Errorf("asking about stream %s: %w", s.Name, err)
}
}
// EnsureConsumer creates or updates one durable consumer.
//
// Explicit acknowledgement throughout: a consumer that acknowledges on delivery cannot redeliver
// work its holder died in the middle of, which is the whole difference between a queue and a
// firehose.
func (j *JetStream) EnsureConsumer(c Consumer) error {
want := &nats.ConsumerConfig{
Durable: c.Name,
AckPolicy: nats.AckExplicitPolicy,
AckWait: time.Duration(c.AckWaitSeconds) * time.Second,
MaxDeliver: c.MaxDeliver,
DeliverGroup: c.Queue,
DeliverSubject: "",
Description: c.Why,
}
switch len(c.Filters) {
case 0:
case 1:
want.FilterSubject = c.Filters[0]
default:
want.FilterSubjects = c.Filters
}
// A queue group needs a delivery subject: a pull consumer has no group, and declaring one
// without the other is refused by the server with a message that does not say which half is
// missing.
if c.Queue != "" || c.Push {
want.DeliverSubject = "_DELIVER." + c.Name
}
switch _, err := j.js.ConsumerInfo(c.Stream, c.Name); {
case err == nil:
if _, err := j.js.UpdateConsumer(c.Stream, want); err != nil {
return fmt.Errorf("bringing consumer %s on %s to match: %w", c.Name, c.Stream, err)
}
return nil
case errors.Is(err, nats.ErrConsumerNotFound):
if _, err := j.js.AddConsumer(c.Stream, want); err != nil {
return fmt.Errorf("creating consumer %s on %s: %w", c.Name, c.Stream, err)
}
return nil
default:
return fmt.Errorf("asking about consumer %s on %s: %w", c.Name, c.Stream, err)
}
}
func retentionOf(r Retention) nats.RetentionPolicy {
switch r {
case RetentionWorkQueue:
return nats.WorkQueuePolicy
default:
return nats.LimitsPolicy
}
}
-71
View File
@@ -1,71 +0,0 @@
package broker
import (
"os"
"testing"
)
// Against a real server, because the questions here are all "does the server accept this" —
// which a mock would answer by agreeing with whatever this file already believes.
//
// Skipped unless MESH_TEST_NATS names one, so the ordinary suite stays fast and offline:
//
// docker run -d --rm --name t -p 14222:4222 nats:2.10-alpine -js
// MESH_TEST_NATS=nats://127.0.0.1:14222 go test ./internal/broker/ -run TestAgainstARealServer
func TestAgainstARealServer(t *testing.T) {
url := os.Getenv("MESH_TEST_NATS")
if url == "" {
t.Skip("MESH_TEST_NATS unset")
}
js, err := Dial(url)
if err != nil {
t.Fatal(err)
}
defer js.Close()
t.Run("the mesh's own streams are accepted", func(t *testing.T) {
if err := AssertMeshStreams(js); err != nil {
t.Fatal(err)
}
})
t.Run("asserting again changes nothing and fails nothing", func(t *testing.T) {
if err := AssertMeshStreams(js); err != nil {
t.Fatalf("the second assertion failed, so the controller cannot restart: %v", err)
}
})
t.Run("a seat's work queue is accepted beside them", func(t *testing.T) {
seats := []DeclaredSeat{{Name: "telegram-sender", Accepts: []string{"send"}}}
for _, s := range SeatStreams(seats) {
if err := js.EnsureStream(s); err != nil {
t.Fatal(err)
}
}
if c := AllOverlaps(seats); len(c) != 0 {
t.Fatalf("overlaps the server would refuse: %v", c)
}
})
t.Run("a module's consumer is accepted and is idempotent", func(t *testing.T) {
c, ok := ConsumerFor(Principal{Kind: KindModule, Node: "one", Module: "audit",
Consumes: []string{"shop.order.placed", "billing.invoice.sent"}, PasswordHash: "x"})
if !ok {
t.Fatal("no consumer derived")
}
if err := js.EnsureConsumer(c); err != nil {
t.Fatal(err)
}
if err := js.EnsureConsumer(c); err != nil {
t.Fatalf("the second assertion failed: %v", err)
}
})
t.Run("a holder's worker is accepted with its queue group", func(t *testing.T) {
c, _ := HolderConsumerFor("one", "telegram",
DeclaredSeat{Name: "telegram-sender", Accepts: []string{"send"}})
if err := js.EnsureConsumer(c); err != nil {
t.Fatal(err)
}
})
}
+366
View File
@@ -0,0 +1,366 @@
package broker
import (
"bytes"
"context"
"encoding/json"
"fmt"
"github.com/novox/mesh-controller/internal/envfile"
"io"
"net/http"
"net/url"
"regexp"
"strings"
"time"
)
// The mesh runs the broker, so there is no chicken-and-egg in a node needing an account before it
// can connect: the account is created when the token is issued, and the one-time secret in that
// token IS the password. A node's first connection is already authenticated, and enrolment is
// what happens over it.
//
// novox/hq ADR 0004's *a node holds its own identity and nothing else* is why the account is per
// node rather than shared. A shared enrolment account would let any node consume another's queue,
// which is the shared-credential fault that record exists to remove, reappearing at the transport.
// ManagementVar holds the broker's management API, credentials included.
const ManagementVar = "MESH_BROKER_MANAGEMENT"
// safeName is what a node may be called at the broker.
//
// The name goes into a URL path and into permission patterns, which are regular expressions. A
// name carrying a `.` or a `*` would silently widen what that node may reach — so it is
// constrained here rather than escaped later, because an escape that is forgotten once is a node
// reading everybody's queues.
var safeName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`)
// Management is the broker's administrative interface.
type Management struct {
base *url.URL
client *http.Client
}
// ManagementFromEnvironment reads where the management API is, if it is configured.
//
// On the port MESH_BROKER_MANAGEMENT_PORT names when the node moved it (novox/hq 04-ISSUES/102);
// the URL's own port otherwise.
func ManagementFromEnvironment() (*Management, error) {
raw, err := envfile.Placed(ManagementVar)
if err != nil {
return nil, err
}
if raw == "" {
return nil, ErrNotConfigured
}
base, err := url.Parse(raw)
if err != nil || base.Host == "" {
// The value carries a password, so it is not quoted back.
return nil, fmt.Errorf("%s is not a usable URL", ManagementVar)
}
return &Management{base: base, client: &http.Client{Timeout: 15 * time.Second}}, nil
}
// QueueFor is the queue a node consumes from. One per node, named after it.
func QueueFor(node string) string { return "node." + node }
// ExchangeName is where nodes publish what they have to say. One exchange, and the control plane
// is the only consumer behind it (novox/hq ADR 0006 — one consumer, so two cannot silently split
// the traffic between them).
const ExchangeName = "mesh"
// BuildQueueName is where build work waits. Duplicated from `link` rather than imported, for the
// same reason QueueFor above is: this package must not depend on the one that uses it, and a
// constant that differed would be caught by the test that asserts they agree.
const BuildQueueName = "builds"
// The events bus (novox/hq ADR 0042): one topic exchange every event rides, a second for tool
// RPC kept apart, and a dead-letter home for a poison event. The foundation owns these — a module's
// account cannot declare them, only bind its own queue to the events one.
const (
EventsExchangeName = "mesh.events"
RPCExchangeName = "mesh.rpc"
DeadExchangeName = "mesh.events.dead"
)
// ModuleQueueFor is the durable queue a module consumes its events from — one per module per node
// (novox/hq ADR 0042), named so the account that may read it is exactly this module's.
func ModuleQueueFor(node, module string) string { return node + "." + module + ".events" }
// modulePermissions is a module's authority on the bus, derived from its manifest (novox/hq
// ADR 0043): what it consumes and what it emits, and nothing else. Pure, so the scope is tested as
// patterns without a broker — the way a builder's is.
//
// A note on the limit: the broker's write permission is per exchange, not per routing key (LavinMQ
// has no topic permissions), so an emitting module is granted the events exchange whole. ADR 0042's
// origin reservation — a module publishes only under `module.<self>.*` — is stamped by the sdk, not
// enforced here; that gap is the broker's, and is recorded rather than hidden. A pure consumer like
// the audit logger is unaffected: it is granted no write to the exchange at all.
func modulePermissions(node, module string, emits, consumes []string) (configure, write, read string) {
queue := regexp.QuoteMeta(ModuleQueueFor(node, module))
events := regexp.QuoteMeta(EventsExchangeName)
rpc := regexp.QuoteMeta(RPCExchangeName)
// A module serves each of its tools on its own queue, namespaced by the module (novox/hq
// ADR 0047) — serve.<module>.<tool> — so the account may declare, bind and read exactly its own,
// and no other module's.
serve := "serve\\." + regexp.QuoteMeta(module) + "\\..*"
// Declare its own events queue and its own tool serve queues.
configure = "^(" + queue + "|" + serve + ")$"
// Write to bind its queue and serve queues (binding is a write on the queue), and to the RPC
// exchange to publish replies (ADR 0047: replies ride mesh.rpc, never the default exchange, which
// would let it publish into any queue). To the events exchange only if it emits.
writes := []string{queue, serve, rpc}
if len(emits) > 0 {
writes = append(writes, events)
}
write = "^(" + strings.Join(writes, "|") + ")$"
// Read its own queue and serve queues to consume them, and the RPC exchange to bind its serve
// queues onto. The events exchange to bind onto only if it consumes.
reads := []string{queue, serve, rpc}
if len(consumes) > 0 {
reads = append(reads, events)
}
read = "^(" + strings.Join(reads, "|") + ")$"
return configure, write, read
}
// CreateModuleAccount gives an assigned module its own broker account, scoped by what it emits and
// consumes (novox/hq ADR 0043). The account name carries the node so the same module on two machines
// holds two accounts, each sealed to its own; the permissions carry the module so one module cannot
// read another's queue. Generic — the builder is one instance of this rule, not a separate kind.
func (m *Management) CreateModuleAccount(ctx context.Context, node, module, password string, emits, consumes []string) (string, error) {
if !safeName.MatchString(node) {
return "", fmt.Errorf("%q cannot be part of a broker account: it is a permission pattern", node)
}
if !safeName.MatchString(module) {
return "", fmt.Errorf("%q cannot be part of a broker account: it is a permission pattern", module)
}
account := node + "-" + module
if !safeName.MatchString(account) {
return "", fmt.Errorf("%q is not a usable broker account name", account)
}
if err := m.put(ctx, "/api/users/"+url.PathEscape(account),
map[string]string{"password": password, "tags": ""}); err != nil {
return "", fmt.Errorf("cannot create the broker account for %s on %s: %w", module, node, err)
}
configure, write, read := modulePermissions(node, module, emits, consumes)
if err := m.put(ctx, "/api/permissions/%2f/"+url.PathEscape(account), map[string]string{
"configure": configure, "write": write, "read": read,
}); err != nil {
return "", fmt.Errorf("cannot scope the broker account for %s on %s: %w", module, node, err)
}
return account, nil
}
// EnsureModuleQueue declares a consuming module's queue with its dead-letter exchange, idempotently.
// The foundation declares it because a scoped module account may not: the broker refuses a queue with
// a dead-letter exchange to a non-administrator (novox/hq ADR 0043), so a consumer passively checks
// the queue the mesh made rather than declaring its own.
func (m *Management) EnsureModuleQueue(ctx context.Context, node, module string) error {
queue := ModuleQueueFor(node, module)
if err := m.put(ctx, "/api/queues/%2f/"+url.PathEscape(queue), map[string]any{
"durable": true,
"arguments": map[string]any{"x-dead-letter-exchange": DeadExchangeName},
}); err != nil {
return fmt.Errorf("cannot declare the queue for %s on %s: %w", module, node, err)
}
return nil
}
// EnsureEventExchanges declares the bus's exchanges and the dead-letter home, idempotently. The
// foundation owns them (a module's account may not declare an exchange), and a dead-letter exchange
// with no queue behind it drops what it receives — so a durable queue bound to `#` retains a poison
// event for inspection, which is the whole reason the trail exists.
func (m *Management) EnsureEventExchanges(ctx context.Context) error {
for _, exchange := range []string{EventsExchangeName, RPCExchangeName, DeadExchangeName} {
if err := m.put(ctx, "/api/exchanges/%2f/"+url.PathEscape(exchange),
map[string]any{"type": "topic", "durable": true}); err != nil {
return fmt.Errorf("cannot declare the %s exchange: %w", exchange, err)
}
}
if err := m.put(ctx, "/api/queues/%2f/"+url.PathEscape(DeadExchangeName),
map[string]any{"durable": true}); err != nil {
return fmt.Errorf("cannot declare the dead-letter queue: %w", err)
}
if err := m.post(ctx, "/api/bindings/%2f/e/"+url.PathEscape(DeadExchangeName)+
"/q/"+url.PathEscape(DeadExchangeName), map[string]string{"routing_key": "#"}); err != nil {
return fmt.Errorf("cannot bind the dead-letter queue: %w", err)
}
return nil
}
// CreateNodeAccount gives a node its own broker account, with the token's secret as the password.
//
// Scoped so a node can reach its own queue and the one exchange, and nothing else. The patterns
// are anchored: a node called `laptop` must not be able to read `laptop-of-somebody-else`.
func (m *Management) CreateNodeAccount(ctx context.Context, node, password string) error {
if !safeName.MatchString(node) {
return fmt.Errorf(
"%q cannot be a broker account name: it becomes part of a permission pattern, so it "+
"is lower-case letters, digits and dashes", node)
}
if err := m.put(ctx, "/api/users/"+url.PathEscape(node),
map[string]string{"password": password, "tags": ""}); err != nil {
return fmt.Errorf("cannot create the broker account for %s: %w", node, err)
}
queue := regexp.QuoteMeta(QueueFor(node))
if err := m.put(ctx, "/api/permissions/%2f/"+url.PathEscape(node), map[string]string{
"configure": "^" + queue + "$",
"write": "^(" + regexp.QuoteMeta(ExchangeName) + "|" + queue + ")$",
"read": "^" + queue + "$",
}); err != nil {
return fmt.Errorf("cannot scope the broker account for %s: %w", node, err)
}
return nil
}
// CreateBuilderAccount scopes an account to taking build work and answering it.
//
// **A build machine is not a node**, and giving it a node's account would let it read another
// machine's declarations. What it needs is narrower and different: read the build queue, and
// write to the exchange and to whatever temporary queue an asker is waiting on.
//
// The reply queues are the reason `write` is not simply the exchange. `RequestBuild` declares an
// exclusive queue with a generated name and waits on it, so a builder that could not write to it
// could take work and never answer — which is the failure that looks like a builder that is not
// running.
func (m *Management) CreateBuilderAccount(ctx context.Context, name, password string) error {
if !safeName.MatchString(name) {
return fmt.Errorf(
"%q cannot be a broker account name: it becomes part of a permission pattern, so it "+
"is lower-case letters, digits and dashes", name)
}
if err := m.put(ctx, "/api/users/"+url.PathEscape(name),
map[string]string{"password": password, "tags": ""}); err != nil {
return fmt.Errorf("cannot create the broker account for %s: %w", name, err)
}
builds := regexp.QuoteMeta(BuildQueueName)
if err := m.put(ctx, "/api/permissions/%2f/"+url.PathEscape(name), map[string]string{
// It declares the build queue, because whichever builder starts first must be able to —
// and a queue nobody may declare is a queue that exists only if the control plane has
// already run, which makes the order they start in matter.
"configure": "^" + builds + "$",
// Two exchanges, and nothing else. **Not the default exchange**: permission there is
// granted per exchange rather than per queue, so a builder allowed to use it could
// publish into any node's queue — the privilege a build machine most obviously should
// not have. Answers go through the node exchange; announcing what was built goes through
// the events exchange, which is a different act with a different audience (novox/hq
// ADR 0072). A builder that could answer and not announce would leave the module graph
// knowing less than the registry does.
"write": "^(" + regexp.QuoteMeta(ExchangeName) + "|" + regexp.QuoteMeta(EventsExchangeName) + ")$",
// The build queue and nothing else. Not another machine's declarations.
"read": "^" + builds + "$",
}); err != nil {
return fmt.Errorf("cannot scope the broker account for %s: %w", name, err)
}
return nil
}
// RemoveNodeAccount withdraws a node's access.
func (m *Management) RemoveNodeAccount(ctx context.Context, node string) error {
if !safeName.MatchString(node) {
return fmt.Errorf("%q is not a broker account name", node)
}
return m.do(ctx, http.MethodDelete, "/api/users/"+url.PathEscape(node), nil)
}
// Accounts lists the broker's users, so a picture can be read from the system rather than assumed
// (novox/hq ADR 0018).
func (m *Management) Accounts(ctx context.Context) ([]string, error) {
body, err := m.get(ctx, "/api/users")
if err != nil {
return nil, err
}
var users []struct {
Name string `json:"name"`
}
if err := json.Unmarshal(body, &users); err != nil {
return nil, err
}
names := make([]string, 0, len(users))
for _, u := range users {
names = append(names, u.Name)
}
return names, nil
}
func (m *Management) put(ctx context.Context, path string, body any) error {
return m.do(ctx, http.MethodPut, path, body)
}
func (m *Management) post(ctx context.Context, path string, body any) error {
return m.do(ctx, http.MethodPost, path, body)
}
func (m *Management) get(ctx context.Context, path string) ([]byte, error) {
request, err := m.request(ctx, http.MethodGet, path, nil)
if err != nil {
return nil, err
}
response, err := m.client.Do(request)
if err != nil {
return nil, err
}
defer response.Body.Close()
if response.StatusCode >= 300 {
return nil, fmt.Errorf("the broker's management API answered %s to GET %s",
response.Status, path)
}
return io.ReadAll(io.LimitReader(response.Body, 1<<20))
}
func (m *Management) do(ctx context.Context, method, path string, body any) error {
request, err := m.request(ctx, method, path, body)
if err != nil {
return err
}
response, err := m.client.Do(request)
if err != nil {
return err
}
defer response.Body.Close()
if response.StatusCode >= 300 {
detail, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
return fmt.Errorf("the broker's management API answered %s to %s %s: %s",
response.Status, method, path, strings.TrimSpace(string(detail)))
}
return nil
}
func (m *Management) request(ctx context.Context, method, path string, body any) (*http.Request, error) {
var payload io.Reader
if body != nil {
raw, err := json.Marshal(body)
if err != nil {
return nil, err
}
payload = bytes.NewReader(raw)
}
// Path joined by hand rather than through url.Parse: %2f is the default vhost and must reach
// the broker still encoded. Parsing would decode it to a slash and address a different route.
target := strings.TrimSuffix(m.base.String(), "/")
if user := m.base.User; user != nil {
target = strings.TrimSuffix(m.base.Scheme+"://"+m.base.Host, "/")
}
request, err := http.NewRequestWithContext(ctx, method, target+path, payload)
if err != nil {
return nil, err
}
if user := m.base.User; user != nil {
password, _ := user.Password()
request.SetBasicAuth(user.Username(), password)
}
if body != nil {
request.Header.Set("Content-Type", "application/json")
}
return request, nil
}
+96
View File
@@ -0,0 +1,96 @@
package broker_test
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"regexp"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/broker"
)
// The audit logger consumes everything and emits nothing. Its account must let it declare and read
// its own queue and read the events exchange to bind onto — and must not reach another module's
// queue, nor grant any write to the events exchange (novox/hq ADR 0043).
func TestAConsumerReadsItsOwnQueueAndTheEventsExchangeAndNoOthers(t *testing.T) {
// Rebuilt through CreateModuleAccount's own path by asking for the same scope it would apply.
// The queue this module reads:
mine := broker.ModuleQueueFor("anchor", "audit-logger")
other := broker.ModuleQueueFor("anchor", "plex")
read := scope(t, "read", "anchor", "audit-logger", nil, []string{"#"})
if !read.MatchString(mine) {
t.Error("the audit logger may not read its own queue, so it consumes nothing")
}
if !read.MatchString(broker.EventsExchangeName) {
t.Error("the audit logger may not read the events exchange, so it cannot bind onto it")
}
if read.MatchString(other) {
t.Error("the audit logger may read another module's queue")
}
// It emits nothing, so it is granted no write to the events exchange — only its own queue, to bind.
write := scope(t, "write", "anchor", "audit-logger", nil, []string{"#"})
if write.MatchString(broker.EventsExchangeName) {
t.Error("a pure consumer was granted write to the events exchange")
}
if !write.MatchString(mine) {
t.Error("the audit logger may not write to its own queue, so it cannot bind it")
}
}
// An emitter is granted the events exchange to write; a consumer is not.
func TestAnEmitterMayWriteTheEventsExchangeAndAConsumerMayNot(t *testing.T) {
emitter := scope(t, "write", "anchor", "umami", []string{"module.umami.site.created"}, nil)
if !emitter.MatchString(broker.EventsExchangeName) {
t.Error("an emitting module may not write the events exchange, so it cannot emit")
}
}
// scope reconstructs one of the three permission patterns CreateModuleAccount would apply, by
// reading it back from a captured request against a stub management API.
func scope(t *testing.T, which, node, module string, emits, consumes []string) *regexp.Regexp {
t.Helper()
pat := capturePermission(t, which, node, module, emits, consumes)
re, err := regexp.Compile(pat)
if err != nil {
t.Fatalf("the %s pattern does not compile: %v", which, err)
}
return re
}
// capturePermission runs CreateModuleAccount against a stub management API and returns the pattern
// it set for `which` ("configure"/"write"/"read"). The scope is tested where it is applied, not
// reconstructed by the test — so a change to the mapping cannot pass a test that hard-codes the old
// one.
func capturePermission(t *testing.T, which, node, module string, emits, consumes []string) string {
t.Helper()
var captured map[string]string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasPrefix(r.URL.Path, "/api/permissions/") {
_ = json.NewDecoder(r.Body).Decode(&captured)
}
w.WriteHeader(http.StatusNoContent)
}))
defer server.Close()
t.Setenv(broker.ManagementVar, server.URL)
m, err := broker.ManagementFromEnvironment()
if err != nil {
t.Fatalf("stub management not usable: %v", err)
}
if _, err := m.CreateModuleAccount(context.Background(), node, module, "pw", emits, consumes); err != nil {
t.Fatalf("CreateModuleAccount: %v", err)
}
if captured == nil {
t.Fatal("no permissions were set")
}
pattern, ok := captured[which]
if !ok {
t.Fatalf("no %s permission was set; got %v", which, captured)
}
return pattern
}
-589
View File
@@ -1,589 +0,0 @@
// Composing the bus's own configuration.
//
// An account is *composed*, never called for: the controller writes accounts, users and
// per-subject permissions into one file the host keeps current, and the server reloads it in
// place (novox/hq ADR 0106 — never through a management API; design 25 §4).
//
// Everything here is pure. Given the principals, it returns the file's text — so the whole of the
// mesh's authority model is testable as strings, with no server.
//
// **Permissions are per subject, so a module's own name is the server's to enforce.** ADR 0042
// reserves a module's origin — it publishes only under its own name — and here that is a refusal
// rather than something a library promises.
package broker
import (
"errors"
"fmt"
"regexp"
"sort"
"strings"
)
// A Kind is what a principal is, which decides the shape of its authority rather than its
// contents: a module's comes from its declaration, a host's from its node, and the controller's
// and the enrolment user's are fixed.
type Kind string
const (
KindModule Kind = "module"
KindNode Kind = "node"
KindController Kind = "controller"
KindEnrolment Kind = "enrolment"
// KindPerson is somebody reaching the mesh's tools from a workstation (design 25 §7). Its
// authority is a list of tools and nothing else — not control, not declarations, not builds,
// and no ability to answer anything, because a person asks.
KindPerson Kind = "person"
)
// Seat is a role on the bus as a principal relates to it: the subjects it accepts, and those it
// emits (novox/hq ADR 0118, design 29 §5).
type Seat struct {
Name string
Accepts []string
Emits []string
Serves []string
Versions []string // protocol versions served beside the current one; empty for v1 only
}
// A Principal is one user of the bus. Its permissions are derived from what it declares and
// nothing else (novox/hq ADR 0043), over the three namespaces of design 29 §2: its own, the seats
// it holds, and the seats it uses.
type Principal struct {
Kind Kind
Node string
Module string
Emits []string
Consumes []string
Serves []string
Holds []Seat
Uses []Seat
// Watches are seats whose events this principal consumes. Separate from Consumes because a
// role's event lives under the seat's namespace and not a module's, and this package cannot tell
// a seat's name from a module's by looking at it — whoever resolved the declaration can, and
// does (novox/hq ADR 0121).
//
// **Found by a consumer reading nothing.** The catalogue consumes the build machine's outcome;
// with that name read as a module's, its subscription pointed at `mesh.mod.mesh-build-machine.…`,
// a namespace no such module owns. Every service started and the graph stayed empty.
Watches []Seat
// Invokes are the tools a person may call, as `<module>.<tool>`; a single `*` is every tool,
// for an administrator. Only meaningful for KindPerson.
//
// **A list, not a role.** A person is not a module and holds no seat: nothing is addressed
// to them, nothing is delivered to them, and they have no durable consumer to acknowledge.
// What they have is permission to ask.
Invokes []string
// PasswordHash is the bcrypt hash the mesh minted. The plaintext is sealed to the principal
// and never appears here: this file is written to a node's disk and read by a server, and a
// secret that can be read from a configuration file is a secret with a wider blast radius
// than the one it protects (novox/hq design 29 §10).
PasswordHash string
}
// meshSeatsTheControllerUses are the roles the mesh's own flows submit work to. Named rather than
// derived from the seat set: the controller is not a module and declares no `uses`, so its side of a
// seat has to be stated, and a list is what makes "which roles does the mesh itself talk to" answerable.
var meshSeatsTheControllerUses = []string{"mesh-build-machine"}
// enrolmentPrefix is the space every enrolling node's user and inbox live under, so the one place the
// controller may answer an enrolment is derived from the same constant the user is named from.
const enrolmentPrefix = "enrol"
// safeSubject refuses anything that would change the meaning of a subject rather than sit inside
// one. A name carrying a dot would silently widen a permission by adding a token; a name carrying
// `>` or `*` would widen it to a wildcard, which is the whole authority model gone.
var safeSubject = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)
// Username is how a principal is named to the server. The node is part of it, so the same module
// on two machines holds two users, each sealed to its own — the rule management.go already
// applies, kept.
func (p Principal) Username() string {
switch p.Kind {
case KindPerson:
return "person." + p.Module
case KindModule:
return p.Node + "." + p.Module
case KindNode:
return "node." + p.Node
case KindController:
return "controller"
case KindEnrolment:
// Per token, not one shared user. **The inbox is the reason**: with a single `enrolment`
// user every machine enrolling at once could read every other's answer, and an answer
// carries that node's credentials sealed to it. Design 25 §6 says the inbox a token
// derives, and a permission belongs to a user, so the user is per token.
//
// Named after the node, which **is** the token's id: a token is issued for a node record,
// the mesh holds one live claim per record, and the node's name is the one identifier both
// sides already have before anything else is agreed. It is also exactly what the other
// transport does, where the account is named after the node and the secret is its password.
return enrolmentPrefix + "." + p.Node
}
return ""
}
// inbox is a principal's own reply space. No user is ever granted a bare `_INBOX.>` (design 25
// §4): with one account, inbox privacy is the permission list or it is nothing, so each user's
// inbox is derived from its own identity and its permissions name that prefix and no other.
func (p Principal) inbox() string { return "_INBOX." + p.Username() + ".>" }
// Permissions is what a principal may publish and subscribe, and whether it may answer.
type Permissions struct {
Publish []string
Subscribe []string
// AllowResponses lets a principal reply to a request it received, on the reply subject that
// request carried, once.
//
// **This is what makes scoped inboxes possible at all**, and design 25 §4 did not say it. If
// every user's inbox is private to it, a module serving a tool cannot publish the answer —
// the answer goes to the *caller's* inbox, which the responder has no permission for. The two
// ways out are granting responders `_INBOX.>`, which is precisely the blanket grant §4
// refuses, or this: the server itself permits one reply to the subject of a message the user
// actually received, and nothing else. The authority is bounded by having been asked.
AllowResponses bool
}
// PermissionsFor derives a principal's authority. Pure, and the only place authority is decided:
// a permission that cannot be derived from a declaration is a permission nobody can explain.
func PermissionsFor(p Principal) (Permissions, error) {
for _, part := range []struct{ what, value string }{
{"node", p.Node}, {"module", p.Module},
} {
if part.value == "" {
continue
}
if !safeSubject.MatchString(part.value) {
return Permissions{}, fmt.Errorf(
"%q cannot be part of a subject: a permission is a subject pattern, and this would widen it", part.value)
}
}
var pub, sub []string
switch p.Kind {
case KindController:
// The controller owns the mesh's own traffic and the streams. It is the only writer of
// stream definitions (design 25 §3), so it alone reaches the JetStream API.
pub = []string{"mesh.control.>", "mesh.node.>", "$JS.API.>"}
// **And where its consumers deliver.** A push consumer delivers on `_DELIVER.<its name>`,
// and a client bound to it subscribes exactly that; the server refused it for every
// principal the first time one bound a consumer (2026-09-28). Each kind below is granted
// its own consumers' delivery subjects and no other's.
sub = []string{"mesh.control.>", "$JS.API.>", "_DELIVER." + ControllerName, "_DELIVER." + ControllerName + ".>"}
// Work the mesh's own flows submit to a role, and the outcomes they wait on (ADR 0121). A
// build is the one today: the controller asks, and reads the answer from the seat's event
// like the catalogue does — which is why no holder needs to publish into anybody's inbox.
for _, seat := range meshSeatsTheControllerUses {
pub = append(pub, "mesh.seat."+seat+".accept.>")
}
// Every module's tools: **the control plane is the way in** (novox/hq ADR 0095). A person
// or an agent asks through it and every question passes one process where an audit
// belongs — so it, alone among principals, may call any tool by name. The first `ask` on
// the new bus was refused the publish (2026-09-28).
pub = append(pub, "mesh.mod.*.tool.>")
// The two events it reacts to, and its ack subject on the stream they arrive from
// (streams.go). **Each named, not a pattern**: `mesh.mod.*.event.>` would make the
// controller a subscriber to every event in the mesh, and its permission list would stop
// saying what it is for. The ack grant below is scoped per stream because the controller's
// consumer name is the same on both and `$JS.ACK.CONTROL.controller.>` does not cover a
// delivery from EVENTS — a consumer that cannot ack has every message redelivered for
// ever, refused by the list it already has.
sub = append(sub, ControllerFollows...)
pub = append(pub, "$JS.ACK.EVENTS."+ControllerName+".>")
// **Where an enrolment's answer goes**, and `allow_responses` does not cover it. That
// permits one reply to the reply subject of a message the user received — and a message a
// JetStream consumer delivers has had that field claimed for the consumer's own ack address
// (design 25 §2), so the address the controller actually answers is the one the request
// carried in its payload, which is not a reply subject as the server understands it.
//
// Verified against a real server before this line existed: the answer was refused with
// "Permissions Violation for Publish to _INBOX.enrol.anchor…", and every enrolment on the
// mesh would have timed out while the controller logged success.
//
// **The enrolment inbox space, not a blanket `_INBOX.>`.** Design 25 §4 refuses that, and
// this is not it: nothing but an enrolling node ever subscribes under this prefix, each
// scoped to its own token's, so the controller publishing here is the mesh answering
// enrolments and can reach nothing else.
pub = append(pub, "_INBOX."+enrolmentPrefix+".>")
case KindPerson:
// Tools, and nothing else. Every subject a person may publish is a tool call; a person
// who could publish an event would be able to claim a module said something.
for _, t := range p.Invokes {
if t == "*" {
pub = append(pub, "mesh.mod.*.tool.>")
continue
}
module, tool, ok := strings.Cut(t, ".")
if !ok {
return Permissions{}, fmt.Errorf(
"%q does not name a tool: a person invokes <module>.<tool>, or * for every one", t)
}
pub = append(pub, "mesh.mod."+module+".tool."+tool)
}
case KindEnrolment:
// A leaked token is useless for anything but enrolling: it cannot read a declaration, hear
// an event, or subscribe any inbox but the one its own token derives (design 25 §6).
//
// **The inbox was missing and the handshake could not have completed without it.** An
// enrolling node publishes its request and waits on an address it states in the payload;
// with nothing to subscribe it waits out its timeout against a mesh that answered. Its own
// and no wider: `_INBOX.enrol.<node>.>`, so what is sealed to one machine cannot be read by
// another enrolling beside it.
if p.Node == "" {
// Refused rather than composed into `_INBOX.enrol..>`, which is a subject with an empty
// token in it — and worse, one every nameless enrolment user would share. A shared
// enrolment inbox is one machine able to read the credentials sealed to another.
return Permissions{}, errors.New(
"an enrolment user names no node, so its inbox would be shared with every other " +
"enrolment: a token is issued for a node record, and that record's name is " +
"the token's id")
}
pub = []string{"mesh.control.enrol"}
sub = []string{p.inbox()}
case KindNode:
// A host publishes its own node's control traffic and subscribes its own declaration —
// and nothing of any other node's.
// And binding to its consumer, which asks the server about it (CONSUMER.INFO) — the one
// thing the host does that nothing granted. Found the first time a machine dialled a
// permissioned server: "this node cannot read its declarations" (2026-09-28). The ack and
// the inbox are granted below with every principal's.
pub = []string{
"mesh.control." + p.Node + ".>",
"$JS.API.CONSUMER.INFO.NODES." + p.Node,
}
sub = []string{"mesh.node." + p.Node + ".declare", "_DELIVER." + p.Node}
case KindModule:
// 1. Its own namespace: it publishes its events there and serves its tools there. Nothing
// else may publish into it, so an event's source is a fact the server enforces rather
// than a claim in the body (design 29 §2).
own := "mesh.mod." + p.Module
for _, e := range p.Emits {
pub = append(pub, own+".event."+e)
}
// Every tool under its own name, not a list: the tools a module serves are what its code
// answers, and a second copy of that list in the manifest would be a second source of
// truth for the mesh to keep in step (2026-09-28: every module that served a tool was
// refused the subscription, because none had written the list twice). Nothing is given
// away — no other principal may subscribe this namespace, and a caller's authority is
// still granted per tool, by name, on the publish side.
sub = append(sub, own+".tool.>")
// 2. What it consumes, by the emitter's own subject — an event is addressed to its
// emitter, because the emitter's identity is the meaning (ADR 0118).
for _, c := range p.Consumes {
subject, err := consumedSubject(c)
if err != nil {
return Permissions{}, err
}
sub = append(sub, subject)
}
// 2b. Events of a role it watches, under the seat's own namespace. Subscribe only: watching a
// role is hearing what it announced, not taking part in it.
for _, w := range p.Watches {
for _, e := range w.Emits {
sub = append(sub, seatSubject(w, "event", e))
}
}
// 2c. Its own consumer, which it **pulls**: the runtime asks for the next message and is
// answered on its own inbox, so what it needs is to ask about the consumer and to ask it
// for messages — its own consumer's name, and no other's. Pulled rather than pushed
// because that is the one shape a runtime's client binds without creating anything; the
// controller and the hosts are pushed to. Named here rather than through ConsumerFor,
// which asks for these permissions to build the consumer and would ask forever. A
// subject for a consumer that turns out not to exist grants nothing anybody can use.
pub = append(pub,
"$JS.API.CONSUMER.INFO."+consumerStream(p)+"."+consumerDurable(p),
"$JS.API.CONSUMER.MSG.NEXT."+consumerStream(p)+"."+consumerDurable(p))
// 3. Seats it holds: full participation.
for _, s := range p.Holds {
// Taking work from the role's queue: the worker consumer it binds (asked about,
// delivered on, acknowledged), each on the seat's own stream. The first machine to
// take work over the new bus was refused the asking (2026-09-28).
worker := "SEAT_" + upperSnake(s.Name) + "_worker"
stream := seatStreamName(s.Name)
sub = append(sub, "_DELIVER."+worker)
pub = append(pub, "$JS.API.CONSUMER.INFO."+stream+"."+worker, "$JS.ACK."+stream+"."+worker+".>")
for _, a := range s.Accepts {
sub = append(sub, seatSubject(s, "accept", a))
}
for _, e := range s.Emits {
pub = append(pub, seatSubject(s, "event", e))
}
for _, t := range s.Serves {
sub = append(sub, seatSubject(s, "tool", t))
}
}
// 4. Seats it uses: publish only, and only the accepts half. A caller cannot subscribe a
// seat's inbound subject and watch other modules' traffic, nor publish its outbound
// events and lie about outcomes (design 29 §2).
for _, s := range p.Uses {
for _, a := range s.Accepts {
pub = append(pub, seatSubject(s, "accept", a))
}
for _, t := range s.Serves {
pub = append(pub, seatSubject(s, "tool", t))
}
}
}
if p.Kind == KindPerson {
// An inbox to hear answers in, and nothing else. No ack subject: a person has no durable
// consumer, because nothing is delivered to a person — they ask and are answered.
sub = append(sub, p.inbox())
}
if p.Kind == KindModule || p.Kind == KindNode || p.Kind == KindController {
// Its own reply space, and nothing wider.
sub = append(sub, p.inbox())
// Acking a JetStream delivery is a publish to that consumer's own ack address — a
// different subject from anything the consumer subscribes. Without it every message a
// module received would be redelivered forever, refused by the permission list it already
// has (design 25 §4). Scoped to this principal's own consumer name, so it can ack its own
// deliveries and no other's.
pub = append(pub, "$JS.ACK."+consumerStream(p)+"."+consumerDurable(p)+".>")
}
sort.Strings(pub)
sort.Strings(sub)
return Permissions{
Publish: pub,
Subscribe: sub,
// A module answers what it was asked — a tool call reaches it on its own namespace, so the
// authority is bounded by having been asked — and so does the controller. A node and a
// person are never asked anything, and are granted nothing here.
AllowResponses: p.Kind == KindModule || p.Kind == KindController,
}, nil
}
// seatSubject places a seat's verb under the kind of traffic it is.
//
// **The kind token is load-bearing, not decoration.** A stream is defined by a subject filter, so
// without it a stream over a seat or a module's namespace would capture that namespace's *tool*
// traffic too — and a tool call must never be persisted (design 25 §3: tools stay on core NATS).
// Found while defining the streams: the first draft of design 29 had one namespace per module
// with no kind, which reads well and cannot be filtered.
//
// A seat serving more than its current protocol version carries the version as a token
// (design 29 §8): the seat stays one role, and v1 and v2 run beside each other until nothing is
// bound to the old one.
func seatSubject(s Seat, kind, verb string) string {
return "mesh.seat." + s.Name + "." + kind + "." + verb
}
// consumerStream and consumerDurable are the two halves of a consumer's identity, and they are
// two functions because conflating them was a real bug.
//
// **A durable name may not contain a dot; an ack subject is built from two names that do.** The
// server acknowledges on `$JS.ACK.<stream>.<consumer>.…`, so a single string "EVENTS.one_audit"
// reads correctly inside the permission and is rejected as a consumer name — *nats: invalid
// consumer name*. Caught against a running server, and worth the comment because the shape of
// the failure if it had not been is the one design 25 §4 warns about: a consumer that cannot ack
// has every message redelivered forever, and its permission list looks right while it happens.
//
// They are derived here, beside the permission that must match them, because two places deriving
// the same name is how a module ends up unable to ack its own deliveries.
// consumedSubject is where a consumed event lands, from the local pattern a module declared.
//
// **The mesh's wildcards become this transport's** (design 29 §1): `*` is one name on both, and `**`
// — the rest — is `>` here. A module writes neither transport's spelling, so a manifest stays correct
// when the wire changes, which is the whole reason names are local.
//
// `**` on its own is every event from every module: the emitter is any, the event is anything. An
// audit logger wants exactly that and says so in one token.
func consumedSubject(pattern string) (string, error) {
if pattern == catalogueTheRest {
return "mesh.mod.*.event.>", nil
}
emitter, event, named := strings.Cut(pattern, ".")
if !named || emitter == "" || event == "" {
return "", fmt.Errorf(
"%q does not name an emitter and an event: a consumed event is <emitter>.<event>, or "+
"%q for every event", pattern, catalogueTheRest)
}
if emitter == catalogueTheRest {
return "", fmt.Errorf("%q stands for the rest of a name, so it cannot name the emitter", catalogueTheRest)
}
// Each name is checked before it becomes a subject: a name carrying a dot would add a token and
// silently widen the permission, which is the whole reason safeSubject exists.
var out []string
for _, part := range strings.Split(event, ".") {
switch part {
case catalogueTheRest:
out = append(out, ">")
case "*":
out = append(out, "*")
default:
if !safeSubject.MatchString(part) {
return "", fmt.Errorf("%q cannot be part of a subject: it would widen the permission", part)
}
out = append(out, part)
}
}
if emitter != "*" && !safeSubject.MatchString(emitter) {
return "", fmt.Errorf("%q cannot name an emitter: it would widen the permission", emitter)
}
return "mesh.mod." + emitter + ".event." + strings.Join(out, "."), nil
}
// catalogueTheRest is the mesh's wildcard for "the rest of a name", duplicated from the catalogue
// package for the one direction of dependency the build queue's name is duplicated for.
const catalogueTheRest = "**"
func consumerStream(p Principal) string {
switch p.Kind {
case KindModule:
return "EVENTS"
case KindNode:
return "NODES"
case KindController:
return "CONTROL"
}
return ""
}
func consumerDurable(p Principal) string {
switch p.Kind {
case KindModule:
return p.Node + "_" + p.Module
case KindNode:
return p.Node
case KindController:
return "controller"
}
return ""
}
// Server is everything the composed file needs that is not a principal.
type Server struct {
// ClientPort carries TLS itself. There is no plaintext port beside it: a bus reachable
// without TLS is one a module can reach without TLS by mistake.
ClientPort int
MonitoringPort int
TLSCert string
TLSKey string
TLSCA string
// StoreDir is a host directory bind, not a named volume — issue 115 is resolved and converted
// four modules away from named volumes; the bus's own data is not the place to bring one back.
StoreDir string
}
// Compose renders the server's whole configuration. The order is stable and the output is
// deterministic, because the file's digest is what the module's entrypoint watches to decide
// whether to reload: a composer that reordered a map on each run would signal a reload every time
// the controller restarted, for a file that had not changed.
func Compose(s Server, principals []Principal) (string, error) {
sorted := append([]Principal(nil), principals...)
sort.Slice(sorted, func(i, j int) bool { return sorted[i].Username() < sorted[j].Username() })
var b strings.Builder
b.WriteString("# Composed by the mesh controller. Do not edit: the next composition overwrites it.\n")
b.WriteString("# Accounts and permissions are derived from what each module declares and nothing\n")
b.WriteString("# else (novox/hq ADR 0043, design 29 §2).\n\n")
fmt.Fprintf(&b, "port: %d\n", s.ClientPort)
fmt.Fprintf(&b, "http: 127.0.0.1:%d\n\n", s.MonitoringPort)
// **No `verify`, and it said `verify: true` until this configuration was run.** That setting
// makes the server demand a *client* certificate, and nothing in the mesh presents one: a host
// pins this server's exact certificate and authenticates with the password the mesh minted
// (ADR 0004, design 25 §4), and so does a module's runtime. With it on, every connection in the
// mesh is refused at the TLS handshake before any password is looked at, and the error —
// "client didn't provide a certificate" — reads as a fault in the client.
//
// TLS is still required: the block is what requires it, and verify only decides whether client
// certificates are checked.
b.WriteString("tls {\n")
fmt.Fprintf(&b, " cert_file: %q\n", s.TLSCert)
fmt.Fprintf(&b, " key_file: %q\n", s.TLSKey)
fmt.Fprintf(&b, " ca_file: %q\n", s.TLSCA)
b.WriteString("}\n\n")
b.WriteString("jetstream {\n")
fmt.Fprintf(&b, " store_dir: %q\n", s.StoreDir)
b.WriteString("}\n\n")
accounts, err := ComposeAccounts(sorted)
if err != nil {
return "", err
}
b.WriteString(accounts)
return b.String(), nil
}
// ComposeAccounts is the accounts block alone — every user, and nothing about the server.
//
// **This is the only part of the configuration the mesh writes, and the split is deliberate.** A
// server's ports, its TLS paths and its store directory are properties of the container the module
// raises: they live in its image and its mounts, and they change when it does. The controller has no
// business knowing them, and a controller that did would have to be kept in step with a Dockerfile
// it never sees. What only the mesh knows is *who may connect*, so that is what it writes, and the
// module's own configuration includes it.
//
// Four things checked against a running server before this shape was committed to: a user in an
// included file authenticates; an unknown user is refused, so the include is the whole authority
// rather than an addition to something; a publish outside a user's grant is refused; and rewriting
// this file alone and signalling a reload makes a new user appear **without dropping the connection
// the mesh already has** — which is what makes every later account, permission or person change cost
// nothing (task 1.2's payoff).
func ComposeAccounts(principals []Principal) (string, error) {
sorted := append([]Principal(nil), principals...)
sort.Slice(sorted, func(i, j int) bool { return sorted[i].Username() < sorted[j].Username() })
var b strings.Builder
b.WriteString("# The mesh's users, composed by the controller. Do not edit: the next\n")
b.WriteString("# composition overwrites it. Permissions are derived from what each module\n")
b.WriteString("# declares and nothing else (novox/hq ADR 0043, design 29 §2).\n\n")
// One account for the mesh: accounts in NATS isolate subject spaces entirely, and the mesh is
// one space (design 25 §4). The cost of that — that permissions are the only isolation — is
// paid in the scoping of every inbox and every ack subject.
// JetStream is enabled per account once accounts exist at all: with only the global block set,
// a user in MESH is told "JetStream not enabled for account" the first time it binds a
// consumer, which is the first thing every host does (2026-09-28).
b.WriteString("accounts {\n MESH {\n jetstream: enabled\n users = [\n")
for _, p := range sorted {
perms, err := PermissionsFor(p)
if err != nil {
return "", err
}
if p.PasswordHash == "" {
return "", fmt.Errorf("%s has no password hash: a user without one is a user anybody is", p.Username())
}
fmt.Fprintf(&b, " { user: %q, password: %q, permissions: {\n", p.Username(), p.PasswordHash)
fmt.Fprintf(&b, " publish: { allow: [%s] }\n", quoted(perms.Publish))
fmt.Fprintf(&b, " subscribe: { allow: [%s] }\n", quoted(perms.Subscribe))
if perms.AllowResponses {
b.WriteString(" allow_responses: { max: 1, ttl: \"1m\" }\n")
}
b.WriteString(" } }\n")
}
b.WriteString(" ]\n }\n}\n")
return b.String(), nil
}
func quoted(values []string) string {
if len(values) == 0 {
return ""
}
out := make([]string, len(values))
for i, v := range values {
out[i] = fmt.Sprintf("%q", v)
}
return strings.Join(out, ", ")
}
-49
View File
@@ -1,49 +0,0 @@
package broker
import (
"flag"
"os"
"path/filepath"
"testing"
)
var update = flag.Bool("update", false, "rewrite the golden composition")
// The composed file is the mesh's whole authority model, so a change to it should be visible in a
// review rather than inferred from a diff of Go. The fixture is also the exact text checked
// against the real server's parser (`nats-server -t`), which is what says this syntax is the
// server's and not one we invented.
func TestTheComposedConfigMatchesTheGolden(t *testing.T) {
seat := Seat{Name: "telegram-sender", Accepts: []string{"send"}, Emits: []string{"delivered", "failed"}}
got, err := Compose(
Server{ClientPort: 4222, MonitoringPort: 8222, StoreDir: "/data",
TLSCert: "/tls/tls.crt", TLSKey: "/tls/tls.key", TLSCA: "/tls/ca.crt"},
[]Principal{
{Kind: KindController, PasswordHash: "$2a$11$cccccccccccccccccccccc"},
{Kind: KindEnrolment, Node: "one", PasswordHash: "$2a$11$eeeeeeeeeeeeeeeeeeeeee"},
{Kind: KindNode, Node: "one", PasswordHash: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn"},
{Kind: KindModule, Node: "one", Module: "telegram", Holds: []Seat{seat},
Serves: []string{"status"}, PasswordHash: "$2a$11$tttttttttttttttttttttt"},
{Kind: KindModule, Node: "two", Module: "shop", Uses: []Seat{seat},
Emits: []string{"order.placed"}, PasswordHash: "$2a$11$ssssssssssssssssssssss"},
{Kind: KindModule, Node: "two", Module: "audit",
Consumes: []string{"shop.order.placed"}, PasswordHash: "$2a$11$aaaaaaaaaaaaaaaaaaaaaa"},
})
if err != nil {
t.Fatal(err)
}
golden := filepath.Join("testdata", "composed.conf")
if *update {
if err := os.WriteFile(golden, []byte(got), 0o644); err != nil {
t.Fatal(err)
}
return
}
want, err := os.ReadFile(golden)
if err != nil {
t.Fatal(err)
}
if got != string(want) {
t.Errorf("composition changed; re-run with -update and read the diff:\n%s", got)
}
}
-361
View File
@@ -1,361 +0,0 @@
package broker
import (
"slices"
"strings"
"testing"
)
func has(t *testing.T, subjects []string, want string) {
t.Helper()
for _, s := range subjects {
if s == want {
return
}
}
t.Fatalf("expected %q among %v", want, subjects)
}
func hasNot(t *testing.T, subjects []string, unwanted string) {
t.Helper()
for _, s := range subjects {
if s == unwanted {
t.Fatalf("did not expect %q among %v", unwanted, subjects)
}
}
}
// A module's authority comes from its declaration and nothing else (novox/hq ADR 0043).
func TestAModulePublishesOnlyWhatItEmits(t *testing.T) {
p := Principal{Kind: KindModule, Node: "one", Module: "billing",
Emits: []string{"order.placed"}, PasswordHash: "x"}
perms, err := PermissionsFor(p)
if err != nil {
t.Fatal(err)
}
has(t, perms.Publish, "mesh.mod.billing.event.order.placed")
hasNot(t, perms.Publish, "mesh.mod.billing.>")
hasNot(t, perms.Publish, "mesh.mod.shipping.event.order.placed")
}
// The gap AMQP left open — an emitter granted the events exchange whole — is closed by per-subject
// permissions. A module cannot publish under another module's name.
func TestAModuleCannotPublishUnderAnothersName(t *testing.T) {
perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "billing",
Emits: []string{"order.placed"}, PasswordHash: "x"})
for _, p := range perms.Publish {
if strings.HasPrefix(p, "mesh.mod.") && !strings.HasPrefix(p, "mesh.mod.billing.") {
t.Fatalf("billing may publish %q, which is not its own namespace", p)
}
}
}
// A caller of a seat may publish what the seat accepts, and nothing else of it: not its outbound
// events, and not a subscription to its inbound queue (design 29 §2).
func TestUsingASeatIsPublishOnlyAndInboundOnly(t *testing.T) {
seat := Seat{Name: "telegram-sender", Accepts: []string{"send"}, Emits: []string{"delivered", "failed"}}
perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "shop",
Uses: []Seat{seat}, PasswordHash: "x"})
has(t, perms.Publish, "mesh.seat.telegram-sender.accept.send")
hasNot(t, perms.Publish, "mesh.seat.telegram-sender.event.delivered")
hasNot(t, perms.Subscribe, "mesh.seat.telegram-sender.accept.send")
}
// The holder is the mirror image: it consumes what the seat accepts and publishes what it emits.
func TestHoldingASeatIsTheMirrorOfUsingIt(t *testing.T) {
seat := Seat{Name: "telegram-sender", Accepts: []string{"send"}, Emits: []string{"delivered", "failed"}}
perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "telegram",
Holds: []Seat{seat}, PasswordHash: "x"})
has(t, perms.Subscribe, "mesh.seat.telegram-sender.accept.send")
has(t, perms.Publish, "mesh.seat.telegram-sender.event.delivered")
hasNot(t, perms.Publish, "mesh.seat.telegram-sender.accept.send")
}
// Without an ack permission a durable consumer never really consumes: every message it receives is
// redelivered forever, refused by the permission list it already has (design 25 §4).
func TestAModuleMayAckItsOwnDeliveriesAndNoOthers(t *testing.T) {
perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "billing",
Consumes: []string{"shop.order.placed"}, PasswordHash: "x"})
has(t, perms.Publish, "$JS.ACK.EVENTS.one_billing.>")
hasNot(t, perms.Publish, "$JS.ACK.>")
hasNot(t, perms.Publish, "$JS.ACK.EVENTS.one_shop.>")
}
// With one account, inbox privacy is the permission list or it is nothing.
func TestAnInboxIsScopedToItsOwner(t *testing.T) {
perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "billing", PasswordHash: "x"})
has(t, perms.Subscribe, "_INBOX.one.billing.>")
hasNot(t, perms.Subscribe, "_INBOX.>")
hasNot(t, perms.Subscribe, "_INBOX.one.shop.>")
}
// A responder answers on the caller's inbox, which it has no permission for. allow_responses is
// what makes a scoped inbox workable at all — the authority is bounded by having been asked. A
// module is asked on its own namespace and may answer; a node and a person are never asked.
func TestOnlyWhatCanBeAskedMayAnswer(t *testing.T) {
module, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "audit",
Consumes: []string{"shop.order.placed"}, PasswordHash: "x"})
if !module.AllowResponses {
t.Fatal("a module cannot answer a tool call on its own namespace")
}
node, _ := PermissionsFor(Principal{Kind: KindNode, Node: "one", PasswordHash: "x"})
if node.AllowResponses {
t.Fatal("a node was granted the right to answer, and nothing asks a node anything")
}
}
// A module serves every tool under its own name, and no other module's.
func TestAModuleServesItsOwnNamespaceAndNoOthers(t *testing.T) {
p, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "gitea", PasswordHash: "x"})
if !slices.Contains(p.Subscribe, "mesh.mod.gitea.tool.>") {
t.Fatalf("a module may not serve its own tools: %v", p.Subscribe)
}
for _, s := range p.Subscribe {
if strings.HasPrefix(s, "mesh.mod.") && !strings.HasPrefix(s, "mesh.mod.gitea.") {
t.Fatalf("a module may subscribe another's namespace: %s", s)
}
}
}
// A host reaches its own node's control traffic and its own declaration, and nothing of any
// other node's.
func TestAHostIsConfinedToItsOwnNode(t *testing.T) {
perms, _ := PermissionsFor(Principal{Kind: KindNode, Node: "one", PasswordHash: "x"})
has(t, perms.Publish, "mesh.control.one.>")
has(t, perms.Subscribe, "mesh.node.one.declare")
hasNot(t, perms.Subscribe, "mesh.node.two.declare")
hasNot(t, perms.Subscribe, "mesh.node.>")
}
// A leaked enrolment token is useless for anything but enrolling (design 25 §6).
func TestTheEnrolmentUserCanOnlyEnrol(t *testing.T) {
perms, err := PermissionsFor(Principal{Kind: KindEnrolment, Node: "anchor", PasswordHash: "x"})
if err != nil {
t.Fatal(err)
}
if len(perms.Publish) != 1 || perms.Publish[0] != "mesh.control.enrol" {
t.Fatalf("enrolment may publish %v", perms.Publish)
}
// Its own inbox and nothing else. **Nothing else** is the point: no declaration, no event, and
// no other machine's answer — and the inbox itself is needed, because a node that cannot
// subscribe one waits out its timeout against a mesh that answered.
if len(perms.Subscribe) != 1 || perms.Subscribe[0] != "_INBOX.enrol.anchor.>" {
t.Fatalf("enrolment may subscribe %v, which is not its own inbox alone", perms.Subscribe)
}
}
// An enrolment user that names no node is refused: its inbox would be an empty subject token, and
// one that every nameless enrolment user shared — which is one machine reading the credentials
// sealed to another.
func TestAnEnrolmentUserWithoutANodeIsRefused(t *testing.T) {
if _, err := PermissionsFor(Principal{Kind: KindEnrolment, PasswordHash: "x"}); err == nil {
t.Fatal("an enrolment user with no node was composed, so its inbox is shared")
}
}
// A name that would widen a permission is refused rather than quietly stretching one.
func TestANameThatWouldWidenAPermissionIsRefused(t *testing.T) {
for _, bad := range []string{"bill.ing", "billing.>", "*", "bil>ling"} {
if _, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: bad, PasswordHash: "x"}); err == nil {
t.Fatalf("%q was accepted as part of a subject", bad)
}
}
}
// The entrypoint reloads on the file's digest changing, so an unchanged mesh must compose an
// identical file — otherwise every controller restart signals a reload of the whole bus.
func TestComposingTwiceGivesTheSameBytes(t *testing.T) {
s := Server{ClientPort: 4222, MonitoringPort: 8222, StoreDir: "/data",
TLSCert: "/tls/tls.crt", TLSKey: "/tls/tls.key", TLSCA: "/tls/ca.crt"}
ps := []Principal{
{Kind: KindModule, Node: "two", Module: "shop", Emits: []string{"order.placed"}, PasswordHash: "b"},
{Kind: KindController, PasswordHash: "c"},
{Kind: KindModule, Node: "one", Module: "billing", Consumes: []string{"shop.order.placed"}, PasswordHash: "a"},
}
first, err := Compose(s, ps)
if err != nil {
t.Fatal(err)
}
shuffled := []Principal{ps[2], ps[0], ps[1]}
second, err := Compose(s, shuffled)
if err != nil {
t.Fatal(err)
}
if first != second {
t.Fatal("composition is order-dependent; every controller restart would reload the bus")
}
}
// A user without a password is a user anybody is.
func TestAUserWithoutAPasswordIsRefused(t *testing.T) {
_, err := Compose(Server{ClientPort: 4222}, []Principal{{Kind: KindController}})
if err == nil {
t.Fatal("composed a user with no password hash")
}
}
// A person reaches the mesh's tools from a workstation (design 25 §7). Their authority is a list
// of tools and nothing else.
func TestAPersonMayAskOnlyTheToolsTheyWereGiven(t *testing.T) {
perms, err := PermissionsFor(Principal{Kind: KindPerson, Module: "jo",
Invokes: []string{"shop.price", "telegram.status"}, PasswordHash: "x"})
if err != nil {
t.Fatal(err)
}
has(t, perms.Publish, "mesh.mod.shop.tool.price")
has(t, perms.Publish, "mesh.mod.telegram.tool.status")
hasNot(t, perms.Publish, "mesh.mod.shop.tool.refund")
hasNot(t, perms.Publish, "mesh.mod.*.tool.>")
}
// An administrator gets every tool, which is a different grant and looks like one.
func TestAnAdministratorMayAskAnyTool(t *testing.T) {
perms, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo",
Invokes: []string{"*"}, PasswordHash: "x"})
has(t, perms.Publish, "mesh.mod.*.tool.>")
}
// **Nothing but tools.** A person who could publish an event would be able to claim a module
// said something; one who could publish control traffic would be a second controller.
func TestAPersonReachesNothingButTools(t *testing.T) {
perms, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo",
Invokes: []string{"*"}, PasswordHash: "x"})
for _, p := range perms.Publish {
if !strings.Contains(p, ".tool.") {
t.Errorf("a person may publish %q, which is not a tool call", p)
}
}
for _, s := range perms.Subscribe {
if !strings.HasPrefix(s, "_INBOX.person.") {
t.Errorf("a person may subscribe %q; only their own inbox should be reachable", s)
}
}
}
// A person has no durable consumer, because nothing is delivered to a person — so no ack
// subject, and an ack permission would be authority over something that does not exist.
func TestAPersonHasNoAckSubject(t *testing.T) {
perms, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo",
Invokes: []string{"*"}, PasswordHash: "x"})
for _, p := range perms.Publish {
if strings.HasPrefix(p, "$JS.ACK") {
t.Errorf("a person was granted %q, and has no consumer to acknowledge", p)
}
}
}
// A person asks and is answered; they never answer. allow_responses would let a person reply to
// a request — which, on a bus where anyone may serve a tool, is somebody impersonating a module.
func TestAPersonMayNotAnswer(t *testing.T) {
perms, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo",
Invokes: []string{"*"}, PasswordHash: "x"})
if perms.AllowResponses {
t.Fatal("a person may answer a request, which is impersonating a module")
}
}
// Two people do not share an inbox, or one would read the other's answers.
func TestTwoPeopleDoNotShareAnInbox(t *testing.T) {
a, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo", Invokes: []string{"*"}, PasswordHash: "x"})
b, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "sam", Invokes: []string{"*"}, PasswordHash: "x"})
if a.Subscribe[0] == b.Subscribe[0] {
t.Fatalf("both read %s", a.Subscribe[0])
}
}
// A malformed grant is refused rather than widened into something that happens to parse.
func TestAToolGrantThatNamesNoToolIsRefused(t *testing.T) {
if _, err := PermissionsFor(Principal{Kind: KindPerson, Module: "jo",
Invokes: []string{"shop"}, PasswordHash: "x"}); err == nil {
t.Fatal("a grant naming a module but no tool was accepted")
}
}
// The controller can answer an enrolment, and reach no other inbox.
//
// **`allow_responses` does not cover this and that is the trap.** It permits one reply to the reply
// subject of a message the user received — and a message a JetStream consumer delivers has had that
// field claimed for the consumer's own ack address, so the address the controller actually answers is
// the one the request carried in its payload, which the server does not recognise as a reply subject
// at all.
//
// Found against a real server, after a live test on an *unpermissioned* one had passed: every
// enrolment on the mesh would have timed out while the controller logged success.
func TestTheControllerCanAnswerAnEnrolmentAndReachNoOtherInbox(t *testing.T) {
ctl, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
if err != nil {
t.Fatal(err)
}
enrolling, err := PermissionsFor(Principal{Kind: KindEnrolment, Node: "anchor", PasswordHash: "x"})
if err != nil {
t.Fatal(err)
}
// Whatever the enrolling node waits on, the controller must be able to publish to.
if len(enrolling.Subscribe) != 1 {
t.Fatalf("an enrolling node subscribes %v, and this test knows only how to check one",
enrolling.Subscribe)
}
waitsOn := enrolling.Subscribe[0]
if !covers(ctl.Publish, waitsOn) {
t.Fatalf("the controller may publish %v, none of which reaches %s — so every enrolment on "+
"the mesh times out while the controller logs success", ctl.Publish, waitsOn)
}
// And nothing wider. A node's own inbox and a module's are not the controller's to write into:
// that is the blanket grant design 25 §4 refuses.
for _, other := range []string{"_INBOX.node.anchor.x", "_INBOX.one.shop.x", "_INBOX.person.ada.x"} {
if covers(ctl.Publish, other) {
t.Errorf("the controller can publish to %s, which is an inbox privacy the permission "+
"list is the only thing protecting", other)
}
}
}
// covers says whether any granted subject pattern admits one concrete subject, with NATS's own
// wildcard meanings: `*` is one token, `>` is the rest.
func covers(granted []string, subject string) bool {
want := strings.Split(subject, ".")
for _, pattern := range granted {
if admits(strings.Split(pattern, "."), want) {
return true
}
}
return false
}
func admits(pattern, subject []string) bool {
for i, token := range pattern {
if token == ">" {
return i < len(subject)
}
if i >= len(subject) {
return false
}
if token != "*" && token != subject[i] {
return false
}
}
return len(pattern) == len(subject)
}
// A module pulls its own consumer — asks about it, asks it for messages — and no other module's.
func TestAModulePullsItsOwnConsumerAndNoOthers(t *testing.T) {
p, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "audit",
Consumes: []string{"shop.order.placed"}, PasswordHash: "x"})
for _, want := range []string{"$JS.API.CONSUMER.INFO.EVENTS.one_audit", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_audit"} {
if !slices.Contains(p.Publish, want) {
t.Errorf("a module cannot bind its own consumer: %v lacks %s", p.Publish, want)
}
}
for _, s := range p.Publish {
if strings.Contains(s, "CONSUMER.") && !strings.HasSuffix(s, ".one_audit") {
t.Errorf("a module may reach another consumer: %s", s)
}
}
for _, s := range p.Subscribe {
if strings.HasPrefix(s, "_DELIVER.") {
t.Errorf("a module is granted a push delivery it never binds: %s", s)
}
}
}
-83
View File
@@ -1,83 +0,0 @@
package broker
import (
"fmt"
"strings"
"github.com/novox/mesh-controller/internal/envfile"
)
// Whether this mesh's own traffic is on the bus being built.
//
// **One switch, read in one place** (novox/hq ADR 0116 step 5). Every seam the bus change went
// behind ships both implementations, and until the rollout every one of them chooses the bus the
// mesh runs on today. This is what the rollout flips, and it is deliberately a single fact rather
// than a fact per component: a controller whose outbound is on one bus and whose inbound is on the
// other is a mesh that hears nothing, and no test of either half would catch it.
// NATSVar is where the controller finds the bus being built. Unset is the ordinary case and means
// the mesh runs on the bus it has always run on.
const NATSVar = "MESH_BUS_NATS"
// OnNATS is the address of the bus being built, and whether the mesh is on it.
//
// Read from the node's own settings rather than baked in, for the reason the broker's address is
// (novox/hq 04-ISSUES/102): an address recorded once does not follow a node's ports.
func OnNATS() (address string, on bool, err error) {
address, err = envfile.Placed(NATSVar)
if err != nil {
return "", false, err
}
address = strings.TrimSpace(address)
if address == "" {
return "", false, nil
}
return address, true, nil
}
// CredentialIn reads the user and password out of a bus address, and the address without them.
//
// The controller's own credential arrives in its address, the way the old bus's does. Split out so the
// controller can record a hash of what it is actually using: its user is created by the installer at a
// bootstrap password, before the controller exists to mint one, and a composition that left itself out
// would produce a bus the writer cannot connect to.
func CredentialIn(address string) (user, password, bare string) {
at := strings.LastIndex(address, "@")
if at < 0 {
return "", "", address
}
scheme := ""
rest := address[:at]
if i := strings.Index(rest, "://"); i >= 0 {
scheme, rest = rest[:i+3], rest[i+3:]
}
user, password, _ = strings.Cut(rest, ":")
return user, password, scheme + address[at+1:]
}
// BareAddress is a bus address with any credential stripped, for something that only needs to know
// whether a server is answering there.
func BareAddress(address string) string {
_, _, bare := CredentialIn(address)
if bare == "" {
return address
}
if strings.Contains(bare, "://") {
return bare
}
return "nats://" + bare
}
// BusAddress is where the mesh's bus is, with this process's credential, and refuses to be empty:
// there is one bus, and a control plane without it can hold records and answer nothing (novox/hq
// ADR 0131, design 28 task 5.5).
func BusAddress() (string, error) {
address, _, err := OnNATS()
if err != nil {
return "", err
}
if address == "" {
return "", fmt.Errorf("this control plane has no %s, so it cannot reach the mesh's bus", NATSVar)
}
return address, nil
}
-28
View File
@@ -1,28 +0,0 @@
package broker
import (
"testing"
)
// Which bus the mesh is on is one fact, and being told about both is refused.
//
// The controller's own credential arrives in its address, and has to be readable out of it — its user
// is created by the installer at a bootstrap password, before the controller exists to mint one.
func TestACredentialIsReadOutOfABusAddress(t *testing.T) {
for _, c := range []struct{ in, user, password, bare string }{
{"nats://controller:secret@127.0.0.1:4222", "controller", "secret", "nats://127.0.0.1:4222"},
{"controller:secret@127.0.0.1:4222", "controller", "secret", "127.0.0.1:4222"},
{"nats://127.0.0.1:4222", "", "", "nats://127.0.0.1:4222"},
{"127.0.0.1:4222", "", "", "127.0.0.1:4222"},
// A password containing an at-sign: split on the last one, or the address becomes part of the
// credential and the connection goes somewhere nobody named.
{"nats://controller:a@b@127.0.0.1:4222", "controller", "a@b", "nats://127.0.0.1:4222"},
} {
user, password, bare := CredentialIn(c.in)
if user != c.user || password != c.password || bare != c.bare {
t.Errorf("%q read as %q/%q at %q; wanted %q/%q at %q",
c.in, user, password, bare, c.user, c.password, c.bare)
}
}
}
-73
View File
@@ -1,73 +0,0 @@
package broker
import "fmt"
// Bringing the bus's own objects into being, in the one order that works.
//
// **Asserted on every start rather than created once at genesis.** A stream somebody deleted, a mesh
// raised from a restored backup, or a bus whose data directory was replaced all have records and no
// objects — and a node whose consumer is missing hears nothing while everything else about it looks
// correct. Idempotence is the whole requirement, and the parts are already idempotent; this is the
// order they have to be asked in.
// Raiser is everything asserting the bus's objects needs of a connection to it.
type Raiser interface {
Asserter
Ensurer
}
// Raise asserts the mesh's streams, the controller's own consumers, and one consumer per node.
//
// **The order is not a preference.** A consumer on a stream that does not exist is refused, and the
// refusal names the stream rather than the order — so somebody reading it goes looking for a deleted
// stream instead of a reversed pair of lines. Nodes last, because the one a node reads lives on a
// stream the mesh's own set defines.
func Raise(r Raiser, nodes []string) error {
if err := AssertMeshStreams(r); err != nil {
return err
}
if err := AssertMeshConsumers(r); err != nil {
return err
}
if err := AssertNodeConsumers(r, nodes); err != nil {
return err
}
return nil
}
// RaiseSeats asserts one work queue per declared seat, and the worker of whoever holds it.
//
// Separate from Raise because it is answered by a different question: the mesh's own objects exist
// because the mesh does, and a seat's exist because a module declaring one was registered. Kept
// beside it so the order is visible — a holder's worker needs the seat's stream, and a seat's stream
// needs nothing.
func RaiseSeats(r Raiser, seats []DeclaredSeat, holders map[string]Holder) error {
for _, s := range SeatStreams(seats) {
if err := r.EnsureStream(s); err != nil {
return fmt.Errorf("asserting the work queue for %s: %w", s.Name, err)
}
}
for _, s := range seats {
h, held := holders[s.Name]
if !held {
// **The stream exists and the consumer does not, on purpose.** Work queues until a
// holder appears, so installing the module a week after something started sending to it
// flushes the backlog instead of having lost it.
continue
}
c, needed := HolderConsumerFor(h.Node, h.Module, s)
if !needed {
continue
}
if err := r.EnsureConsumer(c); err != nil {
return fmt.Errorf("asserting how %s on %s works %s: %w", h.Module, h.Node, s.Name, err)
}
}
return nil
}
// Holder is which module on which machine holds a seat.
type Holder struct {
Node string
Module string
}
-196
View File
@@ -1,196 +0,0 @@
package broker
import (
"os"
"testing"
"github.com/nats-io/nats.go"
)
// Raising the bus's objects against a real server.
//
// The pure tests above say what is asked for and in what order. Only a server can say whether it
// accepts them — and two of these are claims about the server's own behaviour that nothing else
// could answer: that asserting twice changes nothing, and that a consumer really is bound to the one
// subject its node is allowed to read.
//
// docker run -d --rm --name t -p 14227:4222 nats:2.10-alpine -js
// MESH_TEST_NATS=nats://127.0.0.1:14227 go test ./internal/broker/ -run TestRaising
func aLiveBus(t *testing.T) *JetStream {
t.Helper()
url := os.Getenv("MESH_TEST_NATS")
if url == "" {
t.Skip("MESH_TEST_NATS unset")
}
js, err := Dial(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
// **Nothing is deleted here, deliberately.** These objects are the mesh's own and every live
// test in every package shares one server: a test that deleted a stream to get a clean slate
// took it out from under whatever was running beside it, and the failure landed in the other
// test as "stream not found" — which reads as a bug in the code under test. Raise is idempotent
// by requirement, so asserting against whatever is already there is both safe and the realistic
// case.
return js
}
// Every object the mesh's own traffic needs, accepted by a real server, and asserting again changes
// nothing — which is the whole requirement, because this runs on every start.
func TestRaisingTheBusIsAcceptedAndIdempotent(t *testing.T) {
js := aLiveBus(t)
if err := Raise(js, []string{"anchor", "laptop"}); err != nil {
t.Fatalf("a real server refused the mesh's own objects: %v", err)
}
// Twice, with nothing in between. A start that failed the second time is a controller that
// cannot restart.
if err := Raise(js, []string{"anchor", "laptop"}); err != nil {
t.Fatalf("asserting the bus's objects a second time failed, so a restart would: %v", err)
}
// And again with a machine that was not there before, which is what enrolling one is.
if err := Raise(js, []string{"anchor", "laptop", "workstation"}); err != nil {
t.Fatalf("a machine joining an already-raised bus was refused: %v", err)
}
for _, s := range MeshStreams() {
if _, err := js.Context().StreamInfo(s.Name); err != nil {
t.Errorf("stream %s is not there: %v", s.Name, err)
}
}
for _, c := range MeshConsumers() {
if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); err != nil {
t.Errorf("the controller's consumer on %s is not there: %v", c.Stream, err)
}
}
for _, node := range []string{"anchor", "laptop", "workstation"} {
info, err := js.Context().ConsumerInfo("NODES", node)
if err != nil {
t.Errorf("%s has no way to hear its declaration: %v", node, err)
continue
}
// **Its own subject and no other node's.** A consumer filtered on anything wider is a node
// reading another machine's declaration, and its own ack grant would not cover it either.
if info.Config.FilterSubject != "mesh.node."+node+".declare" {
t.Errorf("%s's consumer reads %q", node, info.Config.FilterSubject)
}
if info.Config.AckPolicy != nats.AckExplicitPolicy {
t.Errorf("%s's consumer acknowledges on delivery, so a declaration it died applying is "+
"never sent again", node)
}
}
}
// The store window needs unlimited redelivery on CONTROL: the bound belongs to the controller, and a
// server that dead-lettered first would discard the push the stream exists to protect.
func TestTheControlConsumerDoesNotDeadLetterBeforeTheControllerGivesUp(t *testing.T) {
js := aLiveBus(t)
if err := Raise(js, nil); err != nil {
t.Fatal(err)
}
info, err := js.Context().ConsumerInfo("CONTROL", ControllerName)
if err != nil {
t.Fatal(err)
}
if info.Config.MaxDeliver > 0 {
t.Fatalf("max-deliver is %d: a push held through a store restart would be dead-lettered "+
"before the controller finished deciding about it", info.Config.MaxDeliver)
}
}
// A role's work queue exists before anybody holds it, against a real server.
//
// **The queue before the holder is the point** (novox/hq ADR 0121): work queues until somebody arrives
// to do it, so assigning a build machine a week after something started asking for builds flushes the
// backlog instead of having lost it. A stream created at assignment would make "the holder is not here
// yet" mean "your requests are gone".
func TestRaisingAMeshRolesWorkQueue(t *testing.T) {
js := aLiveBus(t)
seats := []DeclaredSeat{{Name: "mesh-build-machine", Accepts: []string{"build"},
Emits: []string{"built"}}}
t.Cleanup(func() { _ = js.Context().DeleteStream("SEAT_MESH_BUILD_MACHINE") })
if err := RaiseSeats(js, seats, nil); err != nil {
t.Fatalf("a real server refused a role's work queue: %v", err)
}
info, err := js.Context().StreamInfo("SEAT_MESH_BUILD_MACHINE")
if err != nil {
t.Fatalf("the role has no work queue: %v", err)
}
if info.Config.Retention != nats.WorkQueuePolicy {
t.Errorf("the queue retains as %v: work a holder took must leave it, or the next holder does "+
"it again", info.Config.Retention)
}
if len(info.Config.Subjects) != 1 || info.Config.Subjects[0] != "mesh.seat.mesh-build-machine.accept.>" {
t.Errorf("it carries %v rather than the role's own inbound subjects", info.Config.Subjects)
}
// Nobody holds it, so there is no worker — and asserting again changes nothing, because this runs
// on every start.
if err := RaiseSeats(js, seats, nil); err != nil {
t.Fatalf("asserting a role's queue a second time failed, so a restart would: %v", err)
}
// And once somebody holds it, the worker appears on that same queue.
if err := RaiseSeats(js, seats, map[string]Holder{
"mesh-build-machine": {Node: "anchor", Module: "builder"},
}); err != nil {
t.Fatal(err)
}
if _, err := js.Context().ConsumerInfo("SEAT_MESH_BUILD_MACHINE",
"SEAT_MESH_BUILD_MACHINE_worker"); err != nil {
t.Fatalf("the holder got no worker on the role's queue: %v", err)
}
}
// **A consumer created after the fact still sees what came before it**, which is why the mesh needs no
// catch-up at all on this bus (novox/hq 04-ISSUES/050).
//
// On the bus the mesh runs on today a queue receives only what is published after it is bound, so
// everything built before the catalogue existed was announced to nobody — and on a fresh mesh that is
// always the foundation, because those are the things the catalogue needed in order to exist. A whole
// mechanism was built for it: the catalogue asks, the controller re-publishes.
//
// A stream is a log and a consumer is a position in it. A consumer created later starts at the
// beginning by default, so the builds are simply there. Asked of a real server rather than assumed,
// because the whole decision about whether to keep that mechanism rests on it.
func TestAConsumerCreatedAfterwardsStillSeesWhatCameBefore(t *testing.T) {
js := aLiveBus(t)
if err := AssertMeshStreams(js); err != nil {
t.Fatal(err)
}
if err := js.Context().PurgeStream("EVENTS"); err != nil {
t.Fatal(err)
}
// Genesis: things are built before anything is listening.
built := []string{"base", "store", "mesh-catalog"}
for _, m := range built {
if _, err := js.Context().Publish("mesh.seat.mesh-build-machine.event.built",
[]byte(`{"module":"`+m+`"}`)); err != nil {
t.Fatal(err)
}
}
// Now the catalogue is installed and the controller creates its consumer.
c, ok := ConsumerFor(Principal{Kind: KindModule, Node: "one", Module: "mesh-catalog",
Watches: []Seat{{Name: "mesh-build-machine", Emits: []string{"built"}}}, PasswordHash: "x"})
if !ok {
t.Fatal("a module that watches a role got no consumer")
}
t.Cleanup(func() { _ = js.Context().DeleteConsumer(c.Stream, c.Name) })
if err := js.EnsureConsumer(c); err != nil {
t.Fatal(err)
}
info, err := js.Context().ConsumerInfo(c.Stream, c.Name)
if err != nil {
t.Fatal(err)
}
if info.NumPending != uint64(len(built)) {
t.Fatalf("a consumer created after %d builds has %d waiting for it — if this is 0 the mesh "+
"does need a catch-up after all, and the reasoning for deleting it is wrong",
len(built), info.NumPending)
}
}
-139
View File
@@ -1,139 +0,0 @@
package broker
import (
"fmt"
"sort"
"strings"
)
// Whether a mesh could move its bus, and what is missing if not.
//
// **Asked before anything moves, and answerable from records alone.** The rollout moves every node at
// once (novox/hq ADR 0116 step 5), so there is no partial state to inspect afterwards and no half to
// roll back: either the mesh was ready or it was not. That makes a readiness question the most
// valuable thing here — it costs nothing, it can be asked of a running mesh any number of times, and
// every answer is a thing somebody can go and fix.
//
// Deliberately pure. It is handed what the mesh knows and returns sentences; nothing here connects to
// anything, so it can be asked on a workstation about a mesh it has never reached.
// Readiness is what the mesh knows about its own ability to move.
type Readiness struct {
// TheBus is the address the mesh's own traffic would move to, empty when nothing names one.
TheBus string
// ServerStanding is whether a bus is reachable at that address, as somebody checked.
ServerStanding bool
// Holder is the node running the module that holds the bus seat, empty when nothing does.
Holder string
// AccountsComposed is whether that node has been sent the composed user list.
AccountsComposed bool
// Nodes is every machine the mesh knows.
Nodes []string
// Credentialled is which of them has a credential for the new bus.
Credentialled map[string]bool
// Modules is every assigned module, as `<node>/<module>`.
Modules []string
// ModuleCredentialled is which of those has one.
ModuleCredentialled map[string]bool
}
// NotReady is every reason this mesh cannot move its bus yet, in the order somebody would fix them.
//
// Empty means ready. **Each entry names one thing and what to do about it**, because a readiness check
// that says "not ready" is a check nobody can act on — and this is read at the point where the next
// step is irreversible.
func NotReady(r Readiness) []string {
var why []string
if strings.TrimSpace(r.TheBus) == "" {
why = append(why, "nothing names the bus to move to: set "+NATSVar+" on the control node "+
"to the address the new server answers on")
}
if !r.ServerStanding {
why = append(why, "no bus is answering at that address. Step 2 of the change raises it beside "+
"the one the mesh is on, carrying nothing — assign the module that holds "+
"mesh-broker and push the machine that runs it")
}
if r.Holder == "" {
why = append(why, "no machine holds mesh-broker, so nothing would compose the bus's user "+
"list. Assign the module that claims it")
} else if !r.AccountsComposed {
why = append(why, fmt.Sprintf(
"%s holds mesh-broker and has not been sent the composed user list, so the bus would "+
"refuse every connection. `push %s`", r.Holder, r.Holder))
}
// A node with no credential cannot come back after the move, and a node that cannot come back is
// a machine the mesh has lost until somebody goes to it.
var missing []string
for _, n := range r.Nodes {
if !r.Credentialled[n] {
missing = append(missing, n)
}
}
sort.Strings(missing)
if len(missing) > 0 {
why = append(why, fmt.Sprintf(
"%d machine(s) have no credential for the new bus and would not come back: %s. Each needs "+
"one minted before the move, not after — after, there is no bus to ask over",
len(missing), strings.Join(missing, ", ")))
}
// A module without one keeps running and stops being reachable, which is a smaller fault and still
// one somebody should choose rather than discover.
var quiet []string
for _, m := range r.Modules {
if !r.ModuleCredentialled[m] {
quiet = append(quiet, m)
}
}
sort.Strings(quiet)
if len(quiet) > 0 {
why = append(why, fmt.Sprintf(
"%d module(s) have no credential for the new bus: %s. Each keeps serving and stops "+
"answering tools and hearing events until it is issued one",
len(quiet), strings.Join(quiet, ", ")))
}
return why
}
// WhatMoves is what the rollout would do, in order, for somebody reading before they commit.
//
// **Written out rather than summarised.** This is the one step with nothing to inspect afterwards, so
// the last useful moment to disagree with it is while reading this.
func WhatMoves(r Readiness) []string {
out := []string{
fmt.Sprintf("compose the bus's user list and send it to %s", holderOr(r.Holder)),
fmt.Sprintf("move this control plane to %s, and confirm it is heard", busOr(r.TheBus)),
}
nodes := append([]string(nil), r.Nodes...)
sort.Strings(nodes)
for _, n := range nodes {
out = append(out, fmt.Sprintf("move %s, and confirm it reports", n))
}
if len(r.Modules) > 0 {
out = append(out, fmt.Sprintf("move %d module runtime(s), and confirm each answers",
len(r.Modules)))
}
// **The old broker goes, and it goes last** (novox/hq ADR 0131). AMQP is not a provision, so once
// every machine reports on the new bus nothing of the mesh is left speaking to it, and its module
// is unassigned. Said as a step so nobody reads the move as leaving a second bus behind.
out = append(out, "then unassign the old broker's module: AMQP is not a provision (ADR 0131), and "+
"once every machine reports on the new bus nothing of the mesh speaks to it")
return out
}
func holderOr(node string) string {
if node == "" {
return "whichever machine holds mesh-broker"
}
return node
}
func busOr(address string) string {
if address == "" {
return "the new bus"
}
return address
}
-98
View File
@@ -1,98 +0,0 @@
package broker
import (
"strings"
"testing"
)
// Whether a mesh could move its bus.
//
// Every case here is a way of moving that leaves something behind, and the one that matters most is a
// machine with no credential: after the move there is no bus to ask it over, so it is lost until
// somebody walks to it.
func aMeshReadyToMove() Readiness {
return Readiness{
TheBus: "nats://127.0.0.1:5671", ServerStanding: true,
Holder: "anchor", AccountsComposed: true,
Nodes: []string{"anchor", "laptop"},
Credentialled: map[string]bool{"anchor": true, "laptop": true},
Modules: []string{"anchor/gitea"},
ModuleCredentialled: map[string]bool{"anchor/gitea": true},
}
}
func TestAMeshWithEverythingInPlaceIsReady(t *testing.T) {
if why := NotReady(aMeshReadyToMove()); len(why) != 0 {
t.Fatalf("a mesh with everything in place was refused: %v", why)
}
}
// **A machine with no credential is the one that must stop this.** It keeps running and cannot come
// back, and there is no bus left to tell it anything over — so the remedy has to happen before, and
// the message says so.
func TestAMachineWithNoCredentialStopsTheMove(t *testing.T) {
r := aMeshReadyToMove()
r.Credentialled = map[string]bool{"anchor": true}
why := NotReady(r)
if len(why) == 0 {
t.Fatal("a machine that could not come back did not stop the move")
}
said := strings.Join(why, "\n")
if !strings.Contains(said, "laptop") {
t.Errorf("the refusal does not name the machine: %s", said)
}
if !strings.Contains(said, "before the move") {
t.Errorf("the refusal does not say the remedy comes first: %s", said)
}
}
// A bus nobody has raised, a seat nobody holds, and a user list nobody has been sent: each stops it,
// and each names its own next step, because "not ready" that cannot be acted on is not an answer.
func TestEachThingMissingNamesItsOwnRemedy(t *testing.T) {
for _, c := range []struct {
what string
break_ func(*Readiness)
says string
}{
{"no address", func(r *Readiness) { r.TheBus = "" }, NATSVar},
{"no server", func(r *Readiness) { r.ServerStanding = false }, "carrying nothing"},
{"no holder", func(r *Readiness) { r.Holder = "" }, "mesh-broker"},
{"no user list", func(r *Readiness) { r.AccountsComposed = false }, "push anchor"},
{"a module with none", func(r *Readiness) {
r.ModuleCredentialled = map[string]bool{}
}, "anchor/gitea"},
} {
r := aMeshReadyToMove()
c.break_(&r)
why := NotReady(r)
if len(why) == 0 {
t.Errorf("%s did not stop the move", c.what)
continue
}
if !strings.Contains(strings.Join(why, "\n"), c.says) {
t.Errorf("%s: the refusal does not mention %q: %v", c.what, c.says, why)
}
}
}
// What the move would do is written out rather than summarised, because this is the one step with
// nothing to inspect afterwards — so reading it is the last chance to disagree.
func TestWhatMovesNamesEveryMachineAndEndsWithTheOldBrokerGoing(t *testing.T) {
r := aMeshReadyToMove()
steps := strings.Join(WhatMoves(r), "\n")
for _, want := range []string{"anchor", "laptop", "user list", "module runtime"} {
if !strings.Contains(steps, want) {
t.Errorf("the plan does not mention %q:\n%s", want, steps)
}
}
// Said explicitly, and last: AMQP is not a provision (novox/hq ADR 0131), so the move ends with
// the old broker's module unassigned, not left behind as a second bus. An earlier version of this
// test pinned the opposite, under a record 0131 superseded.
lines := WhatMoves(r)
if last := lines[len(lines)-1]; !strings.Contains(last, "unassign the old broker") {
t.Errorf("the plan does not end with the old broker going:\n%s", steps)
}
}
-241
View File
@@ -1,241 +0,0 @@
package broker
import (
"fmt"
"sort"
)
// The mesh's own streams.
//
// **These four and no more** (novox/hq ADR 0116 task 1.4, as revised by ADR 0118). An earlier
// reading had the controller create *every* stream at genesis, from a fixed set. That is only the
// mesh's own half: a seat's streams are created when the module declaring it is registered, and a
// module's durable consumers when it is assigned — neither of which has happened at genesis. What
// is here is the foundation, which exists before any module does.
//
// The controller is the only writer of stream definitions (design 25 §3). A module declares
// nothing about them and cannot reach the JetStream API to make one.
// Retention is how a stream decides what to keep, which is the whole of what distinguishes the
// mesh's four relationships on the wire (design 29 §4).
type Retention string
const (
// RetentionWorkQueue: a message is removed once a consumer acknowledges it. Exactly one
// worker does the work, and a worker that dies has its message redelivered.
RetentionWorkQueue Retention = "workqueue"
// RetentionLastPerSubject: only the newest message on each subject survives. This is the
// state shape — a node that was away gets exactly the current declaration and nothing older.
RetentionLastPerSubject Retention = "last_per_subject"
// RetentionLimits: kept until it ages or the stream fills. Events, where a subscriber that
// was down catches up and nobody is obliged to act.
RetentionLimits Retention = "limits"
)
// A Stream is one of the mesh's own, as the controller asserts it.
type Stream struct {
Name string
Subjects []string
Retention Retention
// MaxAge in seconds, zero for unbounded. Per stream — JetStream has no per-subject age,
// which is why differing retention between modules would mean a stream each.
MaxAge int
// MaxMsgsPerSubject caps each subject independently, so one noisy emitter cannot push
// another's events out of a shared stream. Verified: with a cap of 3, ten messages on one
// subject and one on another leave four in the stream, not three.
MaxMsgsPerSubject int
// Why is carried into the assertion so an operator reading the server's own state finds the
// reason there, rather than only in a repository they may not have.
Why string
}
// MeshStreams is the foundation set, in the order a person reads it.
//
// **CONTROL names its subjects rather than taking `mesh.control.>`**, because heartbeats live
// under that prefix and must not be persisted: a lost heartbeat is the next heartbeat, and a
// stream of them is a stream of the least valuable messages the mesh sends, competing for the
// same retention as the ones that matter.
//
// **EVENTS filters on the `event` token**, which is the reason that token exists. A module's
// namespace carries both its events and its tool calls; a filter of `mesh.mod.*.>` would persist
// every tool invocation in the mesh, and a tool call must never be persisted (design 25 §3 keeps
// tools on core NATS, where a lost call is a timeout the caller already handles).
func MeshStreams() []Stream {
return []Stream{
{
Name: "CONTROL",
// A build's outcome is no longer here: it is the build-machine seat's own event, so one
// publish reaches whoever asked, the controller and the catalogue (novox/hq ADR 0121).
Subjects: []string{"mesh.control.*.report", "mesh.control.enrol"},
Retention: RetentionWorkQueue,
Why: "the store-window guarantee (ADR 0083): the controller naks with a delay while its " +
"store is away and the message is redelivered; nothing is dropped",
},
{
Name: "NODES",
Subjects: []string{"mesh.node.*.declare"},
Retention: RetentionLastPerSubject,
Why: "one declaration per node, always the newest; a node that sees sequence n refuses " +
"n-1 by construction (issue 107)",
},
{
Name: "EVENTS",
// A seat's own events ride here too: they are 1:many like any event, and the
// `event` token keeps them clear of both the seat's work queue (`accept`) and its
// tools (`tool`), which must not be persisted.
Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"},
Retention: RetentionLimits,
MaxAge: 7 * 24 * 60 * 60,
MaxMsgsPerSubject: 10000,
Why: "a subscriber that was down catches up; tool traffic under the same prefix is " +
"excluded by the event token; per-subject caps keep a noisy emitter from " +
"evicting a quiet one without splitting the stream",
},
}
}
// An Asserter is the part of a JetStream connection stream assertion needs. Narrow on purpose: it
// keeps this testable without a server, and keeps the client library out of everything that only
// wants to know what the streams are.
type Asserter interface {
// EnsureStream creates the stream if absent and updates it to match if present. It must be
// idempotent: the controller asserts on every start, not only at genesis.
EnsureStream(s Stream) error
}
// AssertMeshStreams brings the foundation set into being, in order, and says which one failed
// rather than that something did.
//
// Asserted on every start rather than created once at genesis, because a stream that was deleted,
// or a mesh raised from a restored backup, must converge rather than run without the guarantee
// its messages assume. Idempotence is the whole requirement.
func AssertMeshStreams(a Asserter) error {
for _, s := range MeshStreams() {
if err := a.EnsureStream(s); err != nil {
return fmt.Errorf("asserting stream %s: %w", s.Name, err)
}
}
return nil
}
// Overlaps reports subject filters claimed by more than one stream.
//
// **Corrected against the server**: an earlier version of this comment said NATS accepts
// overlapping streams and stores the message twice. It does not — it refuses the second stream
// with "subjects overlap with an existing stream" (verified against nats-server 2.10). The check
// still earns its place, for a different reason: the server's refusal arrives when the controller
// is applying, naming one stream, at a moment when the mesh is half-configured. This one arrives
// where the set is written, names both, and cannot reach a running mesh.
//
// It also decides a design question. Because overlap is refused rather than merged, a shared
// EVENTS stream and a per-module stream cannot coexist — the module's would be refused — so
// "one stream for most, its own for a module that wants different retention" is not an option
// the server allows. It is all of one or all of the other.
func Overlaps() []string {
seen := map[string]string{}
var clashes []string
for _, s := range MeshStreams() {
for _, subject := range s.Subjects {
if first, ok := seen[subject]; ok {
clashes = append(clashes, fmt.Sprintf("%s and %s both claim %s", first, s.Name, subject))
continue
}
seen[subject] = s.Name
}
}
sort.Strings(clashes)
return clashes
}
// The mesh's own consumers.
//
// A seat's streams and a module's consumers are derived from declarations (derived.go). These two
// are not: **the controller is not a module and files no manifest**, so its authority and its
// subscriptions cannot come from a declaration that does not exist. They are named here, where the
// mesh's own streams are named, and narrowly — a controller subscribing `mesh.mod.*.event.>` would
// hear every event in the mesh, which it has no business doing and which would make its permission
// list stop explaining anything.
// ControllerName is the controller's durable consumer on each stream it reads, and the name its
// ack subject is derived from (nats.go: `$JS.ACK.<stream>.controller.>`).
const ControllerName = "controller"
// ControllerFollows are the events the controller reacts to: the catalogue saying a module's
// current version moved, and a catalogue that has just started saying it may have missed builds.
//
// **Derived the same way a module's subscription is**, from the emitter and the bare local event
// name, rather than written out. They were written out while the catalogue still spelled its events
// as the old bus's routing keys, and the moment those were converted (novox/hq 04-ISSUES/127) a
// hard-coded pair became a controller listening to a subject nothing publishes — the same fault, from
// the other side. Deriving them means the conversion could not leave these behind.
var ControllerFollows = []string{
moduleEventSubject("mesh-catalog", "upgraded"),
moduleEventSubject("mesh-catalog", "catching-up"),
// A build's outcome, which is the build-machine role's own event now (ADR 0121) rather than a
// message on the control branch. Same three audiences, one publish: whoever asked, this, and the
// catalogue.
seatEventSubject("mesh-build-machine", "built"),
// The forge's merges: what moved a source, so the mesh builds what that source produces
// without anybody telling it (novox/hq 04-ISSUES/131). Appended, because the index is a name.
moduleEventSubject("gitea", "pull.merged"),
}
// moduleEventSubject is where one module's event lands. The same derivation PermissionsFor uses, so
// what the controller subscribes and what the emitter is permitted to publish cannot drift apart.
func moduleEventSubject(module, event string) string {
return "mesh.mod." + module + ".event." + event
}
// seatEventSubject is where a role's own event lands, derived the same way a holder's permission is.
func seatEventSubject(seat, verb string) string {
return "mesh.seat." + seat + ".event." + verb
}
// MeshConsumers is what the controller consumes, in the order a person reads it.
//
// **Unlimited redelivery on CONTROL, deliberately.** The store window's bound is the controller's,
// not the server's (window.go): a message is held with a nak-and-delay until the controller either
// takes it or gives up and says so. A max-deliver here would dead-letter a push that was being
// held through a store restart — the exact message the stream exists to protect — some minutes
// before the controller had finished deciding about it.
func MeshConsumers() []Consumer {
return []Consumer{
{
Name: ControllerName,
Stream: "CONTROL",
Push: true,
AckWaitSeconds: 30,
Why: "the controller is the single consumer of what nodes say; explicit ack and no " +
"max-deliver, because the store window's bound is the controller's own",
},
{
Name: ControllerName,
Stream: "EVENTS",
Filters: ControllerFollows,
Push: true,
AckWaitSeconds: 30,
MaxDeliver: 5,
Why: "the two events the mesh's own controller reacts to; after max-deliver it " +
"dead-letters, because an announcement it cannot act on will not become actionable",
},
}
}
// Ensurer is the part of a JetStream connection consumer assertion needs, narrow for the reason
// Asserter is.
type Ensurer interface {
EnsureConsumer(c Consumer) error
}
// AssertMeshConsumers brings the controller's own consumers into being, and says which one failed.
//
// After the streams, necessarily: a consumer on a stream that does not exist is refused, and the
// refusal names the stream rather than the order.
func AssertMeshConsumers(e Ensurer) error {
for _, c := range MeshConsumers() {
if err := e.EnsureConsumer(c); err != nil {
return fmt.Errorf("asserting consumer %s on %s: %w", c.Name, c.Stream, err)
}
}
return nil
}
-235
View File
@@ -1,235 +0,0 @@
package broker
import (
"errors"
"strings"
"testing"
)
type recorder struct {
seen []Stream
fail string
}
func (r *recorder) EnsureStream(s Stream) error {
if s.Name == r.fail {
return errors.New("refused")
}
r.seen = append(r.seen, s)
return nil
}
// The controller asserts on every start, not only at genesis: a stream that was deleted, or a mesh
// raised from a backup, must converge rather than run without the guarantee its messages assume.
func TestAssertingTwiceIsTheSameAsOnce(t *testing.T) {
a, b := &recorder{}, &recorder{}
if err := AssertMeshStreams(a); err != nil {
t.Fatal(err)
}
if err := AssertMeshStreams(a); err != nil {
t.Fatal(err)
}
if err := AssertMeshStreams(b); err != nil {
t.Fatal(err)
}
if len(a.seen) != 2*len(b.seen) {
t.Fatalf("asserted %d then %d; assertion is not repeatable", len(a.seen), len(b.seen))
}
}
func TestAFailedAssertionNamesItsStream(t *testing.T) {
err := AssertMeshStreams(&recorder{fail: "NODES"})
if err == nil || !strings.Contains(err.Error(), "NODES") {
t.Fatalf("got %v, which does not say which stream failed", err)
}
}
// Two streams matching one subject is accepted by NATS and stores the message twice under two
// retentions. Nothing reports that, so it is refused where the set is written.
func TestNoTwoStreamsClaimTheSameSubject(t *testing.T) {
if clashes := Overlaps(); len(clashes) != 0 {
t.Fatalf("overlapping subject filters: %v", clashes)
}
}
// A heartbeat under mesh.control.> must not be persisted: a lost one is the next one, and a
// stream of them competes for retention with the messages that matter.
func TestHeartbeatsAreNotInTheControlStream(t *testing.T) {
for _, s := range MeshStreams() {
for _, subject := range s.Subjects {
if subject == "mesh.control.>" || strings.Contains(subject, "alive") {
t.Fatalf("stream %s claims %q, which captures heartbeats", s.Name, subject)
}
}
}
}
// The reason the kind token exists: a filter over a module's whole namespace would persist every
// tool call in the mesh.
func TestTheEventsStreamDoesNotCaptureToolCalls(t *testing.T) {
var events Stream
for _, s := range MeshStreams() {
if s.Name == "EVENTS" {
events = s
}
}
// Nothing a tool call rides may match any of the filters — a module's or a seat's.
for _, tool := range []string{
"mesh.mod.billing.tool.status",
"mesh.seat.telegram-sender.tool.status",
"mesh.seat.telegram-sender.accept.send", // work, not an event: its own stream
} {
for _, f := range events.Subjects {
if subjectMatches(f, tool) {
t.Fatalf("%q matches the events filter %q, so it would be persisted here", tool, f)
}
}
}
// And both kinds of event do match.
for _, event := range []string{
"mesh.mod.billing.event.order.placed",
"mesh.seat.telegram-sender.event.delivered",
} {
matched := false
for _, f := range events.Subjects {
if subjectMatches(f, event) {
matched = true
}
}
if !matched {
t.Fatalf("%q matches no events filter, so nothing would keep it", event)
}
}
}
// subjectMatches is NATS subject matching, enough for these filters: `*` is one token, `>` is the
// rest.
func subjectMatches(filter, subject string) bool {
f, s := strings.Split(filter, "."), strings.Split(subject, ".")
for i, tok := range f {
if tok == ">" {
return i <= len(s)
}
if i >= len(s) {
return false
}
if tok != "*" && tok != s[i] {
return false
}
}
return len(f) == len(s)
}
// Each relationship's retention is the thing that makes it what it is (design 29 §4).
func TestEachStreamCarriesTheRetentionItsShapeNeeds(t *testing.T) {
want := map[string]Retention{
"CONTROL": RetentionWorkQueue,
"NODES": RetentionLastPerSubject,
"EVENTS": RetentionLimits,
}
got := map[string]Retention{}
for _, s := range MeshStreams() {
got[s.Name] = s.Retention
if s.Why == "" {
t.Errorf("stream %s says no reason it exists", s.Name)
}
}
if len(got) != len(want) {
t.Fatalf("the foundation set is %v", got)
}
for name, r := range want {
if got[name] != r {
t.Errorf("%s retains as %q, expected %q", name, got[name], r)
}
}
}
// The order the bus's objects are asserted in, because getting it wrong is a refusal that names the
// wrong thing: a consumer on a stream that does not exist is refused naming the *stream*, so
// somebody reading it goes looking for a deletion instead of a reversed pair of lines.
func TestTheBusesObjectsAreAssertedStreamsBeforeConsumers(t *testing.T) {
r := &recording{}
if err := Raise(r, []string{"anchor", "laptop"}); err != nil {
t.Fatal(err)
}
// Every stream before every consumer.
firstConsumer := -1
for i, step := range r.steps {
if strings.HasPrefix(step, "consumer ") && firstConsumer < 0 {
firstConsumer = i
}
if strings.HasPrefix(step, "stream ") && firstConsumer >= 0 {
t.Fatalf("a stream was asserted after a consumer: %v", r.steps)
}
}
if firstConsumer < 0 {
t.Fatalf("no consumer was asserted: %v", r.steps)
}
// And every node got one, named after it — without which that node hears nothing while
// everything else about it looks correct.
for _, node := range []string{"anchor", "laptop"} {
if !containsStep(r.steps, "consumer NODES/"+node) {
t.Errorf("%s was given no way to hear its declaration: %v", node, r.steps)
}
}
// And the controller its own, on both streams it reads.
for _, want := range []string{"consumer CONTROL/controller", "consumer EVENTS/controller"} {
if !containsStep(r.steps, want) {
t.Errorf("the controller is missing %s: %v", want, r.steps)
}
}
}
// A seat's work queue is asserted whether or not anybody holds it; the holder's worker only when
// somebody does. **The stream without the consumer is the point**: work queues until a holder
// appears, so installing the module later flushes the backlog instead of having lost it.
func TestASeatsQueueExistsBeforeItsHolderDoes(t *testing.T) {
seats := []DeclaredSeat{{Name: "telegram-sender", Accepts: []string{"send"}}}
unheld := &recording{}
if err := RaiseSeats(unheld, seats, nil); err != nil {
t.Fatal(err)
}
if !containsStep(unheld.steps, "stream SEAT_TELEGRAM_SENDER") {
t.Fatalf("a declared seat got no work queue: %v", unheld.steps)
}
for _, step := range unheld.steps {
if strings.HasPrefix(step, "consumer ") {
t.Fatalf("a seat nobody holds got a worker: %v", unheld.steps)
}
}
held := &recording{}
if err := RaiseSeats(held, seats, map[string]Holder{
"telegram-sender": {Node: "anchor", Module: "telegram"},
}); err != nil {
t.Fatal(err)
}
if !containsStep(held.steps, "consumer SEAT_TELEGRAM_SENDER/SEAT_TELEGRAM_SENDER_worker") {
t.Fatalf("the seat's holder got no worker: %v", held.steps)
}
}
// recording is a connection to the bus that writes down what it was asked for.
type recording struct{ steps []string }
func (r *recording) EnsureStream(s Stream) error {
r.steps = append(r.steps, "stream "+s.Name)
return nil
}
func (r *recording) EnsureConsumer(c Consumer) error {
r.steps = append(r.steps, "consumer "+c.Stream+"/"+c.Name)
return nil
}
func containsStep(steps []string, want string) bool {
for _, s := range steps {
if s == want {
return true
}
}
return false
}
-56
View File
@@ -1,56 +0,0 @@
# Composed by the mesh controller. Do not edit: the next composition overwrites it.
# Accounts and permissions are derived from what each module declares and nothing
# else (novox/hq ADR 0043, design 29 §2).
port: 4222
http: 127.0.0.1:8222
tls {
cert_file: "/tls/tls.crt"
key_file: "/tls/tls.key"
ca_file: "/tls/ca.crt"
}
jetstream {
store_dir: "/data"
}
# The mesh's users, composed by the controller. Do not edit: the next
# composition overwrites it. Permissions are derived from what each module
# declares and nothing else (novox/hq ADR 0043, design 29 §2).
accounts {
MESH {
jetstream: enabled
users = [
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>"] }
subscribe: { allow: ["$JS.API.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built"] }
allow_responses: { max: 1, ttl: "1m" }
} }
{ user: "enrol.one", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: {
publish: { allow: ["mesh.control.enrol"] }
subscribe: { allow: ["_INBOX.enrol.one.>"] }
} }
{ user: "node.one", password: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn", permissions: {
publish: { allow: ["$JS.ACK.NODES.one.>", "$JS.API.CONSUMER.INFO.NODES.one", "mesh.control.one.>"] }
subscribe: { allow: ["_DELIVER.one", "_INBOX.node.one.>", "mesh.node.one.declare"] }
} }
{ user: "one.telegram", password: "$2a$11$tttttttttttttttttttttt", permissions: {
publish: { allow: ["$JS.ACK.EVENTS.one_telegram.>", "$JS.ACK.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker.>", "$JS.API.CONSUMER.INFO.EVENTS.one_telegram", "$JS.API.CONSUMER.INFO.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_telegram", "mesh.seat.telegram-sender.event.delivered", "mesh.seat.telegram-sender.event.failed"] }
subscribe: { allow: ["_DELIVER.SEAT_TELEGRAM_SENDER_worker", "_INBOX.one.telegram.>", "mesh.mod.telegram.tool.>", "mesh.seat.telegram-sender.accept.send"] }
allow_responses: { max: 1, ttl: "1m" }
} }
{ user: "two.audit", password: "$2a$11$aaaaaaaaaaaaaaaaaaaaaa", permissions: {
publish: { allow: ["$JS.ACK.EVENTS.two_audit.>", "$JS.API.CONSUMER.INFO.EVENTS.two_audit", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_audit"] }
subscribe: { allow: ["_INBOX.two.audit.>", "mesh.mod.audit.tool.>", "mesh.mod.shop.event.order.placed"] }
allow_responses: { max: 1, ttl: "1m" }
} }
{ user: "two.shop", password: "$2a$11$ssssssssssssssssssssss", permissions: {
publish: { allow: ["$JS.ACK.EVENTS.two_shop.>", "$JS.API.CONSUMER.INFO.EVENTS.two_shop", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_shop", "mesh.mod.shop.event.order.placed", "mesh.seat.telegram-sender.accept.send"] }
subscribe: { allow: ["_INBOX.two.shop.>", "mesh.mod.shop.tool.>"] }
allow_responses: { max: 1, ttl: "1m" }
} }
]
}
}
-127
View File
@@ -1,127 +0,0 @@
package broker
import (
"fmt"
"sort"
)
// Every user the composed file should contain, derived from what the mesh knows.
//
// **The list is derived, never kept.** A stored user list would be a second account of who may
// reach the bus, able to disagree with the records it came from — and the disagreement would be
// invisible, because both would look internally consistent. So this is a pure function of the
// mesh's records, run again every time the file is written.
//
// Records are mirrored into this package's own types rather than imported from the catalogue, for
// the reason DeclaredSeat is: composing authority is a different job from parsing a manifest, and
// this package stays free of the other's types so a change to a manifest field cannot quietly widen
// a permission.
// Declared is one module on one node, as composing its authority needs it.
type Declared struct {
Module string
Emits []string
Consumes []string
Serves []string
// Holds are the seats this module claims, with the protocol each seat declares. A seat the
// mesh defines for itself declares no protocol, so holding one grants nothing on the bus —
// which is right: those seats are about who does a job, not about who may say what.
Holds []Seat
// Uses are the seats this module sends to.
Uses []Seat
// Watches are the seats whose events it consumes.
Watches []Seat
}
// Records is what composing a user list needs to know about the mesh, and nothing more.
type Records struct {
// Nodes is every machine the mesh knows. Each gets a host user.
Nodes []string
// Assigned is the modules on each node, as they declare themselves.
Assigned map[string][]Declared
// Enrolling is every node with a live token — one enrolment user each, because the inbox an
// answer goes to is scoped to the token and a shared one is one machine reading another's
// sealed credentials (design 25 §6).
Enrolling []string
// People is each person's name against the tools they may invoke, `*` for an administrator.
People map[string][]string
}
// Users is every user the composed file should contain, in the order it will be written.
//
// The controller is always first and always present: a mesh whose own controller is not in the file
// is a mesh that cannot be told anything, and there is no state of the records in which that is
// correct.
func Users(r Records) ([]Principal, error) {
out := []Principal{{Kind: KindController}}
for _, node := range sortedCopy(r.Nodes) {
out = append(out, Principal{Kind: KindNode, Node: node})
for _, d := range r.Assigned[node] {
out = append(out, Principal{
Kind: KindModule, Node: node, Module: d.Module,
Emits: d.Emits, Consumes: d.Consumes, Serves: d.Serves,
Holds: d.Holds, Uses: d.Uses, Watches: d.Watches,
})
}
}
for _, node := range sortedCopy(r.Enrolling) {
out = append(out, Principal{Kind: KindEnrolment, Node: node})
}
for _, person := range sortedNames(r.People) {
out = append(out, Principal{Kind: KindPerson, Module: person, Invokes: r.People[person]})
}
// Refused here rather than discovered by the server. Two users with one name is a file the
// server reads as one of them, and which one depends on the order — so a module assigned to a
// node twice, or a person named after nothing, is a composition that must not be written.
seen := map[string]string{}
for _, p := range out {
name := p.Username()
if name == "" || name == "." {
return nil, fmt.Errorf("a %s user has no name, so nothing could authenticate as it", p.Kind)
}
if first, already := seen[name]; already {
return nil, fmt.Errorf(
"two users would be called %q (a %s and a %s): the server would read the file as "+
"one of them, and which one depends on the order", name, first, p.Kind)
}
seen[name] = string(p.Kind)
}
return out, nil
}
// WithPasswords fills each user's hash from what the mesh minted, and says which users have none.
//
// **Separated from Users because they fail differently.** A user missing from the records is a bug
// in deriving them; a user with no password is a step that has not happened yet — a module assigned
// but never given a credential, a node enrolled before this existed. The second is ordinary and its
// remedy is to mint one, so it is named rather than returned as an error, and the caller decides
// whether a partial composition is worth writing.
func WithPasswords(principals []Principal, hashes map[string]string) (filled []Principal, missing []string) {
for _, p := range principals {
hash, ok := hashes[p.Username()]
if !ok || hash == "" {
missing = append(missing, p.Username())
continue
}
p.PasswordHash = hash
filled = append(filled, p)
}
return filled, missing
}
func sortedCopy(in []string) []string {
out := append([]string(nil), in...)
sort.Strings(out)
return out
}
func sortedNames(in map[string][]string) []string {
out := make([]string, 0, len(in))
for k := range in {
out = append(out, k)
}
sort.Strings(out)
return out
}
-247
View File
@@ -1,247 +0,0 @@
package broker
import (
"strings"
"testing"
)
// Deriving the bus's user list from the mesh's records.
//
// Every test here is about a way the list could be wrong that the server would not tell anybody
// about: a user missing, a user named twice, a user with authority it did not declare.
func someRecords() Records {
return Records{
Nodes: []string{"two", "one"},
Assigned: map[string][]Declared{
"one": {{Module: "telegram", Serves: []string{"status"}}},
"two": {{Module: "shop", Emits: []string{"order.placed"}}},
},
Enrolling: []string{"three"},
People: map[string][]string{"ada": {"mesh-catalog.catalog_tools"}},
}
}
func namesOf(t *testing.T, r Records) []string {
t.Helper()
users, err := Users(r)
if err != nil {
t.Fatal(err)
}
out := make([]string, 0, len(users))
for _, u := range users {
out = append(out, u.Username())
}
return out
}
// The controller is always there. A mesh whose own controller is not in the file is a mesh that
// cannot be told anything, and there is no state of the records in which that is correct.
func TestTheControllerIsAlwaysInTheList(t *testing.T) {
for _, r := range []Records{{}, someRecords()} {
names := namesOf(t, r)
if len(names) == 0 || names[0] != "controller" {
t.Fatalf("the controller is not first in %v", names)
}
}
}
// One user per node, one per module per node, one per live token and one per person — and nothing
// else, because a user nobody derived is a user nobody can explain.
func TestEveryRecordBecomesExactlyOneUser(t *testing.T) {
names := namesOf(t, someRecords())
want := []string{
"controller",
"node.one", "one.telegram",
"node.two", "two.shop",
"enrol.three",
"person.ada",
}
if strings.Join(names, ",") != strings.Join(want, ",") {
t.Fatalf("derived %v\n want %v", names, want)
}
}
// Two users with one name is a file the server reads as one of them, and which one depends on the
// order. Refused here, where both can be named, rather than left to be whichever the server picked.
func TestTwoUsersWithOneNameAreRefused(t *testing.T) {
r := someRecords()
r.Assigned["one"] = append(r.Assigned["one"], Declared{Module: "telegram"})
_, err := Users(r)
if err == nil {
t.Fatal("a module assigned twice to one node composed two users with one name")
}
if !strings.Contains(err.Error(), "one.telegram") {
t.Fatalf("the refusal does not name the user: %v", err)
}
}
// A module's authority is what it declared and nothing more, carried through the derivation intact —
// because this is the step where a mistake would grant something no manifest asked for.
func TestAModulesAuthorityIsWhatItDeclared(t *testing.T) {
seat := Seat{Name: "telegram-sender", Accepts: []string{"send"}, Emits: []string{"delivered"}}
users, err := Users(Records{
Nodes: []string{"one"},
Assigned: map[string][]Declared{"one": {{
Module: "shop", Emits: []string{"order.placed"}, Uses: []Seat{seat},
}}},
})
if err != nil {
t.Fatal(err)
}
perms, err := PermissionsFor(users[len(users)-1])
if err != nil {
t.Fatal(err)
}
has(t, perms.Publish, "mesh.mod.shop.event.order.placed")
has(t, perms.Publish, "mesh.seat.telegram-sender.accept.send")
// A seat it uses, not one it holds: it may submit work and may not publish the seat's own
// events, or it could lie about outcomes on a role somebody else fills.
hasNot(t, perms.Publish, "mesh.seat.telegram-sender.event.delivered")
hasNot(t, perms.Subscribe, "mesh.seat.telegram-sender.accept.send")
}
// A user the mesh has never minted a password for is named rather than silently dropped or
// composed as a user anybody is. It is an ordinary situation — a module assigned a moment ago — and
// the remedy is to mint one, so the caller decides whether to write a partial file.
func TestAUserWithNoPasswordIsNamedRatherThanWritten(t *testing.T) {
users, err := Users(someRecords())
if err != nil {
t.Fatal(err)
}
filled, missing := WithPasswords(users, map[string]string{
"controller": "$2a$hash", "node.one": "$2a$hash",
})
if len(filled) != 2 {
t.Fatalf("composed %d users from two hashes", len(filled))
}
if len(missing) != len(users)-2 {
t.Fatalf("%d users are missing a password, of %d: %v", len(missing), len(users), missing)
}
for _, p := range filled {
if p.PasswordHash == "" {
t.Fatalf("%s was kept with no password, which is a user anybody is", p.Username())
}
}
}
// And the whole thing composes: records in, a file the server would read out.
func TestRecordsComposeIntoAFile(t *testing.T) {
users, err := Users(someRecords())
if err != nil {
t.Fatal(err)
}
hashes := map[string]string{}
for _, u := range users {
hashes[u.Username()] = "$2a$11$" + strings.Repeat("x", 22)
}
filled, missing := WithPasswords(users, hashes)
if len(missing) != 0 {
t.Fatalf("users with no password: %v", missing)
}
got, err := Compose(Server{ClientPort: 4222, MonitoringPort: 8222, StoreDir: "/data",
TLSCert: "/tls/tls.crt", TLSKey: "/tls/tls.key", TLSCA: "/tls/ca.crt"}, filled)
if err != nil {
t.Fatal(err)
}
for _, want := range []string{
`user: "controller"`, `user: "node.one"`, `user: "one.telegram"`,
`user: "enrol.three"`, `user: "person.ada"`,
`"_INBOX.enrol.three.>"`, `"mesh.mod.mesh-catalog.tool.catalog_tools"`,
} {
if !strings.Contains(got, want) {
t.Errorf("the composed file does not contain %s", want)
}
}
}
// The accounts block alone is what the mesh writes, and it holds nothing about the server.
//
// **The split is the whole design decision** (ComposeAccounts): ports, TLS paths and a store
// directory are properties of the container the module raises, and a controller that wrote them
// would have to be kept in step with a Dockerfile it never sees. So this test says what must not be
// in the file as plainly as what must.
func TestWhatTheMeshWritesIsUsersAndNothingAboutTheServer(t *testing.T) {
users, err := Users(someRecords())
if err != nil {
t.Fatal(err)
}
hashes := map[string]string{}
for _, u := range users {
hashes[u.Username()] = "$2a$11$" + strings.Repeat("x", 22)
}
filled, missing := WithPasswords(users, hashes)
if len(missing) != 0 {
t.Fatalf("users with no password: %v", missing)
}
got, err := ComposeAccounts(filled)
if err != nil {
t.Fatal(err)
}
for _, want := range []string{"accounts {", `user: "controller"`, `user: "one.telegram"`} {
if !strings.Contains(got, want) {
t.Errorf("the accounts file does not contain %s", want)
}
}
// None of the server's own settings. Each of these in the mesh's file is a value the controller
// would then own, and the module could no longer change its own image without the mesh agreeing.
// `jetstream {` is the server's block (its store, its limits); `jetstream: enabled` inside the
// account is the account's, and the mesh owns the account — a user in it is told "JetStream
// not enabled for account" without it (2026-09-28).
if !strings.Contains(got, "jetstream: enabled") {
t.Errorf("the account does not enable JetStream, so no user in it can bind a consumer")
}
for _, absent := range []string{"port:", "http:", "jetstream {", "tls {", "store_dir", "cert_file"} {
if strings.Contains(got, absent) {
t.Errorf("the accounts file contains %q, which belongs to the module that raises the "+
"server, not to the mesh", absent)
}
}
}
// A user with no password is refused here too, not only by the whole-file composition: this is the
// function the controller actually calls, and a user without a password is a user anybody is.
func TestTheAccountsFileRefusesAUserWithNoPassword(t *testing.T) {
if _, err := ComposeAccounts([]Principal{{Kind: KindController}}); err == nil {
t.Fatal("a user with no password hash was written")
}
}
// **A user list is composed for a bus the mesh has not moved onto yet**, and that is the whole of
// step 2 (novox/hq ADR 0116): the server stands in the mesh carrying nothing, on its own ports, while
// every node is still on the bus it was on.
//
// Pinned because the first version of the composing step got it backwards — it wrote the list only
// once the controller was already on the new bus, which is a step that cannot be taken: the module
// comes up, finds no accounts file, and waits for one the controller had decided not to write.
func TestAUserListIsComposedBeforeAnythingMovesOntoTheBus(t *testing.T) {
// Exactly the records of a mesh mid-change: everything running, nothing on the new bus.
users, err := Users(Records{
Nodes: []string{"anchor"},
Assigned: map[string][]Declared{"anchor": {{Module: "nats"}}},
})
if err != nil {
t.Fatal(err)
}
hashes := map[string]string{}
for _, u := range users {
hashes[u.Username()] = "$2a$11$" + strings.Repeat("x", 22)
}
filled, missing := WithPasswords(users, hashes)
if len(missing) != 0 {
t.Fatalf("users with no credential: %v", missing)
}
accounts, err := ComposeAccounts(filled)
if err != nil {
t.Fatal(err)
}
// The controller's own user above all: a file without it is a bus its writer cannot connect to,
// which is what the server would be left holding the moment it starts.
if !strings.Contains(accounts, `user: "controller"`) {
t.Fatalf("the composed list does not contain the controller:\n%s", accounts)
}
if !strings.Contains(accounts, `user: "node.anchor"`) {
t.Errorf("the composed list does not contain the machine running the bus")
}
}
+22 -155
View File
@@ -61,25 +61,6 @@ type Result struct {
Commit string
// Built is each artifact, for reporting.
Built []catalogue.Built
// Read is every repository this build read source from besides the module's own — the second
// repository an artifact's recipe names (ArtifactContext). Reported because the manifest the
// mesh keeps carries no build section, so nothing else could say that a merge there is a
// change to this module (novox/hq 04-ISSUES/131).
Read []catalogue.ArtifactContext
}
// GitCredential is the forge credential a clone may present when the server asks for one.
//
// **Offered, never pushed.** It is written as a git credential-store file and named to git with
// `-c credential.helper=store`, so git itself decides when it applies: only on an authentication
// challenge, and only for the URL it was written for — scheme, host and port included. A public
// repository clones exactly as before, and a repository on any other host is never shown it.
type GitCredential struct {
// URL is the credential-store line — scheme://user:password@host[:port] — naming the one
// server this credential belongs to. Empty means the builder holds none and every clone is
// anonymous, as it always was.
URL string
}
// Build clones a repository at a ref, reads its manifest, produces what it declares, publishes
@@ -89,8 +70,7 @@ type GitCredential struct {
// archive failed would otherwise leave half of itself in the store under a digest the mesh never
// records — reachable, unreferenced, and indistinguishable from something in use.
func Build(ctx context.Context, run Runner, publish Publisher,
repository, path, ref, workspace string, held map[string]string, npmrc Npmrc,
forge GitCredential, log Log) (Result, error) {
repository, path, ref, workspace string, held map[string]string, npmrc Npmrc, log Log) (Result, error) {
say := logging(log)
say("clone", "%s%s at %s", repository, describePath(path), refOrHead(ref))
@@ -100,15 +80,6 @@ func Build(ctx context.Context, run Runner, publish Publisher,
if err := os.MkdirAll(workspace, 0o755); err != nil {
return Result{}, err
}
// The credential is a file git reads, never an argument: a URL carrying a password in argv
// would be readable by anything that can list processes for as long as a clone runs.
credentials := ""
if forge.URL != "" {
credentials = filepath.Join(workspace, "git-credentials")
if err := os.WriteFile(credentials, []byte(forge.URL+"\n"), 0o600); err != nil {
return Result{}, err
}
}
tree := filepath.Join(workspace, "source")
if err := os.RemoveAll(tree); err != nil {
return Result{}, err
@@ -116,7 +87,7 @@ func Build(ctx context.Context, run Runner, publish Publisher,
// A fresh clone every time rather than a fetch into a tree that is already there. A build
// that reuses a working tree can succeed because of something a previous build left behind,
// and that is a build nobody can reproduce.
if _, err := run(ctx, workspace, "git", cloneWith(credentials, "clone", "--quiet", repository, tree)...); err != nil {
if _, err := run(ctx, workspace, "git", "clone", "--quiet", repository, tree); err != nil {
say("clone", "FAILED: %v", err)
return Result{}, fmt.Errorf("cannot clone %s: %w", repository, err)
}
@@ -175,8 +146,6 @@ func Build(ctx context.Context, run Runner, publish Publisher,
}
var built []catalogue.Built
// stoodOn is every base the build was handed, as resolved — the edges the catalogue derives.
var stoodOn []string
if manifest.Build != nil {
// What this module said it stands on, answered with what this mesh actually holds. Done
// before anything is built, so a missing base is refused in front of the person who can
@@ -194,12 +163,11 @@ func Build(ctx context.Context, run Runner, publish Publisher,
}
return from, nil
}
args, bases, err := standingOn(ctx, manifest, held, mirror)
args, err := standingOn(ctx, manifest, held, mirror)
if err != nil {
say("bases", "UNMET: %v", err)
return Result{}, err
}
stoodOn = bases
if len(args) > 0 {
say("bases", "%d resolved from what the mesh holds", len(args)/2)
}
@@ -209,7 +177,7 @@ func Build(ctx context.Context, run Runner, publish Publisher,
sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name })
for _, a := range artifacts {
say("artifact", "%s (%s%s) — starting", a.Name, a.Kind, langSuffix(a))
made, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, npmrcPath, say)
made, err := one(ctx, run, publish, manifest.Module, within, commit, a, args, held, npmrcPath, say)
if err != nil {
say("artifact", "%s FAILED: %v", a.Name, err)
return Result{}, err
@@ -226,7 +194,7 @@ func Build(ctx context.Context, run Runner, publish Publisher,
}
say("done", "%s at %s — %d artifact(s) pinned", manifest.Module, short(commit), len(built))
return Result{Manifest: resolved, Commit: commit, Built: built,
Against: against(within, manifest, stoodOn), Read: readBy(manifest)}, nil
Against: against(within, manifest)}, nil
}
// Log is where a build says what it is doing, step by step. Nil is silent — the tests pass none,
@@ -242,43 +210,6 @@ func logging(log Log) func(step, format string, args ...any) {
}
}
// contextFrom clones an image artifact's own build context, when it names one apart from this
// module's own repository — a fresh tree, the same way the module's own is, keyed by artifact
// name so two artifacts of one module naming different contexts do not collide.
func contextFrom(ctx context.Context, run Runner, workspace, artifact, credentials string,
from catalogue.ArtifactContext, say func(step, format string, args ...any)) (string, error) {
say("context", "cloning %s at %s for %s", from.Repository, refOrHead(from.Ref), artifact)
dir := filepath.Join(workspace, "context-"+artifact)
if err := os.RemoveAll(dir); err != nil {
return "", err
}
if _, err := run(ctx, workspace, "git", cloneWith(credentials, "clone", "--quiet", from.Repository, dir)...); err != nil {
return "", fmt.Errorf("cannot clone %s: %w", from.Repository, err)
}
if from.Ref != "" {
if _, err := run(ctx, dir, "git", "checkout", "--quiet", from.Ref); err != nil {
return "", fmt.Errorf("%s has no %s: %w", from.Repository, from.Ref, err)
}
}
say("context", "done")
return dir, nil
}
// cloneWith is a git invocation that may offer a stored credential.
//
// The first `-c credential.helper=` clears every helper the environment might carry, so exactly
// one place answers an authentication challenge: the file the builder wrote. Without a file, the
// invocation is exactly what it always was.
func cloneWith(credentials string, rest ...string) []string {
if credentials == "" {
return rest
}
return append([]string{
"-c", "credential.helper=",
"-c", "credential.helper=store --file=" + credentials,
}, rest...)
}
func describePath(path string) string {
if path == "" {
return ""
@@ -348,27 +279,14 @@ func describe(path string) string {
// allowed to name — a tag is something somebody else can move under you.
var pinnedImage = regexp.MustCompile(`[A-Za-z0-9][A-Za-z0-9._/:-]*@sha256:[0-9a-f]{64}`)
// against is what this module's image artifacts are built on top of: every base the mesh resolved
// and handed the recipe as a build argument (`build.on`), and any image a recipe pins by digest
// itself. Nothing is guessed: a reference that was neither resolved nor written down is not
// reported.
//
// **The resolved bases are the edges.** A recipe reads its base from an argument (`FROM
// ${RUNTIME_BASE}`), so the digest is never in the file, and a derivation that read files alone
// recorded no edge for any module on the mesh — which is why nothing knew what a changed base
// meant to rebuild (novox/hq 04-ISSUES/131).
func against(within string, manifest catalogue.Manifest, resolved []string) []string {
// against reads what this module's image artifacts are built on top of, out of the files that
// build them. Nothing is guessed: a reference that is not written down is not reported.
func against(within string, manifest catalogue.Manifest) []string {
if manifest.Build == nil {
return nil
}
seen := map[string]bool{}
var out []string
for _, r := range resolved {
if r != "" && !seen[r] {
seen[r] = true
out = append(out, r)
}
}
for _, a := range manifest.Build.Artifacts {
if a.Kind != catalogue.ArtifactImage || a.From == "" {
continue
@@ -415,7 +333,7 @@ func wantsPackages(manifest catalogue.Manifest, within string) bool {
}
func one(ctx context.Context, run Runner, publish Publisher,
module, tree, workspace, commit, credentials string, a catalogue.Artifact, args []string,
module, tree, commit string, a catalogue.Artifact, args []string,
held map[string]string, npmrc string, say func(step, format string, args ...any)) (catalogue.Built, error) {
switch a.Kind {
@@ -487,27 +405,7 @@ func one(ctx context.Context, run Runner, publish Publisher,
"and start FROM ${<NAME>} (novox/hq ADR 0097)",
module, a.From, strings.Join(bases, ", "))
}
// The recipe is always read from this module's own tree, at this module's own commit — only
// the context docker build's final argument names can come from somewhere else, when the
// artifact says so.
recipePath := a.From
buildDir := tree
if a.Context != nil {
cloned, err := contextFrom(ctx, run, workspace, a.Name, credentials, *a.Context, say)
if err != nil {
return catalogue.Built{}, fmt.Errorf("%s: %s's context: %w", module, a.Name, err)
}
// docker build accepts -f outside the context it is given; the recipe stays exactly
// where it was read from and validated against, absolute so the working directory
// switching to the cloned context does not change which file that is.
absRecipe, err := filepath.Abs(filepath.Join(tree, a.From))
if err != nil {
return catalogue.Built{}, fmt.Errorf("%s: %s's recipe: %w", module, a.Name, err)
}
recipePath = absRecipe
buildDir = cloned
}
invocation := append([]string{"build", "-f", recipePath, "-t", local}, args...)
invocation := append([]string{"build", "-f", a.From, "-t", local}, args...)
if a.Target != "" {
invocation = append(invocation, "--target", a.Target)
}
@@ -519,8 +417,8 @@ func one(ctx context.Context, run Runner, publish Publisher,
invocation = append(invocation, "--network", "host")
}
invocation = append(invocation, ".")
say("image", "docker build -f %s", recipePath)
if _, err := run(ctx, buildDir, "docker", invocation...); err != nil {
say("image", "docker build -f %s", a.From)
if _, err := run(ctx, tree, "docker", invocation...); err != nil {
return catalogue.Built{}, fmt.Errorf("%s: building %s failed: %w", module, a.Name, err)
}
say("image", "built, publishing")
@@ -726,42 +624,40 @@ var _ io.Writer = (*stringWriter)(nil)
// built cannot be built here yet, and the useful sentence names which module is missing — not the
// one a container runtime produces when a recipe's first line refers to an image nobody has.
//
// The order is fixed so two builds of one commit invoke the same command. Returned alongside the
// arguments is every reference they resolved to, which is what the build stood on.
// The order is fixed so two builds of one commit invoke the same command.
func standingOn(ctx context.Context, manifest catalogue.Manifest, held map[string]string,
mirror func(ctx context.Context, from, repository string) (string, error)) ([]string, []string, error) {
mirror func(ctx context.Context, from, repository string) (string, error)) ([]string, error) {
if manifest.Build == nil || len(manifest.Build.On) == 0 {
return nil, nil, nil
return nil, nil
}
on := append([]catalogue.BuildsOn{}, manifest.Build.On...)
sort.Slice(on, func(i, j int) bool { return on[i].Arg < on[j].Arg })
var args, resolved []string
var args []string
for _, base := range on {
if base.Image != "" {
// A vendor's image, declared (novox/hq 04-ISSUES/064, ADR 0097). Pinned, because a tag
// is what somebody else can move; copied into the mesh's registry, because a build
// that reaches a public registry on its own is a build that works sometimes.
if base.Arg == "" || base.Module != "" || base.Artifact != "" {
return nil, nil, fmt.Errorf(
return nil, fmt.Errorf(
"%s stands on the image %s, and a base is either a module's artifact or an "+
"image — never both — read from one build argument", manifest.Module, base.Image)
}
if !strings.Contains(base.Image, "@sha256:") {
return nil, nil, fmt.Errorf(
return nil, fmt.Errorf(
"%s stands on the image %q, which is not pinned by digest. A tag is what "+
"somebody else can move; name it as <image>@sha256:…", manifest.Module, base.Image)
}
reference, err := mirror(ctx, base.Image, manifest.Module+"/on-"+strings.ToLower(base.Arg))
if err != nil {
return nil, nil, fmt.Errorf("%s stands on %s: %w", manifest.Module, base.Image, err)
return nil, fmt.Errorf("%s stands on %s: %w", manifest.Module, base.Image, err)
}
args = append(args, "--build-arg", base.Arg+"="+reference)
resolved = append(resolved, reference)
continue
}
if base.Arg == "" || base.Module == "" || base.Artifact == "" {
return nil, nil, fmt.Errorf(
return nil, fmt.Errorf(
"%s says its build stands on something, and does not say all of what: a base "+
"needs the module, the artifact, and the build argument the recipe reads it "+
"from", manifest.Module)
@@ -769,15 +665,14 @@ func standingOn(ctx context.Context, manifest catalogue.Manifest, held map[strin
key := base.Module + "/" + base.Artifact
reference, has := held[key]
if !has {
return nil, nil, fmt.Errorf(
return nil, fmt.Errorf(
"%s builds on %s, and this mesh has not built it. Build %s first — every module "+
"in this toolchain stands on it, so it is the thing to have before anything "+
"else", manifest.Module, key, base.Module)
}
args = append(args, "--build-arg", base.Arg+"="+reference)
resolved = append(resolved, reference)
}
return args, resolved, nil
return args, nil
}
// compile runs a module's own code through its toolchain, and says where the result is.
@@ -1022,31 +917,3 @@ func instructions(recipe string) []string {
flush()
return out
}
// readBy is every repository other than the module's own that this build's recipes read source from,
// each once and in a fixed order, so two builds of one commit report the same thing the same way.
func readBy(manifest catalogue.Manifest) []catalogue.ArtifactContext {
if manifest.Build == nil {
return nil
}
seen := map[string]bool{}
var out []catalogue.ArtifactContext
for _, a := range manifest.Build.Artifacts {
if a.Context == nil || a.Context.Repository == "" {
continue
}
key := a.Context.Repository + "#" + a.Context.Ref
if seen[key] {
continue
}
seen[key] = true
out = append(out, *a.Context)
}
sort.Slice(out, func(i, j int) bool {
if out[i].Repository != out[j].Repository {
return out[i].Repository < out[j].Repository
}
return out[i].Ref < out[j].Ref
})
return out
}
+14 -199
View File
@@ -18,19 +18,13 @@ import (
// tree, that two builds of one commit produce one digest. Running docker here would test docker.
type recorded struct {
ran []string
// dirs is the directory each entry in ran was run from, same index — so a test can ask not
// only what ran but where.
dirs []string
ran []string
images map[string]string
archives map[string]string
failPush bool
// contents is what a clone of this repository lands, so the fake clone can restore the tree
// Build deliberately removes first.
contents map[string]string
// secondary is what a clone of a repository OTHER than the one under test lands, keyed by
// that repository's URL — an artifact's own build context, cloned apart from the module.
secondary map[string]map[string]string
// stamped is the modification time the clone gives every file. Set differently between two
// builds of one commit, because otherwise both land in the same second and a packer that
// carried timestamps would still produce one digest — which is a test that passes for a
@@ -41,28 +35,13 @@ type recorded struct {
func (r *recorded) run(_ context.Context, dir, name string, args ...string) (string, error) {
line := name + " " + strings.Join(args, " ")
r.ran = append(r.ran, line)
r.dirs = append(r.dirs, dir)
// A clone may carry `-c` configuration in front of the verb — the credential store — so the
// verb is found rather than assumed first.
isClone := false
for _, a := range args {
if a == "clone" {
isClone = true
break
}
}
switch {
case name == "git" && isClone:
repository := args[len(args)-2]
case name == "git" && len(args) > 0 && args[0] == "clone":
tree := args[len(args)-1]
if err := os.MkdirAll(tree, 0o755); err != nil {
return "", err
}
lands := r.contents
if by, is := r.secondary[repository]; is {
lands = by
}
for path, body := range lands {
for path, body := range r.contents {
full := filepath.Join(tree, path)
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
return "", err
@@ -80,6 +59,7 @@ func (r *recorded) run(_ context.Context, dir, name string, args ...string) (str
case name == "git" && len(args) > 0 && args[0] == "rev-parse":
return "c0ffeec0ffeec0ffeec0ffeec0ffeec0ffeec0ff\n", nil
}
_ = dir
return "", nil
}
@@ -128,7 +108,7 @@ func TestABuildProducesAManifestThePinsAreIn(t *testing.T) {
r, workspace := aRepository(t, withBoth, map[string]string{
"Dockerfile": "FROM scratch", "files/theme.conf": "dark",
})
got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil)
got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, Npmrc{}, nil)
if err != nil {
t.Fatal(err)
}
@@ -154,7 +134,7 @@ func TestTwoBuildsOfOneCommitProduceOneDigest(t *testing.T) {
})
// A year apart, so a packer carrying timestamps cannot accidentally agree.
r.stamped = time.Date(2020+i, time.March, 3, 4, 5, 6, 0, time.UTC)
got, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil)
got, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, nil)
if err != nil {
t.Fatal(err)
}
@@ -174,7 +154,7 @@ func TestNothingIsPublishedUntilEverythingIsBuilt(t *testing.T) {
// unreferenced, and indistinguishable from something in use.
r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch"})
// `files` is missing, so packing the archive fails — after the image would have been pushed.
_, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil)
_, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, nil)
if err == nil {
t.Fatal("a build with a missing input succeeded")
}
@@ -186,7 +166,7 @@ func TestNothingIsPublishedUntilEverythingIsBuilt(t *testing.T) {
func TestARepositoryWithNoManifestSaysSo(t *testing.T) {
workspace := t.TempDir()
r := &recorded{contents: map[string]string{"README.md": "nothing to see"}}
_, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil)
_, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, nil)
if err == nil {
t.Fatal("a repository with nothing saying what it is was built")
}
@@ -199,7 +179,7 @@ func TestAModuleThatBuildsNothingStillProducesAManifest(t *testing.T) {
// Most of what a person installs is configuration.
r, workspace := aRepository(t, `{"module":"shell","version":"1","resources":[
{"id":"rc","type":"file","path":"/etc/zsh/zshrc","content":"setopt"}]}`, nil)
got, err := Build(context.Background(), r.run, r, "https://forge.invalid/shell.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil)
got, err := Build(context.Background(), r.run, r, "https://forge.invalid/shell.git", "", "", workspace, nil, Npmrc{}, nil)
if err != nil {
t.Fatal(err)
}
@@ -229,7 +209,7 @@ func TestTheTreeIsFreshEveryTime(t *testing.T) {
if err := os.WriteFile(leftover, []byte("stale"), 0o644); err != nil {
t.Fatal(err)
}
if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil); err != nil {
if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, nil); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(leftover); err == nil {
@@ -242,7 +222,7 @@ func TestABuildThatCannotPushFails(t *testing.T) {
"Dockerfile": "FROM scratch", "files/a": "b",
})
r.failPush = true
if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil); err == nil {
if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, nil); err == nil {
t.Fatal("a build that could publish nothing reported success")
}
}
@@ -256,7 +236,7 @@ func TestAnUpstreamImageIsMirroredRatherThanBuilt(t *testing.T) {
"resources":[{"id":"db","type":"container","name":"mesh-postgres","artifact":"store"}]}`
r, workspace := aRepository(t, mirrors, nil)
got, err := Build(context.Background(), r.run, r, "https://forge.invalid/postgres.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil)
got, err := Build(context.Background(), r.run, r, "https://forge.invalid/postgres.git", "", "", workspace, nil, Npmrc{}, nil)
if err != nil {
t.Fatal(err)
}
@@ -322,7 +302,7 @@ func TestAModuleIsBuiltFromItsPathWithinTheRepository(t *testing.T) {
"modules/other/" + ManifestName: `{"module":"other","version":"1"}`,
}}
got, err := Build(context.Background(), r.run, r,
"https://forge.invalid/catalogue.git", "modules/shell", "", t.TempDir(), nil, Npmrc{}, GitCredential{}, nil)
"https://forge.invalid/catalogue.git", "modules/shell", "", t.TempDir(), nil, Npmrc{}, nil)
if err != nil {
t.Fatal(err)
}
@@ -341,7 +321,7 @@ func TestAPathThatLeavesTheRepositoryIsRefused(t *testing.T) {
for _, escaping := range []string{"../../etc", "/etc"} {
r := &recorded{contents: map[string]string{ManifestName: withBoth}}
_, err := Build(context.Background(), r.run, r,
"https://forge.invalid/x.git", escaping, "", t.TempDir(), nil, Npmrc{}, GitCredential{}, nil)
"https://forge.invalid/x.git", escaping, "", t.TempDir(), nil, Npmrc{}, nil)
if err == nil {
t.Fatalf("%q was accepted as a module's path", escaping)
}
@@ -351,168 +331,3 @@ func TestAPathThatLeavesTheRepositoryIsRefused(t *testing.T) {
}
}
}
// **Packaging and source are allowed to live apart** — a module that ships only the recipe for
// source that lives in a second repository (the reference route-proxy, packaged in the catalogue
// but built from mesh-controller's own repository) names where that source actually is, rather
// than vendoring a second copy the two could drift from.
func TestAnArtifactWithItsOwnContextIsBuiltFromThere(t *testing.T) {
const withContext = `{"module":"route-proxy","version":"1",
"build":{"artifacts":[
{"name":"server","kind":"image","from":"Dockerfile",
"context":{"repository":"https://forge.invalid/source.git","ref":"main"}}]}}`
r := &recorded{
contents: map[string]string{
ManifestName: withContext,
// The recipe lives with the packaging, not the source — read from here regardless of
// where the build context comes from. FROM scratch declares no base, so what is under
// test — where the context comes from — is not entangled with ADR 0097's own checks.
"Dockerfile": "FROM scratch\nCOPY go.mod ./\n",
},
secondary: map[string]map[string]string{
// go.mod exists only in the second repository. A build context taken from the wrong
// place would never find it, which a real docker build would refuse on — the fake
// does not read files, so what is checked below is that the build was even pointed
// at the right place, not that COPY would have succeeded.
"https://forge.invalid/source.git": {"go.mod": "module route-proxy\n"},
},
}
_, err := Build(context.Background(), r.run, r,
"https://forge.invalid/catalogue.git", "", "", t.TempDir(), nil, Npmrc{}, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
var clonedSource bool
for _, line := range r.ran {
if strings.HasPrefix(line, "git clone") && strings.Contains(line, "https://forge.invalid/source.git") {
clonedSource = true
}
}
if !clonedSource {
t.Fatalf("the artifact's own context was never cloned: %v", r.ran)
}
buildIndex := -1
for i, line := range r.ran {
if strings.HasPrefix(line, "docker build ") {
buildIndex = i
}
}
if buildIndex == -1 {
t.Fatal("no docker build was run")
}
build := r.ran[buildIndex]
buildDir := r.dirs[buildIndex]
if !strings.Contains(buildDir, "context-server") {
t.Errorf("docker build ran from %q, not the artifact's own cloned context", buildDir)
}
recipe := strings.SplitN(strings.SplitN(build, "-f ", 2)[1], " ", 2)[0]
if !filepath.IsAbs(recipe) {
t.Errorf("the recipe %q is not an absolute path, so it is read relative to whatever "+
"directory the build context moved to rather than where it actually is", recipe)
}
if !strings.HasSuffix(recipe, string(filepath.Separator)+"Dockerfile") {
t.Errorf("the recipe is not the module's own Dockerfile: %q", recipe)
}
if !strings.HasSuffix(build, " .") {
t.Errorf("the build was not given a context: %s", build)
}
}
// The forge credential is offered through git's own credential store — a file, never argv — and
// git decides when it applies. What is checked: the clone names the store, the secret never
// appears in a command line, and the file holds exactly the URL at 0600.
func TestABuildOffersTheForgesCredentialThroughGitsOwnStore(t *testing.T) {
r, workspace := aRepository(t, withBoth, map[string]string{
"Dockerfile": "FROM scratch", "files/theme.conf": "dark",
})
_, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "",
workspace, nil, Npmrc{},
GitCredential{URL: "http://mesh_novox_builder:sw0rdfi5h@forge.invalid:20000"}, nil)
if err != nil {
t.Fatal(err)
}
stored := filepath.Join(workspace, "git-credentials")
clone := r.ran[0]
if !strings.Contains(clone, "credential.helper=store --file="+stored) {
t.Fatalf("the clone does not name the credential store: %s", clone)
}
for _, line := range r.ran {
if strings.Contains(line, "sw0rdfi5h") {
t.Fatalf("the secret is in a command line, readable by anything that can list processes: %s", line)
}
}
raw, err := os.ReadFile(stored)
if err != nil {
t.Fatal(err)
}
if strings.TrimSpace(string(raw)) != "http://mesh_novox_builder:sw0rdfi5h@forge.invalid:20000" {
t.Fatalf("the store does not hold the credential as given: %q", raw)
}
info, err := os.Stat(stored)
if err != nil {
t.Fatal(err)
}
if info.Mode().Perm() != 0o600 {
t.Fatalf("the credential file is readable beyond its owner: %v", info.Mode())
}
}
// Without a credential, a clone is exactly the invocation it always was, and no credential file
// appears — the builder a mesh of public repositories runs is unchanged.
func TestABuildWithNoCredentialClonesExactlyAsBefore(t *testing.T) {
r, workspace := aRepository(t, withBoth, map[string]string{
"Dockerfile": "FROM scratch", "files/theme.conf": "dark",
})
_, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "",
workspace, nil, Npmrc{}, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if !strings.HasPrefix(r.ran[0], "git clone --quiet ") {
t.Fatalf("a credential-less clone grew flags: %s", r.ran[0])
}
if _, err := os.Stat(filepath.Join(workspace, "git-credentials")); !os.IsNotExist(err) {
t.Fatal("a credential file was written with no credential to put in it")
}
}
// An artifact's own context is cloned with the same offer: a private module whose context is a
// second private repository on the same forge builds, and the secret still never reaches argv.
func TestAContextCloneCarriesTheSameCredentialStore(t *testing.T) {
const withContext = `{"module":"route-proxy","version":"1",
"build":{"artifacts":[
{"name":"server","kind":"image","from":"Dockerfile",
"context":{"repository":"https://forge.invalid/source.git","ref":"main"}}]}}`
r := &recorded{
contents: map[string]string{
ManifestName: withContext,
"Dockerfile": "FROM scratch\nCOPY go.mod ./\n",
},
secondary: map[string]map[string]string{
"https://forge.invalid/source.git": {"go.mod": "module route-proxy\n"},
},
}
workspace := t.TempDir()
_, err := Build(context.Background(), r.run, r,
"https://forge.invalid/catalogue.git", "", "", workspace, nil, Npmrc{},
GitCredential{URL: "https://builder:s3cret@forge.invalid"}, nil)
if err != nil {
t.Fatal(err)
}
stored := filepath.Join(workspace, "git-credentials")
var contextClone string
for _, line := range r.ran {
if strings.Contains(line, "clone") && strings.Contains(line, "source.git") {
contextClone = line
}
}
if contextClone == "" {
t.Fatalf("the context was never cloned: %v", r.ran)
}
if !strings.Contains(contextClone, "credential.helper=store --file="+stored) {
t.Fatalf("the context clone does not name the credential store: %s", contextClone)
}
}
+4 -4
View File
@@ -54,7 +54,7 @@ func TestABundleIsCompiledAndPackedWithNoDockerfile(t *testing.T) {
held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)}
got, err := Build(context.Background(), compiling{r}.run, r,
"https://forge.invalid/greeter.git", "", "", workspace, held, Npmrc{}, GitCredential{}, nil)
"https://forge.invalid/greeter.git", "", "", workspace, held, Npmrc{}, nil)
if err != nil {
t.Fatalf("a module with a language and no Dockerfile did not build: %v", err)
}
@@ -91,7 +91,7 @@ func TestABundleWhoseToolchainIsNotHeldIsRefusedFirst(t *testing.T) {
r, workspace := aRepository(t, aBundle, map[string]string{"index.ts": "console.log(1)"})
_, err := Build(context.Background(), compiling{r}.run, r,
"https://forge.invalid/greeter.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil)
"https://forge.invalid/greeter.git", "", "", workspace, nil, Npmrc{}, nil)
if err == nil {
t.Fatal("a bundle was built with no toolchain to compile it in")
}
@@ -112,7 +112,7 @@ func TestABundleInAnUnknownLanguageIsRefused(t *testing.T) {
_, err := Build(context.Background(), compiling{r}.run, r,
"https://forge.invalid/greeter.git", "", "", workspace,
map[string]string{"mesh-tools/build": "registry.invalid/x@sha256:" + strings.Repeat("c", 64)}, Npmrc{}, GitCredential{}, nil)
map[string]string{"mesh-tools/build": "registry.invalid/x@sha256:" + strings.Repeat("c", 64)}, Npmrc{}, nil)
if err == nil {
t.Fatal("a language nothing can compile was accepted")
}
@@ -140,7 +140,7 @@ func TestTwoBundlesInOneModuleArePackedSeparately(t *testing.T) {
held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)}
got, err := Build(context.Background(), compiling{r}.run, r,
"https://forge.invalid/greeter.git", "", "", workspace, held, Npmrc{}, GitCredential{}, nil)
"https://forge.invalid/greeter.git", "", "", workspace, held, Npmrc{}, nil)
if err != nil {
t.Fatalf("a module with two bundles did not build: %v", err)
}
-14
View File
@@ -180,20 +180,6 @@ func (r Registry) MirrorImage(ctx context.Context, from, repository string) (str
if err != nil {
return "", err
}
// **Already held is already mirrored.** A base is named by digest, and a digest this registry
// holds under the module's repository is the same bytes whatever upstream would say — so
// upstream is not asked. Asked every build, the public hub's anonymous pull limit was reached
// on the first merge that rebuilt a whole catalogue (2026-09-28), and every module whose base
// lives there failed on a copy it did not need.
if strings.HasPrefix(where.reference, "sha256:") {
held, err := r.has(ctx, "http://"+r.Address+"/v2/"+repository+"/manifests/"+where.reference, manifestAccept)
if err != nil {
return "", fmt.Errorf("asking %s whether it holds %s: %w", r.Address, from, err)
}
if held {
return r.Address + "/" + repository + "@" + where.reference, nil
}
}
src := &source{client: r.client()}
digest, err := r.copyManifest(ctx, src, where, where.reference, repository)
if err != nil {
-38
View File
@@ -103,20 +103,6 @@ func (m *theMeshsRegistry) handler() http.Handler {
m.mu.Lock()
defer m.mu.Unlock()
switch {
case r.Method == http.MethodHead && strings.Contains(r.URL.Path, "/manifests/"):
// **As strictly as a real registry.** A manifest is answered only in a media type the
// caller named; a request with no Accept is answered as if nothing were there. The fake
// used to answer regardless, which is why it could not catch a check that asked without
// one — and the mesh copied every base again (2026-09-28).
if !strings.Contains(r.Header.Get("Accept"), "manifest") && !strings.Contains(r.Header.Get("Accept"), "index") {
w.WriteHeader(http.StatusNotFound)
return
}
if _, ok := m.manifests[r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]]; ok {
w.WriteHeader(http.StatusOK)
} else {
w.WriteHeader(http.StatusNotFound)
}
case r.Method == http.MethodHead && strings.Contains(r.URL.Path, "/blobs/"):
if _, ok := m.blobs[r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]]; ok {
w.WriteHeader(http.StatusOK)
@@ -233,27 +219,3 @@ func TestATagBeforeTheDigestIsNotPartOfTheRepository(t *testing.T) {
t.Fatalf("got %+v", got)
}
}
// A base this registry already holds by digest is not asked of upstream at all: the public hub
// limits anonymous pulls, and a catalogue rebuilt on one merge asked it once per module.
func TestABaseAlreadyHeldIsNotAskedOfUpstream(t *testing.T) {
src, indexDigest, _ := anUpstreamRegistry(t)
dst := &theMeshsRegistry{blobs: map[string][]byte{}, manifests: map[string][]byte{}}
dstServer := httptest.NewServer(dst.handler())
defer dstServer.Close()
address := strings.TrimPrefix(dstServer.URL, "http://")
r := Registry{Address: address, HTTP: src.Client()}
host := strings.TrimPrefix(src.URL, "http://")
if _, err := r.MirrorImage(context.Background(), host+"/library/thing:latest", "hello-web/server"); err != nil {
t.Fatal(err)
}
// Upstream gone: the pinned base is answered from what the mesh holds.
src.Close()
reference, err := r.MirrorImage(context.Background(), host+"/library/thing@"+indexDigest, "hello-web/server")
if err != nil {
t.Fatalf("a base the registry holds was asked of an upstream that is gone: %v", err)
}
if reference != address+"/hello-web/server@"+indexDigest {
t.Fatalf("pinned as %q", reference)
}
}
+5 -5
View File
@@ -71,7 +71,7 @@ func TestAnImageBuildGetsTheCredentialInTheContextAndHostNetwork(t *testing.T) {
r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch\nCOPY .npmrc ./", "files/x": "y"})
n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm/", Token: "t"}
if _, err := Build(context.Background(), r.run, r,
"https://forge.invalid/meshboard.git", "", "", workspace, nil, n, GitCredential{}, nil); err != nil {
"https://forge.invalid/meshboard.git", "", "", workspace, nil, n, nil); err != nil {
t.Fatalf("the build failed: %v", err)
}
@@ -101,7 +101,7 @@ func TestAnImageBuildGetsTheCredentialInTheContextAndHostNetwork(t *testing.T) {
func TestAnImageBuildWithoutACredentialGetsNoHostNetwork(t *testing.T) {
r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch", "files/x": "y"})
if _, err := Build(context.Background(), r.run, r,
"https://forge.invalid/meshboard.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil); err != nil {
"https://forge.invalid/meshboard.git", "", "", workspace, nil, Npmrc{}, nil); err != nil {
t.Fatalf("the build failed: %v", err)
}
for _, line := range r.ran {
@@ -127,7 +127,7 @@ func TestAPackageIsBuiltOnAPublicBaseAndPublishedByVersion(t *testing.T) {
})
n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm/", Token: "t"}
got, err := Build(context.Background(), r.run, r,
"https://forge.invalid/mesh-sdk.git", "", "", workspace, nil, n, GitCredential{}, nil)
"https://forge.invalid/mesh-sdk.git", "", "", workspace, nil, n, nil)
if err != nil {
t.Fatalf("the package did not build: %v", err)
}
@@ -159,7 +159,7 @@ func TestAPackageWithNoRegistryIsRefused(t *testing.T) {
"package.json": `{"name":"@novox/mesh-sdk","version":"0.1.0"}`,
})
_, err := Build(context.Background(), r.run, r,
"https://forge.invalid/mesh-sdk.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil)
"https://forge.invalid/mesh-sdk.git", "", "", workspace, nil, Npmrc{}, nil)
if err == nil {
t.Fatal("a package built with no registry to publish to, silently")
}
@@ -206,7 +206,7 @@ func TestAnImageThatDoesNotAskForTheCredentialDoesNotGetIt(t *testing.T) {
r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch\nCOPY . .", "files/x": "y"})
n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm/", Token: "t"}
if _, err := Build(context.Background(), r.run, r,
"https://forge.invalid/meshboard.git", "", "", workspace, nil, n, GitCredential{}, nil); err != nil {
"https://forge.invalid/meshboard.git", "", "", workspace, nil, n, nil); err != nil {
t.Fatalf("the build failed: %v", err)
}
for _, line := range r.ran {
+1 -13
View File
@@ -122,23 +122,11 @@ func (r Registry) PublishArchive(ctx context.Context, repository string, body []
return final, nil
}
// has is whether this registry already holds what is at that URL.
//
// **A manifest HEAD must say what it accepts.** A registry answers a manifest request only in a media
// type the caller named, and a bare HEAD — no Accept at all — is answered 404 for a manifest it holds
// perfectly well. Measured against the mesh's own registry (2026-09-28): the same digest answered 200
// with the manifest media types and 404 without them, so a check written without them concluded the
// registry held nothing, copied every base again, and exhausted the public hub's pull limit. A blob
// needs no Accept, which is why this went unnoticed: the same helper was right for blobs and wrong
// for manifests.
func (r Registry) has(ctx context.Context, url string, accept ...string) (bool, error) {
func (r Registry) has(ctx context.Context, url string) (bool, error) {
request, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
if err != nil {
return false, err
}
for _, media := range accept {
request.Header.Set("Accept", media)
}
response, err := r.client().Do(request)
if err != nil {
return false, fmt.Errorf("cannot reach the registry at %s: %w", r.Address, err)
+6 -55
View File
@@ -22,7 +22,7 @@ func TestABaseTheMeshHasNotBuiltIsRefused(t *testing.T) {
On: []catalogue.BuildsOn{{Arg: "RUNTIME_BASE", Module: "mesh-tools", Artifact: "runtime"}},
},
}
_, _, err := standingOn(context.Background(), manifest, map[string]string{}, noMirror)
_, err := standingOn(context.Background(), manifest, map[string]string{}, noMirror)
if err == nil {
t.Fatal("a base nothing has built was accepted; the build would have failed on its first line")
}
@@ -42,7 +42,7 @@ func TestABaseTheMeshHoldsBecomesABuildArgument(t *testing.T) {
},
}
held := map[string]string{"mesh-tools/runtime": "127.0.0.1:5000/mesh-tools/runtime@sha256:" + strings.Repeat("a", 64)}
args, _, err := standingOn(context.Background(), manifest, held, noMirror)
args, err := standingOn(context.Background(), manifest, held, noMirror)
if err != nil {
t.Fatalf("a base this mesh holds was refused: %v", err)
}
@@ -54,7 +54,7 @@ func TestABaseTheMeshHoldsBecomesABuildArgument(t *testing.T) {
// A module naming no base asks for nothing, which is most modules.
func TestAModuleNamingNoBaseAddsNoArguments(t *testing.T) {
args, _, err := standingOn(context.Background(), catalogue.Manifest{Module: "hello-web", Build: &catalogue.Build{}}, nil, noMirror)
args, err := standingOn(context.Background(), catalogue.Manifest{Module: "hello-web", Build: &catalogue.Build{}}, nil, noMirror)
if err != nil || args != nil {
t.Fatalf("a module naming no base produced %v, %v", args, err)
}
@@ -66,7 +66,7 @@ func TestAnIncompleteBaseIsRefused(t *testing.T) {
Module: "postgres",
Build: &catalogue.Build{On: []catalogue.BuildsOn{{Module: "mesh-tools", Artifact: "runtime"}}},
}
if _, _, err := standingOn(context.Background(), manifest, map[string]string{"mesh-tools/runtime": "x"}, noMirror); err == nil {
if _, err := standingOn(context.Background(), manifest, map[string]string{"mesh-tools/runtime": "x"}, noMirror); err == nil {
t.Fatal("a base with no build argument was accepted; nothing would have read it")
}
}
@@ -86,7 +86,7 @@ func TestADeclaredVendorImageIsCopiedInAndHandedToTheRecipe(t *testing.T) {
},
}
var asked []string
args, _, err := standingOn(context.Background(), manifest, nil, func(_ context.Context, from, repository string) (string, error) {
args, err := standingOn(context.Background(), manifest, nil, func(_ context.Context, from, repository string) (string, error) {
asked = append(asked, from+" -> "+repository)
return "127.0.0.1:5000/" + repository + "@sha256:" + strings.Repeat("d", 64), nil
})
@@ -101,7 +101,7 @@ func TestADeclaredVendorImageIsCopiedInAndHandedToTheRecipe(t *testing.T) {
}
// Unpinned, it is refused: a tag is what somebody else can move.
manifest.Build.On[0].Image = "quay.io/minio/mc:latest"
if _, _, err := standingOn(context.Background(), manifest, nil, noMirror); err == nil || !strings.Contains(err.Error(), "not pinned") {
if _, err := standingOn(context.Background(), manifest, nil, noMirror); err == nil || !strings.Contains(err.Error(), "not pinned") {
t.Fatalf("an unpinned vendor image was accepted: %v", err)
}
}
@@ -151,52 +151,3 @@ func TestARecipeIsReadAsInstructions(t *testing.T) {
t.Fatalf("a heredoc line or a continued stage was read as a base: %v", bases)
}
}
// What a build was handed as its bases is what it stood on — recorded, so a changed base knows what
// to rebuild (novox/hq 04-ISSUES/131). A recipe reads the base from an argument, so nothing else
// could know.
func TestTheBasesABuildWasHandedAreWhatItStoodOn(t *testing.T) {
manifest := catalogue.Manifest{
Module: "gitea",
Build: &catalogue.Build{
On: []catalogue.BuildsOn{
{Arg: "RUNTIME_BASE", Module: "mesh-tools", Artifact: "runtime"},
{Arg: "BUILD_BASE", Module: "mesh-tools", Artifact: "build"},
},
Artifacts: []catalogue.Artifact{{Name: "runtime", Kind: catalogue.ArtifactImage, From: "Dockerfile"}},
},
}
held := map[string]string{
"mesh-tools/runtime": "127.0.0.1:5000/mesh-tools/runtime@sha256:" + strings.Repeat("a", 64),
"mesh-tools/build": "127.0.0.1:5000/mesh-tools/build@sha256:" + strings.Repeat("b", 64),
}
_, resolved, err := standingOn(context.Background(), manifest, held, noMirror)
if err != nil {
t.Fatal(err)
}
got := against(t.TempDir(), manifest, resolved)
if len(got) != 2 || got[0] != held["mesh-tools/build"] || got[1] != held["mesh-tools/runtime"] {
t.Fatalf("the bases the build was handed were not what it stood on: %v", got)
}
}
// What a build read besides its module's own repository is the second repository its recipes name,
// each once: a module that packages source living elsewhere is affected when that source moves.
func TestWhatABuildReadIsTheRepositoriesItsRecipesName(t *testing.T) {
elsewhere := catalogue.ArtifactContext{Repository: "http://forge.internal:20000/novox/mesh-controller.git", Ref: "main"}
manifest := catalogue.Manifest{
Module: "builder",
Build: &catalogue.Build{Artifacts: []catalogue.Artifact{
{Name: "server", Kind: catalogue.ArtifactImage, From: "Dockerfile", Context: &elsewhere},
{Name: "tools", Kind: catalogue.ArtifactImage, From: "Dockerfile", Context: &elsewhere},
{Name: "config", Kind: catalogue.ArtifactArchive, From: "etc"},
}},
}
read := readBy(manifest)
if len(read) != 1 || read[0] != elsewhere {
t.Fatalf("the repositories this build read are %+v", read)
}
if readBy(catalogue.Manifest{Module: "gitea", Build: &catalogue.Build{}}) != nil {
t.Fatal("a module whose recipes name no other repository read one")
}
}
@@ -1,48 +0,0 @@
package catalogue
import (
"os"
"path/filepath"
"strings"
"testing"
)
// **The word does not come back through a manifest** (novox/hq ADR 0131). A module that wants
// messaging wants the mesh's bus, reached through the sdk and named by the `mesh-broker` seat. Naming
// the old wire protocol asks for the one server being retired, so both directions are refused at the
// parser — this is judged from the manifest alone, no store needed.
func TestAManifestProvidingAmqpIsRefused(t *testing.T) {
raw := []byte(`{"module":"old-broker","version":"1","provides":[{"name":"amqp","scope":"mesh"}]}`)
_, err := ParseManifest(raw)
if err == nil || !strings.Contains(err.Error(), `provides "amqp", which is not a provision`) {
t.Fatalf("a module providing amqp was not refused, or not for the reason: %v", err)
}
}
func TestAManifestRequiringAmqpIsRefused(t *testing.T) {
raw := []byte(`{"module":"forwarder","version":"1","requires":["amqp"]}`)
_, err := ParseManifest(raw)
if err == nil || !strings.Contains(err.Error(), `requires "amqp", which is not a provision`) {
t.Fatalf("a module requiring amqp was not refused, or not for the reason: %v", err)
}
}
// And the catalogue as checked out beside this repository names it nowhere — the three modules that
// did are removed under design 28 task 5.4, not converted.
func TestNoCatalogueManifestNamesAmqp(t *testing.T) {
modules, err := filepath.Glob("../../../mesh-catalog/modules/*/module.json")
if err != nil || len(modules) == 0 {
t.Skip("the catalogue is not checked out beside this repository")
}
for _, path := range modules {
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(raw), `"amqp"`) {
t.Errorf("%s names amqp, which is not a provision (novox/hq ADR 0131)",
filepath.Base(filepath.Dir(path)))
}
}
}
@@ -1,38 +0,0 @@
package catalogue
import (
"strings"
"testing"
)
// The artifact store's seat is one per mesh, read from the catalogue beside this checkout.
//
// **A second store anywhere is refused by name, not discovered as a consumer failure.** The seat
// was node-scoped, so a second `distribution` on another machine resolved cleanly there — and a
// node-scoped requirement with one candidate installs that candidate on the node, so anything that
// required the store's presence beside it would have raised a fresh, empty store on the wrong
// machine. Only afterwards did the mesh notice: `artifact-store` offered by two nodes, and every
// consumer elsewhere refusing to choose. The claim says it first, where the second store is
// assigned.
func TestASecondArtifactStoreAnywhereIsRefusedByName(t *testing.T) {
store := catalogueManifest(t, "distribution")
// The first store resolves as it always has.
if _, err := Resolve(shelf(store), []string{"distribution"}, workstation(), World{}); err != nil {
t.Fatalf("the store alone does not resolve: %v", err)
}
// A second one, on any other machine, is refused — and the refusal names the seat.
elsewhere := World{Held: []Held{{Claim: "the-artifact-store", Scope: ScopeMesh,
Node: "anchor", Module: "distribution"}}}
other := workstation()
other.Name = "laptop"
_, err := Resolve(shelf(store), []string{"distribution"}, other, elsewhere)
if err == nil {
t.Fatal("a second store was accepted on another machine; it would offer artifact-store a " +
"second time and every consumer elsewhere would refuse to choose")
}
if !strings.Contains(err.Error(), "the-artifact-store") || !strings.Contains(err.Error(), "one per mesh") {
t.Fatalf("refused without naming the seat: %v", err)
}
}
-9
View File
@@ -97,15 +97,6 @@ func Rerouted(reference, address string) string {
//
// A kept reference with no store to route it through is refused: sent as it is, the runtime would
// refuse the scheme on the machine, one push away from the reason.
//
// **What this does not reach: the images genesis pinned.** The installer builds the control plane
// and the builder before the mesh exists, pushes them itself and pins their manifests to
// `<registry>:<port>/mesh-controller@…` and `<registry>:<port>/mesh-builder@…` — single-segment
// repositories with no build record, so `with.Built` does not name them and they are left as
// written until each is rebuilt through the mesh, which records it by digest and path. Until then
// a registry that moves strands exactly those two on a recreate, and the control plane's is the
// one that cannot be repaired through the mesh. Rebuild both through `build` before moving the
// store (novox/hq 04-ISSUES/102, finding F4).
func artifactsInto(resource map[string]any, module string, with Rendering) error {
for _, key := range []string{"image", "source"} {
written, ok := resource[key].(string)
-87
View File
@@ -1,87 +0,0 @@
package catalogue
import (
"strings"
"testing"
)
// The mesh's bus is one per mesh, read from the catalogue beside this checkout.
//
// **This is step 2's claim, and it is checked here rather than in a bed** (novox/hq ADR 0116):
// adoption puts the NATS server into the `mesh-broker` seat on a mesh that is already running,
// and the property that matters is that a second one anywhere is refused *when it is assigned*,
// not discovered later as two servers holding different halves of the mesh's traffic. A second
// bus is not a degraded mesh; it is two meshes that both believe they are the one.
func TestASecondMeshBusAnywhereIsRefusedByName(t *testing.T) {
nats := catalogueManifest(t, "nats")
if _, err := Resolve(shelf(nats), []string{"nats"}, workstation(), World{}); err != nil {
t.Fatalf("the bus alone does not resolve: %v", err)
}
elsewhere := World{Held: []Held{{Claim: "mesh-broker", Scope: ScopeMesh,
Node: "anchor", Module: "nats"}}}
other := workstation()
other.Name = "laptop"
_, err := Resolve(shelf(nats), []string{"nats"}, other, elsewhere)
if err == nil {
t.Fatal("a second bus was accepted on another machine")
}
if !strings.Contains(err.Error(), "mesh-broker") || !strings.Contains(err.Error(), "one per mesh") {
t.Fatalf("refused without naming the seat: %v", err)
}
}
// The seat is the server's role, not the product's name (novox/hq ADR 0079). A different
// implementation of the bus claims the same seat, and the mesh refuses it for the same reason —
// which is the property that lets the bus be replaced at all.
func TestTheSeatRefusesADifferentBusToo(t *testing.T) {
nats := catalogueManifest(t, "nats")
held := World{Held: []Held{{Claim: "mesh-broker", Scope: ScopeMesh,
Node: "anchor", Module: "some-other-broker"}}}
other := workstation()
other.Name = "laptop"
if _, err := Resolve(shelf(nats), []string{"nats"}, other, held); err == nil {
t.Fatal("the seat admitted a second holder because the module's name differed")
}
}
// The old broker is gone from the catalogue (novox/hq ADR 0131, design 28 task 5.4), so it is no
// longer a fixture here. That two eligible holders stand beside each other with one on record is
// pinned in holdings_test.go against manifests this package owns.
// **A seat and the interface it delivers are different names, and renaming one must not rename
// the other** (novox/hq ADR 0118). This nearly went wrong: the seats were renamed to the `mesh-*`
// prefix, and a blanket search-and-replace also renamed `npm-package-registry` and `git` where
// they are *provisions* — which a consumer requires and a provider offers. The tests failed with
// "the package registry is served on <nil>", which does not say "you renamed an interface".
func TestRenamingASeatDidNotRenameTheInterfaceItDelivers(t *testing.T) {
for _, pair := range []struct{ seat, delivers string }{
{"git", "git"},
{"npm-package-registry", "npm-package-registry"},
{"the-artifact-store", "artifact-store"},
{"mesh-store", "postgres-database"},
{"mesh-broker", "mesh-bus"},
} {
s, known := SeatNamed(pair.seat)
if !known {
t.Fatalf("%q is not a seat", pair.seat)
}
if s.Delivers != pair.delivers {
t.Errorf("the %s seat delivers %q, expected %q — renaming the seat moved the "+
"interface with it, and every consumer requiring it would stop resolving",
pair.seat, s.Delivers, pair.delivers)
}
// **Three of these deliberately share a name with what they deliver**, and that is not an
// incomplete rename. Renaming a seat that delivers a provision cascades to every consumer
// requiring it, with a mesh-wide window where a holder stops resolving mid-flight — so the
// trunk deferred exactly those three (novox/hq ADR 0121) while renaming the node-scoped ones.
// What this test is for is the other direction: that renaming a seat never moves the
// interface, which once produced "the package registry is served on <nil>".
}
}
// A manifest written against an old seat name is told what it became rather than refused as
// unknown. **That map is the controller's store now, not this package** (novox/hq ADR 0122): a
// rename is a row, so the courtesy survives a rename nobody recompiled for. Checked where the
// table is read, not here, where there is no longer a hardcoded list to check against.
+18 -269
View File
@@ -97,29 +97,6 @@ type Rendering struct {
// compose it a second time.
Suffix string
// BusUsers is the mesh's composed user list, for the module holding `mesh-broker`. Empty on
// every other node, and on this one until the controller has composed it.
//
// **Only the users, never the server's own settings**: those are the module's, in its image and
// its mounts (Manifest.BusUsers).
BusUsers string
// BusMembership is this machine's membership for the bus the mesh is moving to, sealed to it
// (design 28, task 5.2). Empty for a machine not being moved. Written as a file the host reads
// after the declaration has applied, so the bus it names is standing before the machine leaves
// the one it is on.
BusMembership string
// MeshRange is the private network's CIDR (the range node addresses are allocated from), for a
// module that must name the whole mesh rather than one machine — an intrusion filter that must
// never ban a tunnel peer, say. A per-mesh value the module cannot know, so it is carried here
// and offered as ${machine:mesh-range}, the same way one machine's address is.
MeshRange string
// Accounts is each machine's operator account, by the same internal name Names uses (novox/hq
// to-be 29). What an ssh Host block's `User` line is composed from; empty for a machine no
// operator account is known on.
Accounts map[string]string
// Kept is every operator-sealed secret in the mesh, for a module that `keeps` them. Nil when
// nothing on this node keeps them, or the mesh has no operator key.
Kept *KeptExport
@@ -137,14 +114,6 @@ type Rendering struct {
// because which machines exist is a fact about the mesh.
Names map[string]string
// Machines is only the machines, by the same internal name — the subset of Names that is a
// node of this mesh rather than a name it was told to serve. Both matter and they are not the
// same set: a container's hosts wants every name, so a routed name resolves to the proxy that
// serves it, while a resolver told the mesh's suffix is authoritative for it answers from what
// it is given and forwards nothing — so a routed name written there is a name nobody asks for,
// standing beside the machines and looking as real as they do.
Machines map[string]string
Settings SettingsBy
Generators map[string]Generator
// Grants are the credentials this node must create, for the provisions it offers. Passed in
@@ -191,13 +160,6 @@ type Rendering struct {
// with an address — before references were kept without one — from an image a module runs
// straight from a public registry.
Built map[string]bool
// DataRoot is where this node keeps the directories the mesh places for its modules
// (novox/hq ADR 0112, to-be 27): a directory resource that states no path resolves to
// <DataRoot>/<module>/<id>, and ${dir:<id>} names that place from the module's own files,
// mounts and environment. A node setting fixed at installation; empty means the default,
// /var/lib — see dir_into.go.
DataRoot string
}
// machinePort is where a module's port lives on this machine, or the port itself when the mesh has
@@ -243,39 +205,10 @@ func (r Resolution) Compose(with Rendering) (Composed, error) {
if err != nil {
return Composed{}, err
}
if with.BusMembership != "" {
// The machine's own, not any module's: how it reaches the mesh from now on. Sealed like a
// secret and placed where the host looks for exactly this (design 28, task 5.2).
resources = append(resources, map[string]any{
"id": BusMembershipID(), "type": "file", "path": BusMembershipPath,
"sealed": with.BusMembership, "mode": "0600",
})
}
return Composed{Resources: resources, Owner: owner}, nil
}
// BusMembershipID names the resource carrying a machine's membership for the new bus, and
// BusMembershipPath is where the host reads it — the same constant on both sides.
func BusMembershipID() string { return "bus-membership" }
const BusMembershipPath = "/var/lib/mesh/membership-next.json"
func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[string]any, error) {
// Every manifest is placed first (novox/hq ADR 0112): the maps naming where its bindings,
// credentials and contributions land are resolved against this node's directories, so every
// reader below — the binding files, the sealed secrets, the grant paths a contribution
// names — sees a concrete place and none learns the vocabulary.
// Into a fresh slice, never the caller's: one resolution may compose for many nodes, and a
// slice element written in place would carry the first node's places into the second's.
placed := make([]Manifest, len(r.Modules))
for i, m := range r.Modules {
var err error
if placed[i], err = placedManifest(m, with); err != nil {
return nil, err
}
}
r.Modules = placed
// Where each provision's credentials land, so a contribution can name the file rather than
// carry a value the mesh does not have.
directories := map[string]string{}
@@ -376,12 +309,6 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri
"content": filtering, "mode": "0600",
})
}
// The node's fail2ban jails, composed from every module it runs (novox/hq to-be 31), written
// where the intrusion-prevention holder owns them. Like the rule set above: gathered from all
// modules, written by the one that holds the role.
if j := m.Jailing; j != nil {
first = append(first, jailsInto(r.Modules, j)...)
}
if c := m.Certificate; c != nil {
if with.Certificate == "" {
// Asked for and not issued. Refused rather than skipped: a module that serves TLS
@@ -402,35 +329,6 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri
})
}
}
if m.BusUsers != "" {
// **The claim authorises it, not the field.** This file holds every user's password
// hash, so a module that could ask for it could read every credential on the bus.
// Checked from this manifest alone, which is the cheapest check there is: whether some
// other module also claims the seat is resolution's business elsewhere, and one holder
// mesh-wide is already guaranteed.
if !m.ClaimsSeat("mesh-broker") {
return nil, fmt.Errorf(
"%s asks for the mesh's user list and does not claim mesh-broker. That file "+
"holds every user's password hash, so the seat is what authorises it",
m.Module)
}
if with.BusUsers == "" {
// Asked for and not composed. Refused rather than skipped, for the reason a
// certificate is: a bus with no user list refuses every connection in the mesh, and
// an empty file would look like a configuration problem on the machine.
return nil, fmt.Errorf(
"%s holds mesh-broker and the mesh composed no user list, so the bus would "+
"refuse every connection", m.Module)
}
first = append(first, map[string]any{
"id": BusUsersID(), "type": "file", "path": m.BusUsers,
"content": with.BusUsers,
// Readable by the server and nothing else. Hashes rather than passwords, so this is
// not a set of working credentials — but a list of every user in the mesh is worth
// keeping to the one process that needs it.
"mode": "0600",
})
}
for _, name := range sortedKeys(m.OwnSecrets) {
sealed := with.Needed[m.Module][name]
if sealed == "" {
@@ -613,19 +511,13 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri
}
// And what its bindings say, for the half of a connection that is not secret.
known := knownFor(m, r.Needs, r.Node)
// And where this node places the directories the module declared without a path
// (novox/hq ADR 0112) — resolved once per module, named by ${dir:…} from any resource.
dirs := dirsFor(m, with)
// And the machine underneath, which no binding of its own can tell it.
thisMachine := machineFacts(r, with.Names, with.MeshRange)
thisMachine := machineFacts(r)
// Which of this module's files carry a secret, for the rule that a container may not read
// one of them as its environment without saying so (ADR 0086, issue 041).
secretFiles := secretFilesOf(resources)
// Which of this module's resources its preparation runs before, if it prepares anything.
prepareBefore := preparationTarget(m)
for _, unsettled := range resources {
resource, err := ApplySettings(unsettled, with.Settings[m.Module])
if err != nil {
@@ -641,13 +533,6 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri
// Said in the catalogue, not on the machine: the host parses strictly and knows no
// such field, and the reason is for a reader of the manifest.
delete(copied, SecretsInEnvironment)
// **Placed before anything reads a path.** A pathless directory receives the path
// this node resolves for it, and every ${dir:…} — in paths, mounts, content and
// environment — becomes that path, so what follows sees only concrete places
// (novox/hq ADR 0112). The host receives paths exactly as it always has.
if err := dirInto(copied, dirs, m.Module); err != nil {
return nil, err
}
// **After settings, and that is the whole reason it is here.** A module's file
// content is where a setting lands, so a placeholder may only exist once the setting
// has been put in — filling secrets first would look at content that is not yet what
@@ -711,18 +596,6 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri
if renamed := reflectsRenamed(m.Module, resource["reload-on"]); renamed != nil {
copied["reload-on"] = renamed
}
// **A version prepares its state before it runs** (novox/hq ADR 0135). Derived from the
// module's own resource rather than declared beside it: what prepares the state is the
// module's own code, so what it is given has to be what that code is given — and a
// second resource written by hand is a second copy to drift from the first. Placed
// immediately before it, because a run-once step stops everything the declaration
// places after it (ADR 0052), which is how a version whose preparation failed does not
// serve.
if prepareBefore != "" && fmt.Sprint(resource["id"]) == prepareBefore {
step := prepared(copied)
owner[fmt.Sprint(step["id"])] = m.Module
out = append(out, step)
}
owner[fmt.Sprint(copied["id"])] = m.Module
out = append(out, copied)
}
@@ -731,7 +604,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri
// plane's; making a name resolve is the module's software. Emitted as ordinary files under
// this module's name, so they are applied, reported and removed exactly as anything else
// it declares.
given, err := FactsInto(m, r, with.Names, with.Machines, with.Accounts, with.Suffix)
given, err := FactsInto(m, r, with.Names, with.Suffix)
if err != nil {
return nil, err
}
@@ -1030,53 +903,30 @@ func (r Resolution) contributions(settings SettingsBy, grants []Grant,
if err != nil {
return nil, fmt.Errorf("%s contributing to %s: %w", m.Module, to, err)
}
composeName(values, r.PublicDomain, r.At)
composeName(values, r.PublicDomain)
out[to] = append(out[to], Contribution{From: m.Module, Values: values})
}
// Several contributions to one requirement (ADR 0094's sibling for `contributes`): an
// object store's data API and its console are two different public names from one module,
// not one. Never in `granted` — a route names a host, not a credential — so every local
// name always reaches the provider from here.
for _, to := range sortedKeys(m.ContributesMany) {
for _, local := range sortedKeys(m.ContributesMany[to]) {
values, err := settle(m.ContributesMany[to][local], settings[m.Module], nil,
m.Module+" contributing "+local+" to "+to)
if err != nil {
return nil, fmt.Errorf("%s contributing %s to %s: %w", m.Module, local, to, err)
}
composeName(values, r.PublicDomain, r.At)
out[to] = append(out[to], Contribution{From: m.Module, Values: values})
}
}
}
return out, nil
}
// composeName joins a contribution's label with a node's public domain, and separately with its
// private one, in place (novox/hq ADR 0056).
// composeName joins a contribution's label with a node's public domain, in place (novox/hq ADR
// 0056).
//
// **The whole of what the mesh does with a route's name: join two given strings — twice.** A
// contribution carries a `label` — the subdomain its operator chose — and the node carries its
// public domain and its own private-network address; the granted names are `<label>.<public-domain>`
// and `<label>.<internal-domain>`, and the mesh interprets none of the halves. It runs on any
// contribution carrying a label, not only a route's, because the mesh does not know what a
// **The whole of what the mesh does with a route's name: join two given strings.** A contribution
// carries a `label` — the subdomain its operator chose — and the node carries its public domain;
// the granted name is `<label>.<public-domain>` and the mesh interprets neither half. It runs on
// any contribution carrying a label, not only a route's, because the mesh does not know what a
// provision means — a name it can compose from parts it was given is the point, whatever the
// provision is called.
//
// **The internal name is not a security boundary.** A predecessor proxy that answered both a
// public and a private-network hostname for the same route did so as a convenience — reaching a
// service over the VPN without a public TLS round trip — not as an access control, and composing
// the same alias here restores that convenience rather than adding one. A route with no internal
// domain to compose against (a node not on the private network) gets no internal name, the same as
// it gets no public one with no public domain.
//
// **Additive, so an unmigrated catalogue still works.** A contribution that already carries a full
// `name` and no `label` is left exactly as it is: the catalogue can migrate module by module while
// the running mesh keeps serving the full names it has. And a labelled contribution on a node with
// no public domain composes nothing — there is nothing to join it to — which reads downstream as a
// route that named no host, the same as it would have before this existed.
func composeName(values map[string]any, publicDomain, internalDomain string) {
if values == nil {
func composeName(values map[string]any, publicDomain string) {
if values == nil || publicDomain == "" {
return
}
if _, already := values["name"]; already {
@@ -1089,25 +939,14 @@ func composeName(values map[string]any, publicDomain, internalDomain string) {
if !ok || strings.TrimSpace(label) == "" {
return
}
trimmed := strings.TrimSpace(label)
if trimmed == "@" {
// The apex: a module served at the bare domain, no subdomain — the zone-file convention
// `@`. Composes to the domain itself, so a node's own site is a label like any other rather
// than the one route that must still carry a full name.
if publicDomain != "" {
values["name"] = publicDomain
}
if internalDomain != "" {
values["internal-name"] = internalDomain
}
if strings.TrimSpace(label) == "@" {
// The apex: a module served at the bare public domain, no subdomain — the zone-file
// convention `@`. Composes to the domain itself, so a node's own site is a label like any
// other rather than the one route that must still carry a full name.
values["name"] = publicDomain
return
}
if publicDomain != "" {
values["name"] = trimmed + "." + publicDomain
}
if internalDomain != "" {
values["internal-name"] = trimmed + "." + internalDomain
}
values["name"] = strings.TrimSpace(label) + "." + publicDomain
}
// receivedFile is the file a provider is given its consumers' contributions in.
@@ -1258,34 +1097,17 @@ func boundFile(n Needed, path, as string) (map[string]any, error) {
// arrangement refused is the ordinary one. A node running eight services against one database is
// not an edge case; it is what a machine looks like. Now each consumer has its own credential and
// there is nothing left to refuse.
//
// **One credential, even where a module contributes several times.** A module may answer one
// requirement more than once (ADR 0094's sibling for `contributes`) — an object store's data API
// and its console are two different names, not one. There is still only one `Needed` for it, one
// credential minted, one grant to settle: a pair credential is not a place to put a label or a
// port. So where several of this module's contributions reach the same requirement, none of them
// is "the" value — settling to the first, arbitrarily, would hand the grant one contribution's
// values under a credential the OTHER contribution's consumer never sees, and would collide with
// that contribution's own entry from contributions() besides. Empty values, still granted: the
// module asked, gets its credential, and each named contribution reaches the provider on its own.
func (r Resolution) ContributionsFrom(requirement, module string, settings SettingsBy) (
map[string]any, bool, error) {
all, err := r.contributions(settings, nil, nil)
if err != nil {
return nil, false, err
}
var mine []map[string]any
for _, g := range all[requirement] {
if g.From == module {
mine = append(mine, g.Values)
return g.Values, true, nil
}
}
if len(mine) == 1 {
return mine[0], true, nil
}
if len(mine) > 1 {
return map[string]any{}, true, nil
}
// It contributes no payload — but a require-only consumer of a parameterless provision (one whose
// `serves` names no consumer-supplied key: `redis-cache`, `amqp`) still ASKS for it and must be
// granted a credential. Keying "asks" on contributions alone marked those grants withdrawn
@@ -1625,76 +1447,3 @@ func atMachinePort(serves map[string]any, module string, ports map[string]map[in
func AtPublishedPort(values map[string]any, module string, published map[int]int) map[string]any {
return atMachinePort(values, module, map[string]map[int]int{module: published})
}
// PreparationArgument is how the mesh asks a module to prepare its state: one word, to the module's
// own program, whatever that program is (novox/hq ADR 0135).
//
// **One word for every kind of module.** A module built as a Go binary receives it as its argument;
// one built as a bundle receives it through the runtime, whose entry takes the same word. So the
// mesh has one way of asking and a module has one way of answering, and neither learns the other's
// shape.
const PreparationArgument = "prepare"
// preparationTarget is the resource a module's preparation runs before: its own workload.
//
// The first container carrying an artifact this module built, and not itself a step — that is the
// thing that runs the module's code, and therefore the thing whose state must be ready. Empty when
// the module prepares nothing, or when nothing it declares could run its code.
//
// **A module with two own workloads gates the first of them.** Five modules in the catalogue declare
// more than one container of their own, none of them preparing anything today. If one ever does and
// its second workload shares the state, the gate is in front of the first — stated here because the
// alternative is a field asking an author to restate what the mesh can see.
func preparationTarget(m Manifest) string {
if !m.Prepares {
return ""
}
for _, r := range m.Resources {
if fmt.Sprint(r["type"]) != "container" || !ownArtifact(r, m.Module) {
continue
}
if once, _ := r["run-once"].(bool); once {
continue
}
return fmt.Sprint(r["id"])
}
return ""
}
// ownArtifact is whether a resource runs something this module built, in either spelling a manifest
// may be in: naming the artifact, before a build resolved it, or carrying the reference a build
// recorded — this mesh's own store, under this module's name.
func ownArtifact(resource map[string]any, module string) bool {
if named, _ := resource["artifact"].(string); named != "" {
return true
}
image, _ := resource["image"].(string)
return strings.HasPrefix(image, ArtifactStoreScheme+module+"/")
}
// prepared is the module's own resource as the step that prepares its state: the same image, the same
// context, run to completion with the mesh's preparation argument.
//
// Three things are taken away rather than copied, each because the step runs while the version it
// prepares for is still running. A published port cannot be bound twice, and a step that tried would
// fail for a reason that has nothing to do with the state. A fixed address cannot be held twice, for
// the same reason. And a cadence is what a step is the opposite of: a container runs once and gates,
// or on a schedule, or stays up, never two (ADR 0053).
func prepared(from map[string]any) map[string]any {
step := map[string]any{}
for k, v := range from {
step[k] = v
}
// **A hyphen, not a dot.** A resource's id is `<module>.<its own id>`, and a module's name may
// itself contain a dot (`novox.be`), so the module is everything before the *last* dot — which
// only works if what the mesh derives adds no dot of its own.
step["id"] = fmt.Sprint(from["id"]) + "-prepare"
step["name"] = fmt.Sprint(from["name"]) + "-prepare"
step["run-once"] = true
step["args"] = []any{PreparationArgument}
delete(step, "ports")
delete(step, "ip")
delete(step, "schedule")
delete(step, "reload-on")
return step
}
-80
View File
@@ -1,80 +0,0 @@
package catalogue
import "testing"
// The mesh's user list reaches the module holding the bus, and nothing else.
//
// Three refusals and one delivery, because each of the refusals would be silent in a different way:
// a module that asked and was given it could read every credential on the bus; a bus given an empty
// file refuses every connection in the mesh and looks like a machine problem; and a bus that never
// asked gets nothing rather than a file it does not read.
func TestTheMeshsUserListGoesOnlyToTheModuleHoldingTheBus(t *testing.T) {
theBus := func() Manifest {
return Manifest{
Module: "nats", Version: "1",
Claims: []Claim{{Name: "mesh-broker", Scope: ScopeMesh}},
BusUsers: "/var/lib/nats-module/conf/accounts.conf",
Resources: []map[string]any{},
}
}
on := func(t *testing.T, m Manifest, with Rendering) ([]map[string]any, error) {
t.Helper()
return Resolution{Node: "anchor", Modules: []Manifest{m}}.Declaration(with)
}
t.Run("the holder is given it", func(t *testing.T) {
resources, err := on(t, theBus(), Rendering{BusUsers: "accounts { MESH { users = [] } }"})
if err != nil {
t.Fatal(err)
}
// Prefixed with the module it came from, like every resource: two modules may reasonably
// both call something "config", and without the prefix the second would silently replace
// the first.
var found map[string]any
for _, r := range resources {
if r["id"] == "nats."+BusUsersID() {
found = r
}
}
if found == nil {
t.Fatalf("the bus was given no user list: %+v", resources)
}
if found["path"] != "/var/lib/nats-module/conf/accounts.conf" {
t.Errorf("written to %v rather than where the module asked", found["path"])
}
if found["mode"] != "0600" {
t.Errorf("mode %v: a list of every user in the mesh belongs to the one process that "+
"needs it", found["mode"])
}
})
t.Run("a module that does not claim the seat is refused", func(t *testing.T) {
m := theBus()
m.Claims = nil
if _, err := on(t, m, Rendering{BusUsers: "accounts {}"}); err == nil {
t.Fatal("a module that claims nothing was handed every user's password hash")
}
})
t.Run("the holder with nothing composed is refused", func(t *testing.T) {
if _, err := on(t, theBus(), Rendering{}); err == nil {
t.Fatal("the bus was given an empty user list, so it would refuse every connection in " +
"the mesh and look like a machine problem")
}
})
t.Run("a module that did not ask gets nothing", func(t *testing.T) {
m := theBus()
m.BusUsers = ""
resources, err := on(t, m, Rendering{BusUsers: "accounts {}"})
if err != nil {
t.Fatal(err)
}
for _, r := range resources {
if r["id"] == "nats."+BusUsersID() {
t.Fatal("a module that asked for no user list was given one")
}
}
})
}
-322
View File
@@ -1,322 +0,0 @@
package catalogue
import (
"fmt"
"regexp"
"sort"
"strings"
)
// A directory the mesh places (novox/hq ADR 0112, to-be 27, issue 119).
//
// **A module definition names no host path.** A directory resource may omit `path`; the mesh
// resolves where it lands when the declaration is composed — `<root>/<module>/<id>`, the root a
// node's own setting with /var/lib as the default. From then on the module's own files, mounts
// and environment name the place as `${dir:<id>}`, the same shape as `${bound:…}` and
// `${secret:…}`: a fact the module asks for by name and never states.
//
// **A directory that states a path keeps it, and still answers `${dir:<id>}`.** That is the
// placement for an adopted machine: data that must sit where the predecessor already put it is
// declared with the path as the exception it is, and everything else in the module names it by
// id — so moving it later is one line, not a search.
//
// **Resolved here, not on the machine.** The host receives concrete paths exactly as it always
// has; nothing new reaches it and it learns no field. Which also means a resolved path changing
// is a spec change like any other — and the spec comparison must see it (novox/hq issue 126).
// defaultDataRoot is where module data lands when a node states no root of its own.
const defaultDataRoot = "/var/lib"
// dirRef is how a module names one of its placed directories: ${dir:<id>}.
var dirRef = regexp.MustCompile(`\$\{dir:([a-z0-9][a-z0-9-]*)\}`)
// dataRoot is the root this node keeps placed directories under.
func dataRoot(with Rendering) string {
if root := strings.TrimRight(strings.TrimSpace(with.DataRoot), "/"); root != "" {
return root
}
return defaultDataRoot
}
// dirsFor is every placed directory of a module, id → the path it resolves to on this node.
//
// A pathless directory saying `"place": "."` is the assignment's own root, <root>/<module> —
// to-be 27's one directory per assignment, which every other placed thing sits beneath. At most
// one makes sense; nothing enforces one, because two ids resolving to one path is a mistake the
// module's own files make visible immediately.
func dirsFor(m Manifest, with Rendering) map[string]string {
dirs := map[string]string{}
for _, r := range m.Resources {
if fmt.Sprint(r["type"]) != "directory" {
continue
}
id := fmt.Sprint(r["id"])
if path, stated := r["path"].(string); stated && path != "" {
dirs[id] = strings.TrimRight(path, "/")
continue
}
if place, said := r["place"].(string); said && place == "." {
dirs[id] = dataRoot(with) + "/" + m.Module
continue
}
dirs[id] = dataRoot(with) + "/" + m.Module + "/" + id
}
return dirs
}
// placedOrAbsolute says a path is usable where the mesh needs one: absolute already, or
// beginning with a placed reference — resolution makes it absolute before anything reads it.
// (unknownDirRefs is what checks the reference names a real directory.)
func placedOrAbsolute(path string) bool {
return strings.HasPrefix(path, "/") ||
(strings.HasPrefix(path, "${dir:") && dirRef.MatchString(path))
}
// dirFill resolves every ${dir:…} in one string, or refuses a reference naming no directory.
func dirFill(s string, dirs map[string]string, module string) (string, error) {
var missing error
out := dirRef.ReplaceAllStringFunc(s, func(ref string) string {
id := dirRef.FindStringSubmatch(ref)[1]
path, has := dirs[id]
if !has {
missing = fmt.Errorf(
"%s says ${dir:%s}, and %s declares no directory %q. It declares %s",
module, id, module, id, orNothing(namesOfDirs(dirs)))
return ref
}
return path
})
return out, missing
}
// placedManifest is the manifest with every path the mesh resolves already resolved: the maps
// naming where bindings, credentials and contributions land are filled against this node's
// placed directories, so everything downstream — the generated binding files, the sealed
// secrets, the grant directories — reads a concrete place and learns nothing new.
func placedManifest(m Manifest, with Rendering) (Manifest, error) {
dirs := dirsFor(m, with)
fillMap := func(in map[string]string) (map[string]string, error) {
if len(in) == 0 {
return in, nil
}
out := make(map[string]string, len(in))
for key, value := range in {
filled, err := dirFill(value, dirs, m.Module)
if err != nil {
return nil, err
}
out[key] = filled
}
return out, nil
}
var err error
if m.Receives, err = fillMap(m.Receives); err != nil {
return m, err
}
if m.Binds, err = fillMap(m.Binds); err != nil {
return m, err
}
if m.Secrets, err = fillMap(m.Secrets); err != nil {
return m, err
}
if m.OwnSecrets, err = fillMap(m.OwnSecrets); err != nil {
return m, err
}
if m.Grants, err = fillMap(m.Grants); err != nil {
return m, err
}
if len(m.SecretsMany) > 0 {
many := make(map[string]map[string]string, len(m.SecretsMany))
for to, locals := range m.SecretsMany {
if many[to], err = fillMap(locals); err != nil {
return m, err
}
}
m.SecretsMany = many
}
return m, nil
}
// dirInto places a resource: a pathless directory is given the path the mesh resolved for it,
// and every ${dir:…} the resource carries — in its path, its content, its mounts, its
// environment and its env-files — becomes that path.
//
// A reference naming no directory of this module is refused. Left as written, the literal
// `${dir:x}` would reach the machine as a path, and the runtime would create and mount a
// directory called `${dir:x}` — real, wrong, and named after the mistake.
func dirInto(resource map[string]any, dirs map[string]string, module string) error {
fill := func(s string) (string, error) { return dirFill(s, dirs, module) }
if fmt.Sprint(resource["type"]) == "directory" {
id := fmt.Sprint(resource["id"])
if path, stated := resource["path"].(string); !stated || path == "" {
resource["path"] = dirs[id]
}
// Said in the catalogue, not on the machine: the host parses strictly and knows no
// such field — resolved, the place IS the path.
delete(resource, "place")
}
var err error
if path, ok := resource["path"].(string); ok {
if resource["path"], err = fill(path); err != nil {
return err
}
}
if content, ok := resource["content"].(string); ok {
if resource["content"], err = fill(content); err != nil {
return err
}
}
// Nested values are rebuilt, never written into: the resource is a shallow copy of the
// manifest's own map, and the manifest is composed once per node — a fill written in place
// would leave the first node's paths inside every later composition.
if volumes, ok := resource["volumes"].([]any); ok {
filled := make([]any, len(volumes))
for i, v := range volumes {
filled[i] = v
if mount, ok := v.(string); ok {
if filled[i], err = fill(mount); err != nil {
return err
}
}
}
resource["volumes"] = filled
}
if env, ok := resource["env"].(map[string]any); ok {
filled := make(map[string]any, len(env))
for key, v := range env {
filled[key] = v
if value, ok := v.(string); ok {
if filled[key], err = fill(value); err != nil {
return err
}
}
}
resource["env"] = filled
}
if files, ok := resource["env-file"].([]any); ok {
filled := make([]any, len(files))
for i, v := range files {
filled[i] = v
if path, ok := v.(string); ok {
if filled[i], err = fill(path); err != nil {
return err
}
}
}
resource["env-file"] = filled
}
return nil
}
// unknownDirRefs is every ${dir:…} in the definition that names no directory the definition
// declares — refused where the author is, not at composition on some later day (the same
// near-versus-far reasoning as the host's strict parse).
func (m Manifest) unknownDirRefs() []string {
declared := map[string]bool{}
for _, r := range m.Resources {
if fmt.Sprint(r["type"]) == "directory" {
declared[fmt.Sprint(r["id"])] = true
}
}
referenced := func(s string) []string {
var ids []string
for _, match := range dirRef.FindAllStringSubmatch(s, -1) {
ids = append(ids, match[1])
}
return ids
}
var problems []string
for _, r := range m.Resources {
place, said := r["place"].(string)
if !said {
continue
}
if fmt.Sprint(r["type"]) != "directory" {
problems = append(problems, fmt.Sprintf(
"%s says place on %v, which is not a directory — only a directory is placed",
m.Module, r["id"]))
continue
}
if path, stated := r["path"].(string); stated && path != "" {
problems = append(problems, fmt.Sprintf(
"%s states both path and place on %v — a stated path IS the placement",
m.Module, r["id"]))
}
if place != "." {
problems = append(problems, fmt.Sprintf(
"%s says place %q on %v, and the only place is %q — the assignment's own root",
m.Module, place, r["id"], "."))
}
}
seen := map[string]bool{}
refuse := func(id string, where any) {
if declared[id] || seen[id] {
return
}
seen[id] = true
problems = append(problems, fmt.Sprintf(
"%s says ${dir:%s} in %v, and declares no directory %q — a reference the mesh "+
"cannot place would reach the machine as a literal path",
m.Module, id, where, id))
}
for _, r := range m.Resources {
for _, field := range []string{"path", "content"} {
if s, ok := r[field].(string); ok {
for _, id := range referenced(s) {
refuse(id, r["id"])
}
}
}
for _, field := range []string{"volumes", "env-file"} {
if list, ok := r[field].([]any); ok {
for _, v := range list {
if s, ok := v.(string); ok {
for _, id := range referenced(s) {
refuse(id, r["id"])
}
}
}
}
}
if env, ok := r["env"].(map[string]any); ok {
for _, v := range env {
if s, ok := v.(string); ok {
for _, id := range referenced(s) {
refuse(id, r["id"])
}
}
}
}
}
maps := map[string]map[string]string{
"receives": m.Receives, "binds": m.Binds, "secrets": m.Secrets,
"own-secrets": m.OwnSecrets, "grants": m.Grants,
}
for field, entries := range maps {
for _, value := range entries {
for _, id := range referenced(value) {
refuse(id, field)
}
}
}
for to, locals := range m.SecretsMany {
for _, value := range locals {
for _, id := range referenced(value) {
refuse(id, "secrets."+to)
}
}
}
sort.Strings(problems)
return problems
}
func namesOfDirs(dirs map[string]string) []string {
var names []string
for id := range dirs {
names = append(names, fmt.Sprintf("%q", id))
}
sort.Strings(names)
return names
}
-252
View File
@@ -1,252 +0,0 @@
package catalogue
// A directory the mesh places (novox/hq ADR 0112). These tests pin the contract: a pathless
// directory resolves under the node's root, ${dir:…} names it from every field a host path can
// live in, a stated path is the adopted-data placement and wins, an unknown reference refuses at
// the manifest, and filling for one node never leaks into the next composition.
import (
"strings"
"testing"
)
func placedModule() Manifest {
return Manifest{
Module: "photos",
Resources: []map[string]any{
{"id": "data", "type": "directory", "mode": "0700"},
{"id": "server-env", "type": "file", "path": "${dir:data}/server.env",
"content": "STORE=${dir:data}/objects\n"},
{"id": "server", "type": "container", "name": "photos-server",
"volumes": []any{"${dir:data}:/data"},
"env": map[string]any{"DATA": "${dir:data}/objects"},
"env-file": []any{"${dir:data}/server.env"}},
},
}
}
func TestAPathlessDirectoryResolvesUnderTheNodesRoot(t *testing.T) {
m := placedModule()
dirs := dirsFor(m, Rendering{})
if dirs["data"] != "/var/lib/photos/data" {
t.Fatalf("the default root is /var/lib and the shape is <root>/<module>/<id>; got %q", dirs["data"])
}
dirs = dirsFor(m, Rendering{DataRoot: "/tank/nox/"})
if dirs["data"] != "/tank/nox/photos/data" {
t.Fatalf("a node's own root is honoured, trailing slash and all; got %q", dirs["data"])
}
}
func TestDirReferencesBecomeThePlaceInEveryField(t *testing.T) {
m := placedModule()
dirs := dirsFor(m, Rendering{})
directory := shallowCopy(m.Resources[0])
if err := dirInto(directory, dirs, m.Module); err != nil {
t.Fatal(err)
}
if directory["path"] != "/var/lib/photos/data" {
t.Fatalf("a pathless directory receives its resolved path; got %v", directory["path"])
}
file := shallowCopy(m.Resources[1])
if err := dirInto(file, dirs, m.Module); err != nil {
t.Fatal(err)
}
if file["path"] != "/var/lib/photos/data/server.env" {
t.Fatalf("a file's path names the place; got %v", file["path"])
}
if file["content"] != "STORE=/var/lib/photos/data/objects\n" {
t.Fatalf("a file's content names the place; got %v", file["content"])
}
container := shallowCopy(m.Resources[2])
if err := dirInto(container, dirs, m.Module); err != nil {
t.Fatal(err)
}
if container["volumes"].([]any)[0] != "/var/lib/photos/data:/data" {
t.Fatalf("a mount names the place; got %v", container["volumes"])
}
if container["env"].(map[string]any)["DATA"] != "/var/lib/photos/data/objects" {
t.Fatalf("an environment value names the place; got %v", container["env"])
}
if container["env-file"].([]any)[0] != "/var/lib/photos/data/server.env" {
t.Fatalf("an env-file names the place; got %v", container["env-file"])
}
}
func TestAStatedPathIsThePlacementAndStillAnswersByName(t *testing.T) {
m := placedModule()
// The adopted-machine case: data that must sit where the predecessor already put it.
m.Resources[0]["path"] = "/services/mssql/data/"
dirs := dirsFor(m, Rendering{})
if dirs["data"] != "/services/mssql/data" {
t.Fatalf("a stated path wins over the root, trimmed; got %q", dirs["data"])
}
container := shallowCopy(m.Resources[2])
if err := dirInto(container, dirs, m.Module); err != nil {
t.Fatal(err)
}
if container["volumes"].([]any)[0] != "/services/mssql/data:/data" {
t.Fatalf("references follow the placement; got %v", container["volumes"])
}
}
func TestFillingForOneNodeLeaksIntoNoOther(t *testing.T) {
m := placedModule()
first := shallowCopy(m.Resources[2])
if err := dirInto(first, dirsFor(m, Rendering{DataRoot: "/first"}), m.Module); err != nil {
t.Fatal(err)
}
second := shallowCopy(m.Resources[2])
if err := dirInto(second, dirsFor(m, Rendering{DataRoot: "/second"}), m.Module); err != nil {
t.Fatal(err)
}
if got := second["volumes"].([]any)[0]; got != "/second/photos/data:/data" {
t.Fatalf("the second composition must see the manifest, not the first fill; got %v", got)
}
if m.Resources[2]["volumes"].([]any)[0] != "${dir:data}:/data" {
t.Fatalf("the manifest itself stays a template; got %v", m.Resources[2]["volumes"])
}
}
func TestAReferenceToNoDirectoryRefusesAtTheManifest(t *testing.T) {
m := placedModule()
m.Resources[2]["volumes"] = []any{"${dir:date}:/data"} // a typo, the likely shape
problems := m.unknownDirRefs()
if len(problems) != 1 || !strings.Contains(problems[0], `${dir:date}`) {
t.Fatalf("a reference naming no directory is a manifest problem; got %v", problems)
}
if got := placedModule().unknownDirRefs(); len(got) != 0 {
t.Fatalf("a correct definition has none; got %v", got)
}
}
func TestAReferenceToNoDirectoryRefusesAtCompositionToo(t *testing.T) {
m := placedModule()
container := shallowCopy(m.Resources[2])
container["env"] = map[string]any{"DATA": "${dir:date}"}
err := dirInto(container, dirsFor(m, Rendering{}), m.Module)
if err == nil || !strings.Contains(err.Error(), `"date"`) || !strings.Contains(err.Error(), `"data"`) {
t.Fatalf("the refusal names the mistake and what exists; got %v", err)
}
}
func TestTheAssignmentsOwnRootIsAPlace(t *testing.T) {
m := Manifest{Module: "mailu", Resources: []map[string]any{
{"id": "state", "type": "directory", "place": ".", "mode": "0700"},
{"id": "data-mail", "type": "directory"},
}}
dirs := dirsFor(m, Rendering{})
if dirs["state"] != "/var/lib/mailu" {
t.Fatalf("place %q is the assignment's root; got %q", ".", dirs["state"])
}
if dirs["data-mail"] != "/var/lib/mailu/data-mail" {
t.Fatalf("everything else sits beneath it; got %q", dirs["data-mail"])
}
root := shallowCopy(m.Resources[0])
if err := dirInto(root, dirs, m.Module); err != nil {
t.Fatal(err)
}
if root["path"] != "/var/lib/mailu" {
t.Fatalf("the root receives its path; got %v", root["path"])
}
if _, still := root["place"]; still {
t.Fatal("place must never reach the host, which parses strictly")
}
}
func TestTheManifestsMapsArePlaced(t *testing.T) {
m := Manifest{
Module: "photos",
Resources: []map[string]any{
{"id": "state", "type": "directory", "place": "."},
},
Binds: map[string]string{"route": "${dir:state}/route.json"},
Secrets: map[string]string{"mongodb-database": "${dir:state}/database.secret"},
OwnSecrets: map[string]string{"admin-key": "${dir:state}/admin-key.secret"},
Receives: map[string]string{"route": "${dir:state}/grants/mesh.json"},
}
placed, err := placedManifest(m, Rendering{})
if err != nil {
t.Fatal(err)
}
if placed.Binds["route"] != "/var/lib/photos/route.json" {
t.Fatalf("binds are placed; got %v", placed.Binds)
}
if placed.Secrets["mongodb-database"] != "/var/lib/photos/database.secret" {
t.Fatalf("secrets are placed; got %v", placed.Secrets)
}
if placed.OwnSecrets["admin-key"] != "/var/lib/photos/admin-key.secret" {
t.Fatalf("own-secrets are placed; got %v", placed.OwnSecrets)
}
if placed.Receives["route"] != "/var/lib/photos/grants/mesh.json" {
t.Fatalf("receives are placed; got %v", placed.Receives)
}
if m.Binds["route"] != "${dir:state}/route.json" {
t.Fatalf("the manifest itself stays a template; got %v", m.Binds)
}
}
func TestAMapReferenceToNoDirectoryRefusesAtTheManifest(t *testing.T) {
m := Manifest{
Module: "photos",
Resources: []map[string]any{{"id": "state", "type": "directory", "place": "."}},
Binds: map[string]string{"route": "${dir:stat}/route.json"},
}
problems := m.unknownDirRefs()
if len(problems) != 1 || !strings.Contains(problems[0], `${dir:stat}`) {
t.Fatalf("a map naming no directory is a manifest problem; got %v", problems)
}
}
func TestPlaceIsValidatedAtTheManifest(t *testing.T) {
both := Manifest{Module: "x", Resources: []map[string]any{
{"id": "d", "type": "directory", "place": ".", "path": "/somewhere"},
}}
if got := both.unknownDirRefs(); len(got) != 1 || !strings.Contains(got[0], "both path and place") {
t.Fatalf("path beside place refuses; got %v", got)
}
elsewhere := Manifest{Module: "x", Resources: []map[string]any{
{"id": "f", "type": "file", "place": ".", "path": "/somewhere", "content": ""},
}}
if got := elsewhere.unknownDirRefs(); len(got) != 1 || !strings.Contains(got[0], "not a directory") {
t.Fatalf("place on a file refuses; got %v", got)
}
wrong := Manifest{Module: "x", Resources: []map[string]any{
{"id": "d", "type": "directory", "place": "sub/dir"},
}}
if got := wrong.unknownDirRefs(); len(got) != 1 || !strings.Contains(got[0], `the only place is "."`) {
t.Fatalf("a place that is not the root refuses; got %v", got)
}
}
func TestTwoModulesPlacedRootsAreNoCollision(t *testing.T) {
a := Manifest{Module: "gitea", Resources: []map[string]any{
{"id": "state", "type": "directory", "place": "."},
{"id": "env", "type": "file", "path": "${dir:state}/server.env", "content": ""},
}}
b := Manifest{Module: "nextcloud", Resources: []map[string]any{
{"id": "state", "type": "directory", "place": "."},
{"id": "env", "type": "file", "path": "${dir:state}/server.env", "content": ""},
}}
if got := checkResources([]Manifest{a, b}); len(got) != 0 {
t.Fatalf("alike templates are different places; got %v", got)
}
// And the real collision is still caught: a module stating another's placed root.
c := Manifest{Module: "squatter", Resources: []map[string]any{
{"id": "nest", "type": "directory", "path": "/var/lib/gitea"},
}}
got := checkResources([]Manifest{a, c})
if len(got) != 1 || !strings.Contains(got[0], `"/var/lib/gitea"`) {
t.Fatalf("a stated path on a placed root collides; got %v", got)
}
}
func shallowCopy(resource map[string]any) map[string]any {
copied := map[string]any{}
for k, v := range resource {
copied[k] = v
}
return copied
}
-161
View File
@@ -1,161 +0,0 @@
package catalogue
import (
"fmt"
"regexp"
"strings"
)
// What a module may call an event, and what a consumer may ask for.
//
// A module names an event **locally**: `order.placed`, not a subject and not a routing key
// (design 29 §1). A consumer names the emitter and the event: `billing.order.placed`. The mesh
// derives the subject from those, so reorganising the subject space leaves every manifest correct.
//
// **Nothing checked this until every manifest in the catalogue was wrong the same way**
// (novox/hq 04-ISSUES/127). All thirty-seven kept the old bus's routing key —
// `module.<module>.<verb>` — which the derivation read as "a module called `module`", so every
// cross-module subscription in the mesh pointed at a namespace nobody publishes to. Nothing failed:
// the services started and none of them reacted. The documentation on these fields taught the old
// form too, which is why the drift was uniform rather than scattered.
// eventName is one name in a local event: lower-case, and no wildcard.
var eventName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`)
// The wildcards a consumer may use, spelled the mesh's way and derived to whatever the transport
// spells them as.
//
// **A manifest holds no transport token**, which is the whole point of naming locally: the bus the
// mesh runs on today spells these `*` and `#`, and the one being built spells them `*` and `>`. A
// manifest that said either would be a manifest that stopped being true when the wire changed.
const (
// OneName stands for exactly one name.
OneName = "*"
// TheRest stands for one or more names, and may only come last.
TheRest = "**"
)
// EventProblems is what is wrong with a manifest's events.
//
// Refused at registration, because the alternative is a module that installs, starts, connects and
// reacts to nothing — and every log line says it is fine.
func EventProblems(m Manifest) []string {
var problems []string
for _, e := range m.Emits {
if was, stale := staleEventForm(e, m.Module); stale {
problems = append(problems, fmt.Sprintf(
"%s emits %q, which is the old bus's routing key. An event is named locally now, so "+
"write %q — the mesh derives the subject (novox/hq design 29 §1)",
m.Module, was, strings.TrimPrefix(was, "module."+m.Module+".")))
continue
}
if strings.HasPrefix(e, "module.") {
problems = append(problems, fmt.Sprintf(
"%s emits %q: `module.` is reserved, because it is how the old bus spelled a "+
"routing key and an event named that way derives into a namespace nobody owns",
m.Module, e))
continue
}
if err := localName(e); err != nil {
problems = append(problems, fmt.Sprintf("%s emits %q: %v", m.Module, e, err))
continue
}
// **Its own name, never another's.** The bus enforces that a namespace belongs to the module
// it is named for, so an event named for somebody else cannot be published at all. If the
// event is about a role rather than about this module, it belongs on the seat: a name that
// is stable across whoever fills it (04-ISSUES/127).
if first, _, split := strings.Cut(e, "."); split && isAModuleNameOtherThan(first, m.Module) {
problems = append(problems, fmt.Sprintf(
"%s emits %q, which reads as another module's event. A module publishes under its "+
"own name only. If this is about a role rather than about %s, declare it on that "+
"seat, where the name survives the holder changing",
m.Module, e, m.Module))
}
}
for _, c := range m.Consumes {
if strings.HasPrefix(c, "module.") {
problems = append(problems, fmt.Sprintf(
"%s consumes %q, which is the old bus's pattern. A consumed event names its emitter "+
"and the event: write %q", m.Module, c, strings.TrimPrefix(c, "module.")))
continue
}
if c == "#" {
problems = append(problems, fmt.Sprintf(
"%s consumes %q, which is the old bus's wildcard for everything. Write %q",
m.Module, c, TheRest))
continue
}
if err := consumePattern(c); err != nil {
problems = append(problems, fmt.Sprintf("%s consumes %q: %v", m.Module, c, err))
}
}
return problems
}
// staleEventForm says an emitted name is this module's own old routing key, and what it was.
func staleEventForm(event, module string) (string, bool) {
return event, module != "" && strings.HasPrefix(event, "module."+module+".")
}
// isAModuleNameOtherThan says a first token names some module of this mesh that is not this one.
//
// Only the mesh's own seats and the catalogue could answer this properly, and neither is reachable
// from a parser given one manifest. So this catches the case that actually happened — a name that
// is a *provision* the mesh defines, which is where "another module's event" comes from in practice
// — and the whole-catalogue check catches the rest.
func isAModuleNameOtherThan(first, module string) bool {
if first == module || first == "" {
return false
}
if _, isASeat := SeatNamed(first); isASeat {
return true
}
if _, isASeat := SeatDelivering(first); isASeat {
return true
}
return false
}
// localName checks one event name: dot-separated names, no wildcards, nothing else.
func localName(event string) error {
if event == "" {
return fmt.Errorf("an event needs a name")
}
for _, part := range strings.Split(event, ".") {
if part == OneName || part == TheRest {
return fmt.Errorf("an emitted event names one event, so it carries no wildcard")
}
if !eventName.MatchString(part) {
return fmt.Errorf("%q is not a usable name: lower-case letters, digits and dashes", part)
}
}
return nil
}
// consumePattern checks a consumed pattern: the emitter, then the event, with wildcards.
func consumePattern(pattern string) error {
if pattern == "" {
return fmt.Errorf("a consumed event needs an emitter and an event")
}
parts := strings.Split(pattern, ".")
for i, part := range parts {
switch {
case part == TheRest:
if i != len(parts)-1 {
return fmt.Errorf("%q stands for the rest of a name, so nothing may follow it", TheRest)
}
case part == OneName:
case !eventName.MatchString(part):
return fmt.Errorf("%q is not a usable name: lower-case letters, digits and dashes", part)
}
}
// `**` alone is every event from every module, which the audit logger wants and says plainly.
if len(parts) == 1 && parts[0] != TheRest {
return fmt.Errorf(
"%q names an emitter and no event. Write <emitter>.<event>, or %q for every event",
pattern, TheRest)
}
return nil
}
+163
View File
@@ -0,0 +1,163 @@
package catalogue
import (
"fmt"
"sort"
"strings"
)
// What only the mesh knows, written where a module asks for it.
//
// **The graph is the control plane's; using it is the module's.** The mesh knows which machines
// exist, what they are called, and where they are. Turning that into a name that resolves is
// somebody's software, and which software is a choice the mesh should not be making.
//
// This replaced three modules — names, a resolver's data, and the private network's own
// configuration — that existed only because computed output needed somewhere to live. They ran no
// software and could not be swapped for anything, which is the test of whether something is a
// module at all (novox/hq ADR 0040).
const (
// FactNodeNames is every machine's name and address, as a hosts file.
//
// Exact names only: `homer` and `homer.internal` resolve to homer. Anything *under* a machine
// is a wildcard, which a hosts file cannot express — that is FactNodeZones.
FactNodeNames = "node-names"
// FactNodeZones is every machine as a wildcard: `*.homer.internal` is homer.
//
// Written in the form a resolver reads. A machine's own name and everything under it are one
// fact — if homer is at an address, so is anything homer serves.
FactNodeZones = "node-zones"
)
// facts is every fact the mesh computes, and what writes it.
//
// **A closed list.** A module asking for a fact the mesh does not have is asking for a file nobody
// will write, and finding that out on a machine — as a daemon that starts, reads nothing, and
// answers no queries — is worse than being told where the manifest is.
var facts = map[string]func(Resolution, map[string]string, string) string{
FactNodeNames: nodeNames,
FactNodeZones: nodeZones,
}
// FactsInto renders the facts a module asked for, as files it will be given.
//
// The module owns everything after the file exists: loading it, restarting on it, what a resolver
// does with it. This only puts it there.
func FactsInto(m Manifest, r Resolution, addresses map[string]string, suffix string) ([]map[string]any, error) {
if len(m.Facts) == 0 {
return nil, nil
}
names := make([]string, 0, len(m.Facts))
for name := range m.Facts {
names = append(names, name)
}
sort.Strings(names)
out := make([]map[string]any, 0, len(names))
for _, name := range names {
write, known := facts[name]
if !known {
return nil, fmt.Errorf(
"%s asks the mesh for %q, which it does not compute. It has %s",
m.Module, name, spokenFacts())
}
path := m.Facts[name]
if !strings.HasPrefix(path, "/") {
return nil, fmt.Errorf(
"%s asks for %q at %q, which is not an absolute path", m.Module, name, path)
}
out = append(out, map[string]any{
"id": "fact-" + name, "type": "file", "path": path, "mode": "0644",
"content": write(r, addresses, suffix),
})
}
return out, nil
}
// spokenFacts lists them, so a refusal says what would have worked.
func spokenFacts() string {
names := make([]string, 0, len(facts))
for name := range facts {
names = append(names, name)
}
sort.Strings(names)
return strings.Join(names, ", ")
}
// nodeNames is every machine's name and address, as a hosts file.
//
// **A machine with no address is left out.** The mesh has a record for it — somebody added it —
// and does not yet know where it is, which is the ordinary state between adding a machine and it
// joining. Writing the name anyway would give a name that resolves to nothing, and a connection to
// that hangs; leaving it out fails at once and says the name is unknown.
func nodeNames(r Resolution, addresses map[string]string, suffix string) string {
var b strings.Builder
b.WriteString("# Generated by the mesh. Do not edit — this file is replaced whenever a machine\n")
b.WriteString("# joins or leaves, and an edit would survive until then and vanish.\n\n")
// The floor every Linux expects, and which removing would break things that have nothing to do
// with the mesh.
b.WriteString("127.0.0.1\tlocalhost\n")
b.WriteString("::1\t\tlocalhost ip6-localhost ip6-loopback\n")
if r.Node != "" {
fmt.Fprintf(&b, "127.0.1.1\t%s\n", r.Node)
}
b.WriteString("\n")
for _, name := range sortedNames(addresses) {
at := addresses[name]
internal, bare := meshName(name, suffix)
// Its mesh name resolves to its address on the private network rather than to loopback,
// so a service binding the name it was given stays reachable from everywhere else.
fmt.Fprintf(&b, "%s\t%s\t%s", at, internal, bare)
if bare == r.Node {
b.WriteString("\t# this machine")
}
b.WriteString("\n")
}
return b.String()
}
// nodeZones is every machine as a wildcard, in the form a resolver reads.
//
// `*.homer.internal` is homer, which is the whole rule: if homer is at an address, so is anything
// homer serves. A module wanting this runs the resolver; the mesh only says what is true.
func nodeZones(_ Resolution, addresses map[string]string, suffix string) string {
var b strings.Builder
b.WriteString("# Generated by the mesh. Do not edit — this file is replaced whenever a machine\n")
b.WriteString("# joins or leaves, and an edit would survive until then and vanish.\n\n")
for _, name := range sortedNames(addresses) {
internal, _ := meshName(name, suffix)
fmt.Fprintf(&b, "address=/%s/%s\n", internal, addresses[name])
}
return b.String()
}
// meshName is a machine's internal name and its bare one, from either. The control plane keys
// the names it hands a resolution by the internal name (`homer.internal`), the same map a
// container gets as its hosts; a caller that keys by the bare name gets the same answer. The
// suffix is the one the control plane composed those names with, handed down rather than written
// here a second time — the alternative was `homer.internal.internal` on every machine.
func meshName(name, suffix string) (internal, bare string) {
if suffix == "" {
suffix = "internal"
}
dotted := "." + strings.TrimPrefix(suffix, ".")
if strings.HasSuffix(name, dotted) {
return name, strings.TrimSuffix(name, dotted)
}
return name + dotted, name
}
func sortedNames(addresses map[string]string) []string {
out := make([]string, 0, len(addresses))
for name, at := range addresses {
// See nodeNames: a machine the mesh cannot place is left out rather than named at nothing.
if at == "" {
continue
}
out = append(out, name)
}
sort.Strings(out)
return out
}
+135
View File
@@ -0,0 +1,135 @@
package catalogue
import (
"strings"
"testing"
)
// Keyed by the internal name, as the control plane hands them (issue 079).
var threeMachines = map[string]string{"homer.internal": "10.42.0.1", "marge.internal": "10.42.0.2", "bart.internal": ""}
// **`*.homer.internal` is homer. That is the whole rule.**
func TestEveryMachineIsAWildcardUnderItsOwnName(t *testing.T) {
out := nodeZones(Resolution{Node: "homer"}, threeMachines, "")
for _, want := range []string{
"address=/homer.internal/10.42.0.1",
"address=/marge.internal/10.42.0.2",
} {
if !strings.Contains(out, want) {
t.Fatalf("missing %q:\n%s", want, out)
}
}
}
// A machine the mesh has a record for and cannot place is left out of both.
//
// **Not an oversight — the alternative is worse.** A name written with no address resolves to
// nothing, and a connection to that hangs. Leaving it out fails at once and says the name is
// unknown, which is a thing somebody can act on.
func TestAMachineWithNoAddressIsNotNamed(t *testing.T) {
for _, out := range []string{
nodeNames(Resolution{Node: "homer"}, threeMachines, ""),
nodeZones(Resolution{Node: "homer"}, threeMachines, ""),
} {
if strings.Contains(out, "bart") {
t.Fatalf("a machine with no address was named, so its name resolves to nothing:\n%s", out)
}
}
}
// A machine's own mesh name points at its address on the private network, not at loopback — or a
// service binding the name it was given is unreachable from everywhere else.
func TestAMachinesOwnNameIsItsMeshAddress(t *testing.T) {
out := nodeNames(Resolution{Node: "homer"}, threeMachines, "")
var line string
for _, l := range strings.Split(out, "\n") {
if strings.Contains(l, "homer.internal") {
line = l
}
}
if !strings.HasPrefix(line, "10.42.0.1") {
t.Fatalf("a machine's own mesh name is not its mesh address: %q", line)
}
// And the loopback floor is still there, or things with nothing to do with the mesh break.
if !strings.Contains(out, "127.0.0.1\tlocalhost") {
t.Fatalf("the loopback floor was removed:\n%s", out)
}
}
// A module says where it wants a fact, and is given a file.
func TestAModuleIsGivenTheFactsItAskedFor(t *testing.T) {
m := Manifest{Module: "dnsmasq", Facts: map[string]string{FactNodeZones: "/etc/mesh/zones.conf"}}
given, err := FactsInto(m, Resolution{Node: "homer"}, threeMachines, "")
if err != nil {
t.Fatal(err)
}
if len(given) != 1 {
t.Fatalf("expected one file, got %d", len(given))
}
if given[0]["path"] != "/etc/mesh/zones.conf" || given[0]["type"] != "file" {
t.Fatalf("not written where it was asked for: %v", given[0])
}
if !strings.Contains(given[0]["content"].(string), "homer.internal") {
t.Fatalf("the file does not hold the fact: %v", given[0]["content"])
}
}
// **Asking for a fact the mesh does not have is refused here, not on a machine.** A daemon that
// starts, reads a file nobody wrote, and answers no queries is a much worse way to find out.
func TestAskingForAFactTheMeshDoesNotHaveIsRefused(t *testing.T) {
m := Manifest{Module: "dnsmasq", Facts: map[string]string{"the-weather": "/etc/weather"}}
_, err := FactsInto(m, Resolution{}, nil, "")
if err == nil {
t.Fatal("a module asked for something nobody computes and was given nothing, silently")
}
for _, known := range []string{FactNodeNames, FactNodeZones} {
if !strings.Contains(err.Error(), known) {
t.Fatalf("the refusal does not say what would have worked: %v", err)
}
}
}
// And a relative path is refused, or a module decides where the mesh writes on a machine.
func TestAFactMustBeAskedForAtAnAbsolutePath(t *testing.T) {
m := Manifest{Module: "dnsmasq", Facts: map[string]string{FactNodeNames: "etc/hosts"}}
if _, err := FactsInto(m, Resolution{}, nil, ""); err == nil {
t.Fatal("a relative path was accepted")
}
}
// **The names the control plane hands a resolution are already internal names** — `homer.internal`,
// the same map every container gets as its hosts. Appending the suffix again wrote
// `homer.internal.internal` into every hosts file and every resolver's zones, and the large mesh
// bed's name test was the first to read it back. Either key gives the same files.
func TestNamesKeyedByInternalNameAreNotSuffixedTwice(t *testing.T) {
internal := map[string]string{"homer.internal": "10.42.0.1", "marge.internal": "10.42.0.2"}
bare := map[string]string{"homer": "10.42.0.1", "marge": "10.42.0.2"}
if a, b := nodeZones(Resolution{Node: "homer"}, internal, ""), nodeZones(Resolution{Node: "homer"}, bare, ""); a != b {
t.Fatalf("the zones differ by how the names were keyed:\n%s\n---\n%s", a, b)
}
if a, b := nodeNames(Resolution{Node: "homer"}, internal, ""), nodeNames(Resolution{Node: "homer"}, bare, ""); a != b {
t.Fatalf("the hosts differ by how the names were keyed:\n%s\n---\n%s", a, b)
}
zones := nodeZones(Resolution{Node: "homer"}, internal, "")
if strings.Contains(zones, "internal.internal") || !strings.Contains(zones, "address=/homer.internal/10.42.0.1") {
t.Fatalf("the zones carry a doubled suffix or miss the name:\n%s", zones)
}
hosts := nodeNames(Resolution{Node: "homer"}, internal, "")
if !strings.Contains(hosts, "10.42.0.1\thomer.internal\thomer\t# this machine") {
t.Fatalf("the hosts line for the machine itself is not name, bare name and the mark:\n%s", hosts)
}
}
// The suffix the control plane composed the names with is the one the facts write — an operator
// who chose another does not get `.internal` appended to it.
func TestTheFactsWriteTheSuffixTheNamesWereComposedWith(t *testing.T) {
names := map[string]string{"homer.lan": "10.42.0.1"}
zones := nodeZones(Resolution{Node: "homer"}, names, "lan")
if !strings.Contains(zones, "address=/homer.lan/10.42.0.1") || strings.Contains(zones, "internal") {
t.Fatalf("the zones do not carry the operator's suffix as given:\n%s", zones)
}
hosts := nodeNames(Resolution{Node: "homer"}, names, "lan")
if !strings.Contains(hosts, "10.42.0.1\thomer.lan\thomer\t# this machine") {
t.Fatalf("the hosts line does not carry the operator's suffix as given:\n%s", hosts)
}
}
+104 -104
View File
@@ -1,6 +1,7 @@
package catalogue
import (
"encoding/json"
"fmt"
"os"
"reflect"
@@ -28,10 +29,8 @@ func TestTheStoreAndTheBrokerSayWhatTheMeshGuards(t *testing.T) {
if got := catalogueManifest(t, "postgres").Guards; !reflect.DeepEqual(got, []int{5432}) {
t.Errorf("postgres guards %v; the store's port must be refused from outside", got)
}
// The bus's monitoring port, not its client port: a node reaches the bus, nobody outside
// reads its state (novox/hq ADR 0131 — the broker that guarded 15672 has left the catalogue).
if got := catalogueManifest(t, "nats").Guards; !reflect.DeepEqual(got, []int{8222}) {
t.Errorf("nats guards %v; the monitoring port must be refused from outside", got)
if got := catalogueManifest(t, "lavinmq").Guards; !reflect.DeepEqual(got, []int{15672}) {
t.Errorf("lavinmq guards %v; the management port must be refused from outside", got)
}
}
@@ -86,8 +85,9 @@ func TestTheFilterModuleNeverFlushesTheRuleset(t *testing.T) {
}
// The package registry's port is the node's, like every other foundation port (novox/hq
// 04-ISSUES/085, ADR 0100). The forge is reached through what it says it serves, and consumers —
// the builder among them — are told that, rather than carrying a number of their own.
// 04-ISSUES/085, ADR 0100). Two halves, because the forge is reached two ways: through what the
// module that serves it says it serves, and — for the genesis window, before any module provides
// `package-registry` at all — through the one binding the builder carries instead of resolving.
func TestTheForgesPortIsGivenLikeAnyOtherProvidersPort(t *testing.T) {
forge := catalogueManifest(t, "gitea")
@@ -104,66 +104,96 @@ func TestTheForgesPortIsGivenLikeAnyOtherProvidersPort(t *testing.T) {
// And every consumer of the package registry is told where the machine actually put it,
// because that is read from what the forge serves rather than written in the consumer.
if got := ServedOn(forge, "npm-package-registry", given)["port"]; got != 3100 {
if got := ServedOn(forge, "package-registry", given)["port"]; got != 3100 {
t.Errorf("the package registry is served on %v, not the port this node gave it", got)
}
if got := ServedOn(forge, "npm-package-registry", nil)["port"]; got != float64(3000) {
if got := ServedOn(forge, "package-registry", nil)["port"]; got != float64(3000) {
t.Errorf("without a setting the forge serves %v, not the catalogue's port", got)
}
// And so is where a repository on it is cloned from (novox/hq ADR 0111), for the same reason:
// a build composes the URL from what the forge serves, so a given port is a followed port.
if got := ServedOn(forge, "git", given)["port"]; got != 3100 {
t.Errorf("git is served on %v, not the port this node gave the forge", got)
}
}
// **The builder requires the registry the npm seat delivers, and carries no binding of its own.**
//
// It used to carry a hand-written binding because nothing provided a package registry to resolve
// one from at genesis. The catalogue now requires it like any consumer, and ADR 0110 makes the
// seat's holder the answer when more than one module provides it — so a carried copy would be a
// second answer to the same question, free to drift from the first. Asserted gone, not merely
// unused.
func TestTheBuilderRequiresTheRegistryTheNpmSeatDelivers(t *testing.T) {
// bindingIn is the package binding the builder carries, as the machine would receive it.
func bindingIn(t *testing.T, m Manifest, layers []Layer) map[string]any {
t.Helper()
for _, r := range m.Resources {
if fmt.Sprint(r["id"]) != "package-binding" {
continue
}
settled, err := ApplySettings(r, layers)
if err != nil {
t.Fatalf("the builder's package binding refused %v: %v", layers, err)
}
if settled["merge"] != nil || settled["protected"] != nil {
t.Fatal("the host would be sent fields it does not know")
}
var out map[string]any
if err := json.Unmarshal([]byte(fmt.Sprint(settled["content"])), &out); err != nil {
t.Fatalf("the builder's package binding is not a binding: %v", err)
}
return out
}
t.Fatal("the builder carries no package binding")
return nil
}
func TestTheBuildersCarriedPackageBindingTakesThePortFromTheNode(t *testing.T) {
builder := catalogueManifest(t, "builder")
seat, _ := SeatNamed("npm-package-registry")
var requires bool
for _, r := range builder.Requires {
requires = requires || r == seat.Delivers
// Nothing set: the catalogue's own number, which is what a mesh raised on the defaults uses.
serves := bindingIn(t, builder, nil)["serves"].(map[string]any)
if serves["port"] != float64(3000) {
t.Fatalf("the builder's binding defaults to %v", serves["port"])
}
if !requires {
t.Fatalf("the builder does not require %q: %v", seat.Delivers, builder.Requires)
// Given a port, the binding dials it — and the rest of what the forge serves survives, because
// a setting is merged into the module's own values rather than replacing them.
moved := bindingIn(t, builder, []Layer{{From: "anchor",
Values: map[string]any{"serves": map[string]any{"port": float64(3100)}}}})
got := moved["serves"].(map[string]any)
if got["port"] != float64(3100) {
t.Errorf("the builder dials %v, not the port this node gave the package registry", got["port"])
}
if builder.Binds[seat.Delivers] == "" {
t.Errorf("the builder is not told where the registry is: binds %v", builder.Binds)
if got["scheme"] != "http" || got["npm-path"] != "/api/packages/novox/npm/" {
t.Errorf("setting the port lost the rest of what the forge serves: %v", got)
}
for _, r := range builder.Resources {
if fmt.Sprint(r["id"]) == "package-binding" {
t.Fatal("the builder carries its own package binding beside the one the mesh resolves")
if moved["as"] != "mesh-builder" || moved["from"] != "gitea" {
t.Errorf("setting the port changed who the binding is with: %v", moved)
}
}
// The two halves are one number. The builder carries a binding because at genesis nothing provides
// `package-registry` to resolve one from; the day the forge is a module, the same consumer is told
// what the forge serves. They have to start from the same port, or a mesh raised on the defaults
// dials one number before the forge is assigned and another after.
func TestTheBuildersCarriedBindingStartsWhereTheForgeServes(t *testing.T) {
forge := ServedOn(catalogueManifest(t, "gitea"), "package-registry", nil)
carried := bindingIn(t, catalogueManifest(t, "builder"), nil)["serves"].(map[string]any)
for _, key := range []string{"port", "scheme", "npm-path"} {
if fmt.Sprint(forge[key]) != fmt.Sprint(carried[key]) {
t.Errorf("the forge serves %s %v and the builder's carried binding says %v — the two "+
"halves of the same registry have drifted apart in the catalogue",
key, forge[key], carried[key])
}
}
}
// The forge holds the seats it answers for (novox/hq ADR 0110, 0111), parsed by the real parser —
// which refuses a delivering seat claimed by a module that does not provide what it delivers.
func TestTheForgeHoldsTheNpmAndGitSeats(t *testing.T) {
forge := catalogueManifest(t, "gitea")
holds := map[string]bool{}
for _, c := range forge.Claims {
holds[c.Name] = true
}
for _, seat := range []string{"npm-package-registry", "git"} {
if !holds[seat] {
t.Errorf("gitea does not claim the %s seat: %+v", seat, forge.Claims)
func TestTheBuildersPackageBindingKeepsItsIdentity(t *testing.T) {
builder := catalogueManifest(t, "builder")
// `at` above all: a setting that moves it points the builder, and the registry password it
// sends as basic auth, at a host somebody else chose.
for _, key := range []string{"provision", "from", "at", "as"} {
var refused error
for _, r := range builder.Resources {
if fmt.Sprint(r["id"]) != "package-binding" {
continue
}
_, refused = ApplySettings(r, []Layer{{From: "anchor",
Values: map[string]any{key: "something else"}}})
}
if refused == nil {
t.Errorf("%q can be set on the builder's package binding, which is not a port but who "+
"the binding is with", key)
}
}
git := ServedOn(forge, "git", nil)
if git["scheme"] != "http" || git["port"] != float64(3000) {
t.Errorf("gitea serves nothing a clone URL can be composed from: %v", git)
}
npm := ServedOn(forge, "npm-package-registry", nil)
if npm["npm-path"] != "/api/packages/novox/npm/" {
t.Errorf("gitea no longer says where its npm registry is: %v", npm)
}
}
@@ -221,13 +251,27 @@ func TestTheForgesOwnAddressFollowsThePortTheNodeGaveIt(t *testing.T) {
}
}
// gitea's own sshd is unmodified — the module's own internal port is 22, the number in
// `listens`, the same convention every other module in the catalogue uses (its internal port,
// not an invented identity). Composed from the manifest in the catalogue beside this checkout,
// because what the mesh publishes is a fact about what the module actually writes.
func declaredGiteaSsh(t *testing.T, given map[int]int) map[string]any {
t.Helper()
// **And the port the forge publishes the long way is the node's too** (novox/hq ADR 0100).
//
// The forge's ssh port is written `2222:22` — the machine's own daemon holds 22, so the module
// takes 2222 and says so in `listens`. A node whose predecessor served git on another number
// cannot be told to leave it there unless the setting may name the machine side of that mapping,
// which is the number the manifest itself uses everywhere else. Composed from the manifest in the
// catalogue beside this checkout, because what the mesh can move is a fact about what the module
// actually writes.
func TestTheForgesSshPortIsGivenByTheNumberTheForgeCallsIt(t *testing.T) {
forge := catalogueManifest(t, "gitea")
given, err := GivenPorts(forge, []Layer{{From: "anchor",
Values: map[string]any{PortsSetting: map[string]any{"2222": float64(222)}}}})
if err != nil {
t.Fatalf("the forge's ssh port cannot be given on a node: %v", err)
}
// Under the number the module listens on — 2222, the machine side of its mapping — which is
// the number the plan, the filter, the openings and the consumer all ask for. One entry.
if want := map[int]int{2222: 222}; !reflect.DeepEqual(given, want) {
t.Fatalf("the forge was given %v, and it names its ssh port %v", given, want)
}
resolved, err := forge.Resolve([]Built{{
Name: "runtime", Kind: ArtifactImage,
Reference: "registry.example/gitea-runtime@sha256:" + strings.Repeat("a", 64),
@@ -242,13 +286,9 @@ func declaredGiteaSsh(t *testing.T, given map[int]int) map[string]any {
{Name: "secret", For: "gitea", From: "anchor", Local: "internal-token", Sealed: "sealed-token"},
{Name: "secret", For: "gitea", From: "anchor", Local: "admin", Sealed: "sealed-admin"},
}}
givenPorts := map[int]int{3000: 3000}
for k, v := range given {
givenPorts[k] = v
}
out, err := r.Declaration(Rendering{
Needed: map[string]map[string]string{"gitea": {"broker": "sealed-broker"}},
Ports: map[string]map[int]int{"gitea": givenPorts},
Ports: map[string]map[int]int{"gitea": {3000: 3000, 2222: 222}},
Given: map[string]map[int]int{"gitea": given},
})
if err != nil {
@@ -258,48 +298,8 @@ func declaredGiteaSsh(t *testing.T, given map[int]int) map[string]any {
if server == nil {
t.Fatalf("the forge's own container is not in the declaration: %v", out)
}
return server
}
// **The forge publishes ssh at the mesh's own fixed convention by default** (novox/hq ADR 0100).
//
// `222` is the mesh's own public convention for the forge's ssh, written directly in the
// manifest's `ports` — every node the forge has run on used the same number, so it needs no
// per-node setting to reach it.
func TestTheForgesSshPortIsTheMeshsFixedConventionByDefault(t *testing.T) {
forge := catalogueManifest(t, "gitea")
// Nothing was given — no node moved this port — which is the ordinary answer: the mesh only
// reports what a setting moved, and the manifest's own `222:22` needs no move to be reached.
given, err := GivenPorts(forge, nil)
if err != nil {
t.Fatalf("the forge's ssh port cannot be given on a node: %v", err)
}
if len(given) != 0 {
t.Fatalf("nothing moved the forge's ssh port, yet it was given %v", given)
}
server := declaredGiteaSsh(t, given)
if published := fmt.Sprint(server["ports"]); !strings.Contains(published, "222:22") {
t.Fatalf("the forge is published on %v, not its own fixed convention", server["ports"])
}
}
// **A node whose predecessor served git on a different number can still be told to leave it
// there.** The setting names the port the module itself listens on — 22, gitea's own sshd, the
// same number `listens` uses — not the mesh's own default machine-side number, so moving it does
// not require guessing what the manifest happens to default to.
func TestANodeMayGiveTheForgesSshPortADifferentNumber(t *testing.T) {
forge := catalogueManifest(t, "gitea")
given, err := GivenPorts(forge, []Layer{{From: "anchor",
Values: map[string]any{PortsSetting: map[string]any{"22": float64(9022)}}}})
if err != nil {
t.Fatalf("the forge's ssh port cannot be moved on a node: %v", err)
}
if want := map[int]int{22: 9022}; !reflect.DeepEqual(given, want) {
t.Fatalf("the forge was given %v, and the setting named %v", given, want)
}
server := declaredGiteaSsh(t, given)
if published := fmt.Sprint(server["ports"]); !strings.Contains(published, "9022:22") ||
strings.Contains(published, "222:22") {
if published := fmt.Sprint(server["ports"]); !strings.Contains(published, "222:22") ||
strings.Contains(published, "2222:22") {
t.Fatalf("the forge is published on %v, not the port this node gave it", server["ports"])
}
}
-160
View File
@@ -1,160 +0,0 @@
package catalogue
import (
"strings"
"testing"
)
// **A seat's holder on record settles who holds it, and lets the next holder stand beside the
// current one** (novox/hq ADR 0131, design 28 task 5.3). Until the record existed, two assignments
// whose modules both claimed a seat were refused outright — which left no way to hand a seat over
// without a moment where nobody held it, and the control plane finds its own bus through one of
// these seats. That moment was the outage of 2026-09-27.
func busSeatDelivering(t *testing.T, delivers string) {
t.Helper()
was := Seats()
t.Cleanup(func() { UseSeats(was) })
UseSeats([]Seat{{Name: "mesh-broker", Scope: ScopeMesh, Delivers: delivers, Decision: "test"}})
}
func oldBroker() Manifest {
return Manifest{Module: "old-broker", Provides: []Offer{{Name: "mesh-bus", Scope: ScopeMesh}},
Claims: []Claim{{Name: "mesh-broker", Scope: ScopeMesh}}}
}
func newBroker() Manifest {
return Manifest{Module: "new-broker", Provides: []Offer{{Name: "mesh-bus", Scope: ScopeMesh}},
Claims: []Claim{{Name: "mesh-broker", Scope: ScopeMesh}}}
}
// Nothing on record: exactly the old rule. One claimant holds; two are refused.
func TestWithNoHolderOnRecordTheSoleClaimantHoldsAndTwoAreRefused(t *testing.T) {
busSeatDelivering(t, "mesh-bus")
node := Node{Name: "anchor"}
held, problems := checkClaims([]Manifest{oldBroker()}, node, nil, nil)
if len(problems) != 0 || len(held) != 1 || held[0].Module != "old-broker" {
t.Fatalf("a sole claimant did not hold the seat: held=%v problems=%v", held, problems)
}
_, problems = checkClaims([]Manifest{oldBroker(), newBroker()}, node, nil, nil)
if len(problems) != 1 || !strings.Contains(problems[0], "both claim") {
t.Fatalf("two claimants with nothing on record were not refused: %v", problems)
}
}
// With a holder on record, the other eligible assignment is silent: not refused, and not holding.
func TestTheHolderOnRecordHoldsAndTheOtherClaimantStandsBesideIt(t *testing.T) {
busSeatDelivering(t, "mesh-bus")
node := Node{Name: "anchor"}
record := []Held{{Claim: "mesh-broker", Scope: ScopeMesh, Node: "anchor", Module: "new-broker"}}
held, problems := checkClaims([]Manifest{oldBroker(), newBroker()}, node, nil, record)
if len(problems) != 0 {
t.Fatalf("the assignment beside the holder was refused: %v", problems)
}
if len(held) != 1 || held[0].Module != "new-broker" {
t.Fatalf("the holder on record is not the one holding: %v", held)
}
}
// The record names a node too: an eligible module on another machine holds nothing, and its
// machine's set still resolves.
func TestAHolderOnRecordElsewhereLeavesThisMachinesClaimantSilent(t *testing.T) {
busSeatDelivering(t, "mesh-bus")
record := []Held{{Claim: "mesh-broker", Scope: ScopeMesh, Node: "anchor", Module: "new-broker"}}
held, problems := checkClaims([]Manifest{oldBroker()}, Node{Name: "laptop"}, nil, record)
if len(problems) != 0 || len(held) != 0 {
t.Fatalf("a claimant elsewhere than the recorded holder was not simply silent: held=%v problems=%v",
held, problems)
}
}
// A record naming a seat's former name still applies to it after a rename (ADR 0122).
func TestAHolderRecordedUnderAFormerNameStillHolds(t *testing.T) {
busSeatDelivering(t, "mesh-bus")
wasAliases := aliases
t.Cleanup(func() { UseAliases(wasAliases) })
UseAliases(map[string]string{"the-broker": "mesh-broker"})
record := []Held{{Claim: "the-broker", Scope: ScopeMesh, Node: "anchor", Module: "new-broker"}}
held, problems := checkClaims([]Manifest{oldBroker(), newBroker()}, Node{Name: "anchor"}, nil, record)
if len(problems) != 0 || len(held) != 1 || held[0].Module != "new-broker" {
t.Fatalf("a record under the former name did not settle the seat: held=%v problems=%v", held, problems)
}
}
// CanHold is the one judgement registration and the handover share, against the store's row.
func TestCanHoldJudgesClaimScopeAndWhatTheSeatDelivers(t *testing.T) {
busSeatDelivering(t, "mesh-bus")
seat, _ := SeatNamed("mesh-broker")
if err := CanHold(newBroker(), seat); err != nil {
t.Fatalf("a module that claims the seat and provides what it delivers was refused: %v", err)
}
noClaim := Manifest{Module: "quiet", Provides: []Offer{{Name: "mesh-bus", Scope: ScopeMesh}}}
if err := CanHold(noClaim, seat); err == nil || !strings.Contains(err.Error(), "does not claim") {
t.Fatalf("a module that never claimed the seat was allowed to hold it: %v", err)
}
wrongScope := newBroker()
wrongScope.Claims[0].Scope = ScopeNode
if err := CanHold(wrongScope, seat); err == nil || !strings.Contains(err.Error(), "scope") {
t.Fatalf("a claim at the wrong scope was allowed: %v", err)
}
cannotAnswer := Manifest{Module: "amqp-only", Provides: []Offer{{Name: "amqp", Scope: ScopeMesh}},
Claims: []Claim{{Name: "mesh-broker", Scope: ScopeMesh}}}
if err := CanHold(cannotAnswer, seat); err == nil || !strings.Contains(err.Error(), `does not provide "mesh-bus"`) {
t.Fatalf("a holder that cannot answer for the seat was allowed: %v", err)
}
// And the judgement follows the store's row, not a compiled copy.
busSeatDelivering(t, "amqp")
seat, _ = SeatNamed("mesh-broker")
if err := CanHold(cannotAnswer, seat); err != nil {
t.Fatalf("with the row saying amqp, an amqp provider was refused: %v", err)
}
}
// A machine being moved is handed its membership for the new bus as a sealed file in its own
// declaration — the machine's, not any module's (design 28, task 5.2).
func TestAMembershipForTheNewBusIsComposedAsASealedFile(t *testing.T) {
r := Resolution{Node: "anchor"}
got, err := r.Compose(Rendering{BusMembership: "sealed-blob"})
if err != nil {
t.Fatal(err)
}
var found map[string]any
for _, res := range got.Resources {
if res["id"] == BusMembershipID() {
found = res
}
}
if found == nil {
t.Fatalf("no membership resource in %v", got.Resources)
}
if found["path"] != BusMembershipPath || found["sealed"] != "sealed-blob" || found["mode"] != "0600" {
t.Fatalf("the membership is not a sealed 0600 file where the host reads it: %v", found)
}
// And a machine not being moved is handed nothing.
got, _ = r.Compose(Rendering{})
for _, res := range got.Resources {
if res["id"] == BusMembershipID() {
t.Fatal("a machine with no membership on record was handed one")
}
}
}
// The store's seat rows have no protocol columns yet; loading them must not drop the protocol the
// bus is derived from, or no role's work queue is ever raised (found live, 2026-09-28).
func TestAStoreRowWithoutAProtocolKeepsTheCompiledOne(t *testing.T) {
was := Seats()
t.Cleanup(func() { UseSeats(was) })
UseSeats([]Seat{{Name: "mesh-build-machine", Scope: ScopeMesh, Decision: "row"}})
got, ok := SeatNamed("mesh-build-machine")
if !ok || len(got.Accepts) == 0 {
t.Fatalf("the build machine's seat lost what it accepts when loaded from the store: %+v", got)
}
if got.Decision != "row" {
t.Fatalf("the store's own columns were not kept: %+v", got)
}
}
-106
View File
@@ -1,106 +0,0 @@
package catalogue_test
import (
"reflect"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/overlay"
)
// novox/hq issue 128, held where the host will see it: the shipped networking module's names,
// through the whole composition, and not only through FactsInto.
//
// Composition prefixes a fact's id with the module that asked for it and passes everything else
// through; a step that dropped `into` on the way would send the region as a whole file, and the
// host would write the machine's hosts file over again with every unit test above still green.
// onTheNetwork stands in for the overlay's generator: the node is part of the private network,
// and what the generator writes is not what is under test here.
type onTheNetwork struct{}
func (onTheNetwork) Resources(string) ([]map[string]any, bool, error) {
return []map[string]any{{"id": "overlay-config", "type": "file",
"path": "/etc/wireguard/mesh0.conf", "mode": "0600", "content": "[Interface]\n"}}, true, nil
}
func TestTheHostsRegionArrivesAsTheHostWillReadIt(t *testing.T) {
shelf := provided(t)
// A resolver restarting on the names another module put on the machine, and one resource it
// only runs at start — neither of which composition has any business changing.
resolver, err := catalogue.ParseManifest([]byte(`{
"module": "resolver", "version": "1", "requires": ["mesh-addressing"],
"resources": [
{"id": "seed", "type": "file", "path": "/etc/resolver/seed", "mode": "0644",
"content": "seed\n", "at": "start"},
{"id": "daemon", "type": "service", "unit": "resolver.service", "state": "running",
"restart-on": ["seed", "mesh-wireguard.fact-node-names"]}
]}`))
if err != nil {
t.Fatal(err)
}
shelf[resolver.Module] = resolver
got, err := catalogue.Resolve(shelf, []string{overlay.Domain, "resolver"},
catalogue.Node{Name: "homer", At: "homer.internal"}, catalogue.World{})
if err != nil {
t.Fatal(err)
}
names := map[string]string{"homer.internal": "10.42.0.1", "marge.internal": "10.42.0.2"}
out, err := got.Declaration(catalogue.Rendering{
Names: names, Machines: names, Suffix: "internal",
Generators: map[string]catalogue.Generator{overlay.Name: onTheNetwork{}},
})
if err != nil {
t.Fatal(err)
}
ids := map[string]map[string]any{}
for _, r := range out {
ids[r["id"].(string)] = r
}
hosts := ids[overlay.Name+".fact-node-names"]
if hosts == nil {
t.Fatalf("no names reached the machine; the declaration has %v", keys(ids))
}
if hosts["path"] != "/etc/hosts" || hosts["into"] != "block" {
t.Fatalf("the hosts file is not written into as a region: %v", hosts)
}
content := hosts["content"].(string)
if !strings.Contains(content, "10.42.0.1\thomer.internal\thomer\t# this machine\n") {
t.Errorf("the region does not name the machine:\n%s", content)
}
for _, floor := range []string{"Generated by the mesh", "localhost", "127.0.1.1"} {
if strings.Contains(content, floor) {
t.Errorf("the region carries %q, which is the machine's:\n%s", floor, content)
}
}
// The resolver's reference to it still names a resource the host will be sent.
daemon := ids["resolver.daemon"]
if daemon == nil {
t.Fatalf("the resolver's service was not composed: %v", keys(ids))
}
for _, named := range daemon["restart-on"].([]any) {
if ids[named.(string)] == nil {
t.Errorf("the resolver restarts on %v, which is nothing the host is sent", named)
}
}
if !reflect.DeepEqual(daemon["restart-on"], []any{"resolver.seed", overlay.Name + ".fact-node-names"}) {
t.Errorf("restart-on is %v", daemon["restart-on"])
}
// And a resource's own `at` passes through as the manifest wrote it.
if seed := ids["resolver.seed"]; seed == nil || seed["at"] != "start" {
t.Errorf("a resource's at did not survive composition: %v", seed)
}
}
func keys(m map[string]map[string]any) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
return out
}
-62
View File
@@ -1,62 +0,0 @@
package catalogue
import (
"fmt"
"sort"
"strings"
)
// A node's fail2ban jails, composed from the modules it runs (novox/hq to-be 31).
//
// **The same shape as the firewall.** Every module's `listens` become the node's rule set; every
// module's `jails` become the node's fail2ban config. A module that runs an authenticating service
// declares what a break-in on it looks like and how to ban it, naming no node and no path (ADR
// 0112); the intrusion-prevention holder — the one module with `jailing` — gathers them and writes
// them where it owns. A node not running a module has none of its jails.
// jailsInto composes every jail declared by the modules on a node into the files the holder writes:
// one jail file (all stanzas, so the fail2ban service restarts on a single resource) and one filter
// file per jail (its failregex, which fail2ban references by the jail's name).
//
// Owned by the holder, because the directory is: two modules writing into one fail2ban is the
// collision the holder model exists to prevent. Empty when nothing declares a jail — then the file
// is written empty rather than absent, so removing the last jail is an ordinary change the service
// restarts on rather than a file that vanishes.
func jailsInto(modules []Manifest, j *Jailing) []map[string]any {
type declared struct {
module string
jail Jail
}
var jails []declared
for _, m := range modules {
for _, jail := range m.Jails {
jails = append(jails, declared{m.Module, jail})
}
}
// A stable order the host applies as given (ADR 0005), and so the same set composes byte for
// byte every time rather than differing by map iteration.
sort.Slice(jails, func(a, b int) bool { return jails[a].jail.Name < jails[b].jail.Name })
var composed strings.Builder
composed.WriteString("# The mesh's jails, composed from the modules this node runs. Do not edit —\n")
composed.WriteString("# replaced whenever the node's modules change (novox/hq to-be 31).\n")
out := make([]map[string]any, 0, len(jails)+1)
for _, d := range jails {
fmt.Fprintf(&composed, "\n# from %s\n[%s]\nenabled = true\nfilter = %s\n%s\n",
d.module, d.jail.Name, d.jail.Name, strings.TrimRight(d.jail.Jail, "\n"))
// The filter is a file of its own, named as the jail's filter= references it.
out = append(out, map[string]any{
"id": "filter-" + d.jail.Name,
"type": "file", "path": strings.TrimRight(j.FilterInto, "/") + "/" + d.jail.Name + ".conf",
"mode": "0644",
"content": "# Generated by the mesh (from module " + d.module + "). Do not edit.\n" +
"[Definition]\nfailregex = " + d.jail.Failregex + "\n",
})
}
// The one jail file, first, with the fixed id the fail2ban service names in its restart-on.
return append([]map[string]any{{
"id": ComposedJailsID(), "type": "file", "path": j.Into, "mode": "0644",
"content": composed.String(),
}}, out...)
}
-46
View File
@@ -1,46 +0,0 @@
package catalogue
import (
"strings"
"testing"
)
// A node's fail2ban jails are composed from the modules it runs (novox/hq to-be 31): the holder
// (jailing) gathers every module's declared jail into one jail file and a filter file per jail.
func TestJailsAreComposedFromTheNodesModules(t *testing.T) {
modules := []Manifest{
{Module: "fail2ban", Jailing: &Jailing{Into: "/etc/fail2ban/jail.d/mesh-composed.conf", FilterInto: "/etc/fail2ban/filter.d"}},
{Module: "postgres", Jails: []Jail{{Name: "postgres-auth", Failregex: "auth failed from <HOST>", Jail: "port = 5432\nmaxretry = 5"}}},
}
files := jailsInto(modules, modules[0].Jailing)
by := map[string]map[string]any{}
for _, f := range files {
by[f["id"].(string)] = f
}
jail := by[ComposedJailsID()]
if jail == nil || jail["path"] != "/etc/fail2ban/jail.d/mesh-composed.conf" {
t.Fatalf("the composed jail file was not written: %v", jail)
}
body := jail["content"].(string)
if !strings.Contains(body, "[postgres-auth]") || !strings.Contains(body, "filter = postgres-auth") ||
!strings.Contains(body, "port = 5432") {
t.Fatalf("the postgres jail stanza was not composed in:\n%s", body)
}
filter := by["filter-postgres-auth"]
if filter == nil || filter["path"] != "/etc/fail2ban/filter.d/postgres-auth.conf" {
t.Fatalf("the jail's filter file was not written: %v", filter)
}
if !strings.Contains(filter["content"].(string), "failregex = auth failed from <HOST>") {
t.Fatalf("the failregex was not written: %v", filter["content"])
}
}
// A holder whose node runs no jail-declaring module still gets the file, empty — so removing the
// last jail is a change the service restarts on, not a file that vanishes.
func TestTheComposedJailFileIsWrittenEvenWhenEmpty(t *testing.T) {
files := jailsInto([]Manifest{{Module: "fail2ban"}}, &Jailing{Into: "/x", FilterInto: "/f"})
if len(files) != 1 || files[0]["id"] != ComposedJailsID() {
t.Fatalf("the empty composed jail file was not written alone: %v", files)
}
}
+19 -57
View File
@@ -22,11 +22,8 @@ import (
// provides, it does not require. Written as a literal it would be a manifest carrying one
// deployment's machine name, which is the shape [ADR 0066] exists to remove.
//
// Three facts, all the mesh's own vocabulary — the same `node` and `at` a contribution already
// carries, and the address behind `at`, for software that takes an address and not a name. The
// case that found the third is a resolver pointing the container runtime at itself: the runtime's
// list of resolvers is addresses, because a name there would have to be resolved by the resolver
// it names. Nothing about what a machine is *for*: that would be the mesh learning what a module
// Two facts, both the mesh's own vocabulary — the same `node` and `at` a contribution already
// carries. Nothing about what a machine is *for*: that would be the mesh learning what a module
// means, which it does not do.
// ofMachine is where a module says a fact about the machine underneath it belongs:
@@ -52,72 +49,37 @@ func machineUsed(content string) []string {
// to be reached at an address that does not exist is a misconfiguration, and it is said here —
// where the module and the machine are both named — rather than discovered later as a certificate
// nobody can verify.
//
// `address` is what `at` resolves to, read from the names the control plane composed — the same map
// the hosts file and the resolver's wildcards are written from, so a file naming the machine's
// address and the file every other machine reaches it by cannot disagree. Absent, like `at`, when
// the machine is off the network or the mesh has not placed it.
func machineFacts(r Resolution, names map[string]string, meshRange string) map[string]string {
func machineFacts(r Resolution) map[string]string {
out := map[string]string{"name": r.Node}
if r.At != "" {
out["at"] = r.At
if address := names[r.At]; address != "" {
out["address"] = address
}
}
// The private network's whole range — a mesh-wide fact, not this machine's, but named here
// because a module cannot know it and sometimes must (an intrusion filter that must never ban a
// tunnel peer). Absent when the mesh has no range to give.
if meshRange != "" {
out["mesh-range"] = meshRange
}
// The operator's login on this machine and where its home is (novox/hq to-be 29), so a module
// that writes operator config names the account and its home rather than a value it cannot know.
// Absent when no operator account is known — a headless box a person never logs into.
if r.Account != "" {
out["account"] = r.Account
out["account-home"] = accountHomeOf(r.Account, r.AccountHome)
}
return out
}
// accountHomeOf is where an account's home is: what was stored, or the derived default — /root for
// root, /home/<account> otherwise. The one place the default is written, so a fact and the store
// cannot disagree about it.
func accountHomeOf(account, home string) string {
if home != "" {
return home
}
if account == "root" {
return "/root"
}
return "/home/" + account
}
// machineInto replaces a file's ${machine:…} placeholders with what the mesh knows about the
// machine the module was assigned to.
//
// A key the mesh does not hold is refused, for the same reason a binding's is: left alone, the
// literal would be written into a configuration file and read as a value.
func machineInto(resource map[string]any, facts map[string]string, module string) error {
// Content, and now the path and owner too: a module that writes into a person's home names it
// with ${machine:account-home} and ${machine:account}, which it cannot know until assigned
// (novox/hq to-be 29), the same reason its content names ${machine:address}.
for _, field := range []string{"path", "owner", "content"} {
s, ok := resource[field].(string)
if !ok {
continue
}
for _, key := range machineUsed(s) {
value, has := facts[key]
if !has {
return fmt.Errorf(
"%s has a %s that says ${machine:%s}, and this machine says %s",
module, field, key, orNothing(namesOfFacts(facts)))
}
s = strings.ReplaceAll(s, fmt.Sprintf("${machine:%s}", key), value)
resource[field] = s
if fmt.Sprint(resource["type"]) != "file" {
return nil
}
content, ok := resource["content"].(string)
if !ok {
return nil
}
for _, key := range machineUsed(content) {
value, has := facts[key]
if !has {
return fmt.Errorf(
"%s has a file that says ${machine:%s}, and this machine says %s",
module, key, orNothing(namesOfFacts(facts)))
}
resource["content"] = strings.ReplaceAll(
content, fmt.Sprintf("${machine:%s}", key), value)
content = resource["content"].(string)
}
return nil
}
@@ -70,59 +70,3 @@ func TestAnAddressAMachineDoesNotHaveIsRefused(t *testing.T) {
t.Errorf("the refusal does not name what was asked for: %v", err)
}
}
// A module that must give software the machine's ADDRESS rather than its name — a resolver pointing
// the container runtime at itself, whose list of resolvers cannot be a name — says
// ${machine:address}, and gets what the machine's name resolves to on the private network: the same
// address every other machine's hosts file carries for it.
func TestAModuleNamesTheAddressBehindItsMachinesName(t *testing.T) {
pointing := Manifest{Module: "pointing", Version: "1", Resources: []map[string]any{{
"id": "runtime", "type": "file", "path": "/etc/runtime.json",
"content": `{"dns":["${machine:address}"]}`,
}}}
got, err := Resolve(shelf(pointing), []string{"pointing"}, anchored(), World{})
if err != nil {
t.Fatal(err)
}
out, err := got.Declaration(Rendering{Names: map[string]string{
"workstation.internal": "10.42.0.7", "anchor.internal": "10.42.0.1"}})
if err != nil {
t.Fatal(err)
}
if content := plainly(out[0]["content"]); content != `{"dns":["10.42.0.7"]}` {
t.Fatalf("the machine's address was not the one its name resolves to: %q", content)
}
// Off the network there is no such address, and the refusal says what the machine does have —
// rather than a placeholder written into the runtime's file and read as an address.
got, err = Resolve(shelf(pointing), []string{"pointing"}, workstation(), World{})
if err != nil {
t.Fatal(err)
}
if _, err := got.Declaration(Rendering{}); err == nil || !strings.Contains(err.Error(), "${machine:address}") {
t.Fatalf("a machine off the network was given an address, or refused for another reason: %v", err)
}
}
// The mesh's private range is offered as ${machine:mesh-range}, so a module names it rather than
// hardcoding a value it cannot know (novox/hq ADR 0112) — the fail2ban ignoreip is the case.
func TestAModuleNamesTheMeshRange(t *testing.T) {
facts := machineFacts(Resolution{Node: "anchor", At: "anchor.internal"},
map[string]string{"anchor.internal": "10.10.0.1"}, "10.10.0.0/24")
if facts["mesh-range"] != "10.10.0.0/24" {
t.Fatalf("the mesh range is not a machine fact: %v", facts)
}
res := map[string]any{"type": "file", "id": "jail", "content": "ignoreip = 127.0.0.1/8 ${machine:mesh-range}\n"}
if err := machineInto(res, facts, "fail2ban"); err != nil {
t.Fatal(err)
}
if got := res["content"].(string); !strings.Contains(got, "10.10.0.0/24") || strings.Contains(got, "${machine:") {
t.Fatalf("the mesh range was not written in: %q", got)
}
// A mesh with no range gives no such fact, and a file that names it is refused rather than
// left with a literal placeholder in it.
none := machineFacts(Resolution{Node: "anchor"}, nil, "")
if _, has := none["mesh-range"]; has {
t.Fatal("a mesh with no range still offered one")
}
}
+36 -340
View File
@@ -176,29 +176,15 @@ type Manifest struct {
// Requires are names that must be provided by something assigned to the same node.
Requires []string `json:"requires,omitempty"`
// Emits are the events this module publishes, named **locally**: `order.placed`, not a subject
// and not a routing key. The mesh derives where it lands (design 29 §1), so reorganising the
// subject space leaves this manifest correct. Events are provisioning's lighter sibling — 1:many
// and broadcast, no credential (novox/hq ADR 0041).
//
// A module publishes under its own name only. If the event is about a *role* rather than about
// this module, it belongs on that seat, where the name outlives whoever holds it.
//
// **This said "dotted topic keys, e.g. module.umami.site.created" until 04-ISSUES/127**, which
// is the old bus's routing key, and is why every manifest in the catalogue had the same mistake:
// nobody was guessing, everybody followed this comment.
// Emits are the event types this module publishes onto the broker — dotted topic keys, e.g.
// "module.umami.site.created". Declared so the mesh knows the event graph; events are
// provisioning's lighter sibling — 1:many and broadcast, no credential (novox/hq ADR 0041).
Emits []string `json:"emits,omitempty"`
// Consumes are the events this module reacts to, each naming its emitter and the event:
// `billing.order.placed`. `*` stands for one name and `**` for the rest, so `*.download.completed`
// is that event from any module and `**` is every event in the mesh.
//
// Spelled the mesh's way rather than the wire's, for the reason Emits is: the bus the mesh runs
// on today spells these `*` and `#`, the one being built spells them `*` and `>`, and a manifest
// naming either would stop being true when the wire changed.
//
// The runtime wires the subscription; the module ships the handler. A Consumes for an event
// nothing on the mesh Emits is a dangling edge.
// Consumes are the event patterns this module subscribes to — topic patterns over module,
// mesh and node events alike, e.g. "node.*.joined" or "#" (the audit logger). The runtime
// wires the subscription; the module ships the handler. A Consumes for an event nothing on
// the mesh Emits is a dangling edge.
Consumes []string `json:"consumes,omitempty"`
// Claims are singular resources. Two modules claiming one thing within a scope cannot both
@@ -206,47 +192,6 @@ type Manifest struct {
// that every new module would force its predecessors to update.
Claims []Claim `json:"claims,omitempty"`
// DefinesSeats are the seats this module defines for itself, with their protocols
// (novox/hq ADR 0121, ADR 0129). The control plane defines the system seats — `mesh-*` and
// `node-*` — and a module may define its own, named outside that namespace, to coordinate its
// own instances: the mesh enforces one-holder-per-scope for it without knowing what it means. A
// module's declared seat is the only non-system name it may then claim; a claim to a name
// neither the mesh nor the module defines is refused.
//
// **Two lines of work built this at once**, one calling it `Seats` with a protocol and one
// `DefinesSeats` without. Same key in the file, so no manifest is affected: this is the trunk's
// name with the richer type, because what a role accepts, emits and serves is what lets the mesh
// check that a holder answers what its seat promises.
DefinesSeats []SeatDeclaration `json:"seats,omitempty"`
// Uses are seats this module sends to. It names the *seat*, never the module holding it, so
// the implementation can be replaced under it and no caller changes. A caller gets publish
// on that seat's inbound subjects and nothing else — not its outbound events, and not a
// subscription to the queue it writes to (design 29 §2).
Uses []string `json:"uses,omitempty"`
// Prepares says this module has state that must be brought to the shape this version needs
// before this version runs, and that the module's own code does it (novox/hq ADR 0135).
//
// **A word, not an arrangement.** The mesh runs the module's own program in its preparation
// mode, in the module's own context — every binding, credential and setting its code receives,
// because it *is* its code. Nothing here names a container, a command, a mount or a variable:
// the module already said all of that once, and a second copy is a second thing to drift.
//
// **Declared, never inferred.** The control plane cannot read what is inside an artifact, so a
// module that ships a migration and does not say this breaks on its first upgrade. That is
// stated in the record rather than guarded here, because nothing mechanical can guard it.
Prepares bool `json:"prepares,omitempty"`
// Tools are the tools this module answers — request and reply, awaited.
//
// **New, and not `serves`**, which this manifest already uses for the facts a consumer needs
// in order to reach a provision. Two meanings under one key would be a footgun in the one
// file a module author reads most. Until now a module's tools were known only at runtime,
// from MESH_TOOL_MODULES in its image; declaring them is what lets the mesh check that a
// module claiming a seat answers what that seat's protocol promises (novox/hq ADR 0118).
Tools []string `json:"tools,omitempty"`
// Capabilities the machine must have. A different field from Requires because the remedy
// differs: a missing module can be assigned, and a missing capability means the wrong
// machine.
@@ -288,18 +233,6 @@ type Manifest struct {
// module that had to say both would eventually say one.
Contributes map[string]map[string]any `json:"contributes,omitempty"`
// ContributesMany is the same key, `contributes`, where a module tells one provider several
// things under local names — `"route": {"api": {"label": "files-api", "port": 9000}, "console":
// {"label": "files", "port": 9001}}` — because a module may answer one requirement more than
// once: an object store with a data API and a console are two different public names, not one
// (novox/hq ADR 0094's sibling for `contributes` rather than `secrets` — "a module may need more
// than one value from a provider that gives one per pair" applies exactly as well to what a
// module gives a provider as to what it keeps from one). Each local name is a contribution of
// its own, reaching the provider as its own entry in the file it receives.
//
// Filled from the manifest's `contributes` object by UnmarshalJSON; never written by hand.
ContributesMany map[string]map[string]map[string]any `json:"-"`
// Receives is where this module wants its consumers' contributions written, per requirement
// it provides.
//
@@ -413,14 +346,6 @@ type Manifest struct {
// that could only see its own ports would write a rule set that closed everything else.
Filtering *Filtering `json:"filtering,omitempty"`
// Jails are the fail2ban jails this module declares for its own service (novox/hq to-be 31).
// Written into whichever node runs the module, the same way `listens` become that node's rules.
Jails []Jail `json:"jails,omitempty"`
// Jailing marks the module that composes the node's fail2ban jails — the intrusion-prevention
// holder. Like Filtering: one module per node gathers what every module declared and writes it.
Jailing *Jailing `json:"jailing,omitempty"`
// Guards are ports of this module's the mesh refuses on an adopted node except from the
// private network and from the machine itself (novox/hq ADR 0100) — the store's port and the
// broker's management port. The ports the software uses; the mesh guards where the machine
@@ -429,29 +354,24 @@ type Manifest struct {
// firewall does. Ignored on a converged node, whose derived filter already closes them.
Guards []int `json:"guards,omitempty"`
// Facts are things only the mesh knows, written where this module asks for them — in the
// module's own format.
// Facts are things only the mesh knows, written where this module asks for them.
//
// **The graph is the control plane's; the format is the module's.** The mesh knows which
// machines exist, what they are called and where they are. Turning that into a name that
// resolves, a peer that is reachable, a host a client trusts, is somebody's software — dnsmasq,
// a resolver, a VPN, ssh — in its own configuration language, and the mesh has no business
// knowing it. So a module gives a path and a template; the mesh renders the roster through it
// and owns nothing of what the file says.
// **The graph is the control plane's; how a machine uses it is the module's.** The mesh knows
// which machines exist, what they are called and where they are. Making a name resolve, or a
// peer reachable, is somebody's software — dnsmasq, a resolver, a VPN — and the mesh has no
// business shipping one, choosing which, or knowing its configuration language.
//
// This used to be a closed list of fact names, each formatted in Go in the control plane, so a
// new consumer meant a new formatter here in the consumer's language. Now the data is the mesh's
// and the format is the module's: the two built-in cases — the network module's `/etc/hosts` and
// dnsmasq's zones — render through the same template path any module uses, and no format lives
// in the control plane at all. See RosterFile for what a template sees.
// So a module says *put the node names here* and owns everything after that. The same shape as
// `filtering`, generalised: a fact, and a path.
//
// It replaces three modules that existed only because computed output needed somewhere to
// live — they ran no software, could not be swapped for anything, and appeared in the graph as
// modules while being a data channel wearing a costume.
//
// Keyed by a name the module chooses, which is the rendered file's id (`fact-<name>`) — what a
// `restart-on` names to restart when the roster changes.
Facts map[string]RosterFile `json:"facts,omitempty"`
// Keyed by fact name; the names are a closed list, because a module asking for one the mesh
// does not compute is asking for something nobody will write, and finding that out on a machine
// is worse than being told here.
Facts map[string]string `json:"facts,omitempty"`
// Certificate is where this module wants a certificate for its machine's name inside the
// mesh, and where the key that goes with it can be found.
@@ -471,20 +391,6 @@ type Manifest struct {
// A directory rather than one document for the same reason as above: each value is sealed
// separately and the mesh cannot open any of them to build a list.
Grants map[string]string `json:"grants,omitempty"`
// BusUsers is where this module wants the mesh's user list written, and it is only ever
// answered for the module holding `mesh-broker`.
//
// **The mesh writes who may connect; the module owns everything else about its server**
// (novox/hq design 25 §4, task 1.7). Ports, TLS paths and a store directory live in this
// module's image and its mounts and change when it does, so the module's own configuration
// carries them and includes this file. A controller that wrote the whole configuration would
// have to be kept in step with a Dockerfile it never sees.
//
// **Asking for it is not enough to receive it.** This file holds every user's password hash, so
// a module that could ask for it could read every credential on the bus — and the claim on
// `mesh-broker` is what authorises it, checked from this manifest alone.
BusUsers string `json:"bus-users,omitempty"`
}
// Build says how to produce this module's artifacts from its source.
@@ -529,18 +435,6 @@ type BuildsOn struct {
Image string `json:"image,omitempty"`
}
// ArtifactContext names the repository an image artifact's build context is cloned from, when
// that is not this module's own repository.
type ArtifactContext struct {
// Repository is cloned fresh, the same way the module's own repository is — a working tree
// nothing has touched, so what was built is reproducible from the two commits named rather
// than from whatever a previous build happened to leave behind.
Repository string `json:"repository"`
// Ref is the branch, tag or commit of that repository to build. Empty means its own default
// branch — the same meaning an empty module ref already has.
Ref string `json:"ref,omitempty"`
}
// Artifact is one thing built from a module's source.
type Artifact struct {
// Name is how resources refer to it. Local to the module.
@@ -560,17 +454,6 @@ type Artifact struct {
// Empty means the whole recipe, which is what a module with one image says by saying nothing.
Target string `json:"target,omitempty"`
// Context names a second repository this image's build reaches into for its own source — the
// recipe itself is still read from this module's own directory, at this module's own commit;
// only the build context `docker build`'s final argument names comes from here instead.
//
// **Packaging and source are allowed to live apart.** A module that only ships the recipe for
// source that lives elsewhere — the reference route-proxy in mesh-controller's own repository,
// packaged as a module in the catalogue rather than vendored a second time the two copies
// could drift from — names where that source actually is. Empty means the ordinary case: an
// image built from this same module's own repository, the same as every other artifact.
Context *ArtifactContext `json:"context,omitempty"`
// Language is what this module's code is written in, for a bundle.
//
// **Declared, never guessed.** Inferring it from what files happen to be present makes a
@@ -684,36 +567,6 @@ func (l Listening) At() string {
return l.Protocol
}
// Jail is a fail2ban jail a module declares for its own service (novox/hq to-be 31).
//
// **The module names no node and no path** (ADR 0112): it says what a break-in on its service looks
// like — the failregex — and the jail's own keys (the port it watches, where it logs, how many
// tries, how long to ban). The mesh writes it into whichever node's fail2ban runs the module, the
// same way a module's `listens` become that node's firewall rules. A node not running the module
// has no such jail.
type Jail struct {
// Name is the jail and its filter, e.g. "postgres-auth". One holder of the name per node.
Name string `json:"name"`
// Failregex is what a failed authentication looks like in the service's log — the filter.
Failregex string `json:"failregex"`
// Jail is the body of the jail's stanza: the keys under [<name>] the module knows and the mesh
// does not — the port it watches, its logpath and backend, maxretry, bantime.
Jail string `json:"jail"`
}
// Jailing says a module composes the node's fail2ban jails — the intrusion-prevention holder. Like
// Filtering for the firewall: one module gathers what every other module declared and writes it
// where it owns. Into is the one jail file the stanzas are composed into (so the fail2ban service
// can restart on a single resource); FilterInto is the directory each jail's filter file goes in.
type Jailing struct {
Into string `json:"into"`
FilterInto string `json:"filter-into"`
}
// ComposedJailsID is the single jail file the mesh composes every declared jail into, so the
// fail2ban service names one resource in its restart-on and a jail added or removed reaches it.
func ComposedJailsID() string { return "composed-jails" }
// Filtering says where a module wants the computed rule set.
type Filtering struct {
// Into is the path to write it to. Whatever loads it is this module's own business — an
@@ -733,22 +586,7 @@ type Certificate struct {
// CertificateID and AuthorityID are the resource identities of what the mesh issued.
func CertificateID() string { return "certificate" }
// ClaimsSeat says whether this manifest claims one named seat.
func (m Manifest) ClaimsSeat(seat string) bool {
for _, c := range m.Claims {
if c.Name == seat {
return true
}
}
return false
}
// BusUsersID names the mesh's composed user list, so it is the same resource across every
// declaration and a change to it is an update rather than a second file beside the old one — which
// on a bus reading a directory would be two account lists, and the server would take both.
func BusUsersID() string { return "bus-users" }
func AuthorityID() string { return "certificate-authority" }
func AuthorityID() string { return "certificate-authority" }
// FilteringID names the computed rule set, so it is the same resource across every declaration
// and a change to it is an update rather than an addition beside the old one.
@@ -777,24 +615,16 @@ func AccessID(path string) string { return "access-" + strings.TrimPrefix(path,
// it contributes to.
func (m Manifest) Wants() []string {
out := append([]string{}, m.Requires...)
add := func(to string) {
for to := range m.Contributes {
var already bool
for _, r := range m.Requires {
if r == to {
return
already = true
}
}
for _, already := range out {
if already == to {
return
}
if !already {
out = append(out, to)
}
out = append(out, to)
}
for to := range m.Contributes {
add(to)
}
for to := range m.ContributesMany {
add(to)
}
sort.Strings(out)
return out
@@ -849,47 +679,6 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error {
}
delete(keys, "secrets")
}
contributesPlain := map[string]map[string]any{}
contributesMany := map[string]map[string]map[string]any{}
if contributes, ok := keys["contributes"]; ok && string(contributes) != "null" {
var byTo map[string]json.RawMessage
if err := json.Unmarshal(contributes, &byTo); err != nil {
return fmt.Errorf("contributes: an object of requirement to values, or to {local name: values}: %w", err)
}
for to, v := range byTo {
// Both shapes are JSON objects, unlike secrets' path-vs-object split, so the shapes are
// told apart by what is INSIDE: an ordinary contribution's fields are scalars (a label,
// a port); the several-instance shape is an object of local names, each itself an
// object of fields. Confirmed against the whole catalogue before relying on it — no
// contribution anywhere has an object-valued field.
var fields map[string]json.RawMessage
if err := json.Unmarshal(v, &fields); err != nil {
return fmt.Errorf("contributes.%s: an object of values, or of local name to values: %w", to, err)
}
many := len(fields) > 0
for _, field := range fields {
trimmed := bytes.TrimSpace(field)
if len(trimmed) == 0 || trimmed[0] != '{' {
many = false
break
}
}
if many {
var locals map[string]map[string]any
if err := json.Unmarshal(v, &locals); err != nil {
return fmt.Errorf("contributes.%s: an object of local name to values: %w", to, err)
}
contributesMany[to] = locals
continue
}
var values map[string]any
if err := json.Unmarshal(v, &values); err != nil {
return fmt.Errorf("contributes.%s: an object of values: %w", to, err)
}
contributesPlain[to] = values
}
delete(keys, "contributes")
}
rest, err := json.Marshal(keys)
if err != nil {
return err
@@ -907,46 +696,22 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error {
if len(many) > 0 {
m.SecretsMany = many
}
if len(contributesPlain) > 0 {
m.Contributes = contributesPlain
}
if len(contributesMany) > 0 {
m.ContributesMany = contributesMany
}
return nil
}
// MarshalJSON writes `secrets` and `contributes` back in the shape they were read: single values,
// and objects of local names.
// MarshalJSON writes `secrets` back in the shape it was read: paths, and objects of local names.
func (m Manifest) MarshalJSON() ([]byte, error) {
raw, err := json.Marshal(manifestFields(m))
if err != nil {
return nil, err
}
if len(m.SecretsMany) == 0 && len(m.ContributesMany) == 0 {
if len(m.SecretsMany) == 0 {
return raw, nil
}
var keys map[string]json.RawMessage
if err := json.Unmarshal(raw, &keys); err != nil {
return nil, err
}
if len(m.ContributesMany) > 0 {
mergedContributes := map[string]any{}
for to, values := range m.Contributes {
mergedContributes[to] = values
}
for to, locals := range m.ContributesMany {
mergedContributes[to] = locals
}
contributes, err := json.Marshal(mergedContributes)
if err != nil {
return nil, err
}
keys["contributes"] = contributes
}
if len(m.SecretsMany) == 0 {
return json.Marshal(keys)
}
merged := map[string]any{}
for to, path := range m.Secrets {
merged[to] = path
@@ -1040,28 +805,6 @@ func ParseManifest(raw []byte) (Manifest, error) {
problems = append(problems, fmt.Sprintf(
"%q is not a usable slug: lower-case letters, digits, dashes and dots", m.Slug))
}
// **`amqp` is not a provision, and not a requirement** (novox/hq ADR 0131). A module that wants
// messaging wants the mesh's bus — it emits and consumes through the sdk, which the mesh hands the
// bus with the module's own credential — and the bus is whatever holds `mesh-broker`, spoken in
// whatever that holder speaks. Naming the old wire protocol asks for a specific server, and the
// only one that could answer is the one being retired. Refused here so the word cannot come back
// through a manifest.
for _, offer := range m.Provides {
if offer.Name == "amqp" {
problems = append(problems, fmt.Sprintf(
"%s provides %q, which is not a provision: the mesh's bus is whatever holds "+
"mesh-broker, and a module provides mesh-bus to be it (novox/hq ADR 0131)",
m.Module, offer.Name))
}
}
for _, r := range m.Requires {
if r == "amqp" {
problems = append(problems, fmt.Sprintf(
"%s requires %q, which is not a provision: a module reaches the mesh's bus through "+
"the sdk, and depends on the mesh-broker seat, not on a protocol (novox/hq ADR 0131)",
m.Module, r))
}
}
for _, offer := range m.Provides {
p := offer.Name
if !name.MatchString(p) {
@@ -1099,15 +842,9 @@ func ParseManifest(raw []byte) (Manifest, error) {
problems = append(problems, fmt.Sprintf("%s requires itself", m.Module))
}
}
// What it may call an event, and what it may ask to hear (events.go). Checked here because a
// module whose event names are wrong installs, starts, connects and reacts to nothing, with
// every log line saying it is fine (novox/hq 04-ISSUES/127).
problems = append(problems, EventProblems(m)...)
wellFormed := true
for _, c := range m.Claims {
if !name.MatchString(c.Name) {
problems = append(problems, fmt.Sprintf("%q is not a usable claim name", c.Name))
wellFormed = false
}
switch c.At() {
case ScopeNode, ScopeSite, ScopeMesh:
@@ -1115,18 +852,8 @@ func ParseManifest(raw []byte) (Manifest, error) {
problems = append(problems, fmt.Sprintf(
"%s claims %s at scope %q; a claim is held per node, per site or per mesh",
m.Module, c.Name, c.Scope))
wellFormed = false
}
}
// Against the seats the mesh defines (novox/hq ADR 0110), once every claim is at least a name
// and a scope — a malformed claim is refused for that, not a second time for being unknown.
if wellFormed {
problems = append(problems, claimProblems(m)...)
}
// What one manifest can be judged on: a declaration's shape, its scope, and the reserved
// prefix. Whether a seat anybody names exists, and whether a holder answers for it, are
// facts about the catalogue and are checked at registration (CatalogueProblems).
problems = append(problems, declaredSeatProblems(m)...)
if m.Computed != "" && len(m.Resources) > 0 {
// One or the other. A module that both ships files and has them computed would leave
// nobody able to say where a given file came from.
@@ -1145,25 +872,6 @@ func ParseManifest(raw []byte) (Manifest, error) {
"%s contributes nothing to %q; if it only needs one, require it", m.Module, to))
}
}
for to, locals := range m.ContributesMany {
if !name.MatchString(to) {
problems = append(problems, fmt.Sprintf("%q is not a usable name to contribute to", to))
}
if len(locals) == 0 {
problems = append(problems, fmt.Sprintf(
"%s contributes nothing to %q; if it only needs one, require it", m.Module, to))
}
for local, values := range locals {
if !name.MatchString(local) {
problems = append(problems, fmt.Sprintf(
"%s contributes to %q under %q, which is not a usable name", m.Module, to, local))
}
if len(values) == 0 {
problems = append(problems, fmt.Sprintf(
"%s contributes nothing to %q under %q", m.Module, to, local))
}
}
}
problems = append(problems, m.Build.problems(m.Module)...)
// **What provides the artifact store cannot be delivered through it** (novox/hq 04-ISSUES/029).
//
@@ -1206,9 +914,9 @@ func ParseManifest(raw []byte) (Manifest, error) {
}
}
for to, where := range m.Binds {
if !placedOrAbsolute(where) {
if !strings.HasPrefix(where, "/") {
problems = append(problems, fmt.Sprintf(
"%s binds %q at %q, which is neither an absolute path nor a placed one", m.Module, to, where))
"%s binds %q at %q, which is not an absolute path", m.Module, to, where))
}
var wanted bool
for _, w := range m.Wants() {
@@ -1289,17 +997,6 @@ func ParseManifest(raw []byte) (Manifest, error) {
"program that reads what the mesh delivered and reconciles",
m.Module, r["id"]))
}
// **A module that prepares its state must have code the mesh can run** (novox/hq ADR 0135). The
// preparation is the module's own program in its preparation mode, so it is derived from the
// resource that runs that program — and a module declaring none has asked for something the mesh
// cannot compose. Said here, where the manifest is read, rather than by a declaration that
// quietly prepares nothing.
if m.Prepares && preparationTarget(m) == "" {
problems = append(problems, fmt.Sprintf(
"%s says it prepares its state, and declares no container running an artifact it built — "+
"the preparation is this module's own program, so there has to be one for the mesh to "+
"run it in", m.Module))
}
// **A run-once container is a step the host runs to completion** (novox/hq ADR 0052). It is a
// boolean modifier on the container shape — the host runs the container, requires it to exit 0,
// and starts whatever the declaration places after it only once it has. A value that is not a
@@ -1351,9 +1048,9 @@ func ParseManifest(raw []byte) (Manifest, error) {
}
}
for name, where := range m.OwnSecrets {
if !placedOrAbsolute(where) {
if !strings.HasPrefix(where, "/") {
problems = append(problems, fmt.Sprintf(
"%s needs %q at %q, which is neither an absolute path nor a placed one", m.Module, name, where))
"%s needs %q at %q, which is not an absolute path", m.Module, name, where))
}
if name == "" {
problems = append(problems, m.Module+" needs a secret with no name")
@@ -1368,9 +1065,9 @@ func ParseManifest(raw []byte) (Manifest, error) {
}
}
for _, f := range m.SecretFiles(to) {
if !placedOrAbsolute(f.Path) {
if !strings.HasPrefix(f.Path, "/") {
problems = append(problems, fmt.Sprintf(
"%s keeps the credential for %q at %q, which is neither an absolute path nor a placed one",
"%s keeps the credential for %q at %q, which is not an absolute path",
m.Module, SecretLocal(to, f.Local), f.Path))
}
if f.Local != "" && !name.MatchString(f.Local) {
@@ -1420,9 +1117,9 @@ func ParseManifest(raw []byte) (Manifest, error) {
}
}
for to, where := range m.Grants {
if !placedOrAbsolute(where) {
if !strings.HasPrefix(where, "/") {
problems = append(problems, fmt.Sprintf(
"%s grants %q into %q, which is neither an absolute path nor a placed one", m.Module, to, where))
"%s grants %q into %q, which is not an absolute path", m.Module, to, where))
}
var offered bool
for _, o := range m.Offers() {
@@ -1443,9 +1140,9 @@ func ParseManifest(raw []byte) (Manifest, error) {
if !name.MatchString(to) {
problems = append(problems, fmt.Sprintf("%q is not a usable name to receive", to))
}
if !placedOrAbsolute(where) {
if !strings.HasPrefix(where, "/") {
problems = append(problems, fmt.Sprintf(
"%s receives %q at %q, which is neither an absolute path nor a placed one", m.Module, to, where))
"%s receives %q at %q, which is not an absolute path", m.Module, to, where))
}
var offered bool
for _, o := range m.Offers() {
@@ -1492,7 +1189,6 @@ func ParseManifest(raw []byte) (Manifest, error) {
// Checked here rather than on the machine because the machine cannot tell the difference: by
// the time it sees the mount it is being asked to create the directory, which it can do.
problems = append(problems, m.undeclaredMounts()...)
problems = append(problems, m.unknownDirRefs()...)
for i, r := range m.Resources {
id, _ := r["id"].(string)
-121
View File
@@ -1,121 +0,0 @@
package catalogue
import (
"encoding/json"
"os"
"path/filepath"
"regexp"
"strings"
"testing"
)
// **A manifest holds no subject** (novox/hq design 29 §1).
//
// A module names its events, tools and seats locally, and the mesh derives where they land. The
// property that buys: reorganise the subject space and every manifest in the catalogue is still
// correct. It holds today by construction — nothing reads a subject from a manifest — and a rule
// held by construction is one a later field breaks quietly, with the symptom appearing as a
// permission that does not match a subject rather than as a manifest that was wrong.
func TestNoManifestContainsASubject(t *testing.T) {
root := filepath.Join("..", "..", "..", "mesh-catalog", "modules")
entries, err := os.ReadDir(root)
if err != nil {
t.Skipf("catalogue sibling not present: %v", err)
}
// Anything in the mesh's own subject space, and anything shaped like a wire address.
subject := regexp.MustCompile(`^(mesh|\$JS)\.[a-zA-Z0-9_*>.-]+$`)
var found []string
var walk func(module string, path string, v any)
walk = func(module, path string, v any) {
switch t := v.(type) {
case string:
if subject.MatchString(t) {
found = append(found, module+" "+path+" = "+t)
}
case map[string]any:
for k, inner := range t {
walk(module, path+"."+k, inner)
}
case []any:
for _, inner := range t {
walk(module, path+"[]", inner)
}
}
}
checked := 0
for _, e := range entries {
if !e.IsDir() {
continue
}
raw, err := os.ReadFile(filepath.Join(root, e.Name(), "module.json"))
if err != nil {
continue
}
var m any
if err := json.Unmarshal(raw, &m); err != nil {
t.Errorf("%s: %v", e.Name(), err)
continue
}
checked++
walk(e.Name(), "", m)
}
if checked == 0 {
t.Skip("no manifests read")
}
if len(found) > 0 {
t.Errorf("a manifest names a subject, so reorganising the subject space would mean "+
"editing the catalogue:\n %s", strings.Join(found, "\n "))
}
t.Logf("%d manifests hold no subject", checked)
}
// **Every module's event names are what design 29 says, across the whole catalogue.**
//
// The rule above holds by construction and turned out to be weaker than it reads: a manifest holds
// no subject, and every manifest in the catalogue still held the old bus's routing key, which
// derives into a namespace nobody owns (novox/hq 04-ISSUES/127). Nothing failed — the services
// started and none of them reacted. This is the check that was missing.
func TestEveryManifestsEventNamesAreLocal(t *testing.T) {
manifests := theCatalogue(t)
var problems []string
for _, m := range manifests {
problems = append(problems, EventProblems(m)...)
}
if len(problems) > 0 {
t.Fatalf("the catalogue holds %d event name(s) the mesh would derive wrongly:\n %s",
len(problems), strings.Join(problems, "\n "))
}
}
// theCatalogue is every manifest beside this checkout, parsed the way registration parses one.
func theCatalogue(t *testing.T) []Manifest {
t.Helper()
root := filepath.Join("..", "..", "..", "mesh-catalog", "modules")
entries, err := os.ReadDir(root)
if err != nil {
t.Skipf("catalogue sibling not present: %v", err)
}
var out []Manifest
for _, e := range entries {
if !e.IsDir() {
continue
}
raw, err := os.ReadFile(filepath.Join(root, e.Name(), "module.json"))
if err != nil {
continue
}
var m Manifest
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatalf("%s: %v", e.Name(), err)
}
out = append(out, m)
}
if len(out) == 0 {
t.Skip("no manifests found beside this checkout")
}
return out
}
-141
View File
@@ -1,141 +0,0 @@
package catalogue
import (
"encoding/json"
"fmt"
"strings"
"testing"
)
// A module version prepares its state before it runs (novox/hq ADR 0135).
//
// What the mesh derives is the module's own resource, run once with one word, placed immediately in
// front of the thing it prepares for. What matters in these tests is that the derivation is a copy
// rather than a second description: the failure it replaces was a hand-written step repeating six
// fields of the resource it preceded, each free to drift from it.
func aPreparingModule() Manifest {
return Manifest{
Module: "gitea",
Prepares: true,
Resources: []map[string]any{
{"id": "state", "type": "directory", "path": "/var/lib/gitea", "mode": "0700"},
{"id": "server", "type": "container", "name": "mesh-gitea-server",
"image": "gitea/gitea@" + digest, "ports": []any{"3000:3000"}},
{"id": "runtime", "type": "container", "name": "mesh-gitea",
"image": ArtifactStoreScheme + "gitea/runtime@" + digest, "network": "host",
"env": map[string]any{"MESH_GITEA_STATE_DIR": "/run/state"},
"volumes": []any{"/var/lib/gitea:/run/state:ro"},
"ports": []any{"9000:9000"}},
},
}
}
func declaredFor(t *testing.T, m Manifest) []map[string]any {
t.Helper()
// The manifest as the mesh holds it: a build resolved the module's own artifact into the
// reference it recorded, which is also how the composition knows whose code a resource runs.
out, err := Resolution{Node: "anchor", Modules: []Manifest{m}}.Declaration(
Rendering{ArtifactStore: "anchor.internal:5100"})
if err != nil {
t.Fatal(err)
}
return out
}
func idsOf(resources []map[string]any) []string {
var ids []string
for _, r := range resources {
ids = append(ids, fmt.Sprint(r["id"]))
}
return ids
}
// The step runs the module's own code, and comes immediately before it — not before the upstream
// server the module packages, which may be the very thing the state lives in.
func TestThePreparationRunsTheModulesOwnCodeAndComesRightBeforeIt(t *testing.T) {
out := declaredFor(t, aPreparingModule())
ids := idsOf(out)
at := -1
for i, id := range ids {
if id == "gitea.runtime-prepare" {
at = i
}
}
if at < 0 {
t.Fatalf("nothing prepares this module's state: %v", ids)
}
// A module's name may contain a dot, so a resource's module is everything before the last one —
// which the derived id must not add to, or a machine reads the wrong owner from it.
if strings.Count("gitea.runtime-prepare", ".") != 1 {
t.Fatal("the derived id adds a dot, so what owns it cannot be read from it")
}
if ids[at+1] != "gitea.runtime" {
t.Fatalf("the preparation is not immediately before the module's own code: %v", ids)
}
for _, id := range ids[:at] {
if id == "gitea.runtime" {
t.Fatalf("the module's own code runs before its state is prepared: %v", ids)
}
}
}
// It is given exactly what the module's own code is given. Asserted field by field against the
// resource it was derived from, because writing it twice is the fault this replaces.
func TestThePreparationIsGivenWhatTheModuleIsGiven(t *testing.T) {
out := declaredFor(t, aPreparingModule())
declared := byID(out)
step, workload := declared["gitea.runtime-prepare"], declared["gitea.runtime"]
if step == nil || workload == nil {
t.Fatalf("expected both, got %v", idsOf(out))
}
for _, field := range []string{"image", "network", "env", "volumes", "type"} {
if fmt.Sprint(step[field]) != fmt.Sprint(workload[field]) {
t.Errorf("the preparation's %s is %v and the module's is %v", field, step[field], workload[field])
}
}
if once, _ := step["run-once"].(bool); !once {
t.Error("the preparation is not a step, so nothing waits for it and nothing is gated by it")
}
if fmt.Sprint(step["args"]) != fmt.Sprint([]any{PreparationArgument}) {
t.Errorf("the preparation is asked for as %v", step["args"])
}
if fmt.Sprint(step["name"]) == fmt.Sprint(workload["name"]) {
t.Error("the preparation and the workload have one name, so one removes the other")
}
// A published port cannot be bound twice, and the version being replaced is still running.
if _, published := step["ports"]; published {
t.Errorf("the preparation publishes a port the running version holds: %v", step["ports"])
}
}
// A module that says nothing about preparing gets nothing, which is most modules.
func TestAModuleThatPreparesNothingGetsNoStep(t *testing.T) {
m := aPreparingModule()
m.Prepares = false
for _, id := range idsOf(declaredFor(t, m)) {
if id == "gitea.runtime-prepare" {
t.Fatal("a module that prepares nothing was given a preparation")
}
}
}
// A module whose own code the mesh cannot find has nothing to ask, and saying so where the manifest
// is read beats a declaration that quietly prepares nothing.
func TestAModuleThatPreparesAndRunsNoneOfItsOwnCodeIsRefused(t *testing.T) {
m := Manifest{
Module: "gitea",
Prepares: true,
Resources: []map[string]any{
// Only the upstream server it packages: nothing here runs gitea's own code.
{"id": "server", "type": "container", "name": "mesh-gitea-server", "image": "gitea/gitea@" + digest},
},
}
raw, err := json.Marshal(m)
if err != nil {
t.Fatal(err)
}
if _, err := ParseManifest(raw); err == nil {
t.Fatal("a module that prepares its state with nothing of its own to run was accepted")
}
}
+1 -1
View File
@@ -54,7 +54,7 @@ func TestTheShippedNetworkingModulesResolveOnTheirOwn(t *testing.T) {
// network is what gives a machine a name, so the provider asks for the node-names fact and
// there is nothing else to bring in. A module that ran nothing used to be here.
for _, m := range got.Modules {
if m.Module == overlay.Name && m.Facts["node-names"].Path == "" {
if m.Module == overlay.Name && m.Facts["node-names"] == "" {
t.Fatalf("the network's provider does not ask for the names: %+v", m.Facts)
}
}
+3 -71
View File
@@ -28,10 +28,6 @@ type Node struct {
// (novox/hq ADR 0066). A route contribution carries only a label — the subdomain — and the mesh
// joins <label>.<public-domain> to make the name it grants, interpreting neither half.
PublicDomain string
// Account is the operator's login on this machine, AccountHome where its home is (novox/hq
// to-be 29). What a home-scoped file is owned by and what ${machine:account} resolves to.
Account string
AccountHome string
}
// World is what the rest of the mesh already has.
@@ -41,11 +37,6 @@ type Node struct {
type World struct {
// Held is the claims already taken, for the scopes wider than one node.
Held []Held
// Holdings is every seat whose holder is **on record** (novox/hq ADR 0131): the one assignment
// that holds it, chosen by a handover. A seat absent here is held by derivation — the sole
// eligible assignment — as it always was. Present, it decides, and any other assignment whose
// module could hold the seat is eligible and silent rather than refused.
Holdings []Held
// Offered is what other nodes provide at mesh scope, and everything needed to use it.
Offered map[string][]Provider
// Pinned is which node this machine was told to get a provision from, by name. Only consulted
@@ -96,10 +87,6 @@ type Provider struct {
At string
// Serves is what the providing module said a consumer needs to know, settled.
Serves map[string]any
// Module is which module on that node provides it. A provider is a (node, module) pair
// (novox/hq to-be 23), and the pair is what tells the holder of a seat apart from another module
// providing the same thing (ADR 0110).
Module string
}
// Held is a claim somebody already has, used for the scopes wider than one node.
@@ -123,11 +110,6 @@ type Resolution struct {
// (novox/hq ADR 0066). Carried from the node so that composing <label>.<public-domain> for a
// route contribution needs no store lookup here — the join is a fact about this one machine.
PublicDomain string
// Account and AccountHome are the operator's login on this machine and where its home is
// (novox/hq to-be 29), carried from the node so a home-scoped file's owner and path resolve
// here without a store lookup.
Account string
AccountHome string
// Modules in the order they were resolved: assigned first, then what they pulled in.
Modules []Manifest
@@ -399,15 +381,6 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
default:
chosenNode, pinned := world.Pinned[want]
if !pinned {
// **The seat's holder answers, when a seat delivers this** (novox/hq ADR 0110).
// Not a guess, which ADR 0009 refuses: the choice was made once, mesh-wide, by
// assigning the holder, where a pin makes it again on every consumer's node. A
// pin still wins — it is a consumer coupled to one provider's contents, and has
// said so.
if holder, held := HolderAmong(want, where, world.Held); held {
take(holder)
break
}
problems = append(problems, fmt.Sprintf(
"%d nodes provide %q, wanted by %s — say which with `pin %s %s <node>`: %s",
len(where), want, because[want], node.Name, want,
@@ -555,14 +528,13 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
}
resolution := Resolution{Node: node.Name, At: node.At, PublicDomain: node.PublicDomain,
Account: node.Account, AccountHome: node.AccountHome,
Because: because, Needs: needs, Unhostable: unhostable}
for _, n := range providersFirst(order, catalogue) {
resolution.Modules = append(resolution.Modules, catalogue[n])
}
problems = append(problems, checkCapabilities(resolution.Modules, node)...)
claims, claimProblems := checkClaims(resolution.Modules, node, elsewhere, world.Holdings)
claims, claimProblems := checkClaims(resolution.Modules, node, elsewhere)
problems = append(problems, claimProblems...)
problems = append(problems, checkResources(resolution.Modules)...)
resolution.Claims = claims
@@ -655,35 +627,14 @@ func checkCapabilities(modules []Manifest, node Node) []string {
//
// Within this node's own set, and against what is already held elsewhere for the wider scopes. A
// claim at mesh scope is the same idea as the mesh's one hub, said once instead of hard-coded.
func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Held) ([]Held, []string) {
func checkClaims(modules []Manifest, node Node, elsewhere []Held) ([]Held, []string) {
var problems []string
var held []Held
// onRecord is the recorded holder of a seat, if a handover ever named one.
onRecord := func(claim, scope string) (Held, bool) {
for _, h := range holdings {
hs, ok := SeatNamed(h.Claim)
cs, cok := SeatNamed(claim)
if ok && cok && hs.Name == cs.Name && h.Scope == scope {
return h, true
}
}
return Held{}, false
}
byScope := map[string]map[string]string{} // scope → claim → module
for _, m := range modules {
for _, c := range m.Claims {
scope := c.At()
// **A recorded holder settles it before any counting.** An assignment that could hold
// the seat but is not the one on record is eligible, and that is all: it is not a second
// holder, so it is not refused, and it does not hold (novox/hq ADR 0131). This is what
// lets the next holder stand beside the current one until the seat is handed over.
if rec, recorded := onRecord(c.Name, scope); recorded {
if rec.Node != node.Name || rec.Module != m.Module {
continue
}
}
if byScope[scope] == nil {
byScope[scope] = map[string]string{}
}
@@ -741,30 +692,11 @@ func checkResources(modules []Manifest) []string {
ownedPath := map[string]string{} // path → owning module, for the access check below
for _, m := range modules {
// Compared placed, not as written (novox/hq ADR 0112): ${dir:state} is the same six
// characters in every module and a different directory in each — two modules' templates
// being spelled alike is not two modules owning one path. The default root serves the
// comparison: a collision is within one node, and any one root keeps distinct modules'
// places distinct. A reference that cannot be placed is left as written — naming what
// does not exist is the manifest's own problem, refused where it was made.
dirs := dirsFor(m, Rendering{})
for _, r := range m.Resources {
for _, field := range []string{"path", "unit", "name", "package"} {
value, ok := r[field].(string)
if !ok || value == "" {
if field == "path" && fmt.Sprint(r["type"]) == "directory" {
// A pathless directory owns its placed path — a module stating that
// very path is exactly the collision this exists to catch.
value = dirs[fmt.Sprint(r["id"])]
}
if value == "" {
continue
}
}
if field == "path" {
if placed, err := dirFill(value, dirs, m.Module); err == nil {
value = placed
}
continue
}
key := field + " " + value
if other, taken := owner[key]; taken && other != m.Module {
@@ -1,191 +0,0 @@
package catalogue
import (
"encoding/json"
"strings"
"testing"
)
// The catalogue's resolver modules as they are, parsed by the real parser and composed as a
// machine would receive them (hal dnsmasq-app conversion, novox/hq 08-connectivity).
//
// The predecessor's resolver answered every name on a machine: the mesh's own itself, the rest
// forwarded to two fixed upstreams, with the machine's resolv.conf naming it alone and the
// container runtime pointed at its private-network address. These hold the mesh's modules to the
// same arrangement, and to the two things a resolver here must never do — read resolv.conf for
// its upstreams, or take an address systemd-resolved holds.
// resolverShelf is the three resolver modules beside something that answers `mesh-addressing`.
// The networking module that really does is composed in the controller and cannot be imported
// here, so a stand-in offers the same word; what is under test is the manifests, not the network.
func resolverShelf(t *testing.T) map[string]Manifest {
t.Helper()
shelf := map[string]Manifest{
"net": {Module: "net", Version: "1", Provides: []Offer{{Name: "mesh-addressing"}}},
}
for _, name := range []string{"dnsmasq", "resolv-conf", "resolved-split-dns"} {
shelf[name] = catalogueManifest(t, name)
}
return shelf
}
// twoMachines is what the control plane hands a rendering: internal names and their addresses.
var twoMachines = map[string]string{"anchor.internal": "10.42.0.1", "laptop.internal": "10.42.0.2"}
// Its configuration forwards to the upstreams the predecessor's module shipped, and gets them from
// nowhere else: `no-resolv` is what makes the documented loop — the resolver finding its own
// address in resolv.conf and becoming its own upstream — impossible.
func TestTheResolverForwardsToFixedUpstreamsAndNeverReadsResolvConf(t *testing.T) {
m := catalogueManifest(t, "dnsmasq")
var config string
for _, r := range m.Resources {
if r["id"] == "config" {
config, _ = r["content"].(string)
}
}
if config == "" {
t.Fatal("the resolver has no configuration file")
}
for _, want := range []string{
"\nno-resolv\n", "\nserver=1.1.1.1\n", "\nserver=8.8.8.8\n",
"\nlisten-address=127.0.0.1\n", "\ninterface=mesh0\n", "\nbind-dynamic\n",
"\ndomain-needed\n", "\nbogus-priv\n",
"\nconf-file=" + m.Facts["node-zones"].Path + "\n",
} {
if !strings.Contains(config, want) {
t.Errorf("the resolver's configuration lacks %q:\n%s", strings.TrimSpace(want), config)
}
}
// Not .53 or .54, which systemd-resolved holds; and not .55 any more, which was a convention
// beside the one every machine already followed — the predecessor's resolv.conf says .1.
for _, taken := range []string{"127.0.0.53", "127.0.0.54", "127.0.0.55"} {
if strings.Contains(config, "listen-address="+taken) {
t.Errorf("the resolver listens on %s", taken)
}
}
// And the file that decides what the machine asks names it there, alone.
var resolv string
for _, r := range catalogueManifest(t, "resolv-conf").Resources {
if r["path"] == "/etc/resolv.conf" {
resolv, _ = r["content"].(string)
}
}
var nameservers []string
for _, line := range strings.Split(resolv, "\n") {
if strings.HasPrefix(line, "nameserver ") {
nameservers = append(nameservers, strings.TrimPrefix(line, "nameserver "))
}
}
if len(nameservers) != 1 || nameservers[0] != "127.0.0.1" {
t.Errorf("resolv.conf names %v; the predecessor's names the mesh's resolver alone at 127.0.0.1", nameservers)
}
// The split-DNS alternative points at the same address, or a machine that keeps
// systemd-resolved in charge would route the mesh's suffix to nothing.
for _, r := range catalogueManifest(t, "resolved-split-dns").Resources {
if content, _ := r["content"].(string); content != "" && !strings.Contains(content, "DNS=127.0.0.1\n") {
t.Errorf("resolved-split-dns does not point at the resolver's address:\n%s", content)
}
}
}
// The resolver and what points the machine at it compose on one machine, and what arrives is the
// mesh's account of every machine as a wildcard, the suffix kept local, the daemon restarting on
// that file, and the runtime pointed at this machine's own address.
func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
got, err := Resolve(resolverShelf(t), []string{"dnsmasq", "resolv-conf"},
Node{Name: "anchor", At: "anchor.internal"}, World{})
if err != nil {
t.Fatal(err)
}
if !strings.Contains(strings.Join(named(got), " "), "net") {
t.Fatalf("the resolver's data is the mesh's addresses, and nothing answering them was taken: %v", named(got))
}
out, err := got.Declaration(Rendering{
// Names is every name the mesh serves; Machines is the subset that is a node (novox/hq
// issue 111) — the resolver's zones read only the second, and in this scenario the two
// happen to be the same map, since nothing routed is part of it.
Names: twoMachines, Machines: twoMachines, Suffix: "internal",
Needed: map[string]map[string]string{"dnsmasq": {"broker": "sealed"}},
})
if err != nil {
t.Fatal(err)
}
ids := byID(out)
zones := ids["dnsmasq.fact-node-zones"]
if zones == nil || zones["path"] != "/etc/mesh-resolver/nodes.conf" {
t.Fatalf("the resolver was not given the machines where its configuration reads them: %v", zones)
}
content, _ := zones["content"].(string)
for _, want := range []string{
"local=/internal/", "address=/anchor.internal/10.42.0.1", "address=/laptop.internal/10.42.0.2",
} {
if !strings.Contains(content, want) {
t.Errorf("the machines file lacks %q:\n%s", want, content)
}
}
service := ids["dnsmasq.service"]
if service == nil {
t.Fatal("no resolver service composed")
}
reflects := map[string]bool{}
for _, id := range service["restart-on"].([]any) {
reflects[id.(string)] = true
}
if !reflects["dnsmasq.config"] || !reflects["dnsmasq.fact-node-zones"] {
t.Errorf("the daemon does not restart on its configuration and the machines file both: %v", service["restart-on"])
}
// The runtime's own file, written into (novox/hq ADR 0102) with the one key this module states.
runtime := ids["dnsmasq.runtime-dns"]
if runtime == nil || runtime["path"] != "/etc/docker/daemon.json" || runtime["into"] != "json" {
t.Fatalf("the runtime's dns is not written into its file: %v", runtime)
}
var keys map[string][]string
if err := json.Unmarshal([]byte(runtime["content"].(string)), &keys); err != nil {
t.Fatalf("the runtime's keys are not JSON: %v", err)
}
if len(keys) != 1 || len(keys["dns"]) != 1 || keys["dns"][0] != "10.42.0.1" {
t.Errorf("the runtime is pointed at %v; containers resolve at this machine's own private-network address, and nothing else is written", keys)
}
for _, r := range out {
if r["type"] == "service" && r["unit"] == "docker.service" && r["id"] != "" &&
strings.HasPrefix(r["id"].(string), "dnsmasq.") {
t.Errorf("the resolver orders the runtime restarted or reloaded, which stops every container (ADR 0102) or does nothing for dns: %v", r)
}
}
resolv := ids["resolv-conf.resolv"]
if resolv == nil || !strings.Contains(resolv["content"].(string), "\nnameserver 127.0.0.1\n") {
t.Fatalf("the machine is not pointed at the resolver: %v", resolv)
}
}
// Two modules deciding what a machine asks are refused on one machine, as before — the claim
// exists so they never take turns overwriting each other.
func TestTwoThingsDecidingWhatAMachineAsksAreRefused(t *testing.T) {
_, err := Resolve(resolverShelf(t), []string{"dnsmasq", "resolv-conf", "resolved-split-dns"},
Node{Name: "anchor", At: "anchor.internal"}, World{})
if err == nil {
t.Fatal("resolv-conf and resolved-split-dns were both assigned to one machine")
}
if !strings.Contains(err.Error(), "node-resolver-config") {
t.Fatalf("the refusal does not say what was claimed: %v", err)
}
}
// A machine that is not on the private network has no address for the runtime to be pointed at.
// Refused where the module and the machine are both named, rather than a placeholder written into
// the runtime's file and read as an address.
func TestTheResolverOnAMachineOffTheNetworkIsRefused(t *testing.T) {
got, err := Resolve(resolverShelf(t), []string{"dnsmasq"}, Node{Name: "anchor"}, World{})
if err != nil {
t.Fatal(err)
}
_, err = got.Declaration(Rendering{Names: twoMachines, Suffix: "internal",
Needed: map[string]map[string]string{"dnsmasq": {"broker": "sealed"}}})
if err == nil || !strings.Contains(err.Error(), "${machine:address}") {
t.Fatalf("a machine off the network was composed a resolver, or refused for another reason: %v", err)
}
}
-210
View File
@@ -1,210 +0,0 @@
package catalogue
import (
"bytes"
"fmt"
"sort"
"strings"
"text/template"
)
// What only the mesh knows, written where a module asks for it — in the module's own format.
//
// **The graph is the control plane's; the format is the module's.** The mesh knows which machines
// exist, what they are called and where they are. Turning that into a hosts file, a resolver's
// zones, an ssh known_hosts is somebody's configuration language, and the mesh has no business
// knowing it. So the mesh hands the roster to a template the module wrote and renders it; it never
// learns what the file means.
//
// This used to be a closed list of fact names, each with its format written in Go here — a hosts
// file, a resolver's zones. Every new consumer meant a new formatter in the control plane, in the
// consumer's configuration language. Now the data is the mesh's and the format is a template the
// module ships: the two built-in cases (the network module's `/etc/hosts`, dnsmasq's zones) render
// the same way any module's would, and the control plane holds no format at all.
//
// It replaced three modules that existed only because computed output needed somewhere to live —
// they ran no software, could not be swapped for anything, and appeared in the graph as modules
// while being a data channel wearing a costume (novox/hq ADR 0040).
// A RosterFile is a file the mesh renders from the roster of machines, in the format the module
// gives as a Go text/template. The template sees a rosterView: `.Node` (this machine's bare name),
// `.Suffix` (what its mesh name ends in), and two sets of `{Name, FQDN, Address}` — `.Names`, every
// name the mesh serves, and `.Machines`, only the nodes of the mesh. Which set a template ranges is
// how the hq issue 111 distinction is drawn: a container's hosts wants every name; a resolver told
// the suffix is its own wants only the machines.
type RosterFile struct {
// Path is where on the machine the rendered file goes. Absolute, or it is refused here rather
// than discovered as a daemon that reads nothing.
Path string `json:"path"`
// Template is the module's format, a Go text/template over the rosterView. It is the module's,
// not the mesh's: the mesh renders it and does not read it.
Template string `json:"template"`
// Shared is whether the file the fact goes to belongs to the machine rather than the mesh. When
// it does, the mesh owns only a marked region of it and keeps the rest byte for byte (novox/hq
// issue 128) — a hosts file is shared, since the distribution's `localhost`, the operator's own
// lines and other tools' blocks live there too; a resolver's zones file is not, the mesh owns it
// whole. A property of the fact, not of the path: the format determines whether the file is
// wholly the mesh's, not where a module happened to ask for it.
Shared bool `json:"shared,omitempty"`
// Home places the file under this node's operator-account home and chowns it to that account,
// rather than at an absolute system path (novox/hq to-be 29). Then Path is home-relative
// (`.ssh/config.d/mesh`), resolved against the account's home on the node it is composed for; a
// node with no operator account gets no such file. This is how the ssh-client config — every
// other node's Host block — is written into a person's home rather than into /etc.
Home bool `json:"home,omitempty"`
}
// rosterView is what a RosterFile's template sees. A closed shape — a template referencing a field
// the mesh does not compute fails to render here, not on a machine.
type rosterView struct {
Node string
Suffix string
Names []rosterEntry
Machines []rosterEntry
}
// rosterEntry is one machine as a template sees it: its bare name, its full mesh name, its address,
// and the operator account to log into it as (novox/hq to-be 29) — empty when none is known, so an
// ssh Host block template can omit the User line for a machine nobody has an account on.
type rosterEntry struct {
Name string
FQDN string
Address string
Account string
}
// FactsInto renders the roster files a module asked for, as files it will be given.
//
// The module owns everything after the file exists: loading it, restarting on it, what a resolver
// or a client does with it. This only puts it there. `every` is every name the mesh serves;
// `machines` is only the machines — the two must not be confused (novox/hq 04-ISSUES/111), so both
// are given and the template chooses.
func FactsInto(m Manifest, r Resolution, every, machines, accounts map[string]string, suffix string) ([]map[string]any, error) {
if len(m.Facts) == 0 {
return nil, nil
}
names := make([]string, 0, len(m.Facts))
for name := range m.Facts {
names = append(names, name)
}
sort.Strings(names)
view := rosterView{
Node: r.Node,
Suffix: strings.TrimPrefix(suffixOr(suffix), "."),
Names: entriesFrom(every, accounts, suffix),
Machines: entriesFrom(machines, accounts, suffix),
}
out := make([]map[string]any, 0, len(names))
for _, name := range names {
fact := m.Facts[name]
content, err := renderRoster(fact.Template, view)
if err != nil {
return nil, fmt.Errorf("%s cannot render %q: %w", m.Module, name, err)
}
// Where the file goes: under the operator's home and chowned to it (a home fact), or at the
// absolute system path it names. A home fact on a machine with no operator account cannot be
// placed, and is left out rather than written to nowhere (novox/hq to-be 29).
path := fact.Path
var owner string
if fact.Home {
if r.Account == "" {
continue
}
path = accountHomeOf(r.Account, r.AccountHome) + "/" + strings.TrimLeft(fact.Path, "/")
owner = r.Account
} else if !strings.HasPrefix(fact.Path, "/") {
return nil, fmt.Errorf(
"%s asks for %q at %q, which is not an absolute path", m.Module, name, fact.Path)
}
file := map[string]any{
"id": "fact-" + name, "type": "file", "path": path, "mode": "0644",
"content": content,
}
if owner != "" {
file["owner"] = owner
}
if fact.Shared {
// The host owns only the lines between `# BEGIN mesh <id>` and `# END mesh <id>` and
// keeps the rest of the file byte for byte; undeclared, the region goes and nothing else
// does (novox/hq issue 128). Every node on the private network receives this, so every
// node's host — the controller's own machine included — must be block-aware before a
// controller emitting it is rolled out: the order ADR 0102 set for `into: json`.
file["into"] = "block"
}
out = append(out, file)
}
return out, nil
}
// renderRoster runs a module's template over the roster. A template that will not parse, or reads
// a field the mesh does not have, is an error here — where the manifest is — rather than an empty
// file on a machine.
func renderRoster(tmpl string, view rosterView) (string, error) {
t, err := template.New("roster").Option("missingkey=error").Parse(tmpl)
if err != nil {
return "", err
}
var b bytes.Buffer
if err := t.Execute(&b, view); err != nil {
return "", err
}
return b.String(), nil
}
// entriesFrom is a name→address map as sorted roster entries.
//
// **A machine with no address is left out.** The mesh has a record for it — somebody added it —
// and does not yet know where it is, which is the ordinary state between adding a machine and it
// joining. Writing the name anyway would give a name that resolves to nothing, and a connection to
// that hangs; leaving it out fails at once and says the name is unknown.
func entriesFrom(addresses, accounts map[string]string, suffix string) []rosterEntry {
out := make([]rosterEntry, 0, len(addresses))
for _, name := range sortedNames(addresses) {
internal, bare := meshName(name, suffix)
// The account is looked up by whichever key the caller keys accounts on — the internal name
// or the bare one — so a template gets the right login however the maps were built.
account := accounts[name]
if account == "" {
account = accounts[bare]
}
out = append(out, rosterEntry{Name: bare, FQDN: internal, Address: addresses[name], Account: account})
}
return out
}
// meshName is a machine's internal name and its bare one, from either. The control plane keys
// the names it hands a resolution by the internal name (`homer.internal`), the same map a
// container gets as its hosts; a caller that keys by the bare name gets the same answer. The
// suffix is the one the control plane composed those names with, handed down rather than written
// here a second time — the alternative was `homer.internal.internal` on every machine.
func meshName(name, suffix string) (internal, bare string) {
dotted := "." + strings.TrimPrefix(suffixOr(suffix), ".")
if strings.HasSuffix(name, dotted) {
return name, strings.TrimSuffix(name, dotted)
}
return name + dotted, name
}
// suffixOr is the suffix given, or the one the mesh composes names with when none was handed down.
// The one place the default is written, so a fact and a name cannot disagree about it.
func suffixOr(suffix string) string {
if suffix == "" {
return "internal"
}
return suffix
}
func sortedNames(addresses map[string]string) []string {
out := make([]string, 0, len(addresses))
for name, at := range addresses {
// A machine the mesh cannot place is left out rather than named at nothing.
if at == "" {
continue
}
out = append(out, name)
}
sort.Strings(out)
return out
}
-260
View File
@@ -1,260 +0,0 @@
package catalogue
import (
"strings"
"testing"
)
// Keyed by the internal name, as the control plane hands them (issue 079). bart has no address —
// the ordinary state between adding a machine and it joining.
var threeMachines = map[string]string{"homer.internal": "10.42.0.1", "marge.internal": "10.42.0.2", "bart.internal": ""}
// A module says where it wants a roster file and in what format, and is given the rendered file.
func TestAModuleIsGivenTheFileItAskedFor(t *testing.T) {
m := Manifest{Module: "resolver", Facts: map[string]RosterFile{
"zones": {Path: "/etc/mesh/zones.conf", Template: "{{range .Machines}}address=/{{.FQDN}}/{{.Address}}\n{{end}}"},
}}
given, err := FactsInto(m, Resolution{Node: "homer"}, threeMachines, threeMachines, nil, "")
if err != nil {
t.Fatal(err)
}
if len(given) != 1 {
t.Fatalf("expected one file, got %d", len(given))
}
if given[0]["path"] != "/etc/mesh/zones.conf" || given[0]["type"] != "file" {
t.Fatalf("not written where it was asked for: %v", given[0])
}
// The id is fact-<name>, which is what a restart-on names when the roster changes.
if given[0]["id"] != "fact-zones" {
t.Fatalf("the file's id is not fact-<name>, so a restart-on cannot find it: %v", given[0]["id"])
}
if !strings.Contains(given[0]["content"].(string), "address=/homer.internal/10.42.0.1") {
t.Fatalf("the file does not hold the fact: %v", given[0]["content"])
}
}
// **A shared fact is written into a region of the machine's file, not over it** (novox/hq issue
// 128). A hosts file is the machine's — its localhost, the operator's lines, other tools' blocks —
// so the mesh owns only a marked region (`into: block`); a resolver's zones file is the mesh's
// whole, and carries no `into`.
func TestASharedFactIsWrittenIntoARegion(t *testing.T) {
roster := map[string]string{"homer.internal": "10.42.0.1"}
m := Manifest{Module: "net", Facts: map[string]RosterFile{
"node-names": {Path: "/etc/hosts", Template: "{{range .Names}}{{.FQDN}}\n{{end}}", Shared: true},
"node-zones": {Path: "/etc/zones", Template: "{{range .Machines}}{{.FQDN}}\n{{end}}"},
}}
given, err := FactsInto(m, Resolution{Node: "homer"}, roster, roster, nil, "")
if err != nil {
t.Fatal(err)
}
by := map[string]map[string]any{}
for _, f := range given {
by[f["path"].(string)] = f
}
if by["/etc/hosts"]["into"] != "block" {
t.Fatalf("a shared fact is not written into a region, so the mesh writes the file whole: %v", by["/etc/hosts"])
}
if _, has := by["/etc/zones"]["into"]; has {
t.Fatalf("an unshared fact was written into a region, so the mesh does not own its own file whole: %v", by["/etc/zones"])
}
}
// **The format is the module's — the mesh renders whatever template it gives.** The same roster
// through two templates is two entirely different files, and the control plane reads neither.
func TestTheFormatIsTheModulesOwn(t *testing.T) {
roster := map[string]string{"homer.internal": "10.42.0.1"}
hostsish := Manifest{Module: "a", Facts: map[string]RosterFile{
"f": {Path: "/f", Template: "{{range .Names}}{{.Address}}\t{{.Name}}\n{{end}}"}}}
sshish := Manifest{Module: "b", Facts: map[string]RosterFile{
"f": {Path: "/f", Template: "{{range .Names}}Host {{.Name}}\n HostName {{.FQDN}}\n{{end}}"}}}
h, err := FactsInto(hostsish, Resolution{Node: "homer"}, roster, roster, nil, "")
if err != nil {
t.Fatal(err)
}
s, err := FactsInto(sshish, Resolution{Node: "homer"}, roster, roster, nil, "")
if err != nil {
t.Fatal(err)
}
if h[0]["content"] != "10.42.0.1\thomer\n" {
t.Fatalf("the hosts-shaped template did not render its format: %q", h[0]["content"])
}
if s[0]["content"] != "Host homer\n HostName homer.internal\n" {
t.Fatalf("the ssh-shaped template did not render its format: %q", s[0]["content"])
}
}
// **A template that will not parse is refused here, not on a machine.** A daemon that starts, reads
// a file the mesh could not render, and answers nothing is a much worse way to find out.
func TestABrokenTemplateIsRefusedHere(t *testing.T) {
m := Manifest{Module: "resolver", Facts: map[string]RosterFile{
"zones": {Path: "/etc/zones", Template: "{{range .Machines}}oops"}}}
_, err := FactsInto(m, Resolution{}, nil, nil, nil, "")
if err == nil {
t.Fatal("a template that does not parse was accepted, so the machine gets an empty file")
}
if !strings.Contains(err.Error(), "resolver") || !strings.Contains(err.Error(), "zones") {
t.Fatalf("the refusal does not say whose template, or which: %v", err)
}
}
// A template reading something the mesh does not compute is refused, not rendered empty. The roster
// is a closed shape; asking it for the weather fails where the manifest is.
func TestATemplateReadingWhatTheMeshDoesNotHaveIsRefused(t *testing.T) {
m := Manifest{Module: "resolver", Facts: map[string]RosterFile{
"zones": {Path: "/etc/zones", Template: "{{.Weather}}"}}}
if _, err := FactsInto(m, Resolution{}, nil, nil, nil, ""); err == nil {
t.Fatal("a template read a field nobody computes and rendered anyway, silently")
}
}
// A relative path is refused, or a module decides where the mesh writes on a machine.
func TestAFactMustBeAskedForAtAnAbsolutePath(t *testing.T) {
m := Manifest{Module: "resolver", Facts: map[string]RosterFile{
"hosts": {Path: "etc/hosts", Template: "x"}}}
if _, err := FactsInto(m, Resolution{}, nil, nil, nil, ""); err == nil {
t.Fatal("a relative path was accepted")
}
}
// A machine the mesh has a record for and cannot place is left out of the roster.
//
// **Not an oversight — the alternative is worse.** A name written with no address resolves to
// nothing, and a connection to that hangs. Leaving it out fails at once and says the name is
// unknown, which is a thing somebody can act on.
func TestAMachineWithNoAddressIsNotInTheRoster(t *testing.T) {
m := Manifest{Module: "a", Facts: map[string]RosterFile{
"f": {Path: "/f", Template: "{{range .Machines}}{{.Name}}\n{{end}}"}}}
given, err := FactsInto(m, Resolution{Node: "homer"}, threeMachines, threeMachines, nil, "")
if err != nil {
t.Fatal(err)
}
if strings.Contains(given[0]["content"].(string), "bart") {
t.Fatalf("a machine with no address was in the roster, so its name resolves to nothing:\n%s", given[0]["content"])
}
}
// **The names the control plane hands a resolution are already internal names** — `homer.internal`,
// the same map every container gets as its hosts. A roster entry's FQDN is that name, not it with
// the suffix appended a second time; either key gives the same entries.
func TestNamesAreNotSuffixedTwice(t *testing.T) {
internal := map[string]string{"homer.internal": "10.42.0.1"}
bare := map[string]string{"homer": "10.42.0.1"}
tmpl := RosterFile{Path: "/f", Template: "{{range .Machines}}{{.FQDN}} {{.Name}}\n{{end}}"}
fromInternal, err := FactsInto(Manifest{Module: "a", Facts: map[string]RosterFile{"f": tmpl}}, Resolution{Node: "homer"}, internal, internal, nil, "")
if err != nil {
t.Fatal(err)
}
fromBare, err := FactsInto(Manifest{Module: "a", Facts: map[string]RosterFile{"f": tmpl}}, Resolution{Node: "homer"}, bare, bare, nil, "")
if err != nil {
t.Fatal(err)
}
if fromInternal[0]["content"] != fromBare[0]["content"] {
t.Fatalf("the roster differs by how the names were keyed:\n%q\n%q", fromInternal[0]["content"], fromBare[0]["content"])
}
got := fromInternal[0]["content"].(string)
if strings.Contains(got, "internal.internal") || !strings.Contains(got, "homer.internal homer") {
t.Fatalf("the entry carries a doubled suffix or the wrong bare name:\n%s", got)
}
}
// The suffix the control plane composed the names with is the one a template sees — an operator who
// chose another does not get `.internal`. `.Suffix` is the bare form, and FQDNs carry it.
func TestTheSuffixIsCarriedAsComposed(t *testing.T) {
names := map[string]string{"homer.lan": "10.42.0.1"}
m := Manifest{Module: "a", Facts: map[string]RosterFile{
"f": {Path: "/f", Template: "local=/{{.Suffix}}/\n{{range .Machines}}{{.FQDN}}\n{{end}}"}}}
given, err := FactsInto(m, Resolution{Node: "homer"}, names, names, nil, "lan")
if err != nil {
t.Fatal(err)
}
got := given[0]["content"].(string)
if !strings.Contains(got, "local=/lan/") || strings.Contains(got, "internal") {
t.Fatalf("the operator's suffix was not carried, so its names would be wrong:\n%s", got)
}
if !strings.Contains(got, "homer.lan") {
t.Fatalf("the FQDN does not carry the operator's suffix:\n%s", got)
}
}
// novox/hq 04-ISSUES/111: a template is given both sets and chooses. `.Names` is every name the mesh
// serves — the machines and the names it was told to route; `.Machines` is only the machines. A
// container's hosts wants every name so a routed name resolves to the machine serving it; a resolver
// told the suffix is its own wants only the machines, or a routed name written there with the suffix
// is a name nobody will ever ask for, standing beside the machines and looking as real.
func TestATemplateChoosesMachinesOrEveryName(t *testing.T) {
machines := map[string]string{"homer.internal": "10.42.0.1", "marge.internal": "10.42.0.2"}
every := map[string]string{
"homer.internal": "10.42.0.1", "marge.internal": "10.42.0.2",
"drive.example.test": "10.42.0.1", "git.example.test": "10.42.0.2",
}
m := Manifest{Module: "resolver", Facts: map[string]RosterFile{
"zones": {Path: "/etc/zones", Template: "{{range .Machines}}{{.FQDN}}\n{{end}}"},
"hosts": {Path: "/etc/hosts", Template: "{{range .Names}}{{.FQDN}}\n{{end}}"},
}}
given, err := FactsInto(m, Resolution{Node: "homer"}, every, machines, nil, "")
if err != nil {
t.Fatal(err)
}
by := map[string]string{}
for _, f := range given {
by[f["path"].(string)] = f["content"].(string)
}
zones := by["/etc/zones"]
for _, served := range []string{"drive.example.test", "git.example.test"} {
if strings.Contains(zones, served) {
t.Fatalf("a template over .Machines saw %q, a name the mesh serves rather than a machine:\n%s", served, zones)
}
}
if !strings.Contains(zones, "homer.internal") {
t.Fatalf("a template over .Machines did not see the machines:\n%s", zones)
}
hosts := by["/etc/hosts"]
for _, name := range []string{"homer.internal", "drive.example.test", "git.example.test"} {
if !strings.Contains(hosts, name) {
t.Fatalf("a template over .Names did not see %q, so a container would not resolve it:\n%s", name, hosts)
}
}
}
// A home fact is placed under the operator account's home and chowned to it, and its template sees
// each node's account (novox/hq to-be 29) — the ssh-client config is the case.
func TestAHomeFactIsPlacedUnderTheAccountsHomeAndOwnedByIt(t *testing.T) {
names := map[string]string{"homer.internal": "10.42.0.1", "marge.internal": "10.42.0.2"}
accounts := map[string]string{"homer": "jo", "marge": "jo"}
m := Manifest{Module: "ssh-client", Facts: map[string]RosterFile{
"ssh-config": {Path: ".ssh/config.d/mesh", Home: true,
Template: "{{range .Names}}Host {{.Name}}\n HostName {{.FQDN}}\n User {{.Account}}\n{{end}}"}}}
given, err := FactsInto(m, Resolution{Node: "homer", Account: "jo"}, names, names, accounts, "")
if err != nil {
t.Fatal(err)
}
f := given[0]
if f["path"] != "/home/jo/.ssh/config.d/mesh" {
t.Fatalf("the home fact was not placed under the account's home: %v", f["path"])
}
if f["owner"] != "jo" {
t.Fatalf("the home fact is not owned by the account: %v", f["owner"])
}
if !strings.Contains(f["content"].(string), "Host marge\n HostName marge.internal\n User jo") {
t.Fatalf("the config does not name the peer's account:\n%s", f["content"])
}
}
// A machine with no operator account gets no home fact — it cannot be placed, so it is left out
// rather than written to nowhere.
func TestAHomeFactIsSkippedWhereThereIsNoAccount(t *testing.T) {
names := map[string]string{"homer.internal": "10.42.0.1"}
m := Manifest{Module: "ssh-client", Facts: map[string]RosterFile{
"ssh-config": {Path: ".ssh/config", Home: true, Template: "x"}}}
given, err := FactsInto(m, Resolution{Node: "homer"}, names, names, nil, "")
if err != nil {
t.Fatal(err)
}
if len(given) != 0 {
t.Fatalf("a home fact was placed on a machine with no operator account: %v", given)
}
}

Some files were not shown because too many files have changed in this diff Show More