Compare commits
236
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c74d990cee | ||
|
|
35252af665 | ||
|
|
aab6ded41b | ||
|
|
aecac5bda2 | ||
|
|
30362118a1 | ||
|
|
81e76fa485 | ||
|
|
12ed35e87d | ||
|
|
525f10b858 | ||
|
|
0547316cf2 | ||
|
|
9fe9b5349c | ||
|
|
d1e488efaf | ||
|
|
c5dc7e732a | ||
|
|
7efcccd013 | ||
|
|
964285f08c | ||
|
|
5698dda11f | ||
|
|
6005a8471f | ||
|
|
e2ee0dfe98 | ||
|
|
70341cfbc7 | ||
|
|
77643aa3f4 | ||
|
|
1fd6194ff8 | ||
|
|
c37018fdd2 | ||
|
|
3907ea0db0 | ||
|
|
40f5e9a41c | ||
|
|
2c2eb51878 | ||
|
|
aa2d0b51ea | ||
|
|
64d154d9d7 | ||
|
|
ffa390f916 | ||
|
|
4d62e6caf1 | ||
|
|
386ae676ca | ||
|
|
f8a9c3d6bc | ||
|
|
83671fae5f | ||
|
|
9b715524a2 | ||
|
|
e06fc1ed16 | ||
|
|
13d7c5c5dd | ||
|
|
7b02feaebb | ||
|
|
bd10e2c695 | ||
|
|
4b209d944d | ||
|
|
84024cbdb6 | ||
|
|
ad2eed2f71 | ||
|
|
e8aa7ed9e7 | ||
|
|
337aaea123 | ||
|
|
4c41628b20 | ||
|
|
ae7fb520d7 | ||
|
|
f325073982 | ||
|
|
33c4e4be34 | ||
|
|
585a6abbdd | ||
|
|
8d52a2cfb0 | ||
|
|
1cfe6be9c4 | ||
|
|
4e4481b6f2 | ||
|
|
63ca073938 | ||
|
|
b244a768a3 | ||
|
|
81d52719f4 | ||
|
|
f6a93fe74c | ||
|
|
497f8ea567 | ||
|
|
dded086b54 | ||
|
|
e7b5100324 | ||
|
|
8ceec32692 | ||
|
|
5fcde512bc | ||
|
|
e5007a7daa | ||
|
|
cb77f35a27 | ||
|
|
71dbca6392 | ||
|
|
47d412e13f | ||
|
|
53e8f5bdd8 | ||
|
|
6ecd631b3e | ||
|
|
19d2725c13 | ||
|
|
8e2824201a | ||
|
|
75dab209d7 | ||
|
|
6da9a5478b | ||
|
|
ff51b329f6 | ||
|
|
2ec0fd218b | ||
|
|
e4e960ec1c | ||
|
|
cc69737934 | ||
|
|
0c83ecf1b5 | ||
|
|
05ff6065d0 | ||
|
|
a7df0fc62f | ||
|
|
1c56210530 | ||
|
|
d65c37caad | ||
|
|
eb72ec36ba | ||
|
|
4de10e32e3 | ||
|
|
f8ab9f2dcf | ||
|
|
ccf50b1269 | ||
|
|
a6d89e3f73 | ||
|
|
ee1b8ffe24 | ||
|
|
0560c792d8 | ||
|
|
966c5ddad3 | ||
|
|
b36f822cb6 | ||
|
|
7180a273a2 | ||
|
|
e65b3950cc | ||
|
|
a0e09695e5 | ||
|
|
88bef39952 | ||
|
|
06cf3c04e5 | ||
|
|
7a8a19b11b | ||
|
|
ce6ac057f6 | ||
|
|
abce68fdf0 | ||
|
|
cc019908c2 | ||
|
|
d0a9abcb1c | ||
|
|
63ba2d178f | ||
|
|
6c12780abe | ||
|
|
92d87b0082 | ||
|
|
2fad32767e | ||
|
|
51163b9f14 | ||
|
|
6557aca750 | ||
|
|
1801f1178e | ||
|
|
f21a84c510 | ||
|
|
e14b02991e | ||
|
|
bf82805cfd | ||
|
|
08ebb8c999 | ||
|
|
0a8a592ef4 | ||
|
|
173c8c7c21 | ||
|
|
98d348d5b9 | ||
|
|
7fc5fd02fd | ||
|
|
50878a9d98 | ||
|
|
cc252472e2 | ||
|
|
aa74bd86ca | ||
|
|
d2ab0b2b82 | ||
|
|
48d8c89749 | ||
|
|
7232d6df4b | ||
|
|
112cbd294d | ||
|
|
553814b6eb | ||
|
|
eeb0560fc2 | ||
|
|
1f36787d75 | ||
|
|
c753f9d5c0 | ||
|
|
2f7b407000 | ||
|
|
952092ccb3 | ||
|
|
fb87f9f7d6 | ||
|
|
ed08cc1adc | ||
|
|
50734095b8 | ||
|
|
95426e25cf | ||
|
|
fda47558d5 | ||
|
|
8ea80f9584 | ||
|
|
2e3b13c0f8 | ||
|
|
cd2481dcd8 | ||
|
|
d2cbc9dbdc | ||
|
|
e88d3bd485 | ||
|
|
a287812e14 | ||
|
|
557f419e71 | ||
|
|
71c8080359 | ||
|
|
cc86f8b433 | ||
|
|
0944311f86 | ||
|
|
7e42380dcd | ||
|
|
41de152739 | ||
|
|
dad0a153ff | ||
|
|
345722a4fd | ||
|
|
f145d17fc8 | ||
|
|
c3458a2546 | ||
|
|
f8919e2079 | ||
|
|
6ac9013d6e | ||
|
|
97448194ac | ||
|
|
5fad1f89cf | ||
|
|
7ffe6ce21b | ||
|
|
20e57c3f51 | ||
|
|
7bf23ea052 | ||
|
|
6da55bdfea | ||
|
|
752abaa81d | ||
|
|
2652287fe1 | ||
|
|
af26ed2e07 | ||
|
|
f996a6707e | ||
|
|
506426cf94 | ||
|
|
e11e1374bd | ||
|
|
008ce39ec0 | ||
|
|
856fabda04 | ||
|
|
8fa5443862 | ||
|
|
3ece1a86d7 | ||
|
|
dc8839246b | ||
|
|
524cc2a3ec | ||
|
|
f4bcb320fe | ||
|
|
caf9746759 | ||
|
|
6090953843 | ||
|
|
2277583e99 | ||
|
|
6bf42025e1 | ||
|
|
0d8264ff55 | ||
|
|
6073e94a4f | ||
|
|
4566c5c9aa | ||
|
|
7ef7669c0c | ||
|
|
cdd3638312 | ||
|
|
e07b56ce43 | ||
|
|
1b5ccf4165 | ||
|
|
26690d89f1 | ||
|
|
3c836f0abb | ||
|
|
8fb32d7ee0 | ||
|
|
7d6f37af54 | ||
|
|
58644fd282 | ||
|
|
91a41b7d20 | ||
|
|
f0049190d7 | ||
|
|
7352c846dd | ||
|
|
45425702d5 | ||
|
|
729537e745 | ||
|
|
0e413e3e7a | ||
|
|
252186d90c | ||
|
|
fad8b30e43 | ||
|
|
1096299e06 | ||
|
|
379f459498 | ||
|
|
d05a5e87af | ||
|
|
01814854b8 | ||
|
|
2cf8739a84 | ||
|
|
87ecc9326e | ||
|
|
0f3eedd163 | ||
|
|
dd6aad4a2f | ||
|
|
65187d4de0 | ||
|
|
2e6b9d30bd | ||
|
|
cc47330884 | ||
|
|
bfe4991dd7 | ||
|
|
689c2c6d33 | ||
|
|
1eff586a40 | ||
|
|
4bb19c9e40 | ||
|
|
9280513afa | ||
|
|
41c300eae0 | ||
|
|
c91fe1a6eb | ||
|
|
ba0f44a36d | ||
|
|
3dc7d0e386 | ||
|
|
ade6b2bfb6 | ||
|
|
a2dfaaf4d1 | ||
|
|
a82bfb41f2 | ||
|
|
8db66e9532 | ||
|
|
28b7fb81ba | ||
|
|
c93128d82f | ||
|
|
dbf62b5212 | ||
|
|
1ea84f8b8f | ||
|
|
28894fa5bd | ||
|
|
c3b1617693 | ||
|
|
a86a6c2974 | ||
|
|
1c32af6a22 | ||
|
|
0a39b7df82 | ||
|
|
4f3b4e6014 | ||
|
|
32b8af6a9b | ||
|
|
4567fa666c | ||
|
|
a3b7e830c8 | ||
|
|
1a41b88ed3 | ||
|
|
b9cdb96a90 | ||
|
|
3fd0c37d22 | ||
|
|
047830a6b5 | ||
|
|
ff26ea959d | ||
|
|
5150c0a0f8 | ||
|
|
6a64aba506 | ||
|
|
8152665298 | ||
|
|
b529c49cff |
@@ -86,13 +86,19 @@ proxy-image:
|
||||
|
||||
# The whole gate. Raises a database, runs everything against it, and takes it down again --
|
||||
# including when the tests fail, which is why the teardown is not conditional.
|
||||
#
|
||||
# **One package at a time (-p 1), and it is not about speed.** The live tests reach one bus, and on
|
||||
# it they assert, read and remove the mesh's own objects -- streams and consumers with fixed names,
|
||||
# because those names are the mesh's and a test cannot choose others. Two packages doing that at once
|
||||
# is one deleting a consumer the other is reading through, and the failure lands in whichever test
|
||||
# was reading, as "no response from stream". That reads as a bug in the code under test.
|
||||
check: fmt vet postgres
|
||||
@go test ./... ; status=$$? ; $(MAKE) postgres-stop ; exit $$status
|
||||
@go test -p 1 ./... ; status=$$? ; $(MAKE) postgres-stop ; exit $$status
|
||||
|
||||
# Without a database the live tests skip rather than fail, so this is the honest subset and not
|
||||
# the gate.
|
||||
# the gate. Serialised for the same reason check is: a bus may be configured even when a store is not.
|
||||
test:
|
||||
go test ./...
|
||||
go test -p 1 ./...
|
||||
|
||||
vet:
|
||||
go vet ./...
|
||||
|
||||
+108
-168
@@ -17,21 +17,15 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
amqp "github.com/rabbitmq/amqp091-go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
@@ -51,7 +45,6 @@ const usage = `mesh-builder — builds modules for the mesh
|
||||
It consumes build requests and answers with what it made. Nothing is listened on and nothing
|
||||
is dialled except the broker.
|
||||
|
||||
MESH_BROKER_AMQP where the broker is, with this builder's own credential
|
||||
MESH_BROKER_FILE a file the mesh sealed to this machine holding the same
|
||||
MESH_REGISTRY host:port to publish artifacts to, when the mesh has not said
|
||||
MESH_BINDING a file the mesh wrote saying where the artifact store is
|
||||
@@ -114,72 +107,48 @@ func run() error {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
conn, err := dial(credential)
|
||||
if err != nil {
|
||||
// Not quoted back: the URL carries this builder's broker password.
|
||||
return fmt.Errorf("cannot reach the broker: %w", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
channel, err := conn.Channel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer channel.Close()
|
||||
|
||||
if _, err := channel.QueueDeclare(link.BuildQueue, true, false, false, false, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
// One at a time. A build machine that took five requests at once would run five container
|
||||
// builds against one runtime and finish all of them slower than it would have finished the
|
||||
// first — and the queue is what shares work between machines, so nothing is lost by it.
|
||||
if err := channel.Qos(1, 0, false); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Not auto-acknowledged. A request acknowledged on arrival is a build that vanishes if this
|
||||
// process dies mid-way, with nobody waiting on it ever hearing why.
|
||||
requests, err := channel.ConsumeWithContext(ctx, link.BuildQueue, "mesh-builder",
|
||||
false, false, false, false, nil)
|
||||
machine, err := takeWorkFrom(credential, on)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer machine.Close()
|
||||
|
||||
fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry)
|
||||
publisher := builder.Registry{Address: registry, Run: builder.Command}
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
fmt.Println("stopping")
|
||||
return nil
|
||||
case delivery, ok := <-requests:
|
||||
if !ok {
|
||||
return fmt.Errorf("the broker closed the connection")
|
||||
}
|
||||
answer(ctx, channel, publisher, on, workspace, delivery)
|
||||
}
|
||||
return machine.Take(ctx, func(ctx context.Context, work link.Build) {
|
||||
answer(ctx, publisher, on, workspace, work)
|
||||
})
|
||||
}
|
||||
|
||||
// takeWorkFrom opens this machine's link to whichever bus the mesh is on.
|
||||
//
|
||||
// **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5): a build
|
||||
// machine told about both would take work from one and answer on the other, and every log line would
|
||||
// say it was fine.
|
||||
func takeWorkFrom(credential Credential, on string) (link.BuildMachine, error) {
|
||||
// **The credential names the bus, and there is one** (novox/hq ADR 0131, design 28 task 5.5).
|
||||
// A credential for the mesh's bus carries user, password and fingerprint beside the address,
|
||||
// and that is enough to dial it, pinned.
|
||||
if !credential.onTheNewBus() {
|
||||
return nil, fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL)
|
||||
}
|
||||
js, err := broker.DialPinned(credential.natsURL(), credential.Fingerprint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return link.MachineOverNATS(js, on), nil
|
||||
}
|
||||
|
||||
// answer does one build and says what happened, whichever way it went.
|
||||
func answer(ctx context.Context, channel *amqp.Channel, publisher builder.Publisher,
|
||||
on, workspace string, delivery amqp.Delivery) {
|
||||
func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build) {
|
||||
request := work.Request()
|
||||
|
||||
// **First thing, and to stdout.** A build request that arrives and produces no visible line
|
||||
// until it either finishes or fails is indistinguishable from one that never arrived — which
|
||||
// cost a long diagnosis against a running mesh, chasing "the handler never fired" when the
|
||||
// truth was only that the handler said nothing until the end.
|
||||
fmt.Fprintf(os.Stderr, "a build request arrived (%d bytes)\n", len(delivery.Body))
|
||||
|
||||
var request link.BuildRequest
|
||||
if err := json.Unmarshal(delivery.Body, &request); err != nil {
|
||||
// Unreadable. Acknowledged and dropped rather than requeued: a message this builder
|
||||
// cannot parse will not become parseable by being delivered again, and requeueing it
|
||||
// would put it in front of every real request for ever.
|
||||
fmt.Fprintf(os.Stderr, "a request could not be read and was dropped: %v\n", err)
|
||||
_ = delivery.Ack(false)
|
||||
return
|
||||
}
|
||||
// **First thing, and to stdout.** A build request that arrives and produces no visible line until
|
||||
// it either finishes or fails is indistinguishable from one that never arrived — which cost a long
|
||||
// diagnosis against a running mesh, chasing "the handler never fired" when the truth was only that
|
||||
// the handler said nothing until the end.
|
||||
fmt.Fprintf(os.Stderr, "a build request arrived for %s\n", request.Repository)
|
||||
|
||||
result := link.BuildResult{
|
||||
ID: request.ID, Repository: request.Repository, Path: request.Path,
|
||||
@@ -198,10 +167,11 @@ func answer(ctx context.Context, channel *amqp.Channel, publisher builder.Publis
|
||||
var built builder.Result
|
||||
if err == nil {
|
||||
// The package-registry credential is a build input, so it is resolved before the clone: a
|
||||
// build that could not have resolved its dependencies is refused in front of the reason,
|
||||
// not after a clone that then fails at npm ci.
|
||||
// build that could not have resolved its dependencies is refused in front of the reason, not
|
||||
// after a clone that then fails at npm ci.
|
||||
built, err = builder.Build(ctx, builder.Command, publisher,
|
||||
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
||||
forgeFrom(),
|
||||
func(step, message string) {
|
||||
fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message)
|
||||
})
|
||||
@@ -228,67 +198,19 @@ func answer(ctx context.Context, channel *amqp.Channel, publisher builder.Publis
|
||||
}
|
||||
}
|
||||
|
||||
body, err := json.Marshal(result)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "cannot report a build: %v\n", err)
|
||||
_ = delivery.Ack(false)
|
||||
if err := work.Announce(ctx, result); err != nil {
|
||||
// Said, not fatal: the build happened. A build reported as failed because announcing it
|
||||
// failed is a lie about work that was done — and the request stays unsettled below only if
|
||||
// nothing was said at all, so another machine can try.
|
||||
fmt.Fprintf(os.Stderr, "cannot say what came of a build: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Always through the exchange, whether or not somebody is waiting.
|
||||
//
|
||||
// **Never the default exchange.** Permission there is granted per exchange rather than per
|
||||
// queue, so a builder allowed to use it could publish into any node's queue — the privilege a
|
||||
// build machine most obviously should not have. An asker binds its own reply queue to this
|
||||
// key and filters by correlation; a control plane that records builds is bound to it too, so
|
||||
// a result nobody asked for is still kept rather than reported into the void.
|
||||
publishCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
if err := channel.PublishWithContext(publishCtx, link.Exchange, link.KeyBuilt, false, false,
|
||||
amqp.Publishing{
|
||||
ContentType: "application/json",
|
||||
CorrelationId: result.ID,
|
||||
Body: body,
|
||||
}); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "cannot answer a build request: %v\n", err)
|
||||
// Settled only once the outcome is away, so a machine that dies before answering leaves the work
|
||||
// for another rather than losing it.
|
||||
if err := work.Done(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "the outcome is away and the request could not be settled: %v\n", err)
|
||||
}
|
||||
// **And announced, which is a different act from answering.** The reply goes to whoever asked
|
||||
// and is correlated to their request; this says to the whole mesh that a module now exists at
|
||||
// a commit, and the catalogue places it in the module graph (novox/hq ADR 0072). A build
|
||||
// nobody asked for still has to be announced, or the graph knows less than the registry does.
|
||||
//
|
||||
// Only on success: a failed build produced no module-version, and announcing one would put
|
||||
// something in the graph that was never made.
|
||||
if result.Failed == "" && result.Commit != "" {
|
||||
announced := map[string]any{
|
||||
"module": moduleOf(result.Manifest), "commit": result.Commit,
|
||||
"repository": result.Repository, "path": result.Path, "ref": result.Ref,
|
||||
"manifest": json.RawMessage(result.Manifest), "against": result.Against,
|
||||
"made": result.Made,
|
||||
}
|
||||
if err := link.EmitEvent(publishCtx, channel, link.KeyModuleBuilt, "builder", on, announced); err != nil {
|
||||
// Said, not fatal: the build happened and was answered. A module the catalogue has not
|
||||
// heard of is a gap somebody can close; a build reported as failed because announcing
|
||||
// it failed is a lie about work that was done.
|
||||
fmt.Fprintf(os.Stderr, " built, but could not announce it: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Acknowledged only once the answer is away, so a builder that dies before answering leaves
|
||||
// the request for another machine rather than losing it.
|
||||
_ = delivery.Ack(false)
|
||||
}
|
||||
|
||||
// moduleOf reads the module's name out of the manifest it just built, which is the only place it is
|
||||
// authoritative — the request named a repository and a path, not a module.
|
||||
func moduleOf(manifest json.RawMessage) string {
|
||||
var named struct {
|
||||
Module string `json:"module"`
|
||||
}
|
||||
if err := json.Unmarshal(manifest, &named); err != nil {
|
||||
return ""
|
||||
}
|
||||
return named.Module
|
||||
}
|
||||
|
||||
// packagesFrom is where a build resolves the mesh's own published packages — the SDK above all
|
||||
@@ -367,6 +289,53 @@ func packagesFrom() (builder.Npmrc, error) {
|
||||
return builder.Npmrc{Scope: scope, Registry: registry, Token: secret}, nil
|
||||
}
|
||||
|
||||
// forgeFrom is the git credential this builder may offer a clone, composed from the same binding
|
||||
// and sealed secret its package-registry half already reads: the forge that answers npm is the
|
||||
// forge that hosts the repositories, and its provisioner applies one password to one user for
|
||||
// both. Anything missing means no credential, and every clone stays anonymous — which is all a
|
||||
// mesh of public repositories ever needs.
|
||||
//
|
||||
// The URL names the binding's own address — the machine the mesh says the forge is on — so a
|
||||
// private repository is registered and built by that address, and a clone of anything else is
|
||||
// never shown this credential (git's credential store matches the whole origin).
|
||||
func forgeFrom() builder.GitCredential {
|
||||
path := strings.TrimSpace(os.Getenv("MESH_PACKAGE_BINDING"))
|
||||
if path == "" {
|
||||
return builder.GitCredential{}
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return builder.GitCredential{}
|
||||
}
|
||||
var told struct {
|
||||
At string `json:"at"`
|
||||
As string `json:"as"`
|
||||
Serves map[string]any `json:"serves"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &told); err != nil || told.At == "" || told.As == "" {
|
||||
return builder.GitCredential{}
|
||||
}
|
||||
secret := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN"))
|
||||
if file := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN_FILE")); file != "" {
|
||||
if raw, err := os.ReadFile(file); err == nil {
|
||||
secret = strings.TrimSpace(string(raw))
|
||||
}
|
||||
}
|
||||
if secret == "" {
|
||||
return builder.GitCredential{}
|
||||
}
|
||||
scheme := "https"
|
||||
if s, ok := told.Serves["scheme"]; ok {
|
||||
scheme = fmt.Sprintf("%v", s)
|
||||
}
|
||||
host := told.At
|
||||
if port, ok := told.Serves["port"]; ok {
|
||||
host = fmt.Sprintf("%s:%v", told.At, port)
|
||||
}
|
||||
made := url.URL{Scheme: scheme, User: url.UserPassword(told.As, secret), Host: host}
|
||||
return builder.GitCredential{URL: made.String()}
|
||||
}
|
||||
|
||||
func short(commit string) string {
|
||||
if len(commit) > 8 {
|
||||
return commit[:8]
|
||||
@@ -450,13 +419,8 @@ func brokerFrom() (Credential, error) {
|
||||
// broker is then verified against whatever this machine already trusts.
|
||||
return Credential{URL: said}, nil
|
||||
}
|
||||
url := strings.TrimSpace(os.Getenv("MESH_BROKER_AMQP"))
|
||||
if url == "" {
|
||||
return Credential{}, fmt.Errorf(
|
||||
"neither MESH_BROKER_FILE nor MESH_BROKER_AMQP: a builder with no broker has " +
|
||||
"nothing to build")
|
||||
}
|
||||
return Credential{URL: url}, nil
|
||||
return Credential{}, fmt.Errorf(
|
||||
"no MESH_BROKER_FILE: a build machine with no credential for the bus has nothing to build")
|
||||
}
|
||||
|
||||
// Credential is what a build machine is given so it can reach the broker.
|
||||
@@ -468,48 +432,24 @@ func brokerFrom() (Credential, error) {
|
||||
// **The same shape a node gets, for the same reason** (novox/hq ADR 0004): the fingerprint travels
|
||||
// out of band — here, sealed with the credential — and the endpoint is verified once at connect.
|
||||
type Credential struct {
|
||||
URL string `json:"url"`
|
||||
// Fingerprint is SHA-256 over the broker certificate's DER bytes, or empty to verify the
|
||||
// ordinary way.
|
||||
URL string `json:"url"`
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
// User and Password ride beside the address on the bus being built (design 25): a credential
|
||||
// embedded in a URL leaks into every log line that prints a connection, so the mesh seals them
|
||||
// as two fields and this machine joins them once, here, to dial.
|
||||
User string `json:"user,omitempty"`
|
||||
Password string `json:"password,omitempty"`
|
||||
}
|
||||
|
||||
// dial opens the connection, pinning the broker's certificate when there is one to pin.
|
||||
func dial(held Credential) (*amqp.Connection, error) {
|
||||
if held.Fingerprint == "" {
|
||||
return amqp.Dial(held.URL)
|
||||
}
|
||||
return amqp.DialTLS(held.URL, pinning(held.Fingerprint))
|
||||
}
|
||||
// onTheNewBus is whether a credential is for the bus being built: its address says so, and the
|
||||
// mesh only ever seals such a credential with the user and password beside it.
|
||||
func (c Credential) onTheNewBus() bool { return strings.HasPrefix(strings.TrimSpace(c.URL), "nats://") }
|
||||
|
||||
// pinning is a TLS configuration that trusts exactly one certificate.
|
||||
//
|
||||
// InsecureSkipVerify with a VerifyPeerCertificate is **pinning, not skipping**: the standard chain
|
||||
// check is replaced, not removed, and what replaces it is stricter — one certificate is accepted
|
||||
// rather than every certificate a public authority would sign.
|
||||
//
|
||||
// Its own function so a test can drive it against a real handshake. A pin check that is only ever
|
||||
// exercised through a broker is a pin check nothing tests.
|
||||
func pinning(fingerprint string) *tls.Config {
|
||||
return &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
VerifyPeerCertificate: func(raw [][]byte, _ [][]*x509.Certificate) error {
|
||||
if len(raw) == 0 {
|
||||
return errors.New("the broker presented no certificate")
|
||||
}
|
||||
// The leaf, and in the same spelling the mesh writes it — `sha256:` and 64 hex
|
||||
// characters. Comparing a bare digest against a written fingerprint never matches,
|
||||
// and the failure is indistinguishable from being pointed at the wrong broker.
|
||||
sum := sha256.Sum256(raw[0])
|
||||
got := "sha256:" + hex.EncodeToString(sum[:])
|
||||
if got != fingerprint {
|
||||
return fmt.Errorf(
|
||||
"this is not the broker this builder was told about\n expected %s\n "+
|
||||
"got %s\nEither this mesh's broker was replaced, or this builder is "+
|
||||
"being pointed at something else. Retrying will not help",
|
||||
fingerprint, got)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
// natsURL is the address with this machine's credential in it, for the one dial that needs it.
|
||||
func (c Credential) natsURL() string {
|
||||
rest := strings.TrimPrefix(strings.TrimSpace(c.URL), "nats://")
|
||||
if c.User == "" {
|
||||
return "nats://" + rest
|
||||
}
|
||||
return "nats://" + c.User + ":" + c.Password + "@" + rest
|
||||
}
|
||||
|
||||
@@ -89,6 +89,7 @@ func buildOnce(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
built, buildErr := builder.Build(ctx, builder.Command, publisher, repository, *path, *ref, where, bases, npmrc,
|
||||
forgeFrom(),
|
||||
func(step, message string) { fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message) })
|
||||
if buildErr != nil {
|
||||
return buildErr
|
||||
|
||||
@@ -14,8 +14,8 @@ func TestBuilderDiagnosticsStayOffStdout(t *testing.T) {
|
||||
allowed := map[string]bool{
|
||||
"string(body)": true, // once.go: the result JSON, which IS stdout
|
||||
"version)": true, // --version
|
||||
`"stopping")`: true, // the loop.s shutdown line
|
||||
"usage)": true, // --help text, for a human
|
||||
`"stopping")`: true, // the loop.s shutdown line
|
||||
"usage)": true, // --help text, for a human
|
||||
}
|
||||
for _, file := range []string{"once.go", "main.go"} {
|
||||
src, err := os.ReadFile(file)
|
||||
|
||||
@@ -15,6 +15,8 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// Where a builder publishes.
|
||||
@@ -193,9 +195,9 @@ func TestThePinIsComparedInTheSpellingTheMeshWritesIt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// handshakeWith runs the builder's own pin check against an address.
|
||||
// handshakeWith runs the pin check the builder dials with against an address.
|
||||
func handshakeWith(address, pin string) error {
|
||||
conn, err := tls.Dial("tcp", address, pinning(pin))
|
||||
conn, err := tls.Dial("tcp", address, broker.PinnedToFingerprint(pin))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -39,9 +39,23 @@ import (
|
||||
// It costs a resolution per machine. Assignment is a person typing a command, and being told which
|
||||
// machines this just blocked is worth more than the milliseconds.
|
||||
func assign(ctx context.Context, open *stores, node, module string) (string, error) {
|
||||
if err := open.inventory.Assign(ctx, node, module); err != nil {
|
||||
// Held while it is recorded, so it cannot land between a converge's preview and its flip and
|
||||
// be taken without ever having been previewed (novox/hq ADR 0100).
|
||||
ctx, release, err := holdNodes(ctx, open, []string{node})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer release()
|
||||
fresh, err := open.inventory.Assign(ctx, node, module)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !fresh {
|
||||
// Nothing changed, and saying "is assigned" would read as an action. One node runs one
|
||||
// of each — the module's name is the assignment's identity (novox/hq ADR 0115).
|
||||
return fmt.Sprintf("%s already runs %s — one node runs one of each (ADR 0115); nothing changed",
|
||||
node, module), nil
|
||||
}
|
||||
said := fmt.Sprintf("%s is assigned %s", node, module)
|
||||
plan, _, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
@@ -71,6 +85,11 @@ func assign(ctx context.Context, open *stores, node, module string) (string, err
|
||||
// module off one machine is the ordinary way to stop providing something to another, and nothing
|
||||
// about the command's own output would ever have said so.
|
||||
func unassign(ctx context.Context, open *stores, node, module string) (string, error) {
|
||||
ctx, release, err := holdNodes(ctx, open, []string{node})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer release()
|
||||
if err := open.inventory.Unassign(ctx, node, module); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,335 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/licences"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// An address is read from the node's settings where it is used, never recorded with a port
|
||||
// (novox/hq 04-ISSUES/102). Three readers did not follow the setting; each is held to it here.
|
||||
|
||||
var aDigest = "sha256:" + strings.Repeat("e", 64)
|
||||
|
||||
// **A build is recorded by digest and path**, whatever address the builder pushed to — and only
|
||||
// what the build made is rewritten: an image the module runs from elsewhere is left where it says.
|
||||
func TestABuildIsRecordedWithoutTheStoresAddress(t *testing.T) {
|
||||
manifest, _ := json.Marshal(map[string]any{
|
||||
"module": "gitea", "version": "1",
|
||||
"resources": []map[string]any{
|
||||
{"id": "server", "type": "container", "name": "mesh-gitea",
|
||||
"image": "anchor.internal:5100/gitea/server@" + aDigest},
|
||||
{"id": "config", "type": "archive", "path": "/etc/gitea", "digest": aDigest,
|
||||
"source": "http://anchor.internal:5100/v2/gitea/config/blobs/" + aDigest},
|
||||
{"id": "cache", "type": "container", "name": "mesh-gitea-cache",
|
||||
"image": "valkey/valkey@" + aDigest},
|
||||
},
|
||||
})
|
||||
kept := buildFrom(link.BuildResult{
|
||||
ID: "b1", Repository: "https://forge.example/gitea.git", Commit: "abc", On: "laptop",
|
||||
Manifest: manifest,
|
||||
Made: []link.MadeArtifact{
|
||||
{Name: "server", Kind: "image", Reference: "anchor.internal:5100/gitea/server@" + aDigest},
|
||||
{Name: "config", Kind: "archive", Reference: "http://anchor.internal:5100/v2/gitea/config/blobs/" + aDigest},
|
||||
},
|
||||
Against: []string{"anchor.internal:5100/mesh-tools/runtime@" + aDigest},
|
||||
})
|
||||
if kept.Module != "gitea" {
|
||||
t.Fatalf("the module was not read from the recorded manifest: %q", kept.Module)
|
||||
}
|
||||
if kept.Made[0].Reference != catalogue.ArtifactStoreScheme+"gitea/server@"+aDigest {
|
||||
t.Errorf("the image is recorded as %q, address and all", kept.Made[0].Reference)
|
||||
}
|
||||
if kept.Made[1].Reference != catalogue.ArtifactStoreScheme+"gitea/config/blobs/"+aDigest {
|
||||
t.Errorf("the archive is recorded as %q, address and all", kept.Made[1].Reference)
|
||||
}
|
||||
recorded, err := catalogue.ParseManifest(kept.Manifest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := recorded.Resources[0]["image"]; got != catalogue.ArtifactStoreScheme+"gitea/server@"+aDigest {
|
||||
t.Errorf("the recorded manifest's image is %v", got)
|
||||
}
|
||||
if got := recorded.Resources[1]["source"]; got != catalogue.ArtifactStoreScheme+"gitea/config/blobs/"+aDigest {
|
||||
t.Errorf("the recorded manifest's archive is %v", got)
|
||||
}
|
||||
if got := recorded.Resources[2]["image"]; got != "valkey/valkey@"+aDigest {
|
||||
t.Errorf("an image the build did not make was rewritten: %v", got)
|
||||
}
|
||||
if strings.Contains(string(kept.Manifest), "anchor.internal:5100") {
|
||||
t.Errorf("the recorded manifest still carries the store's address:\n%s", kept.Manifest)
|
||||
}
|
||||
if kept.Against[0] != "anchor.internal:5100/mesh-tools/runtime@"+aDigest {
|
||||
t.Errorf("what the build stood on was rewritten: %v", kept.Against)
|
||||
}
|
||||
}
|
||||
|
||||
// aStore is a module offering the artifact store on 5000, published the long way as the
|
||||
// distribution module does, so a node may be given another number for it.
|
||||
func aStore() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "distribution", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: catalogue.ArtifactStoreProvision, Scope: catalogue.ScopeMesh}},
|
||||
Serves: map[string]map[string]any{catalogue.ArtifactStoreProvision: {"port": float64(5000)}},
|
||||
Listens: []catalogue.Listening{{Port: 5000, From: catalogue.FromMesh}},
|
||||
Resources: []map[string]any{{"id": "store", "type": "container", "name": "mesh-registry",
|
||||
"ports": []any{"5000:5000"}, "image": "registry@" + aDigest}}}
|
||||
}
|
||||
|
||||
// **The trust a machine writes for the store, and the address every built image is fetched
|
||||
// through, say the port the node gave the store** — not the catalogue's number.
|
||||
func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aStore())
|
||||
if _, err := assign(ctx, open, "anchor", "distribution"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "distribution",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5101}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A module the mesh built, recorded by digest and path, running on the other machine.
|
||||
if err := open.inventory.RecordBuild(ctx, inventory.Build{
|
||||
ID: "b1", Repository: "r", Module: "app", Commit: "abc",
|
||||
Made: []inventory.Artifact{{Name: "server", Kind: "image",
|
||||
Reference: catalogue.ArtifactStoreScheme + "app/server@" + aDigest}},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "app", Version: "1",
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-app",
|
||||
"image": catalogue.ArtifactStoreScheme + "app/server@" + aDigest}}})
|
||||
if _, err := assign(ctx, open, "laptop", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
on := map[string]bool{"anchor": true, "laptop": true}
|
||||
node, port, found, err := artifactStoreOnNetwork(ctx, open.inventory, on)
|
||||
if err != nil || !found || node != "anchor" || port != "5101" {
|
||||
t.Fatalf("the store is found on %q:%q (%v, %v); the node put it on 5101", node, port, found, err)
|
||||
}
|
||||
|
||||
var trust string
|
||||
for _, r := range composed(t, open, "laptop").Resources {
|
||||
if r["path"] == "/etc/docker/daemon.json" {
|
||||
trust, _ = r["content"].(string)
|
||||
}
|
||||
if r["id"] == "app.server" && r["image"] != "anchor.internal:5101/app/server@"+aDigest {
|
||||
t.Errorf("the image the mesh built is fetched as %v", r["image"])
|
||||
}
|
||||
}
|
||||
if !strings.Contains(trust, "anchor.internal:5101") || strings.Contains(trust, ":5000") {
|
||||
t.Fatalf("the runtime is told to trust %q; the node put the store on 5101", trust)
|
||||
}
|
||||
|
||||
// And a replay to the catalogue says where the store is now.
|
||||
announced, err := following{open}.Announceable(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(announced) != 1 || announced[0].Made[0].Reference != "anchor.internal:5101/app/server@"+aDigest {
|
||||
t.Fatalf("the replay announces %+v", announced)
|
||||
}
|
||||
}
|
||||
|
||||
// **The control plane's own connections say the port the node gave the store and the broker.**
|
||||
//
|
||||
// Composed from the control plane's own manifest against a real inventory: the store's module is
|
||||
// given 6852 on this node the way genesis or an operator gives it, and the control plane's
|
||||
// container is told so beside the sealed connection genesis wrote.
|
||||
func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
control, err := withSeatPorts(m).Resolve([]catalogue.Built{{Name: "server", Kind: catalogue.ArtifactImage,
|
||||
Reference: "registry.example/control@" + aDigest}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, control)
|
||||
register(t, open, catalogue.Manifest{Module: "postgres", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}},
|
||||
Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store",
|
||||
"ports": []any{"5432:5432"}, "image": "pg@" + aDigest}}})
|
||||
register(t, open, catalogue.Manifest{Module: "lavinmq", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}},
|
||||
Listens: []catalogue.Listening{{Port: 5671, From: catalogue.FromMesh},
|
||||
{Port: 5672, From: catalogue.FromMesh}},
|
||||
Guards: []int{15672},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-broker",
|
||||
"ports": []any{"5671:5671", "5672:5672", "127.0.0.1:15672:15672"}, "image": "mq@" + aDigest}}})
|
||||
if _, err := assign(ctx, open, "anchor", "mesh-controller"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The store's module is registered and given its port, and NOT assigned: the state genesis
|
||||
// leaves a given-port node in before the foundation is adopted as modules (04-ISSUES/085).
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "postgres",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5432": 6852}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "anchor", "lavinmq"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "lavinmq",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5672": 5679}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var env map[string]any
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
if r["id"] == "mesh-controller.server" {
|
||||
env, _ = r["env"].(map[string]any)
|
||||
}
|
||||
}
|
||||
if env == nil {
|
||||
t.Fatal("the control plane's container is not in its own node's declaration")
|
||||
}
|
||||
for key, want := range map[string]string{
|
||||
"MESH_STORE_INVENTORY_PORT": "6852",
|
||||
"MESH_STORE_IDENTITY_PORT": "6852",
|
||||
"MESH_STORE_LICENCES_PORT": "6852",
|
||||
"MESH_BROKER_AMQP_PORT": "5679",
|
||||
// Neither given nor assigned by the mesh: the manifest's own number is NOT the answer,
|
||||
// because the sealed value beside it carries the port genesis wrote (finding F3).
|
||||
"MESH_BROKER_ADDRESS_PORT": "",
|
||||
"MESH_BROKER_MANAGEMENT_PORT": "",
|
||||
} {
|
||||
if env[key] != want {
|
||||
t.Errorf("the control plane is told %s=%v; the node says %q", key, env[key], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// withSeatPorts is the control plane's manifest with the seat placeholders in its environment —
|
||||
// added here until module.json carries them (the manifest lands one commit after the code that
|
||||
// fills it, so a control plane one build behind never sees a placeholder it cannot fill).
|
||||
func withSeatPorts(m catalogue.Manifest) catalogue.Manifest {
|
||||
seatPorts := map[string]string{
|
||||
"MESH_STORE_INVENTORY_PORT": "${seat:mesh-store:5432}",
|
||||
"MESH_STORE_IDENTITY_PORT": "${seat:mesh-store:5432}",
|
||||
"MESH_STORE_LICENCES_PORT": "${seat:mesh-store:5432}",
|
||||
"MESH_BROKER_AMQP_PORT": "${seat:mesh-broker:5672}",
|
||||
"MESH_BROKER_MANAGEMENT_PORT": "${seat:mesh-broker:15672}",
|
||||
"MESH_BROKER_ADDRESS_PORT": "${seat:mesh-broker:5671}",
|
||||
}
|
||||
out := m
|
||||
out.Resources = nil
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] != "container" {
|
||||
out.Resources = append(out.Resources, r)
|
||||
continue
|
||||
}
|
||||
copied := map[string]any{}
|
||||
for k, v := range r {
|
||||
copied[k] = v
|
||||
}
|
||||
env := map[string]any{}
|
||||
if had, ok := r["env"].(map[string]any); ok {
|
||||
for k, v := range had {
|
||||
env[k] = v
|
||||
}
|
||||
}
|
||||
for k, v := range seatPorts {
|
||||
if _, said := env[k]; !said {
|
||||
env[k] = v
|
||||
}
|
||||
}
|
||||
copied["env"] = env
|
||||
out.Resources = append(out.Resources, copied)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// aLoneNode is one capable machine with nothing placed on any network — the control-node during
|
||||
// genesis, before the "network" step, which is after the store, the broker, the vault and the
|
||||
// catalogue have each been built and pushed (finding F2).
|
||||
func aLoneNode(t *testing.T) *stores {
|
||||
t.Helper()
|
||||
inventory.ForTest(t)
|
||||
licences.ForTest(t)
|
||||
open, err := openStores(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(open.Close)
|
||||
for _, m := range provided {
|
||||
if err := open.inventory.Provide(t.Context(), m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
record, err := open.inventory.AddNode(t.Context(), "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reported, _ := json.Marshal(map[string]any{"capabilities": []map[string]any{
|
||||
{"name": "container-runtime", "present": true}}})
|
||||
var profile map[string]any
|
||||
_ = json.Unmarshal(reported, &profile)
|
||||
if err := open.inventory.RecordProfile(t.Context(), record.ID, profile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSealingKey(t.Context(), record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return open
|
||||
}
|
||||
|
||||
// **Before the network exists, the store's own node reaches it by loopback** — never refused,
|
||||
// never handed the scheme: a genesis pushes the store, the broker, the vault and the catalogue to
|
||||
// a node on no network, and builds the catalogue on a base it must be able to pull.
|
||||
func TestOnANodeWithNoNetworkTheStoreIsReachedByLoopback(t *testing.T) {
|
||||
open := aLoneNode(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aStore())
|
||||
register(t, open, catalogue.Manifest{Module: "builder", Version: "1",
|
||||
Requires: []string{catalogue.ArtifactStoreProvision},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-builder",
|
||||
"image": "registry.example/mesh-builder@" + aDigest}}})
|
||||
if err := open.inventory.RecordBuild(ctx, inventory.Build{
|
||||
ID: "b1", Repository: "r", Module: "postgres", Commit: "abc",
|
||||
Made: []inventory.Artifact{{Name: "runtime", Kind: "image",
|
||||
Reference: catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "postgres", Version: "1",
|
||||
Resources: []map[string]any{{"id": "runtime", "type": "container", "name": "mesh-postgres",
|
||||
"image": catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}}})
|
||||
for _, module := range []string{"distribution", "builder", "postgres"} {
|
||||
if _, err := assign(ctx, open, "anchor", module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "distribution",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5100}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var image any
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
if r["id"] == "postgres.runtime" {
|
||||
image = r["image"]
|
||||
}
|
||||
}
|
||||
if image != "127.0.0.1:5100/postgres/runtime@"+aDigest {
|
||||
t.Fatalf("on the store's own node, off any network, the image is fetched as %v", image)
|
||||
}
|
||||
held := heldBy(ctx)
|
||||
if got := held["postgres/runtime"]; got != "127.0.0.1:5100/postgres/runtime@"+aDigest {
|
||||
t.Fatalf("a builder beside the store is handed the base %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,534 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0100: taking a module is its cutover; converging a node is one act, previewed, and
|
||||
// refused while a found container is held; returning to adopted keeps what was taken.
|
||||
|
||||
// anAdoptedAnchor is aMesh with the anchor adopted, running a served module the predecessor also
|
||||
// runs and a module with only a file, and a filter module in the catalogue.
|
||||
func anAdoptedAnchor(t *testing.T) (*stores, *[]string) {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "hello-web", Version: "1",
|
||||
Listens: []catalogue.Listening{{Port: 8080, From: catalogue.FromEverywhere}},
|
||||
Resources: []map[string]any{
|
||||
{"id": "page", "type": "file", "path": "/var/lib/hello-web/index.html", "content": "hi"},
|
||||
{"id": "server", "type": "container", "name": "hello-web", "ports": []any{"8080:80"},
|
||||
"image": "registry.example/hello@sha256:" + strings.Repeat("a", 64)},
|
||||
}})
|
||||
register(t, open, catalogue.Manifest{Module: "notes", Version: "1",
|
||||
Resources: []map[string]any{
|
||||
{"id": "conf", "type": "file", "path": "/etc/notes.conf", "content": "x"},
|
||||
}})
|
||||
register(t, open, catalogue.Manifest{Module: "nftables", Version: "1",
|
||||
Filtering: &catalogue.Filtering{Into: "/etc/nftables.conf"},
|
||||
Resources: []map[string]any{{"id": "load", "type": "service", "unit": "mesh-filter.service",
|
||||
"state": "running", "restart-on": []any{"filtering"}}}})
|
||||
if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range []string{"hello-web", "notes"} {
|
||||
if _, err := assign(ctx, open, "anchor", m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
sent := &[]string{}
|
||||
saved := sendNodes
|
||||
sendNodes = func(_ context.Context, _ *stores, names []string) error {
|
||||
*sent = append(*sent, names...)
|
||||
return nil
|
||||
}
|
||||
t.Cleanup(func() { sendNodes = saved })
|
||||
return open, sent
|
||||
}
|
||||
|
||||
// reportsHolding has the anchor report, on what it was last sent, holding what is given.
|
||||
func reportsHolding(t *testing.T, open *stores, held ...link.Held) {
|
||||
t.Helper()
|
||||
reportsReaching(t, open, []link.Reach{
|
||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"},
|
||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true,
|
||||
ContainerPort: 80},
|
||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 5000, By: "predecessor-registry",
|
||||
Published: true, ContainerPort: 5000},
|
||||
{Protocol: "tcp", Address: "127.0.0.1", Port: 15672, By: "mesh-broker",
|
||||
Published: true, ContainerPort: 15672},
|
||||
}, held...)
|
||||
}
|
||||
|
||||
// reportsReaching has the anchor report, on what it was last sent, what is reachable on it and
|
||||
// holding what is given.
|
||||
func reportsReaching(t *testing.T, open *stores, reachable []link.Reach, held ...link.Held) {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
body, err := composed(t, open, "anchor").Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, err := open.inventory.NodeByName(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSent(ctx, record.ID, digestOf(body)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := (link.Enrolment{Inventory: open.inventory}).Heard(ctx, link.Report{
|
||||
Node: "anchor", Applied: []string{"hello-web.x"}, Declared: digestOf(body),
|
||||
Firewall: "ufw", Held: held, Reachable: reachable,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
heldContainer = link.Held{ID: "hello-web.server", Module: "hello-web", Kind: "container",
|
||||
Target: "hello-web", Since: time.Now()}
|
||||
heldFile = link.Held{ID: "notes.conf", Module: "notes", Kind: "file",
|
||||
Target: "/etc/notes.conf", Since: time.Now(), Kept: "/var/lib/mesh-host/kept/abc-notes.conf"}
|
||||
)
|
||||
|
||||
func TestTakingAModuleNotOnTheNodeIsRefused(t *testing.T) {
|
||||
open, _ := anAdoptedAnchor(t)
|
||||
if _, err := take(t.Context(), open, "anchor", "nftables"); !errors.Is(err, inventory.ErrNotAssigned) {
|
||||
t.Fatalf("taking an unassigned module gave %v", err)
|
||||
}
|
||||
if _, err := take(t.Context(), open, "laptop", "network"); !errors.Is(err, inventory.ErrNotAdopted) {
|
||||
t.Fatalf("taking on a converged node gave %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvergingIsRefusedOnAPreviewThatWouldBeStale(t *testing.T) {
|
||||
open, sent := anAdoptedAnchor(t)
|
||||
_, err := converge(t.Context(), open, "anchor", false, "", "")
|
||||
if err == nil || !strings.Contains(err.Error(), "has not reported") {
|
||||
t.Fatalf("a node that never reported was previewed: %v", err)
|
||||
}
|
||||
if len(*sent) != 0 {
|
||||
t.Fatal("a refused converge sent something")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheFlipIsRefusedWhileAFoundContainerIsHeld(t *testing.T) {
|
||||
open, sent := anAdoptedAnchor(t)
|
||||
reportsHolding(t, open, heldContainer, heldFile)
|
||||
_, err := converge(t.Context(), open, "anchor", true, "", "")
|
||||
if err == nil || !strings.Contains(err.Error(), "take anchor hello-web once its data has moved") {
|
||||
t.Fatalf("the flip was not refused while hello-web holds its found container: %v", err)
|
||||
}
|
||||
if n, _ := open.inventory.NodeByName(t.Context(), "anchor"); !n.Adopted || len(*sent) != 0 {
|
||||
t.Fatal("a refused flip changed something")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTakingNamesWhatItReplaces(t *testing.T) {
|
||||
open, _ := anAdoptedAnchor(t)
|
||||
reportsHolding(t, open, heldContainer, heldFile)
|
||||
said, err := take(t.Context(), open, "anchor", "hello-web")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(said, "container hello-web (hello-web.server)") ||
|
||||
!strings.Contains(said, "push anchor") {
|
||||
t.Fatalf("taking did not say what it replaces and what to run:\n%s", said)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvergingPreviewsThenChangesAndAdoptingKeepsWhatWasTaken(t *testing.T) {
|
||||
open, sent := anAdoptedAnchor(t)
|
||||
ctx := t.Context()
|
||||
if _, err := take(ctx, open, "anchor", "hello-web"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reportsHolding(t, open, heldFile)
|
||||
|
||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"tcp/8080 hello-web (published, container port 80)",
|
||||
"declared by hello-web (from anywhere)",
|
||||
"WILL CLOSE — no module assigned here declares it",
|
||||
// The anchor faces inward and is on the private network: the derived filter admits ssh
|
||||
// from the mesh only.
|
||||
"WILL CLOSE to everything outside the private network — ssh stays open from the mesh",
|
||||
"notes\n replacing the found file /etc/notes.conf (notes.conf), original kept at",
|
||||
"assigns nftables",
|
||||
"the found firewall (ufw) is disabled, never flushed",
|
||||
// What it routes is not a listener: said not to be previewed, and to be dropped.
|
||||
"not previewed: traffic the machine routes that is not a published port",
|
||||
"the derived filter drops it unless a module declares it",
|
||||
} {
|
||||
if !strings.Contains(preview, want) {
|
||||
t.Errorf("the preview does not say %q:\n%s", want, preview)
|
||||
}
|
||||
}
|
||||
if strings.Contains(preview, "15672") {
|
||||
t.Errorf("a loopback listener is in the preview:\n%s", preview)
|
||||
}
|
||||
for _, line := range strings.Split(preview, "\n") {
|
||||
if strings.Contains(line, "5000") && !strings.Contains(line, "WILL CLOSE") {
|
||||
t.Errorf("an undeclared published port is not said to close: %s", line)
|
||||
}
|
||||
}
|
||||
if n, _ := open.inventory.NodeByName(ctx, "anchor"); !n.Adopted || len(*sent) != 0 {
|
||||
t.Fatal("the preview changed something")
|
||||
}
|
||||
|
||||
if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n, _ := open.inventory.NodeByName(ctx, "anchor")
|
||||
if n.Adopted {
|
||||
t.Fatal("converge --yes left the node adopted")
|
||||
}
|
||||
taken, _ := open.inventory.Taken(ctx, "anchor")
|
||||
if !reflect.DeepEqual(taken, []string{"hello-web", overlay.Name, "nftables", "notes"}) {
|
||||
t.Fatalf("the flip took %v", taken)
|
||||
}
|
||||
if !reflect.DeepEqual(*sent, []string{"anchor"}) {
|
||||
t.Fatalf("the flip sent %v", *sent)
|
||||
}
|
||||
declared := composed(t, open, "anchor")
|
||||
if declared.Adoption != nil {
|
||||
t.Fatal("a converged node is still sent an adoption envelope")
|
||||
}
|
||||
if !hasID(declared.Resources, "nftables.filtering") {
|
||||
t.Fatal("the converged node is not declared the mesh's filter")
|
||||
}
|
||||
|
||||
if _, err := adopt(ctx, open, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := adopt(ctx, open, "anchor"); err == nil {
|
||||
t.Fatal("adopting an adopted node was not refused")
|
||||
}
|
||||
again, _ := open.inventory.Taken(ctx, "anchor")
|
||||
if !reflect.DeepEqual(again, taken) {
|
||||
t.Fatalf("returning to adopted lost what was taken: %v", again)
|
||||
}
|
||||
declared = composed(t, open, "anchor")
|
||||
if declared.Adoption == nil || len(declared.Adoption.Untaken) != 0 {
|
||||
t.Fatalf("returned to adopted, the envelope is %+v", declared.Adoption)
|
||||
}
|
||||
if hasID(declared.Resources, "nftables.filtering") || hasID(declared.Resources, "nftables.load") {
|
||||
t.Fatal("returned to adopted, the mesh's filter is still declared")
|
||||
}
|
||||
}
|
||||
|
||||
func hasID(resources []map[string]any, id string) bool {
|
||||
for _, r := range resources {
|
||||
if r["id"] == id {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// The command API refuses a flip exactly as the command line does, in the same words.
|
||||
func TestTheApiRefusesTheFlipInTheCommandLinesWords(t *testing.T) {
|
||||
open, _ := anAdoptedAnchor(t)
|
||||
reportsHolding(t, open, heldContainer)
|
||||
_, direct := converge(t.Context(), open, "anchor", true, "", "")
|
||||
if direct == nil {
|
||||
t.Fatal("the flip was not refused")
|
||||
}
|
||||
got := asking(t, letIn{}, "POST", "/converge", `{"node":"anchor","yes":true}`)
|
||||
if got.Code != http.StatusConflict {
|
||||
t.Fatalf("got %d: %s", got.Code, got.Body.String())
|
||||
}
|
||||
var said map[string]any
|
||||
if err := json.Unmarshal(got.Body.Bytes(), &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if said["refused"] != direct.Error() {
|
||||
t.Fatalf("the API said %q and the command line %q", said["refused"], direct.Error())
|
||||
}
|
||||
if got := asking(t, letIn{}, "POST", "/take", `{"node":"anchor"}`); got.Code != http.StatusBadRequest {
|
||||
t.Fatalf("a take naming no module got %d", got.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0100: the preview says what the derived filter does, rendered as it is rendered. On
|
||||
// a machine that faces inward, ssh is admitted from the private network only, and a port a module
|
||||
// admits from the mesh only closes to everything outside it: both are said to close.
|
||||
func TestThePreviewSaysWhatNarrowsToTheMeshCloses(t *testing.T) {
|
||||
open, _ := anAdoptedAnchor(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "store", Version: "1",
|
||||
Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}}})
|
||||
if _, err := assign(ctx, open, "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reportsReaching(t, open, []link.Reach{
|
||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"},
|
||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 5432, By: "postgres"},
|
||||
{Protocol: "tcp", Address: "10.77.0.1", Port: 5432, By: "postgres"},
|
||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true,
|
||||
ContainerPort: 80},
|
||||
})
|
||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lines := map[string]string{}
|
||||
for _, line := range strings.Split(preview, "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) > 1 && strings.HasPrefix(fields[0], "tcp/") {
|
||||
lines[fields[0]+" "+fields[1]] += line + "\n"
|
||||
}
|
||||
}
|
||||
if got := lines["tcp/22 sshd"]; !strings.Contains(got, "WILL CLOSE to everything outside "+
|
||||
"the private network") {
|
||||
t.Errorf("ssh on an inward machine is not said to close outside the mesh:\n%s", preview)
|
||||
}
|
||||
store := lines["tcp/5432 postgres"]
|
||||
if strings.Count(store, "WILL CLOSE to everything outside the private network") != 1 ||
|
||||
!strings.Contains(store, "declared by store (from mesh)") {
|
||||
t.Errorf("the store's narrowing is not said to close, or its mesh address is:\n%s", preview)
|
||||
}
|
||||
if got := lines["tcp/8080 hello-web"]; !strings.Contains(got, "declared by hello-web (from anywhere)") {
|
||||
t.Errorf("a port open to everywhere is not said to stay:\n%s", preview)
|
||||
}
|
||||
}
|
||||
|
||||
// digestIn is the digest a converge preview printed.
|
||||
func digestIn(t *testing.T, preview string) string {
|
||||
t.Helper()
|
||||
for _, line := range strings.Split(preview, "\n") {
|
||||
if fields := strings.Fields(line); len(fields) == 2 && fields[0] == "preview" {
|
||||
return fields[1]
|
||||
}
|
||||
}
|
||||
t.Fatalf("the preview printed no digest:\n%s", preview)
|
||||
return ""
|
||||
}
|
||||
|
||||
// The flip acts on the preview the operator saw: it names that preview's digest, and it is refused
|
||||
// when the digest is missing, when anything the preview says has changed since, or when the node's
|
||||
// account of itself is too old to be the machine as it is.
|
||||
func TestTheFlipActsOnlyOnThePreviewTheOperatorSaw(t *testing.T) {
|
||||
open, sent := anAdoptedAnchor(t)
|
||||
ctx := t.Context()
|
||||
if _, err := take(ctx, open, "anchor", "hello-web"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reportsHolding(t, open, heldFile)
|
||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saw := digestIn(t, preview)
|
||||
if !strings.Contains(preview, "converge anchor --yes "+saw) {
|
||||
t.Fatalf("the preview does not say how to act on it:\n%s", preview)
|
||||
}
|
||||
unchanged := func() {
|
||||
t.Helper()
|
||||
if n, _ := open.inventory.NodeByName(ctx, "anchor"); !n.Adopted || len(*sent) != 0 {
|
||||
t.Fatal("a refused flip changed something")
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := converge(ctx, open, "anchor", true, "", ""); err == nil ||
|
||||
!strings.Contains(err.Error(), "--yes "+saw) {
|
||||
t.Fatalf("a flip naming no preview was not refused: %v", err)
|
||||
}
|
||||
unchanged()
|
||||
|
||||
// Something new is reachable: the preview the operator saw is not what would happen.
|
||||
reportsReaching(t, open, []link.Reach{
|
||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"},
|
||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true,
|
||||
ContainerPort: 80},
|
||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 6000, By: "something-new"},
|
||||
}, heldFile)
|
||||
_, err = converge(ctx, open, "anchor", true, saw, "")
|
||||
if err == nil || !strings.Contains(err.Error(), "has changed since preview "+saw) {
|
||||
t.Fatalf("a flip on a changed preview was not refused: %v", err)
|
||||
}
|
||||
unchanged()
|
||||
|
||||
// An account older than the flip trusts is refused, whatever digest is named.
|
||||
again, err := converge(ctx, open, "anchor", false, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saved := reportFreshFor
|
||||
reportFreshFor = time.Nanosecond
|
||||
_, err = converge(ctx, open, "anchor", true, digestIn(t, again), "")
|
||||
reportFreshFor = saved
|
||||
if err == nil || !strings.Contains(err.Error(), "wait for its next report") {
|
||||
t.Fatalf("a flip on an old account was not refused: %v", err)
|
||||
}
|
||||
unchanged()
|
||||
|
||||
if _, err := converge(ctx, open, "anchor", true, digestIn(t, again), ""); err != nil {
|
||||
t.Fatalf("the flip on the preview just seen was refused: %v", err)
|
||||
}
|
||||
if n, _ := open.inventory.NodeByName(ctx, "anchor"); n.Adopted {
|
||||
t.Fatal("the flip did not converge the node")
|
||||
}
|
||||
}
|
||||
|
||||
// The flip holds the node from its checks to its send, so a push composed meanwhile waits and is
|
||||
// composed after it — never sent after it with the node still adopted. And the send inside the
|
||||
// flip is not made to wait on the flip's own hold.
|
||||
func TestTheFlipHoldsTheNodeWhileItSends(t *testing.T) {
|
||||
open, _ := anAdoptedAnchor(t)
|
||||
ctx := t.Context()
|
||||
if _, err := take(ctx, open, "anchor", "hello-web"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reportsHolding(t, open, heldFile)
|
||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var heldElsewhere, heldHere error
|
||||
sendNodes = func(inner context.Context, open *stores, names []string) error {
|
||||
// Another caller cannot hold the node while the flip sends it.
|
||||
waiting, cancel := context.WithTimeout(ctx, 300*time.Millisecond)
|
||||
defer cancel()
|
||||
if release, err := open.inventory.HoldNodes(waiting, names); err == nil {
|
||||
release()
|
||||
heldElsewhere = errors.New("another caller held the node while the flip sent it")
|
||||
}
|
||||
// The flip's own send holds it without waiting on itself.
|
||||
_, release, err := holdNodes(inner, open, names)
|
||||
if err != nil {
|
||||
heldHere = err
|
||||
return err
|
||||
}
|
||||
release()
|
||||
return nil
|
||||
}
|
||||
if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if heldElsewhere != nil || heldHere != nil {
|
||||
t.Fatalf("%v %v", heldElsewhere, heldHere)
|
||||
}
|
||||
// And given back once it is done.
|
||||
release, err := open.inventory.HoldNodes(ctx, []string{"anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
release()
|
||||
}
|
||||
|
||||
// novox/hq ADR 0103: the preview names every kind of thing a module the flip takes holds as found,
|
||||
// not only its files, and the digest changes when any of them does.
|
||||
func TestThePreviewNamesEveryHeldKind(t *testing.T) {
|
||||
open, _ := anAdoptedAnchor(t)
|
||||
ctx := t.Context()
|
||||
if _, err := take(ctx, open, "anchor", "hello-web"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
since := time.Now()
|
||||
held := []link.Held{heldFile,
|
||||
{ID: "notes.data", Module: "notes", Kind: "directory", Target: "/var/lib/notes", Since: since},
|
||||
{ID: "notes.daemon", Module: "notes", Kind: "service", Target: "notes.service", Since: since},
|
||||
{ID: "notes.seed", Module: "notes", Kind: "archive", Target: "/srv/notes", Since: since},
|
||||
{ID: "notes.worker", Module: "notes", Kind: "process", Target: "notes-worker", Since: since},
|
||||
{ID: "notes.account", Module: "notes", Kind: "user", Target: "notes", Since: since},
|
||||
}
|
||||
reportsHolding(t, open, held...)
|
||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"replacing the found directory /var/lib/notes (notes.data)",
|
||||
"replacing the found service notes.service (notes.daemon)",
|
||||
"replacing the found archive /srv/notes (notes.seed)",
|
||||
"replacing the found process notes-worker (notes.worker)",
|
||||
"replacing the found user notes (notes.account)",
|
||||
} {
|
||||
if !strings.Contains(preview, want) {
|
||||
t.Errorf("the preview does not say %q:\n%s", want, preview)
|
||||
}
|
||||
}
|
||||
reportsHolding(t, open, held[:len(held)-1]...)
|
||||
fewer, err := converge(ctx, open, "anchor", false, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if digestIn(t, fewer) == digestIn(t, preview) {
|
||||
t.Fatal("the digest does not change with what is held")
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0100: the flip acts on what the node said is reachable, so an account naming nothing
|
||||
// is refused. Every machine that is up answers on ssh; nothing reported means the host's collectors
|
||||
// did not, and flipping would close ports the preview never named.
|
||||
func TestTheFlipIsRefusedOnAnAccountNamingNothingReachable(t *testing.T) {
|
||||
open, sent := anAdoptedAnchor(t)
|
||||
ctx := t.Context()
|
||||
if _, err := take(ctx, open, "anchor", "hello-web"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Only a loopback listener: nothing off the machine, which is the same silence.
|
||||
reportsReaching(t, open, []link.Reach{
|
||||
{Protocol: "tcp", Address: "127.0.0.1", Port: 15672, By: "mesh-broker"},
|
||||
}, heldFile)
|
||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(preview, "this account looks partial") {
|
||||
t.Errorf("the preview does not mark a partial account:\n%s", preview)
|
||||
}
|
||||
_, err = converge(ctx, open, "anchor", true, digestIn(t, preview), "")
|
||||
if err == nil || !strings.Contains(err.Error(), "says nothing is reachable on it") {
|
||||
t.Fatalf("the flip was not refused on an account naming nothing: %v", err)
|
||||
}
|
||||
if n, _ := open.inventory.NodeByName(ctx, "anchor"); !n.Adopted || len(*sent) != 0 {
|
||||
t.Fatal("a refused flip changed something")
|
||||
}
|
||||
}
|
||||
|
||||
// An assignment cannot land between a preview and the flip that takes every module: assigning
|
||||
// holds the node, so it waits for whatever is converging it.
|
||||
func TestAssigningWaitsForWhateverIsConvergingTheNode(t *testing.T) {
|
||||
open, _ := anAdoptedAnchor(t)
|
||||
ctx := t.Context()
|
||||
if _, err := take(ctx, open, "anchor", "hello-web"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reportsHolding(t, open, heldFile)
|
||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saved, savedPoll := inventory.HoldWaitFor, inventory.HoldPoll
|
||||
inventory.HoldWaitFor, inventory.HoldPoll = time.Second, 50*time.Millisecond
|
||||
defer func() { inventory.HoldWaitFor, inventory.HoldPoll = saved, savedPoll }()
|
||||
|
||||
var whileFlipping error
|
||||
sendNodes = func(context.Context, *stores, []string) error {
|
||||
_, whileFlipping = assign(ctx, open, "anchor", "notes")
|
||||
return nil
|
||||
}
|
||||
if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !errors.Is(whileFlipping, inventory.ErrNodeBusy) {
|
||||
t.Fatalf("an assignment landed while the node was being converged: %v", whileFlipping)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,621 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// A node is adopted or converged (novox/hq ADR 0100), and it is said to be adopted wherever the
|
||||
// mesh reports a node's state: node list, node show, status and the board.
|
||||
|
||||
// showMode is the node show lines about a node's mode and what was taken on it.
|
||||
func showMode(ctx context.Context, inv *inventory.Inventory, node inventory.Node) error {
|
||||
if !node.Adopted {
|
||||
fmt.Printf(" mode converged\n")
|
||||
return nil
|
||||
}
|
||||
fmt.Printf(" mode adopted since %s\n",
|
||||
node.AdoptedSince.Local().Format(time.DateTime))
|
||||
taken, err := inv.Taken(ctx, node.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(taken) == 0 {
|
||||
fmt.Printf(" taken nothing yet\n")
|
||||
} else {
|
||||
fmt.Printf(" taken %s\n", strings.Join(taken, ", "))
|
||||
}
|
||||
said, err := inv.AdoptionOf(ctx, node.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if said.At.IsZero() {
|
||||
fmt.Printf(" it has not yet said what it found\n")
|
||||
return nil
|
||||
}
|
||||
fmt.Printf(" firewall found %s\n", orNone(said.Firewall))
|
||||
if err := showTunnel(ctx, inv, node.Name); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(said.Held) == 0 {
|
||||
fmt.Printf(" holding nothing found\n")
|
||||
}
|
||||
for _, h := range said.Held {
|
||||
// A held thing that changed is how a predecessor still writing is caught: said first.
|
||||
line := fmt.Sprintf(" holds %-11s %s %s, for %s", h.Kind, h.Target, h.ID, h.Module)
|
||||
if h.Changed != "" {
|
||||
line += " — " + strings.ToUpper(h.Changed) + " by something other than the mesh"
|
||||
}
|
||||
fmt.Println(line)
|
||||
if h.Kept != "" {
|
||||
fmt.Printf(" %-17s original kept at %s\n", "", h.Kept)
|
||||
}
|
||||
}
|
||||
fmt.Printf(" as of %s\n", said.At.Local().Format(time.DateTime))
|
||||
return nil
|
||||
}
|
||||
|
||||
// showTunnel is the node show lines about the tunnel an adopted node found and carried (novox/hq
|
||||
// ADR 0105): what it presented at enrolment, and what it last said about taking it over.
|
||||
func showTunnel(ctx context.Context, inv *inventory.Inventory, name string) error {
|
||||
tunnel, err := inv.TunnelOf(ctx, name)
|
||||
if errors.Is(err, inventory.ErrNoTunnel) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf(" tunnel found %s on port %d, %s in %s, %d peer(s)\n",
|
||||
tunnel.Interface, tunnel.Port, tunnel.Address, tunnel.Range, len(tunnel.Peers))
|
||||
carried, said, err := inv.CarriedTunnelOf(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch {
|
||||
case !said:
|
||||
fmt.Printf(" %-17s not yet taken over — the node has not said so\n", "")
|
||||
case carried.State == inventory.CarriedTaken:
|
||||
fmt.Printf(" %-17s taken over: %s is down and disabled, never flushed; the mesh's interface "+
|
||||
"runs with its key, port and %d peer(s)\n", "", carried.Interface, carried.Peers)
|
||||
case carried.State == inventory.CarriedDown:
|
||||
fmt.Printf(" %-17s TUNNEL DOWN: %s is stopped and the mesh's interface is not up — the peers "+
|
||||
"reach nothing. On the machine: systemctl start %s\n", "", carried.Interface,
|
||||
"wg-quick@"+carried.Interface)
|
||||
default:
|
||||
fmt.Printf(" %-17s NOT taken over: %s is still the interface the peers reach\n", "", carried.Interface)
|
||||
}
|
||||
if said && carried.Note != "" {
|
||||
fmt.Printf(" %-17s %s\n", "", carried.Note)
|
||||
}
|
||||
if said && carried.Kept != "" {
|
||||
fmt.Printf(" %-17s its configuration's original kept at %s\n", "", carried.Kept)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func orNone(s string) string {
|
||||
if s == "" {
|
||||
return "none reported"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// adoptedNodes are the names of every adopted node, in the order given.
|
||||
func adoptedNodes(nodes []inventory.Node) []string {
|
||||
var out []string
|
||||
for _, n := range nodes {
|
||||
if n.Adopted {
|
||||
out = append(out, n.Name)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// The operator's acts on an adopted node (novox/hq ADR 0100). Called by the command line and the
|
||||
// command API alike, so a refusal is the same refusal in the same words at both (ADR 0035).
|
||||
|
||||
// sendNodes sends the named machines what they should be now. A variable so a test can see what
|
||||
// an act would send without a broker.
|
||||
var sendNodes = sendTo
|
||||
|
||||
// DefaultFilter is the module converging a node assigns to load the mesh's derived filter.
|
||||
const DefaultFilter = "nftables"
|
||||
|
||||
// take is a module's cutover on an adopted node: the operator's act, done when that module's data
|
||||
// has moved. From the next push its resources converge there like any other, replacing what the
|
||||
// node found and holds for it.
|
||||
func take(ctx context.Context, open *stores, node, module string) (string, error) {
|
||||
inv := open.inventory
|
||||
assigned, err := inv.Assigned(ctx, node)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !slices.Contains(assigned, module) {
|
||||
if plan, _, err := planFor(ctx, open, node); err == nil {
|
||||
if why, runs := plan.Because[module]; runs {
|
||||
return "", fmt.Errorf("%w: %s runs on %s because %s — assign it to %s to take it",
|
||||
inventory.ErrNotAssigned, module, node, why, node)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := inv.Take(ctx, node, module); err != nil {
|
||||
return "", err
|
||||
}
|
||||
said := fmt.Sprintf("%s is taken on %s", module, node)
|
||||
reported, err := inv.AdoptionOf(ctx, node)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var replaces []string
|
||||
for _, h := range reported.Held {
|
||||
if h.Module == module {
|
||||
replaces = append(replaces, " "+heldLine(h))
|
||||
}
|
||||
}
|
||||
if len(replaces) > 0 {
|
||||
said += "; the next push replaces what the node found and holds for it:\n" +
|
||||
strings.Join(replaces, "\n")
|
||||
}
|
||||
return said + fmt.Sprintf("\n run `push %s` to cut it over", node), nil
|
||||
}
|
||||
|
||||
// reportFreshFor is how old a node's account of itself may be for the flip to act on it. A
|
||||
// variable so a test can age a report without waiting.
|
||||
var reportFreshFor = 15 * time.Minute
|
||||
|
||||
// converge previews, and with yes makes, the flip of an adopted node to converged: every module it
|
||||
// runs is taken, the filter module is assigned to load the mesh's derived filter in place of the
|
||||
// guard, and the found firewall is retired — disabled, never flushed — by the host.
|
||||
//
|
||||
// Refused while an assigned module still holds a found container: each service is taken on its
|
||||
// own, when its data has moved, never by the flip. And refused on a preview that would be stale:
|
||||
// what is reachable is the node's last account, so that account must be of what it was last sent.
|
||||
//
|
||||
// **The flip acts on the preview the operator saw** and on nothing else. The preview ends with a
|
||||
// short digest of what it said — every reachable thing and its fate, the modules the flip takes and
|
||||
// the filter — and yes must name that digest: if anything the preview would say has changed since,
|
||||
// the flip is refused rather than done on a preview nobody read. And it is refused on an account
|
||||
// older than reportFreshFor: what was reachable then is not evidence of what is reachable now.
|
||||
func converge(ctx context.Context, open *stores, node string, yes bool, digest string,
|
||||
filter string) (string, error) {
|
||||
inv := open.inventory
|
||||
if filter == "" {
|
||||
filter = DefaultFilter
|
||||
}
|
||||
if yes {
|
||||
// Held from the checks to the send, so no push composed before the flip is sent after it
|
||||
// and returns the node to adopted.
|
||||
held, release, err := holdNodes(ctx, open, []string{node})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer release()
|
||||
ctx = held
|
||||
}
|
||||
record, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !record.Adopted {
|
||||
return "", fmt.Errorf("%s is converged already; there is nothing to flip", node)
|
||||
}
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
current := false
|
||||
for _, r := range reports {
|
||||
if r.Node == node {
|
||||
current = r.Current
|
||||
}
|
||||
}
|
||||
reported, err := inv.AdoptionOf(ctx, node)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !current || reported.At.IsZero() {
|
||||
return "", fmt.Errorf("%s has not reported on the declaration it was last sent, so what it "+
|
||||
"says is reachable may not be the machine as it is: run `push %s --wait 2m` and "+
|
||||
"converge once it has applied", node, node)
|
||||
}
|
||||
|
||||
assignedWhenPreviewed, err := inv.Assigned(ctx, node)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
plan, settings, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
runs := map[string]bool{}
|
||||
for _, m := range plan.Modules {
|
||||
runs[m.Module] = true
|
||||
}
|
||||
var holding []string
|
||||
for _, h := range reported.Held {
|
||||
if h.Kind == "container" && runs[h.Module] {
|
||||
holding = append(holding, fmt.Sprintf(" %s holds the found container %s — take %s %s "+
|
||||
"once its data has moved", h.Module, h.Target, node, h.Module))
|
||||
}
|
||||
}
|
||||
if len(holding) > 0 {
|
||||
sort.Strings(holding)
|
||||
return "", fmt.Errorf("%s still holds what it found, and a service is taken on its own, "+
|
||||
"never by the flip:\n%s", node, strings.Join(holding, "\n"))
|
||||
}
|
||||
// And refused while a peer of the tunnel this hub took over has not enrolled (novox/hq ADR
|
||||
// 0105): the flip loads the derived filter and retires the found firewall, and a machine the
|
||||
// mesh has no record of is not one the filter admits — it would go dark.
|
||||
if _, hubName, adopted, err := inv.AdoptedTunnel(ctx); err != nil {
|
||||
return "", err
|
||||
} else if adopted && hubName == node {
|
||||
carried, err := inv.CarriedPeers(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var waiting []string
|
||||
for _, c := range carried {
|
||||
if c.EnrolledAs == "" {
|
||||
waiting = append(waiting, fmt.Sprintf(" %s at %s", overlay.CarriedName(c.PublicKey), c.Address))
|
||||
}
|
||||
}
|
||||
if len(waiting) > 0 {
|
||||
return "", fmt.Errorf("%s carries peers of the tunnel it took over that have not enrolled, and "+
|
||||
"converging would cut them off — enrol each first (`overlay show` says which are enrolled):\n%s",
|
||||
node, strings.Join(waiting, "\n"))
|
||||
}
|
||||
}
|
||||
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
filterModule, known := shelf[filter]
|
||||
if !known {
|
||||
return "", fmt.Errorf("%w: %s — converging assigns it to load the mesh's filter; "+
|
||||
"name another with --filter", inventory.ErrNoSuchModule, filter)
|
||||
}
|
||||
if filterModule.Filtering == nil {
|
||||
return "", fmt.Errorf("%s loads no filter of the mesh's; name a module that does with --filter",
|
||||
filter)
|
||||
}
|
||||
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Reading)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
rules, err := plan.Rules(with)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
taken, err := inv.Taken(ctx, node)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
derived := derivedFilter{rules: rules, foundation: with.Foundation, mesh: with.Mesh,
|
||||
outward: plan.PublicDomain != ""}
|
||||
preview, saw := previewOf(node, reported, derived, plan, taken, filter, runs[filter])
|
||||
preview += "\n\n preview " + saw
|
||||
if !yes {
|
||||
return preview + fmt.Sprintf("\n\nNothing has changed. Run `converge %s --yes %s` to do "+
|
||||
"it.", node, saw), nil
|
||||
}
|
||||
// An account naming nothing reachable is not an account of a machine: every machine answers
|
||||
// on ssh, and the host's collectors failing — `ss` refusing, or the container runtime not
|
||||
// answering, which drops every published port at once — leaves exactly this. Flipping on it
|
||||
// would close ports the preview never named.
|
||||
if yes && countReachable(reported) == 0 {
|
||||
return preview, fmt.Errorf("%s says nothing is reachable on it, which no machine that is "+
|
||||
"up ever is: its account looks partial — whatever reads what is listening, or what "+
|
||||
"the container runtime publishes, did not answer. Fix that on the machine and run "+
|
||||
"`push %s --wait 2m`, then preview again", node, node)
|
||||
}
|
||||
if age := time.Since(reported.At); age > reportFreshFor {
|
||||
return preview, fmt.Errorf("%s last said what is reachable on it %s ago, and the flip acts "+
|
||||
"only on an account newer than %s: wait for its next report, or run `push %s --wait 2m`, "+
|
||||
"then preview again", node, age.Round(time.Second), reportFreshFor, node)
|
||||
}
|
||||
if digest == "" {
|
||||
return preview, fmt.Errorf("converging %s acts on the preview you saw: name its digest, "+
|
||||
"`converge %s --yes %s`, once you have read it", node, node, saw)
|
||||
}
|
||||
if digest != saw {
|
||||
return preview, fmt.Errorf("what converging %s would do has changed since preview %s "+
|
||||
"(it is now %s): read the preview above, and run `converge %s --yes %s` if it is "+
|
||||
"what you want", node, digest, saw, node, saw)
|
||||
}
|
||||
|
||||
// The flip. The filter first, and only kept if the node still resolves with it: a node that
|
||||
// cannot be worked out would be sent nothing, and would sit with its guard and no filter.
|
||||
assigned, err := inv.Assigned(ctx, node)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// And nothing assigned since the preview was composed: the flip takes every module the node
|
||||
// runs, and one assigned in between would be taken without ever having been previewed.
|
||||
if !slices.Equal(assigned, assignedWhenPreviewed) {
|
||||
return preview, fmt.Errorf("what %s runs changed while this was converging (it is now %s): "+
|
||||
"the flip takes every module on the node, so read the preview again", node,
|
||||
strings.Join(assigned, ", "))
|
||||
}
|
||||
if !slices.Contains(assigned, filter) {
|
||||
if _, err := inv.Assign(ctx, node, filter); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, _, err := planFor(ctx, open, node); err != nil {
|
||||
_ = inv.Unassign(ctx, node, filter)
|
||||
return "", fmt.Errorf("%s cannot run %s, so it was not converged: %w", node, filter, err)
|
||||
}
|
||||
}
|
||||
took, err := inv.Converge(ctx, node)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
said := preview + fmt.Sprintf("\n\n%s is converged", node)
|
||||
if len(took) > 0 {
|
||||
said += "; took " + strings.Join(took, ", ")
|
||||
}
|
||||
if err := sendNodes(ctx, open, []string{node}); err != nil {
|
||||
return said + "\n and it could not be sent: run `push " + node + "`", err
|
||||
}
|
||||
return said + "\n sent: the host loads the mesh's filter and disables the firewall it found", nil
|
||||
}
|
||||
|
||||
// previewOf is what converging a node will change, before it changes it, and a short digest of
|
||||
// what it said: every reachable thing and its fate, the modules the flip takes and the filter. The
|
||||
// digest is what the flip is asked to act on, so it changes whenever any of those would.
|
||||
func previewOf(node string, reported inventory.Adoption, derived derivedFilter,
|
||||
plan catalogue.Resolution, taken []string, filter string, filterAssigned bool) (string, string) {
|
||||
var said []string
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "converging %s\n", node)
|
||||
fmt.Fprintf(&b, "\n reachable on the machine now, as it reported at %s:\n",
|
||||
reported.At.Local().Format(time.DateTime))
|
||||
for _, r := range reported.Reachable {
|
||||
if loopback(r.Address) {
|
||||
continue
|
||||
}
|
||||
what := fmt.Sprintf("%s/%d", r.Protocol, r.Port)
|
||||
if r.By != "" {
|
||||
what += " " + r.By
|
||||
}
|
||||
if r.Published {
|
||||
what += fmt.Sprintf(" (published, container port %d)", r.ContainerPort)
|
||||
}
|
||||
fate := derived.fate(r)
|
||||
fmt.Fprintf(&b, " %-44s %s\n", what, fate)
|
||||
said = append(said, fmt.Sprintf("reach %s %s %s", r.Address, what, fate))
|
||||
}
|
||||
if countReachable(reported) == 0 {
|
||||
// Said as what it is: no machine that is up is reachable on nothing, so this is an
|
||||
// account that did not come back, not a machine with nothing on it.
|
||||
b.WriteString(" nothing reported — this account looks partial, and the flip is " +
|
||||
"refused on it\n")
|
||||
said = append(said, "reach nothing reported")
|
||||
}
|
||||
// What the machine routes for others is not a listener and not a published port, so nothing
|
||||
// above can show it; the derived filter's forward chain drops it all the same.
|
||||
b.WriteString(" not previewed: traffic the machine routes that is not a published port " +
|
||||
"(a tunnel, NAT in the found firewall) — the derived filter drops it unless a module " +
|
||||
"declares it\n")
|
||||
|
||||
isTaken := map[string]bool{}
|
||||
for _, m := range taken {
|
||||
isTaken[m] = true
|
||||
}
|
||||
var takes []string
|
||||
for _, m := range plan.Modules {
|
||||
if !isTaken[m.Module] {
|
||||
takes = append(takes, m.Module)
|
||||
}
|
||||
}
|
||||
if !filterAssigned && !isTaken[filter] {
|
||||
takes = append(takes, filter)
|
||||
}
|
||||
sort.Strings(takes)
|
||||
b.WriteString("\n the flip takes:\n")
|
||||
if len(takes) == 0 {
|
||||
b.WriteString(" nothing — every module is taken already\n")
|
||||
}
|
||||
for _, m := range takes {
|
||||
fmt.Fprintf(&b, " %s\n", m)
|
||||
said = append(said, "take "+m)
|
||||
// Every kind it holds — a directory, a service, an archive, a process, a user as well as a
|
||||
// file (novox/hq ADR 0103) — each said, and each part of what the flip is asked to act on.
|
||||
for _, h := range reported.Held {
|
||||
if h.Module != m {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(&b, " replacing the found %s", heldLine(h))
|
||||
if h.Kept != "" {
|
||||
fmt.Fprintf(&b, ", original kept at %s", h.Kept)
|
||||
}
|
||||
b.WriteString("\n")
|
||||
said = append(said, "replace "+m+" "+heldLine(h)+" "+h.Kept)
|
||||
}
|
||||
}
|
||||
if !filterAssigned {
|
||||
fmt.Fprintf(&b, "\n and assigns %s, which loads the mesh's filter in place of its guard\n", filter)
|
||||
}
|
||||
fw := reported.Firewall
|
||||
if fw == "" || fw == "none" {
|
||||
b.WriteString(" no firewall was found on the machine; the mesh's filter is its first\n")
|
||||
} else {
|
||||
fmt.Fprintf(&b, " the found firewall (%s) is disabled, never flushed: its configuration stays on disk\n", fw)
|
||||
}
|
||||
said = append(said, fmt.Sprintf("filter %s assigned=%t firewall=%s", filter, filterAssigned, fw))
|
||||
// Sorted: the same account, reported in another order, is the same preview.
|
||||
sort.Strings(said)
|
||||
sum := sha256.Sum256([]byte(strings.Join(said, "\n")))
|
||||
return strings.TrimRight(b.String(), "\n"), hex.EncodeToString(sum[:])[:12]
|
||||
}
|
||||
|
||||
// derivedFilter is what the filter the flip loads is rendered from, as AsNftables renders it.
|
||||
type derivedFilter struct {
|
||||
rules []catalogue.Rule
|
||||
foundation []int
|
||||
// mesh is every address on the private network; outward says the machine faces outside.
|
||||
mesh []string
|
||||
outward bool
|
||||
}
|
||||
|
||||
// closesOutside is what a narrowing from everywhere to the private network is called: it closes.
|
||||
const closesOutside = "WILL CLOSE to everything outside the private network"
|
||||
|
||||
// fate is what the derived filter does to one reachable thing: which module declares it and from
|
||||
// where, or that it will close — wholly, or to everything outside the private network. Rendered
|
||||
// exactly as AsNftables admits it, ssh included.
|
||||
func (d derivedFilter) fate(r inventory.Reach) string {
|
||||
// Bound to an address on the private network, it was never reachable from outside it, so
|
||||
// admitting it from the mesh narrows nothing.
|
||||
onMesh := slices.Contains(d.mesh, strings.Trim(r.Address, "[]"))
|
||||
if r.Protocol == "tcp" && r.Port == catalogue.SSHPort {
|
||||
// From everywhere only when the machine faces outward or the mesh has no addresses to
|
||||
// narrow it to; otherwise from the private network only.
|
||||
if d.outward || len(d.mesh) == 0 || onMesh {
|
||||
return "stays open — ssh is never closed"
|
||||
}
|
||||
return closesOutside + " — ssh stays open from the mesh, never closed there"
|
||||
}
|
||||
for _, port := range d.foundation {
|
||||
if r.Protocol == "tcp" && r.Port == port {
|
||||
return "stays open — the mesh's own, from anywhere"
|
||||
}
|
||||
}
|
||||
for _, rule := range d.rules {
|
||||
if rule.Port != r.Port || rule.Protocol != r.Protocol {
|
||||
continue
|
||||
}
|
||||
by := strings.Join(rule.Because, ", ")
|
||||
switch rule.From {
|
||||
case catalogue.FromMachine:
|
||||
return fmt.Sprintf("WILL CLOSE to the network — declared by %s for this machine only", by)
|
||||
case catalogue.FromMesh:
|
||||
if len(d.mesh) == 0 {
|
||||
return fmt.Sprintf("WILL CLOSE — declared by %s from the mesh, and this node "+
|
||||
"knows no mesh addresses", by)
|
||||
}
|
||||
if !onMesh {
|
||||
return fmt.Sprintf("%s — declared by %s from the mesh only", closesOutside, by)
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("declared by %s (from %s)", by, rule.From)
|
||||
}
|
||||
return "WILL CLOSE — no module assigned here declares it"
|
||||
}
|
||||
|
||||
// countReachable is how much of a node's account of itself names something off the machine.
|
||||
// Loopback is left out for the same reason the preview leaves it out: nothing outside reaches it,
|
||||
// so a report of loopback alone says nothing about what the filter would close.
|
||||
func countReachable(reported inventory.Adoption) int {
|
||||
n := 0
|
||||
for _, r := range reported.Reachable {
|
||||
if !loopback(r.Address) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// heldLine is one thing a node holds as found, as take and the converge preview both say it.
|
||||
func heldLine(h inventory.Held) string {
|
||||
return fmt.Sprintf("%s %s (%s)", h.Kind, h.Target, h.ID)
|
||||
}
|
||||
|
||||
// loopback is an address nothing off the machine reaches.
|
||||
func loopback(address string) bool {
|
||||
a := strings.Trim(address, "[]")
|
||||
return strings.HasPrefix(a, "127.") || a == "::1" || a == "localhost"
|
||||
}
|
||||
|
||||
// adopt returns a converged node to adopted: the mesh's filter is unloaded, the guard restored,
|
||||
// the found firewall enabled again and the openings converged through it once more. What was
|
||||
// taken stays taken.
|
||||
func adopt(ctx context.Context, open *stores, node string) (string, error) {
|
||||
inv := open.inventory
|
||||
record, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if record.Adopted {
|
||||
return "", fmt.Errorf("%s is adopted already", node)
|
||||
}
|
||||
held, release, err := holdNodes(ctx, open, []string{node})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer release()
|
||||
ctx = held
|
||||
if err := inv.SetAdopted(ctx, node, true); err != nil {
|
||||
return "", err
|
||||
}
|
||||
said := fmt.Sprintf("%s is adopted; what was taken on it stays taken", node)
|
||||
if err := sendNodes(ctx, open, []string{node}); err != nil {
|
||||
return said + "\n and it could not be sent: run `push " + node + "`", err
|
||||
}
|
||||
return said + "\n sent: the host unloads the mesh's filter and enables the firewall it found", nil
|
||||
}
|
||||
|
||||
// takeCommand, convergeCommand and adoptCommand are the command line's adapters to the acts above.
|
||||
func takeCommand(ctx context.Context, args []string) error {
|
||||
if len(args) != 2 {
|
||||
return errors.New("take <node> <module>")
|
||||
}
|
||||
return runAct(ctx, func(open *stores) (string, error) { return take(ctx, open, args[0], args[1]) })
|
||||
}
|
||||
|
||||
func convergeCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("converge", flag.ContinueOnError)
|
||||
yes := set.String("yes", "", "do it, naming the digest the preview printed; without it, only "+
|
||||
"the preview")
|
||||
filter := set.String("filter", DefaultFilter, "the module that loads the mesh's filter")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("converge <node> [--yes <digest>] [--filter nftables]")
|
||||
}
|
||||
return runAct(ctx, func(open *stores) (string, error) {
|
||||
return converge(ctx, open, positionals[0], *yes != "", *yes, *filter)
|
||||
})
|
||||
}
|
||||
|
||||
func adoptCommand(ctx context.Context, args []string) error {
|
||||
if len(args) != 1 {
|
||||
return errors.New("adopt <node>")
|
||||
}
|
||||
return runAct(ctx, func(open *stores) (string, error) { return adopt(ctx, open, args[0]) })
|
||||
}
|
||||
|
||||
func runAct(ctx context.Context, act func(*stores) (string, error)) error {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
said, err := act(open)
|
||||
if said != "" {
|
||||
fmt.Println(said)
|
||||
}
|
||||
if err != nil && said != "" {
|
||||
fmt.Println()
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -94,19 +94,29 @@ func (n notYet) Who(*http.Request) (string, error) {
|
||||
func commands(who Authenticator) http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
|
||||
mux.HandleFunc("POST /assign", acting(who, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||
mux.HandleFunc("POST /assign", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||
return assign(ctx, open, in.Node, in.Module)
|
||||
}))
|
||||
mux.HandleFunc("POST /unassign", acting(who, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||
mux.HandleFunc("POST /unassign", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||
return unassign(ctx, open, in.Node, in.Module)
|
||||
}))
|
||||
// Adoption (novox/hq ADR 0100): the same acts as `take`, `converge` and `adopt`.
|
||||
mux.HandleFunc("POST /take", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||
return take(ctx, open, in.Node, in.Module)
|
||||
}))
|
||||
mux.HandleFunc("POST /converge", acting(who, false, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||
return converge(ctx, open, in.Node, in.Yes, in.Digest, in.Filter)
|
||||
}))
|
||||
mux.HandleFunc("POST /adopt", acting(who, false, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||
return adopt(ctx, open, in.Node)
|
||||
}))
|
||||
|
||||
// Anything else is said plainly, because a command surface answering 404 to a verb somebody
|
||||
// expected is indistinguishable from one that is down.
|
||||
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
refuse(w, http.StatusNotFound, fmt.Errorf(
|
||||
"%s %s is not something this mesh can be asked; it accepts POST /assign and "+
|
||||
"POST /unassign", r.Method, r.URL.Path))
|
||||
"%s %s is not something this mesh can be asked; it accepts POST /assign, "+
|
||||
"POST /unassign, POST /take, POST /converge and POST /adopt", r.Method, r.URL.Path))
|
||||
})
|
||||
return mux
|
||||
}
|
||||
@@ -114,11 +124,17 @@ func commands(who Authenticator) http.Handler {
|
||||
type request struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
// Yes, Digest and Filter are converge's: do it rather than preview it, the digest of the
|
||||
// preview it acts on, and which module loads the mesh's filter.
|
||||
Yes bool `json:"yes,omitempty"`
|
||||
Digest string `json:"digest,omitempty"`
|
||||
Filter string `json:"filter,omitempty"`
|
||||
}
|
||||
|
||||
// acting is the shape every route shares: authenticate, read, act, answer.
|
||||
func acting(
|
||||
who Authenticator,
|
||||
needsModule bool,
|
||||
do func(context.Context, *stores, request) (string, error),
|
||||
) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -131,8 +147,12 @@ func acting(
|
||||
refuse(w, http.StatusBadRequest, fmt.Errorf("this is not a request this understands: %w", err))
|
||||
return
|
||||
}
|
||||
if in.Node == "" || in.Module == "" {
|
||||
refuse(w, http.StatusBadRequest, errors.New(`both "node" and "module" are needed`))
|
||||
if in.Node == "" || (needsModule && in.Module == "") {
|
||||
if needsModule {
|
||||
refuse(w, http.StatusBadRequest, errors.New(`both "node" and "module" are needed`))
|
||||
} else {
|
||||
refuse(w, http.StatusBadRequest, errors.New(`"node" is needed`))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -37,13 +37,13 @@ func askCommand(ctx context.Context, args []string) error {
|
||||
arguments = json.RawMessage(positionals[2])
|
||||
}
|
||||
|
||||
server, err := link.Connect(nil, nil)
|
||||
server, err := connectLink(ctx, nil, nil, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer server.Close()
|
||||
|
||||
answer, err := link.Ask(ctx, server.Channel(), module, tool, arguments, *wait)
|
||||
answer, err := link.Ask(ctx, server.Bus(), module, tool, arguments, *wait)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -137,6 +137,9 @@ type view struct {
|
||||
Unresolved []blockedMachine
|
||||
// Network is why the private network could not be computed, when it could not.
|
||||
Network string
|
||||
// Adopted is every node still adopted (novox/hq ADR 0100). Not broken: nothing forces the
|
||||
// flip, so a node left adopted is shown rather than read as converged.
|
||||
Adopted []string
|
||||
At string
|
||||
}
|
||||
|
||||
@@ -181,7 +184,7 @@ type staleModule struct {
|
||||
|
||||
func viewOf(asked answers) view {
|
||||
out := view{Machines: len(asked.nodes), At: time.Now().Format("15:04:05"),
|
||||
Network: asked.network}
|
||||
Network: asked.network, Adopted: adoptedNodes(asked.nodes)}
|
||||
var blocked []string
|
||||
for name := range asked.refused {
|
||||
blocked = append(blocked, name)
|
||||
@@ -311,6 +314,10 @@ new, switched off, or unreachable.</p>
|
||||
<p class="quiet">Never told is not out of date: nobody has asked that machine to be anything yet.
|
||||
Both are sent by <code>push --behind</code>.</p>
|
||||
{{else}}<p class="quiet">Every machine is running what the mesh would send it.</p>{{end}}
|
||||
{{if .Adopted}}
|
||||
<h2>Which machines are adopted?</h2>
|
||||
<ul>{{range .Adopted}}<li><strong>{{.}}</strong> <span class="quiet">adopted — what was found on it is kept until each module is taken</span></li>{{end}}</ul>
|
||||
{{end}}
|
||||
{{end}}
|
||||
|
||||
<footer>Read at {{.At}}. This page holds nothing and changes nothing.</footer>
|
||||
|
||||
+207
-98
@@ -2,8 +2,6 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
@@ -31,6 +29,51 @@ import (
|
||||
// control plane may send a machine is bounded by the declaration language. This is the shape the
|
||||
// builder module will take when it is given work over the broker; today a person runs it, and the
|
||||
// mesh records the result the same way either way.
|
||||
// buildOn rebuilds every module the mesh holds that stands on the named module's artifacts — the
|
||||
// rebuild a changed base needs, which nothing else asks for: their sources did not move, and
|
||||
// "behind" does not see a base that did (novox/hq 04-ISSUES/131). Bases first among them too.
|
||||
func buildOn(ctx context.Context, base string, wait time.Duration) error {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
held, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
against, err := open.inventory.BuiltAgainst(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var on []inventory.Entry
|
||||
for _, e := range held {
|
||||
if standsOnModule(e, base, against) {
|
||||
on = append(on, e)
|
||||
}
|
||||
}
|
||||
if len(on) == 0 {
|
||||
fmt.Printf("nothing the mesh holds stands on %s\n", base)
|
||||
return nil
|
||||
}
|
||||
on = orderByBases(on, against)
|
||||
fmt.Printf("%d module(s) stand on %s:\n", len(on), base)
|
||||
var failed []string
|
||||
for _, e := range on {
|
||||
fmt.Printf("--- %s\n", e.Manifest.Module)
|
||||
source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
|
||||
if err := buildOne(ctx, source, e.Source.Path, e.Source.Ref, wait); err != nil {
|
||||
fmt.Printf(" %v\n", err)
|
||||
failed = append(failed, e.Manifest.Module)
|
||||
}
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
return fmt.Errorf("%d of %d could not be built: %s", len(failed), len(on), strings.Join(failed, ", "))
|
||||
}
|
||||
fmt.Printf("\n%d module(s) rebuilt on %s. `push --behind` sends them on\n", len(on), base)
|
||||
return nil
|
||||
}
|
||||
|
||||
func buildCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("build", flag.ContinueOnError)
|
||||
ref := set.String("ref", "", "the branch, tag or commit to build")
|
||||
@@ -46,28 +89,52 @@ func buildCommand(ctx context.Context, args []string) error {
|
||||
// retype each repository is asking them to be the loop. Naming a repository and asking which
|
||||
// ones need building are different requests, so they are not combined.
|
||||
behind := set.Bool("behind", false, "every module the mesh holds older than its source has")
|
||||
on := set.String("on", "", "rebuild every module that stands on this module's artifacts — the rebuild a changed base needs")
|
||||
// A repository on the mesh's own forge, named by its path there (novox/hq ADR 0111). Without it
|
||||
// the repository is external, cloned exactly as given — see source.go.
|
||||
self := set.Bool("self", false, "the repository is a path on the forge holding the git seat")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *on != "" {
|
||||
if len(positionals) != 0 || *behind || *self {
|
||||
return errors.New("build --on <module> names a base and nothing else")
|
||||
}
|
||||
return buildOn(ctx, *on, *wait)
|
||||
}
|
||||
|
||||
if *behind {
|
||||
if len(positionals) != 0 {
|
||||
if len(positionals) != 0 || *self {
|
||||
return errors.New("build <repository> or build --behind, not both: one names a " +
|
||||
"repository and the other asks which need building")
|
||||
}
|
||||
return buildBehind(ctx, *wait)
|
||||
}
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("build <repository> [--ref R] [--wait D] [--dry-run]")
|
||||
return errors.New("build <repository> [--self] [--path P] [--ref R] [--wait D] [--dry-run] | build --behind | build --on <module>")
|
||||
}
|
||||
source := buildSource{Repository: positionals[0]}
|
||||
if *self {
|
||||
if err := onASeat(source.Repository); err != nil {
|
||||
return err
|
||||
}
|
||||
source.Seat = gitSeat
|
||||
}
|
||||
|
||||
if *dryRun {
|
||||
return buildAndShow(ctx, positionals[0], *path, *ref, *wait)
|
||||
return buildAndShow(ctx, source, *path, *ref, *wait)
|
||||
}
|
||||
return buildOne(ctx, positionals[0], *path, *ref, *wait)
|
||||
return buildOne(ctx, source, *path, *ref, *wait)
|
||||
}
|
||||
|
||||
// buildFrom turns what a builder said into what the mesh keeps.
|
||||
//
|
||||
// **By digest and path, never by where it was pushed** (novox/hq 04-ISSUES/102). The builder
|
||||
// says `<registry>:<port>/<module>/<artifact>@sha256:…`; the mesh records the artifact-store
|
||||
// reference and composes the store's address back in where a reference is used. `against` — what
|
||||
// the build stood on, the catalogue's edge — is recorded the same way, so an edge names a module's
|
||||
// artifact and not the machine it was pulled from.
|
||||
func buildFrom(result link.BuildResult) inventory.Build {
|
||||
kept := inventory.Build{
|
||||
ID: result.ID, Repository: result.Repository, Ref: result.Ref,
|
||||
@@ -77,16 +144,24 @@ func buildFrom(result link.BuildResult) inventory.Build {
|
||||
// edges, and it is not always listening when a build happens — on a fresh mesh it cannot
|
||||
// be, for exactly the modules it needs most. Keeping them is what makes a replay able to
|
||||
// rebuild the graph rather than a list of names.
|
||||
Path: result.Path, Manifest: result.Manifest, Against: result.Against,
|
||||
Path: result.Path,
|
||||
}
|
||||
for _, ref := range result.Against {
|
||||
kept.Against = append(kept.Against, catalogue.Recorded(ref))
|
||||
}
|
||||
var announced []inventory.Artifact
|
||||
for _, made := range result.Made {
|
||||
kept.Made = append(kept.Made, inventory.Artifact{
|
||||
announced = append(announced, inventory.Artifact{
|
||||
Name: made.Name, Kind: made.Kind, Reference: made.Reference,
|
||||
})
|
||||
kept.Made = append(kept.Made, inventory.Artifact{
|
||||
Name: made.Name, Kind: made.Kind, Reference: catalogue.Recorded(made.Reference),
|
||||
})
|
||||
}
|
||||
kept.Manifest = recordedManifest(result.Manifest, announced)
|
||||
// The module name comes from the manifest, which only exists when the build got that far.
|
||||
if len(result.Manifest) > 0 {
|
||||
if m, err := catalogue.ParseManifest(result.Manifest); err == nil {
|
||||
if len(kept.Manifest) > 0 {
|
||||
if m, err := catalogue.ParseManifest(kept.Manifest); err == nil {
|
||||
kept.Module = m.Module
|
||||
}
|
||||
}
|
||||
@@ -186,85 +261,45 @@ func builderCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
name := positionals[1]
|
||||
|
||||
management, err := broker.ManagementFromEnvironment()
|
||||
// **The build machine's credential is a module's credential** (novox/hq ADR 0131, design 28
|
||||
// task 5.5): minted into the mesh's records and sealed to the machine as the builder module's
|
||||
// broker secret, usable at the next push — the same act `module issue` performs, and the same
|
||||
// account the composed user list carries. Nothing is created on a server; the bus reads the list.
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// The same shape of secret a token carries: enough entropy that guessing is not a strategy,
|
||||
// and safe to put in a URL because that is where it goes.
|
||||
raw := make([]byte, 32)
|
||||
if _, err := rand.Read(raw); err != nil {
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
password := base64.RawURLEncoding.EncodeToString(raw)
|
||||
if err := management.CreateBuilderAccount(ctx, name, password); err != nil {
|
||||
m, known := shelf[*module]
|
||||
if !known {
|
||||
return fmt.Errorf("%s is not in the catalogue; `module add` it first", *module)
|
||||
}
|
||||
fmt.Printf("build machine %s: ", name)
|
||||
node := *forNode
|
||||
if node == "" {
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Manifest.Module == *module && len(e.On) > 0 {
|
||||
node = e.On[0]
|
||||
}
|
||||
}
|
||||
}
|
||||
if node == "" {
|
||||
return fmt.Errorf("%s is assigned nowhere; `assign <machine> %s` first, or say --node", *module, *module)
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Printf("broker account %s created, scoped to the %s queue and the %s exchange\n\n",
|
||||
name, link.BuildQueue, link.Exchange)
|
||||
|
||||
if *forNode != "" {
|
||||
known, err := broker.FromEnvironment()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot deliver a credential without knowing where the broker is: %w", err)
|
||||
}
|
||||
inv, err := openInventory(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer inv.Close()
|
||||
|
||||
brokerAddr, err := brokerReachableAt(ctx, inv, known, *forNode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// The URL and what verifies the broker, together. A mesh's broker presents a certificate
|
||||
// of the mesh's own, which is in no public trust store — so a URL on its own reaches only
|
||||
// a broker somebody else vouches for, and the connection fails at TLS with an error about
|
||||
// an unknown authority rather than about a missing pin.
|
||||
//
|
||||
// **The same two facts a node's token carries** (novox/hq ADR 0004), delivered the same
|
||||
// way: out of band relative to the broker, so what is trusted does not come from the thing
|
||||
// being trusted.
|
||||
held, err := json.Marshal(struct {
|
||||
URL string `json:"url"`
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
}{
|
||||
URL: fmt.Sprintf("amqps://%s:%s@%s/", name, password, brokerAddr),
|
||||
Fingerprint: known.Fingerprint,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.AcceptSecretForModule(ctx, *forNode, *module, "broker", string(held)); err != nil {
|
||||
return err
|
||||
}
|
||||
// Not printed. It is sealed to that machine and the mesh cannot read it back, which is
|
||||
// the whole point — printing it here would put the one copy that matters on a terminal.
|
||||
fmt.Printf(" sealed to %s, for the %s module. It arrives with the next push.\n",
|
||||
*forNode, *module)
|
||||
fmt.Printf(" run `push %s` to send it\n", *forNode)
|
||||
return nil
|
||||
}
|
||||
|
||||
// The whole line only when the address is known. A URL with a placeholder where the host
|
||||
// should be is a URL somebody pastes and then debugs, and the placeholder is the last thing
|
||||
// they look at.
|
||||
if known, err := broker.FromEnvironment(); err == nil {
|
||||
fmt.Printf(" MESH_BROKER_AMQP=amqps://%s:%s@%s/\n\n", name, password, known.Address)
|
||||
} else {
|
||||
fmt.Printf(" the password is %s\n\n", password)
|
||||
fmt.Printf(" This control plane has no %s, so it cannot say where the broker is.\n"+
|
||||
" Put the password in MESH_BROKER_AMQP on the build machine.\n\n",
|
||||
broker.AddressVar)
|
||||
}
|
||||
// Shown once, like a token, and for the same reason: what is stored is the broker's own hash
|
||||
// of it, and a control plane that could show it back would be a control plane that holds it.
|
||||
fmt.Println("This is the only time it is shown.")
|
||||
return nil
|
||||
return issueOnTheNewBus(ctx, inv, m, node, address)
|
||||
}
|
||||
|
||||
// buildBehind builds every module the mesh holds older than its source has.
|
||||
@@ -309,13 +344,22 @@ func buildBehind(ctx context.Context, wait time.Duration) error {
|
||||
}
|
||||
fmt.Println()
|
||||
|
||||
// Bases first: a module built before the module it stands on is built against the old one
|
||||
// and reports success (novox/hq 04-ISSUES/131).
|
||||
against, err := inv.BuiltAgainst(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stale = orderByBases(stale, against)
|
||||
|
||||
var failed []string
|
||||
for _, e := range stale {
|
||||
fmt.Printf("--- %s\n", e.Manifest.Module)
|
||||
// Its own recorded ref, not its head commit: a module tracking a branch should be built
|
||||
// from that branch, and pinning to the commit the mesh happened to notice would quietly
|
||||
// turn a tracked branch into a pin.
|
||||
if err := buildOne(ctx, e.Source.Repository, e.Source.Path, e.Source.Ref, wait); err != nil {
|
||||
source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
|
||||
if err := buildOne(ctx, source, e.Source.Path, e.Source.Ref, wait); err != nil {
|
||||
fmt.Printf(" %v\n", err)
|
||||
failed = append(failed, e.Manifest.Module)
|
||||
}
|
||||
@@ -333,14 +377,21 @@ func buildBehind(ctx context.Context, wait time.Duration) error {
|
||||
// buildOne asks a build machine for one repository and records everything that came back.
|
||||
//
|
||||
// Separated from the command so `--behind` can walk a list without a second path to the same act.
|
||||
func buildOne(ctx context.Context, repository, path, ref string, wait time.Duration) error {
|
||||
func buildOne(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error {
|
||||
// Before anything is asked of a builder: a source on a seat nobody holds is refused here, with
|
||||
// the reason, rather than sent to a machine to fail at `git clone`.
|
||||
repository, err := cloneFrom(ctx, source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ident, err := openIdentity(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer ident.Close()
|
||||
|
||||
server, err := link.Connect(nil, nil)
|
||||
server, err := connectLink(ctx, nil, nil, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -355,7 +406,10 @@ func buildOne(ctx context.Context, repository, path, ref string, wait time.Durat
|
||||
Ref: ref,
|
||||
Held: heldBy(ctx),
|
||||
}
|
||||
fmt.Printf("asked for %s", request.Repository)
|
||||
fmt.Printf("asked for %s", source)
|
||||
if source.Seat != "" {
|
||||
fmt.Printf(" (%s)", repository)
|
||||
}
|
||||
if path != "" {
|
||||
fmt.Printf(" at %s", path)
|
||||
}
|
||||
@@ -364,7 +418,13 @@ func buildOne(ctx context.Context, repository, path, ref string, wait time.Durat
|
||||
}
|
||||
fmt.Println()
|
||||
|
||||
result, err := link.RequestBuild(ctx, server.Channel(), request, wait)
|
||||
ask, err := askOver(server)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer ask.Close()
|
||||
|
||||
result, err := ask.Submit(ctx, request, wait)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -378,7 +438,8 @@ func buildOne(ctx context.Context, repository, path, ref string, wait time.Durat
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
if err := inv.RecordBuild(ctx, buildFrom(result)); err != nil {
|
||||
kept := buildFrom(result)
|
||||
if err := inv.RecordBuild(ctx, kept); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -388,24 +449,33 @@ func buildOne(ctx context.Context, repository, path, ref string, wait time.Durat
|
||||
return fmt.Errorf("%s could not build %s:\n%s", result.On, result.Repository, result.Failed)
|
||||
}
|
||||
|
||||
for _, made := range result.Made {
|
||||
// Said as recorded: what each artifact is, not where this builder happened to push it.
|
||||
for _, made := range kept.Made {
|
||||
fmt.Printf(" %-12s %s %s\n", made.Name, made.Kind, made.Reference)
|
||||
}
|
||||
|
||||
// Parsed with the same parser a hand-written manifest goes through. A second path would be a
|
||||
// second thing to disagree about what a manifest is.
|
||||
manifest, err := catalogue.ParseManifest(result.Manifest)
|
||||
// second thing to disagree about what a manifest is. The manifest as recorded, so the catalogue
|
||||
// holds references by digest and path and every declaration composes the store's address in.
|
||||
manifest, err := catalogue.ParseManifest(kept.Manifest)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s built %s and what came back is not a manifest: %w",
|
||||
result.On, result.Repository, err)
|
||||
}
|
||||
|
||||
// Recorded with where it came from, so "is this current?" is answerable without building it
|
||||
// again (novox/hq ADR 0009).
|
||||
if err := inv.RegisterModule(ctx, manifest, inventory.Source{
|
||||
// again (novox/hq ADR 0009). **For a source on a seat, as the path and the seat, never the URL
|
||||
// just cloned** (ADR 0111): the URL is where the forge runs today, and recording it would put
|
||||
// the forge's address back into every module built from it. The build log above keeps the URL,
|
||||
// because that is what was cloned.
|
||||
recorded := inventory.Source{
|
||||
Repository: result.Repository, Path: result.Path, Ref: result.Ref,
|
||||
BuiltFrom: result.Commit, Head: result.Commit,
|
||||
}); err != nil {
|
||||
}
|
||||
if source.Seat != "" {
|
||||
recorded.Repository, recorded.Seat = source.Repository, source.Seat
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, manifest, recorded); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("\n%s %s, built on %s from %s\n",
|
||||
@@ -415,19 +485,29 @@ func buildOne(ctx context.Context, repository, path, ref string, wait time.Durat
|
||||
}
|
||||
|
||||
// buildAndShow builds and prints the manifest without recording anything.
|
||||
func buildAndShow(ctx context.Context, repository, path, ref string, wait time.Duration) error {
|
||||
func buildAndShow(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error {
|
||||
repository, err := cloneFrom(ctx, source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ident, err := openIdentity(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer ident.Close()
|
||||
server, err := link.Connect(nil, nil)
|
||||
server, err := connectLink(ctx, nil, nil, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer server.Close()
|
||||
|
||||
result, err := link.RequestBuild(ctx, server.Channel(), link.BuildRequest{
|
||||
ask, err := askOver(server)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer ask.Close()
|
||||
|
||||
result, err := ask.Submit(ctx, link.BuildRequest{
|
||||
ID: fmt.Sprintf("%s-%d", "build", time.Now().UnixNano()),
|
||||
Repository: repository, Path: path, Ref: ref,
|
||||
Held: heldBy(ctx),
|
||||
@@ -481,6 +561,9 @@ type answers struct {
|
||||
// all, and one that does gets a refusal naming exactly what is missing — which is a better sentence
|
||||
// than a build command refusing to start because a query did not run. So the store not opening is
|
||||
// reported and the build goes ahead without it.
|
||||
//
|
||||
// Routed through the artifact store as the network reaches it now (novox/hq 04-ISSUES/102): a
|
||||
// base is recorded by digest and path, and a build machine needs something it can pull.
|
||||
func heldBy(ctx context.Context) map[string]string {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
@@ -494,5 +577,31 @@ func heldBy(ctx context.Context) map[string]string {
|
||||
fmt.Fprintf(os.Stderr, "could not read what this mesh has built: %v\n", err)
|
||||
return nil
|
||||
}
|
||||
return held
|
||||
address, err := whereABuilderReachesTheStore(ctx, open.inventory)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not find the artifact store on this mesh's network, so a "+
|
||||
"module naming a base will be handed a reference nothing can fetch: %v\n", err)
|
||||
return held
|
||||
}
|
||||
if address == "" {
|
||||
return held
|
||||
}
|
||||
routed := make(map[string]string, len(held))
|
||||
for repository, reference := range held {
|
||||
routed[repository] = catalogue.Rerouted(reference, address)
|
||||
}
|
||||
return routed
|
||||
}
|
||||
|
||||
// askOver opens the way a build is asked for, on whichever bus the mesh is on.
|
||||
//
|
||||
// **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5). On the bus
|
||||
// the mesh runs on today this needs the controller's own connection, so it is handed one; on the bus
|
||||
// being built it dials, because a build request is a one-shot and holds nothing else.
|
||||
func askOver(_ *link.Server) (link.Builders, error) {
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return link.BuildsOverNATS(address)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
package main
|
||||
|
||||
// The broker opening belongs only on the node that listens on it (novox/hq: it leaked onto
|
||||
// every enrolled node's declaration, opening a from-anywhere hole for a port nothing there
|
||||
// serves). foundationPortsFor is the scope.
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
func TestTheBrokerHostGetsTheFoundationOpening(t *testing.T) {
|
||||
broker := catalogue.Manifest{Module: "lavinmq", Listens: []catalogue.Listening{
|
||||
{Port: 5671, Protocol: "tcp", From: "mesh"},
|
||||
{Port: 5672, Protocol: "tcp", From: "mesh"},
|
||||
}}
|
||||
got := foundationPortsFor(5671, []catalogue.Manifest{broker})
|
||||
if len(got) != 1 || got[0] != 5671 {
|
||||
t.Fatalf("the node that listens on the broker port keeps it; got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodeThatOnlyDialsTheBrokerGetsNoOpening(t *testing.T) {
|
||||
// ace's set: things that reach the broker as a client, none listening on 5671.
|
||||
ace := []catalogue.Manifest{
|
||||
{Module: "plex", Listens: []catalogue.Listening{{Port: 32400, Protocol: "tcp", From: "anywhere"}}},
|
||||
{Module: "postgres", Listens: []catalogue.Listening{{Port: 5432, Protocol: "tcp", From: "mesh"}}},
|
||||
}
|
||||
if got := foundationPortsFor(5671, ace); got != nil {
|
||||
t.Fatalf("a node that only dials out opens nothing for the broker; got %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
)
|
||||
|
||||
// heldKey carries the nodes this call already holds, so an act that holds a node and then sends
|
||||
// through sendTo does not wait on itself.
|
||||
type heldKey struct{}
|
||||
|
||||
// holdNodes holds the named nodes for composing and sending their declarations (novox/hq ADR
|
||||
// 0100), skipping any the context already holds, and returns a context that says it holds them.
|
||||
// Release gives back only what this call took.
|
||||
func holdNodes(ctx context.Context, open *stores, names []string) (context.Context, func(), error) {
|
||||
already, _ := ctx.Value(heldKey{}).(map[string]bool)
|
||||
var take []string
|
||||
for _, n := range names {
|
||||
if !already[n] {
|
||||
take = append(take, n)
|
||||
}
|
||||
}
|
||||
if len(take) == 0 {
|
||||
return ctx, func() {}, nil
|
||||
}
|
||||
release, err := open.inventory.HoldNodes(ctx, take)
|
||||
if err != nil {
|
||||
return ctx, nil, err
|
||||
}
|
||||
held := map[string]bool{}
|
||||
for n := range already {
|
||||
held[n] = true
|
||||
}
|
||||
for _, n := range take {
|
||||
held[n] = true
|
||||
}
|
||||
return context.WithValue(ctx, heldKey{}, held), release, nil
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A cascade round gives its hold back on every way out: a declaration that cannot be marshalled
|
||||
// and a send that fails leave nobody waiting for the node.
|
||||
func TestASendRoundGivesItsHoldBackOnEveryWayOut(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
unmarshallable := func(context.Context, string) (sendable, error) {
|
||||
return sendable{Resources: []map[string]any{{"id": "x", "bad": make(chan int)}}}, nil
|
||||
}
|
||||
plain := func(context.Context, string) (sendable, error) {
|
||||
return sendable{Resources: []map[string]any{{"id": "x"}}}, nil
|
||||
}
|
||||
failing := func(readyNode, []byte) error { return errors.New("the broker went away") }
|
||||
fine := func(readyNode, []byte) error { return nil }
|
||||
|
||||
for name, round := range map[string]func() error{
|
||||
"a body that cannot be marshalled": func() error {
|
||||
_, err := sendRound(ctx, open, []string{"anchor"}, unmarshallable, fine)
|
||||
return err
|
||||
},
|
||||
"a send that fails": func() error {
|
||||
_, err := sendRound(ctx, open, []string{"anchor"}, plain, failing)
|
||||
return err
|
||||
},
|
||||
} {
|
||||
if err := round(); err == nil {
|
||||
t.Fatalf("%s was not an error", name)
|
||||
}
|
||||
waiting, cancel := context.WithTimeout(ctx, 2*time.Second)
|
||||
release, err := open.inventory.HoldNodes(waiting, []string{"anchor"})
|
||||
cancel()
|
||||
if err != nil {
|
||||
t.Fatalf("after %s the node is still held: %v", name, err)
|
||||
}
|
||||
release()
|
||||
}
|
||||
}
|
||||
@@ -98,6 +98,12 @@ func run() error {
|
||||
return moduleCommand(ctx, args[1:])
|
||||
case "assign", "unassign":
|
||||
return assignCommand(ctx, args[0], args[1:])
|
||||
case "take":
|
||||
return takeCommand(ctx, args[1:])
|
||||
case "converge":
|
||||
return convergeCommand(ctx, args[1:])
|
||||
case "adopt":
|
||||
return adoptCommand(ctx, args[1:])
|
||||
case "settings":
|
||||
return settingsCommand(ctx, args[1:])
|
||||
case "secret":
|
||||
@@ -108,6 +114,12 @@ func run() error {
|
||||
return planCommand(ctx, args[1:])
|
||||
case "push":
|
||||
return pushCommand(ctx, args[1:])
|
||||
case "rollout":
|
||||
return rolloutCommand(ctx, args[1:])
|
||||
case "seats":
|
||||
return seatsCommand(ctx, args[1:])
|
||||
case "seat":
|
||||
return seatCommand(ctx, args[1:])
|
||||
case "status":
|
||||
return statusCommand(ctx, args[1:])
|
||||
case "version":
|
||||
@@ -126,7 +138,7 @@ func usage() {
|
||||
fmt.Fprint(os.Stderr, `mesh-controller — the control plane
|
||||
|
||||
migrate bring each context's schema up to date
|
||||
node add <name> create a node record
|
||||
node add <name> [--adopted] create a node record; --adopted: the machine is in use
|
||||
node list the nodes this mesh knows about
|
||||
node show <name> what one machine reported it can do, and why
|
||||
node public-domain <name> the domain it composes its routed names under
|
||||
@@ -134,6 +146,7 @@ func usage() {
|
||||
node public-domain <name> --clear ...it faces the outside no longer
|
||||
token issue --node <name> a one-time right to join, for an existing record
|
||||
token issue --new <name> create the record and issue for it
|
||||
token issue ... --adopted ...for a machine in use, which joins adopted
|
||||
identity show this control plane's signing key
|
||||
broker show where the broker is, and what to expect there
|
||||
serve consume what nodes say, and answer
|
||||
@@ -150,10 +163,14 @@ func usage() {
|
||||
upgrade <name> roll-out [--together] ...send it to the machines running it
|
||||
upgrade <name> record ...record that they are behind, and send nothing
|
||||
status [--json] what is wrong, what is quiet, and what is out of date
|
||||
seats [--json] every seat this mesh defines, what it delivers, and who holds it
|
||||
board [--listen ADDR] the same three questions, as a page that holds nothing
|
||||
api --issuer URL [--listen A] assign and unassign over http, for a surface that is not here
|
||||
assign <node> <module> put a module on a node
|
||||
unassign <node> <module> take it off
|
||||
take <node> <module> cut a module over on an adopted node, once its data has moved
|
||||
converge <node> [--yes <digest>] [--filter nftables] preview, then make, an adopted node converged
|
||||
adopt <node> return a converged node to adopted; what was taken stays taken
|
||||
settings set <module> <file> what a module's config should say, for the whole mesh
|
||||
settings set <module> <file> --node <n> ...or for one machine
|
||||
settings clear <module> [--node <n>] take a layer away
|
||||
@@ -167,6 +184,7 @@ func usage() {
|
||||
operator key show the operator key, and what it can recover
|
||||
build <repository> [--ref R] have a build machine build it, and record what came out
|
||||
build --behind build every module the mesh holds older than its source
|
||||
build --on <module> rebuild every module that stands on this module's artifacts, bases first
|
||||
builds [<module>] what has been built lately, and what came of it
|
||||
builder issue <name> a broker account for a build machine, scoped to build work,
|
||||
delivered as the builder module's broker secret (module add it first)
|
||||
|
||||
@@ -73,7 +73,7 @@ func aMesh(t *testing.T) *stores {
|
||||
if err := open.inventory.RecordOverlayKey(t.Context(), record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.Assign(t.Context(), name, overlay.Name); err != nil {
|
||||
if _, err := open.inventory.Assign(t.Context(), name, overlay.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,8 +2,6 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
@@ -257,66 +255,15 @@ func moduleCommand(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
management, err := broker.ManagementFromEnvironment()
|
||||
// Which bus this mesh is on. A module gets a credential for exactly one, and the two are
|
||||
// made in entirely different ways: on the bus the mesh runs on today an account is a
|
||||
// management call, and on the bus being built it is a row the next composition writes into
|
||||
// the server's user list (novox/hq design 25 §4).
|
||||
busAddress, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The foundation owns the bus; make sure it exists before a module binds onto it.
|
||||
if err := management.EnsureEventExchanges(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
secret := make([]byte, 32)
|
||||
if _, err := rand.Read(secret); err != nil {
|
||||
return err
|
||||
}
|
||||
password := base64.RawURLEncoding.EncodeToString(secret)
|
||||
account, err := management.CreateModuleAccount(ctx, *forNode, module, password, m.Emits, m.Consumes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// A consumer's queue, with its dead-letter, is the foundation's to declare — its own account
|
||||
// may not (ADR 0043). Made now, so it exists before the module binds onto it.
|
||||
if len(m.Consumes) > 0 {
|
||||
if err := management.EnsureModuleQueue(ctx, *forNode, module); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
known, err := broker.FromEnvironment()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot deliver a credential without knowing where the broker is: %w", err)
|
||||
}
|
||||
brokerAddr, err := brokerReachableAt(ctx, inv, known, *forNode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The URL and what verifies the broker, together — a mesh's broker presents its own
|
||||
// certificate, in no public trust store, so a URL alone fails at TLS (as `builder issue`).
|
||||
held, err := json.Marshal(struct {
|
||||
URL string `json:"url"`
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
}{
|
||||
URL: fmt.Sprintf("amqps://%s:%s@%s/", account, password, brokerAddr),
|
||||
Fingerprint: known.Fingerprint,
|
||||
// The node and module the account is for, so the runtime names its queue as the mesh
|
||||
// scoped it (<node>.<module>.events) without a manifest having to interpolate a node.
|
||||
Node: *forNode,
|
||||
Module: module,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.AcceptSecretForModule(ctx, *forNode, module, "broker", string(held)); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("broker account %s created for %s, scoped to what it emits and consumes\n",
|
||||
account, module)
|
||||
fmt.Printf(" sealed to %s. It arrives with the next push — `push %s` to send it\n",
|
||||
*forNode, *forNode)
|
||||
return nil
|
||||
return issueOnTheNewBus(ctx, inv, m, *forNode, busAddress)
|
||||
|
||||
default:
|
||||
return fmt.Errorf("module has no %q; it has add, list, moved, forget and issue", args[0])
|
||||
@@ -567,3 +514,91 @@ func mayIssue(m catalogue.Manifest) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// issueOnTheNewBus gives an assigned module its credential on the bus being built.
|
||||
//
|
||||
// **Three things differ from a management call, and each is the point of the move.** The credential
|
||||
// is minted into the mesh's records and becomes usable at the next composition, so there is no
|
||||
// server to be reachable for this to work. The password travels beside the address rather than inside
|
||||
// it, because the runtime's contract already separates them and a credential embedded in a URL is one
|
||||
// that leaks into every log line that prints a connection. And the module's durable consumer is
|
||||
// derived from what it declared rather than declared by name, so a module cannot ask for delivery of
|
||||
// something it did not say it consumes.
|
||||
func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest,
|
||||
node, busAddress string) error {
|
||||
|
||||
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: m.Module}.Username()
|
||||
password, err := inv.MintBusPassword(ctx, inventory.BusUser{
|
||||
Username: user, Kind: inventory.BusModule, Node: node, Module: m.Module,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Where the module is told to find the bus, and what certificate it must present. The same pair
|
||||
// a node is told, for the same reason: a mesh's bus presents its own certificate, in no public
|
||||
// trust store, so an address alone fails at TLS.
|
||||
known, err := broker.FromEnvironment()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot deliver a credential without knowing where the bus is: %w", err)
|
||||
}
|
||||
reachable, err := brokerReachableAt(ctx, inv, known, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password)
|
||||
}
|
||||
|
||||
// issueWith is the delivery half: the minted password sealed to the machine as the module's broker
|
||||
// secret, and the module's consumer created where the bus can be reached. Split from the minting
|
||||
// so the move can issue every module against a bus whose address it worked out itself
|
||||
// (`rollout mint`, design 28 task 5.2) rather than the one in this process's environment.
|
||||
func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest,
|
||||
node, busAddress string, known broker.Broker, reachable, user, password string) error {
|
||||
held, err := json.Marshal(struct {
|
||||
URL string `json:"url"`
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
User string `json:"user"`
|
||||
Password string `json:"password"`
|
||||
}{
|
||||
URL: "nats://" + reachable, Fingerprint: known.Fingerprint,
|
||||
Node: node, Module: m.Module, User: user, Password: password,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.AcceptSecretForModule(ctx, node, m.Module, "broker", string(held)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// And how it hears what it consumes. Derived from its declaration, and only when it declared
|
||||
// something: a module that consumes nothing needs no consumer, and creating one would be a
|
||||
// durable subscription nobody reads.
|
||||
if consumer, needed := broker.ConsumerFor(broker.Principal{
|
||||
Kind: broker.KindModule, Node: node, Module: m.Module,
|
||||
Emits: m.Emits, Consumes: m.Consumes, Serves: m.Tools,
|
||||
}); needed {
|
||||
if busAddress == "" {
|
||||
fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push`, then "+
|
||||
"`rollout mint` again is harmless)\n", m.Module)
|
||||
} else {
|
||||
js, err := broker.Dial(busAddress)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the credential is minted and the mesh cannot reach the bus to create "+
|
||||
"how %s hears what it consumes: %w", m.Module, err)
|
||||
}
|
||||
defer js.Close()
|
||||
if err := js.EnsureConsumer(consumer); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("bus user %s minted for %s, scoped to what it emits and consumes\n", user, m.Module)
|
||||
fmt.Printf(" sealed to %s. It arrives with the next push — `push %s` to send it\n", node, node)
|
||||
fmt.Printf(" and it works once the bus has been told: the user list is composed into the " +
|
||||
"machine holding mesh-broker\n")
|
||||
return nil
|
||||
}
|
||||
|
||||
+312
-37
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -21,16 +22,33 @@ import (
|
||||
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
|
||||
// nothing in it was wrong, and no one edit was the one that should have been a new file.
|
||||
|
||||
func overlayCIDR() string {
|
||||
if v := strings.TrimSpace(os.Getenv(OverlayCIDRVar)); v != "" {
|
||||
return v
|
||||
// DefaultOverlayCIDR is the range the mesh allocates from when nothing says another.
|
||||
const DefaultOverlayCIDR = "10.42.0.0/16"
|
||||
|
||||
// overlayRange is the range the mesh allocates node addresses from.
|
||||
//
|
||||
// **The adopted tunnel's range first** (novox/hq ADR 0105): a hub that took over the tunnel it
|
||||
// found is at that tunnel's address, its peers are at theirs, and every node's address is
|
||||
// composed from the same range — the hub's, and every binding, hosts entry and endpoint derived
|
||||
// from it. Those are readers of this; none of them stores the range. Without an adopted tunnel,
|
||||
// the range genesis was told, or the default.
|
||||
func overlayRange(ctx context.Context, inv *inventory.Inventory) (string, error) {
|
||||
tunnel, _, adopted, err := inv.AdoptedTunnel(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "10.42.0.0/16"
|
||||
if adopted {
|
||||
return tunnel.Range, nil
|
||||
}
|
||||
if v := strings.TrimSpace(os.Getenv(OverlayCIDRVar)); v != "" {
|
||||
return v, nil
|
||||
}
|
||||
return DefaultOverlayCIDR, nil
|
||||
}
|
||||
|
||||
func overlayCommand(ctx context.Context, args []string) error {
|
||||
if len(args) == 0 {
|
||||
return errors.New("overlay place <node> [flags], or overlay show")
|
||||
return errors.New("overlay place <node> [flags], overlay name <address> <name>, or overlay show")
|
||||
}
|
||||
// Answered before anything is opened. A message about which command to use should not need a
|
||||
// database to say so, and needing one turns a redirect into a connection error.
|
||||
@@ -51,12 +69,29 @@ func overlayCommand(ctx context.Context, args []string) error {
|
||||
return overlayPlace(ctx, inv, args[1:])
|
||||
case "show":
|
||||
return overlayShow(ctx, open)
|
||||
case "name":
|
||||
return overlayName(ctx, inv, args[1:])
|
||||
|
||||
default:
|
||||
return fmt.Errorf("overlay has no %q; it has place and show", args[0])
|
||||
return fmt.Errorf("overlay has no %q; it has place, name and show", args[0])
|
||||
}
|
||||
}
|
||||
|
||||
// overlayName is the operator saying which machine a carried address is (novox/hq issue 112),
|
||||
// so the mesh answers for its name until the machine enrols and verifies it.
|
||||
func overlayName(ctx context.Context, inv *inventory.Inventory, args []string) error {
|
||||
if len(args) != 2 {
|
||||
return errors.New("overlay name <carried-address> <node-name>")
|
||||
}
|
||||
address, name := args[0], args[1]
|
||||
if err := inv.NamePeer(ctx, address, name); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("the peer at %s is %s until it enrols — the mesh answers for %s.<suffix> from the "+
|
||||
"operator's word, and enrolment under this key must use this name\n", address, name, name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func overlayPlace(ctx context.Context, inv *inventory.Inventory, args []string) error {
|
||||
if len(args) == 0 {
|
||||
return errors.New(
|
||||
@@ -110,16 +145,46 @@ func overlayPlace(ctx context.Context, inv *inventory.Inventory, args []string)
|
||||
}
|
||||
}
|
||||
|
||||
// A hub that took over a tunnel listens on that tunnel's port — it is what the peers dial, and
|
||||
// the reason the port is worth having (novox/hq ADR 0105). An endpoint on another port would
|
||||
// have the mesh's interface up where no peer is listening for it.
|
||||
found, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var tunnel inventory.Tunnel
|
||||
adoptsTunnel := false
|
||||
if *hub && found.Adopted {
|
||||
if t, err := inv.TunnelOf(ctx, node); err == nil {
|
||||
tunnel = t
|
||||
placed, _ := inv.Overlays(ctx)
|
||||
for _, o := range placed {
|
||||
if o.Name == node && o.Key == t.PublicKey {
|
||||
adoptsTunnel = true
|
||||
}
|
||||
}
|
||||
} else if !errors.Is(err, inventory.ErrNoTunnel) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if adoptsTunnel {
|
||||
if port := portOfEndpoint(*endpoint); port != strconv.Itoa(tunnel.Port) {
|
||||
return fmt.Errorf("%s takes over the tunnel it found on %s, which listens on port %d, and "+
|
||||
"its endpoint %q names another port: the peers dial the tunnel's port, so the hub's "+
|
||||
"endpoint must be on it", node, tunnel.Interface, tunnel.Port, *endpoint)
|
||||
}
|
||||
}
|
||||
|
||||
// Declared, all three. The address is evidence of reachability and is not the fact, and hub
|
||||
// election by address prefix fails silently (novox/hq ADR 0007).
|
||||
if err := inv.SetPlace(ctx, node, *endpoint, *site, *hub, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
found, err := inv.NodeByName(ctx, node)
|
||||
cidr, err := overlayRange(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
address, err := inv.AssignAddress(ctx, found.ID, overlayCIDR())
|
||||
address, err := inv.AssignAddress(ctx, found.ID, cidr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -128,6 +193,10 @@ func overlayPlace(ctx context.Context, inv *inventory.Inventory, args []string)
|
||||
fmt.Println(" credentials issued for it before this placement keep their old broker address —" +
|
||||
" `module issue` them again and push (novox/hq issue 059)")
|
||||
switch {
|
||||
case adoptsTunnel:
|
||||
fmt.Printf(" the hub — it takes over the tunnel it found on %s: range %s, port %d, "+
|
||||
"%d peer(s) carried until they enrol\n", tunnel.Interface, tunnel.Range, tunnel.Port,
|
||||
len(tunnel.Peers))
|
||||
case *hub:
|
||||
fmt.Println(" the hub — every node not sharing a site routes through it")
|
||||
case *endpoint == "":
|
||||
@@ -154,15 +223,47 @@ func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool,
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The tunnels adopted nodes take over, and the peers the hub's carries (novox/hq ADR 0105).
|
||||
tunnels, err := inv.Tunnels(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
carried, err := inv.CarriedPeers(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nodes := make([]overlay.Node, 0, len(places))
|
||||
for _, p := range places {
|
||||
if !on[p.Name] {
|
||||
continue
|
||||
}
|
||||
nodes = append(nodes, overlay.Node{
|
||||
n := overlay.Node{
|
||||
Name: p.Name, Key: p.Key, Endpoint: p.Endpoint,
|
||||
Site: p.Site, Hub: p.Hub, Address: p.Address,
|
||||
})
|
||||
}
|
||||
if t, takes := tunnels[p.Name]; takes && t.NodeAdopted {
|
||||
// Only an adopted node is told to take the found unit over: on a converged one there
|
||||
// is nothing found to keep, and the host refuses the field. The range and the carried
|
||||
// peers do not depend on the mode; the takeover does.
|
||||
//
|
||||
// **Refused, not composed, when the hub's record disagrees with the tunnel.** A
|
||||
// declaration that stopped the found unit and raised the mesh's interface on another
|
||||
// port or address would leave every peer dark while reporting the tunnel taken — so a
|
||||
// hub placed before it took the tunnel over (or at the wrong port) is named here, and
|
||||
// nothing is sent until it is re-placed.
|
||||
if wrong := disagrees(p, t.Tunnel); wrong != "" {
|
||||
return nil, fmt.Errorf("%s takes over the tunnel on %s and its placement disagrees with it: %s. "+
|
||||
"Re-place it — `overlay place %s --hub --endpoint <host>:%d …` — and push again; "+
|
||||
"nothing was composed", p.Name, t.Interface, wrong, p.Name, t.Port)
|
||||
}
|
||||
n.TakesOver = &overlay.TakeOver{Interface: t.Interface, Unit: t.Unit, Config: t.Config, Port: t.Port, MTU: t.MTU}
|
||||
}
|
||||
if p.Hub {
|
||||
for _, c := range carried {
|
||||
n.Carried = append(n.Carried, overlay.Carried{Key: c.PublicKey, Address: c.Address})
|
||||
}
|
||||
}
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
// Nobody was given it. An empty network is a legitimate mesh, not a broken one, so this
|
||||
@@ -170,7 +271,11 @@ func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool,
|
||||
// "no hub" to somebody who never asked for a network would be a lie about the cause.
|
||||
return overlay.Empty(), nil
|
||||
}
|
||||
g, err := overlay.From(nodes, overlayCIDR(), "")
|
||||
cidr, err := overlayRange(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
g, err := overlay.From(nodes, cidr, "")
|
||||
if g != nil {
|
||||
// The artifact store, as this network reaches it. Found rather than configured: the
|
||||
// provider is whichever module offers it, on whichever machine holds that module — and if
|
||||
@@ -292,6 +397,17 @@ func overlayShow(ctx context.Context, open *stores) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// The tunnel the hub took over, if any, and the peers carried from it (novox/hq ADR 0105):
|
||||
// listed apart from the nodes, because they are peers of the tunnel and not nodes of the
|
||||
// mesh until they enrol — and once one has, it is listed as the node it became.
|
||||
tunnel, hubName, adopted, err := open.inventory.AdoptedTunnel(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
carried, err := open.inventory.CarriedPeers(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, n := range nodes {
|
||||
place := n.Address
|
||||
if place == "" {
|
||||
@@ -301,22 +417,74 @@ func overlayShow(ctx context.Context, open *stores) error {
|
||||
}
|
||||
fmt.Printf("%-16s %-14s", n.Name, place)
|
||||
switch {
|
||||
case n.Hub && adopted:
|
||||
fmt.Printf(" hub — over the tunnel it took over on %s (range %s, port %d)",
|
||||
tunnel.Interface, tunnel.Range, tunnel.Port)
|
||||
case n.Hub && hubName == n.Name && tunnel.Interface != "":
|
||||
fmt.Printf(" hub — found a tunnel on %s and did NOT take it over: its key is not the tunnel's; "+
|
||||
"`mesh-host overlay take --tunnel %s` on the machine takes it", tunnel.Interface, tunnel.Interface)
|
||||
case n.Hub:
|
||||
fmt.Print(" hub")
|
||||
fmt.Print(" hub — found no tunnel; if the machine runs the predecessor's, " +
|
||||
"`mesh-host overlay take --tunnel <iface>` there adopts it (novox/hq ADR 0105)")
|
||||
case !n.Reachable():
|
||||
fmt.Print(" not dialable")
|
||||
}
|
||||
if n.Site != "" {
|
||||
fmt.Printf(" at %s", n.Site)
|
||||
}
|
||||
if n.TakesOver != nil && !n.Hub {
|
||||
fmt.Printf(" takes over %s", n.TakesOver.Interface)
|
||||
}
|
||||
fmt.Println()
|
||||
for _, p := range computed[n.Name] {
|
||||
fmt.Printf(" → %-14s %-18s %s\n", p.Name, p.Allowed, p.Why)
|
||||
}
|
||||
}
|
||||
if len(carried) > 0 {
|
||||
fmt.Printf("\npeers of the tunnel %s took over — not nodes of the mesh until they enrol:\n", hubName)
|
||||
for _, c := range carried {
|
||||
state := "not yet enrolled"
|
||||
if c.EnrolledAs != "" {
|
||||
state = "enrolled as " + c.EnrolledAs + ", which keeps this address"
|
||||
}
|
||||
fmt.Printf(" %-16s %-14s %s\n", overlay.CarriedName(c.PublicKey), c.Address, state)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// disagrees says how a node's placement differs from the tunnel it takes over — its address not
|
||||
// the tunnel's, its endpoint not on the tunnel's port — or nothing when both agree.
|
||||
func disagrees(p inventory.Overlay, t inventory.Tunnel) string {
|
||||
var wrong []string
|
||||
want := t.Address
|
||||
if i := strings.Index(want, "/"); i >= 0 {
|
||||
want = want[:i]
|
||||
}
|
||||
if p.Address != want {
|
||||
wrong = append(wrong, fmt.Sprintf("its address is %s and the tunnel's is %s", orNothing(p.Address), want))
|
||||
}
|
||||
if p.Reachable() && portOfEndpoint(p.Endpoint) != strconv.Itoa(t.Port) {
|
||||
wrong = append(wrong, fmt.Sprintf("its endpoint %s is not on the tunnel's port %d", p.Endpoint, t.Port))
|
||||
}
|
||||
return strings.Join(wrong, "; ")
|
||||
}
|
||||
|
||||
func orNothing(s string) string {
|
||||
if s == "" {
|
||||
return "unset"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// portOfEndpoint is the port in host:port, or empty.
|
||||
func portOfEndpoint(endpoint string) string {
|
||||
if i := strings.LastIndex(endpoint, ":"); i >= 0 {
|
||||
return endpoint[i+1:]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// SilentFor is how long a node may be quiet before the mesh says so.
|
||||
//
|
||||
// A node speaks every minute, so three of them missed is a gap rather than a slow one. The number
|
||||
@@ -348,11 +516,37 @@ func whereEveryoneIs(ctx context.Context, inv *inventory.Inventory,
|
||||
return nil, errors.New(
|
||||
"asked where everyone is without the catalogue, which cannot be answered")
|
||||
}
|
||||
places, err := inv.Overlays(ctx)
|
||||
places, err := onTheNetwork(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]string{}
|
||||
for _, p := range places {
|
||||
out[p.Name] = overlay.InternalName(p.Name)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// onTheNetwork is every placed machine that resolves the private network — has an address AND
|
||||
// runs what puts it there. "Has an address" alone was true of every placed machine and told you
|
||||
// nothing about whether anything could reach it; a name written for such a machine resolves to
|
||||
// an address that does not answer, and a connection to it hangs (novox/hq issue 079).
|
||||
func onTheNetwork(ctx context.Context, inv *inventory.Inventory,
|
||||
shelf map[string]catalogue.Manifest) ([]inventory.Overlay, error) {
|
||||
places, err := inv.Overlays(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// **With the seat holders on record**, or a machine running the next holder of a seat beside
|
||||
// the current one resolves as two holders, is refused, and drops out of the map — taking the
|
||||
// address every other machine composes for what it offers (novox/hq ADR 0131). Found live:
|
||||
// the control node vanished from the private network the moment the new bus was assigned
|
||||
// beside the old one.
|
||||
holdings, err := inv.Holdings(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []inventory.Overlay
|
||||
for _, p := range places {
|
||||
if p.Address == "" {
|
||||
continue
|
||||
@@ -364,14 +558,14 @@ func whereEveryoneIs(ctx context.Context, inv *inventory.Inventory,
|
||||
caps, _ := inv.ProfileOf(ctx, p.Name)
|
||||
got, err := catalogue.Resolve(shelf, assigned,
|
||||
catalogue.Node{Name: p.Name, Site: p.Site, Capabilities: caps},
|
||||
catalogue.World{Unchecked: true})
|
||||
catalogue.World{Unchecked: true, Holdings: holdings})
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, m := range got.Modules {
|
||||
for _, offered := range m.Offers() {
|
||||
if offered == overlay.Requirement {
|
||||
out[p.Name] = overlay.InternalName(p.Name)
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -379,14 +573,17 @@ func whereEveryoneIs(ctx context.Context, inv *inventory.Inventory,
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// onThePrivateNetwork is every node's address on the overlay, sorted.
|
||||
// onThePrivateNetwork is every node's address on the private network, sorted — the same set
|
||||
// the names and the resolver mean by it (onTheNetwork), so a rule saying "from the mesh" admits
|
||||
// exactly the machines the mesh names.
|
||||
//
|
||||
// A node with no address is left out rather than rendered as an empty source: an empty entry in a
|
||||
// source set is a syntax error in the rule file, and a rule file that does not load leaves the
|
||||
// node filtering whatever it was filtering before -- the one outcome worse than a wrong rule,
|
||||
// because nothing reports it.
|
||||
func onThePrivateNetwork(ctx context.Context, inv *inventory.Inventory) ([]string, error) {
|
||||
places, err := inv.Overlays(ctx)
|
||||
func onThePrivateNetwork(ctx context.Context, inv *inventory.Inventory,
|
||||
shelf map[string]catalogue.Manifest) ([]string, error) {
|
||||
places, err := onTheNetwork(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -400,29 +597,49 @@ func onThePrivateNetwork(ctx context.Context, inv *inventory.Inventory) ([]strin
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// namesInTheMesh is every machine's internal name and the address behind it.
|
||||
// namesInTheMesh is every machine's internal name and the address behind it — every machine
|
||||
// that is on the private network, the same set the resolver means by that.
|
||||
//
|
||||
// A machine with no address has no name: writing one that resolves to nothing is worse than not
|
||||
// A machine that is not has no name: writing one that resolves to nothing is worse than not
|
||||
// writing it, because a connection to an address that does not answer hangs where a name that
|
||||
// does not resolve fails at once and says so. That is the rule the hosts file already follows,
|
||||
// and this is the same set read the same way.
|
||||
func namesInTheMesh(ctx context.Context, inv *inventory.Inventory) (map[string]string, error) {
|
||||
places, err := inv.Overlays(ctx)
|
||||
// does not resolve fails at once and says so. A machine placed on the overlay but not running
|
||||
// the module that puts it there is exactly that (novox/hq issue 079).
|
||||
func namesInTheMesh(ctx context.Context, inv *inventory.Inventory,
|
||||
shelf map[string]catalogue.Manifest) (map[string]string, error) {
|
||||
places, err := onTheNetwork(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]string{}
|
||||
for _, p := range places {
|
||||
if strings.TrimSpace(p.Address) == "" {
|
||||
out[overlay.InternalName(p.Name)] = p.Address
|
||||
}
|
||||
// And the carried peers the operator has named (novox/hq issue 112): machines the
|
||||
// predecessor's resolver answers for and the mesh routes to, known by name on the operator's
|
||||
// word until they enrol — at which point enrolment verifies the name and the node's own
|
||||
// entry takes over above. A name the predecessor answers for must keep resolving until the
|
||||
// machine behind it is a node; without these, taking the resolver silences three machines.
|
||||
carried, err := inv.CarriedPeers(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, p := range carried {
|
||||
if p.Named == "" || p.EnrolledAs != "" {
|
||||
continue
|
||||
}
|
||||
out[overlay.InternalName(p.Name)] = p.Address
|
||||
if _, taken := out[overlay.InternalName(p.Named)]; taken {
|
||||
continue // a node of the mesh owns the name; the stale statement loses
|
||||
}
|
||||
out[overlay.InternalName(p.Named)] = p.Address
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// artifactStoreOnNetwork is the machine and port the mesh's artifact store answers on, when a
|
||||
// module providing it is assigned to a machine that is on the private network.
|
||||
// artifactStoreOnNetwork is the machine on this network that offers the artifact store, and the
|
||||
// port THAT MACHINE put it on — the node's setting when it was given one (novox/hq ADR 0100,
|
||||
// 04-ISSUES/102), the mesh's assignment when it made one, and the manifest's own number only when
|
||||
// neither says anything. Read exactly as a consumer's binding is, because the trust a machine
|
||||
// writes for the store and the address it pulls from are the same fact as what a consumer is told.
|
||||
//
|
||||
// A lookup failure is an error, never "not found": collapsing the two composed a declaration
|
||||
// without the trust whenever the inventory hiccuped, delivered by a push that reported success —
|
||||
@@ -435,29 +652,87 @@ func artifactStoreOnNetwork(ctx context.Context, inv *inventory.Inventory,
|
||||
if err != nil {
|
||||
return "", "", false, fmt.Errorf("reading the catalogue: %w", err)
|
||||
}
|
||||
providers := map[string]string{} // module -> served port
|
||||
providers := map[string]catalogue.Manifest{}
|
||||
for name, m := range shelf {
|
||||
served, offers := m.Serves[catalogue.ArtifactStoreProvision]
|
||||
if !offers {
|
||||
continue
|
||||
}
|
||||
if p, ok := served["port"]; ok {
|
||||
providers[name] = fmt.Sprintf("%v", p)
|
||||
if _, offers := m.Serves[catalogue.ArtifactStoreProvision]; offers {
|
||||
providers[name] = m
|
||||
}
|
||||
}
|
||||
if len(providers) == 0 {
|
||||
return "", "", false, nil
|
||||
}
|
||||
// In a stated order, so two machines offering it would always answer the same one.
|
||||
machines := make([]string, 0, len(on))
|
||||
for machine := range on {
|
||||
machines = append(machines, machine)
|
||||
}
|
||||
sort.Strings(machines)
|
||||
for _, machine := range machines {
|
||||
assigned, err := inv.Assigned(ctx, machine)
|
||||
if err != nil {
|
||||
return "", "", false, fmt.Errorf("reading what %s is assigned: %w", machine, err)
|
||||
}
|
||||
for _, a := range assigned {
|
||||
if p, ok := providers[a]; ok {
|
||||
return machine, p, true, nil
|
||||
m, offers := providers[a]
|
||||
if !offers {
|
||||
continue
|
||||
}
|
||||
serves, err := servedOnNode(ctx, inv, machine, m, catalogue.ArtifactStoreProvision)
|
||||
if err != nil {
|
||||
return "", "", false, fmt.Errorf("reading where %s puts the artifact store: %w", machine, err)
|
||||
}
|
||||
if p, ok := serves["port"]; ok {
|
||||
return machine, fmt.Sprintf("%v", p), true, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", "", false, nil
|
||||
}
|
||||
|
||||
// artifactStoreAddress is the artifact store as `forNode` reaches it: `<node>.internal:<port>`
|
||||
// over the private network, or — when nothing is on the network yet — `127.0.0.1:<port>` for the
|
||||
// node that holds the store itself, and "" for any other. The address composed into every
|
||||
// reference the mesh built, at the moment it is used and never before (novox/hq 04-ISSUES/102).
|
||||
//
|
||||
// **Genesis places the network after the store, the broker, the vault and the catalogue.** Each
|
||||
// of those is built and pushed to a node that is on no network, and the store is on that same
|
||||
// node; an answer of "no store" there would refuse every one of those pushes and hand every one
|
||||
// of those builds a base nothing can pull. Loopback is the truth on that machine, and it is the
|
||||
// address genesis itself reaches the store by.
|
||||
func artifactStoreAddress(ctx context.Context, inv *inventory.Inventory,
|
||||
shelf map[string]catalogue.Manifest, forNode string) (string, error) {
|
||||
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
on := map[string]bool{}
|
||||
for name := range onNetwork {
|
||||
on[name] = true
|
||||
}
|
||||
node, port, found, err := artifactStoreOnNetwork(ctx, inv, on)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if found {
|
||||
return overlay.InternalName(node) + ":" + port, nil
|
||||
}
|
||||
holder, port, found, err := artifactStoreHolder(ctx, inv)
|
||||
if err != nil || !found || holder != forNode {
|
||||
return "", err
|
||||
}
|
||||
return "127.0.0.1:" + port, nil
|
||||
}
|
||||
|
||||
// artifactStoreHolder is whichever node is assigned a module offering the artifact store, on or
|
||||
// off the network, and the port that node put it on.
|
||||
func artifactStoreHolder(ctx context.Context, inv *inventory.Inventory) (node, port string, found bool, err error) {
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return "", "", false, err
|
||||
}
|
||||
all := map[string]bool{}
|
||||
for _, n := range nodes {
|
||||
all[n.Name] = true
|
||||
}
|
||||
return artifactStoreOnNetwork(ctx, inv, all)
|
||||
}
|
||||
|
||||
@@ -2,10 +2,13 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// placementOf is what the mesh holds about where one node is.
|
||||
@@ -76,3 +79,227 @@ func TestAPlacementAndNothingTogetherIsRefused(t *testing.T) {
|
||||
t.Fatal("a placement and --nothing together was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// A machine is named for the others only while it is on the private network — placed AND running
|
||||
// what puts it there — the same set the resolver means by "on the private network". A machine
|
||||
// that has an address and no networking is not named: a name resolving to an address that does
|
||||
// not answer hangs where an unknown name fails at once (novox/hq issue 079).
|
||||
func TestOnlyAMachineOnThePrivateNetworkIsNamed(t *testing.T) {
|
||||
open := aMesh(t) // two placed machines, both assigned what puts them on the private network
|
||||
ctx := t.Context()
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
names, err := namesInTheMesh(ctx, open.inventory, shelf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if names["anchor.internal"] != "10.77.0.1" || names["laptop.internal"] != "10.77.0.2" {
|
||||
t.Fatalf("two machines on the network are not both named: %v", names)
|
||||
}
|
||||
// The laptop keeps its place and its address, and stops running the network.
|
||||
if err := open.inventory.Unassign(ctx, "laptop", overlay.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
names, err = namesInTheMesh(ctx, open.inventory, shelf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, still := names["laptop.internal"]; still || names["anchor.internal"] != "10.77.0.1" {
|
||||
t.Fatalf("a machine that left the network is still named, or the one that stayed is not: %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0105: the range every address is composed from is the adopted tunnel's, read from
|
||||
// the tunnel the hub holds — never stored anywhere else.
|
||||
func TestTheOverlaysRangeIsTheAdoptedTunnels(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
t.Setenv(OverlayCIDRVar, "10.99.0.0/16")
|
||||
|
||||
before, err := overlayRange(ctx, inv)
|
||||
if err != nil || before != "10.99.0.0/16" {
|
||||
t.Fatalf("without an adopted tunnel the range is not what genesis said: %q %v", before, err)
|
||||
}
|
||||
|
||||
// The hub becomes what genesis makes of a machine in use: adopted, enrolled with the found
|
||||
// tunnel's key, and presenting the tunnel.
|
||||
if err := inv.SetAdopted(ctx, "anchor", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hub, err := inv.NodeByName(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const key = "THE-TUNNELS-KEY========================="
|
||||
if err := inv.RecordOverlayKey(ctx, hub.ID, key); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordTunnel(ctx, hub.ID, inventory.Tunnel{
|
||||
Interface: "wg0", Unit: "wg-quick@wg0", Config: "/etc/wireguard/wg0.conf", Port: 51900,
|
||||
Address: "192.0.2.1/24", Range: "192.0.2.0/24", PublicKey: key,
|
||||
Peers: []inventory.TunnelPeer{{PublicKey: "PEER-TWO", Address: "192.0.2.2"}},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
after, err := overlayRange(ctx, inv)
|
||||
if err != nil || after != "192.0.2.0/24" {
|
||||
t.Fatalf("with an adopted tunnel the range is %q (%v), not the tunnel's", after, err)
|
||||
}
|
||||
|
||||
// A placement whose endpoint is on another port than the tunnel's is refused: the peers dial
|
||||
// the tunnel's port.
|
||||
err = overlayPlace(ctx, inv, []string{"anchor", "--endpoint", "198.51.100.10:51820", "--site", "hosting", "--hub"})
|
||||
if err == nil || !strings.Contains(err.Error(), "51900") {
|
||||
t.Fatalf("an endpoint off the tunnel's port was accepted: %v", err)
|
||||
}
|
||||
// On the tunnel's port, the hub is placed at the tunnel's address — whatever it had before.
|
||||
if err := inv.SetPlace(ctx, "anchor", "", "", false, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := overlayPlace(ctx, inv, []string{"anchor", "--endpoint", "198.51.100.10:51900", "--site", "hosting", "--hub"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if placed := placementOf(t, ctx, inv, "anchor"); placed.Address != "192.0.2.1" {
|
||||
t.Fatalf("the hub was placed at %s, not the tunnel's own address", placed.Address)
|
||||
}
|
||||
|
||||
// And the hub's declaration carries the peer and the takeover.
|
||||
nodes, computed, err := graph(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var hubNode overlay.Node
|
||||
for _, n := range nodes {
|
||||
if n.Name == "anchor" {
|
||||
hubNode = n
|
||||
}
|
||||
}
|
||||
if hubNode.TakesOver == nil || hubNode.TakesOver.Unit != "wg-quick@wg0" {
|
||||
t.Errorf("the hub is not told to take over the found tunnel: %+v", hubNode)
|
||||
}
|
||||
carried := false
|
||||
for _, p := range computed["anchor"] {
|
||||
if p.Key == "PEER-TWO" && p.Allowed == "192.0.2.2/32" {
|
||||
carried = true
|
||||
}
|
||||
}
|
||||
if !carried {
|
||||
t.Errorf("the hub's peer list does not carry the tunnel's peer: %+v", computed["anchor"])
|
||||
}
|
||||
}
|
||||
|
||||
// A takeover is composed only for a hub whose placement agrees with the tunnel: an address or an
|
||||
// endpoint port that differs would have the host stop the found interface and raise the mesh's
|
||||
// where no peer is listening.
|
||||
func TestATakeoverIsNotComposedForAHubPlacedOffItsTunnel(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if err := inv.SetAdopted(ctx, "anchor", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hub, err := inv.NodeByName(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const key = "THE-TUNNELS-KEY========================="
|
||||
if err := inv.RecordOverlayKey(ctx, hub.ID, key); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordTunnel(ctx, hub.ID, inventory.Tunnel{
|
||||
Interface: "wg0", Unit: "wg-quick@wg0", Config: "/etc/wireguard/wg0.conf", Port: 51900,
|
||||
Address: "192.0.2.1/24", Range: "192.0.2.0/24", PublicKey: key}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// aMesh placed anchor at 10.77.0.1 on :51820 — the record of a hub placed before it took the
|
||||
// tunnel over.
|
||||
_, _, err = graph(ctx, open)
|
||||
if err == nil {
|
||||
t.Fatal("a takeover was composed for a hub whose address and port are not the tunnel's")
|
||||
}
|
||||
for _, want := range []string{"10.77.0.1", "192.0.2.1", "51820", "51900", "overlay place anchor"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not say %q: %v", want, err)
|
||||
}
|
||||
}
|
||||
// Re-placed on the tunnel, it composes.
|
||||
if err := inv.SetPlace(ctx, "anchor", "", "", false, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := overlayPlace(ctx, inv, []string{"anchor", "--endpoint", "198.51.100.10:51900", "--site", "here", "--hub"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := graph(ctx, open); err != nil {
|
||||
t.Fatalf("re-placed on the tunnel, the graph still refuses: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// theResolver is the catalogue's dnsmasq module as it is, or the test is skipped where the
|
||||
// catalogue is not beside this checkout.
|
||||
func theResolver(t *testing.T) catalogue.Manifest {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile("../../../mesh-catalog/modules/dnsmasq/module.json")
|
||||
if err != nil {
|
||||
t.Skipf("the catalogue is not beside this checkout: %v", err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("dnsmasq does not parse:\n%v", err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// The resolver is handed every machine on the private network as a wildcard, the same set and the
|
||||
// same source as the hosts file, and is handed it again when a machine leaves — through the
|
||||
// module's own manifest asking for the fact, with no module of the mesh's own in between (hal
|
||||
// dnsmasq-app conversion, novox/hq 08-connectivity). The runtime on that machine is pointed at the
|
||||
// machine's own address, where the resolver answers for its containers.
|
||||
func TestTheResolverIsToldEveryMachineOnTheNetworkAndToldAgainWhenOneLeaves(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, theResolver(t))
|
||||
if _, err := assign(ctx, open, "anchor", "dnsmasq"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
zones := func() string {
|
||||
t.Helper()
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
if r["id"] == "dnsmasq.fact-node-zones" {
|
||||
if r["path"] != "/etc/mesh-resolver/nodes.conf" {
|
||||
t.Fatalf("the machines were written somewhere the resolver does not read: %v", r["path"])
|
||||
}
|
||||
return r["content"].(string)
|
||||
}
|
||||
}
|
||||
t.Fatal("the resolver was not handed the machines")
|
||||
return ""
|
||||
}
|
||||
first := zones()
|
||||
for _, want := range []string{
|
||||
"local=/internal/", "address=/anchor.internal/10.77.0.1\n", "address=/laptop.internal/10.77.0.2\n",
|
||||
} {
|
||||
if !strings.Contains(first, want) {
|
||||
t.Errorf("the resolver's machines lack %q:\n%s", want, first)
|
||||
}
|
||||
}
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
if r["id"] == "dnsmasq.runtime-dns" {
|
||||
if !strings.Contains(r["content"].(string), `"10.77.0.1"`) || r["into"] != "json" {
|
||||
t.Errorf("the runtime is not pointed at this machine's own address, written into its file: %v", r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The laptop keeps its place and its address, and stops running the network.
|
||||
if err := open.inventory.Unassign(ctx, "laptop", overlay.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after := zones()
|
||||
if strings.Contains(after, "laptop") || !strings.Contains(after, "address=/anchor.internal/10.77.0.1\n") {
|
||||
t.Fatalf("a machine that left the network is still a wildcard, or the one that stayed is not:\n%s", after)
|
||||
}
|
||||
}
|
||||
|
||||
+121
-34
@@ -10,7 +10,6 @@ import (
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/token"
|
||||
)
|
||||
|
||||
@@ -38,15 +37,7 @@ func nodeCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
return showNode(ctx, inv, args[1])
|
||||
case "add":
|
||||
if len(args) != 2 {
|
||||
return errors.New("node add <name>")
|
||||
}
|
||||
node, err := inv.AddNode(ctx, args[1])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("added %s (%s)\n", node.Name, node.ID)
|
||||
return nil
|
||||
return addNode(ctx, inv, args[1:])
|
||||
|
||||
case "list":
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
@@ -61,7 +52,7 @@ func nodeCommand(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
}
|
||||
for _, n := range nodes {
|
||||
fmt.Printf("%-20s %-14s %s\n", n.Name, heardFrom(n), n.ID)
|
||||
fmt.Printf("%-20s %-14s %-9s %s\n", n.Name, heardFrom(n), modeOf(n), n.ID)
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -75,12 +66,84 @@ func nodeCommand(ctx context.Context, args []string) error {
|
||||
// because the damage is already done by the time it prints.
|
||||
return publicDomain(ctx, inv, args[1:])
|
||||
|
||||
case "account":
|
||||
// The operator's login on this machine (novox/hq to-be 29): what a home-scoped file is
|
||||
// owned by and which account `ssh <node>` uses. Reports with no argument; sets with one;
|
||||
// an optional second argument is the home when it is not /home/<account>.
|
||||
return nodeAccount(ctx, inv, args[1:])
|
||||
|
||||
default:
|
||||
return fmt.Errorf("node has no %q; it has add, list, show and public-domain", args[0])
|
||||
return fmt.Errorf("node has no %q; it has add, list, show, public-domain and account", args[0])
|
||||
}
|
||||
}
|
||||
|
||||
// addNode creates a node record, adopted when the operator says so (novox/hq ADR 0100).
|
||||
func addNode(ctx context.Context, inv *inventory.Inventory, args []string) error {
|
||||
set := flag.NewFlagSet("node add", flag.ContinueOnError)
|
||||
adopted := set.Bool("adopted", false,
|
||||
"the machine is in use: keep what is found on it until each module is taken")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("node add <name> [--adopted]")
|
||||
}
|
||||
node, err := inv.AddNodeAs(ctx, positionals[0], *adopted)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("added %s (%s)", node.Name, node.ID)
|
||||
if node.Adopted {
|
||||
fmt.Print(", adopted")
|
||||
}
|
||||
fmt.Println()
|
||||
return nil
|
||||
}
|
||||
|
||||
// modeOf is a node's mode as a word (novox/hq ADR 0100): an adopted node is said to be adopted
|
||||
// wherever the mesh reports a node's state.
|
||||
func modeOf(n inventory.Node) string {
|
||||
if n.Adopted {
|
||||
return "adopted"
|
||||
}
|
||||
return "converged"
|
||||
}
|
||||
|
||||
// publicDomainUsage is the one description of the three forms, so a refusal and the help agree.
|
||||
// nodeAccount reports or sets a node's operator account (novox/hq to-be 29). Read-shaped with no
|
||||
// argument, like public-domain: `node account novox` answers, it does not change anything.
|
||||
func nodeAccount(ctx context.Context, inv *inventory.Inventory, positionals []string) error {
|
||||
if len(positionals) == 0 || len(positionals) > 3 {
|
||||
return errors.New("node account <name> — what it is now; " +
|
||||
"node account <name> <account> [home] — set it (home defaults to /home/<account>)")
|
||||
}
|
||||
node := positionals[0]
|
||||
if len(positionals) == 1 {
|
||||
who, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if who.Account == "" {
|
||||
fmt.Printf("%s has no operator account known\n", node)
|
||||
fmt.Printf(" `node account %s <account>` sets it\n", node)
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("%s logs a person in as %s (home %s)\n", node, who.Account, who.Home())
|
||||
return nil
|
||||
}
|
||||
home := ""
|
||||
if len(positionals) == 3 {
|
||||
home = positionals[2]
|
||||
}
|
||||
if err := inv.SetAccount(ctx, node, positionals[1], home); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s logs a person in as %s\n", node, positionals[1])
|
||||
fmt.Printf(" run `push %s` once ssh-client is assigned, to send its operator config\n", node)
|
||||
return nil
|
||||
}
|
||||
|
||||
const publicDomainUsage = "node public-domain <name> — what it is now; " +
|
||||
"<name> <domain> to set it; <name> --clear to take it away"
|
||||
|
||||
@@ -153,6 +216,8 @@ func tokenCommand(ctx context.Context, args []string) error {
|
||||
existing := set.String("node", "", "issue for a node record that already exists")
|
||||
fresh := set.String("new", "", "create the node record, then issue for it")
|
||||
validFor := set.Duration("for", time.Hour, "how long the token may be used")
|
||||
adopted := set.Bool("adopted", false,
|
||||
"the machine joining is in use: it is adopted, and keeps what is found on it")
|
||||
if err := set.Parse(args[1:]); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -171,16 +236,7 @@ func tokenCommand(ctx context.Context, args []string) error {
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
name := *existing
|
||||
if *fresh != "" {
|
||||
node, err := inv.AddNode(ctx, *fresh)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
name = node.Name
|
||||
}
|
||||
|
||||
issued, err := inv.IssueToken(ctx, name, *validFor)
|
||||
issued, err := issueFor(ctx, inv, *existing, *fresh, *adopted, *validFor)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -201,17 +257,9 @@ func tokenCommand(ctx context.Context, args []string) error {
|
||||
// chicken-and-egg entirely: the mesh runs the broker, so a joining node's credentials can
|
||||
// exist before it does. The one-time secret IS the password, so a node's first connection is
|
||||
// already authenticated and enrolment is what happens over it.
|
||||
if management, err := broker.ManagementFromEnvironment(); err == nil {
|
||||
if err := management.CreateNodeAccount(ctx, issued.Node.Name, issued.Secret); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("broker account %s created, scoped to %s and the %s exchange\n\n",
|
||||
issued.Node.Name, link.QueueFor(issued.Node.Name), link.Exchange)
|
||||
} else if !errors.Is(err, broker.ErrNotConfigured) {
|
||||
return err
|
||||
}
|
||||
|
||||
made := token.Token{Node: issued.Node.Name, Signer: key.Public, Secret: issued.Secret}
|
||||
made := token.Token{Node: issued.Node.Name, Signer: key.Public, Secret: issued.Secret,
|
||||
Adopted: issued.Node.Adopted}
|
||||
|
||||
// Absent is a state, not a failure: a control plane can hold records and a key before it has
|
||||
// a broker. What it cannot do is issue a token anybody could use, and Missing() says so.
|
||||
@@ -228,8 +276,12 @@ func tokenCommand(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Printf("token for %s, usable once, until %s\n\n %s\n\n",
|
||||
issued.Node.Name, issued.Expires.Format(time.RFC3339), encoded)
|
||||
joins := ""
|
||||
if made.Adopted {
|
||||
joins = ", joining adopted"
|
||||
}
|
||||
fmt.Printf("token for %s%s, usable once, until %s\n\n %s\n\n",
|
||||
issued.Node.Name, joins, issued.Expires.Format(time.RFC3339), encoded)
|
||||
fmt.Println("This is the only time it is shown. What is stored is a hash of the secret.")
|
||||
|
||||
if missing := made.Missing(); len(missing) > 0 {
|
||||
@@ -243,6 +295,38 @@ func tokenCommand(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// issueFor is the inventory's half of issuing a token: the record, made when it is new, adopted
|
||||
// when the operator says so, and the one-time secret for it. The node in what it returns carries
|
||||
// its mode, which is what the token says.
|
||||
func issueFor(ctx context.Context, inv *inventory.Inventory, existing, fresh string, adopted bool,
|
||||
validFor time.Duration) (inventory.Issued, error) {
|
||||
name := existing
|
||||
if fresh != "" {
|
||||
node, err := inv.AddNodeAs(ctx, fresh, adopted)
|
||||
if err != nil {
|
||||
return inventory.Issued{}, err
|
||||
}
|
||||
name = node.Name
|
||||
}
|
||||
// Not saying adopted leaves the node as it is: re-issuing a token for an adopted node does not
|
||||
// converge it — converging is its own act, previewed (novox/hq ADR 0100). And saying it for a
|
||||
// node already converged is refused rather than done quietly: returning a node to adopted is
|
||||
// its own act too, which unloads the mesh's filter and enables the found firewall again.
|
||||
if adopted && fresh == "" {
|
||||
node, err := inv.NodeByName(ctx, name)
|
||||
if err != nil {
|
||||
return inventory.Issued{}, err
|
||||
}
|
||||
if !node.Adopted {
|
||||
return inventory.Issued{}, fmt.Errorf("%s is converged, and a token does not change "+
|
||||
"that: run `adopt %s` to return it to adopted, then issue the token without "+
|
||||
"--adopted", name, name)
|
||||
}
|
||||
}
|
||||
|
||||
return inv.IssueToken(ctx, name, validFor)
|
||||
}
|
||||
|
||||
func identityCommand(ctx context.Context, args []string) error {
|
||||
if len(args) == 0 || args[0] != "show" {
|
||||
return errors.New("identity show")
|
||||
@@ -334,6 +418,9 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
|
||||
}
|
||||
fmt.Printf("%s\n", node.Name)
|
||||
fmt.Printf(" last heard from %s\n", heardFrom(node))
|
||||
if err := showMode(ctx, inv, node); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// The domain its routed names are composed under, when it has one (novox/hq ADR 0066). Shown
|
||||
// only when set: a machine that serves nothing to the outside has no domain, and saying so of
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **A read-shaped invocation is never a destructive write.**
|
||||
@@ -97,3 +98,57 @@ func TestAskingAboutAMachineTheMeshHasNeverHeardOfIsRefused(t *testing.T) {
|
||||
t.Fatal("a name the mesh does not know was answered as if it were a machine")
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0100: the operator says a node is adopted — `node add --adopted` for a record, and
|
||||
// the token for a machine joining.
|
||||
func TestANodeAddedAdoptedIsAdopted(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
if err := addNode(ctx, open.inventory, []string{"joiner", "--adopted"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n, err := open.inventory.NodeByName(ctx, "joiner")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !n.Adopted {
|
||||
t.Fatal("node add --adopted made a converged node")
|
||||
}
|
||||
if err := addNode(ctx, open.inventory, []string{"plain"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n, _ := open.inventory.NodeByName(ctx, "plain"); n.Adopted {
|
||||
t.Fatal("node add without --adopted made an adopted node")
|
||||
}
|
||||
}
|
||||
|
||||
func TestATokenIssuedAdoptedSaysSoAndReissuingDoesNotConverge(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
issued, err := issueFor(ctx, open.inventory, "", "joiner", true, time.Hour)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !issued.Node.Adopted {
|
||||
t.Fatal("a token issued --adopted is for a node that is not adopted")
|
||||
}
|
||||
again, err := issueFor(ctx, open.inventory, "joiner", "", false, time.Hour)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !again.Node.Adopted {
|
||||
t.Fatal("re-issuing without --adopted converged the node; converging is its own act")
|
||||
}
|
||||
// --adopted for a node already converged is refused, and points at the act that does it.
|
||||
if _, err := issueFor(ctx, open.inventory, "laptop", "", true, time.Hour); err == nil ||
|
||||
!strings.Contains(err.Error(), "adopt laptop") {
|
||||
t.Fatalf("--adopted on a converged node was not refused: %v", err)
|
||||
}
|
||||
if n, _ := open.inventory.NodeByName(ctx, "laptop"); n.Adopted {
|
||||
t.Fatal("a refused token flipped the node to adopted")
|
||||
}
|
||||
// And said for a node that is adopted already, it is the ordinary re-issue.
|
||||
if _, err := issueFor(ctx, open.inventory, "joiner", "", true, time.Hour); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,12 +2,15 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/secrets"
|
||||
)
|
||||
|
||||
@@ -26,9 +29,25 @@ import (
|
||||
// operator key make [--out <file>] make a keypair: private half to the file, public half printed
|
||||
// operator key set <public> tell the mesh which key to seal to
|
||||
// operator key show the public key, its fingerprint, and what it can recover
|
||||
const operatorUsage = "operator key make [--out <file>] | operator key set <public> [--replace] | operator key show"
|
||||
const operatorUsage = "operator key make [--out <file>] | operator key set <public> [--replace] | " +
|
||||
"operator key show | operator issue <name> --invokes <tool,tool|*> | operator revoke <name> | " +
|
||||
"operator list"
|
||||
|
||||
func operatorCommand(ctx context.Context, args []string) error {
|
||||
if len(args) == 0 {
|
||||
return errors.New(operatorUsage)
|
||||
}
|
||||
// The people who may reach the mesh's tools (design 25 §7). Beside the operator's key because
|
||||
// both answer "who, other than a machine, may do something here" — and a person reading this
|
||||
// command's usage is asking exactly that.
|
||||
switch args[0] {
|
||||
case "issue":
|
||||
return personIssue(ctx, args[1:])
|
||||
case "revoke":
|
||||
return personRevoke(ctx, args[1:])
|
||||
case "list":
|
||||
return personList(ctx)
|
||||
}
|
||||
if len(args) < 2 || args[0] != "key" {
|
||||
return errors.New(operatorUsage)
|
||||
}
|
||||
@@ -160,3 +179,125 @@ func readPrivateKey(path string) (string, error) {
|
||||
}
|
||||
return strings.TrimSpace(string(raw)), nil
|
||||
}
|
||||
|
||||
// personIssue gives somebody a credential for the mesh's tools, and prints it once.
|
||||
//
|
||||
// **Printed, not stored.** The mesh keeps a hash and nothing else, so this is the only moment the
|
||||
// credential exists anywhere but on the workstation that will use it — the same contract a token has,
|
||||
// and for the same reason: a credential recoverable from the mesh's store has the store's blast
|
||||
// radius.
|
||||
func personIssue(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("operator issue", flag.ContinueOnError)
|
||||
invokes := set.String("invokes", "", "the tools this person may call, comma-separated, or * for every one")
|
||||
if err := set.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
if set.NArg() != 1 {
|
||||
return errors.New("operator issue <name> --invokes <tool,tool|*>")
|
||||
}
|
||||
name := set.Arg(0)
|
||||
if *invokes == "" {
|
||||
return errors.New(
|
||||
"say what this person may call: --invokes mesh-catalog.catalog_tools,gitea.repo_create, " +
|
||||
"or --invokes '*' for an administrator")
|
||||
}
|
||||
var tools []string
|
||||
for _, t := range strings.Split(*invokes, ",") {
|
||||
if t = strings.TrimSpace(t); t != "" {
|
||||
tools = append(tools, t)
|
||||
}
|
||||
}
|
||||
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
if err := inv.RecordPerson(ctx, inventory.Person{Name: name, Invokes: tools}); err != nil {
|
||||
return err
|
||||
}
|
||||
// Refused here rather than at the next composition, where it would stop the whole file being
|
||||
// written for everybody. A name that cannot be part of a subject is one the server would read as
|
||||
// a wider permission than anybody granted.
|
||||
if _, err := broker.PermissionsFor(broker.Principal{
|
||||
Kind: broker.KindPerson, Module: name, Invokes: tools, PasswordHash: "x",
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
user := broker.Principal{Kind: broker.KindPerson, Module: name}.Username()
|
||||
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: user, Kind: inventory.BusPerson})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
where, err := broker.FromEnvironment()
|
||||
if err != nil && !errors.Is(err, broker.ErrNotConfigured) {
|
||||
return err
|
||||
}
|
||||
held, err := json.Marshal(struct {
|
||||
URL string `json:"url"`
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
User string `json:"user"`
|
||||
Password string `json:"password"`
|
||||
Person string `json:"person"`
|
||||
Invokes []string `json:"invokes"`
|
||||
}{
|
||||
URL: "nats://" + where.Address, Fingerprint: where.Fingerprint,
|
||||
User: user, Password: password, Person: name, Invokes: tools,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Printf("issued %s, who may call %s\n", name, strings.Join(tools, ", "))
|
||||
fmt.Println(" this is the only time the credential is printed; the mesh keeps a hash")
|
||||
fmt.Println(" it works once the bus has been told, which is the next push to the machine holding mesh-broker")
|
||||
fmt.Println()
|
||||
fmt.Println(string(held))
|
||||
return nil
|
||||
}
|
||||
|
||||
func personRevoke(ctx context.Context, args []string) error {
|
||||
if len(args) != 1 {
|
||||
return errors.New("operator revoke <name>")
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
|
||||
if err := open.inventory.ForgetPerson(ctx, args[0]); err != nil {
|
||||
return err
|
||||
}
|
||||
// **Revoked at the next composition, not now.** The bus's users are a file, so a credential stops
|
||||
// working when the file no longer names it. Said plainly, because "revoked" that still works for
|
||||
// another minute is worth knowing about.
|
||||
fmt.Printf("%s is forgotten, and their credential stops working at the next composition — "+
|
||||
"push the machine holding mesh-broker to make it so\n", args[0])
|
||||
return nil
|
||||
}
|
||||
|
||||
func personList(ctx context.Context) error {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
|
||||
people, err := open.inventory.People(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(people) == 0 {
|
||||
fmt.Println("nobody but machines reaches this mesh")
|
||||
return nil
|
||||
}
|
||||
for _, p := range people {
|
||||
fmt.Printf("%-20s %s\n", p.Name, strings.Join(p.Invokes, ", "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// entry is a module as the catalogue holds it: built, so its manifest carries no `build` any more.
|
||||
func entry(module string, _ ...string) inventory.Entry {
|
||||
return inventory.Entry{Manifest: catalogue.Manifest{Module: module}}
|
||||
}
|
||||
|
||||
// stoodOn is what each module's newest build recorded it was handed.
|
||||
func stoodOn(edges map[string][]string) map[string][]string {
|
||||
out := map[string][]string{}
|
||||
for module, bases := range edges {
|
||||
for _, b := range bases {
|
||||
out[module] = append(out[module], catalogue.ArtifactStoreScheme+b+"/runtime@sha256:"+strings.Repeat("0", 64))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// A module built before the module it stands on is built against the old one and reports success
|
||||
// (novox/hq 04-ISSUES/131). So bases come first, however the set arrived.
|
||||
func TestBasesAreBuiltBeforeWhatStandsOnThem(t *testing.T) {
|
||||
in := []inventory.Entry{entry("app"), entry("runtime"), entry("other"), entry("base")}
|
||||
edges := stoodOn(map[string][]string{"app": {"runtime"}, "runtime": {"base"}})
|
||||
got := orderByBases(in, edges)
|
||||
pos := map[string]int{}
|
||||
for i, e := range got {
|
||||
pos[e.Manifest.Module] = i
|
||||
}
|
||||
if !(pos["base"] < pos["runtime"] && pos["runtime"] < pos["app"]) {
|
||||
t.Fatalf("bases not first: %v", pos)
|
||||
}
|
||||
if len(got) != 4 {
|
||||
t.Fatalf("an entry was lost or doubled: %d", len(got))
|
||||
}
|
||||
// A base outside the set is not waited for: it is not being rebuilt.
|
||||
got = orderByBases([]inventory.Entry{entry("app")}, stoodOn(map[string][]string{"app": {"elsewhere"}}))
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("a dependency outside the set changed the set: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A module registered from its manifest and never built still names its bases there; once built,
|
||||
// the recorded edge is what says so. Both are read, and a module never stands on itself.
|
||||
func TestWhatStandsOnAModuleIsReadFromItsBuildOrItsManifest(t *testing.T) {
|
||||
built := entry("gitea")
|
||||
edges := stoodOn(map[string][]string{"gitea": {"mesh-tools"}})
|
||||
if !standsOnModule(built, "mesh-tools", edges) {
|
||||
t.Fatal("a recorded edge was not read")
|
||||
}
|
||||
if standsOnModule(built, "gitea", edges) || standsOnModule(built, "postgres", edges) {
|
||||
t.Fatal("an edge was invented")
|
||||
}
|
||||
fresh := inventory.Entry{Manifest: catalogue.Manifest{Module: "plex", Build: &catalogue.Build{
|
||||
On: []catalogue.BuildsOn{{Arg: "RUNTIME_BASE", Module: "mesh-tools", Artifact: "runtime"}},
|
||||
}}}
|
||||
if !standsOnModule(fresh, "mesh-tools", nil) {
|
||||
t.Fatal("a manifest's own base was not read")
|
||||
}
|
||||
}
|
||||
|
||||
// A merge names a repository the way the forge does; a source is recorded the way a build was
|
||||
// asked for. The two meet on owner/repo and branch, whichever form the record took.
|
||||
func TestAMergeMatchesTheSourcesBuiltFromIt(t *testing.T) {
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main",
|
||||
CloneURL: "http://forge.internal:20000/novox/mesh-controller.git"}
|
||||
for _, s := range []inventory.Source{
|
||||
{Repository: "http://forge.internal:20000/novox/mesh-controller.git", Ref: "main"},
|
||||
{Repository: "novox/mesh-controller", Seat: "git", Ref: ""},
|
||||
{Repository: "https://elsewhere.example/novox/mesh-controller", Ref: "main"},
|
||||
} {
|
||||
if !sourceIs(s, m) {
|
||||
t.Errorf("%+v was not matched by the merge", s)
|
||||
}
|
||||
}
|
||||
for _, s := range []inventory.Source{
|
||||
{Repository: "novox/mesh-host", Seat: "git"},
|
||||
{Repository: "http://forge.internal:20000/novox/mesh-controller.git", Ref: "release"},
|
||||
} {
|
||||
if sourceIs(s, m) {
|
||||
t.Errorf("%+v was matched by a merge that is not its", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
+448
-41
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@@ -13,6 +14,7 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/licences"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
"net"
|
||||
"strconv"
|
||||
)
|
||||
@@ -81,9 +83,17 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
|
||||
return catalogue.Resolution{}, nil, err
|
||||
}
|
||||
|
||||
// The operator account this node logs a person in as, and where its home is (novox/hq to-be
|
||||
// 29) — carried so a home-scoped file's owner and path resolve for this machine.
|
||||
who, err := inv.NodeByName(ctx, nodeName)
|
||||
if err != nil {
|
||||
return catalogue.Resolution{}, nil, err
|
||||
}
|
||||
|
||||
resolved, err := catalogue.Resolve(shelf, assigned,
|
||||
catalogue.Node{Name: nodeName, Site: site, Capabilities: capabilities,
|
||||
At: onNetwork[nodeName], PublicDomain: publicDomain}, world)
|
||||
At: onNetwork[nodeName], PublicDomain: publicDomain,
|
||||
Account: who.Account, AccountHome: who.AccountHome}, world)
|
||||
if err != nil {
|
||||
return catalogue.Resolution{}, nil, err
|
||||
}
|
||||
@@ -175,6 +185,15 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
|
||||
|
||||
// Every node, not only the placed ones. A machine that was never put on the private network
|
||||
// still runs modules, still holds claims, and still offers whatever it offers.
|
||||
// **Who holds each seat on record, before anything is resolved** (novox/hq ADR 0131). Both
|
||||
// passes below need it: without it, the assignment standing beside a seat's holder — the next
|
||||
// holder, waiting for the handover — is refused as a second holder, and its node's whole set
|
||||
// with it.
|
||||
holdings, err := inv.Holdings(ctx)
|
||||
if err != nil {
|
||||
return catalogue.World{}, err
|
||||
}
|
||||
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return catalogue.World{}, err
|
||||
@@ -215,35 +234,26 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
|
||||
}
|
||||
|
||||
offered := map[string][]catalogue.Provider{}
|
||||
var firstHeld []catalogue.Held
|
||||
for _, o := range others {
|
||||
got, err := catalogue.Resolve(shelf, o.assigned, o.node, catalogue.World{Unchecked: true})
|
||||
got, err := catalogue.Resolve(shelf, o.assigned, o.node, catalogue.World{Unchecked: true, Holdings: holdings})
|
||||
if err != nil {
|
||||
// Their set does not resolve for some other reason. Not this node's problem to
|
||||
// report, and nothing of theirs is running, so it offers nothing.
|
||||
continue
|
||||
}
|
||||
firstHeld = append(firstHeld, got.Claims...)
|
||||
for _, m := range got.Modules {
|
||||
for _, name := range m.OffersAt(catalogue.ScopeMesh) {
|
||||
// What that module says a consumer needs to know, with that node's settings on
|
||||
// it: a port somebody moved on the provider is a port its consumers must be told
|
||||
// about, and the two coming from different places is how they come to disagree.
|
||||
assigned, err := portsOn(ctx, inv, o.node.Name, m.Module)
|
||||
serves, err := servedOnNode(ctx, inv, o.node.Name, m, name)
|
||||
if err != nil {
|
||||
return catalogue.World{}, err
|
||||
}
|
||||
serves := catalogue.ServedOn(m, name, assigned)
|
||||
if len(serves) > 0 {
|
||||
layers, err := inv.SettingsFor(ctx, o.node.Name, m.Module)
|
||||
if err != nil {
|
||||
return catalogue.World{}, err
|
||||
}
|
||||
serves, err = catalogue.Settle(serves, layers)
|
||||
if err != nil {
|
||||
return catalogue.World{}, err
|
||||
}
|
||||
}
|
||||
offered[name] = append(offered[name], catalogue.Provider{
|
||||
Node: o.node.Name, At: o.node.At, Serves: serves})
|
||||
Node: o.node.Name, At: o.node.At, Serves: serves, Module: m.Module})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -253,14 +263,20 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
|
||||
})
|
||||
}
|
||||
|
||||
world := catalogue.World{Offered: offered}
|
||||
// **The second pass is given the first pass's holdings.** A seat's holder answers a requirement
|
||||
// with several providers (novox/hq ADR 0110), so a node consuming one resolves only once the
|
||||
// holder is known. Without them its set is refused here, and a refused node's own claims drop
|
||||
// out of what the mesh holds — so a second holder of one of its seats would pass unrefused.
|
||||
world := catalogue.World{Offered: offered, Held: firstHeld, Holdings: holdings}
|
||||
var held []catalogue.Held
|
||||
for _, o := range others {
|
||||
got, err := catalogue.Resolve(shelf, o.assigned, o.node, world)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
world.Held = append(world.Held, got.Claims...)
|
||||
held = append(held, got.Claims...)
|
||||
}
|
||||
world.Held = held
|
||||
return world, nil
|
||||
}
|
||||
|
||||
@@ -270,10 +286,10 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
|
||||
// silently produced a declaration missing them — a difference between what `plan` showed and what
|
||||
// `plan --json` handed to anything reading it.
|
||||
func declarationFor(ctx context.Context, open *stores, node string,
|
||||
plan catalogue.Resolution, settings catalogue.SettingsBy) ([]map[string]any, error) {
|
||||
plan catalogue.Resolution, settings catalogue.SettingsBy) (sendable, error) {
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return sendable{}, err
|
||||
}
|
||||
// **Allocating, because `plan` is the send without the sending.** It is one machine, named by
|
||||
// a person, who is asking what a push would do — so the port it shows and the secret it seals
|
||||
@@ -315,11 +331,31 @@ const (
|
||||
|
||||
func declarationWith(ctx context.Context, open *stores, node string,
|
||||
plan catalogue.Resolution, settings catalogue.SettingsBy,
|
||||
gens map[string]catalogue.Generator, choosing Choosing) ([]map[string]any, error) {
|
||||
gens map[string]catalogue.Generator, choosing Choosing) (sendable, error) {
|
||||
with, record, err := renderingFor(ctx, open, node, plan, settings, gens, choosing)
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
composed, err := plan.Compose(with)
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
// And what was taken on it, said in every declaration it is sent from this one place.
|
||||
adoption, err := adoptionOf(ctx, open.inventory, record, plan, composed)
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
return sendable{Resources: composed.Resources, Adoption: adoption}, nil
|
||||
}
|
||||
|
||||
// renderingFor is everything a node's declaration is composed with, and the node's record.
|
||||
func renderingFor(ctx context.Context, open *stores, node string,
|
||||
plan catalogue.Resolution, settings catalogue.SettingsBy,
|
||||
gens map[string]catalogue.Generator, choosing Choosing) (catalogue.Rendering, inventory.Node, error) {
|
||||
inv := open.inventory
|
||||
grants, err := grantsFor(ctx, open, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
// Where this machine puts what each module needs reachable (novox/hq ADR 0038).
|
||||
//
|
||||
@@ -332,7 +368,7 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
||||
if choosing == Reading {
|
||||
held, err := inv.PortsFor(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
for _, a := range held {
|
||||
if already[a.Module] == nil {
|
||||
@@ -342,9 +378,44 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
||||
}
|
||||
}
|
||||
|
||||
// The machine ports this node was given for its modules (novox/hq ADR 0100): the foundation's
|
||||
// ports, as genesis chose them. A given port wins over anything assigned and over a manifest's
|
||||
// own long-form mapping.
|
||||
given := map[string]map[int]int{}
|
||||
for _, m := range plan.Modules {
|
||||
g, err := catalogue.GivenPorts(m, settings[m.Module])
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
if g != nil {
|
||||
given[m.Module] = g
|
||||
}
|
||||
}
|
||||
// And one holder per machine port across the node's modules: settings written before this was
|
||||
// refused where they are set are refused here rather than composed into two containers on
|
||||
// one port.
|
||||
holders := map[int]string{}
|
||||
for _, m := range plan.Modules {
|
||||
for _, at := range given[m.Module] {
|
||||
if other, twice := holders[at]; twice && other != m.Module {
|
||||
return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf("%w: %s and %s are "+
|
||||
"both given machine port %d on %s", inventory.ErrPortTaken, other, m.Module,
|
||||
at, node)
|
||||
}
|
||||
holders[at] = m.Module
|
||||
}
|
||||
}
|
||||
|
||||
ports := map[string]map[int]int{}
|
||||
for _, m := range plan.Modules {
|
||||
for _, l := range m.Listens {
|
||||
if at, isGiven := given[m.Module][l.Port]; isGiven {
|
||||
if ports[m.Module] == nil {
|
||||
ports[m.Module] = map[int]int{}
|
||||
}
|
||||
ports[m.Module][l.Port] = at
|
||||
continue
|
||||
}
|
||||
// **Only a port the module actually publishes is the mesh's to move.** A container's
|
||||
// mapping is the thing that translates; without one the software binds what it binds,
|
||||
// and an assignment would not move the service — it would open the wrong number in the
|
||||
@@ -356,7 +427,7 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
||||
case mayAssign && choosing == Allocating:
|
||||
at, err := inv.PortFor(ctx, node, m.Module, l.Port, l.Fixed)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf(
|
||||
return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf(
|
||||
"%s needs %d reachable on %s and it could not be assigned: %w",
|
||||
m.Module, l.Port, node, err)
|
||||
}
|
||||
@@ -397,7 +468,7 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
if needed[m.Module] == nil {
|
||||
needed[m.Module] = map[string]string{}
|
||||
@@ -415,7 +486,7 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
||||
}
|
||||
issued, meshCA, err := certificateFor(ctx, open, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
certificate, authority = issued, meshCA
|
||||
break
|
||||
@@ -425,26 +496,73 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
||||
// resolves to. Every node's address, including this one's: a machine reaching itself by its
|
||||
// own overlay address rather than by loopback is ordinary, and leaving it out would filter
|
||||
// the node's own traffic to itself with no rule naming why.
|
||||
private, err := onThePrivateNetwork(ctx, inv)
|
||||
// One reading of the catalogue for the three questions below that resolve the whole mesh.
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
private, err := onThePrivateNetwork(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
// The private network's range, offered to a module as ${machine:mesh-range} — a module that must
|
||||
// name the whole mesh (an intrusion filter that must never ban a tunnel peer) names it here
|
||||
// rather than hardcoding a value it cannot know.
|
||||
meshRange, err := overlayRange(ctx, inv)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
|
||||
// The artifact store as this node reaches it now — the address every image and archive the
|
||||
// mesh built is fetched through, composed here and recorded nowhere — with what the mesh has
|
||||
// built, so a reference recorded with an address before that is re-routed too.
|
||||
artifactStore, err := artifactStoreAddress(ctx, inv, shelf, node)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
held, err := inv.Held(ctx)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
built := make(map[string]bool, len(held))
|
||||
for repository := range held {
|
||||
built[repository] = true
|
||||
}
|
||||
|
||||
// And every machine's name, so a container can reach one. The same set that writes the
|
||||
// machine's own hosts file — one reading, so a container and its machine cannot disagree
|
||||
// about where another machine is.
|
||||
names, err := namesInTheMesh(ctx, inv)
|
||||
names, err := namesInTheMesh(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
|
||||
// Each machine's operator account, so an ssh Host block can name the login for every node
|
||||
// (novox/hq to-be 29). Keyed by the bare node name, which entriesFrom falls back to.
|
||||
allNodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
accounts := map[string]string{}
|
||||
for _, n := range allNodes {
|
||||
if n.Account != "" {
|
||||
accounts[n.Name] = n.Account
|
||||
}
|
||||
}
|
||||
|
||||
// And every routed name → the node that serves it (novox/hq ADR 0066). Alongside the
|
||||
// `<node>.internal` names above, so a container — or an internal ACME validator — resolves a
|
||||
// routed name to the proxy that serves it, mesh-wide. The mesh publishes the names it was told
|
||||
// to serve and knows nothing about what they mean.
|
||||
// Kept apart from the machines, because a fact about the machines must not be handed the names
|
||||
// the mesh merely serves (novox/hq 04-ISSUES/111).
|
||||
machines := make(map[string]string, len(names))
|
||||
for name, at := range names {
|
||||
machines[name] = at
|
||||
}
|
||||
routes, err := routeNamesInTheMesh(ctx, open)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
for name, at := range routes {
|
||||
names[name] = at
|
||||
@@ -453,11 +571,19 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
||||
// The ports the mesh itself needs open, which no module declares. Read from the broker this
|
||||
// control plane was told about rather than written down twice: the address a node is handed in
|
||||
// its token and the port its machine must accept on are the same fact.
|
||||
//
|
||||
// **Only on the node that listens on it** (novox/hq issue: the broker opening leaked onto
|
||||
// every node). The opening exists to WIDEN the broker's port to from-anywhere — a machine
|
||||
// enrolling is not on the mesh yet, so the broker's own `from: mesh` listen would refuse its
|
||||
// first dial. That widening belongs on the broker's host and nowhere else: a node that only
|
||||
// dials out needs no incoming rule, and an opening for a port nothing here listens on is a
|
||||
// from-anywhere hole for a dead port. So the foundation port is kept only when a module
|
||||
// resolved onto THIS node actually listens on it.
|
||||
var foundation []int
|
||||
if b, err := broker.FromEnvironment(); err == nil {
|
||||
if _, port, err := net.SplitHostPort(b.Address); err == nil {
|
||||
if n, err := strconv.Atoi(port); err == nil {
|
||||
foundation = append(foundation, n)
|
||||
foundation = foundationPortsFor(n, plan.Modules)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -473,15 +599,57 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
||||
continue
|
||||
}
|
||||
if kept, err = inv.OperatorExport(ctx); err != nil {
|
||||
return nil, err
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
return plan.Declaration(catalogue.Rendering{
|
||||
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
|
||||
// Whether this node is adopted (novox/hq ADR 0100): then the found firewall stays in force, and
|
||||
// the declaration carries openings and the mesh's guard in place of a filter.
|
||||
record, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
// And, on an adopted node, which modules were taken there: the guard is derived from those
|
||||
// only (novox/hq ADR 0103).
|
||||
var taken map[string]bool
|
||||
if record.Adopted {
|
||||
list, err := inv.Taken(ctx, node)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
taken = map[string]bool{}
|
||||
for _, m := range list {
|
||||
taken[m] = true
|
||||
}
|
||||
}
|
||||
// Where this node put the foundation's servers, for the control plane's own connections
|
||||
// (novox/hq 04-ISSUES/102): read from the node's settings for whatever claims each seat,
|
||||
// exactly as a consumer's binding is, never from what genesis wrote into a secret.
|
||||
seats, err := seatsOn(ctx, inv, shelf, node, plan.Modules, record.Adopted, taken)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
// The bus's user list, for the machine that runs the bus. Composed per push rather than kept,
|
||||
// because it is a function of the mesh's records and a kept copy could disagree with them.
|
||||
busUsers, err := composeBusUsers(ctx, inv, plan.Modules)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
memberships, err := inv.BusMemberships(ctx)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
return catalogue.Rendering{
|
||||
BusMembership: memberships[node],
|
||||
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
|
||||
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
||||
Foundation: foundation, Kept: kept})
|
||||
Machines: machines,
|
||||
Suffix: overlay.Suffix(), MeshRange: meshRange, Accounts: accounts, Foundation: foundation,
|
||||
Kept: kept, Adopted: record.Adopted,
|
||||
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, Built: built,
|
||||
BusUsers: busUsers,
|
||||
}, record, nil
|
||||
}
|
||||
|
||||
// routeNamesInTheMesh is every routed name and the address of the node that serves it (novox/hq
|
||||
@@ -698,6 +866,22 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// listensLines is what a person is told about what this module would open, and why — the same
|
||||
// `why` every listens entry already carries for the firewall it also feeds (novox/hq ADR 0007), so
|
||||
// deciding whether to assign a module can see what it would open before it opens it, not only
|
||||
// after. A module with nothing to listen on prints nothing extra, same as today.
|
||||
func listensLines(m catalogue.Manifest) []string {
|
||||
var out []string
|
||||
for _, l := range m.Listens {
|
||||
if l.Why == "" {
|
||||
out = append(out, fmt.Sprintf(" listens %d/%s from %s", l.Port, l.At(), l.From))
|
||||
continue
|
||||
}
|
||||
out = append(out, fmt.Sprintf(" listens %d/%s from %s — %s", l.Port, l.At(), l.From, l.Why))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func planCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("plan", flag.ContinueOnError)
|
||||
// Because "one resource" does not tell you whether the settings landed. Being able to read
|
||||
@@ -730,22 +914,30 @@ func planCommand(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
}
|
||||
if *asJSON {
|
||||
resources, err := declarationFor(ctx, open, args[0], plan, settings)
|
||||
declared, err := declarationFor(ctx, open, args[0], plan, settings)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := json.MarshalIndent(
|
||||
map[string]any{"declaration": 1, "resources": resources}, "", " ")
|
||||
// The bytes a push would send, indented: one marshaller, so `plan --json` cannot show an
|
||||
// envelope other than the one sent.
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
var indented bytes.Buffer
|
||||
if err := json.Indent(&indented, body, "", " "); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(indented.String())
|
||||
return nil
|
||||
}
|
||||
|
||||
fmt.Printf("%s would run:\n", args[0])
|
||||
for _, m := range plan.Modules {
|
||||
fmt.Printf(" %-20s %s\n", m.Module, plan.Because[m.Module])
|
||||
for _, line := range listensLines(m) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
}
|
||||
// What was assigned here and cannot run here. Said with the rest rather than as a refusal: it is
|
||||
// one module on the wrong machine, the others still run, and the remedy is to move this one.
|
||||
@@ -763,10 +955,11 @@ func planCommand(ctx context.Context, args []string) error {
|
||||
for _, n := range plan.Needs {
|
||||
fmt.Printf(" needs %s from %s, for %s\n", n.Name, n.From, n.For)
|
||||
}
|
||||
resources, err := declarationFor(ctx, open, args[0], plan, settings)
|
||||
declared, err := declarationFor(ctx, open, args[0], plan, settings)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resources := declared.Resources
|
||||
for module, layers := range settings {
|
||||
for _, layer := range layers {
|
||||
fmt.Printf(" %-20s settings from %s\n", module, layer.From)
|
||||
@@ -780,12 +973,26 @@ func planCommand(ctx context.Context, args []string) error {
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
fmt.Printf("\n--- %v %v ---\n%s", r["id"], r["path"], content)
|
||||
fmt.Printf("\n--- %s ---\n%s", shownAs(r), content)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// shownAs is the heading `plan --show` puts over a resource's content.
|
||||
//
|
||||
// **A file written into says so.** Its content is the mesh's part of a file that is otherwise the
|
||||
// machine's — the keys of a JSON document (novox/hq ADR 0102), the region of a hosts file (issue
|
||||
// 128). Shown under a bare path it reads as the whole file, and a person checking what a take
|
||||
// replaces would see a hosts file of a dozen lines where the machine keeps thirty.
|
||||
func shownAs(r map[string]any) string {
|
||||
heading := fmt.Sprintf("%v %v", r["id"], r["path"])
|
||||
if into, ok := r["into"].(string); ok && into != "" {
|
||||
heading += fmt.Sprintf(" (written into, %s)", into)
|
||||
}
|
||||
return heading
|
||||
}
|
||||
|
||||
// licencesFor is what this node can be answered with by record, and what it was put on.
|
||||
//
|
||||
// A mesh with no licences at all is the ordinary case and must not be an error: every existing
|
||||
@@ -871,6 +1078,119 @@ func managerPublicKeyFor(
|
||||
return inv.SealingKeyOf(ctx, node)
|
||||
}
|
||||
|
||||
// servedOnNode is what a module on a node tells a consumer of one of its provisions, with THAT
|
||||
// node's ports on it: the port the node was given (novox/hq ADR 0100) over the one the mesh
|
||||
// assigned over the manifest's own, settled with the node's settings layers.
|
||||
//
|
||||
// **The one derivation** for every reader of a provider's address — a consumer's binding, the
|
||||
// artifact store's trust and the references composed through it (04-ISSUES/102). Unreadable
|
||||
// given ports are that node's refusal to report, not this reader's.
|
||||
func servedOnNode(ctx context.Context, inv *inventory.Inventory, node string,
|
||||
m catalogue.Manifest, provision string) (map[string]any, error) {
|
||||
ports, layers, err := portsGivenOn(ctx, inv, node, m)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
serves := catalogue.ServedOn(m, provision, ports)
|
||||
if len(serves) > 0 {
|
||||
serves, err = catalogue.Settle(serves, layers)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return serves, nil
|
||||
}
|
||||
|
||||
// portsGivenOn is where a node puts a module's ports — assigned, then given over them — and the
|
||||
// node's settings layers for the module, read once for both.
|
||||
func portsGivenOn(ctx context.Context, inv *inventory.Inventory, node string,
|
||||
m catalogue.Manifest) (map[int]int, []catalogue.Layer, error) {
|
||||
ports, err := portsOn(ctx, inv, node, m.Module)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
layers, err := inv.SettingsFor(ctx, node, m.Module)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if given, err := catalogue.GivenPorts(m, layers); err == nil {
|
||||
for wanted, at := range given {
|
||||
ports[wanted] = at
|
||||
}
|
||||
}
|
||||
return ports, layers, nil
|
||||
}
|
||||
|
||||
// seatsOn is where a node put the holder of each mesh-scoped seat, by seat and by the port the
|
||||
// holder's software uses — what ${seat:…} answers with (novox/hq 04-ISSUES/102).
|
||||
//
|
||||
// Read for every module in the catalogue that claims a seat, in this node's set or not: the store
|
||||
// and the broker are given their ports at genesis, as settings on a module that may be registered
|
||||
// and not yet assigned (04-ISSUES/085), and the control plane must follow that setting from the
|
||||
// first declaration it composes for itself. A holder in this node's set wins over one that is not.
|
||||
func seatsOn(ctx context.Context, inv *inventory.Inventory, shelf map[string]catalogue.Manifest,
|
||||
node string, inSet []catalogue.Manifest, adopted bool, taken map[string]bool) (map[string]map[int]int, error) {
|
||||
assigned := map[string]bool{}
|
||||
for _, m := range inSet {
|
||||
assigned[m.Module] = true
|
||||
}
|
||||
names := make([]string, 0, len(shelf))
|
||||
for name := range shelf {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
seats := map[string]map[int]int{}
|
||||
for _, name := range names {
|
||||
m := shelf[name]
|
||||
var claims []string
|
||||
for _, c := range m.Claims {
|
||||
if c.At() == catalogue.ScopeMesh {
|
||||
claims = append(claims, c.Name)
|
||||
}
|
||||
}
|
||||
if len(claims) == 0 {
|
||||
continue
|
||||
}
|
||||
// **Only a port the node was given or the mesh assigned — never the manifest's own
|
||||
// number.** The sealed value the answer sits beside carries the port genesis wrote, which
|
||||
// on a given-port node is the predecessor's; a manifest's long-form mapping is the
|
||||
// catalogue's default, and answering with it would override the right number with one
|
||||
// the mesh never checked (the contract in seat_into.go). And on an adopted node a holder
|
||||
// assigned but not yet taken is the found container, on the ports it was found with, not
|
||||
// the declaration's — so its mesh-assigned ports do not count there either; a given port
|
||||
// does, because a given port is the found one by construction (ADR 0100).
|
||||
ports, _, err := portsGivenOn(ctx, inv, node, m)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if adopted && !taken[name] {
|
||||
layers, err := inv.SettingsFor(ctx, node, m.Module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ports = map[int]int{}
|
||||
if given, err := catalogue.GivenPorts(m, layers); err == nil {
|
||||
ports = given
|
||||
}
|
||||
}
|
||||
if len(ports) == 0 {
|
||||
continue
|
||||
}
|
||||
for _, seat := range claims {
|
||||
if seats[seat] == nil {
|
||||
seats[seat] = map[int]int{}
|
||||
}
|
||||
for wanted, at := range ports {
|
||||
if _, said := seats[seat][wanted]; said && !assigned[name] {
|
||||
continue
|
||||
}
|
||||
seats[seat][wanted] = at
|
||||
}
|
||||
}
|
||||
}
|
||||
return seats, nil
|
||||
}
|
||||
|
||||
// portsOn is one module's assignments on one machine, by the port the software uses.
|
||||
func portsOn(
|
||||
ctx context.Context, inv *inventory.Inventory, node, module string,
|
||||
@@ -887,3 +1207,90 @@ func portsOn(
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// composeBusUsers is the bus's user list, for a push to the machine that runs the bus.
|
||||
//
|
||||
// Empty for every other machine, and for every machine while the mesh is on the bus it runs on
|
||||
// today — where accounts are a management call and there is no file to write.
|
||||
//
|
||||
// **Composed on each push, never kept.** The list is a function of the mesh's records (who exists,
|
||||
// what runs where, what each declares), and a stored copy would be a second account of who may reach
|
||||
// the bus, able to disagree with the records while both looked internally consistent (ADR 0043).
|
||||
//
|
||||
// A user the mesh has never minted a password for is **left out and said**, not written as a user
|
||||
// without one — the composer refuses that, because a user with no password is a user anybody is. That
|
||||
// is an ordinary situation with an obvious remedy (`module issue`, or enrolling), so the push carries
|
||||
// the rest rather than failing: a bus that is missing one module's user is a mesh where that module
|
||||
// cannot connect, and a bus with no file at all is a mesh where nothing can.
|
||||
func composeBusUsers(ctx context.Context, inv *inventory.Inventory,
|
||||
onThisNode []catalogue.Manifest) (string, error) {
|
||||
|
||||
// **Not gated on which bus the controller is on, and that was a bug.** It read "compose this only
|
||||
// once the mesh is on the new bus" — which cannot work, because the server needs its user list
|
||||
// *before* anything moves onto it. Step 2 of the change is exactly that: the server stands in the
|
||||
// mesh carrying nothing, on its own ports, while every node is still on the old bus (novox/hq
|
||||
// ADR 0116). Under the old gating that step could not happen: the module would come up, find no
|
||||
// accounts file, and its entrypoint would wait for one the controller had decided not to write.
|
||||
//
|
||||
// So the question is only whether this machine runs the module that asked for the file. A mesh
|
||||
// that never moves has written a user list nothing reads, which costs a few hundred bytes on one
|
||||
// node; the reverse cost a step that cannot be taken.
|
||||
//
|
||||
// Asked of what this push resolves to rather than of the seat's holder mesh-wide: the file is a
|
||||
// resource of that module, so the question is whether it is here.
|
||||
holdsTheBus := false
|
||||
for _, m := range onThisNode {
|
||||
if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") {
|
||||
holdsTheBus = true
|
||||
}
|
||||
}
|
||||
if !holdsTheBus {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
records, err := inv.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
kept, err := inv.BusUsers(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
hashes := make(map[string]string, len(kept))
|
||||
for name, u := range kept {
|
||||
hashes[name] = u.PasswordHash
|
||||
}
|
||||
filled, missing := broker.WithPasswords(users, hashes)
|
||||
if len(missing) > 0 {
|
||||
fmt.Printf("the bus's user list leaves out %d user(s) the mesh has minted no credential "+
|
||||
"for: %s. Each is a user that cannot connect until one is issued\n",
|
||||
len(missing), strings.Join(missing, ", "))
|
||||
}
|
||||
if len(filled) == 0 {
|
||||
return "", fmt.Errorf(
|
||||
"this machine runs the bus and not one user has a credential, so the composed list " +
|
||||
"would refuse every connection in the mesh")
|
||||
}
|
||||
return broker.ComposeAccounts(filled)
|
||||
}
|
||||
|
||||
// foundationPortsFor is the broker port, kept only when a module resolved onto this node listens
|
||||
// on it (novox/hq issue: the broker opening leaked onto every node). The foundation opening
|
||||
// exists to WIDEN the broker's `from: mesh` port to from-anywhere, because a machine enrolling is
|
||||
// not on the mesh yet and its first dial would be refused. That widening belongs on the broker's
|
||||
// host alone: a node that only dials out needs no incoming rule, and an opening for a port
|
||||
// nothing here listens on is a from-anywhere hole for a dead port.
|
||||
func foundationPortsFor(brokerPort int, modules []catalogue.Manifest) []int {
|
||||
for _, m := range modules {
|
||||
for _, l := range m.Listens {
|
||||
if l.Port == brokerPort {
|
||||
return []int{brokerPort}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// `plan` tells a person what a module would open and why, from the same `why` every listens
|
||||
// entry already carries for the firewall (novox/hq ADR 0007) — so deciding whether to assign a
|
||||
// module does not need reading its manifest first.
|
||||
func TestListensLinesShowWhatAModuleWouldOpenAndWhy(t *testing.T) {
|
||||
m := catalogue.Manifest{Module: "minio", Listens: []catalogue.Listening{
|
||||
{Port: 9000, From: catalogue.FromMesh, Why: "the S3 endpoint"},
|
||||
{Port: 9001, From: catalogue.FromMesh},
|
||||
}}
|
||||
got := listensLines(m)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("two listens entries, got %d: %v", len(got), got)
|
||||
}
|
||||
if !strings.Contains(got[0], "9000/tcp") || !strings.Contains(got[0], "the S3 endpoint") {
|
||||
t.Errorf("the port and its why did not both appear: %q", got[0])
|
||||
}
|
||||
if strings.Contains(got[1], "—") {
|
||||
t.Errorf("a listens entry with no why should not print a dash: %q", got[1])
|
||||
}
|
||||
if !strings.Contains(got[1], "9001/tcp") {
|
||||
t.Errorf("the port still appears without a why: %q", got[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestListensLinesAreEmptyForAModuleWithNothingToListenOn(t *testing.T) {
|
||||
if got := listensLines(catalogue.Manifest{Module: "board"}); len(got) != 0 {
|
||||
t.Errorf("a module with no listens should print nothing, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// `plan --show` says when a file is written into rather than over (novox/hq issue 128), or the
|
||||
// mesh's region of a hosts file reads as the whole file.
|
||||
func TestAFileWrittenIntoIsShownAsSuch(t *testing.T) {
|
||||
region := shownAs(map[string]any{
|
||||
"id": "mesh-wireguard.fact-node-names", "path": "/etc/hosts", "into": "block"})
|
||||
if region != "mesh-wireguard.fact-node-names /etc/hosts (written into, block)" {
|
||||
t.Errorf("the region is shown as %q", region)
|
||||
}
|
||||
whole := shownAs(map[string]any{"id": "dnsmasq.fact-node-zones", "path": "/etc/mesh-resolver/nodes.conf"})
|
||||
if strings.Contains(whole, "written into") {
|
||||
t.Errorf("a whole file is shown as written into: %q", whole)
|
||||
}
|
||||
}
|
||||
+232
-62
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
@@ -39,6 +38,27 @@ func reportUnhostable(node string, plan catalogue.Resolution) {
|
||||
// nothing in it was wrong, and no one edit was the one that should have been a new file.
|
||||
|
||||
// serve is the control plane running: one connection to the broker, one queue, one consumer.
|
||||
// connectLink opens the controller's link over whichever bus this process is on (design 25: one
|
||||
// variable moves it). The streams and this controller's consumers are raised first on the new bus,
|
||||
// so nothing served here finds them missing.
|
||||
func connectLink(ctx context.Context, inv *inventory.Inventory, enroller link.Enroller, listener link.Listener) (*link.Server, error) {
|
||||
busAddress, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if inv != nil {
|
||||
if err := raiseTheBus(ctx, inv, busAddress); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
js, err := broker.Dial(busAddress)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the mesh is on the bus at %s and this control plane cannot reach it: %w",
|
||||
broker.BareAddress(busAddress), err)
|
||||
}
|
||||
return link.ConnectNats(js, enroller, listener), nil
|
||||
}
|
||||
|
||||
func serve(ctx context.Context) error {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
@@ -62,11 +82,6 @@ func serve(ctx context.Context) error {
|
||||
}
|
||||
fmt.Printf("signing as %s\n", key.Fingerprint()[:16])
|
||||
|
||||
management, err := broker.ManagementFromEnvironment()
|
||||
if err != nil && !errors.Is(err, broker.ErrNotConfigured) {
|
||||
return err
|
||||
}
|
||||
|
||||
// Where the broker is and what to expect there, so a node can be told how to come back
|
||||
// without a person and a new token.
|
||||
known, err := broker.FromEnvironment()
|
||||
@@ -78,12 +93,22 @@ func serve(ctx context.Context) error {
|
||||
"reconnect. Set %s and %s.\n", broker.AddressVar, broker.CertificateVar)
|
||||
}
|
||||
|
||||
work := link.Enrolment{Inventory: inv, Identity: ident, Management: management, Broker: known}
|
||||
server, err := link.Connect(work, work)
|
||||
// **Which bus this mesh is on, read once** (novox/hq ADR 0116 step 5). Both clients ship; both
|
||||
// being live is refused, because a mesh half on each is one where a declaration goes out on one
|
||||
// and the report comes back on the other, and every component logs success while it happens.
|
||||
|
||||
work := link.Enrolment{Inventory: inv, Identity: ident, Broker: known,
|
||||
OnNATS: true}
|
||||
server, err := connectLink(ctx, inv, work, work)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer server.Close()
|
||||
|
||||
// The bus's own objects, asserted on every start. **Not created once at genesis**: a stream
|
||||
// somebody deleted, a mesh raised from a restored backup, or a bus whose data directory was
|
||||
// replaced all have records and no objects — and a node whose consumer is missing hears nothing
|
||||
// while everything else about it looks correct.
|
||||
// And build results nobody was waiting for. A build triggered any other way than `build`
|
||||
// would otherwise be reported into the void, which is the same as not reporting it.
|
||||
server.Records(builds{inv})
|
||||
@@ -137,13 +162,13 @@ func declare(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
server, err := link.Connect(nil, nil)
|
||||
server, err := connectLink(ctx, nil, nil, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer server.Close()
|
||||
|
||||
if err := link.Declare(ctx, server.Channel(), ident, node, raw, 15*time.Second); err != nil {
|
||||
if err := link.Declare(ctx, server.Bus(), ident, node, raw, 15*time.Second); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw))
|
||||
@@ -250,7 +275,7 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
server, err := link.Connect(nil, nil)
|
||||
server, err := connectLink(ctx, nil, nil, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -283,10 +308,16 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
asked = append(asked, n.Name)
|
||||
}
|
||||
|
||||
sending, refusals := composeEach(asked, func(node string) ([]map[string]any, error) {
|
||||
plan, settings, err := planFor(ctx, open, node)
|
||||
// Held from composing to sending, so a converge on one of them cannot send between the two
|
||||
// and be overtaken by what was composed before it (novox/hq ADR 0100).
|
||||
held, release, err := holdNodes(ctx, open, asked)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sending, refusals := composeEach(asked, func(node string) (sendable, error) {
|
||||
plan, settings, err := planFor(held, open, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return sendable{}, err
|
||||
}
|
||||
// A module assigned here that this machine cannot host is said and left out, not fatal: the
|
||||
// healthy modules beside it are still resolved and sent. Reported so it is not silently
|
||||
@@ -295,16 +326,17 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
// The private network is in here with everything else. It used to be composed separately
|
||||
// and prepended, which meant every machine with an address was on it and no machine could
|
||||
// be kept off. It is a module now, so it arrives the way a module does.
|
||||
return declarationWith(ctx, open, node, plan, settings, gens, Allocating)
|
||||
return declarationWith(held, open, node, plan, settings, gens, Allocating)
|
||||
})
|
||||
|
||||
sentDigest := map[string]string{}
|
||||
defer release()
|
||||
for _, s := range sending {
|
||||
body, err := json.Marshal(map[string]any{"declaration": 1, "resources": s.resources})
|
||||
body, err := s.declared.Body()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := link.Declare(ctx, server.Channel(), ident, s.node, body, 15*time.Second); err != nil {
|
||||
if err := link.Declare(ctx, server.Bus(), ident, s.node, body, 15*time.Second); err != nil {
|
||||
return err
|
||||
}
|
||||
// After it is away, not before. A digest recorded for something that failed to send would
|
||||
@@ -318,8 +350,9 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
sentDigest[s.node] = digest
|
||||
fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.resources))
|
||||
fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.declared.Resources))
|
||||
}
|
||||
release()
|
||||
fmt.Printf("\n%d node(s) told\n", len(sending))
|
||||
|
||||
// **A named push leaves the mesh consistent, not just the machine it named** (novox/hq
|
||||
@@ -374,31 +407,35 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
// (novox/hq ADR 0066). The earlier cut routed these through sendTo, which is
|
||||
// all-or-nothing — so one swept machine's compose error failed the operator's named
|
||||
// push and skipped its --wait, the very intolerance the main path exists to avoid.
|
||||
sending, refused := composeEach(also, func(node string) ([]map[string]any, error) {
|
||||
plan, settings, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reportUnhostable(node, plan)
|
||||
return declarationWith(ctx, open, node, plan, settings, gens, Allocating)
|
||||
})
|
||||
// Held for this round only, and after the last round's were given back, so two pushes
|
||||
// cascading into each other's machines never each wait on the other.
|
||||
refused, err := sendRound(ctx, open, also,
|
||||
func(held context.Context, node string) (sendable, error) {
|
||||
plan, settings, err := planFor(held, open, node)
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
reportUnhostable(node, plan)
|
||||
return declarationWith(held, open, node, plan, settings, gens, Allocating)
|
||||
},
|
||||
func(s readyNode, body []byte) error {
|
||||
if err := link.Declare(ctx, server.Bus(), ident, s.node, body,
|
||||
15*time.Second); err != nil {
|
||||
return err
|
||||
}
|
||||
record, err := inv.NodeByName(ctx, s.node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.RecordSent(ctx, record.ID, digestOf(body)); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.declared.Resources))
|
||||
return nil
|
||||
})
|
||||
refusals = append(refusals, refused...)
|
||||
for _, s := range sending {
|
||||
body, err := json.Marshal(map[string]any{"declaration": 1, "resources": s.resources})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := link.Declare(ctx, server.Channel(), ident, s.node, body, 15*time.Second); err != nil {
|
||||
return err
|
||||
}
|
||||
record, err := inv.NodeByName(ctx, s.node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.RecordSent(ctx, record.ID, digestOf(body)); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.resources))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Every candidate this round is marked handled — the sent ones so they are not
|
||||
// re-listed, and the refused ones so a machine that cannot be composed does not make
|
||||
@@ -458,8 +495,8 @@ func waitForApplied(ctx context.Context, inv *inventory.Inventory, node, digest
|
||||
|
||||
// readyNode is one machine and the declaration it would be sent.
|
||||
type readyNode struct {
|
||||
node string
|
||||
resources []map[string]any
|
||||
node string
|
||||
declared sendable
|
||||
}
|
||||
|
||||
// composeEach works out what each named machine should be, and never lets one machine's answer
|
||||
@@ -480,25 +517,58 @@ type readyNode struct {
|
||||
// The all-or-nothing rule is kept where it means something — sendTo, which rotates a credential
|
||||
// across two machines that must agree — and dropped here, where it never did.
|
||||
func composeEach(names []string,
|
||||
compose func(node string) ([]map[string]any, error)) ([]readyNode, []string) {
|
||||
compose func(node string) (sendable, error)) ([]readyNode, []string) {
|
||||
|
||||
var sending []readyNode
|
||||
var refusals []string
|
||||
for _, name := range names {
|
||||
resources, err := compose(name)
|
||||
declared, err := compose(name)
|
||||
if err != nil {
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
}
|
||||
if len(resources) == 0 {
|
||||
fmt.Printf("%s is assigned nothing — skipped\n", name)
|
||||
continue
|
||||
if len(declared.Resources) == 0 {
|
||||
// Sent, not skipped (novox/hq issue 127). A node whose declaration composes to
|
||||
// nothing may have HELD something before — the broker opening a placement gave it,
|
||||
// say — and skipping the empty declaration leaves that last resource in force
|
||||
// forever, re-applied by the node's own heartbeat, with no way for the mesh to say
|
||||
// it is gone. An empty declaration is the correction: the host drops what the mesh
|
||||
// owned and keeps what it found (the adoption envelope still rides along). A node
|
||||
// that never held anything applies it as the no-op it is.
|
||||
fmt.Printf("%s owns nothing now — sent so it drops what it last held\n", name)
|
||||
}
|
||||
sending = append(sending, readyNode{name, resources})
|
||||
sending = append(sending, readyNode{name, declared})
|
||||
}
|
||||
return sending, refusals
|
||||
}
|
||||
|
||||
// sendRound holds the named nodes, composes each and sends each that composed, and gives the hold
|
||||
// back on every way out — a body that cannot be marshalled and a send that fails included
|
||||
// (novox/hq ADR 0100). A node that cannot be composed is a refusal, not an error: the others are
|
||||
// still sent.
|
||||
func sendRound(ctx context.Context, open *stores, names []string,
|
||||
compose func(held context.Context, node string) (sendable, error),
|
||||
send func(s readyNode, body []byte) error) ([]string, error) {
|
||||
held, release, err := holdNodes(ctx, open, names)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer release()
|
||||
sending, refused := composeEach(names, func(node string) (sendable, error) {
|
||||
return compose(held, node)
|
||||
})
|
||||
for _, s := range sending {
|
||||
body, err := s.declared.Body()
|
||||
if err != nil {
|
||||
return refused, err
|
||||
}
|
||||
if err := send(s, body); err != nil {
|
||||
return refused, err
|
||||
}
|
||||
}
|
||||
return refused, nil
|
||||
}
|
||||
|
||||
// couldNotBeResolved is what a push ends with when some machines could not be worked out.
|
||||
//
|
||||
// **After the rest have been sent, never instead of sending them.** It is still an error, because
|
||||
@@ -533,11 +603,15 @@ func sendTo(ctx context.Context, open *stores, names []string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
type ready struct {
|
||||
node string
|
||||
resources []map[string]any
|
||||
// Held from composing to sending (novox/hq ADR 0100); a caller that holds them already —
|
||||
// converge, which flips the node and then sends it — is not made to wait on itself.
|
||||
ctx, release, err := holdNodes(ctx, open, names)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var sending []ready
|
||||
defer release()
|
||||
|
||||
var sending []readyNode
|
||||
var refusals []string
|
||||
for _, name := range names {
|
||||
plan, settings, err := planFor(ctx, open, name)
|
||||
@@ -546,30 +620,30 @@ func sendTo(ctx context.Context, open *stores, names []string) error {
|
||||
continue
|
||||
}
|
||||
reportUnhostable(name, plan)
|
||||
resources, err := declarationWith(ctx, open, name, plan, settings, gens, Allocating)
|
||||
declared, err := declarationWith(ctx, open, name, plan, settings, gens, Allocating)
|
||||
if err != nil {
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
}
|
||||
sending = append(sending, ready{name, resources})
|
||||
sending = append(sending, readyNode{name, declared})
|
||||
}
|
||||
if len(refusals) > 0 {
|
||||
return fmt.Errorf("nothing was sent. %d machine(s) could not be resolved:\n\n%s",
|
||||
len(refusals), strings.Join(refusals, "\n\n"))
|
||||
}
|
||||
|
||||
server, err := link.Connect(nil, nil)
|
||||
server, err := connectLink(ctx, nil, nil, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer server.Close()
|
||||
|
||||
for _, s := range sending {
|
||||
body, err := json.Marshal(map[string]any{"declaration": 1, "resources": s.resources})
|
||||
body, err := s.declared.Body()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := link.Declare(ctx, server.Channel(), ident, s.node, body, 15*time.Second); err != nil {
|
||||
if err := link.Declare(ctx, server.Bus(), ident, s.node, body, 15*time.Second); err != nil {
|
||||
return err
|
||||
}
|
||||
record, err := inv.NodeByName(ctx, s.node)
|
||||
@@ -579,7 +653,7 @@ func sendTo(ctx context.Context, open *stores, names []string) error {
|
||||
if err := inv.RecordSent(ctx, record.ID, digestOf(body)); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf(" sent %s %d resource(s)\n", s.node, len(s.resources))
|
||||
fmt.Printf(" sent %s %d resource(s)\n", s.node, len(s.declared.Resources))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -610,11 +684,11 @@ func wouldSend(ctx context.Context, open *stores,
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
resources, err := declarationWith(ctx, open, n.Name, plan, settings, gens, Reading)
|
||||
declared, err := declarationWith(ctx, open, n.Name, plan, settings, gens, Reading)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
body, err := json.Marshal(map[string]any{"declaration": 1, "resources": resources})
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -622,3 +696,99 @@ func wouldSend(ctx context.Context, open *stores,
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// raiseTheBus asserts the streams and consumers the mesh's own traffic needs.
|
||||
//
|
||||
// **Every start, and it says what it did.** The objects are the mesh's, created by nothing else —
|
||||
// the controller is their only writer (design 25 §3) — so a mesh that came up without them is one
|
||||
// where nodes connect, authenticate, and hear nothing. Said rather than silent for the reason the
|
||||
// first line of `serve` is said: a log that is quiet on success and loud on failure reads as broken
|
||||
// when it is working.
|
||||
func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string) error {
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the mesh is on the bus at %s and this control plane cannot reach it: %w",
|
||||
broker.BareAddress(address), err)
|
||||
}
|
||||
defer js.Close()
|
||||
|
||||
// **Its own user, before anything else.** The controller's account is created by the installer at
|
||||
// a bootstrap password, before there is a controller to mint one — so nothing recorded a hash for
|
||||
// it, and the first composition would leave the writer out of the file it was writing. Recorded
|
||||
// only if absent: a credential the mesh minted since is the one that counts.
|
||||
// **Its own user, before anything else it does here.** The controller's account is created by the
|
||||
// installer at a bootstrap password, before there is a controller to mint one — so nothing
|
||||
// recorded a hash for it, and the first composition would leave the writer out of the file it was
|
||||
// writing: a bus nothing can connect to, produced by the thing connected to it. Recorded only if
|
||||
// absent, so a restart cannot put the bootstrap credential back over a rotated one.
|
||||
if user, password, _ := broker.CredentialIn(address); user != "" && password != "" {
|
||||
if err := inv.SeedBusUser(ctx, inventory.BusUser{
|
||||
Username: user, Kind: inventory.BusController,
|
||||
}, password); err != nil {
|
||||
return fmt.Errorf("cannot record the credential this control plane is using: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
names := make([]string, 0, len(nodes))
|
||||
for _, n := range nodes {
|
||||
names = append(names, n.Name)
|
||||
}
|
||||
if err := broker.Raise(js, names); err != nil {
|
||||
return err
|
||||
}
|
||||
// The work queues of the mesh's own roles (novox/hq ADR 0121). The queue before the holder,
|
||||
// deliberately: work queues until somebody arrives to do it, so assigning a build machine a week
|
||||
// after something started asking for builds flushes the backlog instead of having lost it.
|
||||
// With the seats' holders, so each role's work queue gets the consumer its holder takes
|
||||
// work from. Passed as nil until the first live raise, which left the build machine bound to a
|
||||
// consumer nothing had created (2026-09-28).
|
||||
holders, err := seatHolders(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := broker.RaiseSeats(js, inventory.MeshSeats(), holders); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("the bus at %s has its streams, and %d machine(s) can hear a declaration\n",
|
||||
broker.BareAddress(address), len(names))
|
||||
return nil
|
||||
}
|
||||
|
||||
// seatHolders is who holds each of the mesh's seats, by seat name: the record where a handover
|
||||
// wrote one, and the assigned module claiming the seat otherwise — the same derivation the
|
||||
// resolver makes, read from the catalogue rather than re-resolved.
|
||||
func seatHolders(ctx context.Context, inv *inventory.Inventory) (map[string]broker.Holder, error) {
|
||||
out := map[string]broker.Holder{}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if len(e.On) == 0 {
|
||||
continue
|
||||
}
|
||||
for _, c := range e.Manifest.Claims {
|
||||
seat, known := catalogue.SeatNamed(c.Name)
|
||||
if !known {
|
||||
continue
|
||||
}
|
||||
if _, taken := out[seat.Name]; !taken {
|
||||
out[seat.Name] = broker.Holder{Node: e.On[0], Module: e.Manifest.Module}
|
||||
}
|
||||
}
|
||||
}
|
||||
recorded, err := inv.Holdings(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, h := range recorded {
|
||||
if seat, known := catalogue.SeatNamed(h.Claim); known {
|
||||
out[seat.Name] = broker.Holder{Node: h.Node, Module: h.Module}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -18,11 +18,11 @@ import (
|
||||
func TestOneUnresolvableNodeStillLetsTheRestBeSent(t *testing.T) {
|
||||
sending, refusals := composeEach(
|
||||
[]string{"anchor", "home-server", "laptop"},
|
||||
func(node string) ([]map[string]any, error) {
|
||||
func(node string) (sendable, error) {
|
||||
if node == "anchor" {
|
||||
return nil, errors.New(`nothing provides "acme-ca", wanted by route-proxy`)
|
||||
return sendable{}, errors.New(`nothing provides "acme-ca", wanted by route-proxy`)
|
||||
}
|
||||
return []map[string]any{{"id": node + ".thing"}}, nil
|
||||
return sendable{Resources: []map[string]any{{"id": node + ".thing"}}}, nil
|
||||
})
|
||||
|
||||
var told []string
|
||||
@@ -39,12 +39,14 @@ func TestOneUnresolvableNodeStillLetsTheRestBeSent(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// And a machine assigned nothing is neither sent nor a refusal — it is nothing to say.
|
||||
func TestAMachineAssignedNothingIsNotARefusal(t *testing.T) {
|
||||
// A machine whose declaration composes to nothing is SENT the empty declaration, not skipped
|
||||
// (novox/hq issue 127): it may have held something before, and only sending the empty
|
||||
// declaration tells it to drop what the mesh owned. It is never a refusal.
|
||||
func TestAnEmptyDeclarationIsSentSoTheNodeDropsWhatItHeld(t *testing.T) {
|
||||
sending, refusals := composeEach([]string{"spare"},
|
||||
func(string) ([]map[string]any, error) { return nil, nil })
|
||||
if len(sending) != 0 || len(refusals) != 0 {
|
||||
t.Errorf("a machine assigned nothing was treated as something: %v / %v", sending, refusals)
|
||||
func(string) (sendable, error) { return sendable{}, nil })
|
||||
if len(sending) != 1 || len(refusals) != 0 {
|
||||
t.Errorf("an empty declaration must be sent, not skipped or refused: %v / %v", sending, refusals)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -54,6 +54,8 @@ type meshStatus struct {
|
||||
// Machines is how many the mesh knows about, so a reader can tell "none wrong" from
|
||||
// "none at all".
|
||||
Machines int `json:"machines"`
|
||||
// Adopted is every node still adopted (novox/hq ADR 0100); absent when none is.
|
||||
Adopted []string `json:"adopted,omitempty"`
|
||||
}
|
||||
|
||||
type machineUnresolved struct {
|
||||
@@ -133,7 +135,7 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
||||
out := meshStatus{Machines: len(nodes), Wrong: []machineDoing{},
|
||||
Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{},
|
||||
Reported: []machineReported{}, Unresolved: []machineUnresolved{},
|
||||
Network: asked.network}
|
||||
Network: asked.network, Adopted: adoptedNodes(nodes)}
|
||||
for name := range asked.refused {
|
||||
out.Unresolved = append(out.Unresolved, machineUnresolved{
|
||||
Node: name, Problem: asked.refused[name]})
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// What a build is recorded as, and what it is announced and handed on as.
|
||||
//
|
||||
// **Recorded by what it is; routed where it is used** (novox/hq 04-ISSUES/102). The builder
|
||||
// announces each artifact by the reference it pushed — `<registry>:<port>/<module>/<artifact>@sha256:…`
|
||||
// — and the manifest with those references in it. The mesh records the digest and the path
|
||||
// (catalogue.Recorded) and composes the store's address back in wherever a machine or a builder
|
||||
// needs a reference it can fetch: a declaration, a replay to the catalogue, the bases a build is
|
||||
// given. Nothing recorded carries a port, so moving the store is a settings change and not a
|
||||
// rebuild of everything the mesh has ever built.
|
||||
|
||||
// recordedManifest is a build's manifest with the store's address taken off every reference the
|
||||
// build itself made. Other references — an image a module runs from a public registry — are what
|
||||
// they were, which is why this rewrites only what `made` names rather than everything that looks
|
||||
// like an address.
|
||||
func recordedManifest(raw json.RawMessage, made []inventory.Artifact) json.RawMessage {
|
||||
announced := map[string]bool{}
|
||||
for _, a := range made {
|
||||
announced[a.Reference] = true
|
||||
}
|
||||
return withReferences(raw, func(ref string) (string, bool) {
|
||||
if !announced[ref] {
|
||||
return ref, false
|
||||
}
|
||||
return catalogue.Recorded(ref), true
|
||||
})
|
||||
}
|
||||
|
||||
// routedManifest is a recorded manifest with the store's address, as this network reaches it now,
|
||||
// composed into every reference the build made — recorded either way, before or after references
|
||||
// were kept without their address.
|
||||
func routedManifest(raw json.RawMessage, made []inventory.Artifact, address string) json.RawMessage {
|
||||
recorded := map[string]bool{}
|
||||
for _, a := range made {
|
||||
recorded[catalogue.Recorded(a.Reference)] = true
|
||||
}
|
||||
return withReferences(raw, func(ref string) (string, bool) {
|
||||
if !recorded[catalogue.Recorded(ref)] {
|
||||
return ref, false
|
||||
}
|
||||
return catalogue.Rerouted(ref, address), true
|
||||
})
|
||||
}
|
||||
|
||||
// withReferences applies `rewrite` to each resource's `image` and `source`, and hands the manifest
|
||||
// back untouched — byte for byte — when nothing changed or it could not be read: what a manifest
|
||||
// is, is the parser's to say, and it says so with a better sentence than anything here would.
|
||||
func withReferences(raw json.RawMessage, rewrite func(string) (string, bool)) json.RawMessage {
|
||||
if len(raw) == 0 {
|
||||
return raw
|
||||
}
|
||||
var manifest map[string]any
|
||||
if err := json.Unmarshal(raw, &manifest); err != nil {
|
||||
return raw
|
||||
}
|
||||
resources, _ := manifest["resources"].([]any)
|
||||
changed := false
|
||||
for _, r := range resources {
|
||||
resource, ok := r.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, key := range []string{"image", "source"} {
|
||||
if written, ok := resource[key].(string); ok {
|
||||
if rewritten, did := rewrite(written); did && rewritten != written {
|
||||
resource[key] = rewritten
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if !changed {
|
||||
return raw
|
||||
}
|
||||
out, err := json.Marshal(manifest)
|
||||
if err != nil {
|
||||
return raw
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// routedArtifacts is a build's artifacts as something can fetch them now.
|
||||
func routedArtifacts(made []inventory.Artifact, address string) []inventory.Artifact {
|
||||
if address == "" {
|
||||
return made
|
||||
}
|
||||
out := make([]inventory.Artifact, 0, len(made))
|
||||
for _, a := range made {
|
||||
out = append(out, inventory.Artifact{Name: a.Name, Kind: a.Kind,
|
||||
Reference: catalogue.Rerouted(a.Reference, address)})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// whereTheStoreIs is the artifact store's address as something on `forNode` reaches it, or "" —
|
||||
// read for a caller that has the inventory open and nothing else in hand. With no node named, the
|
||||
// store's own node: loopback when nothing is on the network yet.
|
||||
func whereTheStoreIs(ctx context.Context, inv *inventory.Inventory, forNode string) (string, error) {
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if forNode == "" {
|
||||
if holder, _, found, err := artifactStoreHolder(ctx, inv); err != nil {
|
||||
return "", err
|
||||
} else if found {
|
||||
forNode = holder
|
||||
}
|
||||
}
|
||||
return artifactStoreAddress(ctx, inv, shelf, forNode)
|
||||
}
|
||||
|
||||
// whereABuilderReachesTheStore is the store's address for the machine that builds: the network's
|
||||
// when there is one, else loopback on the store's own node — when that node also holds a module
|
||||
// requiring the store, which is what a builder is (genesis: one node holds both).
|
||||
func whereABuilderReachesTheStore(ctx context.Context, inv *inventory.Inventory) (string, error) {
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
holder, _, found, err := artifactStoreHolder(ctx, inv)
|
||||
if err != nil || !found {
|
||||
return "", err
|
||||
}
|
||||
assigned, err := inv.Assigned(ctx, holder)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
besideIt := false
|
||||
for _, a := range assigned {
|
||||
for _, r := range shelf[a].Requires {
|
||||
if r == catalogue.ArtifactStoreProvision {
|
||||
besideIt = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !besideIt {
|
||||
holder = ""
|
||||
}
|
||||
return artifactStoreAddress(ctx, inv, shelf, holder)
|
||||
}
|
||||
@@ -0,0 +1,451 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/secrets"
|
||||
)
|
||||
|
||||
// Moving the mesh's own traffic to the bus being built (novox/hq ADR 0116 step 5).
|
||||
//
|
||||
// **The whole mesh moves at once, so there is nothing to inspect afterwards.** Every seam ships both
|
||||
// transports and every one of them chooses by a single fact; this is the step that flips it. That
|
||||
// shape is deliberate — steps 1 to 4 leave every node where it is, so the cost of being wrong stays
|
||||
// bounded until here — and it means the useful work is almost all in the checking.
|
||||
//
|
||||
// So `rollout check` is the command that matters and the one that can be run any number of times
|
||||
// against a mesh that is serving. It answers from records: what is missing, and what would happen.
|
||||
// `rollout` itself refuses unless the check is clean.
|
||||
//
|
||||
// **The old broker goes with the move, and goes last** (novox/hq ADR 0131): AMQP is not a provision,
|
||||
// so once every machine reports on the new bus its module is unassigned. Only the mesh's own traffic
|
||||
// is what moves, which is why this is survivable at all: what breaks if it goes wrong is the mesh's
|
||||
// ability to change things, not the services its modules are serving — measured on 2026-09-27, when
|
||||
// a seat emptied mid-change and the control plane looped for two hours while every service stayed up.
|
||||
|
||||
const rolloutUsage = "rollout check | rollout mint [--again] | rollout hand <node> | rollout --confirm"
|
||||
|
||||
func rolloutCommand(ctx context.Context, args []string) error {
|
||||
switch {
|
||||
case len(args) == 1 && args[0] == "check":
|
||||
return rolloutCheck(ctx)
|
||||
case len(args) == 1 && args[0] == "mint":
|
||||
return rolloutMint(ctx, false)
|
||||
case len(args) == 2 && args[0] == "hand":
|
||||
return rolloutHand(ctx, args[1])
|
||||
case len(args) == 2 && args[0] == "mint" && args[1] == "--again":
|
||||
// Every credential minted afresh, whether or not one exists — for a mint that was wrong
|
||||
// before anything was pushed. Afterwards nothing that received the old one still works,
|
||||
// which is fine exactly when nothing received it.
|
||||
return rolloutMint(ctx, true)
|
||||
case len(args) == 1 && args[0] == "--confirm":
|
||||
return errors.New(
|
||||
"the rollout itself is not built yet: `rollout check` answers whether it could run, and " +
|
||||
"what is missing. Moving every node at once is the one step with nothing to inspect " +
|
||||
"afterwards, so it is not being written before the check it depends on has been run " +
|
||||
"against a real mesh")
|
||||
default:
|
||||
return errors.New(rolloutUsage)
|
||||
}
|
||||
}
|
||||
|
||||
// rolloutCheck says whether the mesh could move, and what would happen if it did.
|
||||
func rolloutCheck(ctx context.Context) error {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
state, err := readinessOf(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Println("the bus this mesh would move to")
|
||||
if state.TheBus == "" {
|
||||
fmt.Printf(" nothing names one (%s is unset)\n", broker.NATSVar)
|
||||
} else {
|
||||
standing := "not answering"
|
||||
if state.ServerStanding {
|
||||
standing = "answering"
|
||||
}
|
||||
fmt.Printf(" %s — %s\n", state.TheBus, standing)
|
||||
}
|
||||
fmt.Println()
|
||||
|
||||
fmt.Println("what would move")
|
||||
for _, step := range broker.WhatMoves(state) {
|
||||
fmt.Printf(" %s\n", step)
|
||||
}
|
||||
fmt.Println()
|
||||
|
||||
why := notReadyOf(state)
|
||||
if len(why) == 0 {
|
||||
fmt.Println("nothing is missing: this mesh could move its bus.")
|
||||
fmt.Println()
|
||||
fmt.Println("Read `what would move` above once more before running it. Every node moves at the")
|
||||
fmt.Println("same moment and there is no half-moved state to look at afterwards.")
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("not ready — %d thing(s) to do first:\n", len(why))
|
||||
for i, w := range why {
|
||||
fmt.Printf(" %d. %s\n", i+1, w)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// readinessOf gathers what the mesh knows about its own ability to move.
|
||||
//
|
||||
// Reads and one dial, and nothing is written. Safe to run on a mesh that is serving, which is the
|
||||
// point: the answer is only useful if it can be had without committing to anything.
|
||||
func readinessOf(ctx context.Context, inv *inventory.Inventory) (broker.Readiness, error) {
|
||||
state := broker.Readiness{
|
||||
Credentialled: map[string]bool{},
|
||||
ModuleCredentialled: map[string]bool{},
|
||||
// The old broker keeps its other clients on this installation, and saying so is how the plan
|
||||
// stops reading as a retirement.
|
||||
}
|
||||
|
||||
address, _, err := broker.OnNATS()
|
||||
if err != nil {
|
||||
return state, err
|
||||
}
|
||||
state.TheBus = address
|
||||
if address != "" {
|
||||
// One dial, briefly. "Is it answering" is the one fact records cannot hold, and a mesh about
|
||||
// to move onto a server that is not there should hear it here rather than afterwards.
|
||||
if conn, err := nats.Connect(broker.BareAddress(address), nats.Timeout(5*time.Second)); err == nil {
|
||||
state.ServerStanding = true
|
||||
conn.Close()
|
||||
}
|
||||
}
|
||||
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return state, err
|
||||
}
|
||||
kept, err := inv.BusUsers(ctx)
|
||||
if err != nil {
|
||||
return state, err
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return state, err
|
||||
}
|
||||
|
||||
for _, n := range nodes {
|
||||
state.Nodes = append(state.Nodes, n.Name)
|
||||
_, has := kept[broker.Principal{Kind: broker.KindNode, Node: n.Name}.Username()]
|
||||
state.Credentialled[n.Name] = has
|
||||
|
||||
assigned, err := inv.Assigned(ctx, n.Name)
|
||||
if err != nil {
|
||||
return state, err
|
||||
}
|
||||
for _, module := range assigned {
|
||||
m, known := shelf[module]
|
||||
if !known {
|
||||
continue
|
||||
}
|
||||
// The machine that holds the bus seat is the one that would be sent the user list.
|
||||
if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") {
|
||||
state.Holder = n.Name
|
||||
state.AccountsComposed = wasSentTheUserList(ctx, inv, n.Name)
|
||||
}
|
||||
// A module that never speaks needs no credential, so it is not counted as missing one.
|
||||
if !speaksOnTheBus(m) {
|
||||
continue
|
||||
}
|
||||
named := n.Name + "/" + module
|
||||
state.Modules = append(state.Modules, named)
|
||||
_, hasOne := kept[broker.Principal{
|
||||
Kind: broker.KindModule, Node: n.Name, Module: module,
|
||||
}.Username()]
|
||||
state.ModuleCredentialled[named] = hasOne
|
||||
}
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
// speaksOnTheBus says whether a module reaches the bus at all.
|
||||
//
|
||||
// A third of the catalogue never does (novox/hq ADR 0120), and counting those as missing a credential
|
||||
// would bury the ones that matter under a list nobody can act on.
|
||||
func speaksOnTheBus(m catalogue.Manifest) bool {
|
||||
return len(m.Emits) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
|
||||
len(m.DefinesSeats) > 0 || len(m.Uses) > 0 || len(m.Claims) > 0
|
||||
}
|
||||
|
||||
// wasSentTheUserList says whether the machine holding the bus has had a declaration since the user
|
||||
// list became part of one.
|
||||
//
|
||||
// Read from what the mesh recorded sending rather than asked of the machine: a machine that is away
|
||||
// has still been sent it, and this question is about whether the mesh did its part.
|
||||
func wasSentTheUserList(ctx context.Context, inv *inventory.Inventory, node string) bool {
|
||||
digest, err := inv.Outstanding(ctx, node)
|
||||
return err == nil && strings.TrimSpace(digest) != ""
|
||||
}
|
||||
|
||||
// notReadyOf is the readiness reasoning, named here so a test can reach it without the command's
|
||||
// printing. The reasoning itself is the broker package's, where it is pure.
|
||||
func notReadyOf(state broker.Readiness) []string { return broker.NotReady(state) }
|
||||
|
||||
// rolloutMint gives every principal the new bus will have a credential it does not yet have, and
|
||||
// puts each where its owner reads it (novox/hq design 28, task 5.2): a machine's as a membership
|
||||
// sealed into its declaration, a module's as its broker secret, the control plane's own as its
|
||||
// `bus` secret. Idempotent: what already has a hash is left alone, so running it again is harmless.
|
||||
//
|
||||
// **Before anything moves, and it is what makes moving possible.** A machine moved without a
|
||||
// credential cannot come back, and afterwards there is no bus to tell it anything over — which is
|
||||
// why `rollout check` refuses until this has run. The bus's address is worked out here, from where
|
||||
// the module that provides it is assigned, rather than read from this process's environment: this
|
||||
// process is still on the old bus when this runs, and must be.
|
||||
func rolloutMint(ctx context.Context, again bool) error {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
known, err := broker.FromEnvironment()
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bus's certificate is not known to this process, and every membership "+
|
||||
"must carry its fingerprint: %w", err)
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var busNode, controllerNode string
|
||||
for _, e := range entries {
|
||||
switch {
|
||||
case e.Manifest.ClaimsSeat("mesh-broker") && providesBus(e.Manifest) && len(e.On) > 0:
|
||||
busNode = e.On[0]
|
||||
case e.Manifest.Module == "mesh-controller" && len(e.On) > 0:
|
||||
controllerNode = e.On[0]
|
||||
}
|
||||
}
|
||||
if busNode == "" {
|
||||
return errors.New("no assigned module provides mesh-bus and claims mesh-broker, so there is no " +
|
||||
"bus to mint credentials for — register and assign it first")
|
||||
}
|
||||
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
busHost := onNetwork[busNode]
|
||||
if busHost == "" {
|
||||
// **The hub is not in that map.** The machine that took over the tunnel is where the current
|
||||
// bus already answers, and every machine dials it at the address the mesh handed them — so
|
||||
// when the new bus runs on the same machine, that address is the one to tell them, with the
|
||||
// new port. Found live: the control node is the hub, and the map lists the machines placed
|
||||
// around it.
|
||||
// The host alone: no scheme (BareAddress adds one where none was, which is the wrong
|
||||
// direction here — every URL built below adds its own) and no port.
|
||||
_, _, host := broker.CredentialIn(known.Address)
|
||||
if host == "" {
|
||||
host = known.Address
|
||||
}
|
||||
if _, after, hasScheme := strings.Cut(host, "://"); hasScheme {
|
||||
host = after
|
||||
}
|
||||
host = strings.TrimSpace(host)
|
||||
if i := strings.LastIndex(host, ":"); i > 0 && !strings.Contains(host[i:], "]") {
|
||||
host = host[:i]
|
||||
}
|
||||
if host == "" {
|
||||
return fmt.Errorf("%s runs the new bus and has no address on the private network, and the "+
|
||||
"current bus's address is unknown too, so no machine could be told where it is", busNode)
|
||||
}
|
||||
busHost = host
|
||||
}
|
||||
busAddress := busHost + ":4222"
|
||||
|
||||
records, err := inv.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
kept, err := inv.BusUsers(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hashes := make(map[string]string, len(kept))
|
||||
for name, u := range kept {
|
||||
hashes[name] = u.PasswordHash
|
||||
}
|
||||
_, missing := broker.WithPasswords(users, hashes)
|
||||
wanted := map[string]bool{}
|
||||
for _, m := range missing {
|
||||
wanted[m] = true
|
||||
}
|
||||
|
||||
var machines, modules, skipped int
|
||||
for _, p := range users {
|
||||
if !again && !wanted[p.Username()] {
|
||||
continue
|
||||
}
|
||||
switch p.Kind {
|
||||
case broker.KindController:
|
||||
if controllerNode == "" {
|
||||
return errors.New("the control plane is not assigned anywhere, so its credential has nowhere to go")
|
||||
}
|
||||
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusController})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
url := "nats://" + p.Username() + ":" + password + "@" + busAddress
|
||||
if err := inv.AcceptSecretForModule(ctx, controllerNode, "mesh-controller", "bus", url); err != nil {
|
||||
return fmt.Errorf("the control plane's credential is minted and could not be sealed to %s: %w", controllerNode, err)
|
||||
}
|
||||
fmt.Printf("control plane: credential minted, sealed to %s as its `bus` secret\n", controllerNode)
|
||||
|
||||
case broker.KindNode:
|
||||
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusNode, Node: p.Node})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
membership, _ := json.Marshal(map[string]string{
|
||||
"broker": busAddress, "fingerprint": known.Fingerprint, "password": password, "transport": "nats",
|
||||
})
|
||||
key, err := inv.SealingKeyOf(ctx, p.Node)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s has no sealing key, so its membership cannot be sealed to it: %w", p.Node, err)
|
||||
}
|
||||
sealed, err := secrets.Seal(key, membership)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.PutBusMembership(ctx, p.Node, sealed); err != nil {
|
||||
return err
|
||||
}
|
||||
machines++
|
||||
|
||||
case broker.KindModule:
|
||||
if p.Module == "mesh-controller" {
|
||||
// The control plane is a module too, and its `broker` secret is the old bus's
|
||||
// credential it is still using while this runs. Writing the new bus's blob there
|
||||
// cut the mesh off from its own old bus mid-move (2026-09-28). Its new-bus credential
|
||||
// is the controller principal's `bus` secret above; nothing else is needed here.
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
m, inShelf := shelf[p.Module]
|
||||
if !inShelf {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
if _, reads := m.OwnSecrets["broker"]; !reads {
|
||||
fmt.Printf(" %s on %s speaks on the bus but declares no `broker` secret to receive a credential in; skipped\n", p.Module, p.Node)
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusModule, Node: p.Node, Module: p.Module})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := issueWith(ctx, inv, m, p.Node, "", known, busAddress, p.Username(), password); err != nil {
|
||||
return err
|
||||
}
|
||||
modules++
|
||||
|
||||
default:
|
||||
skipped++
|
||||
}
|
||||
}
|
||||
fmt.Printf("minted for %d machine(s) and %d module runtime(s); %d skipped; the bus is at %s\n",
|
||||
machines, modules, skipped, busAddress)
|
||||
fmt.Println(" each machine's membership and each module's credential arrive with the next push of its machine;")
|
||||
fmt.Println(" push the machine running the bus first, so the bus stands with its user list before anything dials it")
|
||||
return nil
|
||||
}
|
||||
|
||||
// providesBus is whether a manifest provides the mesh's bus.
|
||||
func providesBus(m catalogue.Manifest) bool {
|
||||
for _, o := range m.Provides {
|
||||
if o.Name == "mesh-bus" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// rolloutHand mints a machine its credential for the new bus afresh and prints its membership
|
||||
// once, for an operator to carry by hand — the rescue for a machine that cannot be reached over
|
||||
// any bus: rotated while it still held the old password, or reachable only by ssh. The plaintext
|
||||
// exists on this terminal and then only where it is written; the store keeps the hash, and the
|
||||
// sealed copy in the machine's declaration is replaced too, so the next push says the same.
|
||||
func rolloutHand(ctx context.Context, node string) error {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
known, err := broker.FromEnvironment()
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bus's certificate is not known to this process: %w", err)
|
||||
}
|
||||
busAddress, _, err := broker.OnNATS()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if busAddress == "" {
|
||||
return errors.New("this control plane is not on the new bus, so there is no membership to hand out")
|
||||
}
|
||||
_, _, bare := broker.CredentialIn(busAddress)
|
||||
if _, after, has := strings.Cut(bare, "://"); has {
|
||||
bare = after
|
||||
}
|
||||
if _, err := inv.NodeByName(ctx, node); err != nil {
|
||||
return err
|
||||
}
|
||||
p := broker.Principal{Kind: broker.KindNode, Node: node}
|
||||
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusNode, Node: node})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
membership, _ := json.Marshal(map[string]string{
|
||||
"broker": bare, "fingerprint": known.Fingerprint, "password": password, "transport": "nats",
|
||||
})
|
||||
key, err := inv.SealingKeyOf(ctx, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sealed, err := secrets.Seal(key, membership)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.PutBusMembership(ctx, node, sealed); err != nil {
|
||||
return err
|
||||
}
|
||||
// The one line of output is the membership itself, so it can be piped to the machine without
|
||||
// being read on the way. Everything else goes to stderr.
|
||||
fmt.Fprintf(os.Stderr, "%s's credential is minted afresh. Write this to %s on it and restart its host; "+
|
||||
"then push the machine running the bus so the user list carries the new hash.\n",
|
||||
node, catalogue.BusMembershipPath)
|
||||
fmt.Println(string(membership))
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// Whether a mesh could move its bus, read from a real store.
|
||||
//
|
||||
// The readiness reasoning has its own tests; this is about the gathering — that the question is
|
||||
// answered from what the mesh actually holds, on a store with machines and modules in it, without
|
||||
// writing anything.
|
||||
|
||||
func TestReadinessIsGatheredFromWhatTheMeshHolds(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
ctx := context.Background()
|
||||
|
||||
// A mesh mid-change: two machines, the bus module on one of them, a module that speaks and a
|
||||
// module that never does.
|
||||
for _, m := range []catalogue.Manifest{
|
||||
{Module: "nats", Version: "1", BusUsers: "/var/lib/nats-module/conf/accounts.conf",
|
||||
Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}}},
|
||||
{Module: "gitea", Version: "1", Tools: []string{"repo_create"}},
|
||||
{Module: "wallpaper", Version: "1"},
|
||||
} {
|
||||
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "/r"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.AddNode(ctx, n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, a := range [][2]string{{"anchor", "nats"}, {"anchor", "gitea"}, {"laptop", "wallpaper"}} {
|
||||
if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
state, err := readinessOf(ctx, inv)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if state.Holder != "anchor" {
|
||||
t.Errorf("the machine holding the bus reads as %q", state.Holder)
|
||||
}
|
||||
if len(state.Nodes) != 2 {
|
||||
t.Errorf("machines read as %v", state.Nodes)
|
||||
}
|
||||
// **A module that never speaks is not counted as missing a credential.** A third of the catalogue
|
||||
// never reaches the bus, and listing those would bury the ones that matter.
|
||||
for _, m := range state.Modules {
|
||||
if strings.HasSuffix(m, "/wallpaper") {
|
||||
t.Errorf("a module that never speaks was counted: %v", state.Modules)
|
||||
}
|
||||
}
|
||||
if len(state.Modules) != 2 {
|
||||
t.Errorf("modules that speak read as %v; expected the bus module and the one with a tool",
|
||||
state.Modules)
|
||||
}
|
||||
|
||||
// Nothing has been minted, so it is not ready — and it says so about each machine by name.
|
||||
why := notReadyOf(state)
|
||||
if len(why) == 0 {
|
||||
t.Fatal("a mesh where nothing has a credential was reported ready to move")
|
||||
}
|
||||
said := strings.Join(why, "\n")
|
||||
for _, name := range []string{"anchor", "laptop"} {
|
||||
if !strings.Contains(said, name) {
|
||||
t.Errorf("the refusal does not name %s: %s", name, said)
|
||||
}
|
||||
}
|
||||
|
||||
// Mint for one machine and it drops out of the complaint, which is how somebody works through it.
|
||||
if _, err := inv.MintBusPassword(ctx, inventory.BusUser{
|
||||
Username: "node.laptop", Kind: inventory.BusNode, Node: "laptop",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state, err = readinessOf(ctx, inv)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !state.Credentialled["laptop"] {
|
||||
t.Error("a machine that was minted a credential still reads as having none")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,267 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"text/tabwriter"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// What this mesh can have one of, and who fills each (novox/hq ADR 0110).
|
||||
//
|
||||
// **Derived every time, never stored.** A seat is held by a module assignment, so the answer is
|
||||
// computed from assignments by the same resolution that decides what every machine runs. A table
|
||||
// of holders kept beside the assignments would be a second copy of one fact, and the first thing
|
||||
// to be wrong about it.
|
||||
|
||||
// seatHolder is one assignment holding a seat.
|
||||
type seatHolder struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
}
|
||||
|
||||
// seatRow is one seat and who holds it. Unheld is an answer — "this mesh has no X" — not a fault.
|
||||
type seatRow struct {
|
||||
Seat string `json:"seat"`
|
||||
Scope string `json:"scope"`
|
||||
Delivers string `json:"delivers,omitempty"`
|
||||
Decision string `json:"decision"`
|
||||
Holders []seatHolder `json:"holders"`
|
||||
}
|
||||
|
||||
// seatsHeld is every seat the mesh defines with its holders, and every claim held that names no
|
||||
// seat in the set.
|
||||
//
|
||||
// **The second list is not empty by construction.** Manifests are held to the set when they are
|
||||
// registered, and a mesh can hold one registered before the set closed. Leaving its claim out of the
|
||||
// overview would make the one thing the overview is for — what does this mesh have — quietly
|
||||
// incomplete.
|
||||
func seatsHeld(seats []catalogue.Seat, held []catalogue.Held) ([]seatRow, []catalogue.Held) {
|
||||
rows := make([]seatRow, 0, len(seats))
|
||||
for _, s := range seats {
|
||||
row := seatRow{Seat: s.Name, Scope: s.Scope, Delivers: s.Delivers, Decision: s.Decision,
|
||||
Holders: []seatHolder{}}
|
||||
seen := map[seatHolder]bool{}
|
||||
for _, h := range held {
|
||||
// Resolve the held claim to a seat rather than comparing names, so a record naming a
|
||||
// seat's former name groups under it after a rename (novox/hq ADR 0122).
|
||||
hs, ok := catalogue.SeatNamed(h.Claim)
|
||||
if !ok || hs.Name != s.Name || h.Scope != s.Scope {
|
||||
continue
|
||||
}
|
||||
holder := seatHolder{Node: h.Node, Module: h.Module}
|
||||
if !seen[holder] {
|
||||
seen[holder] = true
|
||||
row.Holders = append(row.Holders, holder)
|
||||
}
|
||||
}
|
||||
sort.Slice(row.Holders, func(i, j int) bool {
|
||||
if row.Holders[i].Node != row.Holders[j].Node {
|
||||
return row.Holders[i].Node < row.Holders[j].Node
|
||||
}
|
||||
return row.Holders[i].Module < row.Holders[j].Module
|
||||
})
|
||||
rows = append(rows, row)
|
||||
}
|
||||
var outside []catalogue.Held
|
||||
for _, h := range held {
|
||||
// Outside the set only if it resolves to no seat at all — a former name still resolves.
|
||||
if _, ok := catalogue.SeatNamed(h.Claim); !ok {
|
||||
outside = append(outside, h)
|
||||
}
|
||||
}
|
||||
sort.Slice(outside, func(i, j int) bool {
|
||||
if outside[i].Claim != outside[j].Claim {
|
||||
return outside[i].Claim < outside[j].Claim
|
||||
}
|
||||
return outside[i].Node < outside[j].Node
|
||||
})
|
||||
return rows, outside
|
||||
}
|
||||
|
||||
// seatCommand changes the set — the whole point of it being data (novox/hq ADR 0122) — and, since
|
||||
// ADR 0131, changes who holds a seat.
|
||||
func seatCommand(ctx context.Context, args []string) error {
|
||||
if len(args) == 3 && args[0] == "rename" {
|
||||
from, to := args[1], args[2]
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
if err := open.inventory.RenameSeat(ctx, from, to); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s is now %s — its former name still resolves, so nothing is rebuilt, "+
|
||||
"re-registered or frozen (novox/hq ADR 0122)\n", from, to)
|
||||
return nil
|
||||
}
|
||||
if len(args) == 3 && args[1] == "--to" {
|
||||
return handOver(ctx, args[0], args[2])
|
||||
}
|
||||
return fmt.Errorf("seat rename <from> <to> | seat <name> --to <node>/<module>")
|
||||
}
|
||||
|
||||
// handOver makes one assignment the holder of a seat, as one act, so the seat is never without a
|
||||
// holder in between (novox/hq ADR 0131, design 28 task 5.3). The control plane finds its own bus
|
||||
// through one of these seats; the day it was left empty mid-change is why this exists.
|
||||
//
|
||||
// Everything that could make the new holder wrong is refused here, before the row is written: the
|
||||
// seat must exist, the assignment must exist, and the module must be able to hold the seat —
|
||||
// claim it at its scope and provide what it delivers, judged against the store's row. What is
|
||||
// **not** checked is whether the module is running yet: that is what `push` confirms afterwards,
|
||||
// and refusing to record a handover to a module the node has not started would make the handover
|
||||
// impossible to do before the switch instead of as the switch.
|
||||
func handOver(ctx context.Context, seatName, to string) error {
|
||||
nodeName, module, ok := strings.Cut(to, "/")
|
||||
if !ok || nodeName == "" || module == "" {
|
||||
return fmt.Errorf("the new holder is named <node>/<module>, not %q", to)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
seat, known := catalogue.SeatNamed(seatName)
|
||||
if !known {
|
||||
return fmt.Errorf("%q is not a seat this mesh defines — `seats` lists them", seatName)
|
||||
}
|
||||
assigned, err := inv.Assigned(ctx, nodeName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !slices.Contains(assigned, module) {
|
||||
return fmt.Errorf("%s is not assigned to %s, so it cannot hold anything there — "+
|
||||
"`assign %s %s` first", module, nodeName, nodeName, module)
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var m *catalogue.Manifest
|
||||
for i := range entries {
|
||||
if entries[i].Manifest.Module == module {
|
||||
m = &entries[i].Manifest
|
||||
}
|
||||
}
|
||||
if m == nil {
|
||||
return fmt.Errorf("%s is assigned but not in the catalogue, which should not happen", module)
|
||||
}
|
||||
var was string
|
||||
holdings, err := inv.Holdings(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, h := range holdings {
|
||||
if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name {
|
||||
was = h.Node
|
||||
}
|
||||
}
|
||||
|
||||
// **Recording who already holds the seat is not making a new holder, and is not judged like
|
||||
// one.** On a mesh that predates the record, the first handover has to begin by writing down
|
||||
// the standing holder — otherwise the next holder cannot be assigned beside it, because two
|
||||
// eligible claimants with nothing on record are refused. That standing holder may no longer
|
||||
// satisfy what the seat delivers (the row moved under it, on purpose, as ADR 0131's first step),
|
||||
// and it holds regardless: derivation never read that column. So when nothing is on record and
|
||||
// the named assignment is the one holding by derivation, only the claim itself is checked here.
|
||||
// Every *change* of holder is judged in full.
|
||||
claimsIt := false
|
||||
for _, c := range m.Claims {
|
||||
if cs, ok := catalogue.SeatNamed(c.Name); ok && cs.Name == seat.Name && c.At() == seat.Scope {
|
||||
claimsIt = true
|
||||
}
|
||||
}
|
||||
if was == "" && claimsIt {
|
||||
fmt.Printf("nothing was on record for %s; recording %s on %s as its standing holder\n",
|
||||
seat.Name, module, nodeName)
|
||||
} else if err := catalogue.CanHold(*m, seat); err != nil {
|
||||
return fmt.Errorf("%s cannot hold %s: %w", module, seat.Name, err)
|
||||
}
|
||||
if err := inv.HoldSeat(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s is held by %s on %s\n", seat.Name, module, nodeName)
|
||||
if was != "" && was != nodeName {
|
||||
fmt.Printf(" `push %s` and `push %s` send both machines what changed\n", was, nodeName)
|
||||
} else {
|
||||
fmt.Printf(" `push %s` sends the machine what changed; every other machine that reads the "+
|
||||
"seat is re-declared by `push --behind`\n", nodeName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func seatsCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("seats", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "the same, as JSON")
|
||||
if err := set.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Every node, none excluded: the same view of what each machine holds that planning uses.
|
||||
world, err := theRestOfTheMesh(ctx, inv, shelf, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rows, outside := seatsHeld(catalogue.Seats(), world.Held)
|
||||
|
||||
if *asJSON {
|
||||
out := struct {
|
||||
Seats []seatRow `json:"seats"`
|
||||
Outside []catalogue.Held `json:"outside,omitempty"`
|
||||
}{rows, outside}
|
||||
body, err := json.MarshalIndent(out, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
|
||||
w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
|
||||
fmt.Fprintln(w, "SEAT\tSCOPE\tDELIVERS\tHELD BY")
|
||||
for _, r := range rows {
|
||||
delivers := r.Delivers
|
||||
if delivers == "" {
|
||||
delivers = "—"
|
||||
}
|
||||
holders := "unheld"
|
||||
if len(r.Holders) > 0 {
|
||||
parts := make([]string, 0, len(r.Holders))
|
||||
for _, h := range r.Holders {
|
||||
parts = append(parts, h.Module+" on "+h.Node)
|
||||
}
|
||||
holders = strings.Join(parts, ", ")
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", r.Seat, r.Scope, delivers, holders)
|
||||
}
|
||||
if err := w.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(outside) > 0 {
|
||||
fmt.Println("\nheld, and not a seat this mesh defines (registered before the set closed — novox/hq ADR 0110):")
|
||||
for _, h := range outside {
|
||||
fmt.Printf(" %s %s on %s\n", h.Claim, h.Module, h.Node)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// The overview of what a mesh has (novox/hq ADR 0110).
|
||||
|
||||
func TestEverySeatIsListedIncludingTheOnesNobodyHolds(t *testing.T) {
|
||||
// An unheld seat is an answer — "this mesh has no forge" — so it is listed rather than omitted.
|
||||
rows, _ := seatsHeld(catalogue.Seats(), []catalogue.Held{
|
||||
{Claim: "mesh-store", Scope: catalogue.ScopeMesh, Node: "anchor", Module: "postgres"},
|
||||
})
|
||||
if len(rows) != len(catalogue.Seats()) {
|
||||
t.Fatalf("%d seats listed of %d", len(rows), len(catalogue.Seats()))
|
||||
}
|
||||
for _, r := range rows {
|
||||
switch r.Seat {
|
||||
case "mesh-store":
|
||||
if len(r.Holders) != 1 || r.Holders[0].Module != "postgres" || r.Holders[0].Node != "anchor" {
|
||||
t.Errorf("mesh-store is held by %+v", r.Holders)
|
||||
}
|
||||
if r.Delivers != "postgres-database" {
|
||||
t.Errorf("mesh-store does not say what it delivers: %q", r.Delivers)
|
||||
}
|
||||
case "git":
|
||||
if len(r.Holders) != 0 {
|
||||
t.Errorf("git is held by %+v in a mesh with no forge", r.Holders)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodeSeatListsEveryMachineHoldingIt(t *testing.T) {
|
||||
rows, _ := seatsHeld(catalogue.Seats(), []catalogue.Held{
|
||||
{Claim: "node-packet-filter", Scope: catalogue.ScopeNode, Node: "node2", Module: "nftables"},
|
||||
{Claim: "node-packet-filter", Scope: catalogue.ScopeNode, Node: "anchor", Module: "nftables"},
|
||||
// Resolved twice, reported once: a machine is one holder however many passes saw it.
|
||||
{Claim: "node-packet-filter", Scope: catalogue.ScopeNode, Node: "anchor", Module: "nftables"},
|
||||
})
|
||||
for _, r := range rows {
|
||||
if r.Seat != "node-packet-filter" {
|
||||
continue
|
||||
}
|
||||
if len(r.Holders) != 2 || r.Holders[0].Node != "anchor" || r.Holders[1].Node != "node2" {
|
||||
t.Fatalf("the packet filter is held by %+v", r.Holders)
|
||||
}
|
||||
return
|
||||
}
|
||||
t.Fatal("the packet filter is not listed")
|
||||
}
|
||||
|
||||
func TestAClaimOutsideTheSetIsShownNotHidden(t *testing.T) {
|
||||
// A manifest registered before the set closed can still hold one. Leaving it out would make
|
||||
// the overview quietly incomplete, which is the one thing it may not be.
|
||||
_, outside := seatsHeld(catalogue.Seats(), []catalogue.Held{
|
||||
{Claim: "the-controller", Scope: catalogue.ScopeMesh, Node: "anchor", Module: "mesh-controller"},
|
||||
})
|
||||
if len(outside) != 1 || outside[0].Claim != "the-controller" {
|
||||
t.Fatalf("a claim outside the set was not shown: %+v", outside)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// sendable is a declaration as a machine is sent it: its resources and, for an adopted node, its
|
||||
// mode and which modules were taken on it (novox/hq ADR 0100).
|
||||
//
|
||||
// **Body is the only place the envelope is marshalled.** It was written by hand at every send site,
|
||||
// in the digest the mesh compares, and in `plan --json`; a key added at one and not another would
|
||||
// make a machine look out of date for ever, or send something `plan` never showed.
|
||||
type sendable struct {
|
||||
Resources []map[string]any
|
||||
// Adoption is nil for a converged node, and then the body is byte for byte what it was before
|
||||
// adoption existed: an older host parses the envelope strictly and would refuse the key.
|
||||
Adoption *adoptionEnvelope
|
||||
}
|
||||
|
||||
// adoptionEnvelope is what an adopted node is told about its mode. Taken is every module taken on
|
||||
// it that it runs; Untaken is, for every module it runs that is not taken, the ids of every one of
|
||||
// that module's resources — what the host keeps as found until the module is taken (ADR 0103). Ids
|
||||
// rather than a rule to split them by, because a module's name may contain a dot.
|
||||
type adoptionEnvelope struct {
|
||||
Taken []string `json:"taken"`
|
||||
Untaken map[string][]string `json:"untaken,omitempty"`
|
||||
}
|
||||
|
||||
// Body is the declaration's bytes, as sent and as digested.
|
||||
func (s sendable) Body() ([]byte, error) {
|
||||
envelope := map[string]any{"declaration": 1, "resources": s.Resources}
|
||||
if s.Adoption != nil {
|
||||
envelope["adoption"] = s.Adoption
|
||||
}
|
||||
// An empty declaration is deliberate here — the node owns nothing the mesh put there
|
||||
// (novox/hq issue 127) — and the host refuses an empty body unless it is told the emptiness
|
||||
// is meant, so a truncated or mis-composed body is never mistaken for "own nothing".
|
||||
if len(s.Resources) == 0 {
|
||||
envelope["owns_nothing"] = true
|
||||
}
|
||||
return json.Marshal(envelope)
|
||||
}
|
||||
|
||||
// adoptionOf is the envelope for a node, nil when it is converged.
|
||||
func adoptionOf(ctx context.Context, inv *inventory.Inventory, record inventory.Node,
|
||||
plan catalogue.Resolution, composed catalogue.Composed) (*adoptionEnvelope, error) {
|
||||
if !record.Adopted {
|
||||
return nil, nil
|
||||
}
|
||||
taken, err := inv.Taken(ctx, record.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return adoptionFor(plan, taken, composed), nil
|
||||
}
|
||||
|
||||
// adoptionFor is the envelope computed from what was taken and who owns each resource.
|
||||
//
|
||||
// Every module the node runs that is not taken is untaken — including one pulled in by another
|
||||
// rather than assigned: what is found is kept until its module is taken, whoever put it there.
|
||||
func adoptionFor(plan catalogue.Resolution, taken []string,
|
||||
composed catalogue.Composed) *adoptionEnvelope {
|
||||
isTaken := map[string]bool{}
|
||||
for _, m := range taken {
|
||||
isTaken[m] = true
|
||||
}
|
||||
out := &adoptionEnvelope{Taken: []string{}}
|
||||
runs := map[string]bool{}
|
||||
for _, m := range plan.Modules {
|
||||
runs[m.Module] = true
|
||||
if isTaken[m.Module] {
|
||||
out.Taken = append(out.Taken, m.Module)
|
||||
}
|
||||
}
|
||||
sort.Strings(out.Taken)
|
||||
for _, r := range composed.Resources {
|
||||
id, _ := r["id"].(string)
|
||||
module, owned := composed.Owner[id]
|
||||
// Every kind, not only files and containers (novox/hq ADR 0103): a directory, a service, a
|
||||
// container mounting what was found and an action run in a held container all reach what
|
||||
// the machine already has. What the mesh declares of its own is never held.
|
||||
if !owned || !runs[module] || isTaken[module] ||
|
||||
strings.HasPrefix(id, catalogue.AdoptionPrefix) {
|
||||
continue
|
||||
}
|
||||
if out.Untaken == nil {
|
||||
out.Untaken = map[string][]string{}
|
||||
}
|
||||
out.Untaken[module] = append(out.Untaken[module], id)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,379 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0100: every declaration an adopted node is sent says it is adopted and which modules
|
||||
// were taken on it; a converged node's declaration is byte for byte what it was.
|
||||
|
||||
// helloWeb is a module the predecessor also runs: a page and a server under names it uses.
|
||||
func helloWeb() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "hello-web", Version: "1",
|
||||
Resources: []map[string]any{
|
||||
{"id": "page", "type": "file", "path": "/var/lib/hello-web/index.html", "content": "hello"},
|
||||
{"id": "server", "type": "container", "name": "hello-web",
|
||||
"image": "registry.example/hello@sha256:" + strings.Repeat("a", 64)},
|
||||
{"id": "served", "type": "directory", "path": "/var/lib/hello-web"},
|
||||
}}
|
||||
}
|
||||
|
||||
// composed is what node would be sent now, as push composes it.
|
||||
func composed(t *testing.T, open *stores, node string) sendable {
|
||||
t.Helper()
|
||||
plan, settings, err := planFor(t.Context(), open, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationFor(t.Context(), open, node, plan, settings)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return declared
|
||||
}
|
||||
|
||||
// convergedBefore is the envelope a converged node was sent before adoption existed, captured by
|
||||
// running this same composition at the commit this branch left main (0a39b7d). The mesh here is
|
||||
// aMesh's laptop with the private network taken off it, so nothing in the declaration is random:
|
||||
// what changes this string is a change to what a converged machine is sent, which is the thing an
|
||||
// older host would refuse.
|
||||
const convergedBefore = `{"declaration":1,"resources":[{"content":"hello","id":"hello-web.page","path":"/var/lib/hello-web/index.html","type":"file"},{"hosts":["anchor.internal:10.77.0.1"],"id":"hello-web.server","image":"registry.example/hello@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","name":"hello-web","type":"container"},{"id":"hello-web.served","path":"/var/lib/hello-web","type":"directory"}]}`
|
||||
|
||||
func TestAConvergedDeclarationIsByteForByteWhatItWas(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, helloWeb())
|
||||
if _, err := unassign(ctx, open, "laptop", overlay.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "hello-web"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared := composed(t, open, "laptop")
|
||||
if declared.Adoption != nil {
|
||||
t.Fatal("a converged node was given an adoption envelope")
|
||||
}
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(body) != convergedBefore {
|
||||
t.Fatalf("a converged declaration changed; an older host parses this strictly:\n%s\n%s",
|
||||
body, convergedBefore)
|
||||
}
|
||||
if bytes.Contains(body, []byte(`"adoption"`)) {
|
||||
t.Fatal("a converged declaration names adoption; an older host would refuse it")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAdoptedDeclarationCarriesItsModeAndWhatWasTaken(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, helloWeb())
|
||||
if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "anchor", "hello-web"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
declared := composed(t, open, "anchor")
|
||||
if declared.Adoption == nil {
|
||||
t.Fatal("an adopted node's declaration does not say it is adopted")
|
||||
}
|
||||
untaken := declared.Adoption.Untaken["hello-web"]
|
||||
// Every kind — its directory too (novox/hq ADR 0103).
|
||||
if !reflect.DeepEqual(untaken, []string{"hello-web.page", "hello-web.server",
|
||||
"hello-web.served"}) {
|
||||
t.Fatalf("hello-web's resources are not all named untaken: %v", declared.Adoption)
|
||||
}
|
||||
if len(declared.Adoption.Taken) != 0 {
|
||||
t.Fatalf("nothing was taken, and the declaration says %v", declared.Adoption.Taken)
|
||||
}
|
||||
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var envelope map[string]any
|
||||
if err := json.Unmarshal(body, &envelope); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
adoption, _ := envelope["adoption"].(map[string]any)
|
||||
if _, ok := adoption["taken"].([]any); !ok {
|
||||
t.Fatalf("taken is not a list on the wire, even empty: %s", body)
|
||||
}
|
||||
|
||||
// The digest the mesh compares is the digest of what is sent: status and push agree.
|
||||
would, err := wouldSend(ctx, open, mustNodes(t, open))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if would["anchor"] != digestOf(body) {
|
||||
t.Fatal("the digest the mesh compares is not of the declaration push sends")
|
||||
}
|
||||
|
||||
// plan --json prints that same envelope.
|
||||
printed := stdoutOf(t, func() error { return planCommand(ctx, []string{"anchor", "--json"}) })
|
||||
var compact bytes.Buffer
|
||||
if err := json.Compact(&compact, []byte(printed)); err != nil {
|
||||
t.Fatalf("plan --json is not JSON: %v\n%s", err, printed)
|
||||
}
|
||||
if digestOf(compact.Bytes()) != digestOf(body) {
|
||||
t.Fatalf("plan --json shows something other than what push sends:\n%s", printed)
|
||||
}
|
||||
|
||||
// Taking the module moves its resources out of untaken.
|
||||
if err := open.inventory.Take(ctx, "anchor", "hello-web"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared = composed(t, open, "anchor")
|
||||
if _, still := declared.Adoption.Untaken["hello-web"]; still {
|
||||
t.Fatalf("a taken module is still untaken: %v", declared.Adoption)
|
||||
}
|
||||
if !reflect.DeepEqual(declared.Adoption.Taken, []string{"hello-web"}) {
|
||||
t.Fatalf("taken is %v", declared.Adoption.Taken)
|
||||
}
|
||||
}
|
||||
|
||||
func mustNodes(t *testing.T, open *stores) []inventory.Node {
|
||||
t.Helper()
|
||||
nodes, err := open.inventory.Nodes(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return nodes
|
||||
}
|
||||
|
||||
// stdoutOf is what run printed.
|
||||
func stdoutOf(t *testing.T, run func() error) string {
|
||||
t.Helper()
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saved := os.Stdout
|
||||
os.Stdout = w
|
||||
done := make(chan string)
|
||||
go func() {
|
||||
all, _ := io.ReadAll(r)
|
||||
done <- string(all)
|
||||
}()
|
||||
runErr := run()
|
||||
os.Stdout = saved
|
||||
w.Close()
|
||||
out := <-done
|
||||
if runErr != nil {
|
||||
t.Fatalf("%v\n%s", runErr, out)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// novox/hq ADR 0100: a port a node was given for the store is where its consumers on other
|
||||
// machines are told to reach it, and a port given for the whole mesh is refused.
|
||||
func TestConsumersAreToldTheGivenPort(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "store", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "database", Scope: catalogue.ScopeMesh}},
|
||||
Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}},
|
||||
Guards: []int{5432},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store",
|
||||
"ports": []any{"5432:5432"},
|
||||
"image": "registry.example/pg@sha256:" + strings.Repeat("b", 64)}}})
|
||||
register(t, open, catalogue.Manifest{Module: "app", Version: "1", Requires: []string{"database"}})
|
||||
if _, err := assign(ctx, open, "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "store",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5433}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
plan, _, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var told any
|
||||
for _, n := range plan.Needs {
|
||||
if n.Name == "database" {
|
||||
told = n.Serves["port"]
|
||||
}
|
||||
}
|
||||
if told != 5433 {
|
||||
t.Fatalf("the consumer is told the store is on %v", told)
|
||||
}
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
if r["id"] == "store.server" && !reflect.DeepEqual(r["ports"], []any{"5433:5432"}) {
|
||||
t.Fatalf("the store publishes %v", r["ports"])
|
||||
}
|
||||
}
|
||||
|
||||
// A port for the whole mesh is refused where it is set, not stored to refuse every node's
|
||||
// declaration afterwards.
|
||||
if err := open.inventory.SetSettings(ctx, "", "store",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5434}}); err == nil ||
|
||||
!strings.Contains(err.Error(), "per node") {
|
||||
t.Fatalf("a port given for the whole mesh was not refused: %v", err)
|
||||
}
|
||||
plan, settings, err := planFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := declarationFor(ctx, open, "anchor", plan, settings); err != nil {
|
||||
t.Fatalf("the refused mesh-wide layer was stored anyway: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0103: the guard an adopted node is sent follows what was taken there. A store
|
||||
// assigned but not taken is not guarded — its port may still be the predecessor's — and taking it
|
||||
// guards it from the next declaration.
|
||||
func TestTheGuardIsSentForTakenModulesOnly(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "store", Version: "1",
|
||||
Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}},
|
||||
Guards: []int{5432},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store",
|
||||
"ports": []any{"5432:5432"},
|
||||
"image": "registry.example/pg@sha256:" + strings.Repeat("b", 64)}}})
|
||||
if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if hasID(composed(t, open, "anchor").Resources, catalogue.GuardID()) {
|
||||
t.Fatal("an untaken store is guarded")
|
||||
}
|
||||
if err := open.inventory.Take(ctx, "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
if r["id"] == catalogue.GuardID() {
|
||||
if r["content"] != catalogue.AsGuard([]int{5432}) {
|
||||
t.Fatalf("the taken store's guard is:\n%s", r["content"])
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("a taken store is not guarded")
|
||||
}
|
||||
|
||||
// novox/hq ADR 0038 and 0100: a module may publish a port the long way — `2222:22`, because the
|
||||
// machine's own ssh daemon holds 22 — and say it listens on the machine side of that mapping,
|
||||
// which is the number anything reaching it dials. A node moves that port by naming it, and the
|
||||
// number has to reach everything derived from it at once: what the runtime is handed, what an
|
||||
// adopted node is told to open, what its guard refuses, and what a consumer elsewhere dials.
|
||||
//
|
||||
// It reached none of them. The setting was refused outright for naming the machine side — so a
|
||||
// module's port could not be put back where the machine it replaces had it, and, worse, the
|
||||
// node's every push failed for as long as the setting existed.
|
||||
func TestTheMachineSideOfAMappingIsMovedEverywhereTheNumberIsUsed(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "forge", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "git-over-ssh", Scope: catalogue.ScopeMesh}},
|
||||
Serves: map[string]map[string]any{"git-over-ssh": {"port": 2222}},
|
||||
Listens: []catalogue.Listening{{Port: 2222, From: catalogue.FromMesh}},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "forge",
|
||||
"ports": []any{"2222:22"},
|
||||
"image": "registry.example/forge@sha256:" + strings.Repeat("c", 64)}}})
|
||||
register(t, open, catalogue.Manifest{Module: "app", Version: "1",
|
||||
Requires: []string{"git-over-ssh"}})
|
||||
if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "anchor", "forge"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.Take(ctx, "anchor", "forge"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "forge",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"2222": 222}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
resources := composed(t, open, "anchor").Resources
|
||||
var container, opening map[string]any
|
||||
for _, r := range resources {
|
||||
switch r["id"] {
|
||||
case "forge.server":
|
||||
container = r
|
||||
case catalogue.OpeningID("tcp", 222, catalogue.PathForwarded):
|
||||
opening = r
|
||||
}
|
||||
if id, _ := r["id"].(string); strings.HasPrefix(id, "adoption.opening-tcp-2222-") {
|
||||
t.Errorf("the adopted node is told to open the port the forge was moved off: %s", id)
|
||||
}
|
||||
}
|
||||
if container == nil || !reflect.DeepEqual(container["ports"], []any{"222:22"}) {
|
||||
t.Fatalf("the forge's container publishes %v", container["ports"])
|
||||
}
|
||||
if opening == nil || opening["to"] != 22 || opening["from"] != catalogue.OpeningFromMesh {
|
||||
t.Fatalf("no opening for the port this node gave the forge: %v", opening)
|
||||
}
|
||||
var guard map[string]any
|
||||
for _, r := range resources {
|
||||
if r["id"] == catalogue.GuardID() {
|
||||
guard = r
|
||||
}
|
||||
}
|
||||
if guard == nil || guard["content"] != catalogue.AsGuard([]int{222}) {
|
||||
t.Fatalf("the guard does not refuse the port the forge is on:\n%v", guard["content"])
|
||||
}
|
||||
|
||||
// And the consumer on the other machine dials the same number.
|
||||
plan, _, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var told any
|
||||
for _, n := range plan.Needs {
|
||||
if n.Name == "git-over-ssh" {
|
||||
told = n.Serves["port"]
|
||||
}
|
||||
}
|
||||
if told != 222 {
|
||||
t.Fatalf("the consumer is told the forge answers on %v", told)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnEmptyDeclarationSaysOwnsNothing(t *testing.T) {
|
||||
// The host refuses an empty body unless told the emptiness is meant (novox/hq issue 127).
|
||||
body, err := sendable{}.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var env map[string]any
|
||||
if err := json.Unmarshal(body, &env); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if env["owns_nothing"] != true {
|
||||
t.Fatalf("an empty declaration must mark owns_nothing; got %v", env)
|
||||
}
|
||||
// A declaration with resources does not carry the marker.
|
||||
body, _ = sendable{Resources: []map[string]any{{"id": "x"}}}.Body()
|
||||
var env2 map[string]any
|
||||
_ = json.Unmarshal(body, &env2)
|
||||
if _, present := env2["owns_nothing"]; present {
|
||||
t.Fatalf("a non-empty declaration must not mark owns_nothing; got %v", env)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// where a build's repository is (novox/hq ADR 0111).
|
||||
//
|
||||
// A repository is on the mesh's own forge, or it is anywhere else. The first is recorded as its path
|
||||
// on the forge holding the git seat, and cloned from wherever that forge runs at the moment of
|
||||
// building; the second is a URL, recorded and cloned exactly as given. The build machine is not told
|
||||
// the difference — it is handed a URL either way — because only the control plane knows where the
|
||||
// seat's holder runs.
|
||||
|
||||
// gitSeat is the seat a self-hosted repository lives on.
|
||||
const gitSeat = "git"
|
||||
|
||||
// buildSource is where a build's repository is: a URL, or a path on a seat's holder.
|
||||
type buildSource struct {
|
||||
Repository string
|
||||
Seat string
|
||||
}
|
||||
|
||||
// String is the source as a person reads it, which for one on a seat is not the URL: the URL is a
|
||||
// fact about where the forge happens to run today.
|
||||
func (s buildSource) String() string {
|
||||
if s.Seat == "" {
|
||||
return s.Repository
|
||||
}
|
||||
return fmt.Sprintf("%s on the %s seat", s.Repository, s.Seat)
|
||||
}
|
||||
|
||||
// onASeat refuses an address given as a path on the forge.
|
||||
//
|
||||
// **A URL here would be recorded as a path**, and then composed onto the forge's address as one —
|
||||
// cloning `http://forge:3000/https://github.com/…`. Refused by what an address plainly looks like,
|
||||
// not repaired: `--self` promises a path, and something that is not one is a mistake to name.
|
||||
func onASeat(repository string) error {
|
||||
if strings.Contains(repository, ":") || strings.HasPrefix(repository, "/") ||
|
||||
strings.Trim(repository, "/") == "" {
|
||||
return fmt.Errorf("--self takes the repository's path on the forge, such as novox/mesh-catalog, "+
|
||||
"and %q is not one — without --self it is built from exactly what is given", repository)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// cloneFrom is the URL a build machine clones for a source.
|
||||
//
|
||||
// A URL is itself. A path on a seat is composed from the seat's holder as the mesh sees it now —
|
||||
// the same view planning takes of every machine, so the forge a build clones from is the forge the
|
||||
// mesh says holds the seat.
|
||||
func cloneFrom(ctx context.Context, source buildSource) (string, error) {
|
||||
if source.Seat == "" {
|
||||
return source.Repository, nil
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer open.Close()
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
world, err := theRestOfTheMesh(ctx, open.inventory, shelf, "")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return clonedFromSeat(world, source.Seat, source.Repository)
|
||||
}
|
||||
|
||||
// clonedFromSeat composes the clone URL for a repository on a seat's holder.
|
||||
//
|
||||
// **Refused, never defaulted, at every step that has no answer.** Nobody holding the seat is a mesh
|
||||
// without a forge of its own: it builds from external repositories and must say so rather than fail
|
||||
// to clone. A holder off the private network cannot be reached by any build machine. A holder that
|
||||
// serves no scheme or port has nothing to compose from — a default port here would be the forge's
|
||||
// address guessed, which is the thing this exists to stop.
|
||||
func clonedFromSeat(world catalogue.World, seatName, repository string) (string, error) {
|
||||
seat, known := catalogue.SeatNamed(seatName)
|
||||
if !known || seat.Delivers == "" {
|
||||
return "", fmt.Errorf("%q is not a seat a repository can live on", seatName)
|
||||
}
|
||||
var holder *catalogue.Held
|
||||
for i, h := range world.Held {
|
||||
if h.Claim == seat.Name && h.Scope == seat.Scope {
|
||||
holder = &world.Held[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if holder == nil {
|
||||
return "", fmt.Errorf("nobody holds the %s seat, so %s cannot be cloned from this mesh's "+
|
||||
"forge — assign a module that claims it, or build from the repository's URL without --self",
|
||||
seat.Name, repository)
|
||||
}
|
||||
var provider *catalogue.Provider
|
||||
for i, p := range world.Offered[seat.Delivers] {
|
||||
if p.Node == holder.Node && p.Module == holder.Module {
|
||||
provider = &world.Offered[seat.Delivers][i]
|
||||
}
|
||||
}
|
||||
if provider == nil {
|
||||
return "", fmt.Errorf("%s on %s holds the %s seat and offers no %q to clone from",
|
||||
holder.Module, holder.Node, seat.Name, seat.Delivers)
|
||||
}
|
||||
if provider.At == "" {
|
||||
return "", fmt.Errorf("%s on %s holds the %s seat and is not on the private network, so no "+
|
||||
"build machine can reach it", holder.Module, holder.Node, seat.Name)
|
||||
}
|
||||
scheme, _ := provider.Serves["scheme"].(string)
|
||||
port := servedPort(provider.Serves["port"])
|
||||
if scheme == "" || port == "" {
|
||||
return "", fmt.Errorf("%s on %s holds the %s seat and does not serve a scheme and a port for %q",
|
||||
holder.Module, holder.Node, seat.Name, seat.Delivers)
|
||||
}
|
||||
path := strings.TrimSuffix(strings.Trim(repository, "/"), ".git")
|
||||
return fmt.Sprintf("%s://%s:%s/%s.git", scheme, provider.At, port, path), nil
|
||||
}
|
||||
|
||||
// servedPort is a served port as text, however the manifest and the node's settings carried it.
|
||||
func servedPort(v any) string {
|
||||
switch p := v.(type) {
|
||||
case float64:
|
||||
return strconv.Itoa(int(p))
|
||||
case int:
|
||||
return strconv.Itoa(p)
|
||||
case string:
|
||||
return p
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// Defends novox/hq ADR 0111: a build source is on the git seat, or it is external.
|
||||
|
||||
func forgeHolding(port any) catalogue.World {
|
||||
return catalogue.World{
|
||||
Held: []catalogue.Held{{Claim: "git", Scope: catalogue.ScopeMesh, Node: "anchor", Module: "gitea"}},
|
||||
Offered: map[string][]catalogue.Provider{"git": {
|
||||
// A second forge that does not hold the seat, so taking the first one found would be wrong.
|
||||
{Node: "archive", At: "archive.internal", Module: "gitea-mirror",
|
||||
Serves: map[string]any{"scheme": "http", "port": float64(3000)}},
|
||||
{Node: "anchor", At: "anchor.internal", Module: "gitea",
|
||||
Serves: map[string]any{"scheme": "http", "port": port}},
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
func TestARepositoryOnTheSeatIsClonedFromItsHolder(t *testing.T) {
|
||||
got, err := clonedFromSeat(forgeHolding(float64(3000)), "git", "novox/mesh-catalog")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != "http://anchor.internal:3000/novox/mesh-catalog.git" {
|
||||
t.Fatalf("cloned from %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMovedForgeIsFollowedWithoutRewritingAnything(t *testing.T) {
|
||||
// The whole point: the node gave the forge another port, and the same recorded path clones
|
||||
// from the new one. Nothing recorded contained the old one to be wrong.
|
||||
got, err := clonedFromSeat(forgeHolding(float64(3100)), "git", "novox/mesh-catalog")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(got, ":3100/") {
|
||||
t.Fatalf("the moved port was not followed: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithNobodyHoldingTheSeatASelfHostedBuildIsRefusedAndSaysWhy(t *testing.T) {
|
||||
_, err := clonedFromSeat(catalogue.World{}, "git", "novox/mesh-catalog")
|
||||
if err == nil {
|
||||
t.Fatal("a repository was cloned from a forge the mesh does not have")
|
||||
}
|
||||
for _, want := range []string{"nobody holds the git seat", "without --self"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("the refusal does not say %q: %v", want, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnExternalRepositoryIsClonedExactlyAsGiven(t *testing.T) {
|
||||
// Unaffected by the seat, held or not: GitHub and GitLab are the ordinary cases.
|
||||
given := "https://github.com/someone/something.git"
|
||||
got, err := cloneFrom(t.Context(), buildSource{Repository: given})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != given {
|
||||
t.Fatalf("an external repository became %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAHolderOffThePrivateNetworkIsRefused(t *testing.T) {
|
||||
world := forgeHolding(float64(3000))
|
||||
world.Offered["git"][1].At = ""
|
||||
if _, err := clonedFromSeat(world, "git", "novox/mesh-catalog"); err == nil ||
|
||||
!strings.Contains(err.Error(), "private network") {
|
||||
t.Fatalf("a forge nothing can reach was cloned from: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAHolderServingNoPortIsRefusedRatherThanGuessed(t *testing.T) {
|
||||
// A default port would be the forge's address guessed, which is what this exists to stop.
|
||||
if _, err := clonedFromSeat(forgeHolding(nil), "git", "novox/mesh-catalog"); err == nil {
|
||||
t.Fatal("a port was guessed for a forge that serves none")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAddressGivenAsAPathOnTheForgeIsRefused(t *testing.T) {
|
||||
for _, bad := range []string{
|
||||
"https://github.com/someone/something.git",
|
||||
"git@anchor:novox/mesh-catalog.git",
|
||||
"/srv/git/mesh-catalog",
|
||||
"",
|
||||
} {
|
||||
if err := onASeat(bad); err == nil {
|
||||
t.Errorf("--self accepted %q as a path on the forge", bad)
|
||||
}
|
||||
}
|
||||
if err := onASeat("novox/mesh-catalog"); err != nil {
|
||||
t.Errorf("a path on the forge was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASourceOnTheSeatReadsAsAPathNotAnAddress(t *testing.T) {
|
||||
s := buildSource{Repository: "novox/mesh-catalog", Seat: "git"}
|
||||
if got := s.String(); got != "novox/mesh-catalog on the git seat" {
|
||||
t.Fatalf("read as %q", got)
|
||||
}
|
||||
}
|
||||
@@ -171,6 +171,13 @@ func statusCommand(ctx context.Context, args []string) error {
|
||||
fmt.Printf("\n `push --behind` sends them\n\n")
|
||||
}
|
||||
|
||||
if adopted := adoptedNodes(nodes); len(adopted) > 0 {
|
||||
// Said, because nothing forces the flip: a node left adopted is visible here rather than
|
||||
// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
|
||||
fmt.Printf("%d machine(s) adopted: %s\n", len(adopted), strings.Join(adopted, ", "))
|
||||
fmt.Printf("\n `converge <node>` previews the flip\n\n")
|
||||
}
|
||||
|
||||
if len(wrong) == 0 && len(quiet) == 0 && len(behind) == 0 && len(asked.waiting) == 0 &&
|
||||
len(asked.refused) == 0 && asked.network == "" {
|
||||
// Said plainly. "Nothing to report" and "nothing was checked" must never look the same,
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/identity"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/licences"
|
||||
@@ -72,6 +73,14 @@ func migrate(ctx context.Context) error {
|
||||
}
|
||||
fmt.Printf("provided %s\n", m.Module)
|
||||
}
|
||||
// The seats the mesh ships with, into the table that now holds the set (novox/hq ADR 0122).
|
||||
// Idempotent: fills an empty table on first boot, adds a seat a release ships, and leaves an
|
||||
// operator's changes in the table as they are.
|
||||
added, err := inv.SeedSeats(ctx, catalogue.DefaultSeats())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("seeded %d seat(s)\n", added)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -85,6 +94,18 @@ func openInventory(ctx context.Context) (*inventory.Inventory, error) {
|
||||
inv.Close()
|
||||
return nil, err
|
||||
}
|
||||
// Load the seat set from the store, so the control plane reads the set as data rather than as
|
||||
// the slice it was compiled with (novox/hq ADR 0122). A store not yet seeded — or one whose
|
||||
// seat table a migration has not reached — returns nothing, and UseSeats leaves the compiled
|
||||
// defaults in force: the set is never emptied by a read that found nothing, which would refuse
|
||||
// every claim. So this can only ever replace the defaults with what the mesh actually holds.
|
||||
if seats, err := inv.Seats(ctx); err == nil {
|
||||
catalogue.UseSeats(seats)
|
||||
}
|
||||
// And the former names, so a reference to a seat's old name resolves after a rename (ADR 0122).
|
||||
if aliases, err := inv.Aliases(ctx); err == nil {
|
||||
catalogue.UseAliases(aliases)
|
||||
}
|
||||
return inv, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -6,7 +6,9 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
@@ -28,13 +30,15 @@ type following struct{ open *stores }
|
||||
func (f following) Upgraded(ctx context.Context, u link.Upgraded) error {
|
||||
inv := f.open.inventory
|
||||
|
||||
// The store read first, and an outage there said as one, so the announcement is held and asked
|
||||
// again (novox/hq issue 083). Only here: a push that fails further down is not asked again.
|
||||
decision, err := inv.UpgradeOf(ctx, u.Module)
|
||||
if err != nil {
|
||||
return err
|
||||
return notNow(err)
|
||||
}
|
||||
on, err := inv.Running(ctx, u.Module)
|
||||
if err != nil {
|
||||
return err
|
||||
return notNow(err)
|
||||
}
|
||||
if len(on) == 0 {
|
||||
fmt.Printf("%s moved to %s; no machine runs it\n", u.Module, shortCommit(u.Commit))
|
||||
@@ -171,11 +175,21 @@ func sayUpgrade(module string, u inventory.Upgrade) string {
|
||||
// catalogue misses nothing — but the modules built before it first ran were announced to a queue
|
||||
// that did not exist, and on a fresh mesh those are always the same three: the shared base, the
|
||||
// store the catalogue runs on, and the catalogue itself.
|
||||
//
|
||||
// **Announced as fetchable, recorded as what it is** (novox/hq 04-ISSUES/102). A build is
|
||||
// recorded by digest and path; the catalogue hears the builder's own announcements, which name
|
||||
// the store's address, so a replay composes the address back in — the store's address as the
|
||||
// network reaches it NOW, which is the whole point of not having recorded the old one. With no
|
||||
// store on the network yet, the recorded form goes as it is.
|
||||
func (f following) Announceable(ctx context.Context) ([]link.Announcement, error) {
|
||||
builds, err := f.open.inventory.Announceable(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
address, err := whereTheStoreIs(ctx, f.open.inventory, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]link.Announcement, 0, len(builds))
|
||||
for _, b := range builds {
|
||||
a := link.Announcement{
|
||||
@@ -184,8 +198,11 @@ func (f following) Announceable(ctx context.Context) ([]link.Announcement, error
|
||||
}
|
||||
if len(b.Manifest) > 0 {
|
||||
a.Manifest = b.Manifest
|
||||
if address != "" {
|
||||
a.Manifest = routedManifest(b.Manifest, b.Made, address)
|
||||
}
|
||||
}
|
||||
for _, made := range b.Made {
|
||||
for _, made := range routedArtifacts(b.Made, address) {
|
||||
a.Made = append(a.Made, link.MadeArtifact{
|
||||
Name: made.Name, Kind: made.Kind, Reference: made.Reference,
|
||||
})
|
||||
@@ -194,3 +211,154 @@ func (f following) Announceable(ctx context.Context) ([]link.Announcement, error
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// notNow marks a store that could not be read right now, so the announcement is held rather than
|
||||
// lost; anything else is returned as it was.
|
||||
func notNow(err error) error {
|
||||
if inventory.Unreachable(err) {
|
||||
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// SourceMoved is the forge announcing a merge: every module recorded as built from that
|
||||
// repository and branch is marked as moved to the merge commit, and built — bases first, so a
|
||||
// module that stands on another's artifact is built after it and not against the old one
|
||||
// (novox/hq 04-ISSUES/131). Nothing is pushed here: what a finished build does to the machines
|
||||
// running the module is the upgrade's decision, taken when the catalogue announces it.
|
||||
func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
||||
inv := f.open.inventory
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
var moved []inventory.Entry
|
||||
for _, e := range entries {
|
||||
if !sourceIs(e.Source, m) {
|
||||
continue
|
||||
}
|
||||
if e.Source.BuiltFrom == m.Commit {
|
||||
continue
|
||||
}
|
||||
if err := inv.SourceMoved(ctx, e.Manifest.Module, m.Commit); err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
moved = append(moved, e)
|
||||
}
|
||||
if len(moved) == 0 {
|
||||
fmt.Printf("%s/%s merged into %s (%.8s); nothing the mesh holds is built from it\n",
|
||||
m.Owner, m.Repo, m.Base, m.Commit)
|
||||
return nil
|
||||
}
|
||||
against, err := inv.BuiltAgainst(ctx)
|
||||
if err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
ordered := orderByBases(moved, against)
|
||||
names := make([]string, 0, len(ordered))
|
||||
for _, e := range ordered {
|
||||
names = append(names, e.Manifest.Module)
|
||||
}
|
||||
fmt.Printf("%s/%s merged into %s (%.8s); building %s\n",
|
||||
m.Owner, m.Repo, m.Base, m.Commit, strings.Join(names, ", "))
|
||||
var failed []string
|
||||
for _, e := range ordered {
|
||||
source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
|
||||
if err := buildOne(ctx, source, e.Source.Path, e.Source.Ref, 20*time.Minute); err != nil {
|
||||
fmt.Printf(" %s: %v\n", e.Manifest.Module, err)
|
||||
failed = append(failed, e.Manifest.Module)
|
||||
// A base that failed is a reason to stop: what stands on it would be built against
|
||||
// the old one, and report success (novox/hq 04-ISSUES/131).
|
||||
if standsOn(ordered, e.Manifest.Module, against) {
|
||||
fmt.Printf(" stopping: %s is a base of what was still to build\n", e.Manifest.Module)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
fmt.Printf("%d of %d not built: %s\n", len(failed), len(ordered), strings.Join(failed, ", "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// sourceIs is whether a recorded source is the repository and branch a merge announced. A source on
|
||||
// the git seat is recorded as its path on the forge; one elsewhere as the URL it was cloned from.
|
||||
// An empty recorded ref is the repository's default branch, which is what a merge into the base
|
||||
// branch of the forge's default means.
|
||||
func sourceIs(s inventory.Source, m link.SourceMoved) bool {
|
||||
want := strings.ToLower(m.Owner + "/" + m.Repo)
|
||||
repo := strings.ToLower(strings.TrimSuffix(s.Repository, ".git"))
|
||||
matches := repo == want || strings.HasSuffix(repo, "/"+want) ||
|
||||
(m.CloneURL != "" && strings.EqualFold(strings.TrimSuffix(s.Repository, ".git"), strings.TrimSuffix(m.CloneURL, ".git")))
|
||||
if !matches {
|
||||
return false
|
||||
}
|
||||
return s.Ref == "" || s.Ref == m.Base
|
||||
}
|
||||
|
||||
// orderByBases is the entries with every base before what stands on it: a module whose build stood
|
||||
// on another's artifact comes after that module. Entries outside the set are not waited for — they
|
||||
// are not being rebuilt. Stable for what has no order between it.
|
||||
//
|
||||
// `against` is what each module's newest build stood on (inventory.BuiltAgainst): the edges are
|
||||
// derived from builds, not declared, because a recorded manifest no longer carries `build.on`.
|
||||
func orderByBases(entries []inventory.Entry, against map[string][]string) []inventory.Entry {
|
||||
inSet := map[string]bool{}
|
||||
for _, e := range entries {
|
||||
inSet[e.Manifest.Module] = true
|
||||
}
|
||||
var out []inventory.Entry
|
||||
placed := map[string]bool{}
|
||||
var place func(e inventory.Entry, seen map[string]bool)
|
||||
place = func(e inventory.Entry, seen map[string]bool) {
|
||||
name := e.Manifest.Module
|
||||
if placed[name] || seen[name] {
|
||||
return
|
||||
}
|
||||
seen[name] = true
|
||||
for _, base := range entries {
|
||||
if base.Manifest.Module != name && inSet[base.Manifest.Module] && standsOnModule(e, base.Manifest.Module, against) {
|
||||
place(base, seen)
|
||||
}
|
||||
}
|
||||
placed[name] = true
|
||||
out = append(out, e)
|
||||
}
|
||||
for _, e := range entries {
|
||||
place(e, map[string]bool{})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// standsOn is whether anything in the set is built on the named module's artifacts.
|
||||
func standsOn(entries []inventory.Entry, module string, against map[string][]string) bool {
|
||||
for _, e := range entries {
|
||||
if standsOnModule(e, module, against) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// standsOnModule is whether an entry's build stood on the named module: by what its newest build
|
||||
// recorded it was handed (`artifact-store://<module>/<artifact>@…`, the module's own artifact), or
|
||||
// — for a module registered from a manifest and not yet built — by the base its manifest names.
|
||||
func standsOnModule(e inventory.Entry, module string, against map[string][]string) bool {
|
||||
if e.Manifest.Module == module {
|
||||
return false
|
||||
}
|
||||
if e.Manifest.Build != nil {
|
||||
for _, on := range e.Manifest.Build.On {
|
||||
if on.Module == module {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
prefix := catalogue.ArtifactStoreScheme + module + "/"
|
||||
for _, ref := range against[e.Manifest.Module] {
|
||||
if strings.HasPrefix(ref, prefix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
package main
|
||||
|
||||
// The challenge path falls through for real. autocert's own HTTPHandler answers 404 itself for a
|
||||
// token it does not hold and never consults its fallback on the challenge path — the
|
||||
// predecessor's fault, the edge owning /.well-known/acme-challenge outright, rediscovered live
|
||||
// when Mailu's renewal died behind this proxy on cutover day (2026-09-26). These tests pin the
|
||||
// three behaviours tokenOrRoute exists for.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"golang.org/x/crypto/acme/autocert"
|
||||
)
|
||||
|
||||
func routedTo(t *testing.T, marker string) http.Handler {
|
||||
t.Helper()
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if _, err := w.Write([]byte(marker)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestATokenNoAuthorityHoldsIsRoutedNot404d(t *testing.T) {
|
||||
m := &autocert.Manager{Prompt: autocert.AcceptTOS, Cache: autocert.DirCache(t.TempDir())}
|
||||
h := tokenOrRoute(routedTo(t, "the workload answered"), m)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, httptest.NewRequest("GET", "http://mail.example/.well-known/acme-challenge/somebody-elses-token", nil))
|
||||
|
||||
if rec.Code != http.StatusOK || rec.Body.String() != "the workload answered" {
|
||||
t.Fatalf("a token no authority holds must reach plain routing; got %d %q", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestATokenAManagerHoldsIsAnsweredByIt(t *testing.T) {
|
||||
// autocert reads a token it does not have in memory from its cache, under "<token>+http-01" —
|
||||
// which is also how a token would survive the manager restarting mid-issuance.
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "held-token+http-01"), []byte("the-key-authorization"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := &autocert.Manager{Prompt: autocert.AcceptTOS, Cache: autocert.DirCache(dir)}
|
||||
h := tokenOrRoute(routedTo(t, "must not be reached"), m)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, httptest.NewRequest("GET", "http://mail.example/.well-known/acme-challenge/held-token", nil))
|
||||
|
||||
if rec.Code != http.StatusOK || rec.Body.String() != "the-key-authorization" {
|
||||
t.Fatalf("the manager holding a token answers it; got %d %q", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestASecondAuthorityIsProbedBeforeRouting(t *testing.T) {
|
||||
first := &autocert.Manager{Prompt: autocert.AcceptTOS, Cache: autocert.DirCache(t.TempDir())}
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "internal-token+http-01"), []byte("internal-key"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second := &autocert.Manager{Prompt: autocert.AcceptTOS, Cache: autocert.DirCache(dir)}
|
||||
h := tokenOrRoute(routedTo(t, "must not be reached"), first, second)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, httptest.NewRequest("GET", "http://git.internal/.well-known/acme-challenge/internal-token", nil))
|
||||
|
||||
if rec.Code != http.StatusOK || rec.Body.String() != "internal-key" {
|
||||
t.Fatalf("the second authority's token is found by probing past the first; got %d %q", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAuthorityWhosePolicyRefusesTheNameIsProbedPast(t *testing.T) {
|
||||
// autocert checks the host policy before the token and answers 403 — the internal authority
|
||||
// does this for every public name. A policy refusal is as much "not mine" as a missing token:
|
||||
// the request must still reach plain routing, where the workload's own ACME client answers.
|
||||
refusing := &autocert.Manager{
|
||||
Prompt: autocert.AcceptTOS,
|
||||
Cache: autocert.DirCache(t.TempDir()),
|
||||
HostPolicy: func(ctx context.Context, host string) error {
|
||||
return fmt.Errorf("no internal-only route for %q in this mesh", host)
|
||||
},
|
||||
}
|
||||
h := tokenOrRoute(routedTo(t, "the workload answered"), refusing)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, httptest.NewRequest("GET", "http://mail.example/.well-known/acme-challenge/mailus-token", nil))
|
||||
|
||||
if rec.Code != http.StatusOK || rec.Body.String() != "the workload answered" {
|
||||
t.Fatalf("a policy refusal must fall through to routing; got %d %q", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOrdinaryPathNeverTouchesTheChallengeMachinery(t *testing.T) {
|
||||
m := &autocert.Manager{Prompt: autocert.AcceptTOS, Cache: autocert.DirCache(t.TempDir())}
|
||||
h := tokenOrRoute(routedTo(t, "routed"), m)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, httptest.NewRequest("GET", "http://site.example/index.html", nil))
|
||||
|
||||
if rec.Code != http.StatusOK || rec.Body.String() != "routed" {
|
||||
t.Fatalf("an ordinary path goes straight to routing; got %d %q", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
+624
-73
@@ -10,10 +10,31 @@
|
||||
// program. What lives here is that contract, written as something that runs so it can be read
|
||||
// rather than described.
|
||||
//
|
||||
// **A route also carries what a request arriving at it may do** (novox/hq ADR 0108). The grant used
|
||||
// to say only where to send traffic, so this proxy applied nothing; the four things the ingress it
|
||||
// replaces actually relies on are now part of the contribution. The set is closed at four, because
|
||||
// an open middleware surface recreates the thing being replaced and is far harder to narrow later
|
||||
// than a closed one is to widen.
|
||||
//
|
||||
// What it is given, written by the host from an ordinary declaration:
|
||||
//
|
||||
// $ROUTES every consumer, the name it asked for, and where the mesh says that machine is
|
||||
//
|
||||
// Each contribution's values carry the name and port as before, and optionally:
|
||||
//
|
||||
// path the path prefix this rule is scoped to; absent means every path
|
||||
// priority which rule wins where two match; higher first, and the order is total
|
||||
// deny refuse the request outright — the shape an incident mitigation needs
|
||||
// redirect answer with a permanent redirect to this name, keeping the path and query
|
||||
// auth the *path of a secret* holding `user:hash` lines, never the credential itself
|
||||
//
|
||||
// A host may appear more than once, which is what path scoping means: one rule refusing a path
|
||||
// while another serves everything else on the same name.
|
||||
//
|
||||
// **`auth` names a secret and never holds one.** A declaration carrying a credential is refused
|
||||
// outright rather than served unprotected, and a secret that cannot be read makes the route refuse
|
||||
// rather than open — a gate that cannot check is not a gate that opens.
|
||||
//
|
||||
// It re-reads on change rather than being restarted, for the same reason the provisioner does:
|
||||
// a route arriving or leaving is an ordinary event and must not drop the connections of every
|
||||
// other workload.
|
||||
@@ -23,6 +44,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/subtle"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
@@ -42,6 +64,7 @@ import (
|
||||
|
||||
"golang.org/x/crypto/acme"
|
||||
"golang.org/x/crypto/acme/autocert"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
)
|
||||
|
||||
// Where public certificates come from when nothing says otherwise.
|
||||
@@ -74,10 +97,23 @@ func issuer() string {
|
||||
// to what it may serve.
|
||||
func onlyWhatTheMeshSaid(held *table) autocert.HostPolicy {
|
||||
return func(_ context.Context, host string) error {
|
||||
if _, known := held.find(host); known {
|
||||
if held.eligibleForACME(host) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("no route for %q in this mesh, so no certificate is asked for", host)
|
||||
return fmt.Errorf("no public route for %q in this mesh, so no certificate is asked for", host)
|
||||
}
|
||||
}
|
||||
|
||||
// onlyInternalNamesTheMeshSaid is onlyWhatTheMeshSaid's mirror for the internal authority — the
|
||||
// same quota-spending concern applies even to an authority with no rate limit of its own, because
|
||||
// an order for a name this proxy does not actually route is a bug worth refusing rather than
|
||||
// serving.
|
||||
func onlyInternalNamesTheMeshSaid(held *table) autocert.HostPolicy {
|
||||
return func(_ context.Context, host string) error {
|
||||
if held.eligibleForInternalACME(host) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("no internal-only route for %q in this mesh, so no certificate is asked for", host)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -95,48 +131,194 @@ type contribution struct {
|
||||
Values map[string]any `json:"values"`
|
||||
}
|
||||
|
||||
// policy is what a rule does with a request that matched it.
|
||||
//
|
||||
// **Decided by the mesh, not here** (novox/hq ADR 0108). A route grant used to hand back a name and
|
||||
// say nothing about what the name admitted, so this proxy admitted everything. The set is closed at
|
||||
// four — authentication, refusal, path scoping, redirect — because an open middleware surface
|
||||
// recreates the thing being replaced and is far harder to narrow later than a closed one is to widen.
|
||||
type policy struct {
|
||||
// deny refuses the request outright, whatever it is.
|
||||
deny bool
|
||||
// redirectTo answers with a permanent redirect instead of proxying. The request's own path and
|
||||
// query are carried across, which is what canonicalising one public name onto another means.
|
||||
redirectTo string
|
||||
// users is what a request must present, read at load time from the secret the declaration
|
||||
// *named*. A declaration never carries the credential itself.
|
||||
users map[string]string
|
||||
// sealed is set when authentication was declared and the secret could not be read. The rule then
|
||||
// refuses everything and says why.
|
||||
//
|
||||
// **Fail closed.** The alternative — serve the route unauthenticated because the gate is
|
||||
// missing — turns an unreadable file into a silently public admin surface, which is the exact
|
||||
// outcome ADR 0108 exists to prevent. A gate that cannot check is not a gate that opens.
|
||||
sealed string
|
||||
}
|
||||
|
||||
// rule is one way a host may be routed. A host may have several, which is what path scoping means.
|
||||
type rule struct {
|
||||
path string // "" matches every path
|
||||
priority int
|
||||
policy policy
|
||||
to *httputil.ReverseProxy
|
||||
target string
|
||||
// insecure skips certificate verification when target is reached over https. For a backend
|
||||
// that terminates TLS with its own certificate this proxy has no reason to trust — Mailu's
|
||||
// webmail front is the first of these — never for anything reached over plain http, where
|
||||
// there is nothing to verify in the first place.
|
||||
insecure bool
|
||||
// maxRequestBody is the largest body, in bytes, this route carries. Zero is no limit, which is
|
||||
// what every route gets by saying nothing: this proxy has never limited a body, and a default
|
||||
// arriving with the field would change every route that never asked for one.
|
||||
//
|
||||
// **Configuration, not a policy** (novox/hq ADR 0108 closed that set at four). It belongs beside
|
||||
// `insecure` for the same reason `insecure` is not a policy: both tune how this proxy carries a
|
||||
// request to a backend, rather than deciding what the name admits or who may reach it. A
|
||||
// registry is the case that needs it — image layers arrive as single requests of gigabytes, and
|
||||
// a proxy's own default refuses them long before the workload is reached.
|
||||
maxRequestBody int64
|
||||
}
|
||||
|
||||
// table is what the proxy is currently serving, replaced whole whenever the file changes.
|
||||
//
|
||||
// Replaced rather than merged: the file is the whole truth about who has a route, so merging
|
||||
// would keep serving a name whose module was unassigned — which is the stale-route fault
|
||||
// 08-connectivity lists as open, reintroduced one level down.
|
||||
//
|
||||
// Keyed by host to an *ordered* list rather than to one target, because two of the four policies
|
||||
// need a single host routed more than one way: a refusal on a path the ordinary route also matches,
|
||||
// and a certificate-challenge path on a host that otherwise serves a workload.
|
||||
type table struct {
|
||||
mu sync.RWMutex
|
||||
to map[string]*httputil.ReverseProxy
|
||||
targets map[string]string
|
||||
mu sync.RWMutex
|
||||
to map[string][]rule
|
||||
// public is which routed hosts are eligible for a real certificate — every host reached as a
|
||||
// route's own `name`, never one reached only as its `internal-name`. A private alias can never
|
||||
// pass ACME's own validation (it has no public DNS to prove it against), so asking for it is
|
||||
// not merely pointless but the failing order onlyWhatTheMeshSaid exists to prevent.
|
||||
public map[string]bool
|
||||
}
|
||||
|
||||
func (t *table) set(routes map[string]string) {
|
||||
made := map[string]*httputil.ReverseProxy{}
|
||||
for name, target := range routes {
|
||||
where, err := url.Parse(target)
|
||||
if err != nil {
|
||||
log.Printf("route %s points at %q, which is not a URL: %v", name, target, err)
|
||||
func (t *table) set(routes map[string][]rule, public map[string]bool) {
|
||||
made := map[string][]rule{}
|
||||
for host, rules := range routes {
|
||||
kept := make([]rule, 0, len(rules))
|
||||
for _, r := range rules {
|
||||
// A rule that only refuses or only redirects has nowhere to send anything, and needs
|
||||
// nowhere: it answers by itself.
|
||||
if r.policy.deny || r.policy.redirectTo != "" {
|
||||
kept = append(kept, r)
|
||||
continue
|
||||
}
|
||||
where, err := url.Parse(r.target)
|
||||
if err != nil {
|
||||
log.Printf("route %s points at %q, which is not a URL: %v", host, r.target, err)
|
||||
continue
|
||||
}
|
||||
r.to = httputil.NewSingleHostReverseProxy(where)
|
||||
if r.insecure {
|
||||
r.to.Transport = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
|
||||
}
|
||||
kept = append(kept, r)
|
||||
}
|
||||
if len(kept) == 0 {
|
||||
continue
|
||||
}
|
||||
made[name] = httputil.NewSingleHostReverseProxy(where)
|
||||
inOrder(kept)
|
||||
made[host] = kept
|
||||
}
|
||||
t.mu.Lock()
|
||||
t.to, t.targets = made, routes
|
||||
t.to = made
|
||||
t.public = public
|
||||
t.mu.Unlock()
|
||||
}
|
||||
|
||||
func (t *table) find(host string) (*httputil.ReverseProxy, bool) {
|
||||
// The port is not part of the name. A request to app.example:8080 is for app.example.
|
||||
// inOrder puts the rules for one host into the order they are matched in, and does so totally.
|
||||
//
|
||||
// **Equal priorities must resolve identically every time** (ADR 0108). Sorting only by priority
|
||||
// leaves rules that share one in whatever order the map produced, so the same declaration would
|
||||
// serve differently between restarts — a proxy that is not reproducible. Longest path first within a
|
||||
// priority is also the intuitive reading: the more specific rule wins. The last two keys exist only
|
||||
// to make the order total.
|
||||
func inOrder(rules []rule) {
|
||||
sort.SliceStable(rules, func(i, j int) bool {
|
||||
a, b := rules[i], rules[j]
|
||||
if a.priority != b.priority {
|
||||
return a.priority > b.priority
|
||||
}
|
||||
if len(a.path) != len(b.path) {
|
||||
return len(a.path) > len(b.path)
|
||||
}
|
||||
if a.path != b.path {
|
||||
return a.path < b.path
|
||||
}
|
||||
return a.target < b.target
|
||||
})
|
||||
}
|
||||
|
||||
// find is the rule that answers this request, or nothing if the host is not routed here at all.
|
||||
func (t *table) find(host, path string) (rule, bool) {
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
for _, r := range t.to[bareHost(host)] {
|
||||
if r.path == "" || strings.HasPrefix(path, r.path) {
|
||||
return r, true
|
||||
}
|
||||
}
|
||||
return rule{}, false
|
||||
}
|
||||
|
||||
// routed says whether this proxy serves the name at all, whatever the path.
|
||||
//
|
||||
// Separate from find because certificate issuance is a question about the *name*: a host whose only
|
||||
// rules are path-scoped is still a name this proxy answers to, and still needs a certificate.
|
||||
// eligibleForACME says whether this proxy may ask a certificate authority for this name — every
|
||||
// host reached as a route's own public `name`, never one reached only as its `internal-name`
|
||||
// alias, which no public CA can ever validate.
|
||||
func (t *table) eligibleForACME(host string) bool {
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
bare := bareHost(host)
|
||||
return len(t.to[bare]) > 0 && t.public[bare]
|
||||
}
|
||||
|
||||
func (t *table) routed(host string) bool {
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
return len(t.to[bareHost(host)]) > 0
|
||||
}
|
||||
|
||||
// eligibleForInternalACME says whether this proxy may ask its *internal* authority for a
|
||||
// certificate for this name — every host it routes that is not also a route's public `name`.
|
||||
//
|
||||
// **The mesh has two name spaces and two authorities** (novox/hq 03-DESIGN/01-to-be/08-connectivity
|
||||
// §2): a public name is certified by a public CA, an internal one by the mesh's own. This is
|
||||
// composed only from `to` and `public`, which routesFrom already builds correctly — a host never
|
||||
// lands in both a route's own `name` and only its `internal-name`, so nothing new has to be
|
||||
// tracked to tell the two apart.
|
||||
func (t *table) eligibleForInternalACME(host string) bool {
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
bare := bareHost(host)
|
||||
return len(t.to[bare]) > 0 && !t.public[bare]
|
||||
}
|
||||
|
||||
// bareHost is the name without the port, lower-cased.
|
||||
//
|
||||
// The port is not part of the name: a request to app.example:8080 is for app.example. Lower-cased
|
||||
// because a Host header is not case-sensitive, and a route that only answers the spelling in the
|
||||
// manifest answers half the requests made to it.
|
||||
func bareHost(host string) string {
|
||||
if h, _, err := net.SplitHostPort(host); err == nil {
|
||||
host = h
|
||||
}
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
p, ok := t.to[strings.ToLower(host)]
|
||||
return p, ok
|
||||
return strings.ToLower(host)
|
||||
}
|
||||
|
||||
func (t *table) names() []string {
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
out := make([]string, 0, len(t.targets))
|
||||
for name := range t.targets {
|
||||
out := make([]string, 0, len(t.to))
|
||||
for name := range t.to {
|
||||
out = append(out, name)
|
||||
}
|
||||
sort.Strings(out)
|
||||
@@ -162,7 +344,7 @@ func run() error {
|
||||
|
||||
held := newTable()
|
||||
read := func() {
|
||||
routes, err := routesFrom(path)
|
||||
routes, public, err := routesFrom(path)
|
||||
if err != nil {
|
||||
// Kept serving what it had. A file being rewritten is momentarily unreadable, and
|
||||
// dropping every route because one read landed mid-write would turn an ordinary
|
||||
@@ -170,7 +352,7 @@ func run() error {
|
||||
log.Printf("cannot read %s, keeping what is already served: %v", path, err)
|
||||
return
|
||||
}
|
||||
held.set(routes)
|
||||
held.set(routes, public)
|
||||
log.Printf("serving %d route(s): %s", len(routes), strings.Join(held.names(), ", "))
|
||||
}
|
||||
read()
|
||||
@@ -197,16 +379,158 @@ func run() error {
|
||||
return fmt.Errorf("TLS_LISTEN is set and ACME_CACHE is not: certificates need somewhere " +
|
||||
"to persist, or every restart orders them again")
|
||||
}
|
||||
client := &acme.Client{DirectoryURL: issuer()}
|
||||
// An issuer that is not one of the public ones serves its own API over TLS with a certificate
|
||||
// nothing trusts yet — the lab's, or an internal step-ca. Trusting it is a deliberate act and
|
||||
// names a file, rather than the client being told to skip verification: *skip* would also
|
||||
// apply on the day this points at a public issuer, and nothing would say so.
|
||||
publicManager, err := newManager(cache, issuer(), strings.TrimSpace(os.Getenv("ACME_CA_BUNDLE")),
|
||||
onlyWhatTheMeshSaid(held))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("issuing public certificates from %s, for whatever the mesh routes here", issuer())
|
||||
|
||||
// The internal authority is optional: unset means this proxy serves internal-only aliases over
|
||||
// plain HTTP exactly as it always has, which is the standalone-binary default and a safe one —
|
||||
// it asks nothing of an authority it was not told about.
|
||||
var internalManager *autocert.Manager
|
||||
if directory := strings.TrimSpace(os.Getenv("INTERNAL_ACME_DIRECTORY")); directory != "" {
|
||||
internalManager, err = newManager(cache, directory, strings.TrimSpace(os.Getenv("INTERNAL_ACME_CA_BUNDLE")),
|
||||
onlyInternalNamesTheMeshSaid(held))
|
||||
if err != nil {
|
||||
return fmt.Errorf("internal certificate authority: %w", err)
|
||||
}
|
||||
log.Printf("issuing internal certificates from %s, for every internal-only alias this routes",
|
||||
directory)
|
||||
}
|
||||
|
||||
// Port 80 answers the HTTP-01 challenge and goes on proxying everything else. The challenge
|
||||
// must be answered *at the name being certified*, which is why issuance happens on the node
|
||||
// that is publicly reachable rather than wherever the workload runs.
|
||||
//
|
||||
// **autocert's own HTTPHandler does not fall through on the challenge path.** For a token it
|
||||
// does not hold it answers 404 itself; its fallback only ever sees non-challenge paths — which
|
||||
// is exactly the predecessor's fault, the edge owning `/.well-known/acme-challenge` outright,
|
||||
// rediscovered live when Mailu's renewal died behind this proxy on cutover day. tokenOrRoute
|
||||
// probes each manager and hands a token neither authority recognises to plain routing, which
|
||||
// is what lets a consumer's own ACME client — Mailu's, certifying its own name for a protocol
|
||||
// this proxy never proxies — answer its own challenge through an ordinary path-scoped route.
|
||||
port80 := tokenOrRoute(handler(held), publicManager)
|
||||
if internalManager != nil {
|
||||
port80 = tokenOrRoute(handler(held), publicManager, internalManager)
|
||||
}
|
||||
go func() {
|
||||
if err := http.ListenAndServe(listen, port80); err != nil {
|
||||
log.Printf("plain HTTP stopped: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
tlsConfig := publicManager.TLSConfig()
|
||||
if internalManager != nil {
|
||||
// Dispatched by which authority may certify this name at all — the same question
|
||||
// eligibleForInternalACME already answers, asked once more at handshake time rather than
|
||||
// only when an order is placed, since a cached certificate is served here on every request
|
||||
// and never goes through HostPolicy again.
|
||||
fromPublic, fromInternal := tlsConfig.GetCertificate, internalManager.TLSConfig().GetCertificate
|
||||
tlsConfig.GetCertificate = func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
|
||||
if held.eligibleForInternalACME(hello.ServerName) {
|
||||
return fromInternal(hello)
|
||||
}
|
||||
return fromPublic(hello)
|
||||
}
|
||||
}
|
||||
|
||||
server := &http.Server{
|
||||
Addr: secure,
|
||||
Handler: handler(held),
|
||||
TLSConfig: tlsConfig,
|
||||
}
|
||||
return server.ListenAndServeTLS("", "")
|
||||
}
|
||||
|
||||
// tokenOrRoute serves port 80: each manager answers the challenge tokens it is itself holding,
|
||||
// and a token none of them holds is routed like any other request instead of being 404'd at the
|
||||
// edge.
|
||||
//
|
||||
// autocert gives no way to ask "is this your token?" — its HTTPHandler both answers and refuses —
|
||||
// so each manager is probed against a buffered writer and its refusal (404 on the challenge path)
|
||||
// is discarded in favour of the next candidate. The probe is cheap: the handler answers from
|
||||
// memory, and the path only carries traffic while an issuance is actually running.
|
||||
func tokenOrRoute(routes http.Handler, managers ...*autocert.Manager) http.Handler {
|
||||
const challengePrefix = "/.well-known/acme-challenge/"
|
||||
// Non-challenge paths never reach a manager at all; autocert's tryHTTP01 switch still has to
|
||||
// be armed, which HTTPHandler is the only exported way to do.
|
||||
probes := make([]http.Handler, len(managers))
|
||||
for i, m := range managers {
|
||||
probes[i] = m.HTTPHandler(routes)
|
||||
}
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if !strings.HasPrefix(r.URL.Path, challengePrefix) {
|
||||
routes.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
for _, probe := range probes {
|
||||
buffered := &probedResponse{header: make(http.Header)}
|
||||
probe.ServeHTTP(buffered, r)
|
||||
// Two shapes of "not mine": 404, a token this manager is not holding — and 403, a
|
||||
// name its host policy would never certify at all (autocert checks the policy before
|
||||
// the token, so the internal authority answers 403 for every public name).
|
||||
if buffered.status == http.StatusNotFound || buffered.status == http.StatusForbidden {
|
||||
continue
|
||||
}
|
||||
buffered.replayTo(w)
|
||||
return
|
||||
}
|
||||
routes.ServeHTTP(w, r) // no authority holds it: the workload behind a routed path may
|
||||
})
|
||||
}
|
||||
|
||||
// probedResponse buffers one handler's answer so a refusal can be discarded unseen.
|
||||
type probedResponse struct {
|
||||
header http.Header
|
||||
status int
|
||||
body bytes.Buffer
|
||||
}
|
||||
|
||||
func (p *probedResponse) Header() http.Header { return p.header }
|
||||
|
||||
func (p *probedResponse) WriteHeader(status int) {
|
||||
if p.status == 0 {
|
||||
p.status = status
|
||||
}
|
||||
}
|
||||
|
||||
func (p *probedResponse) Write(b []byte) (int, error) {
|
||||
if p.status == 0 {
|
||||
p.status = http.StatusOK
|
||||
}
|
||||
return p.body.Write(b)
|
||||
}
|
||||
|
||||
func (p *probedResponse) replayTo(w http.ResponseWriter) {
|
||||
for k, vs := range p.header {
|
||||
for _, v := range vs {
|
||||
w.Header().Add(k, v)
|
||||
}
|
||||
}
|
||||
status := p.status
|
||||
if status == 0 {
|
||||
status = http.StatusOK
|
||||
}
|
||||
w.WriteHeader(status)
|
||||
_, _ = w.Write(p.body.Bytes())
|
||||
}
|
||||
|
||||
// newManager is one ACME authority's autocert manager: where to ask, what to trust it with, and
|
||||
// which names it may be asked to certify.
|
||||
//
|
||||
// **Trusting an authority names a file rather than skipping verification.** An issuer that is not
|
||||
// one of the public ones — the lab's, or the mesh's own step-ca — serves its own ACME API over TLS
|
||||
// with a certificate nothing trusts yet. *Skip* would also apply the day this points at a public
|
||||
// issuer, and nothing would say so; naming a bundle is a deliberate, visible act instead.
|
||||
func newManager(cache, directory, bundle string, policy autocert.HostPolicy) (*autocert.Manager, error) {
|
||||
client := &acme.Client{DirectoryURL: directory}
|
||||
var root []byte
|
||||
if bundle := strings.TrimSpace(os.Getenv("ACME_CA_BUNDLE")); bundle != "" {
|
||||
if bundle != "" {
|
||||
read, err := os.ReadFile(bundle)
|
||||
if err != nil {
|
||||
return fmt.Errorf("ACME_CA_BUNDLE names %s and it cannot be read: %w", bundle, err)
|
||||
return nil, fmt.Errorf("the CA bundle names %s and it cannot be read: %w", bundle, err)
|
||||
}
|
||||
root = read
|
||||
// An empty bundle means the issuer's root is already in the system trust store — a public
|
||||
@@ -218,7 +542,7 @@ func run() error {
|
||||
if strings.TrimSpace(string(root)) != "" {
|
||||
pool := x509.NewCertPool()
|
||||
if !pool.AppendCertsFromPEM(root) {
|
||||
return fmt.Errorf("%s holds no certificate this can trust", bundle)
|
||||
return nil, fmt.Errorf("%s holds no certificate this can trust", bundle)
|
||||
}
|
||||
client.HTTPClient = &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
@@ -227,31 +551,16 @@ func run() error {
|
||||
}
|
||||
}
|
||||
// Where this authority's account and certificates are kept. Per authority, not per proxy — see
|
||||
// forThisAuthority, which is what makes a re-initialised CA heal itself.
|
||||
mine := forThisAuthority(cache, issuer(), root)
|
||||
manager := &autocert.Manager{
|
||||
// forThisAuthority, which is what makes a re-initialised CA heal itself, and what lets the
|
||||
// public and internal authorities share one ACME_CACHE without colliding: they hash to
|
||||
// different names because their directories differ.
|
||||
mine := forThisAuthority(cache, directory, root)
|
||||
return &autocert.Manager{
|
||||
Cache: autocert.DirCache(mine),
|
||||
Prompt: autocert.AcceptTOS,
|
||||
HostPolicy: onlyWhatTheMeshSaid(held),
|
||||
HostPolicy: policy,
|
||||
Client: client,
|
||||
}
|
||||
log.Printf("issuing from %s into %s, for whatever the mesh routes here", issuer(), mine)
|
||||
|
||||
// Port 80 answers the HTTP-01 challenge and goes on proxying everything else. The challenge
|
||||
// must be answered *at the name being certified*, which is why issuance happens on the node
|
||||
// that is publicly reachable rather than wherever the workload runs.
|
||||
go func() {
|
||||
if err := http.ListenAndServe(listen, manager.HTTPHandler(handler(held))); err != nil {
|
||||
log.Printf("plain HTTP stopped: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
server := &http.Server{
|
||||
Addr: secure,
|
||||
Handler: handler(held),
|
||||
TLSConfig: manager.TLSConfig(),
|
||||
}
|
||||
return server.ListenAndServeTLS("", "")
|
||||
}, nil
|
||||
}
|
||||
|
||||
// forThisAuthority is where one ACME authority's account and certificates are kept.
|
||||
@@ -285,61 +594,303 @@ func forThisAuthority(cache, directory string, root []byte) string {
|
||||
|
||||
// newTable is an empty routing table.
|
||||
func newTable() *table {
|
||||
return &table{to: map[string]*httputil.ReverseProxy{}, targets: map[string]string{}}
|
||||
return &table{to: map[string][]rule{}}
|
||||
}
|
||||
|
||||
// handler is the proxy itself, separated so it can be driven by a test without a listener.
|
||||
func handler(held *table) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
proxy, known := held.find(r.Host)
|
||||
matched, known := held.find(r.Host, r.URL.Path)
|
||||
if !known {
|
||||
// **Named, not a bare 404.** A route that was withdrawn and a name that never existed
|
||||
// are different things, and a proxy that says only "not found" makes an operator go
|
||||
// and read the mesh to tell them apart. What it is serving is the answer to both.
|
||||
//
|
||||
// And since a host may now be routed only on some paths, those are a third thing:
|
||||
// saying "no route for this name" while listing that very name as served is a
|
||||
// contradiction an operator would have to disbelieve the proxy to get past.
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
if held.routed(r.Host) {
|
||||
fmt.Fprintf(w, "%s is served here, but no route covers %q.\n",
|
||||
bareHost(r.Host), r.URL.Path)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(w, "no route for %q in this mesh.\nserving: %s\n",
|
||||
r.Host, strings.Join(held.names(), ", "))
|
||||
return
|
||||
}
|
||||
proxy.ServeHTTP(w, r)
|
||||
|
||||
switch {
|
||||
case matched.policy.sealed != "":
|
||||
// Declared a gate, cannot check it. Refused, and says why — an operator reading this
|
||||
// learns the secret is missing, rather than wondering why a protected name is 503.
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
fmt.Fprintf(w, "this route requires authentication and its credentials cannot be read: %s\n",
|
||||
matched.policy.sealed)
|
||||
return
|
||||
|
||||
case matched.policy.deny:
|
||||
http.Error(w, "this path is not served to you", http.StatusForbidden)
|
||||
return
|
||||
|
||||
case matched.policy.redirectTo != "":
|
||||
http.Redirect(w, r, canonical(matched.policy.redirectTo, r.URL), http.StatusMovedPermanently)
|
||||
return
|
||||
|
||||
case len(matched.policy.users) > 0 && !allowed(matched.policy.users, r):
|
||||
// The realm is the name asked for, so a browser's prompt says which route it is for.
|
||||
w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q, charset=\"UTF-8\"", bareHost(r.Host)))
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
|
||||
if matched.maxRequestBody > 0 {
|
||||
// Refused on the declared length where there is one, so an upload that cannot succeed
|
||||
// is answered before it is carried; and capped while reading for a chunked body, which
|
||||
// declares no length at all. Without the second, a limit is advice.
|
||||
if r.ContentLength > matched.maxRequestBody {
|
||||
http.Error(w, fmt.Sprintf("request body too large for this route: %d bytes is the most it carries",
|
||||
matched.maxRequestBody), http.StatusRequestEntityTooLarge)
|
||||
return
|
||||
}
|
||||
r.Body = http.MaxBytesReader(w, r.Body, matched.maxRequestBody)
|
||||
}
|
||||
|
||||
matched.to.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// routesFrom reads what the mesh wrote and turns it into name → target.
|
||||
func routesFrom(path string) (map[string]string, error) {
|
||||
// canonical is where a redirect sends this request.
|
||||
//
|
||||
// The declaration names the destination *name*; the request keeps its own path and query. That is
|
||||
// what canonicalising one public name onto another means — a link to a page under the old name has
|
||||
// to arrive at the same page under the new one, or the redirect silently loses every deep link.
|
||||
func canonical(to string, from *url.URL) string {
|
||||
where, err := url.Parse(to)
|
||||
if err != nil {
|
||||
return to
|
||||
}
|
||||
if where.Path == "" || where.Path == "/" {
|
||||
where.Path = from.Path
|
||||
}
|
||||
if where.RawQuery == "" {
|
||||
where.RawQuery = from.RawQuery
|
||||
}
|
||||
return where.String()
|
||||
}
|
||||
|
||||
// allowed says whether the request presented credentials this route accepts.
|
||||
//
|
||||
// **Every path costs one bcrypt comparison**, including an unknown user, which is why the miss
|
||||
// compares against a fixed hash rather than returning early. Returning early would make an unknown
|
||||
// user measurably faster than a known one with a wrong password, and that difference is a way to
|
||||
// enumerate the users of a route from outside it.
|
||||
func allowed(users map[string]string, r *http.Request) bool {
|
||||
// A hash of nothing anybody knows. Its only job is to cost what a real comparison costs.
|
||||
const absent = "$2a$10$N9qo8uLOickgx2ZMRZoMyeIjZAgcfl7p92ldGxad68LJZdL17lhWy"
|
||||
user, password, ok := r.BasicAuth()
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
want, known := users[user]
|
||||
if !known {
|
||||
want = absent
|
||||
}
|
||||
if err := bcrypt.CompareHashAndPassword([]byte(want), []byte(password)); err != nil {
|
||||
return false
|
||||
}
|
||||
// `known` is checked after the comparison, not instead of it, so the timing is the same either
|
||||
// way. subtle.ConstantTimeByteEq keeps the branch from being the thing that differs.
|
||||
return subtle.ConstantTimeByteEq(boolByte(known), 1) == 1
|
||||
}
|
||||
|
||||
func boolByte(b bool) byte {
|
||||
if b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// routesFrom reads what the mesh wrote and turns it into host → the rules for that host, and
|
||||
// which of those hosts is a public name — the second is `name`, ACME-eligible; a host reached
|
||||
// only through `internal-name` never appears there.
|
||||
func routesFrom(path string) (map[string][]rule, map[string]bool, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
var said given
|
||||
if err := json.Unmarshal(raw, &said); err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
out := map[string]string{}
|
||||
out := map[string][]rule{}
|
||||
public := map[string]bool{}
|
||||
for _, c := range said.Given {
|
||||
name, _ := c.Values["name"].(string)
|
||||
if name == "" {
|
||||
log.Printf("%s on %s asked for a route and named nothing; skipped", c.From, c.Node)
|
||||
continue
|
||||
}
|
||||
port, ok := asPort(c.Values["port"])
|
||||
if !ok {
|
||||
log.Printf("%s on %s asked for route %q and gave no usable port; skipped",
|
||||
c.From, c.Node, name)
|
||||
host := strings.ToLower(name)
|
||||
public[host] = true
|
||||
|
||||
made := rule{path: asPath(c.Values["path"])}
|
||||
if p, ok := asWhole(c.Values["priority"]); ok {
|
||||
made.priority = p
|
||||
}
|
||||
made.policy.deny, _ = c.Values["deny"].(bool)
|
||||
made.policy.redirectTo, _ = c.Values["redirect"].(string)
|
||||
|
||||
if named, carried := c.Values["auth"].(string); carried && strings.TrimSpace(named) != "" {
|
||||
// **A declaration names a secret; it never holds one** (ADR 0108). Refused rather than
|
||||
// tolerated, and the whole rule is dropped rather than served unprotected — the
|
||||
// rejected option cannot come back by accident, which is the failure this check exists
|
||||
// to make impossible.
|
||||
if looksLikeACredential(named) {
|
||||
log.Printf("%s on %s declared route %q with a credential in the declaration rather "+
|
||||
"than the name of a secret; the whole route is refused (novox/hq ADR 0108)",
|
||||
c.From, c.Node, name)
|
||||
continue
|
||||
}
|
||||
users, err := usersFrom(named)
|
||||
if err != nil {
|
||||
// Fail closed: the rule is kept so the name stays routed and answers, and it
|
||||
// answers by refusing. Dropping it instead would make the name 404 and read as a
|
||||
// withdrawn route rather than an unreadable secret.
|
||||
made.policy.sealed = err.Error()
|
||||
}
|
||||
made.policy.users = users
|
||||
}
|
||||
|
||||
// Only a rule that actually proxies needs somewhere to send the request.
|
||||
if !made.policy.deny && made.policy.redirectTo == "" {
|
||||
port, ok := asPort(c.Values["port"])
|
||||
if !ok {
|
||||
log.Printf("%s on %s asked for route %q and gave no usable port; skipped",
|
||||
c.From, c.Node, name)
|
||||
continue
|
||||
}
|
||||
// Where the mesh says that machine is. Empty means it is this one — a workload beside
|
||||
// the proxy is ordinary, and reaching it over loopback is both correct and the only
|
||||
// thing that works when there is no private network.
|
||||
at := c.At
|
||||
if at == "" {
|
||||
at = "127.0.0.1"
|
||||
}
|
||||
// http unless the contribution says otherwise. A backend that terminates its own TLS
|
||||
// with a certificate this proxy has no reason to trust — Mailu's webmail front is the
|
||||
// first of these — is the reason `insecure` exists, and it stays the exception: every
|
||||
// other target the mesh hands this proxy is a plain workload on the private network.
|
||||
scheme, _ := c.Values["scheme"].(string)
|
||||
scheme = strings.ToLower(strings.TrimSpace(scheme))
|
||||
if scheme == "" {
|
||||
scheme = "http"
|
||||
}
|
||||
if scheme != "http" && scheme != "https" {
|
||||
log.Printf("%s on %s asked for route %q with scheme %q, which is neither http "+
|
||||
"nor https; skipped", c.From, c.Node, name, scheme)
|
||||
continue
|
||||
}
|
||||
made.insecure, _ = c.Values["insecure"].(bool)
|
||||
// A limit this proxy cannot read is a route it does not serve, named like a port that
|
||||
// is not a port. Serving it without the limit would carry exactly what the module said
|
||||
// not to carry, and report success doing it.
|
||||
if asked, said := c.Values["max-request-body"]; said {
|
||||
bytes, whole := asWhole(asked)
|
||||
if !whole || bytes <= 0 {
|
||||
log.Printf("%s on %s asked for route %q with a max-request-body of %v, which is "+
|
||||
"not a whole positive number of bytes; skipped", c.From, c.Node, name, asked)
|
||||
continue
|
||||
}
|
||||
made.maxRequestBody = int64(bytes)
|
||||
}
|
||||
made.target = fmt.Sprintf("%s://%s:%d", scheme, at, port)
|
||||
}
|
||||
|
||||
out[host] = append(out[host], made)
|
||||
|
||||
// The internal-network alias, the same rule under a second host — a predecessor proxy
|
||||
// answered both for one route, as a convenience (reaching a service over the VPN without a
|
||||
// public TLS round trip), not as an access boundary; composing it here restores exactly
|
||||
// that, nothing more. Absent whenever the node composed no internal name (novox/hq ADR
|
||||
// 0056's internalDomain half) — the same "nothing to join a label to" case the public name
|
||||
// already has.
|
||||
if internal, _ := c.Values["internal-name"].(string); strings.TrimSpace(internal) != "" {
|
||||
out[strings.ToLower(internal)] = append(out[strings.ToLower(internal)], made)
|
||||
}
|
||||
}
|
||||
return out, public, nil
|
||||
}
|
||||
|
||||
// asWhole is any whole number the mesh wrote, whatever its magnitude.
|
||||
//
|
||||
// **Not asPort.** Priority was read with the port reader first, which caps at 65535 — so a rule
|
||||
// declared at a priority above that silently became priority 0 and stopped shadowing the route it
|
||||
// exists to shadow. The one real rule this has to reproduce is declared at 100000, so the bug was
|
||||
// exactly load-bearing. A priority is an ordering, not a port: it has no range.
|
||||
func asWhole(v any) (int, bool) {
|
||||
switch n := v.(type) {
|
||||
case float64:
|
||||
// JSON makes a float of every number, so a non-integral one was not meant as a priority.
|
||||
if n != float64(int(n)) {
|
||||
return 0, false
|
||||
}
|
||||
return int(n), true
|
||||
case int:
|
||||
return n, true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// asPath is the path prefix a rule is scoped to, or "" for every path.
|
||||
func asPath(v any) string {
|
||||
p, _ := v.(string)
|
||||
p = strings.TrimSpace(p)
|
||||
if p == "" {
|
||||
return ""
|
||||
}
|
||||
if !strings.HasPrefix(p, "/") {
|
||||
p = "/" + p
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// looksLikeACredential is the check that keeps a secret out of a declaration.
|
||||
//
|
||||
// It errs towards refusing: a value holding a `:` (the htpasswd separator) or opening with a bcrypt
|
||||
// identifier is a credential, not a path, and no filesystem path the mesh writes needs either. A
|
||||
// false refusal is a loud log and a route that does not serve; a false accept is a credential
|
||||
// committed to a declaration, which is the thing being prevented.
|
||||
func looksLikeACredential(v string) bool {
|
||||
v = strings.TrimSpace(v)
|
||||
return strings.Contains(v, ":") || strings.HasPrefix(v, "$2")
|
||||
}
|
||||
|
||||
// usersFrom reads the credentials the mesh mounted, in the one format every htpasswd already is.
|
||||
func usersFrom(path string) (map[string]string, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot read the secret named for this route: %w", err)
|
||||
}
|
||||
users := map[string]string{}
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
// Where the mesh says that machine is. Empty means it is this one — a workload beside the
|
||||
// proxy is ordinary, and reaching it over loopback is both correct and the only thing
|
||||
// that works when there is no private network.
|
||||
at := c.At
|
||||
if at == "" {
|
||||
at = "127.0.0.1"
|
||||
user, hash, ok := strings.Cut(line, ":")
|
||||
if !ok || user == "" || hash == "" {
|
||||
continue
|
||||
}
|
||||
out[strings.ToLower(name)] = fmt.Sprintf("http://%s:%d", at, port)
|
||||
users[user] = hash
|
||||
}
|
||||
return out, nil
|
||||
if len(users) == 0 {
|
||||
return nil, fmt.Errorf("the secret named for this route holds no usable credentials")
|
||||
}
|
||||
return users, nil
|
||||
}
|
||||
|
||||
// asPort accepts what JSON makes of a number, which is a float even when it was written 8080.
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
)
|
||||
|
||||
// What a route carries about the requests arriving at it — novox/hq ADR 0108.
|
||||
//
|
||||
// Each test here is one of the four capabilities that record closed the set at, plus the negative
|
||||
// case it promised would be refused. The negative case is the one that rots quietly: nothing fails
|
||||
// if it stops working, so nothing tells you it has.
|
||||
|
||||
// served starts a workload and gives back the host and port the mesh would have recorded for it.
|
||||
func served(t *testing.T, body string) (string, int) {
|
||||
t.Helper()
|
||||
workload := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte(body))
|
||||
}))
|
||||
t.Cleanup(workload.Close)
|
||||
host, port, _ := strings.Cut(strings.TrimPrefix(workload.URL, "http://"), ":")
|
||||
n, err := strconv.Atoi(port)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return host, n
|
||||
}
|
||||
|
||||
// ask makes one request through the proxy for a given name and path, without following redirects.
|
||||
func ask(t *testing.T, proxy, name, path string, auth [2]string) *http.Response {
|
||||
t.Helper()
|
||||
req, err := http.NewRequest(http.MethodGet, proxy+path, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.Host = name
|
||||
if auth[0] != "" {
|
||||
req.SetBasicAuth(auth[0], auth[1])
|
||||
}
|
||||
client := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error {
|
||||
return http.ErrUseLastResponse
|
||||
}}
|
||||
answer, err := client.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = answer.Body.Close() })
|
||||
return answer
|
||||
}
|
||||
|
||||
func proxyFor(t *testing.T, routesJSON string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "routes.json")
|
||||
if err := os.WriteFile(path, []byte(routesJSON), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
routes, public, err := routesFrom(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held := newTable()
|
||||
held.set(routes, public)
|
||||
server := httptest.NewServer(handler(held))
|
||||
t.Cleanup(server.Close)
|
||||
return server.URL
|
||||
}
|
||||
|
||||
// A refusal on a path shadows the ordinary route for that path and leaves every other path alone.
|
||||
//
|
||||
// **This is why path scoping is a prerequisite and not a sibling capability.** The rule being
|
||||
// reproduced matches a path on a host that is already routed to a workload, so a table mapping a
|
||||
// host to one target cannot express it at all — no amount of authentication or source filtering
|
||||
// would have helped.
|
||||
func TestARefusedPathShadowsTheRouteAndLeavesTheRestServed(t *testing.T) {
|
||||
at, port := served(t, "the workload")
|
||||
proxy := proxyFor(t, `{"given":[
|
||||
{"from":"forge","node":"anchor","at":"`+at+`","values":{"name":"forge.example","port":`+strconv.Itoa(port)+`}},
|
||||
{"from":"forge","node":"anchor","values":{"name":"forge.example","path":"/api/internal","priority":100,"deny":true}}
|
||||
]}`)
|
||||
|
||||
if got := ask(t, proxy, "forge.example", "/api/internal/hook", [2]string{}).StatusCode; got != http.StatusForbidden {
|
||||
t.Fatalf("the refused path answered %d, so the block that was put in front of it during an "+
|
||||
"incident is not in front of it any more", got)
|
||||
}
|
||||
if got := ask(t, proxy, "forge.example", "/", [2]string{}).StatusCode; got != http.StatusOK {
|
||||
t.Fatalf("refusing one path took the whole route with it: %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A redirect answers with the redirect, and the request keeps its own path and query.
|
||||
//
|
||||
// Losing the path would turn canonicalising one name onto another into "every deep link now lands
|
||||
// on the front page", which is the kind of breakage that produces no error anywhere.
|
||||
func TestARedirectKeepsThePathAndQuery(t *testing.T) {
|
||||
proxy := proxyFor(t, `{"given":[
|
||||
{"from":"site","node":"anchor","values":{"name":"www.example","redirect":"https://example/"}}
|
||||
]}`)
|
||||
|
||||
answer := ask(t, proxy, "www.example", "/deep/page?ref=1", [2]string{})
|
||||
if answer.StatusCode != http.StatusMovedPermanently {
|
||||
t.Fatalf("a declared redirect answered %d", answer.StatusCode)
|
||||
}
|
||||
where := answer.Header.Get("Location")
|
||||
if !strings.Contains(where, "/deep/page") || !strings.Contains(where, "ref=1") {
|
||||
t.Fatalf("the redirect dropped the path or the query: %q", where)
|
||||
}
|
||||
}
|
||||
|
||||
// Authentication refuses a request with no credentials, admits one with the right ones, and refuses
|
||||
// the wrong ones — with the credentials read from the secret the declaration *named*.
|
||||
func TestAuthenticationAdmitsOnlyWhatTheSecretSays(t *testing.T) {
|
||||
at, port := served(t, "the console")
|
||||
hash, err := bcrypt.GenerateFromPassword([]byte("correct horse"), bcrypt.MinCost)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secret := filepath.Join(t.TempDir(), "console-auth")
|
||||
if err := os.WriteFile(secret, []byte("# a comment\nadmin:"+string(hash)+"\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
proxy := proxyFor(t, `{"given":[
|
||||
{"from":"console","node":"anchor","at":"`+at+`","values":{"name":"console.example","port":`+strconv.Itoa(port)+`,"auth":"`+secret+`"}}
|
||||
]}`)
|
||||
|
||||
if got := ask(t, proxy, "console.example", "/", [2]string{}).StatusCode; got != http.StatusUnauthorized {
|
||||
t.Fatalf("an admin surface with no login of its own answered %d without credentials", got)
|
||||
}
|
||||
if got := ask(t, proxy, "console.example", "/", [2]string{"admin", "wrong"}).StatusCode; got != http.StatusUnauthorized {
|
||||
t.Fatalf("the wrong password answered %d", got)
|
||||
}
|
||||
if got := ask(t, proxy, "console.example", "/", [2]string{"admin", "correct horse"}).StatusCode; got != http.StatusOK {
|
||||
t.Fatalf("the right password answered %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The negative case ADR 0108 promised would be refused: a credential in the declaration.
|
||||
//
|
||||
// **Refused whole, not tolerated and not served unprotected.** A hash carried in a declaration was
|
||||
// the rejected option; nothing in the running system should quietly accept it later, because the
|
||||
// precedent is far easier to set than to withdraw. If this test is deleted the option returns and
|
||||
// nothing else notices.
|
||||
func TestACredentialInTheDeclarationIsRefusedRatherThanServed(t *testing.T) {
|
||||
inline := []string{
|
||||
`{"given":[{"from":"c","node":"n","at":"127.0.0.1","values":{"name":"c.example","port":8080,"auth":"admin:$2a$10$abcdefghijklmnopqrstuv"}}]}`,
|
||||
`{"given":[{"from":"c","node":"n","at":"127.0.0.1","values":{"name":"c.example","port":8080,"auth":"$2a$10$abcdefghijklmnopqrstuv"}}]}`,
|
||||
}
|
||||
for _, body := range inline {
|
||||
path := filepath.Join(t.TempDir(), "routes.json")
|
||||
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
routes, _, err := routesFrom(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(routes) != 0 {
|
||||
t.Fatalf("a declaration carrying a credential was served anyway: %v", routes)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Authentication declared, secret unreadable: the route refuses. It does not serve unprotected.
|
||||
//
|
||||
// **Fail closed.** The alternative turns a missing file into a silently public admin surface, which
|
||||
// is the outcome the whole record exists to prevent. It answers rather than 404s, so an operator
|
||||
// sees "cannot read the credentials" instead of concluding the route was withdrawn.
|
||||
func TestAnUnreadableSecretFailsClosed(t *testing.T) {
|
||||
at, port := served(t, "the console")
|
||||
missing := filepath.Join(t.TempDir(), "not-mounted")
|
||||
|
||||
proxy := proxyFor(t, `{"given":[
|
||||
{"from":"console","node":"anchor","at":"`+at+`","values":{"name":"console.example","port":`+strconv.Itoa(port)+`,"auth":"`+missing+`"}}
|
||||
]}`)
|
||||
|
||||
answer := ask(t, proxy, "console.example", "/", [2]string{})
|
||||
if answer.StatusCode == http.StatusOK {
|
||||
t.Fatal("a route whose credentials could not be read served the workload unprotected")
|
||||
}
|
||||
if answer.StatusCode != http.StatusServiceUnavailable {
|
||||
t.Fatalf("expected the route to say it cannot check, got %d", answer.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
// Equal priorities resolve the same way every time, so the same declaration serves the same way
|
||||
// after a restart.
|
||||
//
|
||||
// Sorting only by priority leaves rules that share one in whatever order the map produced. The
|
||||
// proxy would still work, and would work differently between restarts — which is the hardest kind
|
||||
// of fault to believe when it is reported.
|
||||
func TestRulesThatShareAPriorityAreStillTotallyOrdered(t *testing.T) {
|
||||
first := []rule{
|
||||
{path: "/a", priority: 10, target: "http://x:1"},
|
||||
{path: "/bb", priority: 10, target: "http://y:2"},
|
||||
{path: "", priority: 10, target: "http://z:3"},
|
||||
}
|
||||
second := []rule{
|
||||
{path: "", priority: 10, target: "http://z:3"},
|
||||
{path: "/bb", priority: 10, target: "http://y:2"},
|
||||
{path: "/a", priority: 10, target: "http://x:1"},
|
||||
}
|
||||
inOrder(first)
|
||||
inOrder(second)
|
||||
for i := range first {
|
||||
if first[i].path != second[i].path || first[i].target != second[i].target {
|
||||
t.Fatalf("two orderings of the same rules disagree at %d: %q vs %q",
|
||||
i, first[i].path, second[i].path)
|
||||
}
|
||||
}
|
||||
// And the more specific rule is matched first, which is the intuitive reading.
|
||||
if first[0].path != "/bb" {
|
||||
t.Fatalf("the longest path is not matched first: %q", first[0].path)
|
||||
}
|
||||
}
|
||||
|
||||
// Priority decides before path length does, so a rule can be made to win regardless of specificity.
|
||||
func TestPriorityOutranksPathLength(t *testing.T) {
|
||||
rules := []rule{
|
||||
{path: "/very/long/path", priority: 1, target: "http://x:1"},
|
||||
{path: "", priority: 100, target: "http://y:2"},
|
||||
}
|
||||
inOrder(rules)
|
||||
if rules[0].priority != 100 {
|
||||
t.Fatalf("a higher priority did not win: %+v", rules[0])
|
||||
}
|
||||
}
|
||||
|
||||
// A priority above a port number survives, because a priority is an ordering and not a port.
|
||||
//
|
||||
// **Found by review, and it was load-bearing.** Priority was first read with the port reader, which
|
||||
// caps at 65535 — so a rule declared above that silently became priority 0 and stopped shadowing the
|
||||
// route it exists to shadow. The one real rule this has to reproduce is declared at 100000, so the
|
||||
// capability would have shipped looking complete and doing nothing.
|
||||
func TestAPriorityAboveAPortNumberSurvives(t *testing.T) {
|
||||
at, port := served(t, "the workload")
|
||||
proxy := proxyFor(t, `{"given":[
|
||||
{"from":"forge","node":"anchor","at":"`+at+`","values":{"name":"forge.example","port":`+strconv.Itoa(port)+`}},
|
||||
{"from":"forge","node":"anchor","values":{"name":"forge.example","path":"/api/internal","priority":100000,"deny":true}}
|
||||
]}`)
|
||||
|
||||
if got := ask(t, proxy, "forge.example", "/api/internal/hook", [2]string{}).StatusCode; got != http.StatusForbidden {
|
||||
t.Fatalf("a rule declared at priority 100000 answered %d instead of refusing", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A host routed only on some paths says so, rather than claiming the name is not served here.
|
||||
//
|
||||
// Saying "no route for this name" while listing that very name as served is a contradiction an
|
||||
// operator has to disbelieve the proxy to get past — and path scoping makes it reachable, because a
|
||||
// host can now have rules that none of this request's paths match.
|
||||
func TestAHostRoutedOnlyOnSomePathsSaysSo(t *testing.T) {
|
||||
proxy := proxyFor(t, `{"given":[
|
||||
{"from":"forge","node":"anchor","values":{"name":"forge.example","path":"/api/internal","deny":true}}
|
||||
]}`)
|
||||
|
||||
answer := ask(t, proxy, "forge.example", "/elsewhere", [2]string{})
|
||||
if answer.StatusCode != http.StatusNotFound {
|
||||
t.Fatalf("an uncovered path answered %d", answer.StatusCode)
|
||||
}
|
||||
body := make([]byte, 256)
|
||||
n, _ := answer.Body.Read(body)
|
||||
said := string(body[:n])
|
||||
if !strings.Contains(said, "is served here") || !strings.Contains(said, "/elsewhere") {
|
||||
t.Fatalf("the refusal does not distinguish an uncovered path from an unserved name: %q", said)
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,8 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
@@ -19,10 +21,37 @@ func write(t *testing.T, body string) string {
|
||||
return path
|
||||
}
|
||||
|
||||
// plain is the table an ordinary set of routes makes: one host, one target, no policy.
|
||||
func plain(routes map[string]string) map[string][]rule {
|
||||
out := map[string][]rule{}
|
||||
for host, target := range routes {
|
||||
out[host] = []rule{{target: target}}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// allPublic is every host in a routes map, ACME-eligible — the ordinary case for a test with no
|
||||
// internal-name alias of its own to distinguish.
|
||||
func allPublic(routes map[string][]rule) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for host := range routes {
|
||||
out[host] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// targetOf is where a host's first matching rule sends a request.
|
||||
func targetOf(routes map[string][]rule, host string) string {
|
||||
if rules := routes[host]; len(rules) > 0 {
|
||||
return rules[0].target
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// A route is a grant: the consumer supplies a target, and where that machine is comes from the
|
||||
// mesh rather than from a naming convention the proxy has to know.
|
||||
func TestARouteGoesToWhereTheMeshSaysTheConsumerIs(t *testing.T) {
|
||||
routes, err := routesFrom(write(t, `{"contributions":1,"requirement":"route","given":[
|
||||
routes, _, err := routesFrom(write(t, `{"contributions":1,"requirement":"route","given":[
|
||||
{"from":"app","node":"laptop","at":"laptop.internal","values":{"name":"App.Example","port":8080}}
|
||||
]}`))
|
||||
if err != nil {
|
||||
@@ -30,28 +59,67 @@ func TestARouteGoesToWhereTheMeshSaysTheConsumerIs(t *testing.T) {
|
||||
}
|
||||
// Lower-cased, because a Host header is not case-sensitive and a route that only answers the
|
||||
// spelling in the manifest answers half the requests made to it.
|
||||
if routes["app.example"] != "http://laptop.internal:8080" {
|
||||
if targetOf(routes, "app.example") != "http://laptop.internal:8080" {
|
||||
t.Fatalf("the route does not point at the consumer: %v", routes)
|
||||
}
|
||||
}
|
||||
|
||||
// A route with an internal-name alias is reachable under both hostnames, pointed at the same
|
||||
// target — the same convenience a predecessor proxy gave for reaching a service over the VPN
|
||||
// without a public TLS round trip.
|
||||
func TestARouteWithAnInternalNameIsReachableUnderBoth(t *testing.T) {
|
||||
routes, public, err := routesFrom(write(t, `{"given":[
|
||||
{"from":"app","node":"anchor","at":"anchor.internal",
|
||||
"values":{"name":"app.example","internal-name":"app.anchor.internal","port":8080}}
|
||||
]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if targetOf(routes, "app.example") != "http://anchor.internal:8080" {
|
||||
t.Fatalf("the public name does not point at the consumer: %v", routes)
|
||||
}
|
||||
if targetOf(routes, "app.anchor.internal") != "http://anchor.internal:8080" {
|
||||
t.Fatalf("the internal alias does not point at the same consumer: %v", routes)
|
||||
}
|
||||
if !public["app.example"] {
|
||||
t.Errorf("the public name is not eligible for a certificate: %v", public)
|
||||
}
|
||||
if public["app.anchor.internal"] {
|
||||
t.Errorf("the internal alias is eligible for a certificate no public CA could ever issue: %v",
|
||||
public)
|
||||
}
|
||||
}
|
||||
|
||||
// A route with no internal-name composed gets no second host — the ordinary case, unchanged.
|
||||
func TestARouteWithNoInternalNameGetsNoAlias(t *testing.T) {
|
||||
routes, _, err := routesFrom(write(t, `{"given":[
|
||||
{"from":"app","node":"anchor","values":{"name":"app.example","port":8080}}
|
||||
]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(routes) != 1 {
|
||||
t.Fatalf("a route with no internal-name grew a second host: %v", routes)
|
||||
}
|
||||
}
|
||||
|
||||
// A workload beside the proxy is ordinary, and reaching it over loopback is both correct and the
|
||||
// only thing that works when there is no private network.
|
||||
func TestAConsumerOnTheProxysOwnMachineIsReachedOverLoopback(t *testing.T) {
|
||||
routes, err := routesFrom(write(t, `{"given":[
|
||||
routes, _, err := routesFrom(write(t, `{"given":[
|
||||
{"from":"app","node":"anchor","values":{"name":"app.example","port":9000}}
|
||||
]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if routes["app.example"] != "http://127.0.0.1:9000" {
|
||||
if targetOf(routes, "app.example") != "http://127.0.0.1:9000" {
|
||||
t.Fatalf("a workload on this machine was not reachable: %v", routes)
|
||||
}
|
||||
}
|
||||
|
||||
// Skipped rather than served wrongly. A route with no port would proxy to :0.
|
||||
func TestAContributionMissingWhatARouteNeedsIsSkipped(t *testing.T) {
|
||||
routes, err := routesFrom(write(t, `{"given":[
|
||||
routes, _, err := routesFrom(write(t, `{"given":[
|
||||
{"from":"a","node":"n","at":"n.internal","values":{"name":"no-port.example"}},
|
||||
{"from":"b","node":"n","at":"n.internal","values":{"port":8080}},
|
||||
{"from":"c","node":"n","at":"n.internal","values":{"name":"fine.example","port":8080}}
|
||||
@@ -59,11 +127,73 @@ func TestAContributionMissingWhatARouteNeedsIsSkipped(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(routes) != 1 || routes["fine.example"] == "" {
|
||||
if len(routes) != 1 || targetOf(routes, "fine.example") == "" {
|
||||
t.Fatalf("an unusable contribution was served: %v", routes)
|
||||
}
|
||||
}
|
||||
|
||||
// A route may name a target reached over https, for a backend that terminates its own TLS — the
|
||||
// shape Mailu's webmail front needs, which this proxy reaches as a plain workload otherwise.
|
||||
func TestARouteMayTargetHttps(t *testing.T) {
|
||||
routes, _, err := routesFrom(write(t, `{"given":[
|
||||
{"from":"mail","node":"anchor","at":"anchor.internal",
|
||||
"values":{"name":"mail.example","port":7443,"scheme":"https","insecure":true}}
|
||||
]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if targetOf(routes, "mail.example") != "https://anchor.internal:7443" {
|
||||
t.Fatalf("an https target was not built as one: %v", routes)
|
||||
}
|
||||
if !routes["mail.example"][0].insecure {
|
||||
t.Fatal("insecure was declared and not carried onto the rule")
|
||||
}
|
||||
}
|
||||
|
||||
// A scheme that is neither http nor https is refused rather than guessed at.
|
||||
func TestARouteWithAnUnknownSchemeIsSkipped(t *testing.T) {
|
||||
routes, _, err := routesFrom(write(t, `{"given":[
|
||||
{"from":"a","node":"n","at":"n.internal","values":{"name":"bad.example","port":80,"scheme":"ftp"}}
|
||||
]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(routes) != 0 {
|
||||
t.Fatalf("a route with an unusable scheme was served: %v", routes)
|
||||
}
|
||||
}
|
||||
|
||||
// End to end: a backend terminating TLS with a certificate nothing would ordinarily trust is still
|
||||
// reached when the route declared `insecure`, and the response comes back through unmodified.
|
||||
func TestTheProxyReachesAnInsecureHttpsBackend(t *testing.T) {
|
||||
workload := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte("the workload, over its own TLS"))
|
||||
}))
|
||||
defer workload.Close()
|
||||
target := strings.TrimPrefix(workload.URL, "https://")
|
||||
|
||||
held := newTable()
|
||||
routes := map[string][]rule{"mail.example": {{target: "https://" + target, insecure: true}}}
|
||||
held.set(routes, allPublic(routes))
|
||||
|
||||
proxy := httptest.NewServer(handler(held))
|
||||
defer proxy.Close()
|
||||
|
||||
asked, err := http.NewRequest(http.MethodGet, proxy.URL, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asked.Host = "mail.example"
|
||||
answer, err := http.DefaultClient.Do(asked)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer answer.Body.Close()
|
||||
if answer.StatusCode != http.StatusOK {
|
||||
t.Fatalf("an insecure https backend was not reached: %d", answer.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
// End to end through the proxy itself: a request for the name reaches the workload, and a name
|
||||
// nobody asked for is refused in a way that says what IS served.
|
||||
func TestTheProxyReachesTheWorkloadAndNamesWhatItServes(t *testing.T) {
|
||||
@@ -75,7 +205,7 @@ func TestTheProxyReachesTheWorkloadAndNamesWhatItServes(t *testing.T) {
|
||||
host, port, _ := strings.Cut(target, ":")
|
||||
|
||||
held := newTable()
|
||||
held.set(map[string]string{"app.example": "http://" + host + ":" + port})
|
||||
held.set(plain(map[string]string{"app.example": "http://" + host + ":" + port}), allPublic(plain(map[string]string{"app.example": "http://" + host + ":" + port})))
|
||||
|
||||
proxy := httptest.NewServer(handler(held))
|
||||
defer proxy.Close()
|
||||
@@ -120,16 +250,17 @@ func TestTheProxyReachesTheWorkloadAndNamesWhatItServes(t *testing.T) {
|
||||
// nothing fails more visibly than a stale grant, which is exactly why it must not survive.
|
||||
func TestWithdrawingARouteStopsServingIt(t *testing.T) {
|
||||
held := newTable()
|
||||
held.set(map[string]string{
|
||||
initial := plain(map[string]string{
|
||||
"going.example": "http://a.internal:80",
|
||||
"staying.example": "http://b.internal:80",
|
||||
})
|
||||
held.set(map[string]string{"staying.example": "http://b.internal:80"})
|
||||
held.set(initial, allPublic(initial))
|
||||
held.set(plain(map[string]string{"staying.example": "http://b.internal:80"}), allPublic(plain(map[string]string{"staying.example": "http://b.internal:80"})))
|
||||
|
||||
if _, still := held.find("going.example"); still {
|
||||
if _, still := held.find("going.example", "/"); still {
|
||||
t.Fatal("a route whose module was unassigned is still served")
|
||||
}
|
||||
if _, kept := held.find("staying.example"); !kept {
|
||||
if _, kept := held.find("staying.example", "/"); !kept {
|
||||
t.Fatal("withdrawing one route took another with it")
|
||||
}
|
||||
}
|
||||
@@ -137,8 +268,8 @@ func TestWithdrawingARouteStopsServingIt(t *testing.T) {
|
||||
// A Host header carries a port and the name does not.
|
||||
func TestARequestNamingAPortStillFindsItsRoute(t *testing.T) {
|
||||
held := newTable()
|
||||
held.set(map[string]string{"app.example": "http://a.internal:8080"})
|
||||
if _, found := held.find("app.example:8080"); !found {
|
||||
held.set(plain(map[string]string{"app.example": "http://a.internal:8080"}), allPublic(plain(map[string]string{"app.example": "http://a.internal:8080"})))
|
||||
if _, found := held.find("app.example:8080", "/"); !found {
|
||||
t.Fatal("a request to app.example:8080 did not find the route for app.example")
|
||||
}
|
||||
}
|
||||
@@ -168,7 +299,7 @@ func TestTheIssuerIsStagingUnlessNamed(t *testing.T) {
|
||||
// rate limit — and the proxy would look healthy throughout.
|
||||
func TestNoCertificateIsAskedForOnAnUnroutedName(t *testing.T) {
|
||||
held := newTable()
|
||||
held.set(map[string]string{"photos.example": "http://127.0.0.1:8080"})
|
||||
held.set(plain(map[string]string{"photos.example": "http://127.0.0.1:8080"}), allPublic(plain(map[string]string{"photos.example": "http://127.0.0.1:8080"})))
|
||||
policy := onlyWhatTheMeshSaid(held)
|
||||
|
||||
if err := policy(context.Background(), "photos.example"); err != nil {
|
||||
@@ -181,18 +312,179 @@ func TestNoCertificateIsAskedForOnAnUnroutedName(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A certificate is asked for on a route's public name, never on its internal-network alias — no
|
||||
// public CA can validate a private name, and asking anyway would only spend the account's rate
|
||||
// limit on an order that can never succeed.
|
||||
func TestNoCertificateIsAskedForOnAnInternalAlias(t *testing.T) {
|
||||
routes, public, err := routesFrom(write(t, `{"given":[
|
||||
{"from":"app","node":"anchor","at":"anchor.internal",
|
||||
"values":{"name":"app.example","internal-name":"app.anchor.internal","port":8080}}
|
||||
]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held := newTable()
|
||||
held.set(routes, public)
|
||||
policy := onlyWhatTheMeshSaid(held)
|
||||
|
||||
if err := policy(context.Background(), "app.example"); err != nil {
|
||||
t.Errorf("the route's public name was refused a certificate: %v", err)
|
||||
}
|
||||
if err := policy(context.Background(), "app.anchor.internal"); err == nil {
|
||||
t.Error("a certificate was ordered for the internal alias, which no public CA can validate")
|
||||
}
|
||||
}
|
||||
|
||||
// A certificate is asked of the *internal* authority only for a name that is routed here and is
|
||||
// not a route's own public name — the internal-network alias, never the route it accompanies.
|
||||
func TestTheInternalAuthorityOnlyCertifiesInternalOnlyAliases(t *testing.T) {
|
||||
routes, public, err := routesFrom(write(t, `{"given":[
|
||||
{"from":"app","node":"anchor","at":"anchor.internal",
|
||||
"values":{"name":"app.example","internal-name":"app.anchor.internal","port":8080}}
|
||||
]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held := newTable()
|
||||
held.set(routes, public)
|
||||
policy := onlyInternalNamesTheMeshSaid(held)
|
||||
|
||||
if err := policy(context.Background(), "app.anchor.internal"); err != nil {
|
||||
t.Errorf("the internal alias was refused by its own authority: %v", err)
|
||||
}
|
||||
if err := policy(context.Background(), "app.example"); err == nil {
|
||||
t.Error("the internal authority certified a route's public name, which the public authority already covers")
|
||||
}
|
||||
if err := policy(context.Background(), "unrouted.internal"); err == nil {
|
||||
t.Error("the internal authority certified a name nobody routed here")
|
||||
}
|
||||
}
|
||||
|
||||
// A route withdrawn stops being certifiable, without the proxy restarting.
|
||||
func TestWithdrawingARouteWithdrawsItsCertificate(t *testing.T) {
|
||||
held := newTable()
|
||||
held.set(map[string]string{"photos.example": "http://127.0.0.1:8080"})
|
||||
held.set(plain(map[string]string{"photos.example": "http://127.0.0.1:8080"}), allPublic(plain(map[string]string{"photos.example": "http://127.0.0.1:8080"})))
|
||||
policy := onlyWhatTheMeshSaid(held)
|
||||
if err := policy(context.Background(), "photos.example"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
held.set(nil)
|
||||
held.set(nil, nil)
|
||||
if err := policy(context.Background(), "photos.example"); err == nil {
|
||||
t.Fatal("a withdrawn route can still order certificates, so the policy read a copy taken " +
|
||||
"once rather than what is served now")
|
||||
}
|
||||
}
|
||||
|
||||
// A route may say the largest body it carries, and the proxy holds requests to it.
|
||||
//
|
||||
// The registry is why: image layers arrive as single requests of gigabytes, and a proxy's own
|
||||
// default refuses them long before the workload is reached. It is configuration beside `insecure`,
|
||||
// not a fifth policy — novox/hq ADR 0108 closed that set at four.
|
||||
func TestARouteMayLimitTheBodyItCarries(t *testing.T) {
|
||||
routes, _, err := routesFrom(write(t, `{"given":[
|
||||
{"from":"registry","node":"anchor","at":"anchor.internal",
|
||||
"values":{"name":"images.example","port":5000,"max-request-body":21474836480}}
|
||||
]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rules := routes["images.example"]
|
||||
if len(rules) != 1 {
|
||||
t.Fatalf("the route is not served once: %v", routes)
|
||||
}
|
||||
if rules[0].maxRequestBody != 21474836480 {
|
||||
t.Fatalf("the limit did not survive the contribution: %d", rules[0].maxRequestBody)
|
||||
}
|
||||
}
|
||||
|
||||
// Saying nothing leaves the route unlimited, which is what every route already got.
|
||||
func TestARouteThatSaysNothingCarriesAnySize(t *testing.T) {
|
||||
routes, _, err := routesFrom(write(t, `{"given":[
|
||||
{"from":"app","node":"anchor","at":"anchor.internal","values":{"name":"app.example","port":8080}}
|
||||
]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := routes["app.example"][0].maxRequestBody; got != 0 {
|
||||
t.Fatalf("a route that asked for no limit got one: %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A limit that is not a whole positive number of bytes takes the route with it. Serving it without
|
||||
// the limit would carry exactly what the module said not to carry, and report success doing it.
|
||||
func TestARouteWithAnUnusableLimitIsSkipped(t *testing.T) {
|
||||
for _, asked := range []string{`"lots"`, `-1`, `0`, `1.5`} {
|
||||
routes, _, err := routesFrom(write(t, `{"given":[
|
||||
{"from":"registry","node":"anchor","at":"anchor.internal",
|
||||
"values":{"name":"images.example","port":5000,"max-request-body":`+asked+`}}
|
||||
]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(routes["images.example"]) != 0 {
|
||||
t.Errorf("a route asking for a max-request-body of %s was served anyway: %v", asked, routes)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A request larger than the route carries is refused by the proxy, with the limit named, and the
|
||||
// workload never sees it. A request within it is proxied normally.
|
||||
func TestABodyOverTheLimitIsRefusedAndOneUnderItIsCarried(t *testing.T) {
|
||||
var reached int
|
||||
workload := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
reached++
|
||||
fmt.Fprintf(w, "carried %d bytes", len(body))
|
||||
}))
|
||||
defer workload.Close()
|
||||
|
||||
at := strings.TrimPrefix(workload.URL, "http://")
|
||||
host, port, _ := strings.Cut(at, ":")
|
||||
routes, _, err := routesFrom(write(t, `{"given":[
|
||||
{"from":"registry","node":"anchor","at":"`+host+`",
|
||||
"values":{"name":"images.example","port":`+port+`,"max-request-body":8}}
|
||||
]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held := newTable()
|
||||
held.set(routes, map[string]bool{})
|
||||
proxy := httptest.NewServer(handler(held))
|
||||
defer proxy.Close()
|
||||
|
||||
over, err := post(proxy.URL, "images.example", "123456789")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer over.Body.Close()
|
||||
if over.StatusCode != http.StatusRequestEntityTooLarge {
|
||||
t.Fatalf("a body over the limit answered %d, not 413", over.StatusCode)
|
||||
}
|
||||
if reached != 0 {
|
||||
t.Fatalf("the workload was reached by a request the route said it would not carry")
|
||||
}
|
||||
|
||||
under, err := post(proxy.URL, "images.example", "1234")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer under.Body.Close()
|
||||
if under.StatusCode != http.StatusOK {
|
||||
t.Fatalf("a body within the limit answered %d, not 200", under.StatusCode)
|
||||
}
|
||||
if reached != 1 {
|
||||
t.Fatalf("the workload was not reached by a request within the limit")
|
||||
}
|
||||
}
|
||||
|
||||
// post sends a body to the proxy as the named route, since a route is found by the Host header.
|
||||
func post(url, host, body string) (*http.Response, error) {
|
||||
asked, err := http.NewRequest(http.MethodPost, url, strings.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
asked.Host = host
|
||||
asked.Header.Set("Content-Type", "application/octet-stream")
|
||||
return http.DefaultClient.Do(asked)
|
||||
}
|
||||
|
||||
@@ -1,19 +1,22 @@
|
||||
module github.com/novox/mesh-controller
|
||||
|
||||
go 1.25.0
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
github.com/jackc/pgx/v5 v5.10.0
|
||||
github.com/rabbitmq/amqp091-go v1.14.0
|
||||
golang.org/x/crypto v0.55.0
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
golang.org/x/crypto v0.57.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
golang.org/x/net v0.57.0 // indirect
|
||||
golang.org/x/sync v0.22.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
golang.org/x/text v0.41.0 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/net v0.58.0 // indirect
|
||||
golang.org/x/sync v0.23.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/text v0.42.0 // indirect
|
||||
)
|
||||
|
||||
@@ -9,27 +9,31 @@ github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0=
|
||||
github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rabbitmq/amqp091-go v1.14.0 h1:RSaT7aOKt/OrkVUyswPDW29lnRz9psuGmfZFBmLqLek=
|
||||
github.com/rabbitmq/amqp091-go v1.14.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
|
||||
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
|
||||
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
|
||||
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
|
||||
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
|
||||
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
|
||||
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
|
||||
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
|
||||
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Do the emitters and the consumers of a catalogue agree?
|
||||
//
|
||||
// **The check that was missing** (novox/hq 04-ISSUES/127). Every manifest was individually
|
||||
// well-formed and every derivation individually correct, and no cross-module subscription in the
|
||||
// mesh matched anything: a consumer's declaration derived into a namespace nobody publishes to.
|
||||
// Nothing failed, because a subscription that matches nothing is not an error — it is silence.
|
||||
//
|
||||
// The comparison has to be over the whole catalogue, because the two halves live in different
|
||||
// manifests, and it cannot simply demand that every consumed event have a live emitter: a module
|
||||
// may be installed long before the one whose events it wants. So the rule is narrower and still
|
||||
// catches this: **where the emitter is present, it must emit what the consumer asked for.**
|
||||
|
||||
// AConsumer is one module's interest in another's events, as this check needs it.
|
||||
type AConsumer struct {
|
||||
Module string
|
||||
Consumes []string
|
||||
}
|
||||
|
||||
// AnEmitter is one module's events.
|
||||
type AnEmitter struct {
|
||||
Module string
|
||||
Emits []string
|
||||
}
|
||||
|
||||
// Disagreements are the consumed events whose emitter is in the catalogue and does not emit them.
|
||||
//
|
||||
// Returned as sentences rather than as structs: every one of them is read by a person deciding
|
||||
// whether a manifest or a catalogue is wrong, and a pair of names without the reason is a puzzle.
|
||||
func Disagreements(emitters []AnEmitter, consumers []AConsumer, seats []DeclaredSeat) []string {
|
||||
emits := map[string]map[string]bool{}
|
||||
for _, e := range emitters {
|
||||
if emits[e.Module] == nil {
|
||||
emits[e.Module] = map[string]bool{}
|
||||
}
|
||||
for _, name := range e.Emits {
|
||||
emits[e.Module][name] = true
|
||||
}
|
||||
}
|
||||
// A seat's events are published by its holder under the seat's name, so a consumer naming the
|
||||
// seat is naming something real even though no module declares it as its own.
|
||||
for _, s := range seats {
|
||||
if len(s.Emits) == 0 {
|
||||
continue
|
||||
}
|
||||
if emits[s.Name] == nil {
|
||||
emits[s.Name] = map[string]bool{}
|
||||
}
|
||||
for _, name := range s.Emits {
|
||||
emits[s.Name][name] = true
|
||||
}
|
||||
}
|
||||
|
||||
var out []string
|
||||
for _, c := range consumers {
|
||||
for _, pattern := range c.Consumes {
|
||||
emitter, event, named := strings.Cut(pattern, ".")
|
||||
// Every event from everyone, or every event from one module: both are deliberate and
|
||||
// neither names a particular event to check.
|
||||
if !named || emitter == "*" || emitter == catalogueTheRest || event == catalogueTheRest {
|
||||
continue
|
||||
}
|
||||
known, present := emits[emitter]
|
||||
if !present {
|
||||
// Not installed here, which is ordinary: a module lives in its own repository and
|
||||
// may be registered later. Nothing to compare, so nothing to say.
|
||||
continue
|
||||
}
|
||||
if matchesAny(event, known) {
|
||||
continue
|
||||
}
|
||||
out = append(out, fmt.Sprintf(
|
||||
"%s consumes %q and %s emits %s — so that subscription would match nothing, and "+
|
||||
"nothing would report it",
|
||||
c.Module, pattern, emitter, listOf(known)))
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// matchesAny says whether one of an emitter's event names satisfies a consumer's pattern.
|
||||
func matchesAny(pattern string, emitted map[string]bool) bool {
|
||||
want := strings.Split(pattern, ".")
|
||||
for name := range emitted {
|
||||
if matches(want, strings.Split(name, ".")) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func matches(pattern, name []string) bool {
|
||||
for i, part := range pattern {
|
||||
if part == catalogueTheRest {
|
||||
return i < len(name)
|
||||
}
|
||||
if i >= len(name) {
|
||||
return false
|
||||
}
|
||||
if part != "*" && part != name[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(pattern) == len(name)
|
||||
}
|
||||
|
||||
func listOf(names map[string]bool) string {
|
||||
if len(names) == 0 {
|
||||
return "nothing"
|
||||
}
|
||||
out := make([]string, 0, len(names))
|
||||
for n := range names {
|
||||
out = append(out, n)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return strings.Join(out, ", ")
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// **Do the catalogue's emitters and consumers agree?**
|
||||
//
|
||||
// This is the check whose absence let issue 127 stand: every manifest was individually well-formed,
|
||||
// every derivation individually correct, and no cross-module subscription in the mesh matched
|
||||
// anything. A subscription that matches nothing is not an error — it is silence — so nothing
|
||||
// anywhere reported it.
|
||||
//
|
||||
// It compares what one manifest asks to hear against what another says it emits. It cannot demand
|
||||
// that every consumed event have a live emitter, because a module lives in its own repository and
|
||||
// may be registered long before the one whose events it wants. Where the emitter *is* here, it must
|
||||
// emit what the consumer asked for.
|
||||
func TestTheCataloguesEmittersAndConsumersAgree(t *testing.T) {
|
||||
emitters, consumers, seats := theCataloguesEvents(t)
|
||||
|
||||
if bad := Disagreements(emitters, consumers, seats); len(bad) > 0 {
|
||||
t.Fatalf("%d subscription(s) in the catalogue would match nothing:\n %s",
|
||||
len(bad), strings.Join(bad, "\n "))
|
||||
}
|
||||
}
|
||||
|
||||
// And the check itself catches the thing it exists for, so it cannot pass by doing nothing.
|
||||
func TestTheAgreementCheckCatchesASubscriptionThatMatchesNothing(t *testing.T) {
|
||||
bad := Disagreements(
|
||||
[]AnEmitter{{Module: "builder", Emits: []string{"built"}}},
|
||||
[]AConsumer{{Module: "mesh-catalog", Consumes: []string{"builder.finished"}}},
|
||||
nil)
|
||||
if len(bad) != 1 {
|
||||
t.Fatalf("a consumer asking for an event its emitter does not emit was not caught: %v", bad)
|
||||
}
|
||||
if !strings.Contains(bad[0], "builder.finished") || !strings.Contains(bad[0], "built") {
|
||||
t.Fatalf("the report names neither what was asked for nor what is emitted: %s", bad[0])
|
||||
}
|
||||
|
||||
// A module that is not here is not a disagreement: it may be registered later.
|
||||
if bad := Disagreements(nil,
|
||||
[]AConsumer{{Module: "plex", Consumes: []string{"sonarr.download.completed"}}}, nil); len(bad) != 0 {
|
||||
t.Fatalf("a consumer whose emitter is not installed was reported: %v", bad)
|
||||
}
|
||||
|
||||
// A wildcard over emitters is deliberate and names no particular event to check.
|
||||
if bad := Disagreements([]AnEmitter{{Module: "sonarr", Emits: []string{"download.completed"}}},
|
||||
[]AConsumer{{Module: "plex", Consumes: []string{"*.download.completed"}}}, nil); len(bad) != 0 {
|
||||
t.Fatalf("a wildcard over emitters was reported: %v", bad)
|
||||
}
|
||||
|
||||
// A consumer of a role's event whose role does not emit it is caught, which is what stops the
|
||||
// catalogue check above from passing by knowing nothing about roles.
|
||||
if bad := Disagreements(nil,
|
||||
[]AConsumer{{Module: "mesh-catalog", Consumes: []string{"mesh-build-machine.finished"}}},
|
||||
[]DeclaredSeat{{Name: "mesh-build-machine", Emits: []string{"built"}}}); len(bad) != 1 {
|
||||
t.Fatalf("a consumer of a role event the role does not emit was not caught: %v", bad)
|
||||
}
|
||||
|
||||
// An event published under a seat's name is real even though no module declares it as its own.
|
||||
if bad := Disagreements(nil,
|
||||
[]AConsumer{{Module: "watcher", Consumes: []string{"mesh-artifact-store.image.pushed"}}},
|
||||
[]DeclaredSeat{{Name: "the-artifact-store", Emits: []string{"image.pushed"}}}); len(bad) != 0 {
|
||||
t.Fatalf("an event a seat emits was reported as matching nothing: %v", bad)
|
||||
}
|
||||
}
|
||||
|
||||
func theCataloguesEvents(t *testing.T) ([]AnEmitter, []AConsumer, []DeclaredSeat) {
|
||||
t.Helper()
|
||||
root := filepath.Join("..", "..", "..", "mesh-catalog", "modules")
|
||||
entries, err := os.ReadDir(root)
|
||||
if err != nil {
|
||||
t.Skipf("catalogue sibling not present: %v", err)
|
||||
}
|
||||
var emitters []AnEmitter
|
||||
var consumers []AConsumer
|
||||
// The mesh's own roles, which emit under the seat's name rather than any module's (novox/hq
|
||||
// ADR 0121). Without these the check skips every consumer of a role's event as "the emitter is
|
||||
// not installed" — which is how it passed vacuously the first time one existed.
|
||||
var seats []DeclaredSeat
|
||||
for _, own := range catalogue.SeatsWithAProtocol() {
|
||||
seats = append(seats, DeclaredSeat{Name: own.Name, Accepts: own.Accepts, Emits: own.Emits})
|
||||
}
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() {
|
||||
continue
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(root, e.Name(), "module.json"))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Emits []string `json:"emits"`
|
||||
Consumes []string `json:"consumes"`
|
||||
Seats []struct {
|
||||
Name string `json:"name"`
|
||||
Emits []string `json:"emits"`
|
||||
} `json:"seats"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatalf("%s: %v", e.Name(), err)
|
||||
}
|
||||
if len(m.Emits) > 0 {
|
||||
emitters = append(emitters, AnEmitter{Module: m.Module, Emits: m.Emits})
|
||||
}
|
||||
if len(m.Consumes) > 0 {
|
||||
consumers = append(consumers, AConsumer{Module: m.Module, Consumes: m.Consumes})
|
||||
}
|
||||
for _, s := range m.Seats {
|
||||
seats = append(seats, DeclaredSeat{Name: s.Name, Emits: s.Emits})
|
||||
}
|
||||
}
|
||||
if len(emitters) == 0 {
|
||||
t.Skip("no manifests found beside this checkout")
|
||||
}
|
||||
return emitters, consumers, seats
|
||||
}
|
||||
|
||||
// **Do the derived subjects meet, not just the names?**
|
||||
//
|
||||
// The check above compares what a consumer asks for against what an emitter says it emits, by name. It
|
||||
// passed while the catalogue's subscription pointed at `mesh.mod.mesh-build-machine.event.built` — a
|
||||
// module namespace for a role's event, which no emitter owns. The names agreed; the subjects did not,
|
||||
// and the graph stayed empty.
|
||||
//
|
||||
// So this compares the thing that actually has to match: the subject a consumer subscribes against the
|
||||
// subject an emitter publishes. It is the last place the two halves can be held together, because
|
||||
// after this the server is the only thing that knows and it says nothing — a subscription that matches
|
||||
// nothing is silence.
|
||||
func TestTheCataloguesDerivedSubjectsMeet(t *testing.T) {
|
||||
emitters, consumers, seats := theCataloguesEvents(t)
|
||||
|
||||
// Every subject something publishes: a module's own events, and the events of every role.
|
||||
published := map[string]bool{}
|
||||
for _, e := range emitters {
|
||||
for _, name := range e.Emits {
|
||||
published["mesh.mod."+e.Module+".event."+name] = true
|
||||
}
|
||||
}
|
||||
for _, s := range seats {
|
||||
for _, name := range s.Emits {
|
||||
published["mesh.seat."+s.Name+".event."+name] = true
|
||||
}
|
||||
}
|
||||
|
||||
byName := map[string]DeclaredSeat{}
|
||||
for _, s := range seats {
|
||||
byName[s.Name] = s
|
||||
}
|
||||
|
||||
var lonely []string
|
||||
for _, c := range consumers {
|
||||
principal := Principal{Kind: KindModule, Node: "one", Module: c.Module, PasswordHash: "x"}
|
||||
for _, want := range c.Consumes {
|
||||
emitter, event, named := strings.Cut(want, ".")
|
||||
if named {
|
||||
if s, isASeat := byName[emitter]; isASeat {
|
||||
principal.Watches = append(principal.Watches,
|
||||
Seat{Name: s.Name, Emits: []string{event}})
|
||||
continue
|
||||
}
|
||||
}
|
||||
principal.Consumes = append(principal.Consumes, want)
|
||||
}
|
||||
perms, err := PermissionsFor(principal)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", c.Module, err)
|
||||
}
|
||||
for _, subject := range perms.Subscribe {
|
||||
if !strings.Contains(subject, ".event.") {
|
||||
continue
|
||||
}
|
||||
if reaches(subject, published) {
|
||||
continue
|
||||
}
|
||||
// A wildcard over emitters reaches whatever arrives later, and an emitter that is not
|
||||
// installed is ordinary — both are already excused by the check above, so only a subject
|
||||
// that can never match anything gets here.
|
||||
if strings.Contains(subject, "*") || strings.Contains(subject, ">") {
|
||||
continue
|
||||
}
|
||||
lonely = append(lonely, c.Module+" subscribes "+subject+", which nothing publishes")
|
||||
}
|
||||
}
|
||||
if len(lonely) > 0 {
|
||||
sort.Strings(lonely)
|
||||
t.Fatalf("%d subscription(s) derive to a subject no emitter owns:\n %s",
|
||||
len(lonely), strings.Join(lonely, "\n "))
|
||||
}
|
||||
}
|
||||
|
||||
// And it catches the thing it exists for: a role's event read as a module's.
|
||||
func TestTheDerivedSubjectCheckCatchesARolesEventReadAsAModules(t *testing.T) {
|
||||
published := map[string]bool{"mesh.seat.mesh-build-machine.event.built": true}
|
||||
// What the derivation produced before a consumed seat name was resolved as one.
|
||||
if reaches("mesh.mod.mesh-build-machine.event.built", published) {
|
||||
t.Fatal("a module namespace was treated as reaching a role's event, which is the bug")
|
||||
}
|
||||
// And the corrected one does reach it.
|
||||
if !reaches("mesh.seat.mesh-build-machine.event.built", published) {
|
||||
t.Fatal("the role's own subject does not reach the role's event")
|
||||
}
|
||||
}
|
||||
|
||||
// reaches says whether a subscribed subject admits any published one.
|
||||
func reaches(subject string, published map[string]bool) bool {
|
||||
for p := range published {
|
||||
if admitsSubject(strings.Split(subject, "."), strings.Split(p, ".")) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func admitsSubject(pattern, subject []string) bool {
|
||||
for i, token := range pattern {
|
||||
if token == ">" {
|
||||
return i < len(subject)
|
||||
}
|
||||
if i >= len(subject) {
|
||||
return false
|
||||
}
|
||||
if token != "*" && token != subject[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(pattern) == len(subject)
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
package broker_test
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The names are written twice, so a test keeps them agreeing.
|
||||
//
|
||||
// `link` imports `broker`, so `broker` cannot import `link` — the queue and exchange names
|
||||
// therefore exist in both. A scoped account naming a queue nothing publishes to produces a
|
||||
// builder that takes no work and says nothing about why, which is the worst kind of silence.
|
||||
//
|
||||
// An external test package, because it may import both without either importing the other.
|
||||
func TestTheNamesTheBrokerScopesAreTheNamesTheLinkUses(t *testing.T) {
|
||||
for _, agreed := range []struct {
|
||||
what string
|
||||
scoped string
|
||||
actually string
|
||||
}{
|
||||
{"the build queue", broker.BuildQueueName, link.BuildQueue},
|
||||
{"the exchange", broker.ExchangeName, link.Exchange},
|
||||
{"a node's queue", broker.QueueFor("somewhere"), link.QueueFor("somewhere")},
|
||||
} {
|
||||
if agreed.scoped != agreed.actually {
|
||||
t.Errorf("%s: the broker scopes %q and the link uses %q",
|
||||
agreed.what, agreed.scoped, agreed.actually)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestABuilderMayWriteToAReplyQueueAndReadNoNodesDeclarations(t *testing.T) {
|
||||
// The scoping, checked as patterns rather than by connecting: what it may write must include
|
||||
// the queue an asker actually waits on, and what it may read must not include any node's.
|
||||
//
|
||||
// This exists because the first version scoped writes to `amq.gen-*` — the name one broker
|
||||
// happens to generate — and the builder built, could not answer, and the connection simply
|
||||
// closed saying only "not allowed to publish to exchange \'\'". Answering through the
|
||||
// exchange is what removed the need for any of that.
|
||||
write := regexp.MustCompile("^" + regexp.QuoteMeta(broker.ExchangeName) + "$")
|
||||
if !write.MatchString(broker.ExchangeName) {
|
||||
t.Error("a builder may not write to the exchange, so it can take work and never answer")
|
||||
}
|
||||
// And not the default exchange, where permission is per exchange rather than per queue — a
|
||||
// builder allowed to use it could publish into any node's queue.
|
||||
//
|
||||
// Confirmed against a real broker as well, and worth recording how that nearly went wrong:
|
||||
// an unconfirmed publish is asynchronous, so a refusal arrives as a channel close afterwards
|
||||
// and a naive check reports success. With publisher confirms the broker's refusal is
|
||||
// immediate. **A negative security assertion made against an asynchronous call is not an
|
||||
// assertion.**
|
||||
if write.MatchString("") {
|
||||
t.Error("a builder may publish to the default exchange, and so into any node's queue")
|
||||
}
|
||||
|
||||
read := regexp.MustCompile("^" + regexp.QuoteMeta(broker.BuildQueueName) + "$")
|
||||
if !read.MatchString(broker.BuildQueueName) {
|
||||
t.Error("a builder may not read the build queue")
|
||||
}
|
||||
if read.MatchString(link.QueueFor("someone-else")) {
|
||||
// A build machine is not a node, and a node's queue carries its declarations.
|
||||
t.Error("a builder may read another machine's declarations")
|
||||
}
|
||||
}
|
||||
@@ -40,8 +40,12 @@ type Broker struct {
|
||||
var ErrNotConfigured = errors.New("this control plane has not been told about its broker")
|
||||
|
||||
// FromEnvironment reads the two settings, if they are there.
|
||||
//
|
||||
// The address's port follows MESH_BROKER_ADDRESS_PORT when the node's settings moved the bus
|
||||
// (novox/hq 04-ISSUES/102): the address genesis wrote is a public name and the port genesis
|
||||
// chose, and only the port is the node's to move.
|
||||
func FromEnvironment() (Broker, error) {
|
||||
address, err := envfile.Value(AddressVar)
|
||||
address, err := envfile.Placed(AddressVar)
|
||||
if err != nil {
|
||||
return Broker{}, err
|
||||
}
|
||||
|
||||
@@ -178,3 +178,19 @@ func TestBothTogetherGiveABroker(t *testing.T) {
|
||||
t.Errorf("got %+v", known)
|
||||
}
|
||||
}
|
||||
|
||||
// The node moved the bus, and the address a token carries follows (novox/hq 04-ISSUES/102).
|
||||
func TestTheAddressPortFollowsThePortTwin(t *testing.T) {
|
||||
t.Setenv(AddressVar, "broker.example:5671")
|
||||
t.Setenv(AddressVar+"_FILE", "")
|
||||
path, _ := writeCertificate(t)
|
||||
t.Setenv(CertificateVar, path)
|
||||
t.Setenv(AddressVar+"_PORT", "5679")
|
||||
b, err := FromEnvironment()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b.Address != "broker.example:5679" {
|
||||
t.Fatalf("the address is %q; the node put the bus on 5679", b.Address)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Streams and consumers derived from what modules declare.
|
||||
//
|
||||
// The mesh's own four exist before any module does (streams.go). Everything here is the other
|
||||
// half: a seat's stream comes into being when the module declaring it is **registered**, and a
|
||||
// consumer when a module is **assigned** — which is why ADR 0116's task 1.4 had to be narrowed to
|
||||
// the foundation set. Neither has happened at genesis.
|
||||
//
|
||||
// All of it is a pure function of declarations. The controller is still the only writer; this is
|
||||
// only what it writes.
|
||||
|
||||
// A Consumer is a durable subscription the controller creates on a module's behalf. A module
|
||||
// declares what it reacts to, never how delivery works, so it does not name these and cannot
|
||||
// misconfigure them.
|
||||
type Consumer struct {
|
||||
Name string
|
||||
Stream string
|
||||
// Filters are the subjects this consumer receives. One consumer per module with several
|
||||
// filters, rather than one per consumed event: its ack subject is derived from its name, and
|
||||
// a module with five consumers would need five ack permissions to ack its own deliveries.
|
||||
Filters []string
|
||||
// Queue is the queue group, set for a seat's worker so that "exactly one holder" survives a
|
||||
// seat later being relaxed to several. Authority and delivery are kept separate on purpose.
|
||||
Queue string
|
||||
// Push asks the server to deliver to a subject rather than wait to be pulled.
|
||||
//
|
||||
// For the mesh's own consumer, where the controller wants every message to arrive in the one
|
||||
// loop it already runs: pulling would mean a second goroutine fetching batches and handing
|
||||
// them over, and a loop that acts on one message at a time is the property the store window
|
||||
// depends on. A queue group implies this, because a group has nothing to pull from.
|
||||
Push bool
|
||||
// AckWaitSeconds before an unacknowledged delivery is redelivered.
|
||||
AckWaitSeconds int
|
||||
// MaxDeliver before the message is dead-lettered; zero for the mesh's default.
|
||||
MaxDeliver int
|
||||
Why string
|
||||
}
|
||||
|
||||
// seatStreamName is the stream holding a seat's inbound work. Named after the seat rather than
|
||||
// the module holding it, because the holder can change and the queued work must not care — which
|
||||
// is the whole reason a caller addresses a seat instead of a module.
|
||||
func seatStreamName(seat string) string { return "SEAT_" + upperSnake(seat) }
|
||||
|
||||
// SeatStreams is one work queue per declared seat, created when the declaring module is
|
||||
// registered rather than when it is assigned.
|
||||
//
|
||||
// **The stream exists before anyone holds the seat, and that is the point.** Work queues until a
|
||||
// holder appears, so installing the telegram module a week after something started sending to it
|
||||
// flushes the backlog instead of having lost it. A stream created at assignment would make "the
|
||||
// holder is not here yet" mean "your messages are gone".
|
||||
func SeatStreams(seats []DeclaredSeat) []Stream {
|
||||
sorted := append([]DeclaredSeat(nil), seats...)
|
||||
sort.Slice(sorted, func(i, j int) bool { return sorted[i].Name < sorted[j].Name })
|
||||
|
||||
var out []Stream
|
||||
for _, s := range sorted {
|
||||
if len(s.Accepts) == 0 {
|
||||
// A seat that only emits and serves needs no stream: its events ride EVENTS and its
|
||||
// tools are core request/reply, which is never persisted.
|
||||
continue
|
||||
}
|
||||
retain := s.RetainSeconds
|
||||
if retain == 0 {
|
||||
retain = 7 * 24 * 60 * 60
|
||||
}
|
||||
out = append(out, Stream{
|
||||
Name: seatStreamName(s.Name),
|
||||
Subjects: []string{"mesh.seat." + s.Name + ".accept.>"},
|
||||
Retention: RetentionWorkQueue,
|
||||
MaxAge: retain,
|
||||
Why: fmt.Sprintf("work submitted to the %s seat; one holder consumes it, and it "+
|
||||
"queues while nobody does", s.Name),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// A DeclaredSeat is a seat as the catalogue knows it. Mirrored here rather than imported so this
|
||||
// package stays free of the catalogue's own types — the same reason the host mirrors the
|
||||
// contracts instead of importing the sdk.
|
||||
type DeclaredSeat struct {
|
||||
Name string
|
||||
Accepts []string
|
||||
// Emits are the verbs the seat's holder publishes under the seat's own name. An event about a
|
||||
// role belongs here rather than in the holder's namespace, because the name then outlives
|
||||
// whoever fills it (novox/hq ADR 0121, 04-ISSUES/127).
|
||||
Emits []string
|
||||
// Serves are the verbs the holder answers, request and reply.
|
||||
Serves []string
|
||||
RetainSeconds int
|
||||
}
|
||||
|
||||
// ConsumerFor is the durable consumer a module's declarations imply, or false when it subscribes
|
||||
// to nothing and needs none.
|
||||
//
|
||||
// One per module, with every consumed subject as a filter, because its ack permission is derived
|
||||
// from its name: a module with a consumer per event would need an ack permission per consumer,
|
||||
// and the permission list would stop being derivable from the declaration.
|
||||
func ConsumerFor(p Principal) (Consumer, bool) {
|
||||
// A module that reacts to anything — a module's events or a role's (novox/hq ADR 0121). Watching
|
||||
// a role was missing here, so the one module that does it got no consumer at all: it started,
|
||||
// connected, and its graph stayed empty with nothing anywhere reporting why.
|
||||
if p.Kind != KindModule || (len(p.Consumes) == 0 && len(p.Watches) == 0) {
|
||||
return Consumer{}, false
|
||||
}
|
||||
perms, err := PermissionsFor(p)
|
||||
if err != nil {
|
||||
return Consumer{}, false
|
||||
}
|
||||
// Events, wherever they live: a module's own namespace, and the namespace of any role it watches
|
||||
// (novox/hq ADR 0121). Tool subjects and inboxes are subscribed directly and are not a consumer's
|
||||
// business, which is why this is a filter and not the whole list.
|
||||
var filters []string
|
||||
for _, s := range perms.Subscribe {
|
||||
if strings.Contains(s, ".event.") {
|
||||
filters = append(filters, s)
|
||||
}
|
||||
}
|
||||
if len(filters) == 0 {
|
||||
return Consumer{}, false
|
||||
}
|
||||
sort.Strings(filters)
|
||||
return Consumer{
|
||||
Name: consumerDurable(p),
|
||||
Stream: consumerStream(p),
|
||||
Filters: filters,
|
||||
AckWaitSeconds: 30,
|
||||
MaxDeliver: 5,
|
||||
Why: "what " + p.Module + " declared it consumes; after max-deliver it dead-letters",
|
||||
}, true
|
||||
}
|
||||
|
||||
// HolderConsumerFor is the worker a seat's holder gets on that seat's work queue.
|
||||
//
|
||||
// **A queue group even though the seat guarantees one holder.** The seat is *authority* — who may
|
||||
// be the telegram sender — and the queue group is *delivery*. Tie delivery to the seat and the
|
||||
// day somebody allows two holders for throughput, every message is processed twice with nothing
|
||||
// reporting it. Kept separate, relaxing one changes nothing about the other.
|
||||
func HolderConsumerFor(node, module string, seat DeclaredSeat) (Consumer, bool) {
|
||||
if len(seat.Accepts) == 0 {
|
||||
return Consumer{}, false
|
||||
}
|
||||
return Consumer{
|
||||
Name: "SEAT_" + upperSnake(seat.Name) + "_worker",
|
||||
Stream: seatStreamName(seat.Name),
|
||||
Filters: []string{"mesh.seat." + seat.Name + ".accept.>"},
|
||||
Queue: "holders",
|
||||
AckWaitSeconds: 60,
|
||||
MaxDeliver: 5,
|
||||
Why: fmt.Sprintf("%s on %s holds %s; it acknowledges after the work is done, so a "+
|
||||
"crash mid-work redelivers rather than loses", module, node, seat.Name),
|
||||
}, true
|
||||
}
|
||||
|
||||
// NodeConsumer is the durable consumer a node reads its own declaration through.
|
||||
//
|
||||
// **Derived from a node existing, and created by the controller, because a host cannot create it.**
|
||||
// A host's account may subscribe its own declaration subject and publish its own ack subject, and
|
||||
// reaches no part of the JetStream API — which is correct (the controller is the only writer of
|
||||
// consumer definitions, design 25 §3) and means the consumer must be waiting before the host binds
|
||||
// to it. Named after the node, because the node's ack grant is `$JS.ACK.NODES.<node>.>` and a
|
||||
// consumer named anything else is one the host cannot acknowledge a delivery from.
|
||||
//
|
||||
// **No max-deliver, and a long ack wait.** A declaration is settled only after the node has applied
|
||||
// it and reported, which is minutes on a machine pulling images; and a declaration the mesh cannot
|
||||
// get a node to accept is not one to dead-letter, because the stream keeps only the newest per node
|
||||
// anyway — so there is exactly one message per node to redeliver, for as long as that node is away.
|
||||
func NodeConsumer(node string) Consumer {
|
||||
return Consumer{
|
||||
Name: node,
|
||||
Stream: "NODES",
|
||||
Filters: []string{"mesh.node." + node + ".declare"},
|
||||
Push: true,
|
||||
AckWaitSeconds: 300,
|
||||
Why: "how " + node + " hears what it should be; last-per-subject, so a node that was away " +
|
||||
"gets exactly the current declaration and nothing older",
|
||||
}
|
||||
}
|
||||
|
||||
// AssertNodeConsumers brings every known node's declaration consumer into being.
|
||||
//
|
||||
// Asserted on start as well as created at enrolment, for the reason the streams are: a mesh raised
|
||||
// from a restored backup, or one whose bus was recreated, has node records and no consumers, and a
|
||||
// node whose consumer is missing hears nothing while everything else about it looks correct.
|
||||
func AssertNodeConsumers(e Ensurer, nodes []string) error {
|
||||
for _, n := range nodes {
|
||||
if err := e.EnsureConsumer(NodeConsumer(n)); err != nil {
|
||||
return fmt.Errorf("asserting how %s hears its declaration: %w", n, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// AllOverlaps reports subject filters claimed by more than one stream, across the mesh's own and
|
||||
// every derived one.
|
||||
//
|
||||
// NATS refuses an overlapping stream rather than merging it (verified against nats-server 2.10:
|
||||
// "subjects overlap with an existing stream"), so this is not a subtle divergence — it is a
|
||||
// registration that fails. Catching it here names both streams, before a half-applied mesh does.
|
||||
func AllOverlaps(seats []DeclaredSeat) []string {
|
||||
all := append(MeshStreams(), SeatStreams(seats)...)
|
||||
seen := map[string]string{}
|
||||
var clashes []string
|
||||
for _, s := range all {
|
||||
for _, subject := range s.Subjects {
|
||||
if first, ok := seen[subject]; ok {
|
||||
clashes = append(clashes, fmt.Sprintf("%s and %s both claim %s", first, s.Name, subject))
|
||||
continue
|
||||
}
|
||||
seen[subject] = s.Name
|
||||
}
|
||||
}
|
||||
sort.Strings(clashes)
|
||||
return clashes
|
||||
}
|
||||
|
||||
// upperSnake makes a stream name from a seat name. NATS stream names may not contain a dot,
|
||||
// a space or a wildcard, and a hyphen is legal but reads badly beside the mesh's own.
|
||||
func upperSnake(s string) string {
|
||||
out := []rune(s)
|
||||
for i, r := range out {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z':
|
||||
out[i] = r - 32
|
||||
case r == '-' || r == '.':
|
||||
out[i] = '_'
|
||||
}
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func telegramSeat() DeclaredSeat {
|
||||
return DeclaredSeat{Name: "telegram-sender", Accepts: []string{"send"}}
|
||||
}
|
||||
|
||||
// The stream exists from registration, not assignment: work queues until a holder appears, so
|
||||
// installing the module a week later flushes the backlog rather than having lost it.
|
||||
func TestASeatGetsAWorkQueueOfItsOwn(t *testing.T) {
|
||||
got := SeatStreams([]DeclaredSeat{telegramSeat()})
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("expected one stream, got %d", len(got))
|
||||
}
|
||||
s := got[0]
|
||||
if s.Retention != RetentionWorkQueue {
|
||||
t.Fatalf("a seat's inbound queue retains as %q; one holder must take each message once", s.Retention)
|
||||
}
|
||||
if s.Subjects[0] != "mesh.seat.telegram-sender.accept.>" {
|
||||
t.Fatalf("filters on %v", s.Subjects)
|
||||
}
|
||||
}
|
||||
|
||||
// A seat that only emits and serves needs no stream: its events ride EVENTS and its tools are
|
||||
// core request/reply, which is never persisted.
|
||||
func TestASeatThatAcceptsNothingGetsNoStream(t *testing.T) {
|
||||
if got := SeatStreams([]DeclaredSeat{{Name: "announcer"}}); len(got) != 0 {
|
||||
t.Fatalf("a seat with no inbound work got %d stream(s)", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
// Retention belongs to whoever owns the namespace, and a seat owns its own.
|
||||
func TestASeatsRetentionIsItsOwn(t *testing.T) {
|
||||
s := SeatStreams([]DeclaredSeat{{Name: "slow", Accepts: []string{"work"}, RetainSeconds: 30 * 24 * 60 * 60}})
|
||||
if s[0].MaxAge != 30*24*60*60 {
|
||||
t.Fatalf("the seat's declared retention was not used: %d", s[0].MaxAge)
|
||||
}
|
||||
d := SeatStreams([]DeclaredSeat{telegramSeat()})
|
||||
if d[0].MaxAge == 0 {
|
||||
t.Fatal("a seat that declares no retention got an unbounded queue")
|
||||
}
|
||||
}
|
||||
|
||||
// NATS refuses an overlapping stream outright, so a clash here is a registration that fails.
|
||||
func TestNoDerivedStreamOverlapsTheMeshsOwn(t *testing.T) {
|
||||
seats := []DeclaredSeat{telegramSeat(), {Name: "licensing-master", Accepts: []string{"report"}}}
|
||||
if c := AllOverlaps(seats); len(c) != 0 {
|
||||
t.Fatalf("overlapping filters: %v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// One consumer per module, with every consumed subject as a filter — because its ack permission
|
||||
// is derived from its name, and a consumer per event would need an ack permission per consumer.
|
||||
func TestAModuleGetsOneConsumerCarryingEveryFilter(t *testing.T) {
|
||||
c, ok := ConsumerFor(Principal{Kind: KindModule, Node: "one", Module: "audit",
|
||||
Consumes: []string{"shop.order.placed", "billing.invoice.sent"}, PasswordHash: "x"})
|
||||
if !ok {
|
||||
t.Fatal("a module that consumes got no consumer")
|
||||
}
|
||||
if len(c.Filters) != 2 {
|
||||
t.Fatalf("expected both subjects as filters, got %v", c.Filters)
|
||||
}
|
||||
perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "audit",
|
||||
Consumes: []string{"shop.order.placed"}, PasswordHash: "x"})
|
||||
ack := "$JS.ACK." + c.Stream + "." + c.Name + ".>"
|
||||
found := false
|
||||
for _, p := range perms.Publish {
|
||||
if p == ack {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("the consumer is named %q but the ack permission is %v; a module could not ack "+
|
||||
"its own deliveries", c.Name, perms.Publish)
|
||||
}
|
||||
}
|
||||
|
||||
// A module that subscribes to nothing needs no consumer, and creating one would leave an object
|
||||
// nothing reads and everything has to maintain.
|
||||
func TestAModuleThatConsumesNothingGetsNoConsumer(t *testing.T) {
|
||||
if _, ok := ConsumerFor(Principal{Kind: KindModule, Node: "one", Module: "shop",
|
||||
Emits: []string{"order.placed"}, PasswordHash: "x"}); ok {
|
||||
t.Fatal("a pure emitter got a consumer")
|
||||
}
|
||||
}
|
||||
|
||||
// The seat is authority and the queue group is delivery. Tie them together and the day somebody
|
||||
// allows two holders, every message is processed twice with nothing reporting it.
|
||||
func TestAHoldersWorkerUsesAQueueGroupAnyway(t *testing.T) {
|
||||
c, ok := HolderConsumerFor("one", "telegram", telegramSeat())
|
||||
if !ok {
|
||||
t.Fatal("the holder of a seat with inbound work got no worker")
|
||||
}
|
||||
if c.Queue == "" {
|
||||
t.Fatal("the worker is not in a queue group, so a second holder would double-process")
|
||||
}
|
||||
if c.Stream != "SEAT_TELEGRAM_SENDER" {
|
||||
t.Fatalf("the worker reads %q, not the seat's own stream", c.Stream)
|
||||
}
|
||||
if c.MaxDeliver == 0 {
|
||||
t.Fatal("a failing worker would redeliver forever rather than dead-letter")
|
||||
}
|
||||
}
|
||||
|
||||
// The stream is named after the seat, not its holder: the holder can change and the queued work
|
||||
// must not care.
|
||||
func TestASeatsStreamIsNamedAfterTheSeat(t *testing.T) {
|
||||
name := seatStreamName("telegram-sender")
|
||||
if strings.Contains(name, "telegram-sender") {
|
||||
t.Fatalf("%q keeps characters a stream name may not hold", name)
|
||||
}
|
||||
if name != "SEAT_TELEGRAM_SENDER" {
|
||||
t.Fatalf("unexpected stream name %q", name)
|
||||
}
|
||||
}
|
||||
|
||||
// A node hears its declaration through a consumer only the controller can make.
|
||||
//
|
||||
// The three things that would each break it silently: a name other than the node's is one the host
|
||||
// cannot acknowledge a delivery from, because its ack grant is derived from the node's name; a
|
||||
// filter other than its own declaration subject is a node reading another's; and a pull consumer is
|
||||
// one the host cannot bind a channel to without creating something, which it has no authority for.
|
||||
func TestANodesDeclarationConsumerIsWhatItsOwnGrantAllows(t *testing.T) {
|
||||
c := NodeConsumer("anchor")
|
||||
if c.Name != "anchor" {
|
||||
t.Fatalf("named %q, so the node cannot ack from it: its grant is $JS.ACK.NODES.anchor.>", c.Name)
|
||||
}
|
||||
if c.Stream != "NODES" {
|
||||
t.Fatalf("on stream %q rather than the one declarations live in", c.Stream)
|
||||
}
|
||||
if len(c.Filters) != 1 || c.Filters[0] != "mesh.node.anchor.declare" {
|
||||
t.Fatalf("filters %v, which is not this node's own declaration and nothing else", c.Filters)
|
||||
}
|
||||
if !c.Push {
|
||||
t.Fatal("pulled, which a host cannot do: pulling needs the JetStream API and a host reaches none of it")
|
||||
}
|
||||
if c.MaxDeliver != 0 {
|
||||
t.Fatalf("max-deliver %d: a declaration a node has not taken yet is not one to dead-letter, "+
|
||||
"because the stream holds exactly one per node", c.MaxDeliver)
|
||||
}
|
||||
|
||||
// And the grant the node actually gets has to match, or none of the above matters.
|
||||
perms, err := PermissionsFor(Principal{Kind: KindNode, Node: "anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Without the ack grant every declaration a node receives is redelivered for ever; without the
|
||||
// subscribe grant its consumer delivers to nobody.
|
||||
has(t, perms.Publish, "$JS.ACK.NODES."+c.Name+".>")
|
||||
has(t, perms.Subscribe, c.Filters[0])
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
)
|
||||
|
||||
// **The first user list the installer carries must be the one the controller would compose.**
|
||||
//
|
||||
// At genesis there is no mesh to write the bus's user list, so the installer carries one: the
|
||||
// controller's own account, at a bootstrap password, the way the store is reached at
|
||||
// `postgres:bootstrap` (novox/hq design 25 §4, task 1.7). It is written by hand in a template and
|
||||
// derived in code here, which is two statements of one fact — so this compares them.
|
||||
//
|
||||
// Getting it wrong is the worst kind of silent: a controller whose carried permissions are narrower
|
||||
// than the ones it derives comes up, connects, and is refused on the first thing it tries, with an
|
||||
// authorisation error that names a subject and not the template that forgot it. And a mesh cannot be
|
||||
// raised twice to find out.
|
||||
func TestTheInstallersFirstUserListIsWhatTheControllerWouldCompose(t *testing.T) {
|
||||
accounts := theCarriedAccounts(t)
|
||||
|
||||
want, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
carriedPub := subjectsIn(accounts, "publish")
|
||||
carriedSub := subjectsIn(accounts, "subscribe")
|
||||
|
||||
if diff := missing(want.Publish, carriedPub); len(diff) > 0 {
|
||||
t.Errorf("the installer's user list does not let the controller publish %v — it would come up "+
|
||||
"and be refused on the first thing it tried", diff)
|
||||
}
|
||||
if diff := missing(want.Subscribe, carriedSub); len(diff) > 0 {
|
||||
t.Errorf("the installer's user list does not let the controller subscribe %v", diff)
|
||||
}
|
||||
// And nothing wider than what it derives, or genesis quietly grants a privilege the composition
|
||||
// takes away again on the first push.
|
||||
if diff := missing(carriedPub, want.Publish); len(diff) > 0 {
|
||||
t.Errorf("the installer's user list lets the controller publish %v, which it does not derive", diff)
|
||||
}
|
||||
if diff := missing(carriedSub, want.Subscribe); len(diff) > 0 {
|
||||
t.Errorf("the installer's user list lets the controller subscribe %v, which it does not derive", diff)
|
||||
}
|
||||
|
||||
// The credential is the bootstrap one and the hash really is of it, because a hash of something
|
||||
// else is a controller that cannot log in to the bus it was just given.
|
||||
hash := regexp.MustCompile(`\$2[aby]?\$[0-9]+\$[A-Za-z0-9./]{53}`).FindString(accounts)
|
||||
if hash == "" {
|
||||
t.Fatal("the installer's user list carries no password hash")
|
||||
}
|
||||
if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte("bootstrap")); err != nil {
|
||||
t.Fatalf("the carried hash does not verify the bootstrap credential the template also carries: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// theCarriedAccounts is the accounts file the installer's template writes at genesis.
|
||||
func theCarriedAccounts(t *testing.T) string {
|
||||
t.Helper()
|
||||
path := filepath.Join("..", "..", "..", "mesh-host", "examples", "foundation-first-node-nats.lock")
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Skipf("the host's checkout is not beside this one: %v", err)
|
||||
}
|
||||
// The template is JSON with line comments, which is how every one of them is written.
|
||||
var lines []string
|
||||
for _, l := range strings.Split(string(raw), "\n") {
|
||||
if !strings.HasPrefix(strings.TrimSpace(l), "//") {
|
||||
lines = append(lines, l)
|
||||
}
|
||||
}
|
||||
var bundle struct {
|
||||
Resources []map[string]any `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(strings.Join(lines, "\n")), &bundle); err != nil {
|
||||
t.Fatalf("the template is not readable: %v", err)
|
||||
}
|
||||
for _, r := range bundle.Resources {
|
||||
if r["id"] == "bus-accounts" {
|
||||
content, _ := r["content"].(string)
|
||||
if content == "" {
|
||||
t.Fatal("the template's accounts file is empty, so the bus would refuse every connection")
|
||||
}
|
||||
return content
|
||||
}
|
||||
}
|
||||
t.Fatal("the template carries no accounts file, so a mesh raised from it has a bus nobody may use")
|
||||
return ""
|
||||
}
|
||||
|
||||
// subjectsIn reads one allow-list out of a composed accounts file.
|
||||
func subjectsIn(accounts, which string) []string {
|
||||
found := regexp.MustCompile(which + `: \{ allow: \[([^\]]*)\]`).FindStringSubmatch(accounts)
|
||||
if len(found) != 2 {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, part := range strings.Split(found[1], ",") {
|
||||
if s := strings.Trim(strings.TrimSpace(part), `"`); s != "" {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// missing is what is in want and not in got.
|
||||
func missing(want, got []string) []string {
|
||||
have := map[string]bool{}
|
||||
for _, g := range got {
|
||||
have[g] = true
|
||||
}
|
||||
var out []string
|
||||
for _, w := range want {
|
||||
if !have[w] {
|
||||
out = append(out, w)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// The JetStream side of the controller: the one place the mesh's streams and consumers are
|
||||
// actually created.
|
||||
//
|
||||
// Everything that decides *what* they are is pure and lives beside this (streams.go, derived.go).
|
||||
// This is only the part that talks to a server, kept small on purpose: a bug in a subject filter
|
||||
// should be findable in a unit test, and only a bug in "did the server accept it" should need one
|
||||
// running.
|
||||
|
||||
// A JetStream is a connection to the bus, as the controller uses it.
|
||||
type JetStream struct {
|
||||
conn *nats.Conn
|
||||
js nats.JetStreamContext
|
||||
}
|
||||
|
||||
// Dial connects and returns the controller's JetStream handle.
|
||||
func Dial(url string, opts ...nats.Option) (*JetStream, error) {
|
||||
opts = append(opts, nats.Name("mesh-controller"), nats.Timeout(10*time.Second))
|
||||
// **Pinned, not named.** The bus presents the mesh's own certificate, which names nothing a
|
||||
// public verifier would accept (design 25 §4: a host pins the server's exact certificate and
|
||||
// checks nothing else, and so does this). Without this, the first connection failed with
|
||||
// "certificate is not valid for any names" against a bus that was answering (2026-09-28).
|
||||
if path := strings.TrimSpace(os.Getenv(CertificateVar)); path != "" {
|
||||
pinned, err := pinnedTo(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
opts = append(opts, nats.Secure(pinned))
|
||||
}
|
||||
// **Its own inbox, and nothing wider.** Every principal is granted `_INBOX.<its user>.>` and
|
||||
// no other inbox; the client's default prefix is random, and the server refused the first
|
||||
// subscription to it (2026-09-28). The user is in the URL, so the prefix follows from it.
|
||||
if user, _, _ := CredentialIn(url); user != "" {
|
||||
opts = append(opts, nats.CustomInboxPrefix("_INBOX."+user))
|
||||
}
|
||||
// The address in an error is the address alone. The URL carries this controller's password,
|
||||
// and an error here is written on the assumption it will be logged.
|
||||
where := BareAddress(url)
|
||||
conn, err := nats.Connect(url, opts...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("connecting to the bus at %s: %w", where, err)
|
||||
}
|
||||
js, err := conn.JetStream()
|
||||
if err != nil {
|
||||
conn.Close()
|
||||
return nil, fmt.Errorf("the bus at %s has no JetStream: %w", where, err)
|
||||
}
|
||||
return &JetStream{conn: conn, js: js}, nil
|
||||
}
|
||||
|
||||
// pinnedTo is a TLS configuration that accepts exactly the certificate in the file and no other:
|
||||
// the leaf's SHA-256, compared on every handshake, with the name and the chain deliberately not
|
||||
// consulted — a self-signed certificate with no names is the ordinary case for a mesh's bus.
|
||||
func pinnedTo(path string) (*tls.Config, error) {
|
||||
want, err := FingerprintOf(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return PinnedToFingerprint(want), nil
|
||||
}
|
||||
|
||||
// DialPinned is Dial with the server's certificate pinned by a fingerprint the caller already holds
|
||||
// — a module or a build machine that was handed one beside its credential, and has no file.
|
||||
func DialPinned(url, fingerprint string, opts ...nats.Option) (*JetStream, error) {
|
||||
if strings.TrimSpace(fingerprint) != "" {
|
||||
opts = append(opts, nats.Secure(PinnedToFingerprint(fingerprint)))
|
||||
}
|
||||
return Dial(url, opts...)
|
||||
}
|
||||
|
||||
// PinnedToFingerprint accepts exactly the certificate with this SHA-256 and no other.
|
||||
func PinnedToFingerprint(want string) *tls.Config {
|
||||
return &tls.Config{
|
||||
InsecureSkipVerify: true, //nolint:gosec // replaced by the pin below, which is stricter
|
||||
MinVersion: tls.VersionTLS12,
|
||||
VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
|
||||
if len(rawCerts) == 0 {
|
||||
return errors.New("the bus presented no certificate")
|
||||
}
|
||||
sum := sha256.Sum256(rawCerts[0])
|
||||
got := "sha256:" + hex.EncodeToString(sum[:])
|
||||
if got != want {
|
||||
return fmt.Errorf("the bus presented a certificate this mesh does not know (%s…), expected %s…", got[:23], want[:23])
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Conn is the connection itself, for what the mesh keeps off JetStream on purpose — a heartbeat,
|
||||
// a tool call — where a lost message is answered by the next one or by a timeout the caller
|
||||
// already handles (design 25 §3).
|
||||
func (j *JetStream) Conn() *nats.Conn { return j.conn }
|
||||
|
||||
// Context is the JetStream handle, for subscribing to what the consumers above define.
|
||||
func (j *JetStream) Context() nats.JetStreamContext { return j.js }
|
||||
|
||||
func (j *JetStream) Close() {
|
||||
if j.conn != nil {
|
||||
j.conn.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// EnsureStream creates the stream if it is absent and brings it to match if it is present.
|
||||
//
|
||||
// **Idempotent, because the controller asserts on every start** rather than creating once at
|
||||
// genesis: a stream somebody deleted, or a mesh raised from a restored backup, has to converge
|
||||
// rather than run without the guarantee its messages assume.
|
||||
//
|
||||
// An update, not a delete and recreate. Recreating would discard every message the stream holds
|
||||
// and every consumer's position in it — which for CONTROL means the pushes being held through a
|
||||
// store restart, exactly the guarantee the stream exists for.
|
||||
func (j *JetStream) EnsureStream(s Stream) error {
|
||||
want := &nats.StreamConfig{
|
||||
Name: s.Name,
|
||||
Subjects: s.Subjects,
|
||||
Retention: retentionOf(s.Retention),
|
||||
MaxAge: time.Duration(s.MaxAge) * time.Second,
|
||||
MaxMsgsPerSubject: int64(s.MaxMsgsPerSubject),
|
||||
Description: s.Why,
|
||||
}
|
||||
if s.Retention == RetentionLastPerSubject {
|
||||
// Last-per-subject is a limits stream with one message kept per subject, not a
|
||||
// retention policy of its own — the state shape, spelled the way the server spells it.
|
||||
want.Retention = nats.LimitsPolicy
|
||||
want.MaxMsgsPerSubject = 1
|
||||
want.MaxAge = 0
|
||||
}
|
||||
|
||||
switch _, err := j.js.StreamInfo(s.Name); {
|
||||
case err == nil:
|
||||
if _, err := j.js.UpdateStream(want); err != nil {
|
||||
return fmt.Errorf("bringing stream %s to match: %w", s.Name, err)
|
||||
}
|
||||
return nil
|
||||
case errors.Is(err, nats.ErrStreamNotFound):
|
||||
if _, err := j.js.AddStream(want); err != nil {
|
||||
return fmt.Errorf("creating stream %s: %w", s.Name, err)
|
||||
}
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("asking about stream %s: %w", s.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
// EnsureConsumer creates or updates one durable consumer.
|
||||
//
|
||||
// Explicit acknowledgement throughout: a consumer that acknowledges on delivery cannot redeliver
|
||||
// work its holder died in the middle of, which is the whole difference between a queue and a
|
||||
// firehose.
|
||||
func (j *JetStream) EnsureConsumer(c Consumer) error {
|
||||
want := &nats.ConsumerConfig{
|
||||
Durable: c.Name,
|
||||
AckPolicy: nats.AckExplicitPolicy,
|
||||
AckWait: time.Duration(c.AckWaitSeconds) * time.Second,
|
||||
MaxDeliver: c.MaxDeliver,
|
||||
DeliverGroup: c.Queue,
|
||||
DeliverSubject: "",
|
||||
Description: c.Why,
|
||||
}
|
||||
switch len(c.Filters) {
|
||||
case 0:
|
||||
case 1:
|
||||
want.FilterSubject = c.Filters[0]
|
||||
default:
|
||||
want.FilterSubjects = c.Filters
|
||||
}
|
||||
// A queue group needs a delivery subject: a pull consumer has no group, and declaring one
|
||||
// without the other is refused by the server with a message that does not say which half is
|
||||
// missing.
|
||||
if c.Queue != "" || c.Push {
|
||||
want.DeliverSubject = "_DELIVER." + c.Name
|
||||
}
|
||||
|
||||
switch _, err := j.js.ConsumerInfo(c.Stream, c.Name); {
|
||||
case err == nil:
|
||||
if _, err := j.js.UpdateConsumer(c.Stream, want); err != nil {
|
||||
return fmt.Errorf("bringing consumer %s on %s to match: %w", c.Name, c.Stream, err)
|
||||
}
|
||||
return nil
|
||||
case errors.Is(err, nats.ErrConsumerNotFound):
|
||||
if _, err := j.js.AddConsumer(c.Stream, want); err != nil {
|
||||
return fmt.Errorf("creating consumer %s on %s: %w", c.Name, c.Stream, err)
|
||||
}
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("asking about consumer %s on %s: %w", c.Name, c.Stream, err)
|
||||
}
|
||||
}
|
||||
|
||||
func retentionOf(r Retention) nats.RetentionPolicy {
|
||||
switch r {
|
||||
case RetentionWorkQueue:
|
||||
return nats.WorkQueuePolicy
|
||||
default:
|
||||
return nats.LimitsPolicy
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Against a real server, because the questions here are all "does the server accept this" —
|
||||
// which a mock would answer by agreeing with whatever this file already believes.
|
||||
//
|
||||
// Skipped unless MESH_TEST_NATS names one, so the ordinary suite stays fast and offline:
|
||||
//
|
||||
// docker run -d --rm --name t -p 14222:4222 nats:2.10-alpine -js
|
||||
// MESH_TEST_NATS=nats://127.0.0.1:14222 go test ./internal/broker/ -run TestAgainstARealServer
|
||||
func TestAgainstARealServer(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
js, err := Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer js.Close()
|
||||
|
||||
t.Run("the mesh's own streams are accepted", func(t *testing.T) {
|
||||
if err := AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("asserting again changes nothing and fails nothing", func(t *testing.T) {
|
||||
if err := AssertMeshStreams(js); err != nil {
|
||||
t.Fatalf("the second assertion failed, so the controller cannot restart: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a seat's work queue is accepted beside them", func(t *testing.T) {
|
||||
seats := []DeclaredSeat{{Name: "telegram-sender", Accepts: []string{"send"}}}
|
||||
for _, s := range SeatStreams(seats) {
|
||||
if err := js.EnsureStream(s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if c := AllOverlaps(seats); len(c) != 0 {
|
||||
t.Fatalf("overlaps the server would refuse: %v", c)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a module's consumer is accepted and is idempotent", func(t *testing.T) {
|
||||
c, ok := ConsumerFor(Principal{Kind: KindModule, Node: "one", Module: "audit",
|
||||
Consumes: []string{"shop.order.placed", "billing.invoice.sent"}, PasswordHash: "x"})
|
||||
if !ok {
|
||||
t.Fatal("no consumer derived")
|
||||
}
|
||||
if err := js.EnsureConsumer(c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.EnsureConsumer(c); err != nil {
|
||||
t.Fatalf("the second assertion failed: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a holder's worker is accepted with its queue group", func(t *testing.T) {
|
||||
c, _ := HolderConsumerFor("one", "telegram",
|
||||
DeclaredSeat{Name: "telegram-sender", Accepts: []string{"send"}})
|
||||
if err := js.EnsureConsumer(c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1,363 +0,0 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/envfile"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The mesh runs the broker, so there is no chicken-and-egg in a node needing an account before it
|
||||
// can connect: the account is created when the token is issued, and the one-time secret in that
|
||||
// token IS the password. A node's first connection is already authenticated, and enrolment is
|
||||
// what happens over it.
|
||||
//
|
||||
// novox/hq ADR 0004's *a node holds its own identity and nothing else* is why the account is per
|
||||
// node rather than shared. A shared enrolment account would let any node consume another's queue,
|
||||
// which is the shared-credential fault that record exists to remove, reappearing at the transport.
|
||||
|
||||
// ManagementVar holds the broker's management API, credentials included.
|
||||
const ManagementVar = "MESH_BROKER_MANAGEMENT"
|
||||
|
||||
// safeName is what a node may be called at the broker.
|
||||
//
|
||||
// The name goes into a URL path and into permission patterns, which are regular expressions. A
|
||||
// name carrying a `.` or a `*` would silently widen what that node may reach — so it is
|
||||
// constrained here rather than escaped later, because an escape that is forgotten once is a node
|
||||
// reading everybody's queues.
|
||||
var safeName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`)
|
||||
|
||||
// Management is the broker's administrative interface.
|
||||
type Management struct {
|
||||
base *url.URL
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// ManagementFromEnvironment reads where the management API is, if it is configured.
|
||||
func ManagementFromEnvironment() (*Management, error) {
|
||||
raw, err := envfile.Value(ManagementVar)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if raw == "" {
|
||||
return nil, ErrNotConfigured
|
||||
}
|
||||
base, err := url.Parse(raw)
|
||||
if err != nil || base.Host == "" {
|
||||
// The value carries a password, so it is not quoted back.
|
||||
return nil, fmt.Errorf("%s is not a usable URL", ManagementVar)
|
||||
}
|
||||
return &Management{base: base, client: &http.Client{Timeout: 15 * time.Second}}, nil
|
||||
}
|
||||
|
||||
// QueueFor is the queue a node consumes from. One per node, named after it.
|
||||
func QueueFor(node string) string { return "node." + node }
|
||||
|
||||
// ExchangeName is where nodes publish what they have to say. One exchange, and the control plane
|
||||
// is the only consumer behind it (novox/hq ADR 0006 — one consumer, so two cannot silently split
|
||||
// the traffic between them).
|
||||
const ExchangeName = "mesh"
|
||||
|
||||
// BuildQueueName is where build work waits. Duplicated from `link` rather than imported, for the
|
||||
// same reason QueueFor above is: this package must not depend on the one that uses it, and a
|
||||
// constant that differed would be caught by the test that asserts they agree.
|
||||
const BuildQueueName = "builds"
|
||||
|
||||
// The events bus (novox/hq ADR 0042): one topic exchange every event rides, a second for tool
|
||||
// RPC kept apart, and a dead-letter home for a poison event. The foundation owns these — a module's
|
||||
// account cannot declare them, only bind its own queue to the events one.
|
||||
const (
|
||||
EventsExchangeName = "mesh.events"
|
||||
RPCExchangeName = "mesh.rpc"
|
||||
DeadExchangeName = "mesh.events.dead"
|
||||
)
|
||||
|
||||
// ModuleQueueFor is the durable queue a module consumes its events from — one per module per node
|
||||
// (novox/hq ADR 0042), named so the account that may read it is exactly this module's.
|
||||
func ModuleQueueFor(node, module string) string { return node + "." + module + ".events" }
|
||||
|
||||
// modulePermissions is a module's authority on the bus, derived from its manifest (novox/hq
|
||||
// ADR 0043): what it consumes and what it emits, and nothing else. Pure, so the scope is tested as
|
||||
// patterns without a broker — the way a builder's is.
|
||||
//
|
||||
// A note on the limit: the broker's write permission is per exchange, not per routing key (LavinMQ
|
||||
// has no topic permissions), so an emitting module is granted the events exchange whole. ADR 0042's
|
||||
// origin reservation — a module publishes only under `module.<self>.*` — is stamped by the sdk, not
|
||||
// enforced here; that gap is the broker's, and is recorded rather than hidden. A pure consumer like
|
||||
// the audit logger is unaffected: it is granted no write to the exchange at all.
|
||||
func modulePermissions(node, module string, emits, consumes []string) (configure, write, read string) {
|
||||
queue := regexp.QuoteMeta(ModuleQueueFor(node, module))
|
||||
events := regexp.QuoteMeta(EventsExchangeName)
|
||||
rpc := regexp.QuoteMeta(RPCExchangeName)
|
||||
// A module serves each of its tools on its own queue, namespaced by the module (novox/hq
|
||||
// ADR 0047) — serve.<module>.<tool> — so the account may declare, bind and read exactly its own,
|
||||
// and no other module's.
|
||||
serve := "serve\\." + regexp.QuoteMeta(module) + "\\..*"
|
||||
|
||||
// Declare its own events queue and its own tool serve queues.
|
||||
configure = "^(" + queue + "|" + serve + ")$"
|
||||
|
||||
// Write to bind its queue and serve queues (binding is a write on the queue), and to the RPC
|
||||
// exchange to publish replies (ADR 0047: replies ride mesh.rpc, never the default exchange, which
|
||||
// would let it publish into any queue). To the events exchange only if it emits.
|
||||
writes := []string{queue, serve, rpc}
|
||||
if len(emits) > 0 {
|
||||
writes = append(writes, events)
|
||||
}
|
||||
write = "^(" + strings.Join(writes, "|") + ")$"
|
||||
|
||||
// Read its own queue and serve queues to consume them, and the RPC exchange to bind its serve
|
||||
// queues onto. The events exchange to bind onto only if it consumes.
|
||||
reads := []string{queue, serve, rpc}
|
||||
if len(consumes) > 0 {
|
||||
reads = append(reads, events)
|
||||
}
|
||||
read = "^(" + strings.Join(reads, "|") + ")$"
|
||||
return configure, write, read
|
||||
}
|
||||
|
||||
// CreateModuleAccount gives an assigned module its own broker account, scoped by what it emits and
|
||||
// consumes (novox/hq ADR 0043). The account name carries the node so the same module on two machines
|
||||
// holds two accounts, each sealed to its own; the permissions carry the module so one module cannot
|
||||
// read another's queue. Generic — the builder is one instance of this rule, not a separate kind.
|
||||
func (m *Management) CreateModuleAccount(ctx context.Context, node, module, password string, emits, consumes []string) (string, error) {
|
||||
if !safeName.MatchString(node) {
|
||||
return "", fmt.Errorf("%q cannot be part of a broker account: it is a permission pattern", node)
|
||||
}
|
||||
if !safeName.MatchString(module) {
|
||||
return "", fmt.Errorf("%q cannot be part of a broker account: it is a permission pattern", module)
|
||||
}
|
||||
account := node + "-" + module
|
||||
if !safeName.MatchString(account) {
|
||||
return "", fmt.Errorf("%q is not a usable broker account name", account)
|
||||
}
|
||||
|
||||
if err := m.put(ctx, "/api/users/"+url.PathEscape(account),
|
||||
map[string]string{"password": password, "tags": ""}); err != nil {
|
||||
return "", fmt.Errorf("cannot create the broker account for %s on %s: %w", module, node, err)
|
||||
}
|
||||
|
||||
configure, write, read := modulePermissions(node, module, emits, consumes)
|
||||
if err := m.put(ctx, "/api/permissions/%2f/"+url.PathEscape(account), map[string]string{
|
||||
"configure": configure, "write": write, "read": read,
|
||||
}); err != nil {
|
||||
return "", fmt.Errorf("cannot scope the broker account for %s on %s: %w", module, node, err)
|
||||
}
|
||||
return account, nil
|
||||
}
|
||||
|
||||
// EnsureModuleQueue declares a consuming module's queue with its dead-letter exchange, idempotently.
|
||||
// The foundation declares it because a scoped module account may not: the broker refuses a queue with
|
||||
// a dead-letter exchange to a non-administrator (novox/hq ADR 0043), so a consumer passively checks
|
||||
// the queue the mesh made rather than declaring its own.
|
||||
func (m *Management) EnsureModuleQueue(ctx context.Context, node, module string) error {
|
||||
queue := ModuleQueueFor(node, module)
|
||||
if err := m.put(ctx, "/api/queues/%2f/"+url.PathEscape(queue), map[string]any{
|
||||
"durable": true,
|
||||
"arguments": map[string]any{"x-dead-letter-exchange": DeadExchangeName},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("cannot declare the queue for %s on %s: %w", module, node, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// EnsureEventExchanges declares the bus's exchanges and the dead-letter home, idempotently. The
|
||||
// foundation owns them (a module's account may not declare an exchange), and a dead-letter exchange
|
||||
// with no queue behind it drops what it receives — so a durable queue bound to `#` retains a poison
|
||||
// event for inspection, which is the whole reason the trail exists.
|
||||
func (m *Management) EnsureEventExchanges(ctx context.Context) error {
|
||||
for _, exchange := range []string{EventsExchangeName, RPCExchangeName, DeadExchangeName} {
|
||||
if err := m.put(ctx, "/api/exchanges/%2f/"+url.PathEscape(exchange),
|
||||
map[string]any{"type": "topic", "durable": true}); err != nil {
|
||||
return fmt.Errorf("cannot declare the %s exchange: %w", exchange, err)
|
||||
}
|
||||
}
|
||||
if err := m.put(ctx, "/api/queues/%2f/"+url.PathEscape(DeadExchangeName),
|
||||
map[string]any{"durable": true}); err != nil {
|
||||
return fmt.Errorf("cannot declare the dead-letter queue: %w", err)
|
||||
}
|
||||
if err := m.post(ctx, "/api/bindings/%2f/e/"+url.PathEscape(DeadExchangeName)+
|
||||
"/q/"+url.PathEscape(DeadExchangeName), map[string]string{"routing_key": "#"}); err != nil {
|
||||
return fmt.Errorf("cannot bind the dead-letter queue: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateNodeAccount gives a node its own broker account, with the token's secret as the password.
|
||||
//
|
||||
// Scoped so a node can reach its own queue and the one exchange, and nothing else. The patterns
|
||||
// are anchored: a node called `laptop` must not be able to read `laptop-of-somebody-else`.
|
||||
func (m *Management) CreateNodeAccount(ctx context.Context, node, password string) error {
|
||||
if !safeName.MatchString(node) {
|
||||
return fmt.Errorf(
|
||||
"%q cannot be a broker account name: it becomes part of a permission pattern, so it "+
|
||||
"is lower-case letters, digits and dashes", node)
|
||||
}
|
||||
|
||||
if err := m.put(ctx, "/api/users/"+url.PathEscape(node),
|
||||
map[string]string{"password": password, "tags": ""}); err != nil {
|
||||
return fmt.Errorf("cannot create the broker account for %s: %w", node, err)
|
||||
}
|
||||
|
||||
queue := regexp.QuoteMeta(QueueFor(node))
|
||||
if err := m.put(ctx, "/api/permissions/%2f/"+url.PathEscape(node), map[string]string{
|
||||
"configure": "^" + queue + "$",
|
||||
"write": "^(" + regexp.QuoteMeta(ExchangeName) + "|" + queue + ")$",
|
||||
"read": "^" + queue + "$",
|
||||
}); err != nil {
|
||||
return fmt.Errorf("cannot scope the broker account for %s: %w", node, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateBuilderAccount scopes an account to taking build work and answering it.
|
||||
//
|
||||
// **A build machine is not a node**, and giving it a node's account would let it read another
|
||||
// machine's declarations. What it needs is narrower and different: read the build queue, and
|
||||
// write to the exchange and to whatever temporary queue an asker is waiting on.
|
||||
//
|
||||
// The reply queues are the reason `write` is not simply the exchange. `RequestBuild` declares an
|
||||
// exclusive queue with a generated name and waits on it, so a builder that could not write to it
|
||||
// could take work and never answer — which is the failure that looks like a builder that is not
|
||||
// running.
|
||||
func (m *Management) CreateBuilderAccount(ctx context.Context, name, password string) error {
|
||||
if !safeName.MatchString(name) {
|
||||
return fmt.Errorf(
|
||||
"%q cannot be a broker account name: it becomes part of a permission pattern, so it "+
|
||||
"is lower-case letters, digits and dashes", name)
|
||||
}
|
||||
|
||||
if err := m.put(ctx, "/api/users/"+url.PathEscape(name),
|
||||
map[string]string{"password": password, "tags": ""}); err != nil {
|
||||
return fmt.Errorf("cannot create the broker account for %s: %w", name, err)
|
||||
}
|
||||
|
||||
builds := regexp.QuoteMeta(BuildQueueName)
|
||||
if err := m.put(ctx, "/api/permissions/%2f/"+url.PathEscape(name), map[string]string{
|
||||
// It declares the build queue, because whichever builder starts first must be able to —
|
||||
// and a queue nobody may declare is a queue that exists only if the control plane has
|
||||
// already run, which makes the order they start in matter.
|
||||
"configure": "^" + builds + "$",
|
||||
// Two exchanges, and nothing else. **Not the default exchange**: permission there is
|
||||
// granted per exchange rather than per queue, so a builder allowed to use it could
|
||||
// publish into any node's queue — the privilege a build machine most obviously should
|
||||
// not have. Answers go through the node exchange; announcing what was built goes through
|
||||
// the events exchange, which is a different act with a different audience (novox/hq
|
||||
// ADR 0072). A builder that could answer and not announce would leave the module graph
|
||||
// knowing less than the registry does.
|
||||
"write": "^(" + regexp.QuoteMeta(ExchangeName) + "|" + regexp.QuoteMeta(EventsExchangeName) + ")$",
|
||||
// The build queue and nothing else. Not another machine's declarations.
|
||||
"read": "^" + builds + "$",
|
||||
}); err != nil {
|
||||
return fmt.Errorf("cannot scope the broker account for %s: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemoveNodeAccount withdraws a node's access.
|
||||
func (m *Management) RemoveNodeAccount(ctx context.Context, node string) error {
|
||||
if !safeName.MatchString(node) {
|
||||
return fmt.Errorf("%q is not a broker account name", node)
|
||||
}
|
||||
return m.do(ctx, http.MethodDelete, "/api/users/"+url.PathEscape(node), nil)
|
||||
}
|
||||
|
||||
// Accounts lists the broker's users, so a picture can be read from the system rather than assumed
|
||||
// (novox/hq ADR 0018).
|
||||
func (m *Management) Accounts(ctx context.Context) ([]string, error) {
|
||||
body, err := m.get(ctx, "/api/users")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var users []struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &users); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names := make([]string, 0, len(users))
|
||||
for _, u := range users {
|
||||
names = append(names, u.Name)
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
|
||||
func (m *Management) put(ctx context.Context, path string, body any) error {
|
||||
return m.do(ctx, http.MethodPut, path, body)
|
||||
}
|
||||
|
||||
func (m *Management) post(ctx context.Context, path string, body any) error {
|
||||
return m.do(ctx, http.MethodPost, path, body)
|
||||
}
|
||||
|
||||
func (m *Management) get(ctx context.Context, path string) ([]byte, error) {
|
||||
request, err := m.request(ctx, http.MethodGet, path, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
response, err := m.client.Do(request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode >= 300 {
|
||||
return nil, fmt.Errorf("the broker's management API answered %s to GET %s",
|
||||
response.Status, path)
|
||||
}
|
||||
return io.ReadAll(io.LimitReader(response.Body, 1<<20))
|
||||
}
|
||||
|
||||
func (m *Management) do(ctx context.Context, method, path string, body any) error {
|
||||
request, err := m.request(ctx, method, path, body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response, err := m.client.Do(request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode >= 300 {
|
||||
detail, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
|
||||
return fmt.Errorf("the broker's management API answered %s to %s %s: %s",
|
||||
response.Status, method, path, strings.TrimSpace(string(detail)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Management) request(ctx context.Context, method, path string, body any) (*http.Request, error) {
|
||||
var payload io.Reader
|
||||
if body != nil {
|
||||
raw, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
payload = bytes.NewReader(raw)
|
||||
}
|
||||
|
||||
// Path joined by hand rather than through url.Parse: %2f is the default vhost and must reach
|
||||
// the broker still encoded. Parsing would decode it to a slash and address a different route.
|
||||
target := strings.TrimSuffix(m.base.String(), "/")
|
||||
if user := m.base.User; user != nil {
|
||||
target = strings.TrimSuffix(m.base.Scheme+"://"+m.base.Host, "/")
|
||||
}
|
||||
request, err := http.NewRequestWithContext(ctx, method, target+path, payload)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if user := m.base.User; user != nil {
|
||||
password, _ := user.Password()
|
||||
request.SetBasicAuth(user.Username(), password)
|
||||
}
|
||||
if body != nil {
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
return request, nil
|
||||
}
|
||||
@@ -1,96 +0,0 @@
|
||||
package broker_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// The audit logger consumes everything and emits nothing. Its account must let it declare and read
|
||||
// its own queue and read the events exchange to bind onto — and must not reach another module's
|
||||
// queue, nor grant any write to the events exchange (novox/hq ADR 0043).
|
||||
func TestAConsumerReadsItsOwnQueueAndTheEventsExchangeAndNoOthers(t *testing.T) {
|
||||
// Rebuilt through CreateModuleAccount's own path by asking for the same scope it would apply.
|
||||
// The queue this module reads:
|
||||
mine := broker.ModuleQueueFor("anchor", "audit-logger")
|
||||
other := broker.ModuleQueueFor("anchor", "plex")
|
||||
|
||||
read := scope(t, "read", "anchor", "audit-logger", nil, []string{"#"})
|
||||
if !read.MatchString(mine) {
|
||||
t.Error("the audit logger may not read its own queue, so it consumes nothing")
|
||||
}
|
||||
if !read.MatchString(broker.EventsExchangeName) {
|
||||
t.Error("the audit logger may not read the events exchange, so it cannot bind onto it")
|
||||
}
|
||||
if read.MatchString(other) {
|
||||
t.Error("the audit logger may read another module's queue")
|
||||
}
|
||||
|
||||
// It emits nothing, so it is granted no write to the events exchange — only its own queue, to bind.
|
||||
write := scope(t, "write", "anchor", "audit-logger", nil, []string{"#"})
|
||||
if write.MatchString(broker.EventsExchangeName) {
|
||||
t.Error("a pure consumer was granted write to the events exchange")
|
||||
}
|
||||
if !write.MatchString(mine) {
|
||||
t.Error("the audit logger may not write to its own queue, so it cannot bind it")
|
||||
}
|
||||
}
|
||||
|
||||
// An emitter is granted the events exchange to write; a consumer is not.
|
||||
func TestAnEmitterMayWriteTheEventsExchangeAndAConsumerMayNot(t *testing.T) {
|
||||
emitter := scope(t, "write", "anchor", "umami", []string{"module.umami.site.created"}, nil)
|
||||
if !emitter.MatchString(broker.EventsExchangeName) {
|
||||
t.Error("an emitting module may not write the events exchange, so it cannot emit")
|
||||
}
|
||||
}
|
||||
|
||||
// scope reconstructs one of the three permission patterns CreateModuleAccount would apply, by
|
||||
// reading it back from a captured request against a stub management API.
|
||||
func scope(t *testing.T, which, node, module string, emits, consumes []string) *regexp.Regexp {
|
||||
t.Helper()
|
||||
pat := capturePermission(t, which, node, module, emits, consumes)
|
||||
re, err := regexp.Compile(pat)
|
||||
if err != nil {
|
||||
t.Fatalf("the %s pattern does not compile: %v", which, err)
|
||||
}
|
||||
return re
|
||||
}
|
||||
|
||||
// capturePermission runs CreateModuleAccount against a stub management API and returns the pattern
|
||||
// it set for `which` ("configure"/"write"/"read"). The scope is tested where it is applied, not
|
||||
// reconstructed by the test — so a change to the mapping cannot pass a test that hard-codes the old
|
||||
// one.
|
||||
func capturePermission(t *testing.T, which, node, module string, emits, consumes []string) string {
|
||||
t.Helper()
|
||||
var captured map[string]string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if strings.HasPrefix(r.URL.Path, "/api/permissions/") {
|
||||
_ = json.NewDecoder(r.Body).Decode(&captured)
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
t.Setenv(broker.ManagementVar, server.URL)
|
||||
m, err := broker.ManagementFromEnvironment()
|
||||
if err != nil {
|
||||
t.Fatalf("stub management not usable: %v", err)
|
||||
}
|
||||
if _, err := m.CreateModuleAccount(context.Background(), node, module, "pw", emits, consumes); err != nil {
|
||||
t.Fatalf("CreateModuleAccount: %v", err)
|
||||
}
|
||||
if captured == nil {
|
||||
t.Fatal("no permissions were set")
|
||||
}
|
||||
pattern, ok := captured[which]
|
||||
if !ok {
|
||||
t.Fatalf("no %s permission was set; got %v", which, captured)
|
||||
}
|
||||
return pattern
|
||||
}
|
||||
@@ -0,0 +1,572 @@
|
||||
// Composing the bus's own configuration.
|
||||
//
|
||||
// An account is *composed*, never called for: the controller writes accounts, users and
|
||||
// per-subject permissions into one file the host keeps current, and the server reloads it in
|
||||
// place (novox/hq ADR 0106 — never through a management API; design 25 §4).
|
||||
//
|
||||
// Everything here is pure. Given the principals, it returns the file's text — so the whole of the
|
||||
// mesh's authority model is testable as strings, with no server.
|
||||
//
|
||||
// **Permissions are per subject, so a module's own name is the server's to enforce.** ADR 0042
|
||||
// reserves a module's origin — it publishes only under its own name — and here that is a refusal
|
||||
// rather than something a library promises.
|
||||
package broker
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// A Kind is what a principal is, which decides the shape of its authority rather than its
|
||||
// contents: a module's comes from its declaration, a host's from its node, and the controller's
|
||||
// and the enrolment user's are fixed.
|
||||
type Kind string
|
||||
|
||||
const (
|
||||
KindModule Kind = "module"
|
||||
KindNode Kind = "node"
|
||||
KindController Kind = "controller"
|
||||
KindEnrolment Kind = "enrolment"
|
||||
// KindPerson is somebody reaching the mesh's tools from a workstation (design 25 §7). Its
|
||||
// authority is a list of tools and nothing else — not control, not declarations, not builds,
|
||||
// and no ability to answer anything, because a person asks.
|
||||
KindPerson Kind = "person"
|
||||
)
|
||||
|
||||
// Seat is a role on the bus as a principal relates to it: the subjects it accepts, and those it
|
||||
// emits (novox/hq ADR 0118, design 29 §5).
|
||||
type Seat struct {
|
||||
Name string
|
||||
Accepts []string
|
||||
Emits []string
|
||||
Serves []string
|
||||
Versions []string // protocol versions served beside the current one; empty for v1 only
|
||||
}
|
||||
|
||||
// A Principal is one user of the bus. Its permissions are derived from what it declares and
|
||||
// nothing else (novox/hq ADR 0043), over the three namespaces of design 29 §2: its own, the seats
|
||||
// it holds, and the seats it uses.
|
||||
type Principal struct {
|
||||
Kind Kind
|
||||
Node string
|
||||
Module string
|
||||
|
||||
Emits []string
|
||||
Consumes []string
|
||||
Serves []string
|
||||
|
||||
Holds []Seat
|
||||
Uses []Seat
|
||||
// Watches are seats whose events this principal consumes. Separate from Consumes because a
|
||||
// role's event lives under the seat's namespace and not a module's, and this package cannot tell
|
||||
// a seat's name from a module's by looking at it — whoever resolved the declaration can, and
|
||||
// does (novox/hq ADR 0121).
|
||||
//
|
||||
// **Found by a consumer reading nothing.** The catalogue consumes the build machine's outcome;
|
||||
// with that name read as a module's, its subscription pointed at `mesh.mod.mesh-build-machine.…`,
|
||||
// a namespace no such module owns. Every service started and the graph stayed empty.
|
||||
Watches []Seat
|
||||
|
||||
// Invokes are the tools a person may call, as `<module>.<tool>`; a single `*` is every tool,
|
||||
// for an administrator. Only meaningful for KindPerson.
|
||||
//
|
||||
// **A list, not a role.** A person is not a module and holds no seat: nothing is addressed
|
||||
// to them, nothing is delivered to them, and they have no durable consumer to acknowledge.
|
||||
// What they have is permission to ask.
|
||||
Invokes []string
|
||||
|
||||
// PasswordHash is the bcrypt hash the mesh minted. The plaintext is sealed to the principal
|
||||
// and never appears here: this file is written to a node's disk and read by a server, and a
|
||||
// secret that can be read from a configuration file is a secret with a wider blast radius
|
||||
// than the one it protects (novox/hq design 29 §10).
|
||||
PasswordHash string
|
||||
}
|
||||
|
||||
// meshSeatsTheControllerUses are the roles the mesh's own flows submit work to. Named rather than
|
||||
// derived from the seat set: the controller is not a module and declares no `uses`, so its side of a
|
||||
// seat has to be stated, and a list is what makes "which roles does the mesh itself talk to" answerable.
|
||||
var meshSeatsTheControllerUses = []string{"mesh-build-machine"}
|
||||
|
||||
// enrolmentPrefix is the space every enrolling node's user and inbox live under, so the one place the
|
||||
// controller may answer an enrolment is derived from the same constant the user is named from.
|
||||
const enrolmentPrefix = "enrol"
|
||||
|
||||
// safeSubject refuses anything that would change the meaning of a subject rather than sit inside
|
||||
// one. A name carrying a dot would silently widen a permission by adding a token; a name carrying
|
||||
// `>` or `*` would widen it to a wildcard, which is the whole authority model gone.
|
||||
var safeSubject = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)
|
||||
|
||||
// Username is how a principal is named to the server. The node is part of it, so the same module
|
||||
// on two machines holds two users, each sealed to its own — the rule management.go already
|
||||
// applies, kept.
|
||||
func (p Principal) Username() string {
|
||||
switch p.Kind {
|
||||
case KindPerson:
|
||||
return "person." + p.Module
|
||||
case KindModule:
|
||||
return p.Node + "." + p.Module
|
||||
case KindNode:
|
||||
return "node." + p.Node
|
||||
case KindController:
|
||||
return "controller"
|
||||
case KindEnrolment:
|
||||
// Per token, not one shared user. **The inbox is the reason**: with a single `enrolment`
|
||||
// user every machine enrolling at once could read every other's answer, and an answer
|
||||
// carries that node's credentials sealed to it. Design 25 §6 says the inbox a token
|
||||
// derives, and a permission belongs to a user, so the user is per token.
|
||||
//
|
||||
// Named after the node, which **is** the token's id: a token is issued for a node record,
|
||||
// the mesh holds one live claim per record, and the node's name is the one identifier both
|
||||
// sides already have before anything else is agreed. It is also exactly what the other
|
||||
// transport does, where the account is named after the node and the secret is its password.
|
||||
return enrolmentPrefix + "." + p.Node
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// inbox is a principal's own reply space. No user is ever granted a bare `_INBOX.>` (design 25
|
||||
// §4): with one account, inbox privacy is the permission list or it is nothing, so each user's
|
||||
// inbox is derived from its own identity and its permissions name that prefix and no other.
|
||||
func (p Principal) inbox() string { return "_INBOX." + p.Username() + ".>" }
|
||||
|
||||
// Permissions is what a principal may publish and subscribe, and whether it may answer.
|
||||
type Permissions struct {
|
||||
Publish []string
|
||||
Subscribe []string
|
||||
// AllowResponses lets a principal reply to a request it received, on the reply subject that
|
||||
// request carried, once.
|
||||
//
|
||||
// **This is what makes scoped inboxes possible at all**, and design 25 §4 did not say it. If
|
||||
// every user's inbox is private to it, a module serving a tool cannot publish the answer —
|
||||
// the answer goes to the *caller's* inbox, which the responder has no permission for. The two
|
||||
// ways out are granting responders `_INBOX.>`, which is precisely the blanket grant §4
|
||||
// refuses, or this: the server itself permits one reply to the subject of a message the user
|
||||
// actually received, and nothing else. The authority is bounded by having been asked.
|
||||
AllowResponses bool
|
||||
}
|
||||
|
||||
// PermissionsFor derives a principal's authority. Pure, and the only place authority is decided:
|
||||
// a permission that cannot be derived from a declaration is a permission nobody can explain.
|
||||
func PermissionsFor(p Principal) (Permissions, error) {
|
||||
for _, part := range []struct{ what, value string }{
|
||||
{"node", p.Node}, {"module", p.Module},
|
||||
} {
|
||||
if part.value == "" {
|
||||
continue
|
||||
}
|
||||
if !safeSubject.MatchString(part.value) {
|
||||
return Permissions{}, fmt.Errorf(
|
||||
"%q cannot be part of a subject: a permission is a subject pattern, and this would widen it", part.value)
|
||||
}
|
||||
}
|
||||
|
||||
var pub, sub []string
|
||||
switch p.Kind {
|
||||
case KindController:
|
||||
// The controller owns the mesh's own traffic and the streams. It is the only writer of
|
||||
// stream definitions (design 25 §3), so it alone reaches the JetStream API.
|
||||
pub = []string{"mesh.control.>", "mesh.node.>", "$JS.API.>"}
|
||||
// **And where its consumers deliver.** A push consumer delivers on `_DELIVER.<its name>`,
|
||||
// and a client bound to it subscribes exactly that; the server refused it for every
|
||||
// principal the first time one bound a consumer (2026-09-28). Each kind below is granted
|
||||
// its own consumers' delivery subjects and no other's.
|
||||
sub = []string{"mesh.control.>", "$JS.API.>", "_DELIVER." + ControllerName, "_DELIVER." + ControllerName + ".>"}
|
||||
|
||||
// Work the mesh's own flows submit to a role, and the outcomes they wait on (ADR 0121). A
|
||||
// build is the one today: the controller asks, and reads the answer from the seat's event
|
||||
// like the catalogue does — which is why no holder needs to publish into anybody's inbox.
|
||||
for _, seat := range meshSeatsTheControllerUses {
|
||||
pub = append(pub, "mesh.seat."+seat+".accept.>")
|
||||
}
|
||||
|
||||
// The two events it reacts to, and its ack subject on the stream they arrive from
|
||||
// (streams.go). **Each named, not a pattern**: `mesh.mod.*.event.>` would make the
|
||||
// controller a subscriber to every event in the mesh, and its permission list would stop
|
||||
// saying what it is for. The ack grant below is scoped per stream because the controller's
|
||||
// consumer name is the same on both and `$JS.ACK.CONTROL.controller.>` does not cover a
|
||||
// delivery from EVENTS — a consumer that cannot ack has every message redelivered for
|
||||
// ever, refused by the list it already has.
|
||||
sub = append(sub, ControllerFollows...)
|
||||
pub = append(pub, "$JS.ACK.EVENTS."+ControllerName+".>")
|
||||
|
||||
// **Where an enrolment's answer goes**, and `allow_responses` does not cover it. That
|
||||
// permits one reply to the reply subject of a message the user received — and a message a
|
||||
// JetStream consumer delivers has had that field claimed for the consumer's own ack address
|
||||
// (design 25 §2), so the address the controller actually answers is the one the request
|
||||
// carried in its payload, which is not a reply subject as the server understands it.
|
||||
//
|
||||
// Verified against a real server before this line existed: the answer was refused with
|
||||
// "Permissions Violation for Publish to _INBOX.enrol.anchor…", and every enrolment on the
|
||||
// mesh would have timed out while the controller logged success.
|
||||
//
|
||||
// **The enrolment inbox space, not a blanket `_INBOX.>`.** Design 25 §4 refuses that, and
|
||||
// this is not it: nothing but an enrolling node ever subscribes under this prefix, each
|
||||
// scoped to its own token's, so the controller publishing here is the mesh answering
|
||||
// enrolments and can reach nothing else.
|
||||
pub = append(pub, "_INBOX."+enrolmentPrefix+".>")
|
||||
|
||||
case KindPerson:
|
||||
// Tools, and nothing else. Every subject a person may publish is a tool call; a person
|
||||
// who could publish an event would be able to claim a module said something.
|
||||
for _, t := range p.Invokes {
|
||||
if t == "*" {
|
||||
pub = append(pub, "mesh.mod.*.tool.>")
|
||||
continue
|
||||
}
|
||||
module, tool, ok := strings.Cut(t, ".")
|
||||
if !ok {
|
||||
return Permissions{}, fmt.Errorf(
|
||||
"%q does not name a tool: a person invokes <module>.<tool>, or * for every one", t)
|
||||
}
|
||||
pub = append(pub, "mesh.mod."+module+".tool."+tool)
|
||||
}
|
||||
|
||||
case KindEnrolment:
|
||||
// A leaked token is useless for anything but enrolling: it cannot read a declaration, hear
|
||||
// an event, or subscribe any inbox but the one its own token derives (design 25 §6).
|
||||
//
|
||||
// **The inbox was missing and the handshake could not have completed without it.** An
|
||||
// enrolling node publishes its request and waits on an address it states in the payload;
|
||||
// with nothing to subscribe it waits out its timeout against a mesh that answered. Its own
|
||||
// and no wider: `_INBOX.enrol.<node>.>`, so what is sealed to one machine cannot be read by
|
||||
// another enrolling beside it.
|
||||
if p.Node == "" {
|
||||
// Refused rather than composed into `_INBOX.enrol..>`, which is a subject with an empty
|
||||
// token in it — and worse, one every nameless enrolment user would share. A shared
|
||||
// enrolment inbox is one machine able to read the credentials sealed to another.
|
||||
return Permissions{}, errors.New(
|
||||
"an enrolment user names no node, so its inbox would be shared with every other " +
|
||||
"enrolment: a token is issued for a node record, and that record's name is " +
|
||||
"the token's id")
|
||||
}
|
||||
pub = []string{"mesh.control.enrol"}
|
||||
sub = []string{p.inbox()}
|
||||
|
||||
case KindNode:
|
||||
// A host publishes its own node's control traffic and subscribes its own declaration —
|
||||
// and nothing of any other node's.
|
||||
// And binding to its consumer, which asks the server about it (CONSUMER.INFO) — the one
|
||||
// thing the host does that nothing granted. Found the first time a machine dialled a
|
||||
// permissioned server: "this node cannot read its declarations" (2026-09-28). The ack and
|
||||
// the inbox are granted below with every principal's.
|
||||
pub = []string{
|
||||
"mesh.control." + p.Node + ".>",
|
||||
"$JS.API.CONSUMER.INFO.NODES." + p.Node,
|
||||
}
|
||||
sub = []string{"mesh.node." + p.Node + ".declare", "_DELIVER." + p.Node}
|
||||
|
||||
case KindModule:
|
||||
// 1. Its own namespace: it publishes its events there and serves its tools there. Nothing
|
||||
// else may publish into it, so an event's source is a fact the server enforces rather
|
||||
// than a claim in the body (design 29 §2).
|
||||
own := "mesh.mod." + p.Module
|
||||
for _, e := range p.Emits {
|
||||
pub = append(pub, own+".event."+e)
|
||||
}
|
||||
for _, t := range p.Serves {
|
||||
sub = append(sub, own+".tool."+t)
|
||||
}
|
||||
|
||||
// 2. What it consumes, by the emitter's own subject — an event is addressed to its
|
||||
// emitter, because the emitter's identity is the meaning (ADR 0118).
|
||||
for _, c := range p.Consumes {
|
||||
subject, err := consumedSubject(c)
|
||||
if err != nil {
|
||||
return Permissions{}, err
|
||||
}
|
||||
sub = append(sub, subject)
|
||||
}
|
||||
|
||||
// 2b. Events of a role it watches, under the seat's own namespace. Subscribe only: watching a
|
||||
// role is hearing what it announced, not taking part in it.
|
||||
for _, w := range p.Watches {
|
||||
for _, e := range w.Emits {
|
||||
sub = append(sub, seatSubject(w, "event", e))
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Seats it holds: full participation.
|
||||
// Its consumer's name, not ConsumerFor: that asks for these permissions to build the
|
||||
// consumer, and would ask forever. A subject for a consumer that turns out not to exist
|
||||
// grants nothing anybody can use.
|
||||
sub = append(sub, "_DELIVER."+consumerDurable(p))
|
||||
for _, s := range p.Holds {
|
||||
// Taking work from the role's queue: the worker consumer it binds (asked about,
|
||||
// delivered on, acknowledged), each on the seat's own stream. The first machine to
|
||||
// take work over the new bus was refused the asking (2026-09-28).
|
||||
worker := "SEAT_" + upperSnake(s.Name) + "_worker"
|
||||
stream := seatStreamName(s.Name)
|
||||
sub = append(sub, "_DELIVER."+worker)
|
||||
pub = append(pub, "$JS.API.CONSUMER.INFO."+stream+"."+worker, "$JS.ACK."+stream+"."+worker+".>")
|
||||
for _, a := range s.Accepts {
|
||||
sub = append(sub, seatSubject(s, "accept", a))
|
||||
}
|
||||
for _, e := range s.Emits {
|
||||
pub = append(pub, seatSubject(s, "event", e))
|
||||
}
|
||||
for _, t := range s.Serves {
|
||||
sub = append(sub, seatSubject(s, "tool", t))
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Seats it uses: publish only, and only the accepts half. A caller cannot subscribe a
|
||||
// seat's inbound subject and watch other modules' traffic, nor publish its outbound
|
||||
// events and lie about outcomes (design 29 §2).
|
||||
for _, s := range p.Uses {
|
||||
for _, a := range s.Accepts {
|
||||
pub = append(pub, seatSubject(s, "accept", a))
|
||||
}
|
||||
for _, t := range s.Serves {
|
||||
pub = append(pub, seatSubject(s, "tool", t))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if p.Kind == KindPerson {
|
||||
// An inbox to hear answers in, and nothing else. No ack subject: a person has no durable
|
||||
// consumer, because nothing is delivered to a person — they ask and are answered.
|
||||
sub = append(sub, p.inbox())
|
||||
}
|
||||
|
||||
if p.Kind == KindModule || p.Kind == KindNode || p.Kind == KindController {
|
||||
// Its own reply space, and nothing wider.
|
||||
sub = append(sub, p.inbox())
|
||||
|
||||
// Acking a JetStream delivery is a publish to that consumer's own ack address — a
|
||||
// different subject from anything the consumer subscribes. Without it every message a
|
||||
// module received would be redelivered forever, refused by the permission list it already
|
||||
// has (design 25 §4). Scoped to this principal's own consumer name, so it can ack its own
|
||||
// deliveries and no other's.
|
||||
pub = append(pub, "$JS.ACK."+consumerStream(p)+"."+consumerDurable(p)+".>")
|
||||
}
|
||||
|
||||
sort.Strings(pub)
|
||||
sort.Strings(sub)
|
||||
return Permissions{
|
||||
Publish: pub,
|
||||
Subscribe: sub,
|
||||
// Only something that serves is ever answering. A pure consumer is granted nothing here.
|
||||
AllowResponses: p.Kind == KindModule && (len(p.Serves) > 0 || len(p.Holds) > 0) ||
|
||||
p.Kind == KindController,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// seatSubject places a seat's verb under the kind of traffic it is.
|
||||
//
|
||||
// **The kind token is load-bearing, not decoration.** A stream is defined by a subject filter, so
|
||||
// without it a stream over a seat or a module's namespace would capture that namespace's *tool*
|
||||
// traffic too — and a tool call must never be persisted (design 25 §3: tools stay on core NATS).
|
||||
// Found while defining the streams: the first draft of design 29 had one namespace per module
|
||||
// with no kind, which reads well and cannot be filtered.
|
||||
//
|
||||
// A seat serving more than its current protocol version carries the version as a token
|
||||
// (design 29 §8): the seat stays one role, and v1 and v2 run beside each other until nothing is
|
||||
// bound to the old one.
|
||||
func seatSubject(s Seat, kind, verb string) string {
|
||||
return "mesh.seat." + s.Name + "." + kind + "." + verb
|
||||
}
|
||||
|
||||
// consumerStream and consumerDurable are the two halves of a consumer's identity, and they are
|
||||
// two functions because conflating them was a real bug.
|
||||
//
|
||||
// **A durable name may not contain a dot; an ack subject is built from two names that do.** The
|
||||
// server acknowledges on `$JS.ACK.<stream>.<consumer>.…`, so a single string "EVENTS.one_audit"
|
||||
// reads correctly inside the permission and is rejected as a consumer name — *nats: invalid
|
||||
// consumer name*. Caught against a running server, and worth the comment because the shape of
|
||||
// the failure if it had not been is the one design 25 §4 warns about: a consumer that cannot ack
|
||||
// has every message redelivered forever, and its permission list looks right while it happens.
|
||||
//
|
||||
// They are derived here, beside the permission that must match them, because two places deriving
|
||||
// the same name is how a module ends up unable to ack its own deliveries.
|
||||
// consumedSubject is where a consumed event lands, from the local pattern a module declared.
|
||||
//
|
||||
// **The mesh's wildcards become this transport's** (design 29 §1): `*` is one name on both, and `**`
|
||||
// — the rest — is `>` here. A module writes neither transport's spelling, so a manifest stays correct
|
||||
// when the wire changes, which is the whole reason names are local.
|
||||
//
|
||||
// `**` on its own is every event from every module: the emitter is any, the event is anything. An
|
||||
// audit logger wants exactly that and says so in one token.
|
||||
func consumedSubject(pattern string) (string, error) {
|
||||
if pattern == catalogueTheRest {
|
||||
return "mesh.mod.*.event.>", nil
|
||||
}
|
||||
emitter, event, named := strings.Cut(pattern, ".")
|
||||
if !named || emitter == "" || event == "" {
|
||||
return "", fmt.Errorf(
|
||||
"%q does not name an emitter and an event: a consumed event is <emitter>.<event>, or "+
|
||||
"%q for every event", pattern, catalogueTheRest)
|
||||
}
|
||||
if emitter == catalogueTheRest {
|
||||
return "", fmt.Errorf("%q stands for the rest of a name, so it cannot name the emitter", catalogueTheRest)
|
||||
}
|
||||
// Each name is checked before it becomes a subject: a name carrying a dot would add a token and
|
||||
// silently widen the permission, which is the whole reason safeSubject exists.
|
||||
var out []string
|
||||
for _, part := range strings.Split(event, ".") {
|
||||
switch part {
|
||||
case catalogueTheRest:
|
||||
out = append(out, ">")
|
||||
case "*":
|
||||
out = append(out, "*")
|
||||
default:
|
||||
if !safeSubject.MatchString(part) {
|
||||
return "", fmt.Errorf("%q cannot be part of a subject: it would widen the permission", part)
|
||||
}
|
||||
out = append(out, part)
|
||||
}
|
||||
}
|
||||
if emitter != "*" && !safeSubject.MatchString(emitter) {
|
||||
return "", fmt.Errorf("%q cannot name an emitter: it would widen the permission", emitter)
|
||||
}
|
||||
return "mesh.mod." + emitter + ".event." + strings.Join(out, "."), nil
|
||||
}
|
||||
|
||||
// catalogueTheRest is the mesh's wildcard for "the rest of a name", duplicated from the catalogue
|
||||
// package for the one direction of dependency the build queue's name is duplicated for.
|
||||
const catalogueTheRest = "**"
|
||||
|
||||
func consumerStream(p Principal) string {
|
||||
switch p.Kind {
|
||||
case KindModule:
|
||||
return "EVENTS"
|
||||
case KindNode:
|
||||
return "NODES"
|
||||
case KindController:
|
||||
return "CONTROL"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func consumerDurable(p Principal) string {
|
||||
switch p.Kind {
|
||||
case KindModule:
|
||||
return p.Node + "_" + p.Module
|
||||
case KindNode:
|
||||
return p.Node
|
||||
case KindController:
|
||||
return "controller"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Server is everything the composed file needs that is not a principal.
|
||||
type Server struct {
|
||||
// ClientPort carries TLS itself. There is no plaintext port beside it: a bus reachable
|
||||
// without TLS is one a module can reach without TLS by mistake.
|
||||
ClientPort int
|
||||
MonitoringPort int
|
||||
TLSCert string
|
||||
TLSKey string
|
||||
TLSCA string
|
||||
// StoreDir is a host directory bind, not a named volume — issue 115 is resolved and converted
|
||||
// four modules away from named volumes; the bus's own data is not the place to bring one back.
|
||||
StoreDir string
|
||||
}
|
||||
|
||||
// Compose renders the server's whole configuration. The order is stable and the output is
|
||||
// deterministic, because the file's digest is what the module's entrypoint watches to decide
|
||||
// whether to reload: a composer that reordered a map on each run would signal a reload every time
|
||||
// the controller restarted, for a file that had not changed.
|
||||
func Compose(s Server, principals []Principal) (string, error) {
|
||||
sorted := append([]Principal(nil), principals...)
|
||||
sort.Slice(sorted, func(i, j int) bool { return sorted[i].Username() < sorted[j].Username() })
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString("# Composed by the mesh controller. Do not edit: the next composition overwrites it.\n")
|
||||
b.WriteString("# Accounts and permissions are derived from what each module declares and nothing\n")
|
||||
b.WriteString("# else (novox/hq ADR 0043, design 29 §2).\n\n")
|
||||
|
||||
fmt.Fprintf(&b, "port: %d\n", s.ClientPort)
|
||||
fmt.Fprintf(&b, "http: 127.0.0.1:%d\n\n", s.MonitoringPort)
|
||||
|
||||
// **No `verify`, and it said `verify: true` until this configuration was run.** That setting
|
||||
// makes the server demand a *client* certificate, and nothing in the mesh presents one: a host
|
||||
// pins this server's exact certificate and authenticates with the password the mesh minted
|
||||
// (ADR 0004, design 25 §4), and so does a module's runtime. With it on, every connection in the
|
||||
// mesh is refused at the TLS handshake before any password is looked at, and the error —
|
||||
// "client didn't provide a certificate" — reads as a fault in the client.
|
||||
//
|
||||
// TLS is still required: the block is what requires it, and verify only decides whether client
|
||||
// certificates are checked.
|
||||
b.WriteString("tls {\n")
|
||||
fmt.Fprintf(&b, " cert_file: %q\n", s.TLSCert)
|
||||
fmt.Fprintf(&b, " key_file: %q\n", s.TLSKey)
|
||||
fmt.Fprintf(&b, " ca_file: %q\n", s.TLSCA)
|
||||
b.WriteString("}\n\n")
|
||||
|
||||
b.WriteString("jetstream {\n")
|
||||
fmt.Fprintf(&b, " store_dir: %q\n", s.StoreDir)
|
||||
b.WriteString("}\n\n")
|
||||
|
||||
accounts, err := ComposeAccounts(sorted)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
b.WriteString(accounts)
|
||||
return b.String(), nil
|
||||
}
|
||||
|
||||
// ComposeAccounts is the accounts block alone — every user, and nothing about the server.
|
||||
//
|
||||
// **This is the only part of the configuration the mesh writes, and the split is deliberate.** A
|
||||
// server's ports, its TLS paths and its store directory are properties of the container the module
|
||||
// raises: they live in its image and its mounts, and they change when it does. The controller has no
|
||||
// business knowing them, and a controller that did would have to be kept in step with a Dockerfile
|
||||
// it never sees. What only the mesh knows is *who may connect*, so that is what it writes, and the
|
||||
// module's own configuration includes it.
|
||||
//
|
||||
// Four things checked against a running server before this shape was committed to: a user in an
|
||||
// included file authenticates; an unknown user is refused, so the include is the whole authority
|
||||
// rather than an addition to something; a publish outside a user's grant is refused; and rewriting
|
||||
// this file alone and signalling a reload makes a new user appear **without dropping the connection
|
||||
// the mesh already has** — which is what makes every later account, permission or person change cost
|
||||
// nothing (task 1.2's payoff).
|
||||
func ComposeAccounts(principals []Principal) (string, error) {
|
||||
sorted := append([]Principal(nil), principals...)
|
||||
sort.Slice(sorted, func(i, j int) bool { return sorted[i].Username() < sorted[j].Username() })
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString("# The mesh's users, composed by the controller. Do not edit: the next\n")
|
||||
b.WriteString("# composition overwrites it. Permissions are derived from what each module\n")
|
||||
b.WriteString("# declares and nothing else (novox/hq ADR 0043, design 29 §2).\n\n")
|
||||
|
||||
// One account for the mesh: accounts in NATS isolate subject spaces entirely, and the mesh is
|
||||
// one space (design 25 §4). The cost of that — that permissions are the only isolation — is
|
||||
// paid in the scoping of every inbox and every ack subject.
|
||||
// JetStream is enabled per account once accounts exist at all: with only the global block set,
|
||||
// a user in MESH is told "JetStream not enabled for account" the first time it binds a
|
||||
// consumer, which is the first thing every host does (2026-09-28).
|
||||
b.WriteString("accounts {\n MESH {\n jetstream: enabled\n users = [\n")
|
||||
for _, p := range sorted {
|
||||
perms, err := PermissionsFor(p)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if p.PasswordHash == "" {
|
||||
return "", fmt.Errorf("%s has no password hash: a user without one is a user anybody is", p.Username())
|
||||
}
|
||||
fmt.Fprintf(&b, " { user: %q, password: %q, permissions: {\n", p.Username(), p.PasswordHash)
|
||||
fmt.Fprintf(&b, " publish: { allow: [%s] }\n", quoted(perms.Publish))
|
||||
fmt.Fprintf(&b, " subscribe: { allow: [%s] }\n", quoted(perms.Subscribe))
|
||||
if perms.AllowResponses {
|
||||
b.WriteString(" allow_responses: { max: 1, ttl: \"1m\" }\n")
|
||||
}
|
||||
b.WriteString(" } }\n")
|
||||
}
|
||||
b.WriteString(" ]\n }\n}\n")
|
||||
return b.String(), nil
|
||||
}
|
||||
|
||||
func quoted(values []string) string {
|
||||
if len(values) == 0 {
|
||||
return ""
|
||||
}
|
||||
out := make([]string, len(values))
|
||||
for i, v := range values {
|
||||
out[i] = fmt.Sprintf("%q", v)
|
||||
}
|
||||
return strings.Join(out, ", ")
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var update = flag.Bool("update", false, "rewrite the golden composition")
|
||||
|
||||
// The composed file is the mesh's whole authority model, so a change to it should be visible in a
|
||||
// review rather than inferred from a diff of Go. The fixture is also the exact text checked
|
||||
// against the real server's parser (`nats-server -t`), which is what says this syntax is the
|
||||
// server's and not one we invented.
|
||||
func TestTheComposedConfigMatchesTheGolden(t *testing.T) {
|
||||
seat := Seat{Name: "telegram-sender", Accepts: []string{"send"}, Emits: []string{"delivered", "failed"}}
|
||||
got, err := Compose(
|
||||
Server{ClientPort: 4222, MonitoringPort: 8222, StoreDir: "/data",
|
||||
TLSCert: "/tls/tls.crt", TLSKey: "/tls/tls.key", TLSCA: "/tls/ca.crt"},
|
||||
[]Principal{
|
||||
{Kind: KindController, PasswordHash: "$2a$11$cccccccccccccccccccccc"},
|
||||
{Kind: KindEnrolment, Node: "one", PasswordHash: "$2a$11$eeeeeeeeeeeeeeeeeeeeee"},
|
||||
{Kind: KindNode, Node: "one", PasswordHash: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn"},
|
||||
{Kind: KindModule, Node: "one", Module: "telegram", Holds: []Seat{seat},
|
||||
Serves: []string{"status"}, PasswordHash: "$2a$11$tttttttttttttttttttttt"},
|
||||
{Kind: KindModule, Node: "two", Module: "shop", Uses: []Seat{seat},
|
||||
Emits: []string{"order.placed"}, PasswordHash: "$2a$11$ssssssssssssssssssssss"},
|
||||
{Kind: KindModule, Node: "two", Module: "audit",
|
||||
Consumes: []string{"shop.order.placed"}, PasswordHash: "$2a$11$aaaaaaaaaaaaaaaaaaaaaa"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
golden := filepath.Join("testdata", "composed.conf")
|
||||
if *update {
|
||||
if err := os.WriteFile(golden, []byte(got), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
want, err := os.ReadFile(golden)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != string(want) {
|
||||
t.Errorf("composition changed; re-run with -update and read the diff:\n%s", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,326 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func has(t *testing.T, subjects []string, want string) {
|
||||
t.Helper()
|
||||
for _, s := range subjects {
|
||||
if s == want {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("expected %q among %v", want, subjects)
|
||||
}
|
||||
|
||||
func hasNot(t *testing.T, subjects []string, unwanted string) {
|
||||
t.Helper()
|
||||
for _, s := range subjects {
|
||||
if s == unwanted {
|
||||
t.Fatalf("did not expect %q among %v", unwanted, subjects)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A module's authority comes from its declaration and nothing else (novox/hq ADR 0043).
|
||||
func TestAModulePublishesOnlyWhatItEmits(t *testing.T) {
|
||||
p := Principal{Kind: KindModule, Node: "one", Module: "billing",
|
||||
Emits: []string{"order.placed"}, PasswordHash: "x"}
|
||||
perms, err := PermissionsFor(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
has(t, perms.Publish, "mesh.mod.billing.event.order.placed")
|
||||
hasNot(t, perms.Publish, "mesh.mod.billing.>")
|
||||
hasNot(t, perms.Publish, "mesh.mod.shipping.event.order.placed")
|
||||
}
|
||||
|
||||
// The gap AMQP left open — an emitter granted the events exchange whole — is closed by per-subject
|
||||
// permissions. A module cannot publish under another module's name.
|
||||
func TestAModuleCannotPublishUnderAnothersName(t *testing.T) {
|
||||
perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "billing",
|
||||
Emits: []string{"order.placed"}, PasswordHash: "x"})
|
||||
for _, p := range perms.Publish {
|
||||
if strings.HasPrefix(p, "mesh.mod.") && !strings.HasPrefix(p, "mesh.mod.billing.") {
|
||||
t.Fatalf("billing may publish %q, which is not its own namespace", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A caller of a seat may publish what the seat accepts, and nothing else of it: not its outbound
|
||||
// events, and not a subscription to its inbound queue (design 29 §2).
|
||||
func TestUsingASeatIsPublishOnlyAndInboundOnly(t *testing.T) {
|
||||
seat := Seat{Name: "telegram-sender", Accepts: []string{"send"}, Emits: []string{"delivered", "failed"}}
|
||||
perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "shop",
|
||||
Uses: []Seat{seat}, PasswordHash: "x"})
|
||||
has(t, perms.Publish, "mesh.seat.telegram-sender.accept.send")
|
||||
hasNot(t, perms.Publish, "mesh.seat.telegram-sender.event.delivered")
|
||||
hasNot(t, perms.Subscribe, "mesh.seat.telegram-sender.accept.send")
|
||||
}
|
||||
|
||||
// The holder is the mirror image: it consumes what the seat accepts and publishes what it emits.
|
||||
func TestHoldingASeatIsTheMirrorOfUsingIt(t *testing.T) {
|
||||
seat := Seat{Name: "telegram-sender", Accepts: []string{"send"}, Emits: []string{"delivered", "failed"}}
|
||||
perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "telegram",
|
||||
Holds: []Seat{seat}, PasswordHash: "x"})
|
||||
has(t, perms.Subscribe, "mesh.seat.telegram-sender.accept.send")
|
||||
has(t, perms.Publish, "mesh.seat.telegram-sender.event.delivered")
|
||||
hasNot(t, perms.Publish, "mesh.seat.telegram-sender.accept.send")
|
||||
}
|
||||
|
||||
// Without an ack permission a durable consumer never really consumes: every message it receives is
|
||||
// redelivered forever, refused by the permission list it already has (design 25 §4).
|
||||
func TestAModuleMayAckItsOwnDeliveriesAndNoOthers(t *testing.T) {
|
||||
perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "billing",
|
||||
Consumes: []string{"shop.order.placed"}, PasswordHash: "x"})
|
||||
has(t, perms.Publish, "$JS.ACK.EVENTS.one_billing.>")
|
||||
hasNot(t, perms.Publish, "$JS.ACK.>")
|
||||
hasNot(t, perms.Publish, "$JS.ACK.EVENTS.one_shop.>")
|
||||
}
|
||||
|
||||
// With one account, inbox privacy is the permission list or it is nothing.
|
||||
func TestAnInboxIsScopedToItsOwner(t *testing.T) {
|
||||
perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "billing", PasswordHash: "x"})
|
||||
has(t, perms.Subscribe, "_INBOX.one.billing.>")
|
||||
hasNot(t, perms.Subscribe, "_INBOX.>")
|
||||
hasNot(t, perms.Subscribe, "_INBOX.one.shop.>")
|
||||
}
|
||||
|
||||
// A responder answers on the caller's inbox, which it has no permission for. allow_responses is
|
||||
// what makes a scoped inbox workable at all — the authority is bounded by having been asked.
|
||||
func TestOnlySomethingThatServesMayAnswer(t *testing.T) {
|
||||
serving, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "billing",
|
||||
Serves: []string{"status"}, PasswordHash: "x"})
|
||||
if !serving.AllowResponses {
|
||||
t.Fatal("a module serving a tool cannot answer the caller's inbox")
|
||||
}
|
||||
consumer, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "audit",
|
||||
Consumes: []string{"shop.order.placed"}, PasswordHash: "x"})
|
||||
if consumer.AllowResponses {
|
||||
t.Fatal("a pure consumer was granted the right to answer, which nothing asked it to do")
|
||||
}
|
||||
}
|
||||
|
||||
// A host reaches its own node's control traffic and its own declaration, and nothing of any
|
||||
// other node's.
|
||||
func TestAHostIsConfinedToItsOwnNode(t *testing.T) {
|
||||
perms, _ := PermissionsFor(Principal{Kind: KindNode, Node: "one", PasswordHash: "x"})
|
||||
has(t, perms.Publish, "mesh.control.one.>")
|
||||
has(t, perms.Subscribe, "mesh.node.one.declare")
|
||||
hasNot(t, perms.Subscribe, "mesh.node.two.declare")
|
||||
hasNot(t, perms.Subscribe, "mesh.node.>")
|
||||
}
|
||||
|
||||
// A leaked enrolment token is useless for anything but enrolling (design 25 §6).
|
||||
func TestTheEnrolmentUserCanOnlyEnrol(t *testing.T) {
|
||||
perms, err := PermissionsFor(Principal{Kind: KindEnrolment, Node: "anchor", PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(perms.Publish) != 1 || perms.Publish[0] != "mesh.control.enrol" {
|
||||
t.Fatalf("enrolment may publish %v", perms.Publish)
|
||||
}
|
||||
// Its own inbox and nothing else. **Nothing else** is the point: no declaration, no event, and
|
||||
// no other machine's answer — and the inbox itself is needed, because a node that cannot
|
||||
// subscribe one waits out its timeout against a mesh that answered.
|
||||
if len(perms.Subscribe) != 1 || perms.Subscribe[0] != "_INBOX.enrol.anchor.>" {
|
||||
t.Fatalf("enrolment may subscribe %v, which is not its own inbox alone", perms.Subscribe)
|
||||
}
|
||||
}
|
||||
|
||||
// An enrolment user that names no node is refused: its inbox would be an empty subject token, and
|
||||
// one that every nameless enrolment user shared — which is one machine reading the credentials
|
||||
// sealed to another.
|
||||
func TestAnEnrolmentUserWithoutANodeIsRefused(t *testing.T) {
|
||||
if _, err := PermissionsFor(Principal{Kind: KindEnrolment, PasswordHash: "x"}); err == nil {
|
||||
t.Fatal("an enrolment user with no node was composed, so its inbox is shared")
|
||||
}
|
||||
}
|
||||
|
||||
// A name that would widen a permission is refused rather than quietly stretching one.
|
||||
func TestANameThatWouldWidenAPermissionIsRefused(t *testing.T) {
|
||||
for _, bad := range []string{"bill.ing", "billing.>", "*", "bil>ling"} {
|
||||
if _, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: bad, PasswordHash: "x"}); err == nil {
|
||||
t.Fatalf("%q was accepted as part of a subject", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The entrypoint reloads on the file's digest changing, so an unchanged mesh must compose an
|
||||
// identical file — otherwise every controller restart signals a reload of the whole bus.
|
||||
func TestComposingTwiceGivesTheSameBytes(t *testing.T) {
|
||||
s := Server{ClientPort: 4222, MonitoringPort: 8222, StoreDir: "/data",
|
||||
TLSCert: "/tls/tls.crt", TLSKey: "/tls/tls.key", TLSCA: "/tls/ca.crt"}
|
||||
ps := []Principal{
|
||||
{Kind: KindModule, Node: "two", Module: "shop", Emits: []string{"order.placed"}, PasswordHash: "b"},
|
||||
{Kind: KindController, PasswordHash: "c"},
|
||||
{Kind: KindModule, Node: "one", Module: "billing", Consumes: []string{"shop.order.placed"}, PasswordHash: "a"},
|
||||
}
|
||||
first, err := Compose(s, ps)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
shuffled := []Principal{ps[2], ps[0], ps[1]}
|
||||
second, err := Compose(s, shuffled)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first != second {
|
||||
t.Fatal("composition is order-dependent; every controller restart would reload the bus")
|
||||
}
|
||||
}
|
||||
|
||||
// A user without a password is a user anybody is.
|
||||
func TestAUserWithoutAPasswordIsRefused(t *testing.T) {
|
||||
_, err := Compose(Server{ClientPort: 4222}, []Principal{{Kind: KindController}})
|
||||
if err == nil {
|
||||
t.Fatal("composed a user with no password hash")
|
||||
}
|
||||
}
|
||||
|
||||
// A person reaches the mesh's tools from a workstation (design 25 §7). Their authority is a list
|
||||
// of tools and nothing else.
|
||||
func TestAPersonMayAskOnlyTheToolsTheyWereGiven(t *testing.T) {
|
||||
perms, err := PermissionsFor(Principal{Kind: KindPerson, Module: "jo",
|
||||
Invokes: []string{"shop.price", "telegram.status"}, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
has(t, perms.Publish, "mesh.mod.shop.tool.price")
|
||||
has(t, perms.Publish, "mesh.mod.telegram.tool.status")
|
||||
hasNot(t, perms.Publish, "mesh.mod.shop.tool.refund")
|
||||
hasNot(t, perms.Publish, "mesh.mod.*.tool.>")
|
||||
}
|
||||
|
||||
// An administrator gets every tool, which is a different grant and looks like one.
|
||||
func TestAnAdministratorMayAskAnyTool(t *testing.T) {
|
||||
perms, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo",
|
||||
Invokes: []string{"*"}, PasswordHash: "x"})
|
||||
has(t, perms.Publish, "mesh.mod.*.tool.>")
|
||||
}
|
||||
|
||||
// **Nothing but tools.** A person who could publish an event would be able to claim a module
|
||||
// said something; one who could publish control traffic would be a second controller.
|
||||
func TestAPersonReachesNothingButTools(t *testing.T) {
|
||||
perms, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo",
|
||||
Invokes: []string{"*"}, PasswordHash: "x"})
|
||||
for _, p := range perms.Publish {
|
||||
if !strings.Contains(p, ".tool.") {
|
||||
t.Errorf("a person may publish %q, which is not a tool call", p)
|
||||
}
|
||||
}
|
||||
for _, s := range perms.Subscribe {
|
||||
if !strings.HasPrefix(s, "_INBOX.person.") {
|
||||
t.Errorf("a person may subscribe %q; only their own inbox should be reachable", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A person has no durable consumer, because nothing is delivered to a person — so no ack
|
||||
// subject, and an ack permission would be authority over something that does not exist.
|
||||
func TestAPersonHasNoAckSubject(t *testing.T) {
|
||||
perms, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo",
|
||||
Invokes: []string{"*"}, PasswordHash: "x"})
|
||||
for _, p := range perms.Publish {
|
||||
if strings.HasPrefix(p, "$JS.ACK") {
|
||||
t.Errorf("a person was granted %q, and has no consumer to acknowledge", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A person asks and is answered; they never answer. allow_responses would let a person reply to
|
||||
// a request — which, on a bus where anyone may serve a tool, is somebody impersonating a module.
|
||||
func TestAPersonMayNotAnswer(t *testing.T) {
|
||||
perms, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo",
|
||||
Invokes: []string{"*"}, PasswordHash: "x"})
|
||||
if perms.AllowResponses {
|
||||
t.Fatal("a person may answer a request, which is impersonating a module")
|
||||
}
|
||||
}
|
||||
|
||||
// Two people do not share an inbox, or one would read the other's answers.
|
||||
func TestTwoPeopleDoNotShareAnInbox(t *testing.T) {
|
||||
a, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo", Invokes: []string{"*"}, PasswordHash: "x"})
|
||||
b, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "sam", Invokes: []string{"*"}, PasswordHash: "x"})
|
||||
if a.Subscribe[0] == b.Subscribe[0] {
|
||||
t.Fatalf("both read %s", a.Subscribe[0])
|
||||
}
|
||||
}
|
||||
|
||||
// A malformed grant is refused rather than widened into something that happens to parse.
|
||||
func TestAToolGrantThatNamesNoToolIsRefused(t *testing.T) {
|
||||
if _, err := PermissionsFor(Principal{Kind: KindPerson, Module: "jo",
|
||||
Invokes: []string{"shop"}, PasswordHash: "x"}); err == nil {
|
||||
t.Fatal("a grant naming a module but no tool was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// The controller can answer an enrolment, and reach no other inbox.
|
||||
//
|
||||
// **`allow_responses` does not cover this and that is the trap.** It permits one reply to the reply
|
||||
// subject of a message the user received — and a message a JetStream consumer delivers has had that
|
||||
// field claimed for the consumer's own ack address, so the address the controller actually answers is
|
||||
// the one the request carried in its payload, which the server does not recognise as a reply subject
|
||||
// at all.
|
||||
//
|
||||
// Found against a real server, after a live test on an *unpermissioned* one had passed: every
|
||||
// enrolment on the mesh would have timed out while the controller logged success.
|
||||
func TestTheControllerCanAnswerAnEnrolmentAndReachNoOtherInbox(t *testing.T) {
|
||||
ctl, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
enrolling, err := PermissionsFor(Principal{Kind: KindEnrolment, Node: "anchor", PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Whatever the enrolling node waits on, the controller must be able to publish to.
|
||||
if len(enrolling.Subscribe) != 1 {
|
||||
t.Fatalf("an enrolling node subscribes %v, and this test knows only how to check one",
|
||||
enrolling.Subscribe)
|
||||
}
|
||||
waitsOn := enrolling.Subscribe[0]
|
||||
if !covers(ctl.Publish, waitsOn) {
|
||||
t.Fatalf("the controller may publish %v, none of which reaches %s — so every enrolment on "+
|
||||
"the mesh times out while the controller logs success", ctl.Publish, waitsOn)
|
||||
}
|
||||
|
||||
// And nothing wider. A node's own inbox and a module's are not the controller's to write into:
|
||||
// that is the blanket grant design 25 §4 refuses.
|
||||
for _, other := range []string{"_INBOX.node.anchor.x", "_INBOX.one.shop.x", "_INBOX.person.ada.x"} {
|
||||
if covers(ctl.Publish, other) {
|
||||
t.Errorf("the controller can publish to %s, which is an inbox privacy the permission "+
|
||||
"list is the only thing protecting", other)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// covers says whether any granted subject pattern admits one concrete subject, with NATS's own
|
||||
// wildcard meanings: `*` is one token, `>` is the rest.
|
||||
func covers(granted []string, subject string) bool {
|
||||
want := strings.Split(subject, ".")
|
||||
for _, pattern := range granted {
|
||||
if admits(strings.Split(pattern, "."), want) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func admits(pattern, subject []string) bool {
|
||||
for i, token := range pattern {
|
||||
if token == ">" {
|
||||
return i < len(subject)
|
||||
}
|
||||
if i >= len(subject) {
|
||||
return false
|
||||
}
|
||||
if token != "*" && token != subject[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(pattern) == len(subject)
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/envfile"
|
||||
)
|
||||
|
||||
// Whether this mesh's own traffic is on the bus being built.
|
||||
//
|
||||
// **One switch, read in one place** (novox/hq ADR 0116 step 5). Every seam the bus change went
|
||||
// behind ships both implementations, and until the rollout every one of them chooses the bus the
|
||||
// mesh runs on today. This is what the rollout flips, and it is deliberately a single fact rather
|
||||
// than a fact per component: a controller whose outbound is on one bus and whose inbound is on the
|
||||
// other is a mesh that hears nothing, and no test of either half would catch it.
|
||||
|
||||
// NATSVar is where the controller finds the bus being built. Unset is the ordinary case and means
|
||||
// the mesh runs on the bus it has always run on.
|
||||
const NATSVar = "MESH_BUS_NATS"
|
||||
|
||||
// OnNATS is the address of the bus being built, and whether the mesh is on it.
|
||||
//
|
||||
// Read from the node's own settings rather than baked in, for the reason the broker's address is
|
||||
// (novox/hq 04-ISSUES/102): an address recorded once does not follow a node's ports.
|
||||
func OnNATS() (address string, on bool, err error) {
|
||||
address, err = envfile.Placed(NATSVar)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
address = strings.TrimSpace(address)
|
||||
if address == "" {
|
||||
return "", false, nil
|
||||
}
|
||||
return address, true, nil
|
||||
}
|
||||
|
||||
// CredentialIn reads the user and password out of a bus address, and the address without them.
|
||||
//
|
||||
// The controller's own credential arrives in its address, the way the old bus's does. Split out so the
|
||||
// controller can record a hash of what it is actually using: its user is created by the installer at a
|
||||
// bootstrap password, before the controller exists to mint one, and a composition that left itself out
|
||||
// would produce a bus the writer cannot connect to.
|
||||
func CredentialIn(address string) (user, password, bare string) {
|
||||
at := strings.LastIndex(address, "@")
|
||||
if at < 0 {
|
||||
return "", "", address
|
||||
}
|
||||
scheme := ""
|
||||
rest := address[:at]
|
||||
if i := strings.Index(rest, "://"); i >= 0 {
|
||||
scheme, rest = rest[:i+3], rest[i+3:]
|
||||
}
|
||||
user, password, _ = strings.Cut(rest, ":")
|
||||
return user, password, scheme + address[at+1:]
|
||||
}
|
||||
|
||||
// BareAddress is a bus address with any credential stripped, for something that only needs to know
|
||||
// whether a server is answering there.
|
||||
func BareAddress(address string) string {
|
||||
_, _, bare := CredentialIn(address)
|
||||
if bare == "" {
|
||||
return address
|
||||
}
|
||||
if strings.Contains(bare, "://") {
|
||||
return bare
|
||||
}
|
||||
return "nats://" + bare
|
||||
}
|
||||
|
||||
// BusAddress is where the mesh's bus is, with this process's credential, and refuses to be empty:
|
||||
// there is one bus, and a control plane without it can hold records and answer nothing (novox/hq
|
||||
// ADR 0131, design 28 task 5.5).
|
||||
func BusAddress() (string, error) {
|
||||
address, _, err := OnNATS()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if address == "" {
|
||||
return "", fmt.Errorf("this control plane has no %s, so it cannot reach the mesh's bus", NATSVar)
|
||||
}
|
||||
return address, nil
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Which bus the mesh is on is one fact, and being told about both is refused.
|
||||
//
|
||||
|
||||
// The controller's own credential arrives in its address, and has to be readable out of it — its user
|
||||
// is created by the installer at a bootstrap password, before the controller exists to mint one.
|
||||
func TestACredentialIsReadOutOfABusAddress(t *testing.T) {
|
||||
for _, c := range []struct{ in, user, password, bare string }{
|
||||
{"nats://controller:secret@127.0.0.1:4222", "controller", "secret", "nats://127.0.0.1:4222"},
|
||||
{"controller:secret@127.0.0.1:4222", "controller", "secret", "127.0.0.1:4222"},
|
||||
{"nats://127.0.0.1:4222", "", "", "nats://127.0.0.1:4222"},
|
||||
{"127.0.0.1:4222", "", "", "127.0.0.1:4222"},
|
||||
// A password containing an at-sign: split on the last one, or the address becomes part of the
|
||||
// credential and the connection goes somewhere nobody named.
|
||||
{"nats://controller:a@b@127.0.0.1:4222", "controller", "a@b", "nats://127.0.0.1:4222"},
|
||||
} {
|
||||
user, password, bare := CredentialIn(c.in)
|
||||
if user != c.user || password != c.password || bare != c.bare {
|
||||
t.Errorf("%q read as %q/%q at %q; wanted %q/%q at %q",
|
||||
c.in, user, password, bare, c.user, c.password, c.bare)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package broker
|
||||
|
||||
import "fmt"
|
||||
|
||||
// Bringing the bus's own objects into being, in the one order that works.
|
||||
//
|
||||
// **Asserted on every start rather than created once at genesis.** A stream somebody deleted, a mesh
|
||||
// raised from a restored backup, or a bus whose data directory was replaced all have records and no
|
||||
// objects — and a node whose consumer is missing hears nothing while everything else about it looks
|
||||
// correct. Idempotence is the whole requirement, and the parts are already idempotent; this is the
|
||||
// order they have to be asked in.
|
||||
|
||||
// Raiser is everything asserting the bus's objects needs of a connection to it.
|
||||
type Raiser interface {
|
||||
Asserter
|
||||
Ensurer
|
||||
}
|
||||
|
||||
// Raise asserts the mesh's streams, the controller's own consumers, and one consumer per node.
|
||||
//
|
||||
// **The order is not a preference.** A consumer on a stream that does not exist is refused, and the
|
||||
// refusal names the stream rather than the order — so somebody reading it goes looking for a deleted
|
||||
// stream instead of a reversed pair of lines. Nodes last, because the one a node reads lives on a
|
||||
// stream the mesh's own set defines.
|
||||
func Raise(r Raiser, nodes []string) error {
|
||||
if err := AssertMeshStreams(r); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := AssertMeshConsumers(r); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := AssertNodeConsumers(r, nodes); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RaiseSeats asserts one work queue per declared seat, and the worker of whoever holds it.
|
||||
//
|
||||
// Separate from Raise because it is answered by a different question: the mesh's own objects exist
|
||||
// because the mesh does, and a seat's exist because a module declaring one was registered. Kept
|
||||
// beside it so the order is visible — a holder's worker needs the seat's stream, and a seat's stream
|
||||
// needs nothing.
|
||||
func RaiseSeats(r Raiser, seats []DeclaredSeat, holders map[string]Holder) error {
|
||||
for _, s := range SeatStreams(seats) {
|
||||
if err := r.EnsureStream(s); err != nil {
|
||||
return fmt.Errorf("asserting the work queue for %s: %w", s.Name, err)
|
||||
}
|
||||
}
|
||||
for _, s := range seats {
|
||||
h, held := holders[s.Name]
|
||||
if !held {
|
||||
// **The stream exists and the consumer does not, on purpose.** Work queues until a
|
||||
// holder appears, so installing the module a week after something started sending to it
|
||||
// flushes the backlog instead of having lost it.
|
||||
continue
|
||||
}
|
||||
c, needed := HolderConsumerFor(h.Node, h.Module, s)
|
||||
if !needed {
|
||||
continue
|
||||
}
|
||||
if err := r.EnsureConsumer(c); err != nil {
|
||||
return fmt.Errorf("asserting how %s on %s works %s: %w", h.Module, h.Node, s.Name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Holder is which module on which machine holds a seat.
|
||||
type Holder struct {
|
||||
Node string
|
||||
Module string
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// Raising the bus's objects against a real server.
|
||||
//
|
||||
// The pure tests above say what is asked for and in what order. Only a server can say whether it
|
||||
// accepts them — and two of these are claims about the server's own behaviour that nothing else
|
||||
// could answer: that asserting twice changes nothing, and that a consumer really is bound to the one
|
||||
// subject its node is allowed to read.
|
||||
//
|
||||
// docker run -d --rm --name t -p 14227:4222 nats:2.10-alpine -js
|
||||
// MESH_TEST_NATS=nats://127.0.0.1:14227 go test ./internal/broker/ -run TestRaising
|
||||
|
||||
func aLiveBus(t *testing.T) *JetStream {
|
||||
t.Helper()
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
js, err := Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
// **Nothing is deleted here, deliberately.** These objects are the mesh's own and every live
|
||||
// test in every package shares one server: a test that deleted a stream to get a clean slate
|
||||
// took it out from under whatever was running beside it, and the failure landed in the other
|
||||
// test as "stream not found" — which reads as a bug in the code under test. Raise is idempotent
|
||||
// by requirement, so asserting against whatever is already there is both safe and the realistic
|
||||
// case.
|
||||
return js
|
||||
}
|
||||
|
||||
// Every object the mesh's own traffic needs, accepted by a real server, and asserting again changes
|
||||
// nothing — which is the whole requirement, because this runs on every start.
|
||||
func TestRaisingTheBusIsAcceptedAndIdempotent(t *testing.T) {
|
||||
js := aLiveBus(t)
|
||||
|
||||
if err := Raise(js, []string{"anchor", "laptop"}); err != nil {
|
||||
t.Fatalf("a real server refused the mesh's own objects: %v", err)
|
||||
}
|
||||
// Twice, with nothing in between. A start that failed the second time is a controller that
|
||||
// cannot restart.
|
||||
if err := Raise(js, []string{"anchor", "laptop"}); err != nil {
|
||||
t.Fatalf("asserting the bus's objects a second time failed, so a restart would: %v", err)
|
||||
}
|
||||
// And again with a machine that was not there before, which is what enrolling one is.
|
||||
if err := Raise(js, []string{"anchor", "laptop", "workstation"}); err != nil {
|
||||
t.Fatalf("a machine joining an already-raised bus was refused: %v", err)
|
||||
}
|
||||
|
||||
for _, s := range MeshStreams() {
|
||||
if _, err := js.Context().StreamInfo(s.Name); err != nil {
|
||||
t.Errorf("stream %s is not there: %v", s.Name, err)
|
||||
}
|
||||
}
|
||||
for _, c := range MeshConsumers() {
|
||||
if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); err != nil {
|
||||
t.Errorf("the controller's consumer on %s is not there: %v", c.Stream, err)
|
||||
}
|
||||
}
|
||||
for _, node := range []string{"anchor", "laptop", "workstation"} {
|
||||
info, err := js.Context().ConsumerInfo("NODES", node)
|
||||
if err != nil {
|
||||
t.Errorf("%s has no way to hear its declaration: %v", node, err)
|
||||
continue
|
||||
}
|
||||
// **Its own subject and no other node's.** A consumer filtered on anything wider is a node
|
||||
// reading another machine's declaration, and its own ack grant would not cover it either.
|
||||
if info.Config.FilterSubject != "mesh.node."+node+".declare" {
|
||||
t.Errorf("%s's consumer reads %q", node, info.Config.FilterSubject)
|
||||
}
|
||||
if info.Config.AckPolicy != nats.AckExplicitPolicy {
|
||||
t.Errorf("%s's consumer acknowledges on delivery, so a declaration it died applying is "+
|
||||
"never sent again", node)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The store window needs unlimited redelivery on CONTROL: the bound belongs to the controller, and a
|
||||
// server that dead-lettered first would discard the push the stream exists to protect.
|
||||
func TestTheControlConsumerDoesNotDeadLetterBeforeTheControllerGivesUp(t *testing.T) {
|
||||
js := aLiveBus(t)
|
||||
if err := Raise(js, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := js.Context().ConsumerInfo("CONTROL", ControllerName)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.Config.MaxDeliver > 0 {
|
||||
t.Fatalf("max-deliver is %d: a push held through a store restart would be dead-lettered "+
|
||||
"before the controller finished deciding about it", info.Config.MaxDeliver)
|
||||
}
|
||||
}
|
||||
|
||||
// A role's work queue exists before anybody holds it, against a real server.
|
||||
//
|
||||
// **The queue before the holder is the point** (novox/hq ADR 0121): work queues until somebody arrives
|
||||
// to do it, so assigning a build machine a week after something started asking for builds flushes the
|
||||
// backlog instead of having lost it. A stream created at assignment would make "the holder is not here
|
||||
// yet" mean "your requests are gone".
|
||||
func TestRaisingAMeshRolesWorkQueue(t *testing.T) {
|
||||
js := aLiveBus(t)
|
||||
seats := []DeclaredSeat{{Name: "mesh-build-machine", Accepts: []string{"build"},
|
||||
Emits: []string{"built"}}}
|
||||
t.Cleanup(func() { _ = js.Context().DeleteStream("SEAT_MESH_BUILD_MACHINE") })
|
||||
|
||||
if err := RaiseSeats(js, seats, nil); err != nil {
|
||||
t.Fatalf("a real server refused a role's work queue: %v", err)
|
||||
}
|
||||
info, err := js.Context().StreamInfo("SEAT_MESH_BUILD_MACHINE")
|
||||
if err != nil {
|
||||
t.Fatalf("the role has no work queue: %v", err)
|
||||
}
|
||||
if info.Config.Retention != nats.WorkQueuePolicy {
|
||||
t.Errorf("the queue retains as %v: work a holder took must leave it, or the next holder does "+
|
||||
"it again", info.Config.Retention)
|
||||
}
|
||||
if len(info.Config.Subjects) != 1 || info.Config.Subjects[0] != "mesh.seat.mesh-build-machine.accept.>" {
|
||||
t.Errorf("it carries %v rather than the role's own inbound subjects", info.Config.Subjects)
|
||||
}
|
||||
// Nobody holds it, so there is no worker — and asserting again changes nothing, because this runs
|
||||
// on every start.
|
||||
if err := RaiseSeats(js, seats, nil); err != nil {
|
||||
t.Fatalf("asserting a role's queue a second time failed, so a restart would: %v", err)
|
||||
}
|
||||
|
||||
// And once somebody holds it, the worker appears on that same queue.
|
||||
if err := RaiseSeats(js, seats, map[string]Holder{
|
||||
"mesh-build-machine": {Node: "anchor", Module: "builder"},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.Context().ConsumerInfo("SEAT_MESH_BUILD_MACHINE",
|
||||
"SEAT_MESH_BUILD_MACHINE_worker"); err != nil {
|
||||
t.Fatalf("the holder got no worker on the role's queue: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// **A consumer created after the fact still sees what came before it**, which is why the mesh needs no
|
||||
// catch-up at all on this bus (novox/hq 04-ISSUES/050).
|
||||
//
|
||||
// On the bus the mesh runs on today a queue receives only what is published after it is bound, so
|
||||
// everything built before the catalogue existed was announced to nobody — and on a fresh mesh that is
|
||||
// always the foundation, because those are the things the catalogue needed in order to exist. A whole
|
||||
// mechanism was built for it: the catalogue asks, the controller re-publishes.
|
||||
//
|
||||
// A stream is a log and a consumer is a position in it. A consumer created later starts at the
|
||||
// beginning by default, so the builds are simply there. Asked of a real server rather than assumed,
|
||||
// because the whole decision about whether to keep that mechanism rests on it.
|
||||
func TestAConsumerCreatedAfterwardsStillSeesWhatCameBefore(t *testing.T) {
|
||||
js := aLiveBus(t)
|
||||
if err := AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.Context().PurgeStream("EVENTS"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Genesis: things are built before anything is listening.
|
||||
built := []string{"base", "store", "mesh-catalog"}
|
||||
for _, m := range built {
|
||||
if _, err := js.Context().Publish("mesh.seat.mesh-build-machine.event.built",
|
||||
[]byte(`{"module":"`+m+`"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Now the catalogue is installed and the controller creates its consumer.
|
||||
c, ok := ConsumerFor(Principal{Kind: KindModule, Node: "one", Module: "mesh-catalog",
|
||||
Watches: []Seat{{Name: "mesh-build-machine", Emits: []string{"built"}}}, PasswordHash: "x"})
|
||||
if !ok {
|
||||
t.Fatal("a module that watches a role got no consumer")
|
||||
}
|
||||
t.Cleanup(func() { _ = js.Context().DeleteConsumer(c.Stream, c.Name) })
|
||||
if err := js.EnsureConsumer(c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
info, err := js.Context().ConsumerInfo(c.Stream, c.Name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.NumPending != uint64(len(built)) {
|
||||
t.Fatalf("a consumer created after %d builds has %d waiting for it — if this is 0 the mesh "+
|
||||
"does need a catch-up after all, and the reasoning for deleting it is wrong",
|
||||
len(built), info.NumPending)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Whether a mesh could move its bus, and what is missing if not.
|
||||
//
|
||||
// **Asked before anything moves, and answerable from records alone.** The rollout moves every node at
|
||||
// once (novox/hq ADR 0116 step 5), so there is no partial state to inspect afterwards and no half to
|
||||
// roll back: either the mesh was ready or it was not. That makes a readiness question the most
|
||||
// valuable thing here — it costs nothing, it can be asked of a running mesh any number of times, and
|
||||
// every answer is a thing somebody can go and fix.
|
||||
//
|
||||
// Deliberately pure. It is handed what the mesh knows and returns sentences; nothing here connects to
|
||||
// anything, so it can be asked on a workstation about a mesh it has never reached.
|
||||
|
||||
// Readiness is what the mesh knows about its own ability to move.
|
||||
type Readiness struct {
|
||||
// TheBus is the address the mesh's own traffic would move to, empty when nothing names one.
|
||||
TheBus string
|
||||
// ServerStanding is whether a bus is reachable at that address, as somebody checked.
|
||||
ServerStanding bool
|
||||
// Holder is the node running the module that holds the bus seat, empty when nothing does.
|
||||
Holder string
|
||||
// AccountsComposed is whether that node has been sent the composed user list.
|
||||
AccountsComposed bool
|
||||
// Nodes is every machine the mesh knows.
|
||||
Nodes []string
|
||||
// Credentialled is which of them has a credential for the new bus.
|
||||
Credentialled map[string]bool
|
||||
// Modules is every assigned module, as `<node>/<module>`.
|
||||
Modules []string
|
||||
// ModuleCredentialled is which of those has one.
|
||||
ModuleCredentialled map[string]bool
|
||||
}
|
||||
|
||||
// NotReady is every reason this mesh cannot move its bus yet, in the order somebody would fix them.
|
||||
//
|
||||
// Empty means ready. **Each entry names one thing and what to do about it**, because a readiness check
|
||||
// that says "not ready" is a check nobody can act on — and this is read at the point where the next
|
||||
// step is irreversible.
|
||||
func NotReady(r Readiness) []string {
|
||||
var why []string
|
||||
|
||||
if strings.TrimSpace(r.TheBus) == "" {
|
||||
why = append(why, "nothing names the bus to move to: set "+NATSVar+" on the control node "+
|
||||
"to the address the new server answers on")
|
||||
}
|
||||
if !r.ServerStanding {
|
||||
why = append(why, "no bus is answering at that address. Step 2 of the change raises it beside "+
|
||||
"the one the mesh is on, carrying nothing — assign the module that holds "+
|
||||
"mesh-broker and push the machine that runs it")
|
||||
}
|
||||
if r.Holder == "" {
|
||||
why = append(why, "no machine holds mesh-broker, so nothing would compose the bus's user "+
|
||||
"list. Assign the module that claims it")
|
||||
} else if !r.AccountsComposed {
|
||||
why = append(why, fmt.Sprintf(
|
||||
"%s holds mesh-broker and has not been sent the composed user list, so the bus would "+
|
||||
"refuse every connection. `push %s`", r.Holder, r.Holder))
|
||||
}
|
||||
|
||||
// A node with no credential cannot come back after the move, and a node that cannot come back is
|
||||
// a machine the mesh has lost until somebody goes to it.
|
||||
var missing []string
|
||||
for _, n := range r.Nodes {
|
||||
if !r.Credentialled[n] {
|
||||
missing = append(missing, n)
|
||||
}
|
||||
}
|
||||
sort.Strings(missing)
|
||||
if len(missing) > 0 {
|
||||
why = append(why, fmt.Sprintf(
|
||||
"%d machine(s) have no credential for the new bus and would not come back: %s. Each needs "+
|
||||
"one minted before the move, not after — after, there is no bus to ask over",
|
||||
len(missing), strings.Join(missing, ", ")))
|
||||
}
|
||||
|
||||
// A module without one keeps running and stops being reachable, which is a smaller fault and still
|
||||
// one somebody should choose rather than discover.
|
||||
var quiet []string
|
||||
for _, m := range r.Modules {
|
||||
if !r.ModuleCredentialled[m] {
|
||||
quiet = append(quiet, m)
|
||||
}
|
||||
}
|
||||
sort.Strings(quiet)
|
||||
if len(quiet) > 0 {
|
||||
why = append(why, fmt.Sprintf(
|
||||
"%d module(s) have no credential for the new bus: %s. Each keeps serving and stops "+
|
||||
"answering tools and hearing events until it is issued one",
|
||||
len(quiet), strings.Join(quiet, ", ")))
|
||||
}
|
||||
|
||||
return why
|
||||
}
|
||||
|
||||
// WhatMoves is what the rollout would do, in order, for somebody reading before they commit.
|
||||
//
|
||||
// **Written out rather than summarised.** This is the one step with nothing to inspect afterwards, so
|
||||
// the last useful moment to disagree with it is while reading this.
|
||||
func WhatMoves(r Readiness) []string {
|
||||
out := []string{
|
||||
fmt.Sprintf("compose the bus's user list and send it to %s", holderOr(r.Holder)),
|
||||
fmt.Sprintf("move this control plane to %s, and confirm it is heard", busOr(r.TheBus)),
|
||||
}
|
||||
nodes := append([]string(nil), r.Nodes...)
|
||||
sort.Strings(nodes)
|
||||
for _, n := range nodes {
|
||||
out = append(out, fmt.Sprintf("move %s, and confirm it reports", n))
|
||||
}
|
||||
if len(r.Modules) > 0 {
|
||||
out = append(out, fmt.Sprintf("move %d module runtime(s), and confirm each answers",
|
||||
len(r.Modules)))
|
||||
}
|
||||
// **The old broker goes, and it goes last** (novox/hq ADR 0131). AMQP is not a provision, so once
|
||||
// every machine reports on the new bus nothing of the mesh is left speaking to it, and its module
|
||||
// is unassigned. Said as a step so nobody reads the move as leaving a second bus behind.
|
||||
out = append(out, "then unassign the old broker's module: AMQP is not a provision (ADR 0131), and "+
|
||||
"once every machine reports on the new bus nothing of the mesh speaks to it")
|
||||
return out
|
||||
}
|
||||
|
||||
func holderOr(node string) string {
|
||||
if node == "" {
|
||||
return "whichever machine holds mesh-broker"
|
||||
}
|
||||
return node
|
||||
}
|
||||
|
||||
func busOr(address string) string {
|
||||
if address == "" {
|
||||
return "the new bus"
|
||||
}
|
||||
return address
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Whether a mesh could move its bus.
|
||||
//
|
||||
// Every case here is a way of moving that leaves something behind, and the one that matters most is a
|
||||
// machine with no credential: after the move there is no bus to ask it over, so it is lost until
|
||||
// somebody walks to it.
|
||||
|
||||
func aMeshReadyToMove() Readiness {
|
||||
return Readiness{
|
||||
TheBus: "nats://127.0.0.1:5671", ServerStanding: true,
|
||||
Holder: "anchor", AccountsComposed: true,
|
||||
Nodes: []string{"anchor", "laptop"},
|
||||
Credentialled: map[string]bool{"anchor": true, "laptop": true},
|
||||
Modules: []string{"anchor/gitea"},
|
||||
ModuleCredentialled: map[string]bool{"anchor/gitea": true},
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMeshWithEverythingInPlaceIsReady(t *testing.T) {
|
||||
if why := NotReady(aMeshReadyToMove()); len(why) != 0 {
|
||||
t.Fatalf("a mesh with everything in place was refused: %v", why)
|
||||
}
|
||||
}
|
||||
|
||||
// **A machine with no credential is the one that must stop this.** It keeps running and cannot come
|
||||
// back, and there is no bus left to tell it anything over — so the remedy has to happen before, and
|
||||
// the message says so.
|
||||
func TestAMachineWithNoCredentialStopsTheMove(t *testing.T) {
|
||||
r := aMeshReadyToMove()
|
||||
r.Credentialled = map[string]bool{"anchor": true}
|
||||
|
||||
why := NotReady(r)
|
||||
if len(why) == 0 {
|
||||
t.Fatal("a machine that could not come back did not stop the move")
|
||||
}
|
||||
said := strings.Join(why, "\n")
|
||||
if !strings.Contains(said, "laptop") {
|
||||
t.Errorf("the refusal does not name the machine: %s", said)
|
||||
}
|
||||
if !strings.Contains(said, "before the move") {
|
||||
t.Errorf("the refusal does not say the remedy comes first: %s", said)
|
||||
}
|
||||
}
|
||||
|
||||
// A bus nobody has raised, a seat nobody holds, and a user list nobody has been sent: each stops it,
|
||||
// and each names its own next step, because "not ready" that cannot be acted on is not an answer.
|
||||
func TestEachThingMissingNamesItsOwnRemedy(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
break_ func(*Readiness)
|
||||
says string
|
||||
}{
|
||||
{"no address", func(r *Readiness) { r.TheBus = "" }, NATSVar},
|
||||
{"no server", func(r *Readiness) { r.ServerStanding = false }, "carrying nothing"},
|
||||
{"no holder", func(r *Readiness) { r.Holder = "" }, "mesh-broker"},
|
||||
{"no user list", func(r *Readiness) { r.AccountsComposed = false }, "push anchor"},
|
||||
{"a module with none", func(r *Readiness) {
|
||||
r.ModuleCredentialled = map[string]bool{}
|
||||
}, "anchor/gitea"},
|
||||
} {
|
||||
r := aMeshReadyToMove()
|
||||
c.break_(&r)
|
||||
why := NotReady(r)
|
||||
if len(why) == 0 {
|
||||
t.Errorf("%s did not stop the move", c.what)
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(strings.Join(why, "\n"), c.says) {
|
||||
t.Errorf("%s: the refusal does not mention %q: %v", c.what, c.says, why)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// What the move would do is written out rather than summarised, because this is the one step with
|
||||
// nothing to inspect afterwards — so reading it is the last chance to disagree.
|
||||
func TestWhatMovesNamesEveryMachineAndEndsWithTheOldBrokerGoing(t *testing.T) {
|
||||
r := aMeshReadyToMove()
|
||||
steps := strings.Join(WhatMoves(r), "\n")
|
||||
|
||||
for _, want := range []string{"anchor", "laptop", "user list", "module runtime"} {
|
||||
if !strings.Contains(steps, want) {
|
||||
t.Errorf("the plan does not mention %q:\n%s", want, steps)
|
||||
}
|
||||
}
|
||||
// Said explicitly, and last: AMQP is not a provision (novox/hq ADR 0131), so the move ends with
|
||||
// the old broker's module unassigned, not left behind as a second bus. An earlier version of this
|
||||
// test pinned the opposite, under a record 0131 superseded.
|
||||
lines := WhatMoves(r)
|
||||
if last := lines[len(lines)-1]; !strings.Contains(last, "unassign the old broker") {
|
||||
t.Errorf("the plan does not end with the old broker going:\n%s", steps)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// The mesh's own streams.
|
||||
//
|
||||
// **These four and no more** (novox/hq ADR 0116 task 1.4, as revised by ADR 0118). An earlier
|
||||
// reading had the controller create *every* stream at genesis, from a fixed set. That is only the
|
||||
// mesh's own half: a seat's streams are created when the module declaring it is registered, and a
|
||||
// module's durable consumers when it is assigned — neither of which has happened at genesis. What
|
||||
// is here is the foundation, which exists before any module does.
|
||||
//
|
||||
// The controller is the only writer of stream definitions (design 25 §3). A module declares
|
||||
// nothing about them and cannot reach the JetStream API to make one.
|
||||
|
||||
// Retention is how a stream decides what to keep, which is the whole of what distinguishes the
|
||||
// mesh's four relationships on the wire (design 29 §4).
|
||||
type Retention string
|
||||
|
||||
const (
|
||||
// RetentionWorkQueue: a message is removed once a consumer acknowledges it. Exactly one
|
||||
// worker does the work, and a worker that dies has its message redelivered.
|
||||
RetentionWorkQueue Retention = "workqueue"
|
||||
// RetentionLastPerSubject: only the newest message on each subject survives. This is the
|
||||
// state shape — a node that was away gets exactly the current declaration and nothing older.
|
||||
RetentionLastPerSubject Retention = "last_per_subject"
|
||||
// RetentionLimits: kept until it ages or the stream fills. Events, where a subscriber that
|
||||
// was down catches up and nobody is obliged to act.
|
||||
RetentionLimits Retention = "limits"
|
||||
)
|
||||
|
||||
// A Stream is one of the mesh's own, as the controller asserts it.
|
||||
type Stream struct {
|
||||
Name string
|
||||
Subjects []string
|
||||
Retention Retention
|
||||
// MaxAge in seconds, zero for unbounded. Per stream — JetStream has no per-subject age,
|
||||
// which is why differing retention between modules would mean a stream each.
|
||||
MaxAge int
|
||||
// MaxMsgsPerSubject caps each subject independently, so one noisy emitter cannot push
|
||||
// another's events out of a shared stream. Verified: with a cap of 3, ten messages on one
|
||||
// subject and one on another leave four in the stream, not three.
|
||||
MaxMsgsPerSubject int
|
||||
// Why is carried into the assertion so an operator reading the server's own state finds the
|
||||
// reason there, rather than only in a repository they may not have.
|
||||
Why string
|
||||
}
|
||||
|
||||
// MeshStreams is the foundation set, in the order a person reads it.
|
||||
//
|
||||
// **CONTROL names its subjects rather than taking `mesh.control.>`**, because heartbeats live
|
||||
// under that prefix and must not be persisted: a lost heartbeat is the next heartbeat, and a
|
||||
// stream of them is a stream of the least valuable messages the mesh sends, competing for the
|
||||
// same retention as the ones that matter.
|
||||
//
|
||||
// **EVENTS filters on the `event` token**, which is the reason that token exists. A module's
|
||||
// namespace carries both its events and its tool calls; a filter of `mesh.mod.*.>` would persist
|
||||
// every tool invocation in the mesh, and a tool call must never be persisted (design 25 §3 keeps
|
||||
// tools on core NATS, where a lost call is a timeout the caller already handles).
|
||||
func MeshStreams() []Stream {
|
||||
return []Stream{
|
||||
{
|
||||
Name: "CONTROL",
|
||||
// A build's outcome is no longer here: it is the build-machine seat's own event, so one
|
||||
// publish reaches whoever asked, the controller and the catalogue (novox/hq ADR 0121).
|
||||
Subjects: []string{"mesh.control.*.report", "mesh.control.enrol"},
|
||||
Retention: RetentionWorkQueue,
|
||||
Why: "the store-window guarantee (ADR 0083): the controller naks with a delay while its " +
|
||||
"store is away and the message is redelivered; nothing is dropped",
|
||||
},
|
||||
{
|
||||
Name: "NODES",
|
||||
Subjects: []string{"mesh.node.*.declare"},
|
||||
Retention: RetentionLastPerSubject,
|
||||
Why: "one declaration per node, always the newest; a node that sees sequence n refuses " +
|
||||
"n-1 by construction (issue 107)",
|
||||
},
|
||||
{
|
||||
Name: "EVENTS",
|
||||
// A seat's own events ride here too: they are 1:many like any event, and the
|
||||
// `event` token keeps them clear of both the seat's work queue (`accept`) and its
|
||||
// tools (`tool`), which must not be persisted.
|
||||
Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"},
|
||||
Retention: RetentionLimits,
|
||||
MaxAge: 7 * 24 * 60 * 60,
|
||||
MaxMsgsPerSubject: 10000,
|
||||
Why: "a subscriber that was down catches up; tool traffic under the same prefix is " +
|
||||
"excluded by the event token; per-subject caps keep a noisy emitter from " +
|
||||
"evicting a quiet one without splitting the stream",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// An Asserter is the part of a JetStream connection stream assertion needs. Narrow on purpose: it
|
||||
// keeps this testable without a server, and keeps the client library out of everything that only
|
||||
// wants to know what the streams are.
|
||||
type Asserter interface {
|
||||
// EnsureStream creates the stream if absent and updates it to match if present. It must be
|
||||
// idempotent: the controller asserts on every start, not only at genesis.
|
||||
EnsureStream(s Stream) error
|
||||
}
|
||||
|
||||
// AssertMeshStreams brings the foundation set into being, in order, and says which one failed
|
||||
// rather than that something did.
|
||||
//
|
||||
// Asserted on every start rather than created once at genesis, because a stream that was deleted,
|
||||
// or a mesh raised from a restored backup, must converge rather than run without the guarantee
|
||||
// its messages assume. Idempotence is the whole requirement.
|
||||
func AssertMeshStreams(a Asserter) error {
|
||||
for _, s := range MeshStreams() {
|
||||
if err := a.EnsureStream(s); err != nil {
|
||||
return fmt.Errorf("asserting stream %s: %w", s.Name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Overlaps reports subject filters claimed by more than one stream.
|
||||
//
|
||||
// **Corrected against the server**: an earlier version of this comment said NATS accepts
|
||||
// overlapping streams and stores the message twice. It does not — it refuses the second stream
|
||||
// with "subjects overlap with an existing stream" (verified against nats-server 2.10). The check
|
||||
// still earns its place, for a different reason: the server's refusal arrives when the controller
|
||||
// is applying, naming one stream, at a moment when the mesh is half-configured. This one arrives
|
||||
// where the set is written, names both, and cannot reach a running mesh.
|
||||
//
|
||||
// It also decides a design question. Because overlap is refused rather than merged, a shared
|
||||
// EVENTS stream and a per-module stream cannot coexist — the module's would be refused — so
|
||||
// "one stream for most, its own for a module that wants different retention" is not an option
|
||||
// the server allows. It is all of one or all of the other.
|
||||
func Overlaps() []string {
|
||||
seen := map[string]string{}
|
||||
var clashes []string
|
||||
for _, s := range MeshStreams() {
|
||||
for _, subject := range s.Subjects {
|
||||
if first, ok := seen[subject]; ok {
|
||||
clashes = append(clashes, fmt.Sprintf("%s and %s both claim %s", first, s.Name, subject))
|
||||
continue
|
||||
}
|
||||
seen[subject] = s.Name
|
||||
}
|
||||
}
|
||||
sort.Strings(clashes)
|
||||
return clashes
|
||||
}
|
||||
|
||||
// The mesh's own consumers.
|
||||
//
|
||||
// A seat's streams and a module's consumers are derived from declarations (derived.go). These two
|
||||
// are not: **the controller is not a module and files no manifest**, so its authority and its
|
||||
// subscriptions cannot come from a declaration that does not exist. They are named here, where the
|
||||
// mesh's own streams are named, and narrowly — a controller subscribing `mesh.mod.*.event.>` would
|
||||
// hear every event in the mesh, which it has no business doing and which would make its permission
|
||||
// list stop explaining anything.
|
||||
|
||||
// ControllerName is the controller's durable consumer on each stream it reads, and the name its
|
||||
// ack subject is derived from (nats.go: `$JS.ACK.<stream>.controller.>`).
|
||||
const ControllerName = "controller"
|
||||
|
||||
// ControllerFollows are the events the controller reacts to: the catalogue saying a module's
|
||||
// current version moved, and a catalogue that has just started saying it may have missed builds.
|
||||
//
|
||||
// **Derived the same way a module's subscription is**, from the emitter and the bare local event
|
||||
// name, rather than written out. They were written out while the catalogue still spelled its events
|
||||
// as the old bus's routing keys, and the moment those were converted (novox/hq 04-ISSUES/127) a
|
||||
// hard-coded pair became a controller listening to a subject nothing publishes — the same fault, from
|
||||
// the other side. Deriving them means the conversion could not leave these behind.
|
||||
var ControllerFollows = []string{
|
||||
moduleEventSubject("mesh-catalog", "upgraded"),
|
||||
moduleEventSubject("mesh-catalog", "catching-up"),
|
||||
// A build's outcome, which is the build-machine role's own event now (ADR 0121) rather than a
|
||||
// message on the control branch. Same three audiences, one publish: whoever asked, this, and the
|
||||
// catalogue.
|
||||
seatEventSubject("mesh-build-machine", "built"),
|
||||
// The forge's merges: what moved a source, so the mesh builds what that source produces
|
||||
// without anybody telling it (novox/hq 04-ISSUES/131). Appended, because the index is a name.
|
||||
moduleEventSubject("gitea", "pull.merged"),
|
||||
}
|
||||
|
||||
// moduleEventSubject is where one module's event lands. The same derivation PermissionsFor uses, so
|
||||
// what the controller subscribes and what the emitter is permitted to publish cannot drift apart.
|
||||
func moduleEventSubject(module, event string) string {
|
||||
return "mesh.mod." + module + ".event." + event
|
||||
}
|
||||
|
||||
// seatEventSubject is where a role's own event lands, derived the same way a holder's permission is.
|
||||
func seatEventSubject(seat, verb string) string {
|
||||
return "mesh.seat." + seat + ".event." + verb
|
||||
}
|
||||
|
||||
// MeshConsumers is what the controller consumes, in the order a person reads it.
|
||||
//
|
||||
// **Unlimited redelivery on CONTROL, deliberately.** The store window's bound is the controller's,
|
||||
// not the server's (window.go): a message is held with a nak-and-delay until the controller either
|
||||
// takes it or gives up and says so. A max-deliver here would dead-letter a push that was being
|
||||
// held through a store restart — the exact message the stream exists to protect — some minutes
|
||||
// before the controller had finished deciding about it.
|
||||
func MeshConsumers() []Consumer {
|
||||
return []Consumer{
|
||||
{
|
||||
Name: ControllerName,
|
||||
Stream: "CONTROL",
|
||||
Push: true,
|
||||
AckWaitSeconds: 30,
|
||||
Why: "the controller is the single consumer of what nodes say; explicit ack and no " +
|
||||
"max-deliver, because the store window's bound is the controller's own",
|
||||
},
|
||||
{
|
||||
Name: ControllerName,
|
||||
Stream: "EVENTS",
|
||||
Filters: ControllerFollows,
|
||||
Push: true,
|
||||
AckWaitSeconds: 30,
|
||||
MaxDeliver: 5,
|
||||
Why: "the two events the mesh's own controller reacts to; after max-deliver it " +
|
||||
"dead-letters, because an announcement it cannot act on will not become actionable",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Ensurer is the part of a JetStream connection consumer assertion needs, narrow for the reason
|
||||
// Asserter is.
|
||||
type Ensurer interface {
|
||||
EnsureConsumer(c Consumer) error
|
||||
}
|
||||
|
||||
// AssertMeshConsumers brings the controller's own consumers into being, and says which one failed.
|
||||
//
|
||||
// After the streams, necessarily: a consumer on a stream that does not exist is refused, and the
|
||||
// refusal names the stream rather than the order.
|
||||
func AssertMeshConsumers(e Ensurer) error {
|
||||
for _, c := range MeshConsumers() {
|
||||
if err := e.EnsureConsumer(c); err != nil {
|
||||
return fmt.Errorf("asserting consumer %s on %s: %w", c.Name, c.Stream, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type recorder struct {
|
||||
seen []Stream
|
||||
fail string
|
||||
}
|
||||
|
||||
func (r *recorder) EnsureStream(s Stream) error {
|
||||
if s.Name == r.fail {
|
||||
return errors.New("refused")
|
||||
}
|
||||
r.seen = append(r.seen, s)
|
||||
return nil
|
||||
}
|
||||
|
||||
// The controller asserts on every start, not only at genesis: a stream that was deleted, or a mesh
|
||||
// raised from a backup, must converge rather than run without the guarantee its messages assume.
|
||||
func TestAssertingTwiceIsTheSameAsOnce(t *testing.T) {
|
||||
a, b := &recorder{}, &recorder{}
|
||||
if err := AssertMeshStreams(a); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := AssertMeshStreams(a); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := AssertMeshStreams(b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(a.seen) != 2*len(b.seen) {
|
||||
t.Fatalf("asserted %d then %d; assertion is not repeatable", len(a.seen), len(b.seen))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailedAssertionNamesItsStream(t *testing.T) {
|
||||
err := AssertMeshStreams(&recorder{fail: "NODES"})
|
||||
if err == nil || !strings.Contains(err.Error(), "NODES") {
|
||||
t.Fatalf("got %v, which does not say which stream failed", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Two streams matching one subject is accepted by NATS and stores the message twice under two
|
||||
// retentions. Nothing reports that, so it is refused where the set is written.
|
||||
func TestNoTwoStreamsClaimTheSameSubject(t *testing.T) {
|
||||
if clashes := Overlaps(); len(clashes) != 0 {
|
||||
t.Fatalf("overlapping subject filters: %v", clashes)
|
||||
}
|
||||
}
|
||||
|
||||
// A heartbeat under mesh.control.> must not be persisted: a lost one is the next one, and a
|
||||
// stream of them competes for retention with the messages that matter.
|
||||
func TestHeartbeatsAreNotInTheControlStream(t *testing.T) {
|
||||
for _, s := range MeshStreams() {
|
||||
for _, subject := range s.Subjects {
|
||||
if subject == "mesh.control.>" || strings.Contains(subject, "alive") {
|
||||
t.Fatalf("stream %s claims %q, which captures heartbeats", s.Name, subject)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The reason the kind token exists: a filter over a module's whole namespace would persist every
|
||||
// tool call in the mesh.
|
||||
func TestTheEventsStreamDoesNotCaptureToolCalls(t *testing.T) {
|
||||
var events Stream
|
||||
for _, s := range MeshStreams() {
|
||||
if s.Name == "EVENTS" {
|
||||
events = s
|
||||
}
|
||||
}
|
||||
// Nothing a tool call rides may match any of the filters — a module's or a seat's.
|
||||
for _, tool := range []string{
|
||||
"mesh.mod.billing.tool.status",
|
||||
"mesh.seat.telegram-sender.tool.status",
|
||||
"mesh.seat.telegram-sender.accept.send", // work, not an event: its own stream
|
||||
} {
|
||||
for _, f := range events.Subjects {
|
||||
if subjectMatches(f, tool) {
|
||||
t.Fatalf("%q matches the events filter %q, so it would be persisted here", tool, f)
|
||||
}
|
||||
}
|
||||
}
|
||||
// And both kinds of event do match.
|
||||
for _, event := range []string{
|
||||
"mesh.mod.billing.event.order.placed",
|
||||
"mesh.seat.telegram-sender.event.delivered",
|
||||
} {
|
||||
matched := false
|
||||
for _, f := range events.Subjects {
|
||||
if subjectMatches(f, event) {
|
||||
matched = true
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
t.Fatalf("%q matches no events filter, so nothing would keep it", event)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// subjectMatches is NATS subject matching, enough for these filters: `*` is one token, `>` is the
|
||||
// rest.
|
||||
func subjectMatches(filter, subject string) bool {
|
||||
f, s := strings.Split(filter, "."), strings.Split(subject, ".")
|
||||
for i, tok := range f {
|
||||
if tok == ">" {
|
||||
return i <= len(s)
|
||||
}
|
||||
if i >= len(s) {
|
||||
return false
|
||||
}
|
||||
if tok != "*" && tok != s[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(f) == len(s)
|
||||
}
|
||||
|
||||
// Each relationship's retention is the thing that makes it what it is (design 29 §4).
|
||||
func TestEachStreamCarriesTheRetentionItsShapeNeeds(t *testing.T) {
|
||||
want := map[string]Retention{
|
||||
"CONTROL": RetentionWorkQueue,
|
||||
"NODES": RetentionLastPerSubject,
|
||||
"EVENTS": RetentionLimits,
|
||||
}
|
||||
got := map[string]Retention{}
|
||||
for _, s := range MeshStreams() {
|
||||
got[s.Name] = s.Retention
|
||||
if s.Why == "" {
|
||||
t.Errorf("stream %s says no reason it exists", s.Name)
|
||||
}
|
||||
}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("the foundation set is %v", got)
|
||||
}
|
||||
for name, r := range want {
|
||||
if got[name] != r {
|
||||
t.Errorf("%s retains as %q, expected %q", name, got[name], r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The order the bus's objects are asserted in, because getting it wrong is a refusal that names the
|
||||
// wrong thing: a consumer on a stream that does not exist is refused naming the *stream*, so
|
||||
// somebody reading it goes looking for a deletion instead of a reversed pair of lines.
|
||||
func TestTheBusesObjectsAreAssertedStreamsBeforeConsumers(t *testing.T) {
|
||||
r := &recording{}
|
||||
if err := Raise(r, []string{"anchor", "laptop"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Every stream before every consumer.
|
||||
firstConsumer := -1
|
||||
for i, step := range r.steps {
|
||||
if strings.HasPrefix(step, "consumer ") && firstConsumer < 0 {
|
||||
firstConsumer = i
|
||||
}
|
||||
if strings.HasPrefix(step, "stream ") && firstConsumer >= 0 {
|
||||
t.Fatalf("a stream was asserted after a consumer: %v", r.steps)
|
||||
}
|
||||
}
|
||||
if firstConsumer < 0 {
|
||||
t.Fatalf("no consumer was asserted: %v", r.steps)
|
||||
}
|
||||
|
||||
// And every node got one, named after it — without which that node hears nothing while
|
||||
// everything else about it looks correct.
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
if !containsStep(r.steps, "consumer NODES/"+node) {
|
||||
t.Errorf("%s was given no way to hear its declaration: %v", node, r.steps)
|
||||
}
|
||||
}
|
||||
// And the controller its own, on both streams it reads.
|
||||
for _, want := range []string{"consumer CONTROL/controller", "consumer EVENTS/controller"} {
|
||||
if !containsStep(r.steps, want) {
|
||||
t.Errorf("the controller is missing %s: %v", want, r.steps)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A seat's work queue is asserted whether or not anybody holds it; the holder's worker only when
|
||||
// somebody does. **The stream without the consumer is the point**: work queues until a holder
|
||||
// appears, so installing the module later flushes the backlog instead of having lost it.
|
||||
func TestASeatsQueueExistsBeforeItsHolderDoes(t *testing.T) {
|
||||
seats := []DeclaredSeat{{Name: "telegram-sender", Accepts: []string{"send"}}}
|
||||
|
||||
unheld := &recording{}
|
||||
if err := RaiseSeats(unheld, seats, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !containsStep(unheld.steps, "stream SEAT_TELEGRAM_SENDER") {
|
||||
t.Fatalf("a declared seat got no work queue: %v", unheld.steps)
|
||||
}
|
||||
for _, step := range unheld.steps {
|
||||
if strings.HasPrefix(step, "consumer ") {
|
||||
t.Fatalf("a seat nobody holds got a worker: %v", unheld.steps)
|
||||
}
|
||||
}
|
||||
|
||||
held := &recording{}
|
||||
if err := RaiseSeats(held, seats, map[string]Holder{
|
||||
"telegram-sender": {Node: "anchor", Module: "telegram"},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !containsStep(held.steps, "consumer SEAT_TELEGRAM_SENDER/SEAT_TELEGRAM_SENDER_worker") {
|
||||
t.Fatalf("the seat's holder got no worker: %v", held.steps)
|
||||
}
|
||||
}
|
||||
|
||||
// recording is a connection to the bus that writes down what it was asked for.
|
||||
type recording struct{ steps []string }
|
||||
|
||||
func (r *recording) EnsureStream(s Stream) error {
|
||||
r.steps = append(r.steps, "stream "+s.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *recording) EnsureConsumer(c Consumer) error {
|
||||
r.steps = append(r.steps, "consumer "+c.Stream+"/"+c.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func containsStep(steps []string, want string) bool {
|
||||
for _, s := range steps {
|
||||
if s == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
# Composed by the mesh controller. Do not edit: the next composition overwrites it.
|
||||
# Accounts and permissions are derived from what each module declares and nothing
|
||||
# else (novox/hq ADR 0043, design 29 §2).
|
||||
|
||||
port: 4222
|
||||
http: 127.0.0.1:8222
|
||||
|
||||
tls {
|
||||
cert_file: "/tls/tls.crt"
|
||||
key_file: "/tls/tls.key"
|
||||
ca_file: "/tls/ca.crt"
|
||||
}
|
||||
|
||||
jetstream {
|
||||
store_dir: "/data"
|
||||
}
|
||||
|
||||
# The mesh's users, composed by the controller. Do not edit: the next
|
||||
# composition overwrites it. Permissions are derived from what each module
|
||||
# declares and nothing else (novox/hq ADR 0043, design 29 §2).
|
||||
|
||||
accounts {
|
||||
MESH {
|
||||
jetstream: enabled
|
||||
users = [
|
||||
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.control.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>"] }
|
||||
subscribe: { allow: ["$JS.API.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
{ user: "enrol.one", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: {
|
||||
publish: { allow: ["mesh.control.enrol"] }
|
||||
subscribe: { allow: ["_INBOX.enrol.one.>"] }
|
||||
} }
|
||||
{ user: "node.one", password: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn", permissions: {
|
||||
publish: { allow: ["$JS.ACK.NODES.one.>", "$JS.API.CONSUMER.INFO.NODES.one", "mesh.control.one.>"] }
|
||||
subscribe: { allow: ["_DELIVER.one", "_INBOX.node.one.>", "mesh.node.one.declare"] }
|
||||
} }
|
||||
{ user: "one.telegram", password: "$2a$11$tttttttttttttttttttttt", permissions: {
|
||||
publish: { allow: ["$JS.ACK.EVENTS.one_telegram.>", "$JS.ACK.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker.>", "$JS.API.CONSUMER.INFO.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "mesh.seat.telegram-sender.event.delivered", "mesh.seat.telegram-sender.event.failed"] }
|
||||
subscribe: { allow: ["_DELIVER.SEAT_TELEGRAM_SENDER_worker", "_DELIVER.one_telegram", "_INBOX.one.telegram.>", "mesh.mod.telegram.tool.status", "mesh.seat.telegram-sender.accept.send"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
{ user: "two.audit", password: "$2a$11$aaaaaaaaaaaaaaaaaaaaaa", permissions: {
|
||||
publish: { allow: ["$JS.ACK.EVENTS.two_audit.>"] }
|
||||
subscribe: { allow: ["_DELIVER.two_audit", "_INBOX.two.audit.>", "mesh.mod.shop.event.order.placed"] }
|
||||
} }
|
||||
{ user: "two.shop", password: "$2a$11$ssssssssssssssssssssss", permissions: {
|
||||
publish: { allow: ["$JS.ACK.EVENTS.two_shop.>", "mesh.mod.shop.event.order.placed", "mesh.seat.telegram-sender.accept.send"] }
|
||||
subscribe: { allow: ["_DELIVER.two_shop", "_INBOX.two.shop.>"] }
|
||||
} }
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// Every user the composed file should contain, derived from what the mesh knows.
|
||||
//
|
||||
// **The list is derived, never kept.** A stored user list would be a second account of who may
|
||||
// reach the bus, able to disagree with the records it came from — and the disagreement would be
|
||||
// invisible, because both would look internally consistent. So this is a pure function of the
|
||||
// mesh's records, run again every time the file is written.
|
||||
//
|
||||
// Records are mirrored into this package's own types rather than imported from the catalogue, for
|
||||
// the reason DeclaredSeat is: composing authority is a different job from parsing a manifest, and
|
||||
// this package stays free of the other's types so a change to a manifest field cannot quietly widen
|
||||
// a permission.
|
||||
|
||||
// Declared is one module on one node, as composing its authority needs it.
|
||||
type Declared struct {
|
||||
Module string
|
||||
Emits []string
|
||||
Consumes []string
|
||||
Serves []string
|
||||
// Holds are the seats this module claims, with the protocol each seat declares. A seat the
|
||||
// mesh defines for itself declares no protocol, so holding one grants nothing on the bus —
|
||||
// which is right: those seats are about who does a job, not about who may say what.
|
||||
Holds []Seat
|
||||
// Uses are the seats this module sends to.
|
||||
Uses []Seat
|
||||
// Watches are the seats whose events it consumes.
|
||||
Watches []Seat
|
||||
}
|
||||
|
||||
// Records is what composing a user list needs to know about the mesh, and nothing more.
|
||||
type Records struct {
|
||||
// Nodes is every machine the mesh knows. Each gets a host user.
|
||||
Nodes []string
|
||||
// Assigned is the modules on each node, as they declare themselves.
|
||||
Assigned map[string][]Declared
|
||||
// Enrolling is every node with a live token — one enrolment user each, because the inbox an
|
||||
// answer goes to is scoped to the token and a shared one is one machine reading another's
|
||||
// sealed credentials (design 25 §6).
|
||||
Enrolling []string
|
||||
// People is each person's name against the tools they may invoke, `*` for an administrator.
|
||||
People map[string][]string
|
||||
}
|
||||
|
||||
// Users is every user the composed file should contain, in the order it will be written.
|
||||
//
|
||||
// The controller is always first and always present: a mesh whose own controller is not in the file
|
||||
// is a mesh that cannot be told anything, and there is no state of the records in which that is
|
||||
// correct.
|
||||
func Users(r Records) ([]Principal, error) {
|
||||
out := []Principal{{Kind: KindController}}
|
||||
|
||||
for _, node := range sortedCopy(r.Nodes) {
|
||||
out = append(out, Principal{Kind: KindNode, Node: node})
|
||||
for _, d := range r.Assigned[node] {
|
||||
out = append(out, Principal{
|
||||
Kind: KindModule, Node: node, Module: d.Module,
|
||||
Emits: d.Emits, Consumes: d.Consumes, Serves: d.Serves,
|
||||
Holds: d.Holds, Uses: d.Uses, Watches: d.Watches,
|
||||
})
|
||||
}
|
||||
}
|
||||
for _, node := range sortedCopy(r.Enrolling) {
|
||||
out = append(out, Principal{Kind: KindEnrolment, Node: node})
|
||||
}
|
||||
for _, person := range sortedNames(r.People) {
|
||||
out = append(out, Principal{Kind: KindPerson, Module: person, Invokes: r.People[person]})
|
||||
}
|
||||
|
||||
// Refused here rather than discovered by the server. Two users with one name is a file the
|
||||
// server reads as one of them, and which one depends on the order — so a module assigned to a
|
||||
// node twice, or a person named after nothing, is a composition that must not be written.
|
||||
seen := map[string]string{}
|
||||
for _, p := range out {
|
||||
name := p.Username()
|
||||
if name == "" || name == "." {
|
||||
return nil, fmt.Errorf("a %s user has no name, so nothing could authenticate as it", p.Kind)
|
||||
}
|
||||
if first, already := seen[name]; already {
|
||||
return nil, fmt.Errorf(
|
||||
"two users would be called %q (a %s and a %s): the server would read the file as "+
|
||||
"one of them, and which one depends on the order", name, first, p.Kind)
|
||||
}
|
||||
seen[name] = string(p.Kind)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// WithPasswords fills each user's hash from what the mesh minted, and says which users have none.
|
||||
//
|
||||
// **Separated from Users because they fail differently.** A user missing from the records is a bug
|
||||
// in deriving them; a user with no password is a step that has not happened yet — a module assigned
|
||||
// but never given a credential, a node enrolled before this existed. The second is ordinary and its
|
||||
// remedy is to mint one, so it is named rather than returned as an error, and the caller decides
|
||||
// whether a partial composition is worth writing.
|
||||
func WithPasswords(principals []Principal, hashes map[string]string) (filled []Principal, missing []string) {
|
||||
for _, p := range principals {
|
||||
hash, ok := hashes[p.Username()]
|
||||
if !ok || hash == "" {
|
||||
missing = append(missing, p.Username())
|
||||
continue
|
||||
}
|
||||
p.PasswordHash = hash
|
||||
filled = append(filled, p)
|
||||
}
|
||||
return filled, missing
|
||||
}
|
||||
|
||||
func sortedCopy(in []string) []string {
|
||||
out := append([]string(nil), in...)
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func sortedNames(in map[string][]string) []string {
|
||||
out := make([]string, 0, len(in))
|
||||
for k := range in {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Deriving the bus's user list from the mesh's records.
|
||||
//
|
||||
// Every test here is about a way the list could be wrong that the server would not tell anybody
|
||||
// about: a user missing, a user named twice, a user with authority it did not declare.
|
||||
|
||||
func someRecords() Records {
|
||||
return Records{
|
||||
Nodes: []string{"two", "one"},
|
||||
Assigned: map[string][]Declared{
|
||||
"one": {{Module: "telegram", Serves: []string{"status"}}},
|
||||
"two": {{Module: "shop", Emits: []string{"order.placed"}}},
|
||||
},
|
||||
Enrolling: []string{"three"},
|
||||
People: map[string][]string{"ada": {"mesh-catalog.catalog_tools"}},
|
||||
}
|
||||
}
|
||||
|
||||
func namesOf(t *testing.T, r Records) []string {
|
||||
t.Helper()
|
||||
users, err := Users(r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := make([]string, 0, len(users))
|
||||
for _, u := range users {
|
||||
out = append(out, u.Username())
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// The controller is always there. A mesh whose own controller is not in the file is a mesh that
|
||||
// cannot be told anything, and there is no state of the records in which that is correct.
|
||||
func TestTheControllerIsAlwaysInTheList(t *testing.T) {
|
||||
for _, r := range []Records{{}, someRecords()} {
|
||||
names := namesOf(t, r)
|
||||
if len(names) == 0 || names[0] != "controller" {
|
||||
t.Fatalf("the controller is not first in %v", names)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// One user per node, one per module per node, one per live token and one per person — and nothing
|
||||
// else, because a user nobody derived is a user nobody can explain.
|
||||
func TestEveryRecordBecomesExactlyOneUser(t *testing.T) {
|
||||
names := namesOf(t, someRecords())
|
||||
want := []string{
|
||||
"controller",
|
||||
"node.one", "one.telegram",
|
||||
"node.two", "two.shop",
|
||||
"enrol.three",
|
||||
"person.ada",
|
||||
}
|
||||
if strings.Join(names, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("derived %v\n want %v", names, want)
|
||||
}
|
||||
}
|
||||
|
||||
// Two users with one name is a file the server reads as one of them, and which one depends on the
|
||||
// order. Refused here, where both can be named, rather than left to be whichever the server picked.
|
||||
func TestTwoUsersWithOneNameAreRefused(t *testing.T) {
|
||||
r := someRecords()
|
||||
r.Assigned["one"] = append(r.Assigned["one"], Declared{Module: "telegram"})
|
||||
_, err := Users(r)
|
||||
if err == nil {
|
||||
t.Fatal("a module assigned twice to one node composed two users with one name")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "one.telegram") {
|
||||
t.Fatalf("the refusal does not name the user: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A module's authority is what it declared and nothing more, carried through the derivation intact —
|
||||
// because this is the step where a mistake would grant something no manifest asked for.
|
||||
func TestAModulesAuthorityIsWhatItDeclared(t *testing.T) {
|
||||
seat := Seat{Name: "telegram-sender", Accepts: []string{"send"}, Emits: []string{"delivered"}}
|
||||
users, err := Users(Records{
|
||||
Nodes: []string{"one"},
|
||||
Assigned: map[string][]Declared{"one": {{
|
||||
Module: "shop", Emits: []string{"order.placed"}, Uses: []Seat{seat},
|
||||
}}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
perms, err := PermissionsFor(users[len(users)-1])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
has(t, perms.Publish, "mesh.mod.shop.event.order.placed")
|
||||
has(t, perms.Publish, "mesh.seat.telegram-sender.accept.send")
|
||||
// A seat it uses, not one it holds: it may submit work and may not publish the seat's own
|
||||
// events, or it could lie about outcomes on a role somebody else fills.
|
||||
hasNot(t, perms.Publish, "mesh.seat.telegram-sender.event.delivered")
|
||||
hasNot(t, perms.Subscribe, "mesh.seat.telegram-sender.accept.send")
|
||||
}
|
||||
|
||||
// A user the mesh has never minted a password for is named rather than silently dropped or
|
||||
// composed as a user anybody is. It is an ordinary situation — a module assigned a moment ago — and
|
||||
// the remedy is to mint one, so the caller decides whether to write a partial file.
|
||||
func TestAUserWithNoPasswordIsNamedRatherThanWritten(t *testing.T) {
|
||||
users, err := Users(someRecords())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
filled, missing := WithPasswords(users, map[string]string{
|
||||
"controller": "$2a$hash", "node.one": "$2a$hash",
|
||||
})
|
||||
if len(filled) != 2 {
|
||||
t.Fatalf("composed %d users from two hashes", len(filled))
|
||||
}
|
||||
if len(missing) != len(users)-2 {
|
||||
t.Fatalf("%d users are missing a password, of %d: %v", len(missing), len(users), missing)
|
||||
}
|
||||
for _, p := range filled {
|
||||
if p.PasswordHash == "" {
|
||||
t.Fatalf("%s was kept with no password, which is a user anybody is", p.Username())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// And the whole thing composes: records in, a file the server would read out.
|
||||
func TestRecordsComposeIntoAFile(t *testing.T) {
|
||||
users, err := Users(someRecords())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hashes := map[string]string{}
|
||||
for _, u := range users {
|
||||
hashes[u.Username()] = "$2a$11$" + strings.Repeat("x", 22)
|
||||
}
|
||||
filled, missing := WithPasswords(users, hashes)
|
||||
if len(missing) != 0 {
|
||||
t.Fatalf("users with no password: %v", missing)
|
||||
}
|
||||
got, err := Compose(Server{ClientPort: 4222, MonitoringPort: 8222, StoreDir: "/data",
|
||||
TLSCert: "/tls/tls.crt", TLSKey: "/tls/tls.key", TLSCA: "/tls/ca.crt"}, filled)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
`user: "controller"`, `user: "node.one"`, `user: "one.telegram"`,
|
||||
`user: "enrol.three"`, `user: "person.ada"`,
|
||||
`"_INBOX.enrol.three.>"`, `"mesh.mod.mesh-catalog.tool.catalog_tools"`,
|
||||
} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("the composed file does not contain %s", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The accounts block alone is what the mesh writes, and it holds nothing about the server.
|
||||
//
|
||||
// **The split is the whole design decision** (ComposeAccounts): ports, TLS paths and a store
|
||||
// directory are properties of the container the module raises, and a controller that wrote them
|
||||
// would have to be kept in step with a Dockerfile it never sees. So this test says what must not be
|
||||
// in the file as plainly as what must.
|
||||
func TestWhatTheMeshWritesIsUsersAndNothingAboutTheServer(t *testing.T) {
|
||||
users, err := Users(someRecords())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hashes := map[string]string{}
|
||||
for _, u := range users {
|
||||
hashes[u.Username()] = "$2a$11$" + strings.Repeat("x", 22)
|
||||
}
|
||||
filled, missing := WithPasswords(users, hashes)
|
||||
if len(missing) != 0 {
|
||||
t.Fatalf("users with no password: %v", missing)
|
||||
}
|
||||
got, err := ComposeAccounts(filled)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for _, want := range []string{"accounts {", `user: "controller"`, `user: "one.telegram"`} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("the accounts file does not contain %s", want)
|
||||
}
|
||||
}
|
||||
// None of the server's own settings. Each of these in the mesh's file is a value the controller
|
||||
// would then own, and the module could no longer change its own image without the mesh agreeing.
|
||||
// `jetstream {` is the server's block (its store, its limits); `jetstream: enabled` inside the
|
||||
// account is the account's, and the mesh owns the account — a user in it is told "JetStream
|
||||
// not enabled for account" without it (2026-09-28).
|
||||
if !strings.Contains(got, "jetstream: enabled") {
|
||||
t.Errorf("the account does not enable JetStream, so no user in it can bind a consumer")
|
||||
}
|
||||
for _, absent := range []string{"port:", "http:", "jetstream {", "tls {", "store_dir", "cert_file"} {
|
||||
if strings.Contains(got, absent) {
|
||||
t.Errorf("the accounts file contains %q, which belongs to the module that raises the "+
|
||||
"server, not to the mesh", absent)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A user with no password is refused here too, not only by the whole-file composition: this is the
|
||||
// function the controller actually calls, and a user without a password is a user anybody is.
|
||||
func TestTheAccountsFileRefusesAUserWithNoPassword(t *testing.T) {
|
||||
if _, err := ComposeAccounts([]Principal{{Kind: KindController}}); err == nil {
|
||||
t.Fatal("a user with no password hash was written")
|
||||
}
|
||||
}
|
||||
|
||||
// **A user list is composed for a bus the mesh has not moved onto yet**, and that is the whole of
|
||||
// step 2 (novox/hq ADR 0116): the server stands in the mesh carrying nothing, on its own ports, while
|
||||
// every node is still on the bus it was on.
|
||||
//
|
||||
// Pinned because the first version of the composing step got it backwards — it wrote the list only
|
||||
// once the controller was already on the new bus, which is a step that cannot be taken: the module
|
||||
// comes up, finds no accounts file, and waits for one the controller had decided not to write.
|
||||
func TestAUserListIsComposedBeforeAnythingMovesOntoTheBus(t *testing.T) {
|
||||
// Exactly the records of a mesh mid-change: everything running, nothing on the new bus.
|
||||
users, err := Users(Records{
|
||||
Nodes: []string{"anchor"},
|
||||
Assigned: map[string][]Declared{"anchor": {{Module: "nats"}}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hashes := map[string]string{}
|
||||
for _, u := range users {
|
||||
hashes[u.Username()] = "$2a$11$" + strings.Repeat("x", 22)
|
||||
}
|
||||
filled, missing := WithPasswords(users, hashes)
|
||||
if len(missing) != 0 {
|
||||
t.Fatalf("users with no credential: %v", missing)
|
||||
}
|
||||
accounts, err := ComposeAccounts(filled)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The controller's own user above all: a file without it is a bus its writer cannot connect to,
|
||||
// which is what the server would be left holding the moment it starts.
|
||||
if !strings.Contains(accounts, `user: "controller"`) {
|
||||
t.Fatalf("the composed list does not contain the controller:\n%s", accounts)
|
||||
}
|
||||
if !strings.Contains(accounts, `user: "node.anchor"`) {
|
||||
t.Errorf("the composed list does not contain the machine running the bus")
|
||||
}
|
||||
}
|
||||
+121
-22
@@ -63,6 +63,19 @@ type Result struct {
|
||||
Built []catalogue.Built
|
||||
}
|
||||
|
||||
// GitCredential is the forge credential a clone may present when the server asks for one.
|
||||
//
|
||||
// **Offered, never pushed.** It is written as a git credential-store file and named to git with
|
||||
// `-c credential.helper=store`, so git itself decides when it applies: only on an authentication
|
||||
// challenge, and only for the URL it was written for — scheme, host and port included. A public
|
||||
// repository clones exactly as before, and a repository on any other host is never shown it.
|
||||
type GitCredential struct {
|
||||
// URL is the credential-store line — scheme://user:password@host[:port] — naming the one
|
||||
// server this credential belongs to. Empty means the builder holds none and every clone is
|
||||
// anonymous, as it always was.
|
||||
URL string
|
||||
}
|
||||
|
||||
// Build clones a repository at a ref, reads its manifest, produces what it declares, publishes
|
||||
// each, and returns the manifest the mesh should hold.
|
||||
//
|
||||
@@ -70,7 +83,8 @@ type Result struct {
|
||||
// archive failed would otherwise leave half of itself in the store under a digest the mesh never
|
||||
// records — reachable, unreferenced, and indistinguishable from something in use.
|
||||
func Build(ctx context.Context, run Runner, publish Publisher,
|
||||
repository, path, ref, workspace string, held map[string]string, npmrc Npmrc, log Log) (Result, error) {
|
||||
repository, path, ref, workspace string, held map[string]string, npmrc Npmrc,
|
||||
forge GitCredential, log Log) (Result, error) {
|
||||
|
||||
say := logging(log)
|
||||
say("clone", "%s%s at %s", repository, describePath(path), refOrHead(ref))
|
||||
@@ -80,6 +94,15 @@ func Build(ctx context.Context, run Runner, publish Publisher,
|
||||
if err := os.MkdirAll(workspace, 0o755); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
// The credential is a file git reads, never an argument: a URL carrying a password in argv
|
||||
// would be readable by anything that can list processes for as long as a clone runs.
|
||||
credentials := ""
|
||||
if forge.URL != "" {
|
||||
credentials = filepath.Join(workspace, "git-credentials")
|
||||
if err := os.WriteFile(credentials, []byte(forge.URL+"\n"), 0o600); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
}
|
||||
tree := filepath.Join(workspace, "source")
|
||||
if err := os.RemoveAll(tree); err != nil {
|
||||
return Result{}, err
|
||||
@@ -87,7 +110,7 @@ func Build(ctx context.Context, run Runner, publish Publisher,
|
||||
// A fresh clone every time rather than a fetch into a tree that is already there. A build
|
||||
// that reuses a working tree can succeed because of something a previous build left behind,
|
||||
// and that is a build nobody can reproduce.
|
||||
if _, err := run(ctx, workspace, "git", "clone", "--quiet", repository, tree); err != nil {
|
||||
if _, err := run(ctx, workspace, "git", cloneWith(credentials, "clone", "--quiet", repository, tree)...); err != nil {
|
||||
say("clone", "FAILED: %v", err)
|
||||
return Result{}, fmt.Errorf("cannot clone %s: %w", repository, err)
|
||||
}
|
||||
@@ -146,6 +169,8 @@ func Build(ctx context.Context, run Runner, publish Publisher,
|
||||
}
|
||||
|
||||
var built []catalogue.Built
|
||||
// stoodOn is every base the build was handed, as resolved — the edges the catalogue derives.
|
||||
var stoodOn []string
|
||||
if manifest.Build != nil {
|
||||
// What this module said it stands on, answered with what this mesh actually holds. Done
|
||||
// before anything is built, so a missing base is refused in front of the person who can
|
||||
@@ -163,11 +188,12 @@ func Build(ctx context.Context, run Runner, publish Publisher,
|
||||
}
|
||||
return from, nil
|
||||
}
|
||||
args, err := standingOn(ctx, manifest, held, mirror)
|
||||
args, bases, err := standingOn(ctx, manifest, held, mirror)
|
||||
if err != nil {
|
||||
say("bases", "UNMET: %v", err)
|
||||
return Result{}, err
|
||||
}
|
||||
stoodOn = bases
|
||||
if len(args) > 0 {
|
||||
say("bases", "%d resolved from what the mesh holds", len(args)/2)
|
||||
}
|
||||
@@ -177,7 +203,7 @@ func Build(ctx context.Context, run Runner, publish Publisher,
|
||||
sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name })
|
||||
for _, a := range artifacts {
|
||||
say("artifact", "%s (%s%s) — starting", a.Name, a.Kind, langSuffix(a))
|
||||
made, err := one(ctx, run, publish, manifest.Module, within, commit, a, args, held, npmrcPath, say)
|
||||
made, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, npmrcPath, say)
|
||||
if err != nil {
|
||||
say("artifact", "%s FAILED: %v", a.Name, err)
|
||||
return Result{}, err
|
||||
@@ -194,7 +220,7 @@ func Build(ctx context.Context, run Runner, publish Publisher,
|
||||
}
|
||||
say("done", "%s at %s — %d artifact(s) pinned", manifest.Module, short(commit), len(built))
|
||||
return Result{Manifest: resolved, Commit: commit, Built: built,
|
||||
Against: against(within, manifest)}, nil
|
||||
Against: against(within, manifest, stoodOn)}, nil
|
||||
}
|
||||
|
||||
// Log is where a build says what it is doing, step by step. Nil is silent — the tests pass none,
|
||||
@@ -210,6 +236,43 @@ func logging(log Log) func(step, format string, args ...any) {
|
||||
}
|
||||
}
|
||||
|
||||
// contextFrom clones an image artifact's own build context, when it names one apart from this
|
||||
// module's own repository — a fresh tree, the same way the module's own is, keyed by artifact
|
||||
// name so two artifacts of one module naming different contexts do not collide.
|
||||
func contextFrom(ctx context.Context, run Runner, workspace, artifact, credentials string,
|
||||
from catalogue.ArtifactContext, say func(step, format string, args ...any)) (string, error) {
|
||||
say("context", "cloning %s at %s for %s", from.Repository, refOrHead(from.Ref), artifact)
|
||||
dir := filepath.Join(workspace, "context-"+artifact)
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := run(ctx, workspace, "git", cloneWith(credentials, "clone", "--quiet", from.Repository, dir)...); err != nil {
|
||||
return "", fmt.Errorf("cannot clone %s: %w", from.Repository, err)
|
||||
}
|
||||
if from.Ref != "" {
|
||||
if _, err := run(ctx, dir, "git", "checkout", "--quiet", from.Ref); err != nil {
|
||||
return "", fmt.Errorf("%s has no %s: %w", from.Repository, from.Ref, err)
|
||||
}
|
||||
}
|
||||
say("context", "done")
|
||||
return dir, nil
|
||||
}
|
||||
|
||||
// cloneWith is a git invocation that may offer a stored credential.
|
||||
//
|
||||
// The first `-c credential.helper=` clears every helper the environment might carry, so exactly
|
||||
// one place answers an authentication challenge: the file the builder wrote. Without a file, the
|
||||
// invocation is exactly what it always was.
|
||||
func cloneWith(credentials string, rest ...string) []string {
|
||||
if credentials == "" {
|
||||
return rest
|
||||
}
|
||||
return append([]string{
|
||||
"-c", "credential.helper=",
|
||||
"-c", "credential.helper=store --file=" + credentials,
|
||||
}, rest...)
|
||||
}
|
||||
|
||||
func describePath(path string) string {
|
||||
if path == "" {
|
||||
return ""
|
||||
@@ -279,14 +342,27 @@ func describe(path string) string {
|
||||
// allowed to name — a tag is something somebody else can move under you.
|
||||
var pinnedImage = regexp.MustCompile(`[A-Za-z0-9][A-Za-z0-9._/:-]*@sha256:[0-9a-f]{64}`)
|
||||
|
||||
// against reads what this module's image artifacts are built on top of, out of the files that
|
||||
// build them. Nothing is guessed: a reference that is not written down is not reported.
|
||||
func against(within string, manifest catalogue.Manifest) []string {
|
||||
// against is what this module's image artifacts are built on top of: every base the mesh resolved
|
||||
// and handed the recipe as a build argument (`build.on`), and any image a recipe pins by digest
|
||||
// itself. Nothing is guessed: a reference that was neither resolved nor written down is not
|
||||
// reported.
|
||||
//
|
||||
// **The resolved bases are the edges.** A recipe reads its base from an argument (`FROM
|
||||
// ${RUNTIME_BASE}`), so the digest is never in the file, and a derivation that read files alone
|
||||
// recorded no edge for any module on the mesh — which is why nothing knew what a changed base
|
||||
// meant to rebuild (novox/hq 04-ISSUES/131).
|
||||
func against(within string, manifest catalogue.Manifest, resolved []string) []string {
|
||||
if manifest.Build == nil {
|
||||
return nil
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, r := range resolved {
|
||||
if r != "" && !seen[r] {
|
||||
seen[r] = true
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
for _, a := range manifest.Build.Artifacts {
|
||||
if a.Kind != catalogue.ArtifactImage || a.From == "" {
|
||||
continue
|
||||
@@ -333,7 +409,7 @@ func wantsPackages(manifest catalogue.Manifest, within string) bool {
|
||||
}
|
||||
|
||||
func one(ctx context.Context, run Runner, publish Publisher,
|
||||
module, tree, commit string, a catalogue.Artifact, args []string,
|
||||
module, tree, workspace, commit, credentials string, a catalogue.Artifact, args []string,
|
||||
held map[string]string, npmrc string, say func(step, format string, args ...any)) (catalogue.Built, error) {
|
||||
|
||||
switch a.Kind {
|
||||
@@ -405,7 +481,27 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
||||
"and start FROM ${<NAME>} (novox/hq ADR 0097)",
|
||||
module, a.From, strings.Join(bases, ", "))
|
||||
}
|
||||
invocation := append([]string{"build", "-f", a.From, "-t", local}, args...)
|
||||
// The recipe is always read from this module's own tree, at this module's own commit — only
|
||||
// the context docker build's final argument names can come from somewhere else, when the
|
||||
// artifact says so.
|
||||
recipePath := a.From
|
||||
buildDir := tree
|
||||
if a.Context != nil {
|
||||
cloned, err := contextFrom(ctx, run, workspace, a.Name, credentials, *a.Context, say)
|
||||
if err != nil {
|
||||
return catalogue.Built{}, fmt.Errorf("%s: %s's context: %w", module, a.Name, err)
|
||||
}
|
||||
// docker build accepts -f outside the context it is given; the recipe stays exactly
|
||||
// where it was read from and validated against, absolute so the working directory
|
||||
// switching to the cloned context does not change which file that is.
|
||||
absRecipe, err := filepath.Abs(filepath.Join(tree, a.From))
|
||||
if err != nil {
|
||||
return catalogue.Built{}, fmt.Errorf("%s: %s's recipe: %w", module, a.Name, err)
|
||||
}
|
||||
recipePath = absRecipe
|
||||
buildDir = cloned
|
||||
}
|
||||
invocation := append([]string{"build", "-f", recipePath, "-t", local}, args...)
|
||||
if a.Target != "" {
|
||||
invocation = append(invocation, "--target", a.Target)
|
||||
}
|
||||
@@ -417,8 +513,8 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
||||
invocation = append(invocation, "--network", "host")
|
||||
}
|
||||
invocation = append(invocation, ".")
|
||||
say("image", "docker build -f %s", a.From)
|
||||
if _, err := run(ctx, tree, "docker", invocation...); err != nil {
|
||||
say("image", "docker build -f %s", recipePath)
|
||||
if _, err := run(ctx, buildDir, "docker", invocation...); err != nil {
|
||||
return catalogue.Built{}, fmt.Errorf("%s: building %s failed: %w", module, a.Name, err)
|
||||
}
|
||||
say("image", "built, publishing")
|
||||
@@ -624,40 +720,42 @@ var _ io.Writer = (*stringWriter)(nil)
|
||||
// built cannot be built here yet, and the useful sentence names which module is missing — not the
|
||||
// one a container runtime produces when a recipe's first line refers to an image nobody has.
|
||||
//
|
||||
// The order is fixed so two builds of one commit invoke the same command.
|
||||
// The order is fixed so two builds of one commit invoke the same command. Returned alongside the
|
||||
// arguments is every reference they resolved to, which is what the build stood on.
|
||||
func standingOn(ctx context.Context, manifest catalogue.Manifest, held map[string]string,
|
||||
mirror func(ctx context.Context, from, repository string) (string, error)) ([]string, error) {
|
||||
mirror func(ctx context.Context, from, repository string) (string, error)) ([]string, []string, error) {
|
||||
if manifest.Build == nil || len(manifest.Build.On) == 0 {
|
||||
return nil, nil
|
||||
return nil, nil, nil
|
||||
}
|
||||
on := append([]catalogue.BuildsOn{}, manifest.Build.On...)
|
||||
sort.Slice(on, func(i, j int) bool { return on[i].Arg < on[j].Arg })
|
||||
|
||||
var args []string
|
||||
var args, resolved []string
|
||||
for _, base := range on {
|
||||
if base.Image != "" {
|
||||
// A vendor's image, declared (novox/hq 04-ISSUES/064, ADR 0097). Pinned, because a tag
|
||||
// is what somebody else can move; copied into the mesh's registry, because a build
|
||||
// that reaches a public registry on its own is a build that works sometimes.
|
||||
if base.Arg == "" || base.Module != "" || base.Artifact != "" {
|
||||
return nil, fmt.Errorf(
|
||||
return nil, nil, fmt.Errorf(
|
||||
"%s stands on the image %s, and a base is either a module's artifact or an "+
|
||||
"image — never both — read from one build argument", manifest.Module, base.Image)
|
||||
}
|
||||
if !strings.Contains(base.Image, "@sha256:") {
|
||||
return nil, fmt.Errorf(
|
||||
return nil, nil, fmt.Errorf(
|
||||
"%s stands on the image %q, which is not pinned by digest. A tag is what "+
|
||||
"somebody else can move; name it as <image>@sha256:…", manifest.Module, base.Image)
|
||||
}
|
||||
reference, err := mirror(ctx, base.Image, manifest.Module+"/on-"+strings.ToLower(base.Arg))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s stands on %s: %w", manifest.Module, base.Image, err)
|
||||
return nil, nil, fmt.Errorf("%s stands on %s: %w", manifest.Module, base.Image, err)
|
||||
}
|
||||
args = append(args, "--build-arg", base.Arg+"="+reference)
|
||||
resolved = append(resolved, reference)
|
||||
continue
|
||||
}
|
||||
if base.Arg == "" || base.Module == "" || base.Artifact == "" {
|
||||
return nil, fmt.Errorf(
|
||||
return nil, nil, fmt.Errorf(
|
||||
"%s says its build stands on something, and does not say all of what: a base "+
|
||||
"needs the module, the artifact, and the build argument the recipe reads it "+
|
||||
"from", manifest.Module)
|
||||
@@ -665,14 +763,15 @@ func standingOn(ctx context.Context, manifest catalogue.Manifest, held map[strin
|
||||
key := base.Module + "/" + base.Artifact
|
||||
reference, has := held[key]
|
||||
if !has {
|
||||
return nil, fmt.Errorf(
|
||||
return nil, nil, fmt.Errorf(
|
||||
"%s builds on %s, and this mesh has not built it. Build %s first — every module "+
|
||||
"in this toolchain stands on it, so it is the thing to have before anything "+
|
||||
"else", manifest.Module, key, base.Module)
|
||||
}
|
||||
args = append(args, "--build-arg", base.Arg+"="+reference)
|
||||
resolved = append(resolved, reference)
|
||||
}
|
||||
return args, nil
|
||||
return args, resolved, nil
|
||||
}
|
||||
|
||||
// compile runs a module's own code through its toolchain, and says where the result is.
|
||||
|
||||
@@ -18,13 +18,19 @@ import (
|
||||
// tree, that two builds of one commit produce one digest. Running docker here would test docker.
|
||||
|
||||
type recorded struct {
|
||||
ran []string
|
||||
ran []string
|
||||
// dirs is the directory each entry in ran was run from, same index — so a test can ask not
|
||||
// only what ran but where.
|
||||
dirs []string
|
||||
images map[string]string
|
||||
archives map[string]string
|
||||
failPush bool
|
||||
// contents is what a clone of this repository lands, so the fake clone can restore the tree
|
||||
// Build deliberately removes first.
|
||||
contents map[string]string
|
||||
// secondary is what a clone of a repository OTHER than the one under test lands, keyed by
|
||||
// that repository's URL — an artifact's own build context, cloned apart from the module.
|
||||
secondary map[string]map[string]string
|
||||
// stamped is the modification time the clone gives every file. Set differently between two
|
||||
// builds of one commit, because otherwise both land in the same second and a packer that
|
||||
// carried timestamps would still produce one digest — which is a test that passes for a
|
||||
@@ -35,13 +41,28 @@ type recorded struct {
|
||||
func (r *recorded) run(_ context.Context, dir, name string, args ...string) (string, error) {
|
||||
line := name + " " + strings.Join(args, " ")
|
||||
r.ran = append(r.ran, line)
|
||||
r.dirs = append(r.dirs, dir)
|
||||
// A clone may carry `-c` configuration in front of the verb — the credential store — so the
|
||||
// verb is found rather than assumed first.
|
||||
isClone := false
|
||||
for _, a := range args {
|
||||
if a == "clone" {
|
||||
isClone = true
|
||||
break
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case name == "git" && len(args) > 0 && args[0] == "clone":
|
||||
case name == "git" && isClone:
|
||||
repository := args[len(args)-2]
|
||||
tree := args[len(args)-1]
|
||||
if err := os.MkdirAll(tree, 0o755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
for path, body := range r.contents {
|
||||
lands := r.contents
|
||||
if by, is := r.secondary[repository]; is {
|
||||
lands = by
|
||||
}
|
||||
for path, body := range lands {
|
||||
full := filepath.Join(tree, path)
|
||||
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
|
||||
return "", err
|
||||
@@ -59,7 +80,6 @@ func (r *recorded) run(_ context.Context, dir, name string, args ...string) (str
|
||||
case name == "git" && len(args) > 0 && args[0] == "rev-parse":
|
||||
return "c0ffeec0ffeec0ffeec0ffeec0ffeec0ffeec0ff\n", nil
|
||||
}
|
||||
_ = dir
|
||||
return "", nil
|
||||
}
|
||||
|
||||
@@ -108,7 +128,7 @@ func TestABuildProducesAManifestThePinsAreIn(t *testing.T) {
|
||||
r, workspace := aRepository(t, withBoth, map[string]string{
|
||||
"Dockerfile": "FROM scratch", "files/theme.conf": "dark",
|
||||
})
|
||||
got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, Npmrc{}, nil)
|
||||
got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -134,7 +154,7 @@ func TestTwoBuildsOfOneCommitProduceOneDigest(t *testing.T) {
|
||||
})
|
||||
// A year apart, so a packer carrying timestamps cannot accidentally agree.
|
||||
r.stamped = time.Date(2020+i, time.March, 3, 4, 5, 6, 0, time.UTC)
|
||||
got, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, nil)
|
||||
got, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -154,7 +174,7 @@ func TestNothingIsPublishedUntilEverythingIsBuilt(t *testing.T) {
|
||||
// unreferenced, and indistinguishable from something in use.
|
||||
r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch"})
|
||||
// `files` is missing, so packing the archive fails — after the image would have been pushed.
|
||||
_, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, nil)
|
||||
_, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||
if err == nil {
|
||||
t.Fatal("a build with a missing input succeeded")
|
||||
}
|
||||
@@ -166,7 +186,7 @@ func TestNothingIsPublishedUntilEverythingIsBuilt(t *testing.T) {
|
||||
func TestARepositoryWithNoManifestSaysSo(t *testing.T) {
|
||||
workspace := t.TempDir()
|
||||
r := &recorded{contents: map[string]string{"README.md": "nothing to see"}}
|
||||
_, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, nil)
|
||||
_, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||
if err == nil {
|
||||
t.Fatal("a repository with nothing saying what it is was built")
|
||||
}
|
||||
@@ -179,7 +199,7 @@ func TestAModuleThatBuildsNothingStillProducesAManifest(t *testing.T) {
|
||||
// Most of what a person installs is configuration.
|
||||
r, workspace := aRepository(t, `{"module":"shell","version":"1","resources":[
|
||||
{"id":"rc","type":"file","path":"/etc/zsh/zshrc","content":"setopt"}]}`, nil)
|
||||
got, err := Build(context.Background(), r.run, r, "https://forge.invalid/shell.git", "", "", workspace, nil, Npmrc{}, nil)
|
||||
got, err := Build(context.Background(), r.run, r, "https://forge.invalid/shell.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -209,7 +229,7 @@ func TestTheTreeIsFreshEveryTime(t *testing.T) {
|
||||
if err := os.WriteFile(leftover, []byte("stale"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, nil); err != nil {
|
||||
if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(leftover); err == nil {
|
||||
@@ -222,7 +242,7 @@ func TestABuildThatCannotPushFails(t *testing.T) {
|
||||
"Dockerfile": "FROM scratch", "files/a": "b",
|
||||
})
|
||||
r.failPush = true
|
||||
if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, nil); err == nil {
|
||||
if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil); err == nil {
|
||||
t.Fatal("a build that could publish nothing reported success")
|
||||
}
|
||||
}
|
||||
@@ -236,7 +256,7 @@ func TestAnUpstreamImageIsMirroredRatherThanBuilt(t *testing.T) {
|
||||
"resources":[{"id":"db","type":"container","name":"mesh-postgres","artifact":"store"}]}`
|
||||
|
||||
r, workspace := aRepository(t, mirrors, nil)
|
||||
got, err := Build(context.Background(), r.run, r, "https://forge.invalid/postgres.git", "", "", workspace, nil, Npmrc{}, nil)
|
||||
got, err := Build(context.Background(), r.run, r, "https://forge.invalid/postgres.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -302,7 +322,7 @@ func TestAModuleIsBuiltFromItsPathWithinTheRepository(t *testing.T) {
|
||||
"modules/other/" + ManifestName: `{"module":"other","version":"1"}`,
|
||||
}}
|
||||
got, err := Build(context.Background(), r.run, r,
|
||||
"https://forge.invalid/catalogue.git", "modules/shell", "", t.TempDir(), nil, Npmrc{}, nil)
|
||||
"https://forge.invalid/catalogue.git", "modules/shell", "", t.TempDir(), nil, Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -321,7 +341,7 @@ func TestAPathThatLeavesTheRepositoryIsRefused(t *testing.T) {
|
||||
for _, escaping := range []string{"../../etc", "/etc"} {
|
||||
r := &recorded{contents: map[string]string{ManifestName: withBoth}}
|
||||
_, err := Build(context.Background(), r.run, r,
|
||||
"https://forge.invalid/x.git", escaping, "", t.TempDir(), nil, Npmrc{}, nil)
|
||||
"https://forge.invalid/x.git", escaping, "", t.TempDir(), nil, Npmrc{}, GitCredential{}, nil)
|
||||
if err == nil {
|
||||
t.Fatalf("%q was accepted as a module's path", escaping)
|
||||
}
|
||||
@@ -331,3 +351,168 @@ func TestAPathThatLeavesTheRepositoryIsRefused(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **Packaging and source are allowed to live apart** — a module that ships only the recipe for
|
||||
// source that lives in a second repository (the reference route-proxy, packaged in the catalogue
|
||||
// but built from mesh-controller's own repository) names where that source actually is, rather
|
||||
// than vendoring a second copy the two could drift from.
|
||||
func TestAnArtifactWithItsOwnContextIsBuiltFromThere(t *testing.T) {
|
||||
const withContext = `{"module":"route-proxy","version":"1",
|
||||
"build":{"artifacts":[
|
||||
{"name":"server","kind":"image","from":"Dockerfile",
|
||||
"context":{"repository":"https://forge.invalid/source.git","ref":"main"}}]}}`
|
||||
r := &recorded{
|
||||
contents: map[string]string{
|
||||
ManifestName: withContext,
|
||||
// The recipe lives with the packaging, not the source — read from here regardless of
|
||||
// where the build context comes from. FROM scratch declares no base, so what is under
|
||||
// test — where the context comes from — is not entangled with ADR 0097's own checks.
|
||||
"Dockerfile": "FROM scratch\nCOPY go.mod ./\n",
|
||||
},
|
||||
secondary: map[string]map[string]string{
|
||||
// go.mod exists only in the second repository. A build context taken from the wrong
|
||||
// place would never find it, which a real docker build would refuse on — the fake
|
||||
// does not read files, so what is checked below is that the build was even pointed
|
||||
// at the right place, not that COPY would have succeeded.
|
||||
"https://forge.invalid/source.git": {"go.mod": "module route-proxy\n"},
|
||||
},
|
||||
}
|
||||
_, err := Build(context.Background(), r.run, r,
|
||||
"https://forge.invalid/catalogue.git", "", "", t.TempDir(), nil, Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var clonedSource bool
|
||||
for _, line := range r.ran {
|
||||
if strings.HasPrefix(line, "git clone") && strings.Contains(line, "https://forge.invalid/source.git") {
|
||||
clonedSource = true
|
||||
}
|
||||
}
|
||||
if !clonedSource {
|
||||
t.Fatalf("the artifact's own context was never cloned: %v", r.ran)
|
||||
}
|
||||
|
||||
buildIndex := -1
|
||||
for i, line := range r.ran {
|
||||
if strings.HasPrefix(line, "docker build ") {
|
||||
buildIndex = i
|
||||
}
|
||||
}
|
||||
if buildIndex == -1 {
|
||||
t.Fatal("no docker build was run")
|
||||
}
|
||||
build := r.ran[buildIndex]
|
||||
buildDir := r.dirs[buildIndex]
|
||||
|
||||
if !strings.Contains(buildDir, "context-server") {
|
||||
t.Errorf("docker build ran from %q, not the artifact's own cloned context", buildDir)
|
||||
}
|
||||
recipe := strings.SplitN(strings.SplitN(build, "-f ", 2)[1], " ", 2)[0]
|
||||
if !filepath.IsAbs(recipe) {
|
||||
t.Errorf("the recipe %q is not an absolute path, so it is read relative to whatever "+
|
||||
"directory the build context moved to rather than where it actually is", recipe)
|
||||
}
|
||||
if !strings.HasSuffix(recipe, string(filepath.Separator)+"Dockerfile") {
|
||||
t.Errorf("the recipe is not the module's own Dockerfile: %q", recipe)
|
||||
}
|
||||
if !strings.HasSuffix(build, " .") {
|
||||
t.Errorf("the build was not given a context: %s", build)
|
||||
}
|
||||
}
|
||||
|
||||
// The forge credential is offered through git's own credential store — a file, never argv — and
|
||||
// git decides when it applies. What is checked: the clone names the store, the secret never
|
||||
// appears in a command line, and the file holds exactly the URL at 0600.
|
||||
func TestABuildOffersTheForgesCredentialThroughGitsOwnStore(t *testing.T) {
|
||||
r, workspace := aRepository(t, withBoth, map[string]string{
|
||||
"Dockerfile": "FROM scratch", "files/theme.conf": "dark",
|
||||
})
|
||||
_, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "",
|
||||
workspace, nil, Npmrc{},
|
||||
GitCredential{URL: "http://mesh_novox_builder:sw0rdfi5h@forge.invalid:20000"}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stored := filepath.Join(workspace, "git-credentials")
|
||||
clone := r.ran[0]
|
||||
if !strings.Contains(clone, "credential.helper=store --file="+stored) {
|
||||
t.Fatalf("the clone does not name the credential store: %s", clone)
|
||||
}
|
||||
for _, line := range r.ran {
|
||||
if strings.Contains(line, "sw0rdfi5h") {
|
||||
t.Fatalf("the secret is in a command line, readable by anything that can list processes: %s", line)
|
||||
}
|
||||
}
|
||||
raw, err := os.ReadFile(stored)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.TrimSpace(string(raw)) != "http://mesh_novox_builder:sw0rdfi5h@forge.invalid:20000" {
|
||||
t.Fatalf("the store does not hold the credential as given: %q", raw)
|
||||
}
|
||||
info, err := os.Stat(stored)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("the credential file is readable beyond its owner: %v", info.Mode())
|
||||
}
|
||||
}
|
||||
|
||||
// Without a credential, a clone is exactly the invocation it always was, and no credential file
|
||||
// appears — the builder a mesh of public repositories runs is unchanged.
|
||||
func TestABuildWithNoCredentialClonesExactlyAsBefore(t *testing.T) {
|
||||
r, workspace := aRepository(t, withBoth, map[string]string{
|
||||
"Dockerfile": "FROM scratch", "files/theme.conf": "dark",
|
||||
})
|
||||
_, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "",
|
||||
workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.HasPrefix(r.ran[0], "git clone --quiet ") {
|
||||
t.Fatalf("a credential-less clone grew flags: %s", r.ran[0])
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(workspace, "git-credentials")); !os.IsNotExist(err) {
|
||||
t.Fatal("a credential file was written with no credential to put in it")
|
||||
}
|
||||
}
|
||||
|
||||
// An artifact's own context is cloned with the same offer: a private module whose context is a
|
||||
// second private repository on the same forge builds, and the secret still never reaches argv.
|
||||
func TestAContextCloneCarriesTheSameCredentialStore(t *testing.T) {
|
||||
const withContext = `{"module":"route-proxy","version":"1",
|
||||
"build":{"artifacts":[
|
||||
{"name":"server","kind":"image","from":"Dockerfile",
|
||||
"context":{"repository":"https://forge.invalid/source.git","ref":"main"}}]}}`
|
||||
r := &recorded{
|
||||
contents: map[string]string{
|
||||
ManifestName: withContext,
|
||||
"Dockerfile": "FROM scratch\nCOPY go.mod ./\n",
|
||||
},
|
||||
secondary: map[string]map[string]string{
|
||||
"https://forge.invalid/source.git": {"go.mod": "module route-proxy\n"},
|
||||
},
|
||||
}
|
||||
workspace := t.TempDir()
|
||||
_, err := Build(context.Background(), r.run, r,
|
||||
"https://forge.invalid/catalogue.git", "", "", workspace, nil, Npmrc{},
|
||||
GitCredential{URL: "https://builder:s3cret@forge.invalid"}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stored := filepath.Join(workspace, "git-credentials")
|
||||
var contextClone string
|
||||
for _, line := range r.ran {
|
||||
if strings.Contains(line, "clone") && strings.Contains(line, "source.git") {
|
||||
contextClone = line
|
||||
}
|
||||
}
|
||||
if contextClone == "" {
|
||||
t.Fatalf("the context was never cloned: %v", r.ran)
|
||||
}
|
||||
if !strings.Contains(contextClone, "credential.helper=store --file="+stored) {
|
||||
t.Fatalf("the context clone does not name the credential store: %s", contextClone)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,7 +54,7 @@ func TestABundleIsCompiledAndPackedWithNoDockerfile(t *testing.T) {
|
||||
held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)}
|
||||
|
||||
got, err := Build(context.Background(), compiling{r}.run, r,
|
||||
"https://forge.invalid/greeter.git", "", "", workspace, held, Npmrc{}, nil)
|
||||
"https://forge.invalid/greeter.git", "", "", workspace, held, Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("a module with a language and no Dockerfile did not build: %v", err)
|
||||
}
|
||||
@@ -91,7 +91,7 @@ func TestABundleWhoseToolchainIsNotHeldIsRefusedFirst(t *testing.T) {
|
||||
r, workspace := aRepository(t, aBundle, map[string]string{"index.ts": "console.log(1)"})
|
||||
|
||||
_, err := Build(context.Background(), compiling{r}.run, r,
|
||||
"https://forge.invalid/greeter.git", "", "", workspace, nil, Npmrc{}, nil)
|
||||
"https://forge.invalid/greeter.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||
if err == nil {
|
||||
t.Fatal("a bundle was built with no toolchain to compile it in")
|
||||
}
|
||||
@@ -112,7 +112,7 @@ func TestABundleInAnUnknownLanguageIsRefused(t *testing.T) {
|
||||
|
||||
_, err := Build(context.Background(), compiling{r}.run, r,
|
||||
"https://forge.invalid/greeter.git", "", "", workspace,
|
||||
map[string]string{"mesh-tools/build": "registry.invalid/x@sha256:" + strings.Repeat("c", 64)}, Npmrc{}, nil)
|
||||
map[string]string{"mesh-tools/build": "registry.invalid/x@sha256:" + strings.Repeat("c", 64)}, Npmrc{}, GitCredential{}, nil)
|
||||
if err == nil {
|
||||
t.Fatal("a language nothing can compile was accepted")
|
||||
}
|
||||
@@ -140,7 +140,7 @@ func TestTwoBundlesInOneModuleArePackedSeparately(t *testing.T) {
|
||||
held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)}
|
||||
|
||||
got, err := Build(context.Background(), compiling{r}.run, r,
|
||||
"https://forge.invalid/greeter.git", "", "", workspace, held, Npmrc{}, nil)
|
||||
"https://forge.invalid/greeter.git", "", "", workspace, held, Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("a module with two bundles did not build: %v", err)
|
||||
}
|
||||
|
||||
@@ -71,7 +71,7 @@ func TestAnImageBuildGetsTheCredentialInTheContextAndHostNetwork(t *testing.T) {
|
||||
r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch\nCOPY .npmrc ./", "files/x": "y"})
|
||||
n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm/", Token: "t"}
|
||||
if _, err := Build(context.Background(), r.run, r,
|
||||
"https://forge.invalid/meshboard.git", "", "", workspace, nil, n, nil); err != nil {
|
||||
"https://forge.invalid/meshboard.git", "", "", workspace, nil, n, GitCredential{}, nil); err != nil {
|
||||
t.Fatalf("the build failed: %v", err)
|
||||
}
|
||||
|
||||
@@ -101,7 +101,7 @@ func TestAnImageBuildGetsTheCredentialInTheContextAndHostNetwork(t *testing.T) {
|
||||
func TestAnImageBuildWithoutACredentialGetsNoHostNetwork(t *testing.T) {
|
||||
r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch", "files/x": "y"})
|
||||
if _, err := Build(context.Background(), r.run, r,
|
||||
"https://forge.invalid/meshboard.git", "", "", workspace, nil, Npmrc{}, nil); err != nil {
|
||||
"https://forge.invalid/meshboard.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil); err != nil {
|
||||
t.Fatalf("the build failed: %v", err)
|
||||
}
|
||||
for _, line := range r.ran {
|
||||
@@ -127,7 +127,7 @@ func TestAPackageIsBuiltOnAPublicBaseAndPublishedByVersion(t *testing.T) {
|
||||
})
|
||||
n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm/", Token: "t"}
|
||||
got, err := Build(context.Background(), r.run, r,
|
||||
"https://forge.invalid/mesh-sdk.git", "", "", workspace, nil, n, nil)
|
||||
"https://forge.invalid/mesh-sdk.git", "", "", workspace, nil, n, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("the package did not build: %v", err)
|
||||
}
|
||||
@@ -159,7 +159,7 @@ func TestAPackageWithNoRegistryIsRefused(t *testing.T) {
|
||||
"package.json": `{"name":"@novox/mesh-sdk","version":"0.1.0"}`,
|
||||
})
|
||||
_, err := Build(context.Background(), r.run, r,
|
||||
"https://forge.invalid/mesh-sdk.git", "", "", workspace, nil, Npmrc{}, nil)
|
||||
"https://forge.invalid/mesh-sdk.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||
if err == nil {
|
||||
t.Fatal("a package built with no registry to publish to, silently")
|
||||
}
|
||||
@@ -206,7 +206,7 @@ func TestAnImageThatDoesNotAskForTheCredentialDoesNotGetIt(t *testing.T) {
|
||||
r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch\nCOPY . .", "files/x": "y"})
|
||||
n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm/", Token: "t"}
|
||||
if _, err := Build(context.Background(), r.run, r,
|
||||
"https://forge.invalid/meshboard.git", "", "", workspace, nil, n, nil); err != nil {
|
||||
"https://forge.invalid/meshboard.git", "", "", workspace, nil, n, GitCredential{}, nil); err != nil {
|
||||
t.Fatalf("the build failed: %v", err)
|
||||
}
|
||||
for _, line := range r.ran {
|
||||
|
||||
@@ -22,7 +22,7 @@ func TestABaseTheMeshHasNotBuiltIsRefused(t *testing.T) {
|
||||
On: []catalogue.BuildsOn{{Arg: "RUNTIME_BASE", Module: "mesh-tools", Artifact: "runtime"}},
|
||||
},
|
||||
}
|
||||
_, err := standingOn(context.Background(), manifest, map[string]string{}, noMirror)
|
||||
_, _, err := standingOn(context.Background(), manifest, map[string]string{}, noMirror)
|
||||
if err == nil {
|
||||
t.Fatal("a base nothing has built was accepted; the build would have failed on its first line")
|
||||
}
|
||||
@@ -42,7 +42,7 @@ func TestABaseTheMeshHoldsBecomesABuildArgument(t *testing.T) {
|
||||
},
|
||||
}
|
||||
held := map[string]string{"mesh-tools/runtime": "127.0.0.1:5000/mesh-tools/runtime@sha256:" + strings.Repeat("a", 64)}
|
||||
args, err := standingOn(context.Background(), manifest, held, noMirror)
|
||||
args, _, err := standingOn(context.Background(), manifest, held, noMirror)
|
||||
if err != nil {
|
||||
t.Fatalf("a base this mesh holds was refused: %v", err)
|
||||
}
|
||||
@@ -54,7 +54,7 @@ func TestABaseTheMeshHoldsBecomesABuildArgument(t *testing.T) {
|
||||
|
||||
// A module naming no base asks for nothing, which is most modules.
|
||||
func TestAModuleNamingNoBaseAddsNoArguments(t *testing.T) {
|
||||
args, err := standingOn(context.Background(), catalogue.Manifest{Module: "hello-web", Build: &catalogue.Build{}}, nil, noMirror)
|
||||
args, _, err := standingOn(context.Background(), catalogue.Manifest{Module: "hello-web", Build: &catalogue.Build{}}, nil, noMirror)
|
||||
if err != nil || args != nil {
|
||||
t.Fatalf("a module naming no base produced %v, %v", args, err)
|
||||
}
|
||||
@@ -66,7 +66,7 @@ func TestAnIncompleteBaseIsRefused(t *testing.T) {
|
||||
Module: "postgres",
|
||||
Build: &catalogue.Build{On: []catalogue.BuildsOn{{Module: "mesh-tools", Artifact: "runtime"}}},
|
||||
}
|
||||
if _, err := standingOn(context.Background(), manifest, map[string]string{"mesh-tools/runtime": "x"}, noMirror); err == nil {
|
||||
if _, _, err := standingOn(context.Background(), manifest, map[string]string{"mesh-tools/runtime": "x"}, noMirror); err == nil {
|
||||
t.Fatal("a base with no build argument was accepted; nothing would have read it")
|
||||
}
|
||||
}
|
||||
@@ -86,7 +86,7 @@ func TestADeclaredVendorImageIsCopiedInAndHandedToTheRecipe(t *testing.T) {
|
||||
},
|
||||
}
|
||||
var asked []string
|
||||
args, err := standingOn(context.Background(), manifest, nil, func(_ context.Context, from, repository string) (string, error) {
|
||||
args, _, err := standingOn(context.Background(), manifest, nil, func(_ context.Context, from, repository string) (string, error) {
|
||||
asked = append(asked, from+" -> "+repository)
|
||||
return "127.0.0.1:5000/" + repository + "@sha256:" + strings.Repeat("d", 64), nil
|
||||
})
|
||||
@@ -101,7 +101,7 @@ func TestADeclaredVendorImageIsCopiedInAndHandedToTheRecipe(t *testing.T) {
|
||||
}
|
||||
// Unpinned, it is refused: a tag is what somebody else can move.
|
||||
manifest.Build.On[0].Image = "quay.io/minio/mc:latest"
|
||||
if _, err := standingOn(context.Background(), manifest, nil, noMirror); err == nil || !strings.Contains(err.Error(), "not pinned") {
|
||||
if _, _, err := standingOn(context.Background(), manifest, nil, noMirror); err == nil || !strings.Contains(err.Error(), "not pinned") {
|
||||
t.Fatalf("an unpinned vendor image was accepted: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -151,3 +151,31 @@ func TestARecipeIsReadAsInstructions(t *testing.T) {
|
||||
t.Fatalf("a heredoc line or a continued stage was read as a base: %v", bases)
|
||||
}
|
||||
}
|
||||
|
||||
// What a build was handed as its bases is what it stood on — recorded, so a changed base knows what
|
||||
// to rebuild (novox/hq 04-ISSUES/131). A recipe reads the base from an argument, so nothing else
|
||||
// could know.
|
||||
func TestTheBasesABuildWasHandedAreWhatItStoodOn(t *testing.T) {
|
||||
manifest := catalogue.Manifest{
|
||||
Module: "gitea",
|
||||
Build: &catalogue.Build{
|
||||
On: []catalogue.BuildsOn{
|
||||
{Arg: "RUNTIME_BASE", Module: "mesh-tools", Artifact: "runtime"},
|
||||
{Arg: "BUILD_BASE", Module: "mesh-tools", Artifact: "build"},
|
||||
},
|
||||
Artifacts: []catalogue.Artifact{{Name: "runtime", Kind: catalogue.ArtifactImage, From: "Dockerfile"}},
|
||||
},
|
||||
}
|
||||
held := map[string]string{
|
||||
"mesh-tools/runtime": "127.0.0.1:5000/mesh-tools/runtime@sha256:" + strings.Repeat("a", 64),
|
||||
"mesh-tools/build": "127.0.0.1:5000/mesh-tools/build@sha256:" + strings.Repeat("b", 64),
|
||||
}
|
||||
_, resolved, err := standingOn(context.Background(), manifest, held, noMirror)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := against(t.TempDir(), manifest, resolved)
|
||||
if len(got) != 2 || got[0] != held["mesh-tools/build"] || got[1] != held["mesh-tools/runtime"] {
|
||||
t.Fatalf("the bases the build was handed were not what it stood on: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// What an adopted node is declared in place of a filter (novox/hq ADR 0100).
|
||||
//
|
||||
// On an adopted node the firewall found on the machine stays in force: the mesh loads no table
|
||||
// that drops by default or holds an accept. What the mesh needs reachable is declared as
|
||||
// openings, which the host converges through the found firewall in its own terms; and the mesh
|
||||
// guards its own foundation ports itself, in a table that only refuses.
|
||||
|
||||
// AdoptionPrefix is the id prefix of what the mesh declares of its own on an adopted node. It is
|
||||
// never a module's, so none of it is ever held as found.
|
||||
const AdoptionPrefix = "adoption."
|
||||
|
||||
// Where an opening admits from, on the wire.
|
||||
const (
|
||||
OpeningFromEverywhere = "everywhere"
|
||||
OpeningFromMesh = "mesh"
|
||||
)
|
||||
|
||||
// The two paths a packet reaches a port by: received by the machine, or forwarded to a
|
||||
// container that publishes it.
|
||||
const (
|
||||
PathIncoming = "incoming"
|
||||
PathForwarded = "forwarded"
|
||||
)
|
||||
|
||||
// Guard resources: the refusal-only table, the unit that loads it, and that unit running.
|
||||
const (
|
||||
GuardPath = "/etc/mesh/guard.nft"
|
||||
GuardUnit = "mesh-guard.service"
|
||||
// GuardUnitPath is where the unit is written.
|
||||
GuardUnitPath = "/etc/systemd/system/" + GuardUnit
|
||||
)
|
||||
|
||||
// GuardID, GuardUnitID and GuardRunningID are the guard's resource identities. The installer
|
||||
// raises the same three on an adopted genesis, so the first push finds them already there.
|
||||
func GuardID() string { return AdoptionPrefix + "guard" }
|
||||
func GuardUnitID() string { return AdoptionPrefix + "guard-unit" }
|
||||
func GuardRunningID() string { return AdoptionPrefix + "guard-running" }
|
||||
|
||||
// GuardPackageID is the tool that loads the guard, declared first: a node joining adopted has
|
||||
// no filter module and may have no nft at all, and a table nothing can load guards nothing.
|
||||
func GuardPackageID() string { return AdoptionPrefix + "guard-package" }
|
||||
|
||||
// GuardPackage is the package that carries nft.
|
||||
const GuardPackage = "nftables"
|
||||
|
||||
// OpeningID is an opening's resource identity: its protocol, port and path say what it is.
|
||||
func OpeningID(protocol string, port int, path string) string {
|
||||
return fmt.Sprintf("%sopening-%s-%d-%s", AdoptionPrefix, protocol, port, path)
|
||||
}
|
||||
|
||||
// Openings are what the mesh needs reachable on an adopted node, from the same inputs as the
|
||||
// filter it would load were the node converged, each from where that filter would admit it.
|
||||
//
|
||||
// `rules` is Filtering's answer — every module's listens, the hub's port, the per-node exposure —
|
||||
// and `foundation` is the ports the mesh itself needs, from everywhere. A rule for this machine
|
||||
// only opens nothing. `published` maps a machine port a container publishes to the container's
|
||||
// port: a published port is forwarded, not received, so its opening names the forwarded path and
|
||||
// the port the packet is forwarded to.
|
||||
func Openings(rules []Rule, foundation []int, published map[string]map[int]int) []map[string]any {
|
||||
type key struct {
|
||||
protocol string
|
||||
port int
|
||||
}
|
||||
from := map[key]string{}
|
||||
var order []key
|
||||
widen := func(k key, f string) {
|
||||
was, seen := from[k]
|
||||
if !seen {
|
||||
order = append(order, k)
|
||||
}
|
||||
if !seen || was != OpeningFromEverywhere {
|
||||
from[k] = f
|
||||
}
|
||||
}
|
||||
for _, rule := range rules {
|
||||
switch rule.From {
|
||||
case FromEverywhere:
|
||||
widen(key{rule.Protocol, rule.Port}, OpeningFromEverywhere)
|
||||
case FromMesh:
|
||||
widen(key{rule.Protocol, rule.Port}, OpeningFromMesh)
|
||||
}
|
||||
}
|
||||
for _, port := range foundation {
|
||||
widen(key{"tcp", port}, OpeningFromEverywhere)
|
||||
}
|
||||
sort.Slice(order, func(a, b int) bool {
|
||||
if order[a].port != order[b].port {
|
||||
return order[a].port < order[b].port
|
||||
}
|
||||
return order[a].protocol < order[b].protocol
|
||||
})
|
||||
out := make([]map[string]any, 0, len(order))
|
||||
for _, k := range order {
|
||||
opening := map[string]any{"type": "opening", "port": k.port, "protocol": k.protocol,
|
||||
"from": from[k]}
|
||||
if to, forwarded := published[k.protocol][k.port]; forwarded {
|
||||
opening["id"] = OpeningID(k.protocol, k.port, PathForwarded)
|
||||
opening["path"] = PathForwarded
|
||||
opening["to"] = to
|
||||
} else {
|
||||
opening["id"] = OpeningID(k.protocol, k.port, PathIncoming)
|
||||
opening["path"] = PathIncoming
|
||||
}
|
||||
out = append(out, opening)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Published is every port the given containers publish on the machine, by protocol and machine
|
||||
// port, mapped to the container's own port. A mapping bound to loopback is left out: nothing off
|
||||
// the machine reaches it, forwarded or not.
|
||||
func Published(resources []map[string]any) map[string]map[int]int {
|
||||
out := map[string]map[int]int{}
|
||||
for _, r := range resources {
|
||||
if fmt.Sprint(r["type"]) != "container" {
|
||||
continue
|
||||
}
|
||||
listed, _ := r["ports"].([]any)
|
||||
for _, entry := range listed {
|
||||
written := strings.TrimSpace(fmt.Sprint(entry))
|
||||
protocol := "tcp"
|
||||
if cut := strings.LastIndex(written, "/"); cut >= 0 {
|
||||
protocol = written[cut+1:]
|
||||
}
|
||||
// Indexed from the end, so an IPv6 address's own colons never shift the ports.
|
||||
outer, inner, address, ok := mapping(written)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
switch strings.Trim(address, "[]") {
|
||||
case "127.0.0.1", "localhost", "::1":
|
||||
continue
|
||||
}
|
||||
if out[protocol] == nil {
|
||||
out[protocol] = map[int]int{}
|
||||
}
|
||||
out[protocol][outer] = inner
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// AsGuard renders the mesh's refusal-only table for the given machine ports.
|
||||
//
|
||||
// It passes everything by default and holds nothing but a refusal, so it cannot close anything
|
||||
// the machine serves; and it is the mesh's own table, so the found firewall reloading does not
|
||||
// touch it. It refuses only packets addressed to this machine, and the ports except from the
|
||||
// machine itself — its loopback and the container runtime's own networks — and from the private
|
||||
// network, known by the interface a packet arrives on and never by its source address. At
|
||||
// prerouting, ahead of the runtime's destination translation, so it matches the port the packet
|
||||
// was sent to; in the inet family, so both address families.
|
||||
//
|
||||
// **The machine's own interfaces are named, where the derived filter names address ranges.** The
|
||||
// filter accepts the container runtime's networks by CIDR; this excludes its bridges by name — lo,
|
||||
// docker0, the br-* a compose network gets, and the mesh's own mesh0. A runtime whose bridge is
|
||||
// named anything else (a podman or libvirt bridge, or a docker network created with a fixed name)
|
||||
// would have its containers' traffic to a guarded port refused, which reads as the port being
|
||||
// down. Names rather than addresses is deliberate: a source address can be claimed by whoever
|
||||
// sends the packet, and this table exists to refuse what the found firewall never sees. Widening
|
||||
// it means adding names here and in the installer's copy together, which the golden test holds to
|
||||
// one text.
|
||||
//
|
||||
// The same text the installer raises on an adopted genesis; a test holds both to it.
|
||||
func AsGuard(ports []int) string {
|
||||
sorted := append([]int{}, ports...)
|
||||
sort.Ints(sorted)
|
||||
listed := make([]string, len(sorted))
|
||||
for i, p := range sorted {
|
||||
listed[i] = strconv.Itoa(p)
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString("table inet mesh_guard {}\n")
|
||||
b.WriteString("delete table inet mesh_guard\n")
|
||||
b.WriteString("table inet mesh_guard {\n")
|
||||
b.WriteString("\tchain prerouting {\n")
|
||||
b.WriteString("\t\ttype filter hook prerouting priority raw; policy accept;\n")
|
||||
// Only packets addressed to this machine: traffic it routes for others — a predecessor's hub,
|
||||
// say — is never the guard's business (novox/hq ADR 0103).
|
||||
fmt.Fprintf(&b, "\t\tfib daddr type local iifname != \"lo\" iifname != \"docker0\" "+
|
||||
"iifname != \"br-*\" iifname != \"mesh0\" tcp dport { %s } drop\n",
|
||||
strings.Join(listed, ", "))
|
||||
b.WriteString("\t}\n")
|
||||
b.WriteString("}\n")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// GuardUnitText is the unit that loads the guard. Stopping it deletes only its own table: never
|
||||
// a flush, which would take the container runtime's rules and the found firewall with it.
|
||||
func GuardUnitText() string {
|
||||
return "[Unit]\n" +
|
||||
"Description=The mesh's guard: refuses its own ports from outside (novox/hq ADR 0100)\n" +
|
||||
// Early, before the network is up, and without the default dependencies that would
|
||||
// order it after the network; stopped at shutdown like any unit.
|
||||
"DefaultDependencies=no\n" +
|
||||
"Wants=network-pre.target\n" +
|
||||
"Before=network-pre.target shutdown.target\n" +
|
||||
"Conflicts=shutdown.target\n" +
|
||||
"\n" +
|
||||
"[Service]\n" +
|
||||
"Type=oneshot\n" +
|
||||
"RemainAfterExit=yes\n" +
|
||||
"ExecStart=nft -f " + GuardPath + "\n" +
|
||||
"ExecReload=nft -f " + GuardPath + "\n" +
|
||||
"ExecStop=nft delete table inet mesh_guard\n" +
|
||||
"\n" +
|
||||
"[Install]\n" +
|
||||
"WantedBy=multi-user.target\n"
|
||||
}
|
||||
|
||||
// GuardResources are the guard as four resources of the existing kinds: the tool that loads it,
|
||||
// the table, the unit, and the unit running — reloaded when the table changes, so the new table
|
||||
// replaces the old in one `nft -f` through the unit's ExecReload with no moment unguarded, and
|
||||
// restarted only when the unit itself changes. Nothing when there is nothing to guard: an empty
|
||||
// set is not a table nft loads.
|
||||
func GuardResources(ports []int) []map[string]any {
|
||||
if len(ports) == 0 {
|
||||
return nil
|
||||
}
|
||||
return []map[string]any{
|
||||
{"id": GuardPackageID(), "type": "package", "package": GuardPackage},
|
||||
{"id": GuardID(), "type": "file", "path": GuardPath, "content": AsGuard(ports),
|
||||
"mode": "0644"},
|
||||
{"id": GuardUnitID(), "type": "file", "path": GuardUnitPath, "content": GuardUnitText(),
|
||||
"mode": "0644"},
|
||||
{"id": GuardRunningID(), "type": "service", "unit": GuardUnit, "state": "running",
|
||||
"boot": "enabled", "restart-on": []any{GuardUnitID()}, "reload-on": []any{GuardID()}},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,675 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0100: on an adopted node the found firewall stays in force. The mesh declares
|
||||
// openings where it would have loaded a filter, and guards its own ports in a table that only
|
||||
// refuses.
|
||||
|
||||
// hub is the private network's generator on the hub: it opens the hub's port from anywhere.
|
||||
type hub struct{}
|
||||
|
||||
func (hub) Resources(string) ([]map[string]any, bool, error) {
|
||||
return []map[string]any{{"id": "config", "type": "file", "path": "/etc/wireguard/mesh0.conf",
|
||||
"content": "[Interface]\n"}}, true, nil
|
||||
}
|
||||
func (hub) Listens(string) ([]Listening, error) {
|
||||
return []Listening{{Port: 51820, Protocol: "udp", From: FromEverywhere}}, nil
|
||||
}
|
||||
|
||||
// anAdoptedAnchor is the control-node's set: the store, the bus, the registry, the private network,
|
||||
// a served module and the filter module.
|
||||
func anAdoptedAnchor() Resolution {
|
||||
return Resolution{Node: "anchor", Modules: []Manifest{
|
||||
{Module: "network", Computed: "overlay"},
|
||||
{Module: "postgres", Guards: []int{5432},
|
||||
Listens: []Listening{{Port: 5432, From: FromMesh}},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store",
|
||||
"ports": []any{"5432:5432"}}}},
|
||||
{Module: "lavinmq", Guards: []int{15672},
|
||||
Listens: []Listening{{Port: 5671, From: FromMesh}, {Port: 5672, From: FromMesh}},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-broker",
|
||||
"ports": []any{"5671:5671", "5672:5672", "127.0.0.1:15672:15672"}}}},
|
||||
{Module: "distribution",
|
||||
Listens: []Listening{{Port: 5000, From: FromMesh}},
|
||||
Resources: []map[string]any{{"id": "store", "type": "container", "name": "registry",
|
||||
"ports": []any{"5000"}}}},
|
||||
{Module: "hello-web",
|
||||
Listens: []Listening{{Port: 8080, From: FromEverywhere}},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "hello-web",
|
||||
"ports": []any{"8080"}}}},
|
||||
{Module: "helper", Listens: []Listening{{Port: 9000, From: FromMachine}}},
|
||||
{Module: "nftables", Filtering: &Filtering{Into: "/etc/nftables.conf"},
|
||||
Resources: []map[string]any{{"id": "load", "type": "service", "unit": "mesh-filter.service",
|
||||
"state": "running", "restart-on": []any{"filtering"}}}},
|
||||
}}
|
||||
}
|
||||
|
||||
func anchorRendering(adopted bool) Rendering {
|
||||
return Rendering{
|
||||
Generators: map[string]Generator{"overlay": hub{}},
|
||||
Ports: map[string]map[int]int{"distribution": {5000: 5000}, "hello-web": {8080: 20001}},
|
||||
Settings: SettingsBy{"distribution": {{From: "node anchor",
|
||||
Values: map[string]any{ExposeSetting: map[string]any{"5000": FromEverywhere}}}}},
|
||||
Mesh: []string{"10.42.0.1"},
|
||||
Foundation: []int{5671},
|
||||
Adopted: adopted,
|
||||
// Genesis takes the foundation's modules.
|
||||
Taken: map[string]bool{"postgres": true, "lavinmq": true},
|
||||
}
|
||||
}
|
||||
|
||||
func byID(resources []map[string]any) map[string]map[string]any {
|
||||
out := map[string]map[string]any{}
|
||||
for _, r := range resources {
|
||||
out[r["id"].(string)] = r
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestAnAdoptedNodeIsDeclaredOpeningsFromTheSameInputsAsTheFilter(t *testing.T) {
|
||||
composed, err := anAdoptedAnchor().Compose(anchorRendering(true))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := byID(composed.Resources)
|
||||
want := map[string]map[string]any{
|
||||
// The hub's port, from anywhere, received.
|
||||
"adoption.opening-udp-51820-incoming": {"port": 51820, "protocol": "udp",
|
||||
"from": "everywhere", "path": "incoming"},
|
||||
// The store's port from the private network only, and forwarded: a container publishes it.
|
||||
"adoption.opening-tcp-5432-forwarded": {"port": 5432, "protocol": "tcp", "from": "mesh",
|
||||
"path": "forwarded", "to": 5432},
|
||||
// The bus from anywhere: a node enrols over it before it has a private address.
|
||||
"adoption.opening-tcp-5671-forwarded": {"port": 5671, "protocol": "tcp",
|
||||
"from": "everywhere", "path": "forwarded", "to": 5671},
|
||||
"adoption.opening-tcp-5672-forwarded": {"port": 5672, "protocol": "tcp", "from": "mesh",
|
||||
"path": "forwarded", "to": 5672},
|
||||
// The registry from anywhere, by its node's exposure setting.
|
||||
"adoption.opening-tcp-5000-forwarded": {"port": 5000, "protocol": "tcp",
|
||||
"from": "everywhere", "path": "forwarded", "to": 5000},
|
||||
// A published port names the machine port and the container port it is forwarded to.
|
||||
"adoption.opening-tcp-20001-forwarded": {"port": 20001, "protocol": "tcp",
|
||||
"from": "everywhere", "path": "forwarded", "to": 8080},
|
||||
}
|
||||
for id, fields := range want {
|
||||
opening, ok := got[id]
|
||||
if !ok {
|
||||
t.Errorf("no %s among %v", id, keys(got))
|
||||
continue
|
||||
}
|
||||
if opening["type"] != "opening" {
|
||||
t.Errorf("%s is a %v", id, opening["type"])
|
||||
}
|
||||
for k, v := range fields {
|
||||
if opening[k] != v {
|
||||
t.Errorf("%s: %s is %v, want %v", id, k, opening[k], v)
|
||||
}
|
||||
}
|
||||
}
|
||||
for id := range got {
|
||||
if strings.HasPrefix(id, "adoption.opening-") && want[id] == nil {
|
||||
t.Errorf("an opening nothing asked for: %s", id)
|
||||
}
|
||||
}
|
||||
// A port for this machine only opens nothing, and the management port is not opened at all.
|
||||
for id := range got {
|
||||
if strings.Contains(id, "-9000-") || strings.Contains(id, "-15672-") {
|
||||
t.Errorf("%s is opened", id)
|
||||
}
|
||||
}
|
||||
|
||||
// And openings come first, in the order the machine applies them.
|
||||
if !strings.HasPrefix(composed.Resources[0]["id"].(string), "adoption.opening-") {
|
||||
t.Errorf("openings are not first: %v", composed.Resources[0]["id"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAdoptedNodeLoadsNoFilterOfTheMeshs(t *testing.T) {
|
||||
composed, err := anAdoptedAnchor().Compose(anchorRendering(true))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range composed.Resources {
|
||||
if r["path"] == "/etc/nftables.conf" || strings.HasPrefix(r["id"].(string), "nftables.") {
|
||||
t.Fatalf("an adopted node is declared the filter module's %v", r["id"])
|
||||
}
|
||||
if content, _ := r["content"].(string); strings.Contains(content, "policy drop") {
|
||||
t.Fatalf("an adopted node is declared a table that drops by default: %v", r["id"])
|
||||
}
|
||||
// Nothing but refusals: the only accept in the guard is its policy.
|
||||
if content, _ := r["content"].(string); r["id"] == GuardID() &&
|
||||
strings.Count(content, "accept") != 1 {
|
||||
t.Fatalf("the guard holds an accept:\n%s", content)
|
||||
}
|
||||
}
|
||||
got := byID(composed.Resources)
|
||||
guard := got[GuardID()]
|
||||
if guard == nil || got[GuardUnitID()] == nil || got[GuardRunningID()] == nil {
|
||||
t.Fatalf("no guard: %v", keys(got))
|
||||
}
|
||||
if guard["content"] != AsGuard([]int{5432, 5672, 15672}) {
|
||||
t.Fatalf("the guard does not guard the store, the broker and its management port:\n%s",
|
||||
guard["content"])
|
||||
}
|
||||
// The tool that loads it comes first, and the table after it: a node joining adopted has no
|
||||
// filter module and may have no nft.
|
||||
pkg, table := -1, -1
|
||||
for i, r := range composed.Resources {
|
||||
switch r["id"] {
|
||||
case GuardPackageID():
|
||||
pkg = i
|
||||
if r["type"] != "package" || r["package"] != "nftables" {
|
||||
t.Fatalf("the guard's package is %v", r)
|
||||
}
|
||||
case GuardID():
|
||||
table = i
|
||||
}
|
||||
}
|
||||
if pkg < 0 || pkg > table {
|
||||
t.Fatalf("nftables is not declared before the guard's table (%d, %d)", pkg, table)
|
||||
}
|
||||
// A changed table is reloaded — one `nft -f`, atomic — never restarted, which would delete the
|
||||
// table and leave the ports unguarded until it is loaded again. Only a changed unit restarts.
|
||||
if !reflect.DeepEqual(got[GuardRunningID()]["reload-on"], []any{GuardID()}) ||
|
||||
!reflect.DeepEqual(got[GuardRunningID()]["restart-on"], []any{GuardUnitID()}) {
|
||||
t.Fatalf("the guard is not reloaded on its table and restarted on its unit: %v",
|
||||
got[GuardRunningID()])
|
||||
}
|
||||
// Nothing of the mesh's own is anybody's to hold.
|
||||
for id, module := range composed.Owner {
|
||||
if strings.HasPrefix(id, AdoptionPrefix) {
|
||||
t.Fatalf("%s is owned by %s", id, module)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAConvergedNodeIsDeclaredItsFilterAndNoOpenings(t *testing.T) {
|
||||
composed, err := anAdoptedAnchor().Compose(anchorRendering(false))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := byID(composed.Resources)
|
||||
if got["nftables.filtering"] == nil || got["nftables.load"] == nil {
|
||||
t.Fatalf("a converged node lost its filter: %v", keys(got))
|
||||
}
|
||||
for id := range got {
|
||||
if strings.HasPrefix(id, AdoptionPrefix) {
|
||||
t.Fatalf("a converged node is declared %s", id)
|
||||
}
|
||||
}
|
||||
plain, err := anAdoptedAnchor().Declaration(anchorRendering(false))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a, _ := json.Marshal(plain)
|
||||
b, _ := json.Marshal(composed.Resources)
|
||||
if string(a) != string(b) {
|
||||
t.Fatal("Compose and Declaration disagree on a converged node")
|
||||
}
|
||||
}
|
||||
|
||||
// The table the installer raises and the controller declares, character for character.
|
||||
func TestTheGuardIsExactlyThisTable(t *testing.T) {
|
||||
const golden = `table inet mesh_guard {}
|
||||
delete table inet mesh_guard
|
||||
table inet mesh_guard {
|
||||
chain prerouting {
|
||||
type filter hook prerouting priority raw; policy accept;
|
||||
fib daddr type local iifname != "lo" iifname != "docker0" iifname != "br-*" iifname != "mesh0" tcp dport { 5432, 15672 } drop
|
||||
}
|
||||
}
|
||||
`
|
||||
if got := AsGuard([]int{15672, 5432}); got != golden {
|
||||
t.Fatalf("the guard changed:\n%s", got)
|
||||
}
|
||||
const unit = `[Unit]
|
||||
Description=The mesh's guard: refuses its own ports from outside (novox/hq ADR 0100)
|
||||
DefaultDependencies=no
|
||||
Wants=network-pre.target
|
||||
Before=network-pre.target shutdown.target
|
||||
Conflicts=shutdown.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=nft -f /etc/mesh/guard.nft
|
||||
ExecReload=nft -f /etc/mesh/guard.nft
|
||||
ExecStop=nft delete table inet mesh_guard
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
`
|
||||
if got := GuardUnitText(); got != unit {
|
||||
t.Fatalf("the guard's unit changed:\n%s", got)
|
||||
}
|
||||
if GuardResources(nil) != nil {
|
||||
t.Fatal("a guard with nothing to guard is an empty set nft refuses to load")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAGuardedPortMustBeAPort(t *testing.T) {
|
||||
if _, err := ParseManifest([]byte(`{"module":"postgres","guards":[5432]}`)); err != nil {
|
||||
t.Fatalf("guards is refused: %v", err)
|
||||
}
|
||||
if _, err := ParseManifest([]byte(`{"module":"postgres","guards":[0]}`)); err == nil {
|
||||
t.Fatal("guarding port 0 was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func keys[V any](m map[string]V) []string {
|
||||
return sortedKeys(m)
|
||||
}
|
||||
|
||||
// novox/hq ADR 0100: the foundation's ports are the node's. Given 5433 for the store, every place
|
||||
// that uses the port reads it from there: the container, the filter, the openings, the guard.
|
||||
func TestAGivenPortIsUsedEverywhereThePortIs(t *testing.T) {
|
||||
given := map[string]map[int]int{"postgres": {5432: 5433}, "lavinmq": {15672: 15673}}
|
||||
for _, adopted := range []bool{true, false} {
|
||||
with := anchorRendering(adopted)
|
||||
with.Given = given
|
||||
with.Ports["postgres"] = map[int]int{5432: 5433}
|
||||
composed, err := anAdoptedAnchor().Compose(with)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := byID(composed.Resources)
|
||||
if ports := got["postgres.server"]["ports"]; !reflect.DeepEqual(ports, []any{"5433:5432"}) {
|
||||
t.Fatalf("the store's container publishes %v", ports)
|
||||
}
|
||||
if ports := got["lavinmq.server"]["ports"]; !reflect.DeepEqual(ports,
|
||||
[]any{"5671:5671", "5672:5672", "127.0.0.1:15673:15672"}) {
|
||||
t.Fatalf("the broker's container publishes %v", ports)
|
||||
}
|
||||
if !adopted {
|
||||
filter, _ := got["nftables.filtering"]["content"].(string)
|
||||
if !strings.Contains(filter, "tcp dport 5433 accept") || strings.Contains(filter, "5432") {
|
||||
t.Fatalf("the filter does not use the given port:\n%s", filter)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if o := got["adoption.opening-tcp-5433-forwarded"]; o == nil || o["to"] != 5432 {
|
||||
t.Fatalf("no opening for the given port: %v", keys(got))
|
||||
}
|
||||
if guard := got[GuardID()]["content"]; guard != AsGuard([]int{5433, 5672, 15673}) {
|
||||
t.Fatalf("the guard does not guard the given ports:\n%s", guard)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAGivenPortIsTheNodesAndReachesSomething(t *testing.T) {
|
||||
store := anAdoptedAnchor().Modules[1]
|
||||
node := func(v any) []Layer {
|
||||
return []Layer{{From: "anchor", Values: map[string]any{PortsSetting: v}}}
|
||||
}
|
||||
if got, err := GivenPorts(store, node(map[string]any{"5432": float64(5433)})); err != nil ||
|
||||
got[5432] != 5433 {
|
||||
t.Fatalf("a node's given port was not read: %v %v", got, err)
|
||||
}
|
||||
if _, err := GivenPorts(store, []Layer{{From: MeshWideLayer,
|
||||
Values: map[string]any{PortsSetting: map[string]any{"5432": float64(5433)}}}}); err == nil {
|
||||
t.Fatal("a port given for the whole mesh was accepted")
|
||||
}
|
||||
if _, err := GivenPorts(store, node(map[string]any{"6000": float64(6001)})); err == nil {
|
||||
t.Fatal("a port the module neither listens on, publishes nor guards was given")
|
||||
}
|
||||
if _, err := GivenPorts(store, node(map[string]any{"5432": float64(70000)})); err == nil {
|
||||
t.Fatal("a machine port that is not a port was given")
|
||||
}
|
||||
if _, err := GivenPorts(store, node(map[string]any{"5432": float64(22)})); err == nil {
|
||||
t.Fatal("ssh's port was given")
|
||||
}
|
||||
broker := anAdoptedAnchor().Modules[2]
|
||||
if _, err := GivenPorts(broker, node(map[string]any{"5671": float64(5700),
|
||||
"5672": float64(5700)})); err == nil {
|
||||
t.Fatal("one machine port was given for two of the module's ports")
|
||||
}
|
||||
if stray := UnusedSettings(store, node(map[string]any{"5432": float64(5433)})); len(stray) != 0 {
|
||||
t.Fatalf("a given port is called stray: %v", stray)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0103: the guard is derived, and from taken modules only — every machine port a taken
|
||||
// module publishes that the filter admits from the private network only, and the ports its
|
||||
// manifest guards. A module assigned but not taken is not guarded: its port may still be the
|
||||
// predecessor's.
|
||||
func TestTheGuardIsDerivedFromTakenModulesOnly(t *testing.T) {
|
||||
guardOf := func(with Rendering) string {
|
||||
t.Helper()
|
||||
composed, err := anAdoptedAnchor().Compose(with)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
content, _ := byID(composed.Resources)[GuardID()]["content"].(string)
|
||||
return content
|
||||
}
|
||||
|
||||
// The broker taken, the store not: the broker's plain port follows from its listens (from
|
||||
// the mesh, published), its management port from its manifest; the store is not guarded, and
|
||||
// neither is the bus, which the mesh needs from everywhere.
|
||||
with := anchorRendering(true)
|
||||
with.Taken = map[string]bool{"lavinmq": true}
|
||||
if got := guardOf(with); got != AsGuard([]int{5672, 15672}) {
|
||||
t.Fatalf("the guard is not the taken broker's ports:\n%s", got)
|
||||
}
|
||||
|
||||
// A taken module publishing a port admitted from everywhere is not guarded; one admitted from
|
||||
// the mesh is. The registry is exposed everywhere on this node, and hello-web listens from
|
||||
// everywhere.
|
||||
with.Taken = map[string]bool{"distribution": true, "hello-web": true}
|
||||
if got := guardOf(with); got != "" {
|
||||
t.Fatalf("a port admitted from everywhere is guarded:\n%s", got)
|
||||
}
|
||||
with.Settings = nil
|
||||
if got := guardOf(with); got != AsGuard([]int{5000}) {
|
||||
t.Fatalf("the registry, from the mesh only, is not guarded:\n%s", got)
|
||||
}
|
||||
|
||||
// Nothing taken, nothing guarded — and no guard at all rather than an empty set.
|
||||
with = anchorRendering(true)
|
||||
with.Taken = nil
|
||||
if got := guardOf(with); got != "" {
|
||||
t.Fatalf("an untaken store is guarded:\n%s", got)
|
||||
}
|
||||
|
||||
// A given port is followed: where the machine put it is what is refused.
|
||||
with = anchorRendering(true)
|
||||
with.Given = map[string]map[int]int{"lavinmq": {5672: 5682, 15672: 15673}}
|
||||
with.Ports["lavinmq"] = map[int]int{5671: 5671, 5672: 5682}
|
||||
if got := guardOf(with); got != AsGuard([]int{5432, 5682, 15673}) {
|
||||
t.Fatalf("the guard does not follow the given ports:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A mapping bound to loopback is not published to anything off the machine — in either address
|
||||
// family — and an address's own colons never shift the ports.
|
||||
func TestPublishedLeavesOutLoopbackInBothFamilies(t *testing.T) {
|
||||
got := Published([]map[string]any{{"type": "container", "ports": []any{
|
||||
"127.0.0.1:15672:15672", "[::1]:8080:80", "localhost:9090:90",
|
||||
"[::]:8443:443", "0.0.0.0:5000:5000", "5353:53/udp"}}})
|
||||
want := map[string]map[int]int{"tcp": {8443: 443, 5000: 5000}, "udp": {5353: 53}}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("published is %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0038: only a published port is the mesh's to move. A module that binds the machine
|
||||
// itself listens where its software was told to, so giving it a machine port is refused.
|
||||
func TestAGivenPortIsRefusedForAPortNoContainerPublishes(t *testing.T) {
|
||||
onTheMachine := Manifest{Module: "daemon",
|
||||
Listens: []Listening{{Port: 9000, From: FromMesh}}, Guards: []int{9000}}
|
||||
layers := []Layer{{From: "node anchor",
|
||||
Values: map[string]any{PortsSetting: map[string]any{"9000": float64(9100)}}}}
|
||||
_, err := GivenPorts(onTheMachine, layers)
|
||||
if err == nil || !strings.Contains(err.Error(), "does not publish") {
|
||||
t.Fatalf("a port no container publishes was given: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0103: a guarded port this node is told to open to everyone is opened, not guarded.
|
||||
// An opening from everywhere beside a guard dropping it is one statement refusing the other.
|
||||
func TestAGuardedPortOpenedToEveryoneIsNotGuarded(t *testing.T) {
|
||||
with := anchorRendering(true)
|
||||
with.Settings["postgres"] = []Layer{{From: "node anchor",
|
||||
Values: map[string]any{ExposeSetting: map[string]any{"5432": FromEverywhere}}}}
|
||||
composed, err := anAdoptedAnchor().Compose(with)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := byID(composed.Resources)
|
||||
if o := got["adoption.opening-tcp-5432-forwarded"]; o == nil || o["from"] != OpeningFromEverywhere {
|
||||
t.Fatalf("the store's port is not opened to everyone: %v", o)
|
||||
}
|
||||
if guard, _ := got[GuardID()]["content"].(string); guard != AsGuard([]int{5672, 15672}) {
|
||||
t.Fatalf("a port opened to everyone is still guarded:\n%s", guard)
|
||||
}
|
||||
}
|
||||
|
||||
// A module that publishes its ssh port the long way — `2222:22`, because the machine's own daemon
|
||||
// holds 22 — and says it listens on the machine side of that mapping, which is what anything
|
||||
// reaching it dials.
|
||||
func aForge() Manifest {
|
||||
return Manifest{Module: "forge",
|
||||
Provides: []Offer{{Name: "git-over-ssh", Scope: ScopeMesh}},
|
||||
Serves: map[string]map[string]any{"git-over-ssh": {"port": 2222}},
|
||||
Listens: []Listening{{Port: 3000, From: FromMesh}, {Port: 2222, From: FromMesh}},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "forge",
|
||||
"ports": []any{"3000", "2222:22"}}}}
|
||||
}
|
||||
|
||||
// portsAsThePlanWould is where this machine puts each of a module's ports, derived the way
|
||||
// cmd/mesh-controller/plan.go derives it: a given port first, looked up by the port the module
|
||||
// says it listens on, and otherwise wherever the manifest's own mapping already put it. Written
|
||||
// here because everything below — the filter, the openings, the guard, what a consumer is told —
|
||||
// reads that map, and a given port that the lookup does not find moves the container's mapping
|
||||
// and nothing else.
|
||||
//
|
||||
// It stands in for the plan only where the plan does not allocate: a port the manifest already
|
||||
// placed, or one a node was given. For a short form with no given port the real plan asks the
|
||||
// inventory for a machine port and may come back with one from the pool, which needs a store and
|
||||
// is what cmd/mesh-controller's own tests exercise. So an assertion here about such a port asserts
|
||||
// this helper, not the mesh; keep the assertions to the ports under test.
|
||||
func portsAsThePlanWould(m Manifest, given map[int]int) map[int]int {
|
||||
out := map[int]int{}
|
||||
for _, l := range m.Listens {
|
||||
if at, is := given[l.Port]; is {
|
||||
out[l.Port] = at
|
||||
continue
|
||||
}
|
||||
at, _ := m.MachineSide(l.Port)
|
||||
out[l.Port] = at
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// novox/hq ADR 0100 and 0038: the machine side of a long-form mapping is a port the module
|
||||
// publishes, so a node may move it — and a module may name a mapping by either end.
|
||||
func TestAGivenPortNamesEitherEndOfWhatTheModulePublishes(t *testing.T) {
|
||||
forge := aForge()
|
||||
node := func(v any) []Layer {
|
||||
return []Layer{{From: "node anchor", Values: map[string]any{PortsSetting: v}}}
|
||||
}
|
||||
|
||||
// The machine side — the number this module says it listens on, and the one the predecessor
|
||||
// had somewhere else. Answered under 2222, the end the module itself names, which is what
|
||||
// every reader of this map asks for. One entry, not two: a second key for the same answer is
|
||||
// an entry another mapping's reader could find instead.
|
||||
given, err := GivenPorts(forge, node(map[string]any{"2222": float64(222)}))
|
||||
if err != nil {
|
||||
t.Fatalf("the machine side of a mapping cannot be given a port: %v", err)
|
||||
}
|
||||
if want := map[int]int{2222: 222}; !reflect.DeepEqual(given, want) {
|
||||
t.Fatalf("the forge was given %v; the mapping it names is filed under %v", given, want)
|
||||
}
|
||||
|
||||
// The container's own port names the same mapping and means the same thing — and is filed
|
||||
// under the same name, because the module's name for it has not changed.
|
||||
if inside, err := GivenPorts(forge, node(map[string]any{"22": float64(222)})); err != nil ||
|
||||
!reflect.DeepEqual(inside, map[int]int{2222: 222}) {
|
||||
t.Fatalf("the container's end of the mapping was given %v: %v", inside, err)
|
||||
}
|
||||
|
||||
// A short form is published on the number it names, and is unchanged by any of this.
|
||||
if short, err := GivenPorts(forge, node(map[string]any{"3000": float64(2999)})); err != nil ||
|
||||
short[3000] != 2999 {
|
||||
t.Fatalf("the short form was given %v: %v", short, err)
|
||||
}
|
||||
|
||||
// A port no container publishes is still refused, in the same words.
|
||||
if _, err := GivenPorts(forge, node(map[string]any{"9000": float64(9100)})); err == nil ||
|
||||
!strings.Contains(err.Error(), "does not publish") {
|
||||
t.Fatalf("a port the forge does not publish was given: %v", err)
|
||||
}
|
||||
|
||||
// And the two ends of one mapping given two different numbers is one setting contradicting
|
||||
// the other: the machine publishes it once.
|
||||
if _, err := GivenPorts(forge, node(map[string]any{
|
||||
"2222": float64(222), "22": float64(300)})); err == nil ||
|
||||
!strings.Contains(err.Error(), "one mapping") {
|
||||
t.Fatalf("the two ends of one mapping were given different ports: %v", err)
|
||||
}
|
||||
// Said at both ends with the same number, it is still said twice, and refused where every
|
||||
// other repeated machine port is — as the inventory refuses it when the setting is stored,
|
||||
// which is the layer that sees it first.
|
||||
if _, err := GivenPorts(forge, node(map[string]any{
|
||||
"2222": float64(222), "22": float64(222)})); err == nil ||
|
||||
!strings.Contains(err.Error(), "to both its 22 and its 2222") {
|
||||
t.Fatalf("one mapping given one machine port at both ends: %v", err)
|
||||
}
|
||||
// A number that names two different mappings names neither: which one to move is not said.
|
||||
twice := aForge()
|
||||
twice.Resources[0]["ports"] = []any{"22", "2222:22"}
|
||||
if _, err := GivenPorts(twice, node(map[string]any{"22": float64(222)})); err == nil ||
|
||||
!strings.Contains(err.Error(), "twice") {
|
||||
t.Fatalf("a number naming two of the module's mappings was accepted: %v", err)
|
||||
}
|
||||
|
||||
// And the same refusal when the number naming two mappings is not the one the setting used
|
||||
// but the one the answer would be filed under. Here `80` is the module's own name for a
|
||||
// mapping, and two mappings wear it; filing an answer there is one container's port standing
|
||||
// where the other's is read, and both containers then publish it.
|
||||
shared := Manifest{Module: "gallery",
|
||||
Listens: []Listening{{Port: 80, From: FromMesh}},
|
||||
Resources: []map[string]any{
|
||||
{"id": "a", "type": "container", "name": "a", "ports": []any{"4001:80"}},
|
||||
{"id": "b", "type": "container", "name": "b", "ports": []any{"4002:80"}}}}
|
||||
if _, err := GivenPorts(shared, node(map[string]any{"4001": float64(1234)})); err == nil ||
|
||||
!strings.Contains(err.Error(), "twice") {
|
||||
t.Fatalf("two containers were put on one machine port: %v", err)
|
||||
}
|
||||
|
||||
// A module whose mappings chain — one's machine side is another's container port — keeps them
|
||||
// apart, because each is filed under the port the module names it by and neither name is
|
||||
// shared. Given both, each moves on its own and neither overwrites the other.
|
||||
chained := Manifest{Module: "chain",
|
||||
Listens: []Listening{{Port: 80, From: FromMesh}, {Port: 9090, From: FromMesh}},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "chain",
|
||||
"ports": []any{"8080:80", "9090:8080"}}}}
|
||||
both, err := GivenPorts(chained, node(map[string]any{"80": float64(1234), "9090": float64(5678)}))
|
||||
if err != nil || !reflect.DeepEqual(both, map[int]int{80: 1234, 9090: 5678}) {
|
||||
t.Fatalf("chained mappings were given %v: %v", both, err)
|
||||
}
|
||||
if moved := givenOuter("8080:80", both); moved != "1234:80" {
|
||||
t.Fatalf("the first mapping moved to %q, and it was given 1234", moved)
|
||||
}
|
||||
if moved := givenOuter("9090:8080", both); moved != "5678:8080" {
|
||||
t.Fatalf("the second mapping moved to %q, and it was given 5678", moved)
|
||||
}
|
||||
}
|
||||
|
||||
// And the number reaches everything derived from it. The fault this is written against moved the
|
||||
// container's mapping alone: the filter opened the port the software had left, the adopted node's
|
||||
// opening named it too, the guard refused it, and a consumer was sent to it.
|
||||
func TestAGivenMachineSideReachesTheFilterTheOpeningAndTheConsumer(t *testing.T) {
|
||||
forge := aForge()
|
||||
given, err := GivenPorts(forge, []Layer{{From: "node anchor",
|
||||
Values: map[string]any{PortsSetting: map[string]any{"2222": float64(222)}}}})
|
||||
if err != nil {
|
||||
t.Fatalf("the machine side of a mapping cannot be given a port: %v", err)
|
||||
}
|
||||
r := Resolution{Node: "anchor", Modules: []Manifest{forge}}
|
||||
with := Rendering{
|
||||
Ports: map[string]map[int]int{"forge": portsAsThePlanWould(forge, given)},
|
||||
Given: map[string]map[int]int{"forge": given},
|
||||
Mesh: []string{"10.77.0.1"},
|
||||
Adopted: true,
|
||||
Taken: map[string]bool{"forge": true},
|
||||
}
|
||||
|
||||
// What the runtime is handed: the machine's own port on the outside, the container's within.
|
||||
composed, err := r.Compose(with)
|
||||
if err != nil {
|
||||
t.Fatalf("the forge does not compose: %v", err)
|
||||
}
|
||||
got := byID(composed.Resources)
|
||||
if ports := got["forge.server"]["ports"]; !reflect.DeepEqual(ports, []any{"3000:3000", "222:22"}) {
|
||||
t.Fatalf("the forge's container publishes %v", ports)
|
||||
}
|
||||
|
||||
// What the filter would open, were the node converged.
|
||||
rules, err := r.Rules(with)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var opened []int
|
||||
for _, rule := range rules {
|
||||
opened = append(opened, rule.Port)
|
||||
}
|
||||
// Only the moved port is asserted: the forge's other port is a short form the real plan would
|
||||
// allocate rather than read off the manifest, so its number here is portsAsThePlanWould's and
|
||||
// not the mesh's.
|
||||
if !slices.Contains(opened, 222) || slices.Contains(opened, 2222) {
|
||||
t.Fatalf("the filter opens %v, not where the machine puts the forge", opened)
|
||||
}
|
||||
|
||||
// And what it is declared instead, adopted: an opening on the machine's port, naming the
|
||||
// container's port on the forwarded path — a published port is forwarded, never received.
|
||||
opening := got[OpeningID("tcp", 222, PathForwarded)]
|
||||
if opening == nil || opening["to"] != 22 || opening["from"] != OpeningFromMesh {
|
||||
t.Fatalf("no opening for the port this node gave the forge: %v", keys(got))
|
||||
}
|
||||
for id := range got {
|
||||
if strings.HasPrefix(id, "adoption.opening-tcp-2222-") {
|
||||
t.Errorf("an opening for the port the forge was moved off: %s", id)
|
||||
}
|
||||
}
|
||||
|
||||
// The guard refuses it where the machine put it, and nothing where it used to be.
|
||||
guard, _ := got[GuardID()]["content"].(string)
|
||||
if !strings.Contains(guard, "222") || strings.Contains(guard, "2222") {
|
||||
t.Fatalf("the guard does not follow the given port:\n%s", guard)
|
||||
}
|
||||
|
||||
// And a consumer is sent to the same number, which is read from what the module serves.
|
||||
if told := ServedOn(forge, "git-over-ssh", with.Ports["forge"])["port"]; told != 222 {
|
||||
t.Fatalf("a consumer is told the forge answers on %v", told)
|
||||
}
|
||||
}
|
||||
|
||||
// And the mapping itself moves under either name, because Rendering.Given is a map anybody
|
||||
// composing a declaration hands in: keyed by the machine side, which is what a module declaring
|
||||
// 2222 calls its port, only the outside moves and the container's own port stays as written.
|
||||
func TestAMappingIsMovedUnderEitherOfItsNames(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
written string
|
||||
given map[int]int
|
||||
want string
|
||||
}{
|
||||
{"2222:22", map[int]int{2222: 222}, "222:22"},
|
||||
{"2222:22", map[int]int{22: 222}, "222:22"},
|
||||
{"127.0.0.1:15672:15672/tcp", map[int]int{15672: 15673}, "127.0.0.1:15673:15672/tcp"},
|
||||
{"2222:22", map[int]int{3000: 2999}, "2222:22"},
|
||||
{"2222:22", nil, "2222:22"},
|
||||
} {
|
||||
if got := givenOuter(c.written, c.given); got != c.want {
|
||||
t.Errorf("%s given %v is published as %s, want %s", c.written, c.given, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The same on a node that was given nothing, which is where the derivation was never at fault:
|
||||
// a long-form mapping is published where the manifest put it, and an adopted node opens that port
|
||||
// on the forwarded path like any other. What broke it was the number reaching only the mapping.
|
||||
func TestALongFormPortIsOpenedWhereTheManifestPublishesIt(t *testing.T) {
|
||||
forge := aForge()
|
||||
r := Resolution{Node: "anchor", Modules: []Manifest{forge}}
|
||||
composed, err := r.Compose(Rendering{
|
||||
Ports: map[string]map[int]int{"forge": portsAsThePlanWould(forge, nil)},
|
||||
Mesh: []string{"10.77.0.1"},
|
||||
Adopted: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := byID(composed.Resources)
|
||||
opening := got[OpeningID("tcp", 2222, PathForwarded)]
|
||||
if opening == nil || opening["to"] != 22 {
|
||||
t.Fatalf("no opening for the forge's published ssh port: %v", keys(got))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// **The word does not come back through a manifest** (novox/hq ADR 0131). A module that wants
|
||||
// messaging wants the mesh's bus, reached through the sdk and named by the `mesh-broker` seat. Naming
|
||||
// the old wire protocol asks for the one server being retired, so both directions are refused at the
|
||||
// parser — this is judged from the manifest alone, no store needed.
|
||||
|
||||
func TestAManifestProvidingAmqpIsRefused(t *testing.T) {
|
||||
raw := []byte(`{"module":"old-broker","version":"1","provides":[{"name":"amqp","scope":"mesh"}]}`)
|
||||
_, err := ParseManifest(raw)
|
||||
if err == nil || !strings.Contains(err.Error(), `provides "amqp", which is not a provision`) {
|
||||
t.Fatalf("a module providing amqp was not refused, or not for the reason: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAManifestRequiringAmqpIsRefused(t *testing.T) {
|
||||
raw := []byte(`{"module":"forwarder","version":"1","requires":["amqp"]}`)
|
||||
_, err := ParseManifest(raw)
|
||||
if err == nil || !strings.Contains(err.Error(), `requires "amqp", which is not a provision`) {
|
||||
t.Fatalf("a module requiring amqp was not refused, or not for the reason: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// And the catalogue as checked out beside this repository names it nowhere — the three modules that
|
||||
// did are removed under design 28 task 5.4, not converted.
|
||||
func TestNoCatalogueManifestNamesAmqp(t *testing.T) {
|
||||
modules, err := filepath.Glob("../../../mesh-catalog/modules/*/module.json")
|
||||
if err != nil || len(modules) == 0 {
|
||||
t.Skip("the catalogue is not checked out beside this repository")
|
||||
}
|
||||
for _, path := range modules {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(raw), `"amqp"`) {
|
||||
t.Errorf("%s names amqp, which is not a provision (novox/hq ADR 0131)",
|
||||
filepath.Base(filepath.Dir(path)))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The artifact store's seat is one per mesh, read from the catalogue beside this checkout.
|
||||
//
|
||||
// **A second store anywhere is refused by name, not discovered as a consumer failure.** The seat
|
||||
// was node-scoped, so a second `distribution` on another machine resolved cleanly there — and a
|
||||
// node-scoped requirement with one candidate installs that candidate on the node, so anything that
|
||||
// required the store's presence beside it would have raised a fresh, empty store on the wrong
|
||||
// machine. Only afterwards did the mesh notice: `artifact-store` offered by two nodes, and every
|
||||
// consumer elsewhere refusing to choose. The claim says it first, where the second store is
|
||||
// assigned.
|
||||
func TestASecondArtifactStoreAnywhereIsRefusedByName(t *testing.T) {
|
||||
store := catalogueManifest(t, "distribution")
|
||||
|
||||
// The first store resolves as it always has.
|
||||
if _, err := Resolve(shelf(store), []string{"distribution"}, workstation(), World{}); err != nil {
|
||||
t.Fatalf("the store alone does not resolve: %v", err)
|
||||
}
|
||||
|
||||
// A second one, on any other machine, is refused — and the refusal names the seat.
|
||||
elsewhere := World{Held: []Held{{Claim: "the-artifact-store", Scope: ScopeMesh,
|
||||
Node: "anchor", Module: "distribution"}}}
|
||||
other := workstation()
|
||||
other.Name = "laptop"
|
||||
_, err := Resolve(shelf(store), []string{"distribution"}, other, elsewhere)
|
||||
if err == nil {
|
||||
t.Fatal("a second store was accepted on another machine; it would offer artifact-store a " +
|
||||
"second time and every consumer elsewhere would refuse to choose")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "the-artifact-store") || !strings.Contains(err.Error(), "one per mesh") {
|
||||
t.Fatalf("refused without naming the seat: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// What the mesh built, named by what it is rather than by where it was pushed.
|
||||
//
|
||||
// **An image is recorded by its digest and its path; the registry's address is a route to it**
|
||||
// (novox/hq 04-ISSUES/102). A build used to be recorded as `<registry>:<port>/<module>/<artifact>@sha256:…`
|
||||
// — the reference the builder pushed to, kept whole — and every declaration carried that literal.
|
||||
// Move the registry's port, or the registry, and every fresh pull of a mesh image fails: a new
|
||||
// node, a recreate after eviction. The digest is the identity; the address is the node's setting
|
||||
// for the module that serves the artifact store, and it is read from there when a reference is
|
||||
// composed, never written into a record.
|
||||
//
|
||||
// So a kept reference has a scheme of the mesh's own, named after the provision that answers it:
|
||||
//
|
||||
// artifact-store://<module>/<artifact>@sha256:<hex> an image
|
||||
// artifact-store://<module>/<artifact>/blobs/sha256:<hex> an archive
|
||||
//
|
||||
// No runtime knows the scheme. That is the point of it being one: a reference that leaks to a
|
||||
// machine uncomposed is refused by the runtime as malformed, in front of whoever sent it, rather
|
||||
// than pulled from a public registry that happens to have a repository by that name — which is
|
||||
// what an address-less `<module>/<artifact>@sha256:…` would be.
|
||||
|
||||
// ArtifactStoreScheme marks a reference to something in the mesh's artifact store, kept without
|
||||
// the store's address.
|
||||
const ArtifactStoreScheme = ArtifactStoreProvision + "://"
|
||||
|
||||
// Recorded is a reference as the mesh records it: the artifact store's address, if the builder wrote
|
||||
// one, taken off.
|
||||
//
|
||||
// For a reference the builder announced — one it pushed to the store — which is the only kind
|
||||
// this is called on. An image the builder named `<host>/<path>@sha256:…` is kept as its path; an
|
||||
// archive it named `http://<host>/v2/<path>/blobs/<digest>` likewise. A reference with no address
|
||||
// in it — an image id from a genesis build that had nowhere to publish, a package version — is
|
||||
// what it was.
|
||||
func Recorded(reference string) string {
|
||||
if strings.HasPrefix(reference, ArtifactStoreScheme) {
|
||||
return reference
|
||||
}
|
||||
if rest, isURL := strings.CutPrefix(reference, "http://"); isURL {
|
||||
if _, path, ok := strings.Cut(rest, "/v2/"); ok && strings.Contains(path, "/blobs/") {
|
||||
return ArtifactStoreScheme + path
|
||||
}
|
||||
return reference
|
||||
}
|
||||
host, path, ok := strings.Cut(reference, "/")
|
||||
if !ok || !isRegistryHost(host) || !strings.Contains(path, "@sha256:") {
|
||||
return reference
|
||||
}
|
||||
return ArtifactStoreScheme + path
|
||||
}
|
||||
|
||||
// isRegistryHost is the runtime's own rule for reading the first component of a reference as a
|
||||
// registry rather than as a namespace: it has a dot or a port in it, or it is localhost.
|
||||
func isRegistryHost(component string) bool {
|
||||
return component == "localhost" || strings.ContainsAny(component, ".:")
|
||||
}
|
||||
|
||||
// InArtifactStore reports whether a reference is a kept one, and what it names there.
|
||||
func InArtifactStore(reference string) (path string, kept bool) {
|
||||
return strings.CutPrefix(reference, ArtifactStoreScheme)
|
||||
}
|
||||
|
||||
// Routed is a kept reference as a machine fetches it, through the artifact store at `address`
|
||||
// (host:port). A reference that is not a kept one is what it was.
|
||||
func Routed(reference, address string) string {
|
||||
path, kept := InArtifactStore(reference)
|
||||
if !kept {
|
||||
return reference
|
||||
}
|
||||
if strings.Contains(path, "/blobs/") {
|
||||
return "http://" + address + "/v2/" + path
|
||||
}
|
||||
return address + "/" + path
|
||||
}
|
||||
|
||||
// Rerouted is a reference the mesh recorded, whichever way it was recorded, as a machine fetches
|
||||
// it now: a kept one composed with the store's address, and one recorded before references were
|
||||
// kept without their address — the builder's own `<host>/<path>@sha256:…` — re-routed to where
|
||||
// the store is now. Only for references that are the mesh's own: everything a build record
|
||||
// holds is, by construction.
|
||||
func Rerouted(reference, address string) string {
|
||||
return Routed(Recorded(reference), address)
|
||||
}
|
||||
|
||||
// artifactsInto composes the artifact store's address into a resource's `image` and `source`.
|
||||
//
|
||||
// **Composed here, at the last moment before a machine, and stored nowhere.** A kept reference is
|
||||
// routed through the store as this network reaches it now. A reference recorded with an address
|
||||
// before references were kept without one is re-routed the same way — but only when the mesh
|
||||
// built it (`with.Built` names every `<module>/<artifact>` it has), because a module may run an
|
||||
// image from a public registry under its own name and that one is exactly where it says.
|
||||
//
|
||||
// A kept reference with no store to route it through is refused: sent as it is, the runtime would
|
||||
// refuse the scheme on the machine, one push away from the reason.
|
||||
//
|
||||
// **What this does not reach: the images genesis pinned.** The installer builds the control plane
|
||||
// and the builder before the mesh exists, pushes them itself and pins their manifests to
|
||||
// `<registry>:<port>/mesh-controller@…` and `<registry>:<port>/mesh-builder@…` — single-segment
|
||||
// repositories with no build record, so `with.Built` does not name them and they are left as
|
||||
// written until each is rebuilt through the mesh, which records it by digest and path. Until then
|
||||
// a registry that moves strands exactly those two on a recreate, and the control plane's is the
|
||||
// one that cannot be repaired through the mesh. Rebuild both through `build` before moving the
|
||||
// store (novox/hq 04-ISSUES/102, finding F4).
|
||||
func artifactsInto(resource map[string]any, module string, with Rendering) error {
|
||||
for _, key := range []string{"image", "source"} {
|
||||
written, ok := resource[key].(string)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if _, kept := InArtifactStore(written); kept {
|
||||
if with.ArtifactStore == "" {
|
||||
return fmt.Errorf(
|
||||
"%s's %v names %s, which is in the mesh's artifact store, and this mesh has no "+
|
||||
"artifact store on its network to fetch it from — nothing assigned offers "+
|
||||
"%s, or the machine offering it is not on the private network",
|
||||
module, resource["id"], written, ArtifactStoreProvision)
|
||||
}
|
||||
resource[key] = Routed(written, with.ArtifactStore)
|
||||
continue
|
||||
}
|
||||
if with.ArtifactStore == "" {
|
||||
continue
|
||||
}
|
||||
recorded := Recorded(written)
|
||||
if recorded == written {
|
||||
continue
|
||||
}
|
||||
path, _ := InArtifactStore(recorded)
|
||||
repository := path
|
||||
if at := strings.IndexAny(path, "@"); at >= 0 {
|
||||
repository = path[:at]
|
||||
} else if blobs := strings.Index(path, "/blobs/"); blobs >= 0 {
|
||||
repository = path[:blobs]
|
||||
}
|
||||
if with.Built[repository] {
|
||||
resource[key] = Routed(recorded, with.ArtifactStore)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A build is recorded by what it is; where it is pushed is composed where it is used (novox/hq
|
||||
// 04-ISSUES/102).
|
||||
|
||||
var digest = "sha256:" + strings.Repeat("d", 64)
|
||||
|
||||
func TestAReferenceIsRecordedWithoutTheStoresAddress(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"anchor.internal:5100/gitea/server@" + digest: ArtifactStoreScheme + "gitea/server@" + digest,
|
||||
"localhost:5000/gitea/server@" + digest: ArtifactStoreScheme + "gitea/server@" + digest,
|
||||
"http://anchor.internal:5100/v2/gitea/config/blobs/" + digest: ArtifactStoreScheme + "gitea/config/blobs/" + digest,
|
||||
ArtifactStoreScheme + "gitea/server@" + digest: ArtifactStoreScheme + "gitea/server@" + digest,
|
||||
digest: digest,
|
||||
"@novox/sdk@1.2.3": "@novox/sdk@1.2.3",
|
||||
"gitea/gitea@" + digest: "gitea/gitea@" + digest,
|
||||
"https://registry.example/v2/gitea/config/blobs/" + digest: "https://registry.example/v2/gitea/config/blobs/" + digest,
|
||||
}
|
||||
for announced, want := range cases {
|
||||
if got := Recorded(announced); got != want {
|
||||
t.Errorf("Recorded(%q) = %q, want %q", announced, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestARecordedReferenceIsRoutedThroughTheStoreAsItIsNow(t *testing.T) {
|
||||
if got := Routed(ArtifactStoreScheme+"gitea/server@"+digest, "anchor.internal:5101"); got != "anchor.internal:5101/gitea/server@"+digest {
|
||||
t.Errorf("an image is fetched as %q", got)
|
||||
}
|
||||
if got := Routed(ArtifactStoreScheme+"gitea/config/blobs/"+digest, "anchor.internal:5101"); got != "http://anchor.internal:5101/v2/gitea/config/blobs/"+digest {
|
||||
t.Errorf("an archive is fetched as %q", got)
|
||||
}
|
||||
if got := Routed("gitea/gitea@"+digest, "anchor.internal:5101"); got != "gitea/gitea@"+digest {
|
||||
t.Errorf("a reference that is not the store's was routed: %q", got)
|
||||
}
|
||||
// One recorded before references were kept without their address follows the store too.
|
||||
if got := Rerouted("anchor.internal:5100/gitea/server@"+digest, "anchor.internal:5101"); got != "anchor.internal:5101/gitea/server@"+digest {
|
||||
t.Errorf("a reference recorded with the old address stays there: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **The address is composed into a declaration, and the record never carries it.**
|
||||
func TestAnImageTheMeshBuiltIsRoutedThroughTheStoreWhenDeclared(t *testing.T) {
|
||||
m := Manifest{Module: "gitea", Version: "1", Resources: []map[string]any{
|
||||
{"id": "server", "type": "container", "name": "mesh-gitea", "artifact": "server"},
|
||||
{"id": "config", "type": "archive", "path": "/etc/gitea", "artifact": "config"},
|
||||
{"id": "cache", "type": "container", "name": "mesh-gitea-cache", "image": "valkey/valkey@" + digest},
|
||||
}, Build: &Build{Artifacts: []Artifact{
|
||||
{Name: "server", Kind: ArtifactImage, From: "Dockerfile"},
|
||||
{Name: "config", Kind: ArtifactArchive, From: "config"},
|
||||
}}}
|
||||
resolved, err := m.Resolve([]Built{
|
||||
{Name: "server", Kind: ArtifactImage, Reference: ArtifactStoreScheme + "gitea/server@" + digest},
|
||||
{Name: "config", Kind: ArtifactArchive, Reference: ArtifactStoreScheme + "gitea/config/blobs/" + digest, Digest: digest},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := Resolution{Node: "anchor", Modules: []Manifest{resolved}}
|
||||
|
||||
out, err := r.Declaration(Rendering{ArtifactStore: "anchor.internal:5101"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := fileNamed(out, "gitea.server")["image"]; got != "anchor.internal:5101/gitea/server@"+digest {
|
||||
t.Errorf("the image the mesh built is fetched as %v", got)
|
||||
}
|
||||
if got := fileNamed(out, "gitea.config")["source"]; got != "http://anchor.internal:5101/v2/gitea/config/blobs/"+digest {
|
||||
t.Errorf("the archive the mesh built is fetched from %v", got)
|
||||
}
|
||||
if got := fileNamed(out, "gitea.cache")["image"]; got != "valkey/valkey@"+digest {
|
||||
t.Errorf("an image from a public registry was routed through the store: %v", got)
|
||||
}
|
||||
// The manifest the mesh holds still says what it is, not where it was fetched from.
|
||||
if got := resolved.Resources[0]["image"]; got != ArtifactStoreScheme+"gitea/server@"+digest {
|
||||
t.Errorf("composing wrote the address into the catalogue's copy: %v", got)
|
||||
}
|
||||
|
||||
// And the store moves: the same record, another address, without a rebuild.
|
||||
out, err = r.Declaration(Rendering{ArtifactStore: "laptop.internal:5000"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := fileNamed(out, "gitea.server")["image"]; got != "laptop.internal:5000/gitea/server@"+digest {
|
||||
t.Errorf("after the store moved, the image is still fetched as %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnImageInTheStoreWithNoStoreToFetchItFromIsRefused(t *testing.T) {
|
||||
m := Manifest{Module: "gitea", Version: "1", Resources: []map[string]any{
|
||||
{"id": "server", "type": "container", "name": "mesh-gitea",
|
||||
"image": ArtifactStoreScheme + "gitea/server@" + digest},
|
||||
}}
|
||||
_, err := Resolution{Node: "anchor", Modules: []Manifest{m}}.Declaration(Rendering{})
|
||||
if err == nil || !strings.Contains(err.Error(), "no artifact store") {
|
||||
t.Fatalf("a reference nothing can fetch was sent to a machine: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// **A reference recorded with an address before this follows the store too** — when the mesh
|
||||
// built it. A module running an image straight from a public registry under its own name is left
|
||||
// exactly where it says: `quay.io/keycloak/keycloak` is not the mesh's, whatever it is called.
|
||||
func TestAReferenceRecordedWithAnAddressFollowsTheStoreWhenTheMeshBuiltIt(t *testing.T) {
|
||||
m := Manifest{Module: "keycloak", Version: "1", Resources: []map[string]any{
|
||||
{"id": "server", "type": "container", "name": "mesh-keycloak",
|
||||
"image": "anchor.internal:5100/keycloak/server@" + digest},
|
||||
{"id": "upstream", "type": "container", "name": "mesh-keycloak-upstream",
|
||||
"image": "quay.io/keycloak/keycloak@" + digest},
|
||||
}}
|
||||
r := Resolution{Node: "anchor", Modules: []Manifest{m}}
|
||||
out, err := r.Declaration(Rendering{
|
||||
ArtifactStore: "anchor.internal:5101",
|
||||
Built: map[string]bool{"keycloak/server": true},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := fileNamed(out, "keycloak.server")["image"]; got != "anchor.internal:5101/keycloak/server@"+digest {
|
||||
t.Errorf("an image the mesh built, recorded with the old address, is fetched as %v", got)
|
||||
}
|
||||
if got := fileNamed(out, "keycloak.upstream")["image"]; got != "quay.io/keycloak/keycloak@"+digest {
|
||||
t.Errorf("a public image was re-routed through the store: %v", got)
|
||||
}
|
||||
// Nothing known to be built: nothing re-routed, nothing refused.
|
||||
out, err = r.Declaration(Rendering{ArtifactStore: "anchor.internal:5101"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := fileNamed(out, "keycloak.server")["image"]; got != "anchor.internal:5100/keycloak/server@"+digest {
|
||||
t.Errorf("with no build record, a reference was rewritten: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The mesh's bus is one per mesh, read from the catalogue beside this checkout.
|
||||
//
|
||||
// **This is step 2's claim, and it is checked here rather than in a bed** (novox/hq ADR 0116):
|
||||
// adoption puts the NATS server into the `mesh-broker` seat on a mesh that is already running,
|
||||
// and the property that matters is that a second one anywhere is refused *when it is assigned*,
|
||||
// not discovered later as two servers holding different halves of the mesh's traffic. A second
|
||||
// bus is not a degraded mesh; it is two meshes that both believe they are the one.
|
||||
func TestASecondMeshBusAnywhereIsRefusedByName(t *testing.T) {
|
||||
nats := catalogueManifest(t, "nats")
|
||||
|
||||
if _, err := Resolve(shelf(nats), []string{"nats"}, workstation(), World{}); err != nil {
|
||||
t.Fatalf("the bus alone does not resolve: %v", err)
|
||||
}
|
||||
|
||||
elsewhere := World{Held: []Held{{Claim: "mesh-broker", Scope: ScopeMesh,
|
||||
Node: "anchor", Module: "nats"}}}
|
||||
other := workstation()
|
||||
other.Name = "laptop"
|
||||
_, err := Resolve(shelf(nats), []string{"nats"}, other, elsewhere)
|
||||
if err == nil {
|
||||
t.Fatal("a second bus was accepted on another machine")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "mesh-broker") || !strings.Contains(err.Error(), "one per mesh") {
|
||||
t.Fatalf("refused without naming the seat: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The seat is the server's role, not the product's name (novox/hq ADR 0079). A different
|
||||
// implementation of the bus claims the same seat, and the mesh refuses it for the same reason —
|
||||
// which is the property that lets the bus be replaced at all.
|
||||
func TestTheSeatRefusesADifferentBusToo(t *testing.T) {
|
||||
nats := catalogueManifest(t, "nats")
|
||||
held := World{Held: []Held{{Claim: "mesh-broker", Scope: ScopeMesh,
|
||||
Node: "anchor", Module: "some-other-broker"}}}
|
||||
other := workstation()
|
||||
other.Name = "laptop"
|
||||
if _, err := Resolve(shelf(nats), []string{"nats"}, other, held); err == nil {
|
||||
t.Fatal("the seat admitted a second holder because the module's name differed")
|
||||
}
|
||||
}
|
||||
|
||||
// The old broker is gone from the catalogue (novox/hq ADR 0131, design 28 task 5.4), so it is no
|
||||
// longer a fixture here. That two eligible holders stand beside each other with one on record is
|
||||
// pinned in holdings_test.go against manifests this package owns.
|
||||
|
||||
// **A seat and the interface it delivers are different names, and renaming one must not rename
|
||||
// the other** (novox/hq ADR 0118). This nearly went wrong: the seats were renamed to the `mesh-*`
|
||||
// prefix, and a blanket search-and-replace also renamed `npm-package-registry` and `git` where
|
||||
// they are *provisions* — which a consumer requires and a provider offers. The tests failed with
|
||||
// "the package registry is served on <nil>", which does not say "you renamed an interface".
|
||||
func TestRenamingASeatDidNotRenameTheInterfaceItDelivers(t *testing.T) {
|
||||
for _, pair := range []struct{ seat, delivers string }{
|
||||
{"git", "git"},
|
||||
{"npm-package-registry", "npm-package-registry"},
|
||||
{"the-artifact-store", "artifact-store"},
|
||||
{"mesh-store", "postgres-database"},
|
||||
{"mesh-broker", "mesh-bus"},
|
||||
} {
|
||||
s, known := SeatNamed(pair.seat)
|
||||
if !known {
|
||||
t.Fatalf("%q is not a seat", pair.seat)
|
||||
}
|
||||
if s.Delivers != pair.delivers {
|
||||
t.Errorf("the %s seat delivers %q, expected %q — renaming the seat moved the "+
|
||||
"interface with it, and every consumer requiring it would stop resolving",
|
||||
pair.seat, s.Delivers, pair.delivers)
|
||||
}
|
||||
// **Three of these deliberately share a name with what they deliver**, and that is not an
|
||||
// incomplete rename. Renaming a seat that delivers a provision cascades to every consumer
|
||||
// requiring it, with a mesh-wide window where a holder stops resolving mid-flight — so the
|
||||
// trunk deferred exactly those three (novox/hq ADR 0121) while renaming the node-scoped ones.
|
||||
// What this test is for is the other direction: that renaming a seat never moves the
|
||||
// interface, which once produced "the package registry is served on <nil>".
|
||||
}
|
||||
}
|
||||
|
||||
// A manifest written against an old seat name is told what it became rather than refused as
|
||||
// unknown. **That map is the controller's store now, not this package** (novox/hq ADR 0122): a
|
||||
// rename is a row, so the courtesy survives a rename nobody recompiled for. Checked where the
|
||||
// table is read, not here, where there is no longer a hardcoded list to check against.
|
||||
@@ -1,6 +1,7 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
@@ -157,3 +158,47 @@ func TestTwoWaysToBeOnAPrivateNetworkRefuseAndNameBoth(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// reloading answers the runtime's trust as the networking module does (novox/hq ADR 0102): a file
|
||||
// written into, and the runtime reloaded on it.
|
||||
type reloading struct{}
|
||||
|
||||
func (reloading) Resources(node string) ([]map[string]any, bool, error) {
|
||||
return []map[string]any{
|
||||
{"id": "registry-trust", "type": "file", "path": "/etc/docker/daemon.json",
|
||||
"merge": MergeJSON, "into": "json", "content": `{"insecure-registries":["r:5000"]}`},
|
||||
{"id": "registry-trust-reload", "type": "service", "unit": "docker.service",
|
||||
"state": "running", "reload-on": []string{"registry-trust"}},
|
||||
}, true, nil
|
||||
}
|
||||
|
||||
func TestWhatAServiceIsReloadedOnIsNamedAsTheHostWillSeeIt(t *testing.T) {
|
||||
// Ids are prefixed with their module on the way out. An unprefixed reload-on would name a
|
||||
// resource the host never sees, and the runtime would never be reloaded for its trust.
|
||||
got, err := Resolve(computedShelf(), []string{"mesh-network"}, workstation(), World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := got.Declaration(Rendering{Generators: map[string]Generator{"mesh-network": reloading{}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var file, service map[string]any
|
||||
for _, r := range out {
|
||||
switch r["type"] {
|
||||
case "file":
|
||||
file = r
|
||||
case "service":
|
||||
service = r
|
||||
}
|
||||
}
|
||||
if file == nil || service == nil {
|
||||
t.Fatalf("got %v", out)
|
||||
}
|
||||
if file["into"] != "json" {
|
||||
t.Errorf("into did not reach the host: %v", file)
|
||||
}
|
||||
if want := "[" + file["id"].(string) + "]"; fmt.Sprint(service["reload-on"]) != want {
|
||||
t.Errorf("reload-on names %v, the file is %v", service["reload-on"], file["id"])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -92,6 +92,33 @@ type Rendering struct {
|
||||
// mesh" resolves to. Passed in for the same reason grants are: who else is on the network is
|
||||
// a fact about the mesh, and resolution answers questions about one machine.
|
||||
Mesh []string
|
||||
// Suffix is what a machine's internal name ends in, as the control plane composed Names —
|
||||
// `internal` unless the operator chose another — so a fact writing those names does not
|
||||
// compose it a second time.
|
||||
Suffix string
|
||||
|
||||
// BusUsers is the mesh's composed user list, for the module holding `mesh-broker`. Empty on
|
||||
// every other node, and on this one until the controller has composed it.
|
||||
//
|
||||
// **Only the users, never the server's own settings**: those are the module's, in its image and
|
||||
// its mounts (Manifest.BusUsers).
|
||||
BusUsers string
|
||||
// BusMembership is this machine's membership for the bus the mesh is moving to, sealed to it
|
||||
// (design 28, task 5.2). Empty for a machine not being moved. Written as a file the host reads
|
||||
// after the declaration has applied, so the bus it names is standing before the machine leaves
|
||||
// the one it is on.
|
||||
BusMembership string
|
||||
|
||||
// MeshRange is the private network's CIDR (the range node addresses are allocated from), for a
|
||||
// module that must name the whole mesh rather than one machine — an intrusion filter that must
|
||||
// never ban a tunnel peer, say. A per-mesh value the module cannot know, so it is carried here
|
||||
// and offered as ${machine:mesh-range}, the same way one machine's address is.
|
||||
MeshRange string
|
||||
|
||||
// Accounts is each machine's operator account, by the same internal name Names uses (novox/hq
|
||||
// to-be 29). What an ssh Host block's `User` line is composed from; empty for a machine no
|
||||
// operator account is known on.
|
||||
Accounts map[string]string
|
||||
|
||||
// Kept is every operator-sealed secret in the mesh, for a module that `keeps` them. Nil when
|
||||
// nothing on this node keeps them, or the mesh has no operator key.
|
||||
@@ -110,6 +137,14 @@ type Rendering struct {
|
||||
// because which machines exist is a fact about the mesh.
|
||||
Names map[string]string
|
||||
|
||||
// Machines is only the machines, by the same internal name — the subset of Names that is a
|
||||
// node of this mesh rather than a name it was told to serve. Both matter and they are not the
|
||||
// same set: a container's hosts wants every name, so a routed name resolves to the proxy that
|
||||
// serves it, while a resolver told the mesh's suffix is authoritative for it answers from what
|
||||
// it is given and forwards nothing — so a routed name written there is a name nobody asks for,
|
||||
// standing beside the machines and looking as real as they do.
|
||||
Machines map[string]string
|
||||
|
||||
Settings SettingsBy
|
||||
Generators map[string]Generator
|
||||
// Grants are the credentials this node must create, for the provisions it offers. Passed in
|
||||
@@ -124,12 +159,54 @@ type Rendering struct {
|
||||
// set, for what a consumer is told, and for what the runtime publishes — and nothing checked
|
||||
// that the three agreed. They are all derived from this.
|
||||
Ports map[string]map[int]int
|
||||
|
||||
// Adopted says the node is adopted (novox/hq ADR 0100): the firewall found on it stays in
|
||||
// force, so no module that loads a filter is declared there, and what the mesh needs
|
||||
// reachable is declared as openings, with its own ports guarded by a table that only refuses.
|
||||
Adopted bool
|
||||
|
||||
// Given is the machine ports this node was given for its modules' ports, by module and by the
|
||||
// port the software uses (novox/hq ADR 0100) — the foundation's ports, as genesis chose them.
|
||||
// They win over anything the mesh would assign and over a manifest's own long-form mapping.
|
||||
Given map[string]map[int]int
|
||||
|
||||
// Taken is the modules taken on this adopted node (novox/hq ADR 0100). The guard is derived
|
||||
// from these only (ADR 0103): a port of a module assigned but not taken may still be the
|
||||
// predecessor's.
|
||||
Taken map[string]bool
|
||||
|
||||
// Seats is where this machine put each mesh-scoped seat's holder, by seat and by the port the
|
||||
// holder's software uses (novox/hq 04-ISSUES/102) — read from the node's settings and
|
||||
// assignments for whichever module claims the seat, whether or not it is in this node's set.
|
||||
// What ${seat:…} answers with; see seat_into.go for why the answer may be absent.
|
||||
Seats map[string]map[int]int
|
||||
|
||||
// ArtifactStore is the mesh's artifact store as this network reaches it (host:port) — the
|
||||
// node holding it and the port that node put it on — or empty when the mesh has none on its
|
||||
// network yet. Composed into every image and archive the mesh built, at this moment and never
|
||||
// stored (novox/hq 04-ISSUES/102).
|
||||
ArtifactStore string
|
||||
|
||||
// Built is every `<module>/<artifact>` the mesh has built. What tells a reference recorded
|
||||
// with an address — before references were kept without one — from an image a module runs
|
||||
// straight from a public registry.
|
||||
Built map[string]bool
|
||||
|
||||
// DataRoot is where this node keeps the directories the mesh places for its modules
|
||||
// (novox/hq ADR 0112, to-be 27): a directory resource that states no path resolves to
|
||||
// <DataRoot>/<module>/<id>, and ${dir:<id>} names that place from the module's own files,
|
||||
// mounts and environment. A node setting fixed at installation; empty means the default,
|
||||
// /var/lib — see dir_into.go.
|
||||
DataRoot string
|
||||
}
|
||||
|
||||
// machinePort is where a module's port lives on this machine, or the port itself when the mesh has
|
||||
// not been asked. Unassigned is not an error here: a module with no `listens` never needed one,
|
||||
// and a caller composing a declaration without a store still gets something coherent.
|
||||
func (r Rendering) machinePort(module string, wanted int) int {
|
||||
if at, given := r.Given[module][wanted]; given {
|
||||
return at
|
||||
}
|
||||
if at, known := r.Ports[module][wanted]; known {
|
||||
return at
|
||||
}
|
||||
@@ -142,6 +219,63 @@ func (r Rendering) machinePort(module string, wanted int) int {
|
||||
// both call something "config", and without this the second would silently replace the first —
|
||||
// the node applying one of them and reporting success.
|
||||
func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) {
|
||||
composed, err := r.Compose(with)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return composed.Resources, nil
|
||||
}
|
||||
|
||||
// Composed is a declaration's resources and which module each came from.
|
||||
//
|
||||
// Owner is kept beside the resources because a resource id cannot be split back into its module:
|
||||
// a module's name may itself contain a dot. What the mesh adds of its own — an opening, the guard —
|
||||
// has no owner.
|
||||
type Composed struct {
|
||||
Resources []map[string]any
|
||||
Owner map[string]string
|
||||
}
|
||||
|
||||
// Compose is Declaration with the owner of every resource said.
|
||||
func (r Resolution) Compose(with Rendering) (Composed, error) {
|
||||
owner := map[string]string{}
|
||||
resources, err := r.compose(with, owner)
|
||||
if err != nil {
|
||||
return Composed{}, err
|
||||
}
|
||||
if with.BusMembership != "" {
|
||||
// The machine's own, not any module's: how it reaches the mesh from now on. Sealed like a
|
||||
// secret and placed where the host looks for exactly this (design 28, task 5.2).
|
||||
resources = append(resources, map[string]any{
|
||||
"id": BusMembershipID(), "type": "file", "path": BusMembershipPath,
|
||||
"sealed": with.BusMembership, "mode": "0600",
|
||||
})
|
||||
}
|
||||
return Composed{Resources: resources, Owner: owner}, nil
|
||||
}
|
||||
|
||||
// BusMembershipID names the resource carrying a machine's membership for the new bus, and
|
||||
// BusMembershipPath is where the host reads it — the same constant on both sides.
|
||||
func BusMembershipID() string { return "bus-membership" }
|
||||
|
||||
const BusMembershipPath = "/var/lib/mesh/membership-next.json"
|
||||
|
||||
func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[string]any, error) {
|
||||
// Every manifest is placed first (novox/hq ADR 0112): the maps naming where its bindings,
|
||||
// credentials and contributions land are resolved against this node's directories, so every
|
||||
// reader below — the binding files, the sealed secrets, the grant paths a contribution
|
||||
// names — sees a concrete place and none learns the vocabulary.
|
||||
// Into a fresh slice, never the caller's: one resolution may compose for many nodes, and a
|
||||
// slice element written in place would carry the first node's places into the second's.
|
||||
placed := make([]Manifest, len(r.Modules))
|
||||
for i, m := range r.Modules {
|
||||
var err error
|
||||
if placed[i], err = placedManifest(m, with); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
r.Modules = placed
|
||||
|
||||
// Where each provision's credentials land, so a contribution can name the file rather than
|
||||
// carry a value the mesh does not have.
|
||||
directories := map[string]string{}
|
||||
@@ -207,17 +341,7 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) {
|
||||
// Once, from every module's listens -- not per module. A module receiving only its own ports
|
||||
// would write a rule set that closed every other module on the machine. Each module's per-node
|
||||
// exposure settings override its listens' source first (novox/hq ADR 0046).
|
||||
exposure := map[string]map[int]string{}
|
||||
for _, m := range r.Modules {
|
||||
e, err := Exposure(m, with.Settings[m.Module])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if e != nil {
|
||||
exposure[m.Module] = e
|
||||
}
|
||||
}
|
||||
rules, err := r.Filtering(with.Generators, with.Ports, exposure)
|
||||
rules, err := r.Rules(with)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -225,6 +349,13 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) {
|
||||
|
||||
var out []map[string]any
|
||||
for _, m := range r.Modules {
|
||||
if with.Adopted && m.Filtering != nil {
|
||||
// Nothing of a module that loads a filter, on an adopted node: its table would drop
|
||||
// by default and hold accepts, and the found firewall stays in force. Every resource,
|
||||
// not only the rule set — its service must not run, and a node returned to adopted
|
||||
// stops it by the ordinary removal of what is no longer declared.
|
||||
continue
|
||||
}
|
||||
resources := m.Resources
|
||||
|
||||
// What the mesh computes for this module goes FIRST, before the module's own resources.
|
||||
@@ -245,6 +376,12 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) {
|
||||
"content": filtering, "mode": "0600",
|
||||
})
|
||||
}
|
||||
// The node's fail2ban jails, composed from every module it runs (novox/hq to-be 31), written
|
||||
// where the intrusion-prevention holder owns them. Like the rule set above: gathered from all
|
||||
// modules, written by the one that holds the role.
|
||||
if j := m.Jailing; j != nil {
|
||||
first = append(first, jailsInto(r.Modules, j)...)
|
||||
}
|
||||
if c := m.Certificate; c != nil {
|
||||
if with.Certificate == "" {
|
||||
// Asked for and not issued. Refused rather than skipped: a module that serves TLS
|
||||
@@ -265,6 +402,35 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) {
|
||||
})
|
||||
}
|
||||
}
|
||||
if m.BusUsers != "" {
|
||||
// **The claim authorises it, not the field.** This file holds every user's password
|
||||
// hash, so a module that could ask for it could read every credential on the bus.
|
||||
// Checked from this manifest alone, which is the cheapest check there is: whether some
|
||||
// other module also claims the seat is resolution's business elsewhere, and one holder
|
||||
// mesh-wide is already guaranteed.
|
||||
if !m.ClaimsSeat("mesh-broker") {
|
||||
return nil, fmt.Errorf(
|
||||
"%s asks for the mesh's user list and does not claim mesh-broker. That file "+
|
||||
"holds every user's password hash, so the seat is what authorises it",
|
||||
m.Module)
|
||||
}
|
||||
if with.BusUsers == "" {
|
||||
// Asked for and not composed. Refused rather than skipped, for the reason a
|
||||
// certificate is: a bus with no user list refuses every connection in the mesh, and
|
||||
// an empty file would look like a configuration problem on the machine.
|
||||
return nil, fmt.Errorf(
|
||||
"%s holds mesh-broker and the mesh composed no user list, so the bus would "+
|
||||
"refuse every connection", m.Module)
|
||||
}
|
||||
first = append(first, map[string]any{
|
||||
"id": BusUsersID(), "type": "file", "path": m.BusUsers,
|
||||
"content": with.BusUsers,
|
||||
// Readable by the server and nothing else. Hashes rather than passwords, so this is
|
||||
// not a set of working credentials — but a list of every user in the mesh is worth
|
||||
// keeping to the one process that needs it.
|
||||
"mode": "0600",
|
||||
})
|
||||
}
|
||||
for _, name := range sortedKeys(m.OwnSecrets) {
|
||||
sealed := with.Needed[m.Module][name]
|
||||
if sealed == "" {
|
||||
@@ -447,8 +613,11 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) {
|
||||
}
|
||||
// And what its bindings say, for the half of a connection that is not secret.
|
||||
known := knownFor(m, r.Needs, r.Node)
|
||||
// And where this node places the directories the module declared without a path
|
||||
// (novox/hq ADR 0112) — resolved once per module, named by ${dir:…} from any resource.
|
||||
dirs := dirsFor(m, with)
|
||||
// And the machine underneath, which no binding of its own can tell it.
|
||||
thisMachine := machineFacts(r)
|
||||
thisMachine := machineFacts(r, with.Names, with.MeshRange)
|
||||
|
||||
// Which of this module's files carry a secret, for the rule that a container may not read
|
||||
// one of them as its environment without saying so (ADR 0086, issue 041).
|
||||
@@ -469,6 +638,13 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) {
|
||||
// Said in the catalogue, not on the machine: the host parses strictly and knows no
|
||||
// such field, and the reason is for a reader of the manifest.
|
||||
delete(copied, SecretsInEnvironment)
|
||||
// **Placed before anything reads a path.** A pathless directory receives the path
|
||||
// this node resolves for it, and every ${dir:…} — in paths, mounts, content and
|
||||
// environment — becomes that path, so what follows sees only concrete places
|
||||
// (novox/hq ADR 0112). The host receives paths exactly as it always has.
|
||||
if err := dirInto(copied, dirs, m.Module); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// **After settings, and that is the whole reason it is here.** A module's file
|
||||
// content is where a setting lands, so a placeholder may only exist once the setting
|
||||
// has been put in — filling secrets first would look at content that is not yet what
|
||||
@@ -492,6 +668,11 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) {
|
||||
if err := portInto(copied, m.Module, m.Listens, with); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// And where this machine put the foundation's servers, for the one module that
|
||||
// reaches them by seat rather than by binding (novox/hq 04-ISSUES/102).
|
||||
if err := seatInto(copied, m.Module, with); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := machineInto(copied, thisMachine, m.Module); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -503,6 +684,11 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) {
|
||||
if err := built(copied, m.Module); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// What the mesh built is kept by digest and path; the store's address is this
|
||||
// network's now, composed here and never recorded (novox/hq 04-ISSUES/102).
|
||||
if err := artifactsInto(copied, m.Module, with); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
publishedOn(copied, m.Module, with)
|
||||
copied["id"] = m.Module + "." + fmt.Sprint(resource["id"])
|
||||
// A service saying what it reflects names resources within its own module, so those
|
||||
@@ -517,6 +703,12 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) {
|
||||
if renamed := reflectsRenamed(m.Module, resource["restart-on"]); renamed != nil {
|
||||
copied["restart-on"] = renamed
|
||||
}
|
||||
// And what it is reloaded on, by the same rule (novox/hq ADR 0102): an id left
|
||||
// unprefixed matches nothing, and the service is never reloaded.
|
||||
if renamed := reflectsRenamed(m.Module, resource["reload-on"]); renamed != nil {
|
||||
copied["reload-on"] = renamed
|
||||
}
|
||||
owner[fmt.Sprint(copied["id"])] = m.Module
|
||||
out = append(out, copied)
|
||||
}
|
||||
|
||||
@@ -524,18 +716,144 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) {
|
||||
// plane's; making a name resolve is the module's software. Emitted as ordinary files under
|
||||
// this module's name, so they are applied, reported and removed exactly as anything else
|
||||
// it declares.
|
||||
given, err := FactsInto(m, r, with.Names)
|
||||
given, err := FactsInto(m, r, with.Names, with.Machines, with.Accounts, with.Suffix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, fact := range given {
|
||||
fact["id"] = m.Module + "." + fmt.Sprint(fact["id"])
|
||||
owner[fmt.Sprint(fact["id"])] = m.Module
|
||||
out = append(out, fact)
|
||||
}
|
||||
}
|
||||
if with.Adopted {
|
||||
// First, before anything a module declares: what the mesh needs reachable, then its guard.
|
||||
// The order a machine applies is the order written here.
|
||||
ours := Openings(rules, with.Foundation, Published(out))
|
||||
ours = append(ours, GuardResources(r.guarded(out, owner, rules, with))...)
|
||||
out = append(ours, out...)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// guarded is what the mesh's guard refuses on an adopted node (novox/hq ADR 0103): derived, and
|
||||
// for taken modules only.
|
||||
//
|
||||
// For each taken module, every machine port its containers publish that the filter would admit
|
||||
// from the private network only — a published port is forwarded, not received, so a found
|
||||
// firewall filtering only what it receives never sees it — together with the ports the module's
|
||||
// manifest guards explicitly (the store's port, the broker's management port), wherever the
|
||||
// machine put them. A port of a module assigned but not taken is not guarded: it may still be the
|
||||
// predecessor's, serving the predecessor's other machines. The foundation's ports are admitted
|
||||
// from everywhere and are never guarded.
|
||||
func (r Resolution) guarded(out []map[string]any, owner map[string]string, rules []Rule,
|
||||
with Rendering) []int {
|
||||
meshOnly := map[int]bool{}
|
||||
fromEverywhere := map[int]bool{}
|
||||
for _, rule := range rules {
|
||||
if rule.Protocol != "tcp" {
|
||||
continue
|
||||
}
|
||||
switch rule.From {
|
||||
case FromMesh:
|
||||
meshOnly[rule.Port] = true
|
||||
case FromEverywhere:
|
||||
fromEverywhere[rule.Port] = true
|
||||
}
|
||||
}
|
||||
for _, port := range with.Foundation {
|
||||
delete(meshOnly, port)
|
||||
fromEverywhere[port] = true
|
||||
}
|
||||
seen := map[int]bool{}
|
||||
var ports []int
|
||||
guard := func(at int) {
|
||||
if !seen[at] {
|
||||
seen[at] = true
|
||||
ports = append(ports, at)
|
||||
}
|
||||
}
|
||||
for _, m := range r.Modules {
|
||||
if !with.Taken[m.Module] {
|
||||
continue
|
||||
}
|
||||
var mine []map[string]any
|
||||
for _, resource := range out {
|
||||
if owner[fmt.Sprint(resource["id"])] == m.Module {
|
||||
mine = append(mine, resource)
|
||||
}
|
||||
}
|
||||
for outer := range Published(mine)["tcp"] {
|
||||
if meshOnly[outer] {
|
||||
guard(outer)
|
||||
}
|
||||
}
|
||||
for _, want := range m.Guards {
|
||||
at := with.machinePort(m.Module, want)
|
||||
for _, resource := range mine {
|
||||
if fmt.Sprint(resource["type"]) != "container" {
|
||||
continue
|
||||
}
|
||||
listed, _ := resource["ports"].([]any)
|
||||
for _, entry := range listed {
|
||||
outer, inner, _, ok := mapping(fmt.Sprint(entry))
|
||||
if ok && inner == want {
|
||||
at = outer
|
||||
}
|
||||
}
|
||||
}
|
||||
// Not what this node is told to open to everyone: a per-node exposure setting that
|
||||
// widens a guarded port is the operator saying so, and declaring an opening for it
|
||||
// and a guard dropping it would be one statement refusing the other.
|
||||
if !fromEverywhere[at] {
|
||||
guard(at)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Ints(ports)
|
||||
return ports
|
||||
}
|
||||
|
||||
// mapping reads a container's port mapping — `[address:]outer:inner[/protocol]`, the address
|
||||
// possibly an IPv6 one in brackets — indexing from the end, so an address's own colons never
|
||||
// shift the ports. Not ok for a short form or anything that is not a mapping.
|
||||
func mapping(written string) (outer, inner int, address string, ok bool) {
|
||||
written = strings.TrimSpace(written)
|
||||
if cut := strings.LastIndex(written, "/"); cut >= 0 {
|
||||
written = written[:cut]
|
||||
}
|
||||
parts := strings.Split(written, ":")
|
||||
if len(parts) < 2 {
|
||||
return 0, 0, "", false
|
||||
}
|
||||
inner, err := strconv.Atoi(parts[len(parts)-1])
|
||||
if err != nil {
|
||||
return 0, 0, "", false
|
||||
}
|
||||
outer, err = strconv.Atoi(parts[len(parts)-2])
|
||||
if err != nil {
|
||||
return 0, 0, "", false
|
||||
}
|
||||
return outer, inner, strings.Join(parts[:len(parts)-2], ":"), true
|
||||
}
|
||||
|
||||
// Rules is the rule set this node's filter is derived from: every module's listens, what was
|
||||
// computed for this machine, and each module's per-node exposure. The same answer whether the node
|
||||
// is adopted or converged — the one loads it as a filter, the other declares it as openings.
|
||||
func (r Resolution) Rules(with Rendering) ([]Rule, error) {
|
||||
exposure := map[string]map[int]string{}
|
||||
for _, m := range r.Modules {
|
||||
e, err := Exposure(m, with.Settings[m.Module])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if e != nil {
|
||||
exposure[m.Module] = e
|
||||
}
|
||||
}
|
||||
return r.Filtering(with.Generators, with.Ports, exposure)
|
||||
}
|
||||
|
||||
// Contribution is one module telling the answer to a requirement what it needs from it.
|
||||
type Contribution struct {
|
||||
// From is the module that said it, so the provider and a person reading the file can tell
|
||||
@@ -697,30 +1015,53 @@ func (r Resolution) contributions(settings SettingsBy, grants []Grant,
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s contributing to %s: %w", m.Module, to, err)
|
||||
}
|
||||
composeName(values, r.PublicDomain)
|
||||
composeName(values, r.PublicDomain, r.At)
|
||||
out[to] = append(out[to], Contribution{From: m.Module, Values: values})
|
||||
}
|
||||
// Several contributions to one requirement (ADR 0094's sibling for `contributes`): an
|
||||
// object store's data API and its console are two different public names from one module,
|
||||
// not one. Never in `granted` — a route names a host, not a credential — so every local
|
||||
// name always reaches the provider from here.
|
||||
for _, to := range sortedKeys(m.ContributesMany) {
|
||||
for _, local := range sortedKeys(m.ContributesMany[to]) {
|
||||
values, err := settle(m.ContributesMany[to][local], settings[m.Module], nil,
|
||||
m.Module+" contributing "+local+" to "+to)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s contributing %s to %s: %w", m.Module, local, to, err)
|
||||
}
|
||||
composeName(values, r.PublicDomain, r.At)
|
||||
out[to] = append(out[to], Contribution{From: m.Module, Values: values})
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// composeName joins a contribution's label with a node's public domain, in place (novox/hq ADR
|
||||
// 0056).
|
||||
// composeName joins a contribution's label with a node's public domain, and separately with its
|
||||
// private one, in place (novox/hq ADR 0056).
|
||||
//
|
||||
// **The whole of what the mesh does with a route's name: join two given strings.** A contribution
|
||||
// carries a `label` — the subdomain its operator chose — and the node carries its public domain;
|
||||
// the granted name is `<label>.<public-domain>` and the mesh interprets neither half. It runs on
|
||||
// any contribution carrying a label, not only a route's, because the mesh does not know what a
|
||||
// **The whole of what the mesh does with a route's name: join two given strings — twice.** A
|
||||
// contribution carries a `label` — the subdomain its operator chose — and the node carries its
|
||||
// public domain and its own private-network address; the granted names are `<label>.<public-domain>`
|
||||
// and `<label>.<internal-domain>`, and the mesh interprets none of the halves. It runs on any
|
||||
// contribution carrying a label, not only a route's, because the mesh does not know what a
|
||||
// provision means — a name it can compose from parts it was given is the point, whatever the
|
||||
// provision is called.
|
||||
//
|
||||
// **The internal name is not a security boundary.** A predecessor proxy that answered both a
|
||||
// public and a private-network hostname for the same route did so as a convenience — reaching a
|
||||
// service over the VPN without a public TLS round trip — not as an access control, and composing
|
||||
// the same alias here restores that convenience rather than adding one. A route with no internal
|
||||
// domain to compose against (a node not on the private network) gets no internal name, the same as
|
||||
// it gets no public one with no public domain.
|
||||
//
|
||||
// **Additive, so an unmigrated catalogue still works.** A contribution that already carries a full
|
||||
// `name` and no `label` is left exactly as it is: the catalogue can migrate module by module while
|
||||
// the running mesh keeps serving the full names it has. And a labelled contribution on a node with
|
||||
// no public domain composes nothing — there is nothing to join it to — which reads downstream as a
|
||||
// route that named no host, the same as it would have before this existed.
|
||||
func composeName(values map[string]any, publicDomain string) {
|
||||
if values == nil || publicDomain == "" {
|
||||
func composeName(values map[string]any, publicDomain, internalDomain string) {
|
||||
if values == nil {
|
||||
return
|
||||
}
|
||||
if _, already := values["name"]; already {
|
||||
@@ -733,14 +1074,25 @@ func composeName(values map[string]any, publicDomain string) {
|
||||
if !ok || strings.TrimSpace(label) == "" {
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(label) == "@" {
|
||||
// The apex: a module served at the bare public domain, no subdomain — the zone-file
|
||||
// convention `@`. Composes to the domain itself, so a node's own site is a label like any
|
||||
// other rather than the one route that must still carry a full name.
|
||||
values["name"] = publicDomain
|
||||
trimmed := strings.TrimSpace(label)
|
||||
if trimmed == "@" {
|
||||
// The apex: a module served at the bare domain, no subdomain — the zone-file convention
|
||||
// `@`. Composes to the domain itself, so a node's own site is a label like any other rather
|
||||
// than the one route that must still carry a full name.
|
||||
if publicDomain != "" {
|
||||
values["name"] = publicDomain
|
||||
}
|
||||
if internalDomain != "" {
|
||||
values["internal-name"] = internalDomain
|
||||
}
|
||||
return
|
||||
}
|
||||
values["name"] = strings.TrimSpace(label) + "." + publicDomain
|
||||
if publicDomain != "" {
|
||||
values["name"] = trimmed + "." + publicDomain
|
||||
}
|
||||
if internalDomain != "" {
|
||||
values["internal-name"] = trimmed + "." + internalDomain
|
||||
}
|
||||
}
|
||||
|
||||
// receivedFile is the file a provider is given its consumers' contributions in.
|
||||
@@ -891,17 +1243,34 @@ func boundFile(n Needed, path, as string) (map[string]any, error) {
|
||||
// arrangement refused is the ordinary one. A node running eight services against one database is
|
||||
// not an edge case; it is what a machine looks like. Now each consumer has its own credential and
|
||||
// there is nothing left to refuse.
|
||||
//
|
||||
// **One credential, even where a module contributes several times.** A module may answer one
|
||||
// requirement more than once (ADR 0094's sibling for `contributes`) — an object store's data API
|
||||
// and its console are two different names, not one. There is still only one `Needed` for it, one
|
||||
// credential minted, one grant to settle: a pair credential is not a place to put a label or a
|
||||
// port. So where several of this module's contributions reach the same requirement, none of them
|
||||
// is "the" value — settling to the first, arbitrarily, would hand the grant one contribution's
|
||||
// values under a credential the OTHER contribution's consumer never sees, and would collide with
|
||||
// that contribution's own entry from contributions() besides. Empty values, still granted: the
|
||||
// module asked, gets its credential, and each named contribution reaches the provider on its own.
|
||||
func (r Resolution) ContributionsFrom(requirement, module string, settings SettingsBy) (
|
||||
map[string]any, bool, error) {
|
||||
all, err := r.contributions(settings, nil, nil)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
var mine []map[string]any
|
||||
for _, g := range all[requirement] {
|
||||
if g.From == module {
|
||||
return g.Values, true, nil
|
||||
mine = append(mine, g.Values)
|
||||
}
|
||||
}
|
||||
if len(mine) == 1 {
|
||||
return mine[0], true, nil
|
||||
}
|
||||
if len(mine) > 1 {
|
||||
return map[string]any{}, true, nil
|
||||
}
|
||||
// It contributes no payload — but a require-only consumer of a parameterless provision (one whose
|
||||
// `serves` names no consumer-supplied key: `redis-cache`, `amqp`) still ASKS for it and must be
|
||||
// granted a credential. Keying "asks" on contributions alone marked those grants withdrawn
|
||||
@@ -1090,7 +1459,11 @@ func publishedOn(resource map[string]any, module string, with Rendering) {
|
||||
for _, entry := range listed {
|
||||
written := fmt.Sprint(entry)
|
||||
if strings.Contains(written, ":") {
|
||||
out = append(out, written)
|
||||
// Written the long way, and left alone — unless this node was given a machine port for
|
||||
// it (novox/hq ADR 0100): the foundation's ports are the node's, and a manifest's
|
||||
// number is only the default. The outer port only; an address and the software's
|
||||
// port stay as written.
|
||||
out = append(out, givenOuter(written, with.Given[module]))
|
||||
continue
|
||||
}
|
||||
wanted, err := strconv.Atoi(strings.TrimSpace(written))
|
||||
@@ -1105,6 +1478,43 @@ func publishedOn(resource map[string]any, module string, with Rendering) {
|
||||
resource["ports"] = out
|
||||
}
|
||||
|
||||
// givenOuter rewrites the machine side of a long-form mapping to the port this node was given for
|
||||
// it, when it was given one.
|
||||
//
|
||||
// **Under either of the mapping's names.** A node gives a port by the number the module names it
|
||||
// by, and a module publishing `"2222:22"` may say it listens on 22 or on 2222 — GivenPorts accepts
|
||||
// both and answers to both, so looking the machine side up first and the container's port second
|
||||
// finds the same number either way. Read only by the container's port, this moved nothing for the
|
||||
// module that declares the machine side, and the setting was refused before it got here.
|
||||
func givenOuter(written string, given map[int]int) string {
|
||||
if len(given) == 0 {
|
||||
return written
|
||||
}
|
||||
mapping, protocol := written, ""
|
||||
if cut := strings.LastIndex(written, "/"); cut >= 0 {
|
||||
mapping, protocol = written[:cut], written[cut:]
|
||||
}
|
||||
parts := strings.Split(mapping, ":")
|
||||
if len(parts) < 2 {
|
||||
return written
|
||||
}
|
||||
inner, err := strconv.Atoi(strings.TrimSpace(parts[len(parts)-1]))
|
||||
if err != nil {
|
||||
return written
|
||||
}
|
||||
at, ok := given[inner]
|
||||
if outer, err := strconv.Atoi(strings.TrimSpace(parts[len(parts)-2])); err == nil {
|
||||
if machine, named := given[outer]; named {
|
||||
at, ok = machine, true
|
||||
}
|
||||
}
|
||||
if !ok {
|
||||
return written
|
||||
}
|
||||
parts[len(parts)-2] = strconv.Itoa(at)
|
||||
return strings.Join(parts, ":") + protocol
|
||||
}
|
||||
|
||||
// ServedOn is what a provider tells a consumer, with the port that machine actually uses.
|
||||
//
|
||||
// **The module writes the port once, in `listens`** (novox/hq ADR 0038). It used to write it three
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
package catalogue
|
||||
|
||||
import "testing"
|
||||
|
||||
// The mesh's user list reaches the module holding the bus, and nothing else.
|
||||
//
|
||||
// Three refusals and one delivery, because each of the refusals would be silent in a different way:
|
||||
// a module that asked and was given it could read every credential on the bus; a bus given an empty
|
||||
// file refuses every connection in the mesh and looks like a machine problem; and a bus that never
|
||||
// asked gets nothing rather than a file it does not read.
|
||||
func TestTheMeshsUserListGoesOnlyToTheModuleHoldingTheBus(t *testing.T) {
|
||||
theBus := func() Manifest {
|
||||
return Manifest{
|
||||
Module: "nats", Version: "1",
|
||||
Claims: []Claim{{Name: "mesh-broker", Scope: ScopeMesh}},
|
||||
BusUsers: "/var/lib/nats-module/conf/accounts.conf",
|
||||
Resources: []map[string]any{},
|
||||
}
|
||||
}
|
||||
|
||||
on := func(t *testing.T, m Manifest, with Rendering) ([]map[string]any, error) {
|
||||
t.Helper()
|
||||
return Resolution{Node: "anchor", Modules: []Manifest{m}}.Declaration(with)
|
||||
}
|
||||
|
||||
t.Run("the holder is given it", func(t *testing.T) {
|
||||
resources, err := on(t, theBus(), Rendering{BusUsers: "accounts { MESH { users = [] } }"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Prefixed with the module it came from, like every resource: two modules may reasonably
|
||||
// both call something "config", and without the prefix the second would silently replace
|
||||
// the first.
|
||||
var found map[string]any
|
||||
for _, r := range resources {
|
||||
if r["id"] == "nats."+BusUsersID() {
|
||||
found = r
|
||||
}
|
||||
}
|
||||
if found == nil {
|
||||
t.Fatalf("the bus was given no user list: %+v", resources)
|
||||
}
|
||||
if found["path"] != "/var/lib/nats-module/conf/accounts.conf" {
|
||||
t.Errorf("written to %v rather than where the module asked", found["path"])
|
||||
}
|
||||
if found["mode"] != "0600" {
|
||||
t.Errorf("mode %v: a list of every user in the mesh belongs to the one process that "+
|
||||
"needs it", found["mode"])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a module that does not claim the seat is refused", func(t *testing.T) {
|
||||
m := theBus()
|
||||
m.Claims = nil
|
||||
if _, err := on(t, m, Rendering{BusUsers: "accounts {}"}); err == nil {
|
||||
t.Fatal("a module that claims nothing was handed every user's password hash")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("the holder with nothing composed is refused", func(t *testing.T) {
|
||||
if _, err := on(t, theBus(), Rendering{}); err == nil {
|
||||
t.Fatal("the bus was given an empty user list, so it would refuse every connection in " +
|
||||
"the mesh and look like a machine problem")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a module that did not ask gets nothing", func(t *testing.T) {
|
||||
m := theBus()
|
||||
m.BusUsers = ""
|
||||
resources, err := on(t, m, Rendering{BusUsers: "accounts {}"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range resources {
|
||||
if r["id"] == "nats."+BusUsersID() {
|
||||
t.Fatal("a module that asked for no user list was given one")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,322 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// A directory the mesh places (novox/hq ADR 0112, to-be 27, issue 119).
|
||||
//
|
||||
// **A module definition names no host path.** A directory resource may omit `path`; the mesh
|
||||
// resolves where it lands when the declaration is composed — `<root>/<module>/<id>`, the root a
|
||||
// node's own setting with /var/lib as the default. From then on the module's own files, mounts
|
||||
// and environment name the place as `${dir:<id>}`, the same shape as `${bound:…}` and
|
||||
// `${secret:…}`: a fact the module asks for by name and never states.
|
||||
//
|
||||
// **A directory that states a path keeps it, and still answers `${dir:<id>}`.** That is the
|
||||
// placement for an adopted machine: data that must sit where the predecessor already put it is
|
||||
// declared with the path as the exception it is, and everything else in the module names it by
|
||||
// id — so moving it later is one line, not a search.
|
||||
//
|
||||
// **Resolved here, not on the machine.** The host receives concrete paths exactly as it always
|
||||
// has; nothing new reaches it and it learns no field. Which also means a resolved path changing
|
||||
// is a spec change like any other — and the spec comparison must see it (novox/hq issue 126).
|
||||
|
||||
// defaultDataRoot is where module data lands when a node states no root of its own.
|
||||
const defaultDataRoot = "/var/lib"
|
||||
|
||||
// dirRef is how a module names one of its placed directories: ${dir:<id>}.
|
||||
var dirRef = regexp.MustCompile(`\$\{dir:([a-z0-9][a-z0-9-]*)\}`)
|
||||
|
||||
// dataRoot is the root this node keeps placed directories under.
|
||||
func dataRoot(with Rendering) string {
|
||||
if root := strings.TrimRight(strings.TrimSpace(with.DataRoot), "/"); root != "" {
|
||||
return root
|
||||
}
|
||||
return defaultDataRoot
|
||||
}
|
||||
|
||||
// dirsFor is every placed directory of a module, id → the path it resolves to on this node.
|
||||
//
|
||||
// A pathless directory saying `"place": "."` is the assignment's own root, <root>/<module> —
|
||||
// to-be 27's one directory per assignment, which every other placed thing sits beneath. At most
|
||||
// one makes sense; nothing enforces one, because two ids resolving to one path is a mistake the
|
||||
// module's own files make visible immediately.
|
||||
func dirsFor(m Manifest, with Rendering) map[string]string {
|
||||
dirs := map[string]string{}
|
||||
for _, r := range m.Resources {
|
||||
if fmt.Sprint(r["type"]) != "directory" {
|
||||
continue
|
||||
}
|
||||
id := fmt.Sprint(r["id"])
|
||||
if path, stated := r["path"].(string); stated && path != "" {
|
||||
dirs[id] = strings.TrimRight(path, "/")
|
||||
continue
|
||||
}
|
||||
if place, said := r["place"].(string); said && place == "." {
|
||||
dirs[id] = dataRoot(with) + "/" + m.Module
|
||||
continue
|
||||
}
|
||||
dirs[id] = dataRoot(with) + "/" + m.Module + "/" + id
|
||||
}
|
||||
return dirs
|
||||
}
|
||||
|
||||
// placedOrAbsolute says a path is usable where the mesh needs one: absolute already, or
|
||||
// beginning with a placed reference — resolution makes it absolute before anything reads it.
|
||||
// (unknownDirRefs is what checks the reference names a real directory.)
|
||||
func placedOrAbsolute(path string) bool {
|
||||
return strings.HasPrefix(path, "/") ||
|
||||
(strings.HasPrefix(path, "${dir:") && dirRef.MatchString(path))
|
||||
}
|
||||
|
||||
// dirFill resolves every ${dir:…} in one string, or refuses a reference naming no directory.
|
||||
func dirFill(s string, dirs map[string]string, module string) (string, error) {
|
||||
var missing error
|
||||
out := dirRef.ReplaceAllStringFunc(s, func(ref string) string {
|
||||
id := dirRef.FindStringSubmatch(ref)[1]
|
||||
path, has := dirs[id]
|
||||
if !has {
|
||||
missing = fmt.Errorf(
|
||||
"%s says ${dir:%s}, and %s declares no directory %q. It declares %s",
|
||||
module, id, module, id, orNothing(namesOfDirs(dirs)))
|
||||
return ref
|
||||
}
|
||||
return path
|
||||
})
|
||||
return out, missing
|
||||
}
|
||||
|
||||
// placedManifest is the manifest with every path the mesh resolves already resolved: the maps
|
||||
// naming where bindings, credentials and contributions land are filled against this node's
|
||||
// placed directories, so everything downstream — the generated binding files, the sealed
|
||||
// secrets, the grant directories — reads a concrete place and learns nothing new.
|
||||
func placedManifest(m Manifest, with Rendering) (Manifest, error) {
|
||||
dirs := dirsFor(m, with)
|
||||
fillMap := func(in map[string]string) (map[string]string, error) {
|
||||
if len(in) == 0 {
|
||||
return in, nil
|
||||
}
|
||||
out := make(map[string]string, len(in))
|
||||
for key, value := range in {
|
||||
filled, err := dirFill(value, dirs, m.Module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[key] = filled
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
var err error
|
||||
if m.Receives, err = fillMap(m.Receives); err != nil {
|
||||
return m, err
|
||||
}
|
||||
if m.Binds, err = fillMap(m.Binds); err != nil {
|
||||
return m, err
|
||||
}
|
||||
if m.Secrets, err = fillMap(m.Secrets); err != nil {
|
||||
return m, err
|
||||
}
|
||||
if m.OwnSecrets, err = fillMap(m.OwnSecrets); err != nil {
|
||||
return m, err
|
||||
}
|
||||
if m.Grants, err = fillMap(m.Grants); err != nil {
|
||||
return m, err
|
||||
}
|
||||
if len(m.SecretsMany) > 0 {
|
||||
many := make(map[string]map[string]string, len(m.SecretsMany))
|
||||
for to, locals := range m.SecretsMany {
|
||||
if many[to], err = fillMap(locals); err != nil {
|
||||
return m, err
|
||||
}
|
||||
}
|
||||
m.SecretsMany = many
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// dirInto places a resource: a pathless directory is given the path the mesh resolved for it,
|
||||
// and every ${dir:…} the resource carries — in its path, its content, its mounts, its
|
||||
// environment and its env-files — becomes that path.
|
||||
//
|
||||
// A reference naming no directory of this module is refused. Left as written, the literal
|
||||
// `${dir:x}` would reach the machine as a path, and the runtime would create and mount a
|
||||
// directory called `${dir:x}` — real, wrong, and named after the mistake.
|
||||
func dirInto(resource map[string]any, dirs map[string]string, module string) error {
|
||||
fill := func(s string) (string, error) { return dirFill(s, dirs, module) }
|
||||
|
||||
if fmt.Sprint(resource["type"]) == "directory" {
|
||||
id := fmt.Sprint(resource["id"])
|
||||
if path, stated := resource["path"].(string); !stated || path == "" {
|
||||
resource["path"] = dirs[id]
|
||||
}
|
||||
// Said in the catalogue, not on the machine: the host parses strictly and knows no
|
||||
// such field — resolved, the place IS the path.
|
||||
delete(resource, "place")
|
||||
}
|
||||
|
||||
var err error
|
||||
if path, ok := resource["path"].(string); ok {
|
||||
if resource["path"], err = fill(path); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if content, ok := resource["content"].(string); ok {
|
||||
if resource["content"], err = fill(content); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Nested values are rebuilt, never written into: the resource is a shallow copy of the
|
||||
// manifest's own map, and the manifest is composed once per node — a fill written in place
|
||||
// would leave the first node's paths inside every later composition.
|
||||
if volumes, ok := resource["volumes"].([]any); ok {
|
||||
filled := make([]any, len(volumes))
|
||||
for i, v := range volumes {
|
||||
filled[i] = v
|
||||
if mount, ok := v.(string); ok {
|
||||
if filled[i], err = fill(mount); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
resource["volumes"] = filled
|
||||
}
|
||||
if env, ok := resource["env"].(map[string]any); ok {
|
||||
filled := make(map[string]any, len(env))
|
||||
for key, v := range env {
|
||||
filled[key] = v
|
||||
if value, ok := v.(string); ok {
|
||||
if filled[key], err = fill(value); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
resource["env"] = filled
|
||||
}
|
||||
if files, ok := resource["env-file"].([]any); ok {
|
||||
filled := make([]any, len(files))
|
||||
for i, v := range files {
|
||||
filled[i] = v
|
||||
if path, ok := v.(string); ok {
|
||||
if filled[i], err = fill(path); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
resource["env-file"] = filled
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// unknownDirRefs is every ${dir:…} in the definition that names no directory the definition
|
||||
// declares — refused where the author is, not at composition on some later day (the same
|
||||
// near-versus-far reasoning as the host's strict parse).
|
||||
func (m Manifest) unknownDirRefs() []string {
|
||||
declared := map[string]bool{}
|
||||
for _, r := range m.Resources {
|
||||
if fmt.Sprint(r["type"]) == "directory" {
|
||||
declared[fmt.Sprint(r["id"])] = true
|
||||
}
|
||||
}
|
||||
referenced := func(s string) []string {
|
||||
var ids []string
|
||||
for _, match := range dirRef.FindAllStringSubmatch(s, -1) {
|
||||
ids = append(ids, match[1])
|
||||
}
|
||||
return ids
|
||||
}
|
||||
var problems []string
|
||||
for _, r := range m.Resources {
|
||||
place, said := r["place"].(string)
|
||||
if !said {
|
||||
continue
|
||||
}
|
||||
if fmt.Sprint(r["type"]) != "directory" {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s says place on %v, which is not a directory — only a directory is placed",
|
||||
m.Module, r["id"]))
|
||||
continue
|
||||
}
|
||||
if path, stated := r["path"].(string); stated && path != "" {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s states both path and place on %v — a stated path IS the placement",
|
||||
m.Module, r["id"]))
|
||||
}
|
||||
if place != "." {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s says place %q on %v, and the only place is %q — the assignment's own root",
|
||||
m.Module, place, r["id"], "."))
|
||||
}
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
refuse := func(id string, where any) {
|
||||
if declared[id] || seen[id] {
|
||||
return
|
||||
}
|
||||
seen[id] = true
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s says ${dir:%s} in %v, and declares no directory %q — a reference the mesh "+
|
||||
"cannot place would reach the machine as a literal path",
|
||||
m.Module, id, where, id))
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
for _, field := range []string{"path", "content"} {
|
||||
if s, ok := r[field].(string); ok {
|
||||
for _, id := range referenced(s) {
|
||||
refuse(id, r["id"])
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, field := range []string{"volumes", "env-file"} {
|
||||
if list, ok := r[field].([]any); ok {
|
||||
for _, v := range list {
|
||||
if s, ok := v.(string); ok {
|
||||
for _, id := range referenced(s) {
|
||||
refuse(id, r["id"])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if env, ok := r["env"].(map[string]any); ok {
|
||||
for _, v := range env {
|
||||
if s, ok := v.(string); ok {
|
||||
for _, id := range referenced(s) {
|
||||
refuse(id, r["id"])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
maps := map[string]map[string]string{
|
||||
"receives": m.Receives, "binds": m.Binds, "secrets": m.Secrets,
|
||||
"own-secrets": m.OwnSecrets, "grants": m.Grants,
|
||||
}
|
||||
for field, entries := range maps {
|
||||
for _, value := range entries {
|
||||
for _, id := range referenced(value) {
|
||||
refuse(id, field)
|
||||
}
|
||||
}
|
||||
}
|
||||
for to, locals := range m.SecretsMany {
|
||||
for _, value := range locals {
|
||||
for _, id := range referenced(value) {
|
||||
refuse(id, "secrets."+to)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(problems)
|
||||
return problems
|
||||
}
|
||||
|
||||
func namesOfDirs(dirs map[string]string) []string {
|
||||
var names []string
|
||||
for id := range dirs {
|
||||
names = append(names, fmt.Sprintf("%q", id))
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
package catalogue
|
||||
|
||||
// A directory the mesh places (novox/hq ADR 0112). These tests pin the contract: a pathless
|
||||
// directory resolves under the node's root, ${dir:…} names it from every field a host path can
|
||||
// live in, a stated path is the adopted-data placement and wins, an unknown reference refuses at
|
||||
// the manifest, and filling for one node never leaks into the next composition.
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func placedModule() Manifest {
|
||||
return Manifest{
|
||||
Module: "photos",
|
||||
Resources: []map[string]any{
|
||||
{"id": "data", "type": "directory", "mode": "0700"},
|
||||
{"id": "server-env", "type": "file", "path": "${dir:data}/server.env",
|
||||
"content": "STORE=${dir:data}/objects\n"},
|
||||
{"id": "server", "type": "container", "name": "photos-server",
|
||||
"volumes": []any{"${dir:data}:/data"},
|
||||
"env": map[string]any{"DATA": "${dir:data}/objects"},
|
||||
"env-file": []any{"${dir:data}/server.env"}},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPathlessDirectoryResolvesUnderTheNodesRoot(t *testing.T) {
|
||||
m := placedModule()
|
||||
dirs := dirsFor(m, Rendering{})
|
||||
if dirs["data"] != "/var/lib/photos/data" {
|
||||
t.Fatalf("the default root is /var/lib and the shape is <root>/<module>/<id>; got %q", dirs["data"])
|
||||
}
|
||||
dirs = dirsFor(m, Rendering{DataRoot: "/tank/nox/"})
|
||||
if dirs["data"] != "/tank/nox/photos/data" {
|
||||
t.Fatalf("a node's own root is honoured, trailing slash and all; got %q", dirs["data"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirReferencesBecomeThePlaceInEveryField(t *testing.T) {
|
||||
m := placedModule()
|
||||
dirs := dirsFor(m, Rendering{})
|
||||
|
||||
directory := shallowCopy(m.Resources[0])
|
||||
if err := dirInto(directory, dirs, m.Module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if directory["path"] != "/var/lib/photos/data" {
|
||||
t.Fatalf("a pathless directory receives its resolved path; got %v", directory["path"])
|
||||
}
|
||||
|
||||
file := shallowCopy(m.Resources[1])
|
||||
if err := dirInto(file, dirs, m.Module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if file["path"] != "/var/lib/photos/data/server.env" {
|
||||
t.Fatalf("a file's path names the place; got %v", file["path"])
|
||||
}
|
||||
if file["content"] != "STORE=/var/lib/photos/data/objects\n" {
|
||||
t.Fatalf("a file's content names the place; got %v", file["content"])
|
||||
}
|
||||
|
||||
container := shallowCopy(m.Resources[2])
|
||||
if err := dirInto(container, dirs, m.Module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if container["volumes"].([]any)[0] != "/var/lib/photos/data:/data" {
|
||||
t.Fatalf("a mount names the place; got %v", container["volumes"])
|
||||
}
|
||||
if container["env"].(map[string]any)["DATA"] != "/var/lib/photos/data/objects" {
|
||||
t.Fatalf("an environment value names the place; got %v", container["env"])
|
||||
}
|
||||
if container["env-file"].([]any)[0] != "/var/lib/photos/data/server.env" {
|
||||
t.Fatalf("an env-file names the place; got %v", container["env-file"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStatedPathIsThePlacementAndStillAnswersByName(t *testing.T) {
|
||||
m := placedModule()
|
||||
// The adopted-machine case: data that must sit where the predecessor already put it.
|
||||
m.Resources[0]["path"] = "/services/mssql/data/"
|
||||
dirs := dirsFor(m, Rendering{})
|
||||
if dirs["data"] != "/services/mssql/data" {
|
||||
t.Fatalf("a stated path wins over the root, trimmed; got %q", dirs["data"])
|
||||
}
|
||||
container := shallowCopy(m.Resources[2])
|
||||
if err := dirInto(container, dirs, m.Module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if container["volumes"].([]any)[0] != "/services/mssql/data:/data" {
|
||||
t.Fatalf("references follow the placement; got %v", container["volumes"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestFillingForOneNodeLeaksIntoNoOther(t *testing.T) {
|
||||
m := placedModule()
|
||||
first := shallowCopy(m.Resources[2])
|
||||
if err := dirInto(first, dirsFor(m, Rendering{DataRoot: "/first"}), m.Module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second := shallowCopy(m.Resources[2])
|
||||
if err := dirInto(second, dirsFor(m, Rendering{DataRoot: "/second"}), m.Module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := second["volumes"].([]any)[0]; got != "/second/photos/data:/data" {
|
||||
t.Fatalf("the second composition must see the manifest, not the first fill; got %v", got)
|
||||
}
|
||||
if m.Resources[2]["volumes"].([]any)[0] != "${dir:data}:/data" {
|
||||
t.Fatalf("the manifest itself stays a template; got %v", m.Resources[2]["volumes"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReferenceToNoDirectoryRefusesAtTheManifest(t *testing.T) {
|
||||
m := placedModule()
|
||||
m.Resources[2]["volumes"] = []any{"${dir:date}:/data"} // a typo, the likely shape
|
||||
problems := m.unknownDirRefs()
|
||||
if len(problems) != 1 || !strings.Contains(problems[0], `${dir:date}`) {
|
||||
t.Fatalf("a reference naming no directory is a manifest problem; got %v", problems)
|
||||
}
|
||||
if got := placedModule().unknownDirRefs(); len(got) != 0 {
|
||||
t.Fatalf("a correct definition has none; got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReferenceToNoDirectoryRefusesAtCompositionToo(t *testing.T) {
|
||||
m := placedModule()
|
||||
container := shallowCopy(m.Resources[2])
|
||||
container["env"] = map[string]any{"DATA": "${dir:date}"}
|
||||
err := dirInto(container, dirsFor(m, Rendering{}), m.Module)
|
||||
if err == nil || !strings.Contains(err.Error(), `"date"`) || !strings.Contains(err.Error(), `"data"`) {
|
||||
t.Fatalf("the refusal names the mistake and what exists; got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheAssignmentsOwnRootIsAPlace(t *testing.T) {
|
||||
m := Manifest{Module: "mailu", Resources: []map[string]any{
|
||||
{"id": "state", "type": "directory", "place": ".", "mode": "0700"},
|
||||
{"id": "data-mail", "type": "directory"},
|
||||
}}
|
||||
dirs := dirsFor(m, Rendering{})
|
||||
if dirs["state"] != "/var/lib/mailu" {
|
||||
t.Fatalf("place %q is the assignment's root; got %q", ".", dirs["state"])
|
||||
}
|
||||
if dirs["data-mail"] != "/var/lib/mailu/data-mail" {
|
||||
t.Fatalf("everything else sits beneath it; got %q", dirs["data-mail"])
|
||||
}
|
||||
root := shallowCopy(m.Resources[0])
|
||||
if err := dirInto(root, dirs, m.Module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if root["path"] != "/var/lib/mailu" {
|
||||
t.Fatalf("the root receives its path; got %v", root["path"])
|
||||
}
|
||||
if _, still := root["place"]; still {
|
||||
t.Fatal("place must never reach the host, which parses strictly")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheManifestsMapsArePlaced(t *testing.T) {
|
||||
m := Manifest{
|
||||
Module: "photos",
|
||||
Resources: []map[string]any{
|
||||
{"id": "state", "type": "directory", "place": "."},
|
||||
},
|
||||
Binds: map[string]string{"route": "${dir:state}/route.json"},
|
||||
Secrets: map[string]string{"mongodb-database": "${dir:state}/database.secret"},
|
||||
OwnSecrets: map[string]string{"admin-key": "${dir:state}/admin-key.secret"},
|
||||
Receives: map[string]string{"route": "${dir:state}/grants/mesh.json"},
|
||||
}
|
||||
placed, err := placedManifest(m, Rendering{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if placed.Binds["route"] != "/var/lib/photos/route.json" {
|
||||
t.Fatalf("binds are placed; got %v", placed.Binds)
|
||||
}
|
||||
if placed.Secrets["mongodb-database"] != "/var/lib/photos/database.secret" {
|
||||
t.Fatalf("secrets are placed; got %v", placed.Secrets)
|
||||
}
|
||||
if placed.OwnSecrets["admin-key"] != "/var/lib/photos/admin-key.secret" {
|
||||
t.Fatalf("own-secrets are placed; got %v", placed.OwnSecrets)
|
||||
}
|
||||
if placed.Receives["route"] != "/var/lib/photos/grants/mesh.json" {
|
||||
t.Fatalf("receives are placed; got %v", placed.Receives)
|
||||
}
|
||||
if m.Binds["route"] != "${dir:state}/route.json" {
|
||||
t.Fatalf("the manifest itself stays a template; got %v", m.Binds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMapReferenceToNoDirectoryRefusesAtTheManifest(t *testing.T) {
|
||||
m := Manifest{
|
||||
Module: "photos",
|
||||
Resources: []map[string]any{{"id": "state", "type": "directory", "place": "."}},
|
||||
Binds: map[string]string{"route": "${dir:stat}/route.json"},
|
||||
}
|
||||
problems := m.unknownDirRefs()
|
||||
if len(problems) != 1 || !strings.Contains(problems[0], `${dir:stat}`) {
|
||||
t.Fatalf("a map naming no directory is a manifest problem; got %v", problems)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlaceIsValidatedAtTheManifest(t *testing.T) {
|
||||
both := Manifest{Module: "x", Resources: []map[string]any{
|
||||
{"id": "d", "type": "directory", "place": ".", "path": "/somewhere"},
|
||||
}}
|
||||
if got := both.unknownDirRefs(); len(got) != 1 || !strings.Contains(got[0], "both path and place") {
|
||||
t.Fatalf("path beside place refuses; got %v", got)
|
||||
}
|
||||
elsewhere := Manifest{Module: "x", Resources: []map[string]any{
|
||||
{"id": "f", "type": "file", "place": ".", "path": "/somewhere", "content": ""},
|
||||
}}
|
||||
if got := elsewhere.unknownDirRefs(); len(got) != 1 || !strings.Contains(got[0], "not a directory") {
|
||||
t.Fatalf("place on a file refuses; got %v", got)
|
||||
}
|
||||
wrong := Manifest{Module: "x", Resources: []map[string]any{
|
||||
{"id": "d", "type": "directory", "place": "sub/dir"},
|
||||
}}
|
||||
if got := wrong.unknownDirRefs(); len(got) != 1 || !strings.Contains(got[0], `the only place is "."`) {
|
||||
t.Fatalf("a place that is not the root refuses; got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTwoModulesPlacedRootsAreNoCollision(t *testing.T) {
|
||||
a := Manifest{Module: "gitea", Resources: []map[string]any{
|
||||
{"id": "state", "type": "directory", "place": "."},
|
||||
{"id": "env", "type": "file", "path": "${dir:state}/server.env", "content": ""},
|
||||
}}
|
||||
b := Manifest{Module: "nextcloud", Resources: []map[string]any{
|
||||
{"id": "state", "type": "directory", "place": "."},
|
||||
{"id": "env", "type": "file", "path": "${dir:state}/server.env", "content": ""},
|
||||
}}
|
||||
if got := checkResources([]Manifest{a, b}); len(got) != 0 {
|
||||
t.Fatalf("alike templates are different places; got %v", got)
|
||||
}
|
||||
// And the real collision is still caught: a module stating another's placed root.
|
||||
c := Manifest{Module: "squatter", Resources: []map[string]any{
|
||||
{"id": "nest", "type": "directory", "path": "/var/lib/gitea"},
|
||||
}}
|
||||
got := checkResources([]Manifest{a, c})
|
||||
if len(got) != 1 || !strings.Contains(got[0], `"/var/lib/gitea"`) {
|
||||
t.Fatalf("a stated path on a placed root collides; got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func shallowCopy(resource map[string]any) map[string]any {
|
||||
copied := map[string]any{}
|
||||
for k, v := range resource {
|
||||
copied[k] = v
|
||||
}
|
||||
return copied
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// What a module may call an event, and what a consumer may ask for.
|
||||
//
|
||||
// A module names an event **locally**: `order.placed`, not a subject and not a routing key
|
||||
// (design 29 §1). A consumer names the emitter and the event: `billing.order.placed`. The mesh
|
||||
// derives the subject from those, so reorganising the subject space leaves every manifest correct.
|
||||
//
|
||||
// **Nothing checked this until every manifest in the catalogue was wrong the same way**
|
||||
// (novox/hq 04-ISSUES/127). All thirty-seven kept the old bus's routing key —
|
||||
// `module.<module>.<verb>` — which the derivation read as "a module called `module`", so every
|
||||
// cross-module subscription in the mesh pointed at a namespace nobody publishes to. Nothing failed:
|
||||
// the services started and none of them reacted. The documentation on these fields taught the old
|
||||
// form too, which is why the drift was uniform rather than scattered.
|
||||
|
||||
// eventName is one name in a local event: lower-case, and no wildcard.
|
||||
var eventName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`)
|
||||
|
||||
// The wildcards a consumer may use, spelled the mesh's way and derived to whatever the transport
|
||||
// spells them as.
|
||||
//
|
||||
// **A manifest holds no transport token**, which is the whole point of naming locally: the bus the
|
||||
// mesh runs on today spells these `*` and `#`, and the one being built spells them `*` and `>`. A
|
||||
// manifest that said either would be a manifest that stopped being true when the wire changed.
|
||||
const (
|
||||
// OneName stands for exactly one name.
|
||||
OneName = "*"
|
||||
// TheRest stands for one or more names, and may only come last.
|
||||
TheRest = "**"
|
||||
)
|
||||
|
||||
// EventProblems is what is wrong with a manifest's events.
|
||||
//
|
||||
// Refused at registration, because the alternative is a module that installs, starts, connects and
|
||||
// reacts to nothing — and every log line says it is fine.
|
||||
func EventProblems(m Manifest) []string {
|
||||
var problems []string
|
||||
|
||||
for _, e := range m.Emits {
|
||||
if was, stale := staleEventForm(e, m.Module); stale {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s emits %q, which is the old bus's routing key. An event is named locally now, so "+
|
||||
"write %q — the mesh derives the subject (novox/hq design 29 §1)",
|
||||
m.Module, was, strings.TrimPrefix(was, "module."+m.Module+".")))
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(e, "module.") {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s emits %q: `module.` is reserved, because it is how the old bus spelled a "+
|
||||
"routing key and an event named that way derives into a namespace nobody owns",
|
||||
m.Module, e))
|
||||
continue
|
||||
}
|
||||
if err := localName(e); err != nil {
|
||||
problems = append(problems, fmt.Sprintf("%s emits %q: %v", m.Module, e, err))
|
||||
continue
|
||||
}
|
||||
// **Its own name, never another's.** The bus enforces that a namespace belongs to the module
|
||||
// it is named for, so an event named for somebody else cannot be published at all. If the
|
||||
// event is about a role rather than about this module, it belongs on the seat: a name that
|
||||
// is stable across whoever fills it (04-ISSUES/127).
|
||||
if first, _, split := strings.Cut(e, "."); split && isAModuleNameOtherThan(first, m.Module) {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s emits %q, which reads as another module's event. A module publishes under its "+
|
||||
"own name only. If this is about a role rather than about %s, declare it on that "+
|
||||
"seat, where the name survives the holder changing",
|
||||
m.Module, e, m.Module))
|
||||
}
|
||||
}
|
||||
|
||||
for _, c := range m.Consumes {
|
||||
if strings.HasPrefix(c, "module.") {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s consumes %q, which is the old bus's pattern. A consumed event names its emitter "+
|
||||
"and the event: write %q", m.Module, c, strings.TrimPrefix(c, "module.")))
|
||||
continue
|
||||
}
|
||||
if c == "#" {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s consumes %q, which is the old bus's wildcard for everything. Write %q",
|
||||
m.Module, c, TheRest))
|
||||
continue
|
||||
}
|
||||
if err := consumePattern(c); err != nil {
|
||||
problems = append(problems, fmt.Sprintf("%s consumes %q: %v", m.Module, c, err))
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// staleEventForm says an emitted name is this module's own old routing key, and what it was.
|
||||
func staleEventForm(event, module string) (string, bool) {
|
||||
return event, module != "" && strings.HasPrefix(event, "module."+module+".")
|
||||
}
|
||||
|
||||
// isAModuleNameOtherThan says a first token names some module of this mesh that is not this one.
|
||||
//
|
||||
// Only the mesh's own seats and the catalogue could answer this properly, and neither is reachable
|
||||
// from a parser given one manifest. So this catches the case that actually happened — a name that
|
||||
// is a *provision* the mesh defines, which is where "another module's event" comes from in practice
|
||||
// — and the whole-catalogue check catches the rest.
|
||||
func isAModuleNameOtherThan(first, module string) bool {
|
||||
if first == module || first == "" {
|
||||
return false
|
||||
}
|
||||
if _, isASeat := SeatNamed(first); isASeat {
|
||||
return true
|
||||
}
|
||||
if _, isASeat := SeatDelivering(first); isASeat {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// localName checks one event name: dot-separated names, no wildcards, nothing else.
|
||||
func localName(event string) error {
|
||||
if event == "" {
|
||||
return fmt.Errorf("an event needs a name")
|
||||
}
|
||||
for _, part := range strings.Split(event, ".") {
|
||||
if part == OneName || part == TheRest {
|
||||
return fmt.Errorf("an emitted event names one event, so it carries no wildcard")
|
||||
}
|
||||
if !eventName.MatchString(part) {
|
||||
return fmt.Errorf("%q is not a usable name: lower-case letters, digits and dashes", part)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// consumePattern checks a consumed pattern: the emitter, then the event, with wildcards.
|
||||
func consumePattern(pattern string) error {
|
||||
if pattern == "" {
|
||||
return fmt.Errorf("a consumed event needs an emitter and an event")
|
||||
}
|
||||
parts := strings.Split(pattern, ".")
|
||||
for i, part := range parts {
|
||||
switch {
|
||||
case part == TheRest:
|
||||
if i != len(parts)-1 {
|
||||
return fmt.Errorf("%q stands for the rest of a name, so nothing may follow it", TheRest)
|
||||
}
|
||||
case part == OneName:
|
||||
case !eventName.MatchString(part):
|
||||
return fmt.Errorf("%q is not a usable name: lower-case letters, digits and dashes", part)
|
||||
}
|
||||
}
|
||||
// `**` alone is every event from every module, which the audit logger wants and says plainly.
|
||||
if len(parts) == 1 && parts[0] != TheRest {
|
||||
return fmt.Errorf(
|
||||
"%q names an emitter and no event. Write <emitter>.<event>, or %q for every event",
|
||||
pattern, TheRest)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,145 +0,0 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// What only the mesh knows, written where a module asks for it.
|
||||
//
|
||||
// **The graph is the control plane's; using it is the module's.** The mesh knows which machines
|
||||
// exist, what they are called, and where they are. Turning that into a name that resolves is
|
||||
// somebody's software, and which software is a choice the mesh should not be making.
|
||||
//
|
||||
// This replaced three modules — names, a resolver's data, and the private network's own
|
||||
// configuration — that existed only because computed output needed somewhere to live. They ran no
|
||||
// software and could not be swapped for anything, which is the test of whether something is a
|
||||
// module at all (novox/hq ADR 0040).
|
||||
|
||||
const (
|
||||
// FactNodeNames is every machine's name and address, as a hosts file.
|
||||
//
|
||||
// Exact names only: `homer` and `homer.internal` resolve to homer. Anything *under* a machine
|
||||
// is a wildcard, which a hosts file cannot express — that is FactNodeZones.
|
||||
FactNodeNames = "node-names"
|
||||
|
||||
// FactNodeZones is every machine as a wildcard: `*.homer.internal` is homer.
|
||||
//
|
||||
// Written in the form a resolver reads. A machine's own name and everything under it are one
|
||||
// fact — if homer is at an address, so is anything homer serves.
|
||||
FactNodeZones = "node-zones"
|
||||
)
|
||||
|
||||
// facts is every fact the mesh computes, and what writes it.
|
||||
//
|
||||
// **A closed list.** A module asking for a fact the mesh does not have is asking for a file nobody
|
||||
// will write, and finding that out on a machine — as a daemon that starts, reads nothing, and
|
||||
// answers no queries — is worse than being told where the manifest is.
|
||||
var facts = map[string]func(Resolution, map[string]string) string{
|
||||
FactNodeNames: nodeNames,
|
||||
FactNodeZones: nodeZones,
|
||||
}
|
||||
|
||||
// FactsInto renders the facts a module asked for, as files it will be given.
|
||||
//
|
||||
// The module owns everything after the file exists: loading it, restarting on it, what a resolver
|
||||
// does with it. This only puts it there.
|
||||
func FactsInto(m Manifest, r Resolution, addresses map[string]string) ([]map[string]any, error) {
|
||||
if len(m.Facts) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
names := make([]string, 0, len(m.Facts))
|
||||
for name := range m.Facts {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
|
||||
out := make([]map[string]any, 0, len(names))
|
||||
for _, name := range names {
|
||||
write, known := facts[name]
|
||||
if !known {
|
||||
return nil, fmt.Errorf(
|
||||
"%s asks the mesh for %q, which it does not compute. It has %s",
|
||||
m.Module, name, spokenFacts())
|
||||
}
|
||||
path := m.Facts[name]
|
||||
if !strings.HasPrefix(path, "/") {
|
||||
return nil, fmt.Errorf(
|
||||
"%s asks for %q at %q, which is not an absolute path", m.Module, name, path)
|
||||
}
|
||||
out = append(out, map[string]any{
|
||||
"id": "fact-" + name, "type": "file", "path": path, "mode": "0644",
|
||||
"content": write(r, addresses),
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// spokenFacts lists them, so a refusal says what would have worked.
|
||||
func spokenFacts() string {
|
||||
names := make([]string, 0, len(facts))
|
||||
for name := range facts {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
return strings.Join(names, ", ")
|
||||
}
|
||||
|
||||
// nodeNames is every machine's name and address, as a hosts file.
|
||||
//
|
||||
// **A machine with no address is left out.** The mesh has a record for it — somebody added it —
|
||||
// and does not yet know where it is, which is the ordinary state between adding a machine and it
|
||||
// joining. Writing the name anyway would give a name that resolves to nothing, and a connection to
|
||||
// that hangs; leaving it out fails at once and says the name is unknown.
|
||||
func nodeNames(r Resolution, addresses map[string]string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("# Generated by the mesh. Do not edit — this file is replaced whenever a machine\n")
|
||||
b.WriteString("# joins or leaves, and an edit would survive until then and vanish.\n\n")
|
||||
// The floor every Linux expects, and which removing would break things that have nothing to do
|
||||
// with the mesh.
|
||||
b.WriteString("127.0.0.1\tlocalhost\n")
|
||||
b.WriteString("::1\t\tlocalhost ip6-localhost ip6-loopback\n")
|
||||
if r.Node != "" {
|
||||
fmt.Fprintf(&b, "127.0.1.1\t%s\n", r.Node)
|
||||
}
|
||||
b.WriteString("\n")
|
||||
for _, name := range sortedNames(addresses) {
|
||||
at := addresses[name]
|
||||
// Its mesh name resolves to its address on the private network rather than to loopback,
|
||||
// so a service binding the name it was given stays reachable from everywhere else.
|
||||
fmt.Fprintf(&b, "%s\t%s.internal\t%s", at, name, name)
|
||||
if name == r.Node {
|
||||
b.WriteString("\t# this machine")
|
||||
}
|
||||
b.WriteString("\n")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// nodeZones is every machine as a wildcard, in the form a resolver reads.
|
||||
//
|
||||
// `*.homer.internal` is homer, which is the whole rule: if homer is at an address, so is anything
|
||||
// homer serves. A module wanting this runs the resolver; the mesh only says what is true.
|
||||
func nodeZones(_ Resolution, addresses map[string]string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("# Generated by the mesh. Do not edit — this file is replaced whenever a machine\n")
|
||||
b.WriteString("# joins or leaves, and an edit would survive until then and vanish.\n\n")
|
||||
for _, name := range sortedNames(addresses) {
|
||||
fmt.Fprintf(&b, "address=/%s.internal/%s\n", name, addresses[name])
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func sortedNames(addresses map[string]string) []string {
|
||||
out := make([]string, 0, len(addresses))
|
||||
for name, at := range addresses {
|
||||
// See nodeNames: a machine the mesh cannot place is left out rather than named at nothing.
|
||||
if at == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, name)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -1,97 +0,0 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var threeMachines = map[string]string{"homer": "10.42.0.1", "marge": "10.42.0.2", "bart": ""}
|
||||
|
||||
// **`*.homer.internal` is homer. That is the whole rule.**
|
||||
func TestEveryMachineIsAWildcardUnderItsOwnName(t *testing.T) {
|
||||
out := nodeZones(Resolution{Node: "homer"}, threeMachines)
|
||||
for _, want := range []string{
|
||||
"address=/homer.internal/10.42.0.1",
|
||||
"address=/marge.internal/10.42.0.2",
|
||||
} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Fatalf("missing %q:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A machine the mesh has a record for and cannot place is left out of both.
|
||||
//
|
||||
// **Not an oversight — the alternative is worse.** A name written with no address resolves to
|
||||
// nothing, and a connection to that hangs. Leaving it out fails at once and says the name is
|
||||
// unknown, which is a thing somebody can act on.
|
||||
func TestAMachineWithNoAddressIsNotNamed(t *testing.T) {
|
||||
for _, out := range []string{
|
||||
nodeNames(Resolution{Node: "homer"}, threeMachines),
|
||||
nodeZones(Resolution{Node: "homer"}, threeMachines),
|
||||
} {
|
||||
if strings.Contains(out, "bart") {
|
||||
t.Fatalf("a machine with no address was named, so its name resolves to nothing:\n%s", out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A machine's own mesh name points at its address on the private network, not at loopback — or a
|
||||
// service binding the name it was given is unreachable from everywhere else.
|
||||
func TestAMachinesOwnNameIsItsMeshAddress(t *testing.T) {
|
||||
out := nodeNames(Resolution{Node: "homer"}, threeMachines)
|
||||
var line string
|
||||
for _, l := range strings.Split(out, "\n") {
|
||||
if strings.Contains(l, "homer.internal") {
|
||||
line = l
|
||||
}
|
||||
}
|
||||
if !strings.HasPrefix(line, "10.42.0.1") {
|
||||
t.Fatalf("a machine's own mesh name is not its mesh address: %q", line)
|
||||
}
|
||||
// And the loopback floor is still there, or things with nothing to do with the mesh break.
|
||||
if !strings.Contains(out, "127.0.0.1\tlocalhost") {
|
||||
t.Fatalf("the loopback floor was removed:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
// A module says where it wants a fact, and is given a file.
|
||||
func TestAModuleIsGivenTheFactsItAskedFor(t *testing.T) {
|
||||
m := Manifest{Module: "dnsmasq", Facts: map[string]string{FactNodeZones: "/etc/mesh/zones.conf"}}
|
||||
given, err := FactsInto(m, Resolution{Node: "homer"}, threeMachines)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(given) != 1 {
|
||||
t.Fatalf("expected one file, got %d", len(given))
|
||||
}
|
||||
if given[0]["path"] != "/etc/mesh/zones.conf" || given[0]["type"] != "file" {
|
||||
t.Fatalf("not written where it was asked for: %v", given[0])
|
||||
}
|
||||
if !strings.Contains(given[0]["content"].(string), "homer.internal") {
|
||||
t.Fatalf("the file does not hold the fact: %v", given[0]["content"])
|
||||
}
|
||||
}
|
||||
|
||||
// **Asking for a fact the mesh does not have is refused here, not on a machine.** A daemon that
|
||||
// starts, reads a file nobody wrote, and answers no queries is a much worse way to find out.
|
||||
func TestAskingForAFactTheMeshDoesNotHaveIsRefused(t *testing.T) {
|
||||
m := Manifest{Module: "dnsmasq", Facts: map[string]string{"the-weather": "/etc/weather"}}
|
||||
_, err := FactsInto(m, Resolution{}, nil)
|
||||
if err == nil {
|
||||
t.Fatal("a module asked for something nobody computes and was given nothing, silently")
|
||||
}
|
||||
for _, known := range []string{FactNodeNames, FactNodeZones} {
|
||||
if !strings.Contains(err.Error(), known) {
|
||||
t.Fatalf("the refusal does not say what would have worked: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// And a relative path is refused, or a module decides where the mesh writes on a machine.
|
||||
func TestAFactMustBeAskedForAtAnAbsolutePath(t *testing.T) {
|
||||
m := Manifest{Module: "dnsmasq", Facts: map[string]string{FactNodeNames: "etc/hosts"}}
|
||||
if _, err := FactsInto(m, Resolution{}, nil); err == nil {
|
||||
t.Fatal("a relative path was accepted")
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -457,3 +458,209 @@ func byFamily(addresses []string) (four []string, six []string) {
|
||||
}
|
||||
return four, six
|
||||
}
|
||||
|
||||
// PortsSetting is the settings key that gives a module's port a machine port on one node (novox/hq
|
||||
// ADR 0100):
|
||||
//
|
||||
// {"ports": {"5432": 5433}}
|
||||
//
|
||||
// puts what the software calls 5432 on the machine's 5433. The foundation's ports are the node's:
|
||||
// every one is an input to genesis, checked free there, and becomes that node's setting for the
|
||||
// foundation's modules — the catalogue's numbers are only their defaults. Keyed by the port the
|
||||
// software uses, like expose; the value is where the machine puts it.
|
||||
const PortsSetting = "ports"
|
||||
|
||||
// MeshWideLayer is what a layer set for the whole mesh is called, rather than for one node.
|
||||
const MeshWideLayer = "the mesh"
|
||||
|
||||
// GivenPorts reads a module's given machine ports from its settings: software port → machine port.
|
||||
//
|
||||
// Refused from a mesh-wide layer — a port is a fact about one machine, and one number for every
|
||||
// machine is the collision this exists to avoid — and for a port the module's containers do not
|
||||
// publish.
|
||||
//
|
||||
// **Only a published port is the mesh's to move** (novox/hq ADR 0038). A container's mapping is
|
||||
// what translates; a module binding the machine's network directly binds the number its software
|
||||
// was configured with, and moving that number would put it in the filter, in the openings and in
|
||||
// what consumers are told while the software still listens on the old one — a port that reads as
|
||||
// moved and is not.
|
||||
//
|
||||
// **Either name of a mapping names it; the module's own name answers.** A short form publishes one
|
||||
// number, which is the container's port and the machine's at once. A mapping written the long way
|
||||
// — `"2222:22"` — has two, and a module reasonably declares it listens on either: the port its
|
||||
// software uses, or the port the machine already serves on. A setting may name either end, because
|
||||
// both are true of the same mapping. The answer comes back under the end the **module** names in
|
||||
// its `listens`, which is the number every reader of this map holds: the ports map, the filter, the
|
||||
// openings, what a consumer is told, and the mapping the runtime is handed. Keyed one way and read
|
||||
// the other, the setting moved the container's mapping and nothing else — a firewall, a set of
|
||||
// openings and a consumer all pointing at a port the software had left.
|
||||
//
|
||||
// One entry per mapping, never two. A second key for the same answer is not a convenience: where
|
||||
// two mappings share a number, it is an entry one of them writes over the other's, and the reader
|
||||
// that finds the survivor disagrees with the reader that recomputes it.
|
||||
func GivenPorts(m Manifest, layers []Layer) (map[int]int, error) {
|
||||
// Every name a setting may use, and the mapping it names.
|
||||
names := map[int][]publishing{}
|
||||
for _, p := range publishedPorts(m) {
|
||||
names[p.inner] = append(names[p.inner], p)
|
||||
if p.machine != p.inner {
|
||||
names[p.machine] = append(names[p.machine], p)
|
||||
}
|
||||
}
|
||||
// And the names the module itself uses. A mapping's answer is filed under these, because they
|
||||
// are what every reader asks for; a mapping the module names at neither end is filed under its
|
||||
// machine side, where nothing looks, which is correct — nothing serves it.
|
||||
declares := map[int]bool{}
|
||||
for _, l := range m.Listens {
|
||||
declares[l.Port] = true
|
||||
}
|
||||
chose := map[int]int{} // the port a setting named → the machine port it gave it
|
||||
out := map[int]int{} // the port the module names → the machine port it is on
|
||||
by := map[int]int{} // and which of the setting's ports put it there, for the refusal
|
||||
for _, layer := range layers {
|
||||
raw, ok := layer.Values[PortsSetting]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if layer.From == MeshWideLayer {
|
||||
return nil, fmt.Errorf("%s: %s is given per node — a port is a fact about one "+
|
||||
"machine; set it with --node", m.Module, PortsSetting)
|
||||
}
|
||||
entries, ok := raw.(map[string]any)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%s: %s is a { port: machine-port } map, and %q set it to "+
|
||||
"something else", m.Module, PortsSetting, layer.From)
|
||||
}
|
||||
for portText, value := range entries {
|
||||
port, err := strconv.Atoi(portText)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s gives %q a port, which is not a port", m.Module, portText)
|
||||
}
|
||||
publishes, known := names[port]
|
||||
if !known {
|
||||
return nil, fmt.Errorf("%s gives port %d a machine port, and no container of its "+
|
||||
"publishes %d — the mesh cannot move a port the module does not publish; the "+
|
||||
"software would go on listening where it was told to", m.Module, port, port)
|
||||
}
|
||||
if err := oneMapping(m.Module, port, publishes); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
at, ok := asPort(value)
|
||||
if !ok || at < 1 || at > 65535 {
|
||||
return nil, fmt.Errorf("%s gives port %d the machine port %v, which is not a port",
|
||||
m.Module, port, value)
|
||||
}
|
||||
if at == SSHPort {
|
||||
return nil, fmt.Errorf("%s gives port %d the machine port %d, which is ssh's — the "+
|
||||
"one port a machine may never lose", m.Module, port, at)
|
||||
}
|
||||
chose[port] = at
|
||||
}
|
||||
}
|
||||
// One holder per machine port, within the module too.
|
||||
holder := map[int]int{}
|
||||
for _, port := range sortedPorts(chose) {
|
||||
at := chose[port]
|
||||
if other, twice := holder[at]; twice {
|
||||
return nil, fmt.Errorf("%s gives machine port %d to both its %d and its %d", m.Module,
|
||||
at, min(port, other), max(port, other))
|
||||
}
|
||||
holder[at] = port
|
||||
}
|
||||
// Filed under the module's own names for the mapping — every one it uses, so a module that
|
||||
// says it listens on both ends is answered at both, and under the machine side when it names
|
||||
// neither.
|
||||
for _, port := range sortedPorts(chose) {
|
||||
at, mapping := chose[port], names[port][0]
|
||||
keys := []int{}
|
||||
for _, end := range []int{mapping.machine, mapping.inner} {
|
||||
if declares[end] && !slices.Contains(keys, end) {
|
||||
keys = append(keys, end)
|
||||
}
|
||||
}
|
||||
if len(keys) == 0 {
|
||||
keys = []int{mapping.machine}
|
||||
}
|
||||
for _, key := range keys {
|
||||
// The key must name this mapping and no other, or the entry is one mapping's answer
|
||||
// standing where another's is read.
|
||||
if err := oneMapping(m.Module, key, names[key]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// And one machine port per mapping: `{"22": 222, "2222": 300}` is two numbers for the
|
||||
// one thing the machine publishes, and neither is more right.
|
||||
if was, twice := out[key]; twice && was != at {
|
||||
return nil, fmt.Errorf("%s gives %s the machine ports %d and %d — its %d and its "+
|
||||
"%d are the two ends of one mapping, and it is published once", m.Module,
|
||||
mapping, min(was, at), max(was, at), min(by[key], port), max(by[key], port))
|
||||
}
|
||||
out[key], by[key] = at, port
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// oneMapping refuses a number that names two of a module's mappings — `"22"` beside `"2222:22"`,
|
||||
// say, or `"4001:80"` beside `"4002:80"` read by their shared `80`. Refused rather than picked:
|
||||
// moving one of them and leaving the other is a mapping the operator did not ask for and cannot
|
||||
// see, and the two readings differ.
|
||||
func oneMapping(module string, port int, publishes []publishing) error {
|
||||
for _, other := range publishes[1:] {
|
||||
if other != publishes[0] {
|
||||
return fmt.Errorf("%s gives port %d a machine port, and its containers publish %d "+
|
||||
"twice — as %s and as %s; which one to move is not said", module, port, port,
|
||||
publishes[0], other)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// publishing is one mapping a module's container writes: the port the container itself uses, and
|
||||
// the machine port the manifest put it on — the same number for a short form, which the runtime
|
||||
// publishes on the port it names.
|
||||
type publishing struct{ machine, inner int }
|
||||
|
||||
func (p publishing) String() string {
|
||||
if p.machine == p.inner {
|
||||
return strconv.Itoa(p.inner)
|
||||
}
|
||||
return fmt.Sprintf("%d:%d", p.machine, p.inner)
|
||||
}
|
||||
|
||||
// publishedPorts is every mapping a module's containers publish, whichever form it is written in.
|
||||
func publishedPorts(m Manifest) []publishing {
|
||||
var out []publishing
|
||||
for _, r := range m.Resources {
|
||||
if fmt.Sprint(r["type"]) != "container" {
|
||||
continue
|
||||
}
|
||||
listed, _ := r["ports"].([]any)
|
||||
for _, entry := range listed {
|
||||
written := strings.TrimSpace(fmt.Sprint(entry))
|
||||
if machine, inner, _, ok := mapping(written); ok {
|
||||
out = append(out, publishing{machine: machine, inner: inner})
|
||||
continue
|
||||
}
|
||||
if cut := strings.LastIndex(written, "/"); cut >= 0 {
|
||||
written = written[:cut]
|
||||
}
|
||||
if n, err := strconv.Atoi(strings.TrimSpace(written)); err == nil {
|
||||
out = append(out, publishing{machine: n, inner: n})
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sortedPorts is a settings map's ports in order, so a refusal reads the same on every run.
|
||||
func sortedPorts(of map[int]int) []int {
|
||||
out := make([]int, 0, len(of))
|
||||
for port := range of {
|
||||
out = append(out, port)
|
||||
}
|
||||
sort.Ints(out)
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -0,0 +1,305 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The catalogue's foundation modules as they are, parsed by the real parser (novox/hq ADR 0100):
|
||||
// the store and the broker say which of their ports the mesh guards on an adopted node, and the
|
||||
// filter module loads its table through a unit of its own whose stop deletes only that table.
|
||||
func catalogueManifest(t *testing.T, module string) Manifest {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile("../../../mesh-catalog/modules/" + module + "/module.json")
|
||||
if err != nil {
|
||||
t.Skipf("the catalogue is not beside this checkout: %v", err)
|
||||
}
|
||||
m, err := ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("%s does not parse:\n%v", module, err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func TestTheStoreAndTheBrokerSayWhatTheMeshGuards(t *testing.T) {
|
||||
if got := catalogueManifest(t, "postgres").Guards; !reflect.DeepEqual(got, []int{5432}) {
|
||||
t.Errorf("postgres guards %v; the store's port must be refused from outside", got)
|
||||
}
|
||||
// The bus's monitoring port, not its client port: a node reaches the bus, nobody outside
|
||||
// reads its state (novox/hq ADR 0131 — the broker that guarded 15672 has left the catalogue).
|
||||
if got := catalogueManifest(t, "nats").Guards; !reflect.DeepEqual(got, []int{8222}) {
|
||||
t.Errorf("nats guards %v; the monitoring port must be refused from outside", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheFilterModuleNeverFlushesTheRuleset(t *testing.T) {
|
||||
m := catalogueManifest(t, "nftables")
|
||||
var unit, stock, load map[string]any
|
||||
for _, r := range m.Resources {
|
||||
switch r["id"] {
|
||||
case "unit":
|
||||
unit = r
|
||||
case "stock-unit-stop":
|
||||
stock = r
|
||||
case "load":
|
||||
load = r
|
||||
}
|
||||
}
|
||||
if load == nil || load["unit"] != "mesh-filter.service" {
|
||||
t.Fatalf("the filter is not loaded by its own unit: %v", load)
|
||||
}
|
||||
content, _ := unit["content"].(string)
|
||||
if unit == nil || unit["path"] != "/etc/systemd/system/mesh-filter.service" {
|
||||
t.Fatalf("the filter's unit is not written: %v", unit)
|
||||
}
|
||||
if strings.Contains(content, "flush") {
|
||||
t.Fatalf("stopping the filter flushes the whole ruleset — the runtime's and the found "+
|
||||
"firewall's with it:\n%s", content)
|
||||
}
|
||||
if !strings.Contains(content, "ExecStop=nft delete table inet mesh\n") ||
|
||||
!strings.Contains(content, "ExecStart=nft -f "+m.Filtering.Into+"\n") {
|
||||
t.Fatalf("the unit does not load the computed rule set and delete only its own table:\n%s",
|
||||
content)
|
||||
}
|
||||
// A node converged before the filter had its own unit still has the stock nftables.service
|
||||
// enabled, whose stop flushes the whole ruleset: a drop-in makes it delete only the mesh's
|
||||
// table, and the load is restarted on it so the host reloads units and the drop-in is read.
|
||||
if stock == nil || stock["path"] != "/etc/systemd/system/nftables.service.d/mesh.conf" ||
|
||||
!strings.HasSuffix(fmt.Sprint(stock["content"]),
|
||||
"[Service]\nExecStop=\nExecStop=nft delete table inet mesh\n") {
|
||||
t.Fatalf("the stock unit's stop is not replaced with deleting the mesh's table: %v", stock)
|
||||
}
|
||||
// A changed rule set is RELOADED — ExecReload replaces the table in one `nft -f`, so the node
|
||||
// is never unfiltered — and only the units themselves restart it, which is the one change a
|
||||
// reload cannot carry.
|
||||
if !reflect.DeepEqual(load["reload-on"], []any{"filtering"}) {
|
||||
t.Fatalf("the filter is restarted rather than reloaded when its rules change, leaving the "+
|
||||
"node unfiltered in between: %v", load)
|
||||
}
|
||||
if !reflect.DeepEqual(load["restart-on"], []any{"unit", "stock-unit-stop"}) {
|
||||
t.Fatalf("the filter is not restarted when its unit or the stock unit's drop-in changes: %v",
|
||||
load["restart-on"])
|
||||
}
|
||||
}
|
||||
|
||||
// The package registry's port is the node's, like every other foundation port (novox/hq
|
||||
// 04-ISSUES/085, ADR 0100). The forge is reached through what it says it serves, and consumers —
|
||||
// the builder among them — are told that, rather than carrying a number of their own.
|
||||
|
||||
func TestTheForgesPortIsGivenLikeAnyOtherProvidersPort(t *testing.T) {
|
||||
forge := catalogueManifest(t, "gitea")
|
||||
|
||||
// The catalogue's number is a default and the node's setting moves it.
|
||||
given, err := GivenPorts(forge, []Layer{{From: "anchor",
|
||||
Values: map[string]any{PortsSetting: map[string]any{"3000": float64(3100)}}}})
|
||||
if err != nil {
|
||||
t.Fatalf("the forge's port cannot be given on a node: %v", err)
|
||||
}
|
||||
if given[3000] != 3100 {
|
||||
t.Fatalf("the forge was given %v", given)
|
||||
}
|
||||
|
||||
// And every consumer of the package registry is told where the machine actually put it,
|
||||
// because that is read from what the forge serves rather than written in the consumer.
|
||||
if got := ServedOn(forge, "npm-package-registry", given)["port"]; got != 3100 {
|
||||
t.Errorf("the package registry is served on %v, not the port this node gave it", got)
|
||||
}
|
||||
if got := ServedOn(forge, "npm-package-registry", nil)["port"]; got != float64(3000) {
|
||||
t.Errorf("without a setting the forge serves %v, not the catalogue's port", got)
|
||||
}
|
||||
// And so is where a repository on it is cloned from (novox/hq ADR 0111), for the same reason:
|
||||
// a build composes the URL from what the forge serves, so a given port is a followed port.
|
||||
if got := ServedOn(forge, "git", given)["port"]; got != 3100 {
|
||||
t.Errorf("git is served on %v, not the port this node gave the forge", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **The builder requires the registry the npm seat delivers, and carries no binding of its own.**
|
||||
//
|
||||
// It used to carry a hand-written binding because nothing provided a package registry to resolve
|
||||
// one from at genesis. The catalogue now requires it like any consumer, and ADR 0110 makes the
|
||||
// seat's holder the answer when more than one module provides it — so a carried copy would be a
|
||||
// second answer to the same question, free to drift from the first. Asserted gone, not merely
|
||||
// unused.
|
||||
func TestTheBuilderRequiresTheRegistryTheNpmSeatDelivers(t *testing.T) {
|
||||
builder := catalogueManifest(t, "builder")
|
||||
seat, _ := SeatNamed("npm-package-registry")
|
||||
var requires bool
|
||||
for _, r := range builder.Requires {
|
||||
requires = requires || r == seat.Delivers
|
||||
}
|
||||
if !requires {
|
||||
t.Fatalf("the builder does not require %q: %v", seat.Delivers, builder.Requires)
|
||||
}
|
||||
if builder.Binds[seat.Delivers] == "" {
|
||||
t.Errorf("the builder is not told where the registry is: binds %v", builder.Binds)
|
||||
}
|
||||
for _, r := range builder.Resources {
|
||||
if fmt.Sprint(r["id"]) == "package-binding" {
|
||||
t.Fatal("the builder carries its own package binding beside the one the mesh resolves")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The forge holds the seats it answers for (novox/hq ADR 0110, 0111), parsed by the real parser —
|
||||
// which refuses a delivering seat claimed by a module that does not provide what it delivers.
|
||||
func TestTheForgeHoldsTheNpmAndGitSeats(t *testing.T) {
|
||||
forge := catalogueManifest(t, "gitea")
|
||||
holds := map[string]bool{}
|
||||
for _, c := range forge.Claims {
|
||||
holds[c.Name] = true
|
||||
}
|
||||
for _, seat := range []string{"npm-package-registry", "git"} {
|
||||
if !holds[seat] {
|
||||
t.Errorf("gitea does not claim the %s seat: %+v", seat, forge.Claims)
|
||||
}
|
||||
}
|
||||
git := ServedOn(forge, "git", nil)
|
||||
if git["scheme"] != "http" || git["port"] != float64(3000) {
|
||||
t.Errorf("gitea serves nothing a clone URL can be composed from: %v", git)
|
||||
}
|
||||
npm := ServedOn(forge, "npm-package-registry", nil)
|
||||
if npm["npm-path"] != "/api/packages/novox/npm/" {
|
||||
t.Errorf("gitea no longer says where its npm registry is: %v", npm)
|
||||
}
|
||||
}
|
||||
|
||||
// **And the forge's own address follows it**, composed from the manifest in the catalogue beside
|
||||
// this checkout (novox/hq 04-ISSUES/088).
|
||||
//
|
||||
// The forge is reached a third way that neither test above covers: by its own sidecar, over the
|
||||
// machine's loopback, told where to go in its environment. The `2999:3000` mapping that lets the
|
||||
// forge go on binding 3000 does nothing for a caller dialling the machine — so a literal there is
|
||||
// wrong on every node whose assignment differs, and wrong for a second reason on a node given the
|
||||
// port (ADR 0100). Composed through the whole path, because what proves the placeholder resolves
|
||||
// in an `env` at all is a declaration, not a manifest.
|
||||
func TestTheForgesOwnAddressFollowsThePortTheNodeGaveIt(t *testing.T) {
|
||||
forge, err := catalogueManifest(t, "gitea").Resolve([]Built{{
|
||||
Name: "runtime", Kind: ArtifactImage,
|
||||
Reference: "registry.example/gitea-runtime@sha256:" + strings.Repeat("a", 64),
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("the forge's manifest does not resolve against its own build: %v", err)
|
||||
}
|
||||
r := Resolution{Node: "anchor", Modules: []Manifest{forge}, Needs: []Needed{
|
||||
{Name: "postgres-database", For: "gitea", From: "anchor", At: "127.0.0.1",
|
||||
Serves: map[string]any{"port": float64(5432)}, Sealed: "sealed-db"},
|
||||
{Name: "route", For: "gitea", From: "anchor"},
|
||||
{Name: "secret", For: "gitea", From: "anchor", Local: "internal-token", Sealed: "sealed-token"},
|
||||
{Name: "secret", For: "gitea", From: "anchor", Local: "admin", Sealed: "sealed-admin"},
|
||||
}}
|
||||
|
||||
// The number this node was given for the forge — the one the machine it is about to run on
|
||||
// already publishes.
|
||||
out, err := r.Declaration(Rendering{
|
||||
Needed: map[string]map[string]string{"gitea": {"broker": "sealed-broker"}},
|
||||
Given: map[string]map[int]int{"gitea": {3000: 2999}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("the forge does not compose: %v", err)
|
||||
}
|
||||
|
||||
// What the machine publishes, and what the forge's sidecar is told to dial: one number.
|
||||
server := fileNamed(out, "gitea.server")
|
||||
if server == nil {
|
||||
t.Fatalf("the forge's own container is not in the declaration: %v", out)
|
||||
}
|
||||
if published := fmt.Sprint(server["ports"]); !strings.Contains(published, "2999:3000") {
|
||||
t.Fatalf("the forge is not published on the port this node gave it: %v", server["ports"])
|
||||
}
|
||||
runtime := fileNamed(out, "gitea.runtime")
|
||||
if runtime == nil {
|
||||
t.Fatalf("the forge's sidecar is not in the declaration: %v", out)
|
||||
}
|
||||
env, _ := runtime["env"].(map[string]any)
|
||||
if env["MESH_GITEA_URL"] != "http://127.0.0.1:2999" {
|
||||
t.Fatalf("the forge's sidecar dials %v while the machine publishes the forge on 2999 — "+
|
||||
"whatever reads it dials a dead port", env["MESH_GITEA_URL"])
|
||||
}
|
||||
}
|
||||
|
||||
// gitea's own sshd is unmodified — the module's own internal port is 22, the number in
|
||||
// `listens`, the same convention every other module in the catalogue uses (its internal port,
|
||||
// not an invented identity). Composed from the manifest in the catalogue beside this checkout,
|
||||
// because what the mesh publishes is a fact about what the module actually writes.
|
||||
func declaredGiteaSsh(t *testing.T, given map[int]int) map[string]any {
|
||||
t.Helper()
|
||||
forge := catalogueManifest(t, "gitea")
|
||||
resolved, err := forge.Resolve([]Built{{
|
||||
Name: "runtime", Kind: ArtifactImage,
|
||||
Reference: "registry.example/gitea-runtime@sha256:" + strings.Repeat("a", 64),
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("the forge's manifest does not resolve against its own build: %v", err)
|
||||
}
|
||||
r := Resolution{Node: "anchor", Modules: []Manifest{resolved}, Needs: []Needed{
|
||||
{Name: "postgres-database", For: "gitea", From: "anchor", At: "127.0.0.1",
|
||||
Serves: map[string]any{"port": float64(5432)}, Sealed: "sealed-db"},
|
||||
{Name: "route", For: "gitea", From: "anchor"},
|
||||
{Name: "secret", For: "gitea", From: "anchor", Local: "internal-token", Sealed: "sealed-token"},
|
||||
{Name: "secret", For: "gitea", From: "anchor", Local: "admin", Sealed: "sealed-admin"},
|
||||
}}
|
||||
givenPorts := map[int]int{3000: 3000}
|
||||
for k, v := range given {
|
||||
givenPorts[k] = v
|
||||
}
|
||||
out, err := r.Declaration(Rendering{
|
||||
Needed: map[string]map[string]string{"gitea": {"broker": "sealed-broker"}},
|
||||
Ports: map[string]map[int]int{"gitea": givenPorts},
|
||||
Given: map[string]map[int]int{"gitea": given},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("the forge does not compose: %v", err)
|
||||
}
|
||||
server := fileNamed(out, "gitea.server")
|
||||
if server == nil {
|
||||
t.Fatalf("the forge's own container is not in the declaration: %v", out)
|
||||
}
|
||||
return server
|
||||
}
|
||||
|
||||
// **The forge publishes ssh at the mesh's own fixed convention by default** (novox/hq ADR 0100).
|
||||
//
|
||||
// `222` is the mesh's own public convention for the forge's ssh, written directly in the
|
||||
// manifest's `ports` — every node the forge has run on used the same number, so it needs no
|
||||
// per-node setting to reach it.
|
||||
func TestTheForgesSshPortIsTheMeshsFixedConventionByDefault(t *testing.T) {
|
||||
forge := catalogueManifest(t, "gitea")
|
||||
// Nothing was given — no node moved this port — which is the ordinary answer: the mesh only
|
||||
// reports what a setting moved, and the manifest's own `222:22` needs no move to be reached.
|
||||
given, err := GivenPorts(forge, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("the forge's ssh port cannot be given on a node: %v", err)
|
||||
}
|
||||
if len(given) != 0 {
|
||||
t.Fatalf("nothing moved the forge's ssh port, yet it was given %v", given)
|
||||
}
|
||||
server := declaredGiteaSsh(t, given)
|
||||
if published := fmt.Sprint(server["ports"]); !strings.Contains(published, "222:22") {
|
||||
t.Fatalf("the forge is published on %v, not its own fixed convention", server["ports"])
|
||||
}
|
||||
}
|
||||
|
||||
// **A node whose predecessor served git on a different number can still be told to leave it
|
||||
// there.** The setting names the port the module itself listens on — 22, gitea's own sshd, the
|
||||
// same number `listens` uses — not the mesh's own default machine-side number, so moving it does
|
||||
// not require guessing what the manifest happens to default to.
|
||||
func TestANodeMayGiveTheForgesSshPortADifferentNumber(t *testing.T) {
|
||||
forge := catalogueManifest(t, "gitea")
|
||||
given, err := GivenPorts(forge, []Layer{{From: "anchor",
|
||||
Values: map[string]any{PortsSetting: map[string]any{"22": float64(9022)}}}})
|
||||
if err != nil {
|
||||
t.Fatalf("the forge's ssh port cannot be moved on a node: %v", err)
|
||||
}
|
||||
if want := map[int]int{22: 9022}; !reflect.DeepEqual(given, want) {
|
||||
t.Fatalf("the forge was given %v, and the setting named %v", given, want)
|
||||
}
|
||||
server := declaredGiteaSsh(t, given)
|
||||
if published := fmt.Sprint(server["ports"]); !strings.Contains(published, "9022:22") ||
|
||||
strings.Contains(published, "222:22") {
|
||||
t.Fatalf("the forge is published on %v, not the port this node gave it", server["ports"])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// **A seat's holder on record settles who holds it, and lets the next holder stand beside the
|
||||
// current one** (novox/hq ADR 0131, design 28 task 5.3). Until the record existed, two assignments
|
||||
// whose modules both claimed a seat were refused outright — which left no way to hand a seat over
|
||||
// without a moment where nobody held it, and the control plane finds its own bus through one of
|
||||
// these seats. That moment was the outage of 2026-09-27.
|
||||
|
||||
func busSeatDelivering(t *testing.T, delivers string) {
|
||||
t.Helper()
|
||||
was := Seats()
|
||||
t.Cleanup(func() { UseSeats(was) })
|
||||
UseSeats([]Seat{{Name: "mesh-broker", Scope: ScopeMesh, Delivers: delivers, Decision: "test"}})
|
||||
}
|
||||
|
||||
func oldBroker() Manifest {
|
||||
return Manifest{Module: "old-broker", Provides: []Offer{{Name: "mesh-bus", Scope: ScopeMesh}},
|
||||
Claims: []Claim{{Name: "mesh-broker", Scope: ScopeMesh}}}
|
||||
}
|
||||
|
||||
func newBroker() Manifest {
|
||||
return Manifest{Module: "new-broker", Provides: []Offer{{Name: "mesh-bus", Scope: ScopeMesh}},
|
||||
Claims: []Claim{{Name: "mesh-broker", Scope: ScopeMesh}}}
|
||||
}
|
||||
|
||||
// Nothing on record: exactly the old rule. One claimant holds; two are refused.
|
||||
func TestWithNoHolderOnRecordTheSoleClaimantHoldsAndTwoAreRefused(t *testing.T) {
|
||||
busSeatDelivering(t, "mesh-bus")
|
||||
node := Node{Name: "anchor"}
|
||||
|
||||
held, problems := checkClaims([]Manifest{oldBroker()}, node, nil, nil)
|
||||
if len(problems) != 0 || len(held) != 1 || held[0].Module != "old-broker" {
|
||||
t.Fatalf("a sole claimant did not hold the seat: held=%v problems=%v", held, problems)
|
||||
}
|
||||
_, problems = checkClaims([]Manifest{oldBroker(), newBroker()}, node, nil, nil)
|
||||
if len(problems) != 1 || !strings.Contains(problems[0], "both claim") {
|
||||
t.Fatalf("two claimants with nothing on record were not refused: %v", problems)
|
||||
}
|
||||
}
|
||||
|
||||
// With a holder on record, the other eligible assignment is silent: not refused, and not holding.
|
||||
func TestTheHolderOnRecordHoldsAndTheOtherClaimantStandsBesideIt(t *testing.T) {
|
||||
busSeatDelivering(t, "mesh-bus")
|
||||
node := Node{Name: "anchor"}
|
||||
record := []Held{{Claim: "mesh-broker", Scope: ScopeMesh, Node: "anchor", Module: "new-broker"}}
|
||||
|
||||
held, problems := checkClaims([]Manifest{oldBroker(), newBroker()}, node, nil, record)
|
||||
if len(problems) != 0 {
|
||||
t.Fatalf("the assignment beside the holder was refused: %v", problems)
|
||||
}
|
||||
if len(held) != 1 || held[0].Module != "new-broker" {
|
||||
t.Fatalf("the holder on record is not the one holding: %v", held)
|
||||
}
|
||||
}
|
||||
|
||||
// The record names a node too: an eligible module on another machine holds nothing, and its
|
||||
// machine's set still resolves.
|
||||
func TestAHolderOnRecordElsewhereLeavesThisMachinesClaimantSilent(t *testing.T) {
|
||||
busSeatDelivering(t, "mesh-bus")
|
||||
record := []Held{{Claim: "mesh-broker", Scope: ScopeMesh, Node: "anchor", Module: "new-broker"}}
|
||||
|
||||
held, problems := checkClaims([]Manifest{oldBroker()}, Node{Name: "laptop"}, nil, record)
|
||||
if len(problems) != 0 || len(held) != 0 {
|
||||
t.Fatalf("a claimant elsewhere than the recorded holder was not simply silent: held=%v problems=%v",
|
||||
held, problems)
|
||||
}
|
||||
}
|
||||
|
||||
// A record naming a seat's former name still applies to it after a rename (ADR 0122).
|
||||
func TestAHolderRecordedUnderAFormerNameStillHolds(t *testing.T) {
|
||||
busSeatDelivering(t, "mesh-bus")
|
||||
wasAliases := aliases
|
||||
t.Cleanup(func() { UseAliases(wasAliases) })
|
||||
UseAliases(map[string]string{"the-broker": "mesh-broker"})
|
||||
record := []Held{{Claim: "the-broker", Scope: ScopeMesh, Node: "anchor", Module: "new-broker"}}
|
||||
|
||||
held, problems := checkClaims([]Manifest{oldBroker(), newBroker()}, Node{Name: "anchor"}, nil, record)
|
||||
if len(problems) != 0 || len(held) != 1 || held[0].Module != "new-broker" {
|
||||
t.Fatalf("a record under the former name did not settle the seat: held=%v problems=%v", held, problems)
|
||||
}
|
||||
}
|
||||
|
||||
// CanHold is the one judgement registration and the handover share, against the store's row.
|
||||
func TestCanHoldJudgesClaimScopeAndWhatTheSeatDelivers(t *testing.T) {
|
||||
busSeatDelivering(t, "mesh-bus")
|
||||
seat, _ := SeatNamed("mesh-broker")
|
||||
|
||||
if err := CanHold(newBroker(), seat); err != nil {
|
||||
t.Fatalf("a module that claims the seat and provides what it delivers was refused: %v", err)
|
||||
}
|
||||
noClaim := Manifest{Module: "quiet", Provides: []Offer{{Name: "mesh-bus", Scope: ScopeMesh}}}
|
||||
if err := CanHold(noClaim, seat); err == nil || !strings.Contains(err.Error(), "does not claim") {
|
||||
t.Fatalf("a module that never claimed the seat was allowed to hold it: %v", err)
|
||||
}
|
||||
wrongScope := newBroker()
|
||||
wrongScope.Claims[0].Scope = ScopeNode
|
||||
if err := CanHold(wrongScope, seat); err == nil || !strings.Contains(err.Error(), "scope") {
|
||||
t.Fatalf("a claim at the wrong scope was allowed: %v", err)
|
||||
}
|
||||
cannotAnswer := Manifest{Module: "amqp-only", Provides: []Offer{{Name: "amqp", Scope: ScopeMesh}},
|
||||
Claims: []Claim{{Name: "mesh-broker", Scope: ScopeMesh}}}
|
||||
if err := CanHold(cannotAnswer, seat); err == nil || !strings.Contains(err.Error(), `does not provide "mesh-bus"`) {
|
||||
t.Fatalf("a holder that cannot answer for the seat was allowed: %v", err)
|
||||
}
|
||||
// And the judgement follows the store's row, not a compiled copy.
|
||||
busSeatDelivering(t, "amqp")
|
||||
seat, _ = SeatNamed("mesh-broker")
|
||||
if err := CanHold(cannotAnswer, seat); err != nil {
|
||||
t.Fatalf("with the row saying amqp, an amqp provider was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine being moved is handed its membership for the new bus as a sealed file in its own
|
||||
// declaration — the machine's, not any module's (design 28, task 5.2).
|
||||
func TestAMembershipForTheNewBusIsComposedAsASealedFile(t *testing.T) {
|
||||
r := Resolution{Node: "anchor"}
|
||||
got, err := r.Compose(Rendering{BusMembership: "sealed-blob"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var found map[string]any
|
||||
for _, res := range got.Resources {
|
||||
if res["id"] == BusMembershipID() {
|
||||
found = res
|
||||
}
|
||||
}
|
||||
if found == nil {
|
||||
t.Fatalf("no membership resource in %v", got.Resources)
|
||||
}
|
||||
if found["path"] != BusMembershipPath || found["sealed"] != "sealed-blob" || found["mode"] != "0600" {
|
||||
t.Fatalf("the membership is not a sealed 0600 file where the host reads it: %v", found)
|
||||
}
|
||||
// And a machine not being moved is handed nothing.
|
||||
got, _ = r.Compose(Rendering{})
|
||||
for _, res := range got.Resources {
|
||||
if res["id"] == BusMembershipID() {
|
||||
t.Fatal("a machine with no membership on record was handed one")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The store's seat rows have no protocol columns yet; loading them must not drop the protocol the
|
||||
// bus is derived from, or no role's work queue is ever raised (found live, 2026-09-28).
|
||||
func TestAStoreRowWithoutAProtocolKeepsTheCompiledOne(t *testing.T) {
|
||||
was := Seats()
|
||||
t.Cleanup(func() { UseSeats(was) })
|
||||
UseSeats([]Seat{{Name: "mesh-build-machine", Scope: ScopeMesh, Decision: "row"}})
|
||||
got, ok := SeatNamed("mesh-build-machine")
|
||||
if !ok || len(got.Accepts) == 0 {
|
||||
t.Fatalf("the build machine's seat lost what it accepts when loaded from the store: %+v", got)
|
||||
}
|
||||
if got.Decision != "row" {
|
||||
t.Fatalf("the store's own columns were not kept: %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package catalogue_test
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// novox/hq issue 128, held where the host will see it: the shipped networking module's names,
|
||||
// through the whole composition, and not only through FactsInto.
|
||||
//
|
||||
// Composition prefixes a fact's id with the module that asked for it and passes everything else
|
||||
// through; a step that dropped `into` on the way would send the region as a whole file, and the
|
||||
// host would write the machine's hosts file over again with every unit test above still green.
|
||||
|
||||
// onTheNetwork stands in for the overlay's generator: the node is part of the private network,
|
||||
// and what the generator writes is not what is under test here.
|
||||
type onTheNetwork struct{}
|
||||
|
||||
func (onTheNetwork) Resources(string) ([]map[string]any, bool, error) {
|
||||
return []map[string]any{{"id": "overlay-config", "type": "file",
|
||||
"path": "/etc/wireguard/mesh0.conf", "mode": "0600", "content": "[Interface]\n"}}, true, nil
|
||||
}
|
||||
|
||||
func TestTheHostsRegionArrivesAsTheHostWillReadIt(t *testing.T) {
|
||||
shelf := provided(t)
|
||||
// A resolver restarting on the names another module put on the machine, and one resource it
|
||||
// only runs at start — neither of which composition has any business changing.
|
||||
resolver, err := catalogue.ParseManifest([]byte(`{
|
||||
"module": "resolver", "version": "1", "requires": ["mesh-addressing"],
|
||||
"resources": [
|
||||
{"id": "seed", "type": "file", "path": "/etc/resolver/seed", "mode": "0644",
|
||||
"content": "seed\n", "at": "start"},
|
||||
{"id": "daemon", "type": "service", "unit": "resolver.service", "state": "running",
|
||||
"restart-on": ["seed", "mesh-wireguard.fact-node-names"]}
|
||||
]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
shelf[resolver.Module] = resolver
|
||||
|
||||
got, err := catalogue.Resolve(shelf, []string{overlay.Domain, "resolver"},
|
||||
catalogue.Node{Name: "homer", At: "homer.internal"}, catalogue.World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
names := map[string]string{"homer.internal": "10.42.0.1", "marge.internal": "10.42.0.2"}
|
||||
out, err := got.Declaration(catalogue.Rendering{
|
||||
Names: names, Machines: names, Suffix: "internal",
|
||||
Generators: map[string]catalogue.Generator{overlay.Name: onTheNetwork{}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ids := map[string]map[string]any{}
|
||||
for _, r := range out {
|
||||
ids[r["id"].(string)] = r
|
||||
}
|
||||
|
||||
hosts := ids[overlay.Name+".fact-node-names"]
|
||||
if hosts == nil {
|
||||
t.Fatalf("no names reached the machine; the declaration has %v", keys(ids))
|
||||
}
|
||||
if hosts["path"] != "/etc/hosts" || hosts["into"] != "block" {
|
||||
t.Fatalf("the hosts file is not written into as a region: %v", hosts)
|
||||
}
|
||||
content := hosts["content"].(string)
|
||||
if !strings.Contains(content, "10.42.0.1\thomer.internal\thomer\t# this machine\n") {
|
||||
t.Errorf("the region does not name the machine:\n%s", content)
|
||||
}
|
||||
for _, floor := range []string{"Generated by the mesh", "localhost", "127.0.1.1"} {
|
||||
if strings.Contains(content, floor) {
|
||||
t.Errorf("the region carries %q, which is the machine's:\n%s", floor, content)
|
||||
}
|
||||
}
|
||||
|
||||
// The resolver's reference to it still names a resource the host will be sent.
|
||||
daemon := ids["resolver.daemon"]
|
||||
if daemon == nil {
|
||||
t.Fatalf("the resolver's service was not composed: %v", keys(ids))
|
||||
}
|
||||
for _, named := range daemon["restart-on"].([]any) {
|
||||
if ids[named.(string)] == nil {
|
||||
t.Errorf("the resolver restarts on %v, which is nothing the host is sent", named)
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(daemon["restart-on"], []any{"resolver.seed", overlay.Name + ".fact-node-names"}) {
|
||||
t.Errorf("restart-on is %v", daemon["restart-on"])
|
||||
}
|
||||
|
||||
// And a resource's own `at` passes through as the manifest wrote it.
|
||||
if seed := ids["resolver.seed"]; seed == nil || seed["at"] != "start" {
|
||||
t.Errorf("a resource's at did not survive composition: %v", seed)
|
||||
}
|
||||
}
|
||||
|
||||
func keys(m map[string]map[string]any) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// A node's fail2ban jails, composed from the modules it runs (novox/hq to-be 31).
|
||||
//
|
||||
// **The same shape as the firewall.** Every module's `listens` become the node's rule set; every
|
||||
// module's `jails` become the node's fail2ban config. A module that runs an authenticating service
|
||||
// declares what a break-in on it looks like and how to ban it, naming no node and no path (ADR
|
||||
// 0112); the intrusion-prevention holder — the one module with `jailing` — gathers them and writes
|
||||
// them where it owns. A node not running a module has none of its jails.
|
||||
|
||||
// jailsInto composes every jail declared by the modules on a node into the files the holder writes:
|
||||
// one jail file (all stanzas, so the fail2ban service restarts on a single resource) and one filter
|
||||
// file per jail (its failregex, which fail2ban references by the jail's name).
|
||||
//
|
||||
// Owned by the holder, because the directory is: two modules writing into one fail2ban is the
|
||||
// collision the holder model exists to prevent. Empty when nothing declares a jail — then the file
|
||||
// is written empty rather than absent, so removing the last jail is an ordinary change the service
|
||||
// restarts on rather than a file that vanishes.
|
||||
func jailsInto(modules []Manifest, j *Jailing) []map[string]any {
|
||||
type declared struct {
|
||||
module string
|
||||
jail Jail
|
||||
}
|
||||
var jails []declared
|
||||
for _, m := range modules {
|
||||
for _, jail := range m.Jails {
|
||||
jails = append(jails, declared{m.Module, jail})
|
||||
}
|
||||
}
|
||||
// A stable order the host applies as given (ADR 0005), and so the same set composes byte for
|
||||
// byte every time rather than differing by map iteration.
|
||||
sort.Slice(jails, func(a, b int) bool { return jails[a].jail.Name < jails[b].jail.Name })
|
||||
|
||||
var composed strings.Builder
|
||||
composed.WriteString("# The mesh's jails, composed from the modules this node runs. Do not edit —\n")
|
||||
composed.WriteString("# replaced whenever the node's modules change (novox/hq to-be 31).\n")
|
||||
|
||||
out := make([]map[string]any, 0, len(jails)+1)
|
||||
for _, d := range jails {
|
||||
fmt.Fprintf(&composed, "\n# from %s\n[%s]\nenabled = true\nfilter = %s\n%s\n",
|
||||
d.module, d.jail.Name, d.jail.Name, strings.TrimRight(d.jail.Jail, "\n"))
|
||||
// The filter is a file of its own, named as the jail's filter= references it.
|
||||
out = append(out, map[string]any{
|
||||
"id": "filter-" + d.jail.Name,
|
||||
"type": "file", "path": strings.TrimRight(j.FilterInto, "/") + "/" + d.jail.Name + ".conf",
|
||||
"mode": "0644",
|
||||
"content": "# Generated by the mesh (from module " + d.module + "). Do not edit.\n" +
|
||||
"[Definition]\nfailregex = " + d.jail.Failregex + "\n",
|
||||
})
|
||||
}
|
||||
// The one jail file, first, with the fixed id the fail2ban service names in its restart-on.
|
||||
return append([]map[string]any{{
|
||||
"id": ComposedJailsID(), "type": "file", "path": j.Into, "mode": "0644",
|
||||
"content": composed.String(),
|
||||
}}, out...)
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A node's fail2ban jails are composed from the modules it runs (novox/hq to-be 31): the holder
|
||||
// (jailing) gathers every module's declared jail into one jail file and a filter file per jail.
|
||||
func TestJailsAreComposedFromTheNodesModules(t *testing.T) {
|
||||
modules := []Manifest{
|
||||
{Module: "fail2ban", Jailing: &Jailing{Into: "/etc/fail2ban/jail.d/mesh-composed.conf", FilterInto: "/etc/fail2ban/filter.d"}},
|
||||
{Module: "postgres", Jails: []Jail{{Name: "postgres-auth", Failregex: "auth failed from <HOST>", Jail: "port = 5432\nmaxretry = 5"}}},
|
||||
}
|
||||
files := jailsInto(modules, modules[0].Jailing)
|
||||
|
||||
by := map[string]map[string]any{}
|
||||
for _, f := range files {
|
||||
by[f["id"].(string)] = f
|
||||
}
|
||||
jail := by[ComposedJailsID()]
|
||||
if jail == nil || jail["path"] != "/etc/fail2ban/jail.d/mesh-composed.conf" {
|
||||
t.Fatalf("the composed jail file was not written: %v", jail)
|
||||
}
|
||||
body := jail["content"].(string)
|
||||
if !strings.Contains(body, "[postgres-auth]") || !strings.Contains(body, "filter = postgres-auth") ||
|
||||
!strings.Contains(body, "port = 5432") {
|
||||
t.Fatalf("the postgres jail stanza was not composed in:\n%s", body)
|
||||
}
|
||||
filter := by["filter-postgres-auth"]
|
||||
if filter == nil || filter["path"] != "/etc/fail2ban/filter.d/postgres-auth.conf" {
|
||||
t.Fatalf("the jail's filter file was not written: %v", filter)
|
||||
}
|
||||
if !strings.Contains(filter["content"].(string), "failregex = auth failed from <HOST>") {
|
||||
t.Fatalf("the failregex was not written: %v", filter["content"])
|
||||
}
|
||||
}
|
||||
|
||||
// A holder whose node runs no jail-declaring module still gets the file, empty — so removing the
|
||||
// last jail is a change the service restarts on, not a file that vanishes.
|
||||
func TestTheComposedJailFileIsWrittenEvenWhenEmpty(t *testing.T) {
|
||||
files := jailsInto([]Manifest{{Module: "fail2ban"}}, &Jailing{Into: "/x", FilterInto: "/f"})
|
||||
if len(files) != 1 || files[0]["id"] != ComposedJailsID() {
|
||||
t.Fatalf("the empty composed jail file was not written alone: %v", files)
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user