Compare commits
67
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
497f8ea567 | ||
|
|
dded086b54 | ||
|
|
e7b5100324 | ||
|
|
8ceec32692 | ||
|
|
5fcde512bc | ||
|
|
e5007a7daa | ||
|
|
cb77f35a27 | ||
|
|
71dbca6392 | ||
|
|
47d412e13f | ||
|
|
53e8f5bdd8 | ||
|
|
6ecd631b3e | ||
|
|
19d2725c13 | ||
|
|
8e2824201a | ||
|
|
75dab209d7 | ||
|
|
6da9a5478b | ||
|
|
ff51b329f6 | ||
|
|
2ec0fd218b | ||
|
|
e4e960ec1c | ||
|
|
cc69737934 | ||
|
|
0c83ecf1b5 | ||
|
|
05ff6065d0 | ||
|
|
a7df0fc62f | ||
|
|
1c56210530 | ||
|
|
d65c37caad | ||
|
|
eb72ec36ba | ||
|
|
4de10e32e3 | ||
|
|
f8ab9f2dcf | ||
|
|
ccf50b1269 | ||
|
|
a6d89e3f73 | ||
|
|
ee1b8ffe24 | ||
|
|
0560c792d8 | ||
|
|
966c5ddad3 | ||
|
|
b36f822cb6 | ||
|
|
7180a273a2 | ||
|
|
e65b3950cc | ||
|
|
a0e09695e5 | ||
|
|
88bef39952 | ||
|
|
06cf3c04e5 | ||
|
|
7a8a19b11b | ||
|
|
ce6ac057f6 | ||
|
|
abce68fdf0 | ||
|
|
cc019908c2 | ||
|
|
d0a9abcb1c | ||
|
|
63ba2d178f | ||
|
|
6c12780abe | ||
|
|
92d87b0082 | ||
|
|
2fad32767e | ||
|
|
51163b9f14 | ||
|
|
6557aca750 | ||
|
|
1801f1178e | ||
|
|
f21a84c510 | ||
|
|
e14b02991e | ||
|
|
bf82805cfd | ||
|
|
08ebb8c999 | ||
|
|
0a8a592ef4 | ||
|
|
173c8c7c21 | ||
|
|
98d348d5b9 | ||
|
|
7fc5fd02fd | ||
|
|
50878a9d98 | ||
|
|
aa74bd86ca | ||
|
|
7232d6df4b | ||
|
|
112cbd294d | ||
|
|
553814b6eb | ||
|
|
eeb0560fc2 | ||
|
|
1f36787d75 | ||
|
|
c753f9d5c0 | ||
|
|
fb87f9f7d6 |
@@ -86,13 +86,19 @@ proxy-image:
|
|||||||
|
|
||||||
# The whole gate. Raises a database, runs everything against it, and takes it down again --
|
# The whole gate. Raises a database, runs everything against it, and takes it down again --
|
||||||
# including when the tests fail, which is why the teardown is not conditional.
|
# including when the tests fail, which is why the teardown is not conditional.
|
||||||
|
#
|
||||||
|
# **One package at a time (-p 1), and it is not about speed.** The live tests reach one bus, and on
|
||||||
|
# it they assert, read and remove the mesh's own objects -- streams and consumers with fixed names,
|
||||||
|
# because those names are the mesh's and a test cannot choose others. Two packages doing that at once
|
||||||
|
# is one deleting a consumer the other is reading through, and the failure lands in whichever test
|
||||||
|
# was reading, as "no response from stream". That reads as a bug in the code under test.
|
||||||
check: fmt vet postgres
|
check: fmt vet postgres
|
||||||
@go test ./... ; status=$$? ; $(MAKE) postgres-stop ; exit $$status
|
@go test -p 1 ./... ; status=$$? ; $(MAKE) postgres-stop ; exit $$status
|
||||||
|
|
||||||
# Without a database the live tests skip rather than fail, so this is the honest subset and not
|
# Without a database the live tests skip rather than fail, so this is the honest subset and not
|
||||||
# the gate.
|
# the gate. Serialised for the same reason check is: a bus may be configured even when a store is not.
|
||||||
test:
|
test:
|
||||||
go test ./...
|
go test -p 1 ./...
|
||||||
|
|
||||||
vet:
|
vet:
|
||||||
go vet ./...
|
go vet ./...
|
||||||
|
|||||||
+57
-114
@@ -29,10 +29,10 @@ import (
|
|||||||
"os/signal"
|
"os/signal"
|
||||||
"strings"
|
"strings"
|
||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
|
||||||
|
|
||||||
amqp "github.com/rabbitmq/amqp091-go"
|
amqp "github.com/rabbitmq/amqp091-go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
"github.com/novox/mesh-controller/internal/builder"
|
"github.com/novox/mesh-controller/internal/builder"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
)
|
)
|
||||||
@@ -115,72 +115,63 @@ func run() error {
|
|||||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||||
defer stop()
|
defer stop()
|
||||||
|
|
||||||
conn, err := dial(credential)
|
machine, err := takeWorkFrom(credential, on)
|
||||||
if err != nil {
|
|
||||||
// Not quoted back: the URL carries this builder's broker password.
|
|
||||||
return fmt.Errorf("cannot reach the broker: %w", err)
|
|
||||||
}
|
|
||||||
defer conn.Close()
|
|
||||||
channel, err := conn.Channel()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer channel.Close()
|
|
||||||
|
|
||||||
if _, err := channel.QueueDeclare(link.BuildQueue, true, false, false, false, nil); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// One at a time. A build machine that took five requests at once would run five container
|
|
||||||
// builds against one runtime and finish all of them slower than it would have finished the
|
|
||||||
// first — and the queue is what shares work between machines, so nothing is lost by it.
|
|
||||||
if err := channel.Qos(1, 0, false); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Not auto-acknowledged. A request acknowledged on arrival is a build that vanishes if this
|
|
||||||
// process dies mid-way, with nobody waiting on it ever hearing why.
|
|
||||||
requests, err := channel.ConsumeWithContext(ctx, link.BuildQueue, "mesh-builder",
|
|
||||||
false, false, false, false, nil)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
defer machine.Close()
|
||||||
|
|
||||||
fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry)
|
fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry)
|
||||||
publisher := builder.Registry{Address: registry, Run: builder.Command}
|
publisher := builder.Registry{Address: registry, Run: builder.Command}
|
||||||
|
|
||||||
for {
|
return machine.Take(ctx, func(ctx context.Context, work link.Build) {
|
||||||
select {
|
answer(ctx, publisher, on, workspace, work)
|
||||||
case <-ctx.Done():
|
})
|
||||||
fmt.Println("stopping")
|
}
|
||||||
return nil
|
|
||||||
case delivery, ok := <-requests:
|
// takeWorkFrom opens this machine's link to whichever bus the mesh is on.
|
||||||
if !ok {
|
//
|
||||||
return fmt.Errorf("the broker closed the connection")
|
// **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5): a build
|
||||||
}
|
// machine told about both would take work from one and answer on the other, and every log line would
|
||||||
answer(ctx, channel, publisher, on, workspace, delivery)
|
// say it was fine.
|
||||||
}
|
func takeWorkFrom(credential Credential, on string) (link.BuildMachine, error) {
|
||||||
|
address, onNATS, err := broker.OnNATS()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
}
|
}
|
||||||
|
if err := broker.MustBeOneBus(credential.URL, address); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if onNATS {
|
||||||
|
js, err := broker.Dial(address)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot reach the bus at %s: %w", address, err)
|
||||||
|
}
|
||||||
|
return link.MachineOverNATS(js, on), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
conn, err := dial(credential)
|
||||||
|
if err != nil {
|
||||||
|
// Not quoted back: the URL carries this builder's broker password.
|
||||||
|
return nil, fmt.Errorf("cannot reach the broker: %w", err)
|
||||||
|
}
|
||||||
|
channel, err := conn.Channel()
|
||||||
|
if err != nil {
|
||||||
|
conn.Close()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return link.MachineOverCurrent(conn, channel, on), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// answer does one build and says what happened, whichever way it went.
|
// answer does one build and says what happened, whichever way it went.
|
||||||
func answer(ctx context.Context, channel *amqp.Channel, publisher builder.Publisher,
|
func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build) {
|
||||||
on, workspace string, delivery amqp.Delivery) {
|
request := work.Request()
|
||||||
|
|
||||||
// **First thing, and to stdout.** A build request that arrives and produces no visible line
|
// **First thing, and to stdout.** A build request that arrives and produces no visible line until
|
||||||
// until it either finishes or fails is indistinguishable from one that never arrived — which
|
// it either finishes or fails is indistinguishable from one that never arrived — which cost a long
|
||||||
// cost a long diagnosis against a running mesh, chasing "the handler never fired" when the
|
// diagnosis against a running mesh, chasing "the handler never fired" when the truth was only that
|
||||||
// truth was only that the handler said nothing until the end.
|
// the handler said nothing until the end.
|
||||||
fmt.Fprintf(os.Stderr, "a build request arrived (%d bytes)\n", len(delivery.Body))
|
fmt.Fprintf(os.Stderr, "a build request arrived for %s\n", request.Repository)
|
||||||
|
|
||||||
var request link.BuildRequest
|
|
||||||
if err := json.Unmarshal(delivery.Body, &request); err != nil {
|
|
||||||
// Unreadable. Acknowledged and dropped rather than requeued: a message this builder
|
|
||||||
// cannot parse will not become parseable by being delivered again, and requeueing it
|
|
||||||
// would put it in front of every real request for ever.
|
|
||||||
fmt.Fprintf(os.Stderr, "a request could not be read and was dropped: %v\n", err)
|
|
||||||
_ = delivery.Ack(false)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
result := link.BuildResult{
|
result := link.BuildResult{
|
||||||
ID: request.ID, Repository: request.Repository, Path: request.Path,
|
ID: request.ID, Repository: request.Repository, Path: request.Path,
|
||||||
@@ -199,8 +190,8 @@ func answer(ctx context.Context, channel *amqp.Channel, publisher builder.Publis
|
|||||||
var built builder.Result
|
var built builder.Result
|
||||||
if err == nil {
|
if err == nil {
|
||||||
// The package-registry credential is a build input, so it is resolved before the clone: a
|
// The package-registry credential is a build input, so it is resolved before the clone: a
|
||||||
// build that could not have resolved its dependencies is refused in front of the reason,
|
// build that could not have resolved its dependencies is refused in front of the reason, not
|
||||||
// not after a clone that then fails at npm ci.
|
// after a clone that then fails at npm ci.
|
||||||
built, err = builder.Build(ctx, builder.Command, publisher,
|
built, err = builder.Build(ctx, builder.Command, publisher,
|
||||||
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
||||||
forgeFrom(),
|
forgeFrom(),
|
||||||
@@ -230,67 +221,19 @@ func answer(ctx context.Context, channel *amqp.Channel, publisher builder.Publis
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
body, err := json.Marshal(result)
|
if err := work.Announce(ctx, result); err != nil {
|
||||||
if err != nil {
|
// Said, not fatal: the build happened. A build reported as failed because announcing it
|
||||||
fmt.Fprintf(os.Stderr, "cannot report a build: %v\n", err)
|
// failed is a lie about work that was done — and the request stays unsettled below only if
|
||||||
_ = delivery.Ack(false)
|
// nothing was said at all, so another machine can try.
|
||||||
|
fmt.Fprintf(os.Stderr, "cannot say what came of a build: %v\n", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Always through the exchange, whether or not somebody is waiting.
|
// Settled only once the outcome is away, so a machine that dies before answering leaves the work
|
||||||
//
|
// for another rather than losing it.
|
||||||
// **Never the default exchange.** Permission there is granted per exchange rather than per
|
if err := work.Done(); err != nil {
|
||||||
// queue, so a builder allowed to use it could publish into any node's queue — the privilege a
|
fmt.Fprintf(os.Stderr, "the outcome is away and the request could not be settled: %v\n", err)
|
||||||
// build machine most obviously should not have. An asker binds its own reply queue to this
|
|
||||||
// key and filters by correlation; a control plane that records builds is bound to it too, so
|
|
||||||
// a result nobody asked for is still kept rather than reported into the void.
|
|
||||||
publishCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
if err := channel.PublishWithContext(publishCtx, link.Exchange, link.KeyBuilt, false, false,
|
|
||||||
amqp.Publishing{
|
|
||||||
ContentType: "application/json",
|
|
||||||
CorrelationId: result.ID,
|
|
||||||
Body: body,
|
|
||||||
}); err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "cannot answer a build request: %v\n", err)
|
|
||||||
}
|
}
|
||||||
// **And announced, which is a different act from answering.** The reply goes to whoever asked
|
|
||||||
// and is correlated to their request; this says to the whole mesh that a module now exists at
|
|
||||||
// a commit, and the catalogue places it in the module graph (novox/hq ADR 0072). A build
|
|
||||||
// nobody asked for still has to be announced, or the graph knows less than the registry does.
|
|
||||||
//
|
|
||||||
// Only on success: a failed build produced no module-version, and announcing one would put
|
|
||||||
// something in the graph that was never made.
|
|
||||||
if result.Failed == "" && result.Commit != "" {
|
|
||||||
announced := map[string]any{
|
|
||||||
"module": moduleOf(result.Manifest), "commit": result.Commit,
|
|
||||||
"repository": result.Repository, "path": result.Path, "ref": result.Ref,
|
|
||||||
"manifest": json.RawMessage(result.Manifest), "against": result.Against,
|
|
||||||
"made": result.Made,
|
|
||||||
}
|
|
||||||
if err := link.EmitEvent(publishCtx, channel, link.KeyModuleBuilt, "builder", on, announced); err != nil {
|
|
||||||
// Said, not fatal: the build happened and was answered. A module the catalogue has not
|
|
||||||
// heard of is a gap somebody can close; a build reported as failed because announcing
|
|
||||||
// it failed is a lie about work that was done.
|
|
||||||
fmt.Fprintf(os.Stderr, " built, but could not announce it: %v\n", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Acknowledged only once the answer is away, so a builder that dies before answering leaves
|
|
||||||
// the request for another machine rather than losing it.
|
|
||||||
_ = delivery.Ack(false)
|
|
||||||
}
|
|
||||||
|
|
||||||
// moduleOf reads the module's name out of the manifest it just built, which is the only place it is
|
|
||||||
// authoritative — the request named a repository and a path, not a module.
|
|
||||||
func moduleOf(manifest json.RawMessage) string {
|
|
||||||
var named struct {
|
|
||||||
Module string `json:"module"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(manifest, &named); err != nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return named.Module
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// packagesFrom is where a build resolves the mesh's own published packages — the SDK above all
|
// packagesFrom is where a build resolves the mesh's own published packages — the SDK above all
|
||||||
|
|||||||
@@ -395,7 +395,13 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti
|
|||||||
}
|
}
|
||||||
fmt.Println()
|
fmt.Println()
|
||||||
|
|
||||||
result, err := link.RequestBuild(ctx, server.Channel(), request, wait)
|
ask, err := askOver(server)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer ask.Close()
|
||||||
|
|
||||||
|
result, err := ask.Submit(ctx, request, wait)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -472,7 +478,13 @@ func buildAndShow(ctx context.Context, source buildSource, path, ref string, wai
|
|||||||
}
|
}
|
||||||
defer server.Close()
|
defer server.Close()
|
||||||
|
|
||||||
result, err := link.RequestBuild(ctx, server.Channel(), link.BuildRequest{
|
ask, err := askOver(server)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer ask.Close()
|
||||||
|
|
||||||
|
result, err := ask.Submit(ctx, link.BuildRequest{
|
||||||
ID: fmt.Sprintf("%s-%d", "build", time.Now().UnixNano()),
|
ID: fmt.Sprintf("%s-%d", "build", time.Now().UnixNano()),
|
||||||
Repository: repository, Path: path, Ref: ref,
|
Repository: repository, Path: path, Ref: ref,
|
||||||
Held: heldBy(ctx),
|
Held: heldBy(ctx),
|
||||||
@@ -557,3 +569,22 @@ func heldBy(ctx context.Context) map[string]string {
|
|||||||
}
|
}
|
||||||
return routed
|
return routed
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// askOver opens the way a build is asked for, on whichever bus the mesh is on.
|
||||||
|
//
|
||||||
|
// **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5). On the bus
|
||||||
|
// the mesh runs on today this needs the controller's own connection, so it is handed one; on the bus
|
||||||
|
// being built it dials, because a build request is a one-shot and holds nothing else.
|
||||||
|
func askOver(server *link.Server) (link.Builders, error) {
|
||||||
|
address, onNATS, err := broker.OnNATS()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := broker.MustBeOneBus(os.Getenv(broker.AMQPVarName), address); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if onNATS {
|
||||||
|
return link.BuildsOverNATS(address)
|
||||||
|
}
|
||||||
|
return link.BuildsOverCurrent(server.Channel()), nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -114,8 +114,12 @@ func run() error {
|
|||||||
return planCommand(ctx, args[1:])
|
return planCommand(ctx, args[1:])
|
||||||
case "push":
|
case "push":
|
||||||
return pushCommand(ctx, args[1:])
|
return pushCommand(ctx, args[1:])
|
||||||
|
case "rollout":
|
||||||
|
return rolloutCommand(ctx, args[1:])
|
||||||
case "seats":
|
case "seats":
|
||||||
return seatsCommand(ctx, args[1:])
|
return seatsCommand(ctx, args[1:])
|
||||||
|
case "seat":
|
||||||
|
return seatCommand(ctx, args[1:])
|
||||||
case "status":
|
case "status":
|
||||||
return statusCommand(ctx, args[1:])
|
return statusCommand(ctx, args[1:])
|
||||||
case "version":
|
case "version":
|
||||||
|
|||||||
@@ -257,6 +257,21 @@ func moduleCommand(ctx context.Context, args []string) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Which bus this mesh is on. A module gets a credential for exactly one, and the two are
|
||||||
|
// made in entirely different ways: on the bus the mesh runs on today an account is a
|
||||||
|
// management call, and on the bus being built it is a row the next composition writes into
|
||||||
|
// the server's user list (novox/hq design 25 §4).
|
||||||
|
busAddress, onNATS, err := broker.OnNATS()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := broker.MustBeOneBus(os.Getenv(broker.AMQPVarName), busAddress); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if onNATS {
|
||||||
|
return issueOnTheNewBus(ctx, inv, m, *forNode, busAddress)
|
||||||
|
}
|
||||||
|
|
||||||
management, err := broker.ManagementFromEnvironment()
|
management, err := broker.ManagementFromEnvironment()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -567,3 +582,77 @@ func mayIssue(m catalogue.Manifest) error {
|
|||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// issueOnTheNewBus gives an assigned module its credential on the bus being built.
|
||||||
|
//
|
||||||
|
// **Three things differ from a management call, and each is the point of the move.** The credential
|
||||||
|
// is minted into the mesh's records and becomes usable at the next composition, so there is no
|
||||||
|
// server to be reachable for this to work. The password travels beside the address rather than inside
|
||||||
|
// it, because the runtime's contract already separates them and a credential embedded in a URL is one
|
||||||
|
// that leaks into every log line that prints a connection. And the module's durable consumer is
|
||||||
|
// derived from what it declared rather than declared by name, so a module cannot ask for delivery of
|
||||||
|
// something it did not say it consumes.
|
||||||
|
func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest,
|
||||||
|
node, busAddress string) error {
|
||||||
|
|
||||||
|
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: m.Module}.Username()
|
||||||
|
password, err := inv.MintBusPassword(ctx, inventory.BusUser{
|
||||||
|
Username: user, Kind: inventory.BusModule, Node: node, Module: m.Module,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Where the module is told to find the bus, and what certificate it must present. The same pair
|
||||||
|
// a node is told, for the same reason: a mesh's bus presents its own certificate, in no public
|
||||||
|
// trust store, so an address alone fails at TLS.
|
||||||
|
known, err := broker.FromEnvironment()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot deliver a credential without knowing where the bus is: %w", err)
|
||||||
|
}
|
||||||
|
reachable, err := brokerReachableAt(ctx, inv, known, node)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
held, err := json.Marshal(struct {
|
||||||
|
URL string `json:"url"`
|
||||||
|
Fingerprint string `json:"fingerprint,omitempty"`
|
||||||
|
Node string `json:"node"`
|
||||||
|
Module string `json:"module"`
|
||||||
|
User string `json:"user"`
|
||||||
|
Password string `json:"password"`
|
||||||
|
}{
|
||||||
|
URL: "nats://" + reachable, Fingerprint: known.Fingerprint,
|
||||||
|
Node: node, Module: m.Module, User: user, Password: password,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := inv.AcceptSecretForModule(ctx, node, m.Module, "broker", string(held)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// And how it hears what it consumes. Derived from its declaration, and only when it declared
|
||||||
|
// something: a module that consumes nothing needs no consumer, and creating one would be a
|
||||||
|
// durable subscription nobody reads.
|
||||||
|
if consumer, needed := broker.ConsumerFor(broker.Principal{
|
||||||
|
Kind: broker.KindModule, Node: node, Module: m.Module,
|
||||||
|
Emits: m.Emits, Consumes: m.Consumes, Serves: m.Tools,
|
||||||
|
}); needed {
|
||||||
|
js, err := broker.Dial(busAddress)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the credential is minted and the mesh cannot reach the bus to create "+
|
||||||
|
"how %s hears what it consumes: %w", m.Module, err)
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
if err := js.EnsureConsumer(consumer); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("bus user %s minted for %s, scoped to what it emits and consumes\n", user, m.Module)
|
||||||
|
fmt.Printf(" sealed to %s. It arrives with the next push — `push %s` to send it\n", node, node)
|
||||||
|
fmt.Printf(" and it works once the bus has been told: the user list is composed into the " +
|
||||||
|
"machine holding mesh-broker\n")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -256,7 +256,7 @@ func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool,
|
|||||||
"Re-place it — `overlay place %s --hub --endpoint <host>:%d …` — and push again; "+
|
"Re-place it — `overlay place %s --hub --endpoint <host>:%d …` — and push again; "+
|
||||||
"nothing was composed", p.Name, t.Interface, wrong, p.Name, t.Port)
|
"nothing was composed", p.Name, t.Interface, wrong, p.Name, t.Port)
|
||||||
}
|
}
|
||||||
n.TakesOver = &overlay.TakeOver{Interface: t.Interface, Unit: t.Unit, Config: t.Config, Port: t.Port}
|
n.TakesOver = &overlay.TakeOver{Interface: t.Interface, Unit: t.Unit, Config: t.Config, Port: t.Port, MTU: t.MTU}
|
||||||
}
|
}
|
||||||
if p.Hub {
|
if p.Hub {
|
||||||
for _, c := range carried {
|
for _, c := range carried {
|
||||||
|
|||||||
@@ -67,8 +67,14 @@ func nodeCommand(ctx context.Context, args []string) error {
|
|||||||
// because the damage is already done by the time it prints.
|
// because the damage is already done by the time it prints.
|
||||||
return publicDomain(ctx, inv, args[1:])
|
return publicDomain(ctx, inv, args[1:])
|
||||||
|
|
||||||
|
case "account":
|
||||||
|
// The operator's login on this machine (novox/hq to-be 29): what a home-scoped file is
|
||||||
|
// owned by and which account `ssh <node>` uses. Reports with no argument; sets with one;
|
||||||
|
// an optional second argument is the home when it is not /home/<account>.
|
||||||
|
return nodeAccount(ctx, inv, args[1:])
|
||||||
|
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("node has no %q; it has add, list, show and public-domain", args[0])
|
return fmt.Errorf("node has no %q; it has add, list, show, public-domain and account", args[0])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -106,6 +112,39 @@ func modeOf(n inventory.Node) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// publicDomainUsage is the one description of the three forms, so a refusal and the help agree.
|
// publicDomainUsage is the one description of the three forms, so a refusal and the help agree.
|
||||||
|
// nodeAccount reports or sets a node's operator account (novox/hq to-be 29). Read-shaped with no
|
||||||
|
// argument, like public-domain: `node account novox` answers, it does not change anything.
|
||||||
|
func nodeAccount(ctx context.Context, inv *inventory.Inventory, positionals []string) error {
|
||||||
|
if len(positionals) == 0 || len(positionals) > 3 {
|
||||||
|
return errors.New("node account <name> — what it is now; " +
|
||||||
|
"node account <name> <account> [home] — set it (home defaults to /home/<account>)")
|
||||||
|
}
|
||||||
|
node := positionals[0]
|
||||||
|
if len(positionals) == 1 {
|
||||||
|
who, err := inv.NodeByName(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if who.Account == "" {
|
||||||
|
fmt.Printf("%s has no operator account known\n", node)
|
||||||
|
fmt.Printf(" `node account %s <account>` sets it\n", node)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
fmt.Printf("%s logs a person in as %s (home %s)\n", node, who.Account, who.Home())
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
home := ""
|
||||||
|
if len(positionals) == 3 {
|
||||||
|
home = positionals[2]
|
||||||
|
}
|
||||||
|
if err := inv.SetAccount(ctx, node, positionals[1], home); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s logs a person in as %s\n", node, positionals[1])
|
||||||
|
fmt.Printf(" run `push %s` once ssh-client is assigned, to send its operator config\n", node)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
const publicDomainUsage = "node public-domain <name> — what it is now; " +
|
const publicDomainUsage = "node public-domain <name> — what it is now; " +
|
||||||
"<name> <domain> to set it; <name> --clear to take it away"
|
"<name> <domain> to set it; <name> --clear to take it away"
|
||||||
|
|
||||||
|
|||||||
@@ -2,12 +2,15 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/secrets"
|
"github.com/novox/mesh-controller/internal/secrets"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -26,9 +29,25 @@ import (
|
|||||||
// operator key make [--out <file>] make a keypair: private half to the file, public half printed
|
// operator key make [--out <file>] make a keypair: private half to the file, public half printed
|
||||||
// operator key set <public> tell the mesh which key to seal to
|
// operator key set <public> tell the mesh which key to seal to
|
||||||
// operator key show the public key, its fingerprint, and what it can recover
|
// operator key show the public key, its fingerprint, and what it can recover
|
||||||
const operatorUsage = "operator key make [--out <file>] | operator key set <public> [--replace] | operator key show"
|
const operatorUsage = "operator key make [--out <file>] | operator key set <public> [--replace] | " +
|
||||||
|
"operator key show | operator issue <name> --invokes <tool,tool|*> | operator revoke <name> | " +
|
||||||
|
"operator list"
|
||||||
|
|
||||||
func operatorCommand(ctx context.Context, args []string) error {
|
func operatorCommand(ctx context.Context, args []string) error {
|
||||||
|
if len(args) == 0 {
|
||||||
|
return errors.New(operatorUsage)
|
||||||
|
}
|
||||||
|
// The people who may reach the mesh's tools (design 25 §7). Beside the operator's key because
|
||||||
|
// both answer "who, other than a machine, may do something here" — and a person reading this
|
||||||
|
// command's usage is asking exactly that.
|
||||||
|
switch args[0] {
|
||||||
|
case "issue":
|
||||||
|
return personIssue(ctx, args[1:])
|
||||||
|
case "revoke":
|
||||||
|
return personRevoke(ctx, args[1:])
|
||||||
|
case "list":
|
||||||
|
return personList(ctx)
|
||||||
|
}
|
||||||
if len(args) < 2 || args[0] != "key" {
|
if len(args) < 2 || args[0] != "key" {
|
||||||
return errors.New(operatorUsage)
|
return errors.New(operatorUsage)
|
||||||
}
|
}
|
||||||
@@ -160,3 +179,125 @@ func readPrivateKey(path string) (string, error) {
|
|||||||
}
|
}
|
||||||
return strings.TrimSpace(string(raw)), nil
|
return strings.TrimSpace(string(raw)), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// personIssue gives somebody a credential for the mesh's tools, and prints it once.
|
||||||
|
//
|
||||||
|
// **Printed, not stored.** The mesh keeps a hash and nothing else, so this is the only moment the
|
||||||
|
// credential exists anywhere but on the workstation that will use it — the same contract a token has,
|
||||||
|
// and for the same reason: a credential recoverable from the mesh's store has the store's blast
|
||||||
|
// radius.
|
||||||
|
func personIssue(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("operator issue", flag.ContinueOnError)
|
||||||
|
invokes := set.String("invokes", "", "the tools this person may call, comma-separated, or * for every one")
|
||||||
|
if err := set.Parse(args); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if set.NArg() != 1 {
|
||||||
|
return errors.New("operator issue <name> --invokes <tool,tool|*>")
|
||||||
|
}
|
||||||
|
name := set.Arg(0)
|
||||||
|
if *invokes == "" {
|
||||||
|
return errors.New(
|
||||||
|
"say what this person may call: --invokes mesh-catalog.catalog_tools,gitea.repo_create, " +
|
||||||
|
"or --invokes '*' for an administrator")
|
||||||
|
}
|
||||||
|
var tools []string
|
||||||
|
for _, t := range strings.Split(*invokes, ",") {
|
||||||
|
if t = strings.TrimSpace(t); t != "" {
|
||||||
|
tools = append(tools, t)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
if err := inv.RecordPerson(ctx, inventory.Person{Name: name, Invokes: tools}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Refused here rather than at the next composition, where it would stop the whole file being
|
||||||
|
// written for everybody. A name that cannot be part of a subject is one the server would read as
|
||||||
|
// a wider permission than anybody granted.
|
||||||
|
if _, err := broker.PermissionsFor(broker.Principal{
|
||||||
|
Kind: broker.KindPerson, Module: name, Invokes: tools, PasswordHash: "x",
|
||||||
|
}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
user := broker.Principal{Kind: broker.KindPerson, Module: name}.Username()
|
||||||
|
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: user, Kind: inventory.BusPerson})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
where, err := broker.FromEnvironment()
|
||||||
|
if err != nil && !errors.Is(err, broker.ErrNotConfigured) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
held, err := json.Marshal(struct {
|
||||||
|
URL string `json:"url"`
|
||||||
|
Fingerprint string `json:"fingerprint,omitempty"`
|
||||||
|
User string `json:"user"`
|
||||||
|
Password string `json:"password"`
|
||||||
|
Person string `json:"person"`
|
||||||
|
Invokes []string `json:"invokes"`
|
||||||
|
}{
|
||||||
|
URL: "nats://" + where.Address, Fingerprint: where.Fingerprint,
|
||||||
|
User: user, Password: password, Person: name, Invokes: tools,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("issued %s, who may call %s\n", name, strings.Join(tools, ", "))
|
||||||
|
fmt.Println(" this is the only time the credential is printed; the mesh keeps a hash")
|
||||||
|
fmt.Println(" it works once the bus has been told, which is the next push to the machine holding mesh-broker")
|
||||||
|
fmt.Println()
|
||||||
|
fmt.Println(string(held))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func personRevoke(ctx context.Context, args []string) error {
|
||||||
|
if len(args) != 1 {
|
||||||
|
return errors.New("operator revoke <name>")
|
||||||
|
}
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
|
||||||
|
if err := open.inventory.ForgetPerson(ctx, args[0]); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// **Revoked at the next composition, not now.** The bus's users are a file, so a credential stops
|
||||||
|
// working when the file no longer names it. Said plainly, because "revoked" that still works for
|
||||||
|
// another minute is worth knowing about.
|
||||||
|
fmt.Printf("%s is forgotten, and their credential stops working at the next composition — "+
|
||||||
|
"push the machine holding mesh-broker to make it so\n", args[0])
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func personList(ctx context.Context) error {
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
|
||||||
|
people, err := open.inventory.People(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(people) == 0 {
|
||||||
|
fmt.Println("nobody but machines reaches this mesh")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, p := range people {
|
||||||
|
fmt.Printf("%-20s %s\n", p.Name, strings.Join(p.Invokes, ", "))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
+123
-3
@@ -83,9 +83,17 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
|
|||||||
return catalogue.Resolution{}, nil, err
|
return catalogue.Resolution{}, nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The operator account this node logs a person in as, and where its home is (novox/hq to-be
|
||||||
|
// 29) — carried so a home-scoped file's owner and path resolve for this machine.
|
||||||
|
who, err := inv.NodeByName(ctx, nodeName)
|
||||||
|
if err != nil {
|
||||||
|
return catalogue.Resolution{}, nil, err
|
||||||
|
}
|
||||||
|
|
||||||
resolved, err := catalogue.Resolve(shelf, assigned,
|
resolved, err := catalogue.Resolve(shelf, assigned,
|
||||||
catalogue.Node{Name: nodeName, Site: site, Capabilities: capabilities,
|
catalogue.Node{Name: nodeName, Site: site, Capabilities: capabilities,
|
||||||
At: onNetwork[nodeName], PublicDomain: publicDomain}, world)
|
At: onNetwork[nodeName], PublicDomain: publicDomain,
|
||||||
|
Account: who.Account, AccountHome: who.AccountHome}, world)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return catalogue.Resolution{}, nil, err
|
return catalogue.Resolution{}, nil, err
|
||||||
}
|
}
|
||||||
@@ -488,6 +496,13 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return catalogue.Rendering{}, inventory.Node{}, err
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
}
|
}
|
||||||
|
// The private network's range, offered to a module as ${machine:mesh-range} — a module that must
|
||||||
|
// name the whole mesh (an intrusion filter that must never ban a tunnel peer) names it here
|
||||||
|
// rather than hardcoding a value it cannot know.
|
||||||
|
meshRange, err := overlayRange(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
|
}
|
||||||
|
|
||||||
// The artifact store as this node reaches it now — the address every image and archive the
|
// The artifact store as this node reaches it now — the address every image and archive the
|
||||||
// mesh built is fetched through, composed here and recorded nowhere — with what the mesh has
|
// mesh built is fetched through, composed here and recorded nowhere — with what the mesh has
|
||||||
@@ -513,6 +528,19 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
|||||||
return catalogue.Rendering{}, inventory.Node{}, err
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Each machine's operator account, so an ssh Host block can name the login for every node
|
||||||
|
// (novox/hq to-be 29). Keyed by the bare node name, which entriesFrom falls back to.
|
||||||
|
allNodes, err := inv.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
|
}
|
||||||
|
accounts := map[string]string{}
|
||||||
|
for _, n := range allNodes {
|
||||||
|
if n.Account != "" {
|
||||||
|
accounts[n.Name] = n.Account
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// And every routed name → the node that serves it (novox/hq ADR 0066). Alongside the
|
// And every routed name → the node that serves it (novox/hq ADR 0066). Alongside the
|
||||||
// `<node>.internal` names above, so a container — or an internal ACME validator — resolves a
|
// `<node>.internal` names above, so a container — or an internal ACME validator — resolves a
|
||||||
// routed name to the proxy that serves it, mesh-wide. The mesh publishes the names it was told
|
// routed name to the proxy that serves it, mesh-wide. The mesh publishes the names it was told
|
||||||
@@ -593,12 +621,20 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return catalogue.Rendering{}, inventory.Node{}, err
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
}
|
}
|
||||||
|
// The bus's user list, for the machine that runs the bus. Composed per push rather than kept,
|
||||||
|
// because it is a function of the mesh's records and a kept copy could disagree with them.
|
||||||
|
busUsers, err := composeBusUsers(ctx, inv, plan.Modules)
|
||||||
|
if err != nil {
|
||||||
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
|
}
|
||||||
return catalogue.Rendering{
|
return catalogue.Rendering{
|
||||||
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
|
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
|
||||||
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
||||||
Machines: machines,
|
Machines: machines,
|
||||||
Suffix: overlay.Suffix(), Foundation: foundation, Kept: kept, Adopted: record.Adopted,
|
Suffix: overlay.Suffix(), MeshRange: meshRange, Accounts: accounts, Foundation: foundation,
|
||||||
|
Kept: kept, Adopted: record.Adopted,
|
||||||
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, Built: built,
|
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, Built: built,
|
||||||
|
BusUsers: busUsers,
|
||||||
}, record, nil
|
}, record, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -923,12 +959,26 @@ func planCommand(ctx context.Context, args []string) error {
|
|||||||
if !ok {
|
if !ok {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
fmt.Printf("\n--- %v %v ---\n%s", r["id"], r["path"], content)
|
fmt.Printf("\n--- %s ---\n%s", shownAs(r), content)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// shownAs is the heading `plan --show` puts over a resource's content.
|
||||||
|
//
|
||||||
|
// **A file written into says so.** Its content is the mesh's part of a file that is otherwise the
|
||||||
|
// machine's — the keys of a JSON document (novox/hq ADR 0102), the region of a hosts file (issue
|
||||||
|
// 128). Shown under a bare path it reads as the whole file, and a person checking what a take
|
||||||
|
// replaces would see a hosts file of a dozen lines where the machine keeps thirty.
|
||||||
|
func shownAs(r map[string]any) string {
|
||||||
|
heading := fmt.Sprintf("%v %v", r["id"], r["path"])
|
||||||
|
if into, ok := r["into"].(string); ok && into != "" {
|
||||||
|
heading += fmt.Sprintf(" (written into, %s)", into)
|
||||||
|
}
|
||||||
|
return heading
|
||||||
|
}
|
||||||
|
|
||||||
// licencesFor is what this node can be answered with by record, and what it was put on.
|
// licencesFor is what this node can be answered with by record, and what it was put on.
|
||||||
//
|
//
|
||||||
// A mesh with no licences at all is the ordinary case and must not be an error: every existing
|
// A mesh with no licences at all is the ordinary case and must not be an error: every existing
|
||||||
@@ -1144,6 +1194,76 @@ func portsOn(
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// composeBusUsers is the bus's user list, for a push to the machine that runs the bus.
|
||||||
|
//
|
||||||
|
// Empty for every other machine, and for every machine while the mesh is on the bus it runs on
|
||||||
|
// today — where accounts are a management call and there is no file to write.
|
||||||
|
//
|
||||||
|
// **Composed on each push, never kept.** The list is a function of the mesh's records (who exists,
|
||||||
|
// what runs where, what each declares), and a stored copy would be a second account of who may reach
|
||||||
|
// the bus, able to disagree with the records while both looked internally consistent (ADR 0043).
|
||||||
|
//
|
||||||
|
// A user the mesh has never minted a password for is **left out and said**, not written as a user
|
||||||
|
// without one — the composer refuses that, because a user with no password is a user anybody is. That
|
||||||
|
// is an ordinary situation with an obvious remedy (`module issue`, or enrolling), so the push carries
|
||||||
|
// the rest rather than failing: a bus that is missing one module's user is a mesh where that module
|
||||||
|
// cannot connect, and a bus with no file at all is a mesh where nothing can.
|
||||||
|
func composeBusUsers(ctx context.Context, inv *inventory.Inventory,
|
||||||
|
onThisNode []catalogue.Manifest) (string, error) {
|
||||||
|
|
||||||
|
// **Not gated on which bus the controller is on, and that was a bug.** It read "compose this only
|
||||||
|
// once the mesh is on the new bus" — which cannot work, because the server needs its user list
|
||||||
|
// *before* anything moves onto it. Step 2 of the change is exactly that: the server stands in the
|
||||||
|
// mesh carrying nothing, on its own ports, while every node is still on the old bus (novox/hq
|
||||||
|
// ADR 0116). Under the old gating that step could not happen: the module would come up, find no
|
||||||
|
// accounts file, and its entrypoint would wait for one the controller had decided not to write.
|
||||||
|
//
|
||||||
|
// So the question is only whether this machine runs the module that asked for the file. A mesh
|
||||||
|
// that never moves has written a user list nothing reads, which costs a few hundred bytes on one
|
||||||
|
// node; the reverse cost a step that cannot be taken.
|
||||||
|
//
|
||||||
|
// Asked of what this push resolves to rather than of the seat's holder mesh-wide: the file is a
|
||||||
|
// resource of that module, so the question is whether it is here.
|
||||||
|
holdsTheBus := false
|
||||||
|
for _, m := range onThisNode {
|
||||||
|
if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") {
|
||||||
|
holdsTheBus = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !holdsTheBus {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
records, err := inv.BusRecords(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
users, err := broker.Users(records)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
kept, err := inv.BusUsers(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
hashes := make(map[string]string, len(kept))
|
||||||
|
for name, u := range kept {
|
||||||
|
hashes[name] = u.PasswordHash
|
||||||
|
}
|
||||||
|
filled, missing := broker.WithPasswords(users, hashes)
|
||||||
|
if len(missing) > 0 {
|
||||||
|
fmt.Printf("the bus's user list leaves out %d user(s) the mesh has minted no credential "+
|
||||||
|
"for: %s. Each is a user that cannot connect until one is issued\n",
|
||||||
|
len(missing), strings.Join(missing, ", "))
|
||||||
|
}
|
||||||
|
if len(filled) == 0 {
|
||||||
|
return "", fmt.Errorf(
|
||||||
|
"this machine runs the bus and not one user has a credential, so the composed list " +
|
||||||
|
"would refuse every connection in the mesh")
|
||||||
|
}
|
||||||
|
return broker.ComposeAccounts(filled)
|
||||||
|
}
|
||||||
|
|
||||||
// foundationPortsFor is the broker port, kept only when a module resolved onto this node listens
|
// foundationPortsFor is the broker port, kept only when a module resolved onto this node listens
|
||||||
// on it (novox/hq issue: the broker opening leaked onto every node). The foundation opening
|
// on it (novox/hq issue: the broker opening leaked onto every node). The foundation opening
|
||||||
// exists to WIDEN the broker's `from: mesh` port to from-anywhere, because a machine enrolling is
|
// exists to WIDEN the broker's `from: mesh` port to from-anywhere, because a machine enrolling is
|
||||||
|
|||||||
@@ -35,3 +35,17 @@ func TestListensLinesAreEmptyForAModuleWithNothingToListenOn(t *testing.T) {
|
|||||||
t.Errorf("a module with no listens should print nothing, got %v", got)
|
t.Errorf("a module with no listens should print nothing, got %v", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// `plan --show` says when a file is written into rather than over (novox/hq issue 128), or the
|
||||||
|
// mesh's region of a hosts file reads as the whole file.
|
||||||
|
func TestAFileWrittenIntoIsShownAsSuch(t *testing.T) {
|
||||||
|
region := shownAs(map[string]any{
|
||||||
|
"id": "mesh-wireguard.fact-node-names", "path": "/etc/hosts", "into": "block"})
|
||||||
|
if region != "mesh-wireguard.fact-node-names /etc/hosts (written into, block)" {
|
||||||
|
t.Errorf("the region is shown as %q", region)
|
||||||
|
}
|
||||||
|
whole := shownAs(map[string]any{"id": "dnsmasq.fact-node-zones", "path": "/etc/mesh-resolver/nodes.conf"})
|
||||||
|
if strings.Contains(whole, "written into") {
|
||||||
|
t.Errorf("a whole file is shown as written into: %q", whole)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -77,12 +77,34 @@ func serve(ctx context.Context) error {
|
|||||||
"reconnect. Set %s and %s.\n", broker.AddressVar, broker.CertificateVar)
|
"reconnect. Set %s and %s.\n", broker.AddressVar, broker.CertificateVar)
|
||||||
}
|
}
|
||||||
|
|
||||||
work := link.Enrolment{Inventory: inv, Identity: ident, Management: management, Broker: known}
|
// **Which bus this mesh is on, read once** (novox/hq ADR 0116 step 5). Both clients ship; both
|
||||||
|
// being live is refused, because a mesh half on each is one where a declaration goes out on one
|
||||||
|
// and the report comes back on the other, and every component logs success while it happens.
|
||||||
|
busAddress, onNATS, err := broker.OnNATS()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := broker.MustBeOneBus(os.Getenv(broker.AMQPVarName), busAddress); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
work := link.Enrolment{Inventory: inv, Identity: ident, Management: management, Broker: known,
|
||||||
|
OnNATS: onNATS}
|
||||||
server, err := link.Connect(work, work)
|
server, err := link.Connect(work, work)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer server.Close()
|
defer server.Close()
|
||||||
|
|
||||||
|
// The bus's own objects, asserted on every start. **Not created once at genesis**: a stream
|
||||||
|
// somebody deleted, a mesh raised from a restored backup, or a bus whose data directory was
|
||||||
|
// replaced all have records and no objects — and a node whose consumer is missing hears nothing
|
||||||
|
// while everything else about it looks correct.
|
||||||
|
if onNATS {
|
||||||
|
if err := raiseTheBus(ctx, inv, busAddress); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
// And build results nobody was waiting for. A build triggered any other way than `build`
|
// And build results nobody was waiting for. A build triggered any other way than `build`
|
||||||
// would otherwise be reported into the void, which is the same as not reporting it.
|
// would otherwise be reported into the void, which is the same as not reporting it.
|
||||||
server.Records(builds{inv})
|
server.Records(builds{inv})
|
||||||
@@ -142,7 +164,7 @@ func declare(ctx context.Context, args []string) error {
|
|||||||
}
|
}
|
||||||
defer server.Close()
|
defer server.Close()
|
||||||
|
|
||||||
if err := link.Declare(ctx, server.Channel(), ident, node, raw, 15*time.Second); err != nil {
|
if err := link.Declare(ctx, link.OverCurrent{Channel: server.Channel()}, ident, node, raw, 15*time.Second); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw))
|
fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw))
|
||||||
@@ -310,7 +332,7 @@ func pushCommand(ctx context.Context, args []string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := link.Declare(ctx, server.Channel(), ident, s.node, body, 15*time.Second); err != nil {
|
if err := link.Declare(ctx, link.OverCurrent{Channel: server.Channel()}, ident, s.node, body, 15*time.Second); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// After it is away, not before. A digest recorded for something that failed to send would
|
// After it is away, not before. A digest recorded for something that failed to send would
|
||||||
@@ -393,7 +415,7 @@ func pushCommand(ctx context.Context, args []string) error {
|
|||||||
return declarationWith(held, open, node, plan, settings, gens, Allocating)
|
return declarationWith(held, open, node, plan, settings, gens, Allocating)
|
||||||
},
|
},
|
||||||
func(s readyNode, body []byte) error {
|
func(s readyNode, body []byte) error {
|
||||||
if err := link.Declare(ctx, server.Channel(), ident, s.node, body,
|
if err := link.Declare(ctx, link.OverCurrent{Channel: server.Channel()}, ident, s.node, body,
|
||||||
15*time.Second); err != nil {
|
15*time.Second); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -502,8 +524,14 @@ func composeEach(names []string,
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if len(declared.Resources) == 0 {
|
if len(declared.Resources) == 0 {
|
||||||
fmt.Printf("%s is assigned nothing — skipped\n", name)
|
// Sent, not skipped (novox/hq issue 127). A node whose declaration composes to
|
||||||
continue
|
// nothing may have HELD something before — the broker opening a placement gave it,
|
||||||
|
// say — and skipping the empty declaration leaves that last resource in force
|
||||||
|
// forever, re-applied by the node's own heartbeat, with no way for the mesh to say
|
||||||
|
// it is gone. An empty declaration is the correction: the host drops what the mesh
|
||||||
|
// owned and keeps what it found (the adoption envelope still rides along). A node
|
||||||
|
// that never held anything applies it as the no-op it is.
|
||||||
|
fmt.Printf("%s owns nothing now — sent so it drops what it last held\n", name)
|
||||||
}
|
}
|
||||||
sending = append(sending, readyNode{name, declared})
|
sending = append(sending, readyNode{name, declared})
|
||||||
}
|
}
|
||||||
@@ -611,7 +639,7 @@ func sendTo(ctx context.Context, open *stores, names []string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := link.Declare(ctx, server.Channel(), ident, s.node, body, 15*time.Second); err != nil {
|
if err := link.Declare(ctx, link.OverCurrent{Channel: server.Channel()}, ident, s.node, body, 15*time.Second); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
record, err := inv.NodeByName(ctx, s.node)
|
record, err := inv.NodeByName(ctx, s.node)
|
||||||
@@ -664,3 +692,57 @@ func wouldSend(ctx context.Context, open *stores,
|
|||||||
}
|
}
|
||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// raiseTheBus asserts the streams and consumers the mesh's own traffic needs.
|
||||||
|
//
|
||||||
|
// **Every start, and it says what it did.** The objects are the mesh's, created by nothing else —
|
||||||
|
// the controller is their only writer (design 25 §3) — so a mesh that came up without them is one
|
||||||
|
// where nodes connect, authenticate, and hear nothing. Said rather than silent for the reason the
|
||||||
|
// first line of `serve` is said: a log that is quiet on success and loud on failure reads as broken
|
||||||
|
// when it is working.
|
||||||
|
func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string) error {
|
||||||
|
js, err := broker.Dial(address)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the mesh is on the bus at %s and this control plane cannot reach it: %w",
|
||||||
|
address, err)
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
|
||||||
|
// **Its own user, before anything else.** The controller's account is created by the installer at
|
||||||
|
// a bootstrap password, before there is a controller to mint one — so nothing recorded a hash for
|
||||||
|
// it, and the first composition would leave the writer out of the file it was writing. Recorded
|
||||||
|
// only if absent: a credential the mesh minted since is the one that counts.
|
||||||
|
// **Its own user, before anything else it does here.** The controller's account is created by the
|
||||||
|
// installer at a bootstrap password, before there is a controller to mint one — so nothing
|
||||||
|
// recorded a hash for it, and the first composition would leave the writer out of the file it was
|
||||||
|
// writing: a bus nothing can connect to, produced by the thing connected to it. Recorded only if
|
||||||
|
// absent, so a restart cannot put the bootstrap credential back over a rotated one.
|
||||||
|
if user, password, _ := broker.CredentialIn(address); user != "" && password != "" {
|
||||||
|
if err := inv.SeedBusUser(ctx, inventory.BusUser{
|
||||||
|
Username: user, Kind: inventory.BusController,
|
||||||
|
}, password); err != nil {
|
||||||
|
return fmt.Errorf("cannot record the credential this control plane is using: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
nodes, err := inv.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(nodes))
|
||||||
|
for _, n := range nodes {
|
||||||
|
names = append(names, n.Name)
|
||||||
|
}
|
||||||
|
if err := broker.Raise(js, names); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// The work queues of the mesh's own roles (novox/hq ADR 0121). The queue before the holder,
|
||||||
|
// deliberately: work queues until somebody arrives to do it, so assigning a build machine a week
|
||||||
|
// after something started asking for builds flushes the backlog instead of having lost it.
|
||||||
|
if err := broker.RaiseSeats(js, inventory.MeshSeats(), nil); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("the bus at %s has its streams, and %d machine(s) can hear a declaration\n",
|
||||||
|
address, len(names))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -39,12 +39,14 @@ func TestOneUnresolvableNodeStillLetsTheRestBeSent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// And a machine assigned nothing is neither sent nor a refusal — it is nothing to say.
|
// A machine whose declaration composes to nothing is SENT the empty declaration, not skipped
|
||||||
func TestAMachineAssignedNothingIsNotARefusal(t *testing.T) {
|
// (novox/hq issue 127): it may have held something before, and only sending the empty
|
||||||
|
// declaration tells it to drop what the mesh owned. It is never a refusal.
|
||||||
|
func TestAnEmptyDeclarationIsSentSoTheNodeDropsWhatItHeld(t *testing.T) {
|
||||||
sending, refusals := composeEach([]string{"spare"},
|
sending, refusals := composeEach([]string{"spare"},
|
||||||
func(string) (sendable, error) { return sendable{}, nil })
|
func(string) (sendable, error) { return sendable{}, nil })
|
||||||
if len(sending) != 0 || len(refusals) != 0 {
|
if len(sending) != 1 || len(refusals) != 0 {
|
||||||
t.Errorf("a machine assigned nothing was treated as something: %v / %v", sending, refusals)
|
t.Errorf("an empty declaration must be sent, not skipped or refused: %v / %v", sending, refusals)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,193 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Moving the mesh's own traffic to the bus being built (novox/hq ADR 0116 step 5).
|
||||||
|
//
|
||||||
|
// **The whole mesh moves at once, so there is nothing to inspect afterwards.** Every seam ships both
|
||||||
|
// transports and every one of them chooses by a single fact; this is the step that flips it. That
|
||||||
|
// shape is deliberate — steps 1 to 4 leave every node where it is, so the cost of being wrong stays
|
||||||
|
// bounded until here — and it means the useful work is almost all in the checking.
|
||||||
|
//
|
||||||
|
// So `rollout check` is the command that matters and the one that can be run any number of times
|
||||||
|
// against a mesh that is serving. It answers from records: what is missing, and what would happen.
|
||||||
|
// `rollout` itself refuses unless the check is clean.
|
||||||
|
//
|
||||||
|
// **The old broker is not switched off by this.** It stays an ordinary provider of `amqp` for whatever
|
||||||
|
// else uses it — on this installation, a whole automation layer that has nothing to do with the mesh
|
||||||
|
// ([ADR 0119](../../02-DECISIONS/0119-amqp-is-a-provision-not-the-bus.md)). Only the mesh's own
|
||||||
|
// traffic moves, which is why this is survivable at all: what breaks if it goes wrong is the mesh's
|
||||||
|
// ability to change things, not the services its modules are serving.
|
||||||
|
|
||||||
|
const rolloutUsage = "rollout check | rollout --confirm"
|
||||||
|
|
||||||
|
func rolloutCommand(ctx context.Context, args []string) error {
|
||||||
|
switch {
|
||||||
|
case len(args) == 1 && args[0] == "check":
|
||||||
|
return rolloutCheck(ctx)
|
||||||
|
case len(args) == 1 && args[0] == "--confirm":
|
||||||
|
return errors.New(
|
||||||
|
"the rollout itself is not built yet: `rollout check` answers whether it could run, and " +
|
||||||
|
"what is missing. Moving every node at once is the one step with nothing to inspect " +
|
||||||
|
"afterwards, so it is not being written before the check it depends on has been run " +
|
||||||
|
"against a real mesh")
|
||||||
|
default:
|
||||||
|
return errors.New(rolloutUsage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// rolloutCheck says whether the mesh could move, and what would happen if it did.
|
||||||
|
func rolloutCheck(ctx context.Context) error {
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
state, err := readinessOf(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println("the bus this mesh would move to")
|
||||||
|
if state.TheBus == "" {
|
||||||
|
fmt.Printf(" nothing names one (%s is unset)\n", broker.NATSVar)
|
||||||
|
} else {
|
||||||
|
standing := "not answering"
|
||||||
|
if state.ServerStanding {
|
||||||
|
standing = "answering"
|
||||||
|
}
|
||||||
|
fmt.Printf(" %s — %s\n", state.TheBus, standing)
|
||||||
|
}
|
||||||
|
fmt.Println()
|
||||||
|
|
||||||
|
fmt.Println("what would move")
|
||||||
|
for _, step := range broker.WhatMoves(state) {
|
||||||
|
fmt.Printf(" %s\n", step)
|
||||||
|
}
|
||||||
|
fmt.Println()
|
||||||
|
|
||||||
|
why := notReadyOf(state)
|
||||||
|
if len(why) == 0 {
|
||||||
|
fmt.Println("nothing is missing: this mesh could move its bus.")
|
||||||
|
fmt.Println()
|
||||||
|
fmt.Println("Read `what would move` above once more before running it. Every node moves at the")
|
||||||
|
fmt.Println("same moment and there is no half-moved state to look at afterwards.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
fmt.Printf("not ready — %d thing(s) to do first:\n", len(why))
|
||||||
|
for i, w := range why {
|
||||||
|
fmt.Printf(" %d. %s\n", i+1, w)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// readinessOf gathers what the mesh knows about its own ability to move.
|
||||||
|
//
|
||||||
|
// Reads and one dial, and nothing is written. Safe to run on a mesh that is serving, which is the
|
||||||
|
// point: the answer is only useful if it can be had without committing to anything.
|
||||||
|
func readinessOf(ctx context.Context, inv *inventory.Inventory) (broker.Readiness, error) {
|
||||||
|
state := broker.Readiness{
|
||||||
|
Credentialled: map[string]bool{},
|
||||||
|
ModuleCredentialled: map[string]bool{},
|
||||||
|
// The old broker keeps its other clients on this installation, and saying so is how the plan
|
||||||
|
// stops reading as a retirement.
|
||||||
|
OldBusHasOtherClients: true,
|
||||||
|
}
|
||||||
|
|
||||||
|
address, _, err := broker.OnNATS()
|
||||||
|
if err != nil {
|
||||||
|
return state, err
|
||||||
|
}
|
||||||
|
state.TheBus = address
|
||||||
|
if address != "" {
|
||||||
|
// One dial, briefly. "Is it answering" is the one fact records cannot hold, and a mesh about
|
||||||
|
// to move onto a server that is not there should hear it here rather than afterwards.
|
||||||
|
if conn, err := nats.Connect(broker.BareAddress(address), nats.Timeout(5*time.Second)); err == nil {
|
||||||
|
state.ServerStanding = true
|
||||||
|
conn.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
nodes, err := inv.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return state, err
|
||||||
|
}
|
||||||
|
kept, err := inv.BusUsers(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return state, err
|
||||||
|
}
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return state, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, n := range nodes {
|
||||||
|
state.Nodes = append(state.Nodes, n.Name)
|
||||||
|
_, has := kept[broker.Principal{Kind: broker.KindNode, Node: n.Name}.Username()]
|
||||||
|
state.Credentialled[n.Name] = has
|
||||||
|
|
||||||
|
assigned, err := inv.Assigned(ctx, n.Name)
|
||||||
|
if err != nil {
|
||||||
|
return state, err
|
||||||
|
}
|
||||||
|
for _, module := range assigned {
|
||||||
|
m, known := shelf[module]
|
||||||
|
if !known {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// The machine that holds the bus seat is the one that would be sent the user list.
|
||||||
|
if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") {
|
||||||
|
state.Holder = n.Name
|
||||||
|
state.AccountsComposed = wasSentTheUserList(ctx, inv, n.Name)
|
||||||
|
}
|
||||||
|
// A module that never speaks needs no credential, so it is not counted as missing one.
|
||||||
|
if !speaksOnTheBus(m) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
named := n.Name + "/" + module
|
||||||
|
state.Modules = append(state.Modules, named)
|
||||||
|
_, hasOne := kept[broker.Principal{
|
||||||
|
Kind: broker.KindModule, Node: n.Name, Module: module,
|
||||||
|
}.Username()]
|
||||||
|
state.ModuleCredentialled[named] = hasOne
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return state, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// speaksOnTheBus says whether a module reaches the bus at all.
|
||||||
|
//
|
||||||
|
// A third of the catalogue never does (novox/hq ADR 0120), and counting those as missing a credential
|
||||||
|
// would bury the ones that matter under a list nobody can act on.
|
||||||
|
func speaksOnTheBus(m catalogue.Manifest) bool {
|
||||||
|
return len(m.Emits) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
|
||||||
|
len(m.DefinesSeats) > 0 || len(m.Uses) > 0 || len(m.Claims) > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// wasSentTheUserList says whether the machine holding the bus has had a declaration since the user
|
||||||
|
// list became part of one.
|
||||||
|
//
|
||||||
|
// Read from what the mesh recorded sending rather than asked of the machine: a machine that is away
|
||||||
|
// has still been sent it, and this question is about whether the mesh did its part.
|
||||||
|
func wasSentTheUserList(ctx context.Context, inv *inventory.Inventory, node string) bool {
|
||||||
|
digest, err := inv.Outstanding(ctx, node)
|
||||||
|
return err == nil && strings.TrimSpace(digest) != ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// notReadyOf is the readiness reasoning, named here so a test can reach it without the command's
|
||||||
|
// printing. The reasoning itself is the broker package's, where it is pure.
|
||||||
|
func notReadyOf(state broker.Readiness) []string { return broker.NotReady(state) }
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Whether a mesh could move its bus, read from a real store.
|
||||||
|
//
|
||||||
|
// The readiness reasoning has its own tests; this is about the gathering — that the question is
|
||||||
|
// answered from what the mesh actually holds, on a store with machines and modules in it, without
|
||||||
|
// writing anything.
|
||||||
|
|
||||||
|
func TestReadinessIsGatheredFromWhatTheMeshHolds(t *testing.T) {
|
||||||
|
inv := inventory.ForTest(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
// A mesh mid-change: two machines, the bus module on one of them, a module that speaks and a
|
||||||
|
// module that never does.
|
||||||
|
for _, m := range []catalogue.Manifest{
|
||||||
|
{Module: "nats", Version: "1", BusUsers: "/var/lib/nats-module/conf/accounts.conf",
|
||||||
|
Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}}},
|
||||||
|
{Module: "gitea", Version: "1", Tools: []string{"repo_create"}},
|
||||||
|
{Module: "wallpaper", Version: "1"},
|
||||||
|
} {
|
||||||
|
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "/r"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, n := range []string{"anchor", "laptop"} {
|
||||||
|
if _, err := inv.AddNode(ctx, n); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, a := range [][2]string{{"anchor", "nats"}, {"anchor", "gitea"}, {"laptop", "wallpaper"}} {
|
||||||
|
if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
state, err := readinessOf(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if state.Holder != "anchor" {
|
||||||
|
t.Errorf("the machine holding the bus reads as %q", state.Holder)
|
||||||
|
}
|
||||||
|
if len(state.Nodes) != 2 {
|
||||||
|
t.Errorf("machines read as %v", state.Nodes)
|
||||||
|
}
|
||||||
|
// **A module that never speaks is not counted as missing a credential.** A third of the catalogue
|
||||||
|
// never reaches the bus, and listing those would bury the ones that matter.
|
||||||
|
for _, m := range state.Modules {
|
||||||
|
if strings.HasSuffix(m, "/wallpaper") {
|
||||||
|
t.Errorf("a module that never speaks was counted: %v", state.Modules)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(state.Modules) != 2 {
|
||||||
|
t.Errorf("modules that speak read as %v; expected the bus module and the one with a tool",
|
||||||
|
state.Modules)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Nothing has been minted, so it is not ready — and it says so about each machine by name.
|
||||||
|
why := notReadyOf(state)
|
||||||
|
if len(why) == 0 {
|
||||||
|
t.Fatal("a mesh where nothing has a credential was reported ready to move")
|
||||||
|
}
|
||||||
|
said := strings.Join(why, "\n")
|
||||||
|
for _, name := range []string{"anchor", "laptop"} {
|
||||||
|
if !strings.Contains(said, name) {
|
||||||
|
t.Errorf("the refusal does not name %s: %s", name, said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mint for one machine and it drops out of the complaint, which is how somebody works through it.
|
||||||
|
if _, err := inv.MintBusPassword(ctx, inventory.BusUser{
|
||||||
|
Username: "node.laptop", Kind: inventory.BusNode, Node: "laptop",
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
state, err = readinessOf(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !state.Credentialled["laptop"] {
|
||||||
|
t.Error("a machine that was minted a credential still reads as having none")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -43,15 +43,16 @@ type seatRow struct {
|
|||||||
// overview would make the one thing the overview is for — what does this mesh have — quietly
|
// overview would make the one thing the overview is for — what does this mesh have — quietly
|
||||||
// incomplete.
|
// incomplete.
|
||||||
func seatsHeld(seats []catalogue.Seat, held []catalogue.Held) ([]seatRow, []catalogue.Held) {
|
func seatsHeld(seats []catalogue.Seat, held []catalogue.Held) ([]seatRow, []catalogue.Held) {
|
||||||
defined := map[string]bool{}
|
|
||||||
rows := make([]seatRow, 0, len(seats))
|
rows := make([]seatRow, 0, len(seats))
|
||||||
for _, s := range seats {
|
for _, s := range seats {
|
||||||
defined[s.Name] = true
|
|
||||||
row := seatRow{Seat: s.Name, Scope: s.Scope, Delivers: s.Delivers, Decision: s.Decision,
|
row := seatRow{Seat: s.Name, Scope: s.Scope, Delivers: s.Delivers, Decision: s.Decision,
|
||||||
Holders: []seatHolder{}}
|
Holders: []seatHolder{}}
|
||||||
seen := map[seatHolder]bool{}
|
seen := map[seatHolder]bool{}
|
||||||
for _, h := range held {
|
for _, h := range held {
|
||||||
if h.Claim != s.Name || h.Scope != s.Scope {
|
// Resolve the held claim to a seat rather than comparing names, so a record naming a
|
||||||
|
// seat's former name groups under it after a rename (novox/hq ADR 0122).
|
||||||
|
hs, ok := catalogue.SeatNamed(h.Claim)
|
||||||
|
if !ok || hs.Name != s.Name || h.Scope != s.Scope {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
holder := seatHolder{Node: h.Node, Module: h.Module}
|
holder := seatHolder{Node: h.Node, Module: h.Module}
|
||||||
@@ -70,7 +71,8 @@ func seatsHeld(seats []catalogue.Seat, held []catalogue.Held) ([]seatRow, []cata
|
|||||||
}
|
}
|
||||||
var outside []catalogue.Held
|
var outside []catalogue.Held
|
||||||
for _, h := range held {
|
for _, h := range held {
|
||||||
if !defined[h.Claim] {
|
// Outside the set only if it resolves to no seat at all — a former name still resolves.
|
||||||
|
if _, ok := catalogue.SeatNamed(h.Claim); !ok {
|
||||||
outside = append(outside, h)
|
outside = append(outside, h)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -83,6 +85,25 @@ func seatsHeld(seats []catalogue.Seat, held []catalogue.Held) ([]seatRow, []cata
|
|||||||
return rows, outside
|
return rows, outside
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// seatCommand changes the set — the whole point of it being data (novox/hq ADR 0122).
|
||||||
|
func seatCommand(ctx context.Context, args []string) error {
|
||||||
|
if len(args) == 3 && args[0] == "rename" {
|
||||||
|
from, to := args[1], args[2]
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
if err := open.inventory.RenameSeat(ctx, from, to); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s is now %s — its former name still resolves, so nothing is rebuilt, "+
|
||||||
|
"re-registered or frozen (novox/hq ADR 0122)\n", from, to)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return fmt.Errorf("seat rename <from> <to>")
|
||||||
|
}
|
||||||
|
|
||||||
func seatsCommand(ctx context.Context, args []string) error {
|
func seatsCommand(ctx context.Context, args []string) error {
|
||||||
set := flag.NewFlagSet("seats", flag.ContinueOnError)
|
set := flag.NewFlagSet("seats", flag.ContinueOnError)
|
||||||
asJSON := set.Bool("json", false, "the same, as JSON")
|
asJSON := set.Bool("json", false, "the same, as JSON")
|
||||||
|
|||||||
@@ -35,13 +35,13 @@ func TestEverySeatIsListedIncludingTheOnesNobodyHolds(t *testing.T) {
|
|||||||
|
|
||||||
func TestANodeSeatListsEveryMachineHoldingIt(t *testing.T) {
|
func TestANodeSeatListsEveryMachineHoldingIt(t *testing.T) {
|
||||||
rows, _ := seatsHeld(catalogue.Seats(), []catalogue.Held{
|
rows, _ := seatsHeld(catalogue.Seats(), []catalogue.Held{
|
||||||
{Claim: "the-packet-filter", Scope: catalogue.ScopeNode, Node: "node2", Module: "nftables"},
|
{Claim: "node-packet-filter", Scope: catalogue.ScopeNode, Node: "node2", Module: "nftables"},
|
||||||
{Claim: "the-packet-filter", Scope: catalogue.ScopeNode, Node: "anchor", Module: "nftables"},
|
{Claim: "node-packet-filter", Scope: catalogue.ScopeNode, Node: "anchor", Module: "nftables"},
|
||||||
// Resolved twice, reported once: a machine is one holder however many passes saw it.
|
// Resolved twice, reported once: a machine is one holder however many passes saw it.
|
||||||
{Claim: "the-packet-filter", Scope: catalogue.ScopeNode, Node: "anchor", Module: "nftables"},
|
{Claim: "node-packet-filter", Scope: catalogue.ScopeNode, Node: "anchor", Module: "nftables"},
|
||||||
})
|
})
|
||||||
for _, r := range rows {
|
for _, r := range rows {
|
||||||
if r.Seat != "the-packet-filter" {
|
if r.Seat != "node-packet-filter" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if len(r.Holders) != 2 || r.Holders[0].Node != "anchor" || r.Holders[1].Node != "node2" {
|
if len(r.Holders) != 2 || r.Holders[0].Node != "anchor" || r.Holders[1].Node != "node2" {
|
||||||
|
|||||||
@@ -38,6 +38,12 @@ func (s sendable) Body() ([]byte, error) {
|
|||||||
if s.Adoption != nil {
|
if s.Adoption != nil {
|
||||||
envelope["adoption"] = s.Adoption
|
envelope["adoption"] = s.Adoption
|
||||||
}
|
}
|
||||||
|
// An empty declaration is deliberate here — the node owns nothing the mesh put there
|
||||||
|
// (novox/hq issue 127) — and the host refuses an empty body unless it is told the emptiness
|
||||||
|
// is meant, so a truncated or mis-composed body is never mistaken for "own nothing".
|
||||||
|
if len(s.Resources) == 0 {
|
||||||
|
envelope["owns_nothing"] = true
|
||||||
|
}
|
||||||
return json.Marshal(envelope)
|
return json.Marshal(envelope)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -355,3 +355,25 @@ func TestTheMachineSideOfAMappingIsMovedEverywhereTheNumberIsUsed(t *testing.T)
|
|||||||
t.Fatalf("the consumer is told the forge answers on %v", told)
|
t.Fatalf("the consumer is told the forge answers on %v", told)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestAnEmptyDeclarationSaysOwnsNothing(t *testing.T) {
|
||||||
|
// The host refuses an empty body unless told the emptiness is meant (novox/hq issue 127).
|
||||||
|
body, err := sendable{}.Body()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var env map[string]any
|
||||||
|
if err := json.Unmarshal(body, &env); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if env["owns_nothing"] != true {
|
||||||
|
t.Fatalf("an empty declaration must mark owns_nothing; got %v", env)
|
||||||
|
}
|
||||||
|
// A declaration with resources does not carry the marker.
|
||||||
|
body, _ = sendable{Resources: []map[string]any{{"id": "x"}}}.Body()
|
||||||
|
var env2 map[string]any
|
||||||
|
_ = json.Unmarshal(body, &env2)
|
||||||
|
if _, present := env2["owns_nothing"]; present {
|
||||||
|
t.Fatalf("a non-empty declaration must not mark owns_nothing; got %v", env)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
"github.com/novox/mesh-controller/internal/identity"
|
"github.com/novox/mesh-controller/internal/identity"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/licences"
|
"github.com/novox/mesh-controller/internal/licences"
|
||||||
@@ -72,6 +73,14 @@ func migrate(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
fmt.Printf("provided %s\n", m.Module)
|
fmt.Printf("provided %s\n", m.Module)
|
||||||
}
|
}
|
||||||
|
// The seats the mesh ships with, into the table that now holds the set (novox/hq ADR 0122).
|
||||||
|
// Idempotent: fills an empty table on first boot, adds a seat a release ships, and leaves an
|
||||||
|
// operator's changes in the table as they are.
|
||||||
|
added, err := inv.SeedSeats(ctx, catalogue.DefaultSeats())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("seeded %d seat(s)\n", added)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -85,6 +94,18 @@ func openInventory(ctx context.Context) (*inventory.Inventory, error) {
|
|||||||
inv.Close()
|
inv.Close()
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
// Load the seat set from the store, so the control plane reads the set as data rather than as
|
||||||
|
// the slice it was compiled with (novox/hq ADR 0122). A store not yet seeded — or one whose
|
||||||
|
// seat table a migration has not reached — returns nothing, and UseSeats leaves the compiled
|
||||||
|
// defaults in force: the set is never emptied by a read that found nothing, which would refuse
|
||||||
|
// every claim. So this can only ever replace the defaults with what the mesh actually holds.
|
||||||
|
if seats, err := inv.Seats(ctx); err == nil {
|
||||||
|
catalogue.UseSeats(seats)
|
||||||
|
}
|
||||||
|
// And the former names, so a reference to a seat's old name resolves after a rename (ADR 0122).
|
||||||
|
if aliases, err := inv.Aliases(ctx); err == nil {
|
||||||
|
catalogue.UseAliases(aliases)
|
||||||
|
}
|
||||||
return inv, nil
|
return inv, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,19 +1,23 @@
|
|||||||
module github.com/novox/mesh-controller
|
module github.com/novox/mesh-controller
|
||||||
|
|
||||||
go 1.25.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/jackc/pgx/v5 v5.10.0
|
github.com/jackc/pgx/v5 v5.10.0
|
||||||
github.com/rabbitmq/amqp091-go v1.14.0
|
github.com/rabbitmq/amqp091-go v1.14.0
|
||||||
golang.org/x/crypto v0.55.0
|
golang.org/x/crypto v0.57.0
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||||
golang.org/x/net v0.57.0 // indirect
|
github.com/klauspost/compress v1.20.0 // indirect
|
||||||
golang.org/x/sync v0.22.0 // indirect
|
github.com/nats-io/nats.go v1.54.0 // indirect
|
||||||
golang.org/x/sys v0.47.0 // indirect
|
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||||
golang.org/x/text v0.41.0 // indirect
|
github.com/nats-io/nuid v1.0.1 // indirect
|
||||||
|
golang.org/x/net v0.58.0 // indirect
|
||||||
|
golang.org/x/sync v0.23.0 // indirect
|
||||||
|
golang.org/x/sys v0.48.0 // indirect
|
||||||
|
golang.org/x/text v0.42.0 // indirect
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -9,6 +9,14 @@ github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0=
|
|||||||
github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||||
|
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||||
|
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||||
|
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||||
|
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||||
|
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||||
|
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||||
|
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||||
|
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
github.com/rabbitmq/amqp091-go v1.14.0 h1:RSaT7aOKt/OrkVUyswPDW29lnRz9psuGmfZFBmLqLek=
|
github.com/rabbitmq/amqp091-go v1.14.0 h1:RSaT7aOKt/OrkVUyswPDW29lnRz9psuGmfZFBmLqLek=
|
||||||
@@ -22,14 +30,24 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
|||||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||||
golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
|
golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
|
||||||
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
|
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
|
||||||
|
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||||
|
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||||
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
|
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
|
||||||
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
|
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
|
||||||
|
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
|
||||||
|
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
|
||||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||||
|
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
|
||||||
|
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||||
|
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||||
|
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||||
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
|
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
|
||||||
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
|
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
|
||||||
|
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
|
||||||
|
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
|
|||||||
@@ -0,0 +1,125 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Do the emitters and the consumers of a catalogue agree?
|
||||||
|
//
|
||||||
|
// **The check that was missing** (novox/hq 04-ISSUES/127). Every manifest was individually
|
||||||
|
// well-formed and every derivation individually correct, and no cross-module subscription in the
|
||||||
|
// mesh matched anything: a consumer's declaration derived into a namespace nobody publishes to.
|
||||||
|
// Nothing failed, because a subscription that matches nothing is not an error — it is silence.
|
||||||
|
//
|
||||||
|
// The comparison has to be over the whole catalogue, because the two halves live in different
|
||||||
|
// manifests, and it cannot simply demand that every consumed event have a live emitter: a module
|
||||||
|
// may be installed long before the one whose events it wants. So the rule is narrower and still
|
||||||
|
// catches this: **where the emitter is present, it must emit what the consumer asked for.**
|
||||||
|
|
||||||
|
// AConsumer is one module's interest in another's events, as this check needs it.
|
||||||
|
type AConsumer struct {
|
||||||
|
Module string
|
||||||
|
Consumes []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// AnEmitter is one module's events.
|
||||||
|
type AnEmitter struct {
|
||||||
|
Module string
|
||||||
|
Emits []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Disagreements are the consumed events whose emitter is in the catalogue and does not emit them.
|
||||||
|
//
|
||||||
|
// Returned as sentences rather than as structs: every one of them is read by a person deciding
|
||||||
|
// whether a manifest or a catalogue is wrong, and a pair of names without the reason is a puzzle.
|
||||||
|
func Disagreements(emitters []AnEmitter, consumers []AConsumer, seats []DeclaredSeat) []string {
|
||||||
|
emits := map[string]map[string]bool{}
|
||||||
|
for _, e := range emitters {
|
||||||
|
if emits[e.Module] == nil {
|
||||||
|
emits[e.Module] = map[string]bool{}
|
||||||
|
}
|
||||||
|
for _, name := range e.Emits {
|
||||||
|
emits[e.Module][name] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A seat's events are published by its holder under the seat's name, so a consumer naming the
|
||||||
|
// seat is naming something real even though no module declares it as its own.
|
||||||
|
for _, s := range seats {
|
||||||
|
if len(s.Emits) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if emits[s.Name] == nil {
|
||||||
|
emits[s.Name] = map[string]bool{}
|
||||||
|
}
|
||||||
|
for _, name := range s.Emits {
|
||||||
|
emits[s.Name][name] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var out []string
|
||||||
|
for _, c := range consumers {
|
||||||
|
for _, pattern := range c.Consumes {
|
||||||
|
emitter, event, named := strings.Cut(pattern, ".")
|
||||||
|
// Every event from everyone, or every event from one module: both are deliberate and
|
||||||
|
// neither names a particular event to check.
|
||||||
|
if !named || emitter == "*" || emitter == catalogueTheRest || event == catalogueTheRest {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
known, present := emits[emitter]
|
||||||
|
if !present {
|
||||||
|
// Not installed here, which is ordinary: a module lives in its own repository and
|
||||||
|
// may be registered later. Nothing to compare, so nothing to say.
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if matchesAny(event, known) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, fmt.Sprintf(
|
||||||
|
"%s consumes %q and %s emits %s — so that subscription would match nothing, and "+
|
||||||
|
"nothing would report it",
|
||||||
|
c.Module, pattern, emitter, listOf(known)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// matchesAny says whether one of an emitter's event names satisfies a consumer's pattern.
|
||||||
|
func matchesAny(pattern string, emitted map[string]bool) bool {
|
||||||
|
want := strings.Split(pattern, ".")
|
||||||
|
for name := range emitted {
|
||||||
|
if matches(want, strings.Split(name, ".")) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func matches(pattern, name []string) bool {
|
||||||
|
for i, part := range pattern {
|
||||||
|
if part == catalogueTheRest {
|
||||||
|
return i < len(name)
|
||||||
|
}
|
||||||
|
if i >= len(name) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if part != "*" && part != name[i] {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return len(pattern) == len(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func listOf(names map[string]bool) string {
|
||||||
|
if len(names) == 0 {
|
||||||
|
return "nothing"
|
||||||
|
}
|
||||||
|
out := make([]string, 0, len(names))
|
||||||
|
for n := range names {
|
||||||
|
out = append(out, n)
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return strings.Join(out, ", ")
|
||||||
|
}
|
||||||
@@ -0,0 +1,236 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **Do the catalogue's emitters and consumers agree?**
|
||||||
|
//
|
||||||
|
// This is the check whose absence let issue 127 stand: every manifest was individually well-formed,
|
||||||
|
// every derivation individually correct, and no cross-module subscription in the mesh matched
|
||||||
|
// anything. A subscription that matches nothing is not an error — it is silence — so nothing
|
||||||
|
// anywhere reported it.
|
||||||
|
//
|
||||||
|
// It compares what one manifest asks to hear against what another says it emits. It cannot demand
|
||||||
|
// that every consumed event have a live emitter, because a module lives in its own repository and
|
||||||
|
// may be registered long before the one whose events it wants. Where the emitter *is* here, it must
|
||||||
|
// emit what the consumer asked for.
|
||||||
|
func TestTheCataloguesEmittersAndConsumersAgree(t *testing.T) {
|
||||||
|
emitters, consumers, seats := theCataloguesEvents(t)
|
||||||
|
|
||||||
|
if bad := Disagreements(emitters, consumers, seats); len(bad) > 0 {
|
||||||
|
t.Fatalf("%d subscription(s) in the catalogue would match nothing:\n %s",
|
||||||
|
len(bad), strings.Join(bad, "\n "))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// And the check itself catches the thing it exists for, so it cannot pass by doing nothing.
|
||||||
|
func TestTheAgreementCheckCatchesASubscriptionThatMatchesNothing(t *testing.T) {
|
||||||
|
bad := Disagreements(
|
||||||
|
[]AnEmitter{{Module: "builder", Emits: []string{"built"}}},
|
||||||
|
[]AConsumer{{Module: "mesh-catalog", Consumes: []string{"builder.finished"}}},
|
||||||
|
nil)
|
||||||
|
if len(bad) != 1 {
|
||||||
|
t.Fatalf("a consumer asking for an event its emitter does not emit was not caught: %v", bad)
|
||||||
|
}
|
||||||
|
if !strings.Contains(bad[0], "builder.finished") || !strings.Contains(bad[0], "built") {
|
||||||
|
t.Fatalf("the report names neither what was asked for nor what is emitted: %s", bad[0])
|
||||||
|
}
|
||||||
|
|
||||||
|
// A module that is not here is not a disagreement: it may be registered later.
|
||||||
|
if bad := Disagreements(nil,
|
||||||
|
[]AConsumer{{Module: "plex", Consumes: []string{"sonarr.download.completed"}}}, nil); len(bad) != 0 {
|
||||||
|
t.Fatalf("a consumer whose emitter is not installed was reported: %v", bad)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A wildcard over emitters is deliberate and names no particular event to check.
|
||||||
|
if bad := Disagreements([]AnEmitter{{Module: "sonarr", Emits: []string{"download.completed"}}},
|
||||||
|
[]AConsumer{{Module: "plex", Consumes: []string{"*.download.completed"}}}, nil); len(bad) != 0 {
|
||||||
|
t.Fatalf("a wildcard over emitters was reported: %v", bad)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A consumer of a role's event whose role does not emit it is caught, which is what stops the
|
||||||
|
// catalogue check above from passing by knowing nothing about roles.
|
||||||
|
if bad := Disagreements(nil,
|
||||||
|
[]AConsumer{{Module: "mesh-catalog", Consumes: []string{"mesh-build-machine.finished"}}},
|
||||||
|
[]DeclaredSeat{{Name: "mesh-build-machine", Emits: []string{"built"}}}); len(bad) != 1 {
|
||||||
|
t.Fatalf("a consumer of a role event the role does not emit was not caught: %v", bad)
|
||||||
|
}
|
||||||
|
|
||||||
|
// An event published under a seat's name is real even though no module declares it as its own.
|
||||||
|
if bad := Disagreements(nil,
|
||||||
|
[]AConsumer{{Module: "watcher", Consumes: []string{"mesh-artifact-store.image.pushed"}}},
|
||||||
|
[]DeclaredSeat{{Name: "the-artifact-store", Emits: []string{"image.pushed"}}}); len(bad) != 0 {
|
||||||
|
t.Fatalf("an event a seat emits was reported as matching nothing: %v", bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func theCataloguesEvents(t *testing.T) ([]AnEmitter, []AConsumer, []DeclaredSeat) {
|
||||||
|
t.Helper()
|
||||||
|
root := filepath.Join("..", "..", "..", "mesh-catalog", "modules")
|
||||||
|
entries, err := os.ReadDir(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Skipf("catalogue sibling not present: %v", err)
|
||||||
|
}
|
||||||
|
var emitters []AnEmitter
|
||||||
|
var consumers []AConsumer
|
||||||
|
// The mesh's own roles, which emit under the seat's name rather than any module's (novox/hq
|
||||||
|
// ADR 0121). Without these the check skips every consumer of a role's event as "the emitter is
|
||||||
|
// not installed" — which is how it passed vacuously the first time one existed.
|
||||||
|
var seats []DeclaredSeat
|
||||||
|
for _, own := range catalogue.SeatsWithAProtocol() {
|
||||||
|
seats = append(seats, DeclaredSeat{Name: own.Name, Accepts: own.Accepts, Emits: own.Emits})
|
||||||
|
}
|
||||||
|
for _, e := range entries {
|
||||||
|
if !e.IsDir() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(filepath.Join(root, e.Name(), "module.json"))
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var m struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
Emits []string `json:"emits"`
|
||||||
|
Consumes []string `json:"consumes"`
|
||||||
|
Seats []struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Emits []string `json:"emits"`
|
||||||
|
} `json:"seats"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &m); err != nil {
|
||||||
|
t.Fatalf("%s: %v", e.Name(), err)
|
||||||
|
}
|
||||||
|
if len(m.Emits) > 0 {
|
||||||
|
emitters = append(emitters, AnEmitter{Module: m.Module, Emits: m.Emits})
|
||||||
|
}
|
||||||
|
if len(m.Consumes) > 0 {
|
||||||
|
consumers = append(consumers, AConsumer{Module: m.Module, Consumes: m.Consumes})
|
||||||
|
}
|
||||||
|
for _, s := range m.Seats {
|
||||||
|
seats = append(seats, DeclaredSeat{Name: s.Name, Emits: s.Emits})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(emitters) == 0 {
|
||||||
|
t.Skip("no manifests found beside this checkout")
|
||||||
|
}
|
||||||
|
return emitters, consumers, seats
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Do the derived subjects meet, not just the names?**
|
||||||
|
//
|
||||||
|
// The check above compares what a consumer asks for against what an emitter says it emits, by name. It
|
||||||
|
// passed while the catalogue's subscription pointed at `mesh.mod.mesh-build-machine.event.built` — a
|
||||||
|
// module namespace for a role's event, which no emitter owns. The names agreed; the subjects did not,
|
||||||
|
// and the graph stayed empty.
|
||||||
|
//
|
||||||
|
// So this compares the thing that actually has to match: the subject a consumer subscribes against the
|
||||||
|
// subject an emitter publishes. It is the last place the two halves can be held together, because
|
||||||
|
// after this the server is the only thing that knows and it says nothing — a subscription that matches
|
||||||
|
// nothing is silence.
|
||||||
|
func TestTheCataloguesDerivedSubjectsMeet(t *testing.T) {
|
||||||
|
emitters, consumers, seats := theCataloguesEvents(t)
|
||||||
|
|
||||||
|
// Every subject something publishes: a module's own events, and the events of every role.
|
||||||
|
published := map[string]bool{}
|
||||||
|
for _, e := range emitters {
|
||||||
|
for _, name := range e.Emits {
|
||||||
|
published["mesh.mod."+e.Module+".event."+name] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, s := range seats {
|
||||||
|
for _, name := range s.Emits {
|
||||||
|
published["mesh.seat."+s.Name+".event."+name] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
byName := map[string]DeclaredSeat{}
|
||||||
|
for _, s := range seats {
|
||||||
|
byName[s.Name] = s
|
||||||
|
}
|
||||||
|
|
||||||
|
var lonely []string
|
||||||
|
for _, c := range consumers {
|
||||||
|
principal := Principal{Kind: KindModule, Node: "one", Module: c.Module, PasswordHash: "x"}
|
||||||
|
for _, want := range c.Consumes {
|
||||||
|
emitter, event, named := strings.Cut(want, ".")
|
||||||
|
if named {
|
||||||
|
if s, isASeat := byName[emitter]; isASeat {
|
||||||
|
principal.Watches = append(principal.Watches,
|
||||||
|
Seat{Name: s.Name, Emits: []string{event}})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
principal.Consumes = append(principal.Consumes, want)
|
||||||
|
}
|
||||||
|
perms, err := PermissionsFor(principal)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: %v", c.Module, err)
|
||||||
|
}
|
||||||
|
for _, subject := range perms.Subscribe {
|
||||||
|
if !strings.Contains(subject, ".event.") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if reaches(subject, published) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// A wildcard over emitters reaches whatever arrives later, and an emitter that is not
|
||||||
|
// installed is ordinary — both are already excused by the check above, so only a subject
|
||||||
|
// that can never match anything gets here.
|
||||||
|
if strings.Contains(subject, "*") || strings.Contains(subject, ">") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
lonely = append(lonely, c.Module+" subscribes "+subject+", which nothing publishes")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(lonely) > 0 {
|
||||||
|
sort.Strings(lonely)
|
||||||
|
t.Fatalf("%d subscription(s) derive to a subject no emitter owns:\n %s",
|
||||||
|
len(lonely), strings.Join(lonely, "\n "))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// And it catches the thing it exists for: a role's event read as a module's.
|
||||||
|
func TestTheDerivedSubjectCheckCatchesARolesEventReadAsAModules(t *testing.T) {
|
||||||
|
published := map[string]bool{"mesh.seat.mesh-build-machine.event.built": true}
|
||||||
|
// What the derivation produced before a consumed seat name was resolved as one.
|
||||||
|
if reaches("mesh.mod.mesh-build-machine.event.built", published) {
|
||||||
|
t.Fatal("a module namespace was treated as reaching a role's event, which is the bug")
|
||||||
|
}
|
||||||
|
// And the corrected one does reach it.
|
||||||
|
if !reaches("mesh.seat.mesh-build-machine.event.built", published) {
|
||||||
|
t.Fatal("the role's own subject does not reach the role's event")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// reaches says whether a subscribed subject admits any published one.
|
||||||
|
func reaches(subject string, published map[string]bool) bool {
|
||||||
|
for p := range published {
|
||||||
|
if admitsSubject(strings.Split(subject, "."), strings.Split(p, ".")) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func admitsSubject(pattern, subject []string) bool {
|
||||||
|
for i, token := range pattern {
|
||||||
|
if token == ">" {
|
||||||
|
return i < len(subject)
|
||||||
|
}
|
||||||
|
if i >= len(subject) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if token != "*" && token != subject[i] {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return len(pattern) == len(subject)
|
||||||
|
}
|
||||||
@@ -0,0 +1,237 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Streams and consumers derived from what modules declare.
|
||||||
|
//
|
||||||
|
// The mesh's own four exist before any module does (streams.go). Everything here is the other
|
||||||
|
// half: a seat's stream comes into being when the module declaring it is **registered**, and a
|
||||||
|
// consumer when a module is **assigned** — which is why ADR 0116's task 1.4 had to be narrowed to
|
||||||
|
// the foundation set. Neither has happened at genesis.
|
||||||
|
//
|
||||||
|
// All of it is a pure function of declarations. The controller is still the only writer; this is
|
||||||
|
// only what it writes.
|
||||||
|
|
||||||
|
// A Consumer is a durable subscription the controller creates on a module's behalf. A module
|
||||||
|
// declares what it reacts to, never how delivery works, so it does not name these and cannot
|
||||||
|
// misconfigure them.
|
||||||
|
type Consumer struct {
|
||||||
|
Name string
|
||||||
|
Stream string
|
||||||
|
// Filters are the subjects this consumer receives. One consumer per module with several
|
||||||
|
// filters, rather than one per consumed event: its ack subject is derived from its name, and
|
||||||
|
// a module with five consumers would need five ack permissions to ack its own deliveries.
|
||||||
|
Filters []string
|
||||||
|
// Queue is the queue group, set for a seat's worker so that "exactly one holder" survives a
|
||||||
|
// seat later being relaxed to several. Authority and delivery are kept separate on purpose.
|
||||||
|
Queue string
|
||||||
|
// Push asks the server to deliver to a subject rather than wait to be pulled.
|
||||||
|
//
|
||||||
|
// For the mesh's own consumer, where the controller wants every message to arrive in the one
|
||||||
|
// loop it already runs: pulling would mean a second goroutine fetching batches and handing
|
||||||
|
// them over, and a loop that acts on one message at a time is the property the store window
|
||||||
|
// depends on. A queue group implies this, because a group has nothing to pull from.
|
||||||
|
Push bool
|
||||||
|
// AckWaitSeconds before an unacknowledged delivery is redelivered.
|
||||||
|
AckWaitSeconds int
|
||||||
|
// MaxDeliver before the message is dead-lettered; zero for the mesh's default.
|
||||||
|
MaxDeliver int
|
||||||
|
Why string
|
||||||
|
}
|
||||||
|
|
||||||
|
// seatStreamName is the stream holding a seat's inbound work. Named after the seat rather than
|
||||||
|
// the module holding it, because the holder can change and the queued work must not care — which
|
||||||
|
// is the whole reason a caller addresses a seat instead of a module.
|
||||||
|
func seatStreamName(seat string) string { return "SEAT_" + upperSnake(seat) }
|
||||||
|
|
||||||
|
// SeatStreams is one work queue per declared seat, created when the declaring module is
|
||||||
|
// registered rather than when it is assigned.
|
||||||
|
//
|
||||||
|
// **The stream exists before anyone holds the seat, and that is the point.** Work queues until a
|
||||||
|
// holder appears, so installing the telegram module a week after something started sending to it
|
||||||
|
// flushes the backlog instead of having lost it. A stream created at assignment would make "the
|
||||||
|
// holder is not here yet" mean "your messages are gone".
|
||||||
|
func SeatStreams(seats []DeclaredSeat) []Stream {
|
||||||
|
sorted := append([]DeclaredSeat(nil), seats...)
|
||||||
|
sort.Slice(sorted, func(i, j int) bool { return sorted[i].Name < sorted[j].Name })
|
||||||
|
|
||||||
|
var out []Stream
|
||||||
|
for _, s := range sorted {
|
||||||
|
if len(s.Accepts) == 0 {
|
||||||
|
// A seat that only emits and serves needs no stream: its events ride EVENTS and its
|
||||||
|
// tools are core request/reply, which is never persisted.
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
retain := s.RetainSeconds
|
||||||
|
if retain == 0 {
|
||||||
|
retain = 7 * 24 * 60 * 60
|
||||||
|
}
|
||||||
|
out = append(out, Stream{
|
||||||
|
Name: seatStreamName(s.Name),
|
||||||
|
Subjects: []string{"mesh.seat." + s.Name + ".accept.>"},
|
||||||
|
Retention: RetentionWorkQueue,
|
||||||
|
MaxAge: retain,
|
||||||
|
Why: fmt.Sprintf("work submitted to the %s seat; one holder consumes it, and it "+
|
||||||
|
"queues while nobody does", s.Name),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// A DeclaredSeat is a seat as the catalogue knows it. Mirrored here rather than imported so this
|
||||||
|
// package stays free of the catalogue's own types — the same reason the host mirrors the
|
||||||
|
// contracts instead of importing the sdk.
|
||||||
|
type DeclaredSeat struct {
|
||||||
|
Name string
|
||||||
|
Accepts []string
|
||||||
|
// Emits are the verbs the seat's holder publishes under the seat's own name. An event about a
|
||||||
|
// role belongs here rather than in the holder's namespace, because the name then outlives
|
||||||
|
// whoever fills it (novox/hq ADR 0121, 04-ISSUES/127).
|
||||||
|
Emits []string
|
||||||
|
// Serves are the verbs the holder answers, request and reply.
|
||||||
|
Serves []string
|
||||||
|
RetainSeconds int
|
||||||
|
}
|
||||||
|
|
||||||
|
// ConsumerFor is the durable consumer a module's declarations imply, or false when it subscribes
|
||||||
|
// to nothing and needs none.
|
||||||
|
//
|
||||||
|
// One per module, with every consumed subject as a filter, because its ack permission is derived
|
||||||
|
// from its name: a module with a consumer per event would need an ack permission per consumer,
|
||||||
|
// and the permission list would stop being derivable from the declaration.
|
||||||
|
func ConsumerFor(p Principal) (Consumer, bool) {
|
||||||
|
// A module that reacts to anything — a module's events or a role's (novox/hq ADR 0121). Watching
|
||||||
|
// a role was missing here, so the one module that does it got no consumer at all: it started,
|
||||||
|
// connected, and its graph stayed empty with nothing anywhere reporting why.
|
||||||
|
if p.Kind != KindModule || (len(p.Consumes) == 0 && len(p.Watches) == 0) {
|
||||||
|
return Consumer{}, false
|
||||||
|
}
|
||||||
|
perms, err := PermissionsFor(p)
|
||||||
|
if err != nil {
|
||||||
|
return Consumer{}, false
|
||||||
|
}
|
||||||
|
// Events, wherever they live: a module's own namespace, and the namespace of any role it watches
|
||||||
|
// (novox/hq ADR 0121). Tool subjects and inboxes are subscribed directly and are not a consumer's
|
||||||
|
// business, which is why this is a filter and not the whole list.
|
||||||
|
var filters []string
|
||||||
|
for _, s := range perms.Subscribe {
|
||||||
|
if strings.Contains(s, ".event.") {
|
||||||
|
filters = append(filters, s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(filters) == 0 {
|
||||||
|
return Consumer{}, false
|
||||||
|
}
|
||||||
|
sort.Strings(filters)
|
||||||
|
return Consumer{
|
||||||
|
Name: consumerDurable(p),
|
||||||
|
Stream: consumerStream(p),
|
||||||
|
Filters: filters,
|
||||||
|
AckWaitSeconds: 30,
|
||||||
|
MaxDeliver: 5,
|
||||||
|
Why: "what " + p.Module + " declared it consumes; after max-deliver it dead-letters",
|
||||||
|
}, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// HolderConsumerFor is the worker a seat's holder gets on that seat's work queue.
|
||||||
|
//
|
||||||
|
// **A queue group even though the seat guarantees one holder.** The seat is *authority* — who may
|
||||||
|
// be the telegram sender — and the queue group is *delivery*. Tie delivery to the seat and the
|
||||||
|
// day somebody allows two holders for throughput, every message is processed twice with nothing
|
||||||
|
// reporting it. Kept separate, relaxing one changes nothing about the other.
|
||||||
|
func HolderConsumerFor(node, module string, seat DeclaredSeat) (Consumer, bool) {
|
||||||
|
if len(seat.Accepts) == 0 {
|
||||||
|
return Consumer{}, false
|
||||||
|
}
|
||||||
|
return Consumer{
|
||||||
|
Name: "SEAT_" + upperSnake(seat.Name) + "_worker",
|
||||||
|
Stream: seatStreamName(seat.Name),
|
||||||
|
Filters: []string{"mesh.seat." + seat.Name + ".accept.>"},
|
||||||
|
Queue: "holders",
|
||||||
|
AckWaitSeconds: 60,
|
||||||
|
MaxDeliver: 5,
|
||||||
|
Why: fmt.Sprintf("%s on %s holds %s; it acknowledges after the work is done, so a "+
|
||||||
|
"crash mid-work redelivers rather than loses", module, node, seat.Name),
|
||||||
|
}, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// NodeConsumer is the durable consumer a node reads its own declaration through.
|
||||||
|
//
|
||||||
|
// **Derived from a node existing, and created by the controller, because a host cannot create it.**
|
||||||
|
// A host's account may subscribe its own declaration subject and publish its own ack subject, and
|
||||||
|
// reaches no part of the JetStream API — which is correct (the controller is the only writer of
|
||||||
|
// consumer definitions, design 25 §3) and means the consumer must be waiting before the host binds
|
||||||
|
// to it. Named after the node, because the node's ack grant is `$JS.ACK.NODES.<node>.>` and a
|
||||||
|
// consumer named anything else is one the host cannot acknowledge a delivery from.
|
||||||
|
//
|
||||||
|
// **No max-deliver, and a long ack wait.** A declaration is settled only after the node has applied
|
||||||
|
// it and reported, which is minutes on a machine pulling images; and a declaration the mesh cannot
|
||||||
|
// get a node to accept is not one to dead-letter, because the stream keeps only the newest per node
|
||||||
|
// anyway — so there is exactly one message per node to redeliver, for as long as that node is away.
|
||||||
|
func NodeConsumer(node string) Consumer {
|
||||||
|
return Consumer{
|
||||||
|
Name: node,
|
||||||
|
Stream: "NODES",
|
||||||
|
Filters: []string{"mesh.node." + node + ".declare"},
|
||||||
|
Push: true,
|
||||||
|
AckWaitSeconds: 300,
|
||||||
|
Why: "how " + node + " hears what it should be; last-per-subject, so a node that was away " +
|
||||||
|
"gets exactly the current declaration and nothing older",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// AssertNodeConsumers brings every known node's declaration consumer into being.
|
||||||
|
//
|
||||||
|
// Asserted on start as well as created at enrolment, for the reason the streams are: a mesh raised
|
||||||
|
// from a restored backup, or one whose bus was recreated, has node records and no consumers, and a
|
||||||
|
// node whose consumer is missing hears nothing while everything else about it looks correct.
|
||||||
|
func AssertNodeConsumers(e Ensurer, nodes []string) error {
|
||||||
|
for _, n := range nodes {
|
||||||
|
if err := e.EnsureConsumer(NodeConsumer(n)); err != nil {
|
||||||
|
return fmt.Errorf("asserting how %s hears its declaration: %w", n, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// AllOverlaps reports subject filters claimed by more than one stream, across the mesh's own and
|
||||||
|
// every derived one.
|
||||||
|
//
|
||||||
|
// NATS refuses an overlapping stream rather than merging it (verified against nats-server 2.10:
|
||||||
|
// "subjects overlap with an existing stream"), so this is not a subtle divergence — it is a
|
||||||
|
// registration that fails. Catching it here names both streams, before a half-applied mesh does.
|
||||||
|
func AllOverlaps(seats []DeclaredSeat) []string {
|
||||||
|
all := append(MeshStreams(), SeatStreams(seats)...)
|
||||||
|
seen := map[string]string{}
|
||||||
|
var clashes []string
|
||||||
|
for _, s := range all {
|
||||||
|
for _, subject := range s.Subjects {
|
||||||
|
if first, ok := seen[subject]; ok {
|
||||||
|
clashes = append(clashes, fmt.Sprintf("%s and %s both claim %s", first, s.Name, subject))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[subject] = s.Name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(clashes)
|
||||||
|
return clashes
|
||||||
|
}
|
||||||
|
|
||||||
|
// upperSnake makes a stream name from a seat name. NATS stream names may not contain a dot,
|
||||||
|
// a space or a wildcard, and a hyphen is legal but reads badly beside the mesh's own.
|
||||||
|
func upperSnake(s string) string {
|
||||||
|
out := []rune(s)
|
||||||
|
for i, r := range out {
|
||||||
|
switch {
|
||||||
|
case r >= 'a' && r <= 'z':
|
||||||
|
out[i] = r - 32
|
||||||
|
case r == '-' || r == '.':
|
||||||
|
out[i] = '_'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return string(out)
|
||||||
|
}
|
||||||
@@ -0,0 +1,155 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func telegramSeat() DeclaredSeat {
|
||||||
|
return DeclaredSeat{Name: "telegram-sender", Accepts: []string{"send"}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The stream exists from registration, not assignment: work queues until a holder appears, so
|
||||||
|
// installing the module a week later flushes the backlog rather than having lost it.
|
||||||
|
func TestASeatGetsAWorkQueueOfItsOwn(t *testing.T) {
|
||||||
|
got := SeatStreams([]DeclaredSeat{telegramSeat()})
|
||||||
|
if len(got) != 1 {
|
||||||
|
t.Fatalf("expected one stream, got %d", len(got))
|
||||||
|
}
|
||||||
|
s := got[0]
|
||||||
|
if s.Retention != RetentionWorkQueue {
|
||||||
|
t.Fatalf("a seat's inbound queue retains as %q; one holder must take each message once", s.Retention)
|
||||||
|
}
|
||||||
|
if s.Subjects[0] != "mesh.seat.telegram-sender.accept.>" {
|
||||||
|
t.Fatalf("filters on %v", s.Subjects)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A seat that only emits and serves needs no stream: its events ride EVENTS and its tools are
|
||||||
|
// core request/reply, which is never persisted.
|
||||||
|
func TestASeatThatAcceptsNothingGetsNoStream(t *testing.T) {
|
||||||
|
if got := SeatStreams([]DeclaredSeat{{Name: "announcer"}}); len(got) != 0 {
|
||||||
|
t.Fatalf("a seat with no inbound work got %d stream(s)", len(got))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Retention belongs to whoever owns the namespace, and a seat owns its own.
|
||||||
|
func TestASeatsRetentionIsItsOwn(t *testing.T) {
|
||||||
|
s := SeatStreams([]DeclaredSeat{{Name: "slow", Accepts: []string{"work"}, RetainSeconds: 30 * 24 * 60 * 60}})
|
||||||
|
if s[0].MaxAge != 30*24*60*60 {
|
||||||
|
t.Fatalf("the seat's declared retention was not used: %d", s[0].MaxAge)
|
||||||
|
}
|
||||||
|
d := SeatStreams([]DeclaredSeat{telegramSeat()})
|
||||||
|
if d[0].MaxAge == 0 {
|
||||||
|
t.Fatal("a seat that declares no retention got an unbounded queue")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NATS refuses an overlapping stream outright, so a clash here is a registration that fails.
|
||||||
|
func TestNoDerivedStreamOverlapsTheMeshsOwn(t *testing.T) {
|
||||||
|
seats := []DeclaredSeat{telegramSeat(), {Name: "licensing-master", Accepts: []string{"report"}}}
|
||||||
|
if c := AllOverlaps(seats); len(c) != 0 {
|
||||||
|
t.Fatalf("overlapping filters: %v", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// One consumer per module, with every consumed subject as a filter — because its ack permission
|
||||||
|
// is derived from its name, and a consumer per event would need an ack permission per consumer.
|
||||||
|
func TestAModuleGetsOneConsumerCarryingEveryFilter(t *testing.T) {
|
||||||
|
c, ok := ConsumerFor(Principal{Kind: KindModule, Node: "one", Module: "audit",
|
||||||
|
Consumes: []string{"shop.order.placed", "billing.invoice.sent"}, PasswordHash: "x"})
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("a module that consumes got no consumer")
|
||||||
|
}
|
||||||
|
if len(c.Filters) != 2 {
|
||||||
|
t.Fatalf("expected both subjects as filters, got %v", c.Filters)
|
||||||
|
}
|
||||||
|
perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "audit",
|
||||||
|
Consumes: []string{"shop.order.placed"}, PasswordHash: "x"})
|
||||||
|
ack := "$JS.ACK." + c.Stream + "." + c.Name + ".>"
|
||||||
|
found := false
|
||||||
|
for _, p := range perms.Publish {
|
||||||
|
if p == ack {
|
||||||
|
found = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatalf("the consumer is named %q but the ack permission is %v; a module could not ack "+
|
||||||
|
"its own deliveries", c.Name, perms.Publish)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A module that subscribes to nothing needs no consumer, and creating one would leave an object
|
||||||
|
// nothing reads and everything has to maintain.
|
||||||
|
func TestAModuleThatConsumesNothingGetsNoConsumer(t *testing.T) {
|
||||||
|
if _, ok := ConsumerFor(Principal{Kind: KindModule, Node: "one", Module: "shop",
|
||||||
|
Emits: []string{"order.placed"}, PasswordHash: "x"}); ok {
|
||||||
|
t.Fatal("a pure emitter got a consumer")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The seat is authority and the queue group is delivery. Tie them together and the day somebody
|
||||||
|
// allows two holders, every message is processed twice with nothing reporting it.
|
||||||
|
func TestAHoldersWorkerUsesAQueueGroupAnyway(t *testing.T) {
|
||||||
|
c, ok := HolderConsumerFor("one", "telegram", telegramSeat())
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("the holder of a seat with inbound work got no worker")
|
||||||
|
}
|
||||||
|
if c.Queue == "" {
|
||||||
|
t.Fatal("the worker is not in a queue group, so a second holder would double-process")
|
||||||
|
}
|
||||||
|
if c.Stream != "SEAT_TELEGRAM_SENDER" {
|
||||||
|
t.Fatalf("the worker reads %q, not the seat's own stream", c.Stream)
|
||||||
|
}
|
||||||
|
if c.MaxDeliver == 0 {
|
||||||
|
t.Fatal("a failing worker would redeliver forever rather than dead-letter")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The stream is named after the seat, not its holder: the holder can change and the queued work
|
||||||
|
// must not care.
|
||||||
|
func TestASeatsStreamIsNamedAfterTheSeat(t *testing.T) {
|
||||||
|
name := seatStreamName("telegram-sender")
|
||||||
|
if strings.Contains(name, "telegram-sender") {
|
||||||
|
t.Fatalf("%q keeps characters a stream name may not hold", name)
|
||||||
|
}
|
||||||
|
if name != "SEAT_TELEGRAM_SENDER" {
|
||||||
|
t.Fatalf("unexpected stream name %q", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A node hears its declaration through a consumer only the controller can make.
|
||||||
|
//
|
||||||
|
// The three things that would each break it silently: a name other than the node's is one the host
|
||||||
|
// cannot acknowledge a delivery from, because its ack grant is derived from the node's name; a
|
||||||
|
// filter other than its own declaration subject is a node reading another's; and a pull consumer is
|
||||||
|
// one the host cannot bind a channel to without creating something, which it has no authority for.
|
||||||
|
func TestANodesDeclarationConsumerIsWhatItsOwnGrantAllows(t *testing.T) {
|
||||||
|
c := NodeConsumer("anchor")
|
||||||
|
if c.Name != "anchor" {
|
||||||
|
t.Fatalf("named %q, so the node cannot ack from it: its grant is $JS.ACK.NODES.anchor.>", c.Name)
|
||||||
|
}
|
||||||
|
if c.Stream != "NODES" {
|
||||||
|
t.Fatalf("on stream %q rather than the one declarations live in", c.Stream)
|
||||||
|
}
|
||||||
|
if len(c.Filters) != 1 || c.Filters[0] != "mesh.node.anchor.declare" {
|
||||||
|
t.Fatalf("filters %v, which is not this node's own declaration and nothing else", c.Filters)
|
||||||
|
}
|
||||||
|
if !c.Push {
|
||||||
|
t.Fatal("pulled, which a host cannot do: pulling needs the JetStream API and a host reaches none of it")
|
||||||
|
}
|
||||||
|
if c.MaxDeliver != 0 {
|
||||||
|
t.Fatalf("max-deliver %d: a declaration a node has not taken yet is not one to dead-letter, "+
|
||||||
|
"because the stream holds exactly one per node", c.MaxDeliver)
|
||||||
|
}
|
||||||
|
|
||||||
|
// And the grant the node actually gets has to match, or none of the above matters.
|
||||||
|
perms, err := PermissionsFor(Principal{Kind: KindNode, Node: "anchor"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Without the ack grant every declaration a node receives is redelivered for ever; without the
|
||||||
|
// subscribe grant its consumer delivers to nobody.
|
||||||
|
has(t, perms.Publish, "$JS.ACK.NODES."+c.Name+".>")
|
||||||
|
has(t, perms.Subscribe, c.Filters[0])
|
||||||
|
}
|
||||||
@@ -0,0 +1,126 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"golang.org/x/crypto/bcrypt"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **The first user list the installer carries must be the one the controller would compose.**
|
||||||
|
//
|
||||||
|
// At genesis there is no mesh to write the bus's user list, so the installer carries one: the
|
||||||
|
// controller's own account, at a bootstrap password, the way the store is reached at
|
||||||
|
// `postgres:bootstrap` (novox/hq design 25 §4, task 1.7). It is written by hand in a template and
|
||||||
|
// derived in code here, which is two statements of one fact — so this compares them.
|
||||||
|
//
|
||||||
|
// Getting it wrong is the worst kind of silent: a controller whose carried permissions are narrower
|
||||||
|
// than the ones it derives comes up, connects, and is refused on the first thing it tries, with an
|
||||||
|
// authorisation error that names a subject and not the template that forgot it. And a mesh cannot be
|
||||||
|
// raised twice to find out.
|
||||||
|
func TestTheInstallersFirstUserListIsWhatTheControllerWouldCompose(t *testing.T) {
|
||||||
|
accounts := theCarriedAccounts(t)
|
||||||
|
|
||||||
|
want, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
carriedPub := subjectsIn(accounts, "publish")
|
||||||
|
carriedSub := subjectsIn(accounts, "subscribe")
|
||||||
|
|
||||||
|
if diff := missing(want.Publish, carriedPub); len(diff) > 0 {
|
||||||
|
t.Errorf("the installer's user list does not let the controller publish %v — it would come up "+
|
||||||
|
"and be refused on the first thing it tried", diff)
|
||||||
|
}
|
||||||
|
if diff := missing(want.Subscribe, carriedSub); len(diff) > 0 {
|
||||||
|
t.Errorf("the installer's user list does not let the controller subscribe %v", diff)
|
||||||
|
}
|
||||||
|
// And nothing wider than what it derives, or genesis quietly grants a privilege the composition
|
||||||
|
// takes away again on the first push.
|
||||||
|
if diff := missing(carriedPub, want.Publish); len(diff) > 0 {
|
||||||
|
t.Errorf("the installer's user list lets the controller publish %v, which it does not derive", diff)
|
||||||
|
}
|
||||||
|
if diff := missing(carriedSub, want.Subscribe); len(diff) > 0 {
|
||||||
|
t.Errorf("the installer's user list lets the controller subscribe %v, which it does not derive", diff)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The credential is the bootstrap one and the hash really is of it, because a hash of something
|
||||||
|
// else is a controller that cannot log in to the bus it was just given.
|
||||||
|
hash := regexp.MustCompile(`\$2[aby]?\$[0-9]+\$[A-Za-z0-9./]{53}`).FindString(accounts)
|
||||||
|
if hash == "" {
|
||||||
|
t.Fatal("the installer's user list carries no password hash")
|
||||||
|
}
|
||||||
|
if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte("bootstrap")); err != nil {
|
||||||
|
t.Fatalf("the carried hash does not verify the bootstrap credential the template also carries: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// theCarriedAccounts is the accounts file the installer's template writes at genesis.
|
||||||
|
func theCarriedAccounts(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
path := filepath.Join("..", "..", "..", "mesh-host", "examples", "foundation-first-node-nats.lock")
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Skipf("the host's checkout is not beside this one: %v", err)
|
||||||
|
}
|
||||||
|
// The template is JSON with line comments, which is how every one of them is written.
|
||||||
|
var lines []string
|
||||||
|
for _, l := range strings.Split(string(raw), "\n") {
|
||||||
|
if !strings.HasPrefix(strings.TrimSpace(l), "//") {
|
||||||
|
lines = append(lines, l)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var bundle struct {
|
||||||
|
Resources []map[string]any `json:"resources"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal([]byte(strings.Join(lines, "\n")), &bundle); err != nil {
|
||||||
|
t.Fatalf("the template is not readable: %v", err)
|
||||||
|
}
|
||||||
|
for _, r := range bundle.Resources {
|
||||||
|
if r["id"] == "bus-accounts" {
|
||||||
|
content, _ := r["content"].(string)
|
||||||
|
if content == "" {
|
||||||
|
t.Fatal("the template's accounts file is empty, so the bus would refuse every connection")
|
||||||
|
}
|
||||||
|
return content
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatal("the template carries no accounts file, so a mesh raised from it has a bus nobody may use")
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// subjectsIn reads one allow-list out of a composed accounts file.
|
||||||
|
func subjectsIn(accounts, which string) []string {
|
||||||
|
found := regexp.MustCompile(which + `: \{ allow: \[([^\]]*)\]`).FindStringSubmatch(accounts)
|
||||||
|
if len(found) != 2 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
for _, part := range strings.Split(found[1], ",") {
|
||||||
|
if s := strings.Trim(strings.TrimSpace(part), `"`); s != "" {
|
||||||
|
out = append(out, s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// missing is what is in want and not in got.
|
||||||
|
func missing(want, got []string) []string {
|
||||||
|
have := map[string]bool{}
|
||||||
|
for _, g := range got {
|
||||||
|
have[g] = true
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
for _, w := range want {
|
||||||
|
if !have[w] {
|
||||||
|
out = append(out, w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,150 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The JetStream side of the controller: the one place the mesh's streams and consumers are
|
||||||
|
// actually created.
|
||||||
|
//
|
||||||
|
// Everything that decides *what* they are is pure and lives beside this (streams.go, derived.go).
|
||||||
|
// This is only the part that talks to a server, kept small on purpose: a bug in a subject filter
|
||||||
|
// should be findable in a unit test, and only a bug in "did the server accept it" should need one
|
||||||
|
// running.
|
||||||
|
|
||||||
|
// A JetStream is a connection to the bus, as the controller uses it.
|
||||||
|
type JetStream struct {
|
||||||
|
conn *nats.Conn
|
||||||
|
js nats.JetStreamContext
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dial connects and returns the controller's JetStream handle.
|
||||||
|
func Dial(url string, opts ...nats.Option) (*JetStream, error) {
|
||||||
|
// A name, because a connection nobody can identify in the server's own monitoring is one
|
||||||
|
// nobody can attribute a problem to.
|
||||||
|
opts = append(opts, nats.Name("mesh-controller"), nats.Timeout(10*time.Second))
|
||||||
|
conn, err := nats.Connect(url, opts...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("connecting to the bus at %s: %w", url, err)
|
||||||
|
}
|
||||||
|
js, err := conn.JetStream()
|
||||||
|
if err != nil {
|
||||||
|
conn.Close()
|
||||||
|
return nil, fmt.Errorf("the bus at %s has no JetStream: %w", url, err)
|
||||||
|
}
|
||||||
|
return &JetStream{conn: conn, js: js}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Conn is the connection itself, for what the mesh keeps off JetStream on purpose — a heartbeat,
|
||||||
|
// a tool call — where a lost message is answered by the next one or by a timeout the caller
|
||||||
|
// already handles (design 25 §3).
|
||||||
|
func (j *JetStream) Conn() *nats.Conn { return j.conn }
|
||||||
|
|
||||||
|
// Context is the JetStream handle, for subscribing to what the consumers above define.
|
||||||
|
func (j *JetStream) Context() nats.JetStreamContext { return j.js }
|
||||||
|
|
||||||
|
func (j *JetStream) Close() {
|
||||||
|
if j.conn != nil {
|
||||||
|
j.conn.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// EnsureStream creates the stream if it is absent and brings it to match if it is present.
|
||||||
|
//
|
||||||
|
// **Idempotent, because the controller asserts on every start** rather than creating once at
|
||||||
|
// genesis: a stream somebody deleted, or a mesh raised from a restored backup, has to converge
|
||||||
|
// rather than run without the guarantee its messages assume.
|
||||||
|
//
|
||||||
|
// An update, not a delete and recreate. Recreating would discard every message the stream holds
|
||||||
|
// and every consumer's position in it — which for CONTROL means the pushes being held through a
|
||||||
|
// store restart, exactly the guarantee the stream exists for.
|
||||||
|
func (j *JetStream) EnsureStream(s Stream) error {
|
||||||
|
want := &nats.StreamConfig{
|
||||||
|
Name: s.Name,
|
||||||
|
Subjects: s.Subjects,
|
||||||
|
Retention: retentionOf(s.Retention),
|
||||||
|
MaxAge: time.Duration(s.MaxAge) * time.Second,
|
||||||
|
MaxMsgsPerSubject: int64(s.MaxMsgsPerSubject),
|
||||||
|
Description: s.Why,
|
||||||
|
}
|
||||||
|
if s.Retention == RetentionLastPerSubject {
|
||||||
|
// Last-per-subject is a limits stream with one message kept per subject, not a
|
||||||
|
// retention policy of its own — the state shape, spelled the way the server spells it.
|
||||||
|
want.Retention = nats.LimitsPolicy
|
||||||
|
want.MaxMsgsPerSubject = 1
|
||||||
|
want.MaxAge = 0
|
||||||
|
}
|
||||||
|
|
||||||
|
switch _, err := j.js.StreamInfo(s.Name); {
|
||||||
|
case err == nil:
|
||||||
|
if _, err := j.js.UpdateStream(want); err != nil {
|
||||||
|
return fmt.Errorf("bringing stream %s to match: %w", s.Name, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
case errors.Is(err, nats.ErrStreamNotFound):
|
||||||
|
if _, err := j.js.AddStream(want); err != nil {
|
||||||
|
return fmt.Errorf("creating stream %s: %w", s.Name, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("asking about stream %s: %w", s.Name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// EnsureConsumer creates or updates one durable consumer.
|
||||||
|
//
|
||||||
|
// Explicit acknowledgement throughout: a consumer that acknowledges on delivery cannot redeliver
|
||||||
|
// work its holder died in the middle of, which is the whole difference between a queue and a
|
||||||
|
// firehose.
|
||||||
|
func (j *JetStream) EnsureConsumer(c Consumer) error {
|
||||||
|
want := &nats.ConsumerConfig{
|
||||||
|
Durable: c.Name,
|
||||||
|
AckPolicy: nats.AckExplicitPolicy,
|
||||||
|
AckWait: time.Duration(c.AckWaitSeconds) * time.Second,
|
||||||
|
MaxDeliver: c.MaxDeliver,
|
||||||
|
DeliverGroup: c.Queue,
|
||||||
|
DeliverSubject: "",
|
||||||
|
Description: c.Why,
|
||||||
|
}
|
||||||
|
switch len(c.Filters) {
|
||||||
|
case 0:
|
||||||
|
case 1:
|
||||||
|
want.FilterSubject = c.Filters[0]
|
||||||
|
default:
|
||||||
|
want.FilterSubjects = c.Filters
|
||||||
|
}
|
||||||
|
// A queue group needs a delivery subject: a pull consumer has no group, and declaring one
|
||||||
|
// without the other is refused by the server with a message that does not say which half is
|
||||||
|
// missing.
|
||||||
|
if c.Queue != "" || c.Push {
|
||||||
|
want.DeliverSubject = "_DELIVER." + c.Name
|
||||||
|
}
|
||||||
|
|
||||||
|
switch _, err := j.js.ConsumerInfo(c.Stream, c.Name); {
|
||||||
|
case err == nil:
|
||||||
|
if _, err := j.js.UpdateConsumer(c.Stream, want); err != nil {
|
||||||
|
return fmt.Errorf("bringing consumer %s on %s to match: %w", c.Name, c.Stream, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
case errors.Is(err, nats.ErrConsumerNotFound):
|
||||||
|
if _, err := j.js.AddConsumer(c.Stream, want); err != nil {
|
||||||
|
return fmt.Errorf("creating consumer %s on %s: %w", c.Name, c.Stream, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("asking about consumer %s on %s: %w", c.Name, c.Stream, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func retentionOf(r Retention) nats.RetentionPolicy {
|
||||||
|
switch r {
|
||||||
|
case RetentionWorkQueue:
|
||||||
|
return nats.WorkQueuePolicy
|
||||||
|
default:
|
||||||
|
return nats.LimitsPolicy
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Against a real server, because the questions here are all "does the server accept this" —
|
||||||
|
// which a mock would answer by agreeing with whatever this file already believes.
|
||||||
|
//
|
||||||
|
// Skipped unless MESH_TEST_NATS names one, so the ordinary suite stays fast and offline:
|
||||||
|
//
|
||||||
|
// docker run -d --rm --name t -p 14222:4222 nats:2.10-alpine -js
|
||||||
|
// MESH_TEST_NATS=nats://127.0.0.1:14222 go test ./internal/broker/ -run TestAgainstARealServer
|
||||||
|
func TestAgainstARealServer(t *testing.T) {
|
||||||
|
url := os.Getenv("MESH_TEST_NATS")
|
||||||
|
if url == "" {
|
||||||
|
t.Skip("MESH_TEST_NATS unset")
|
||||||
|
}
|
||||||
|
js, err := Dial(url)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
|
||||||
|
t.Run("the mesh's own streams are accepted", func(t *testing.T) {
|
||||||
|
if err := AssertMeshStreams(js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("asserting again changes nothing and fails nothing", func(t *testing.T) {
|
||||||
|
if err := AssertMeshStreams(js); err != nil {
|
||||||
|
t.Fatalf("the second assertion failed, so the controller cannot restart: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("a seat's work queue is accepted beside them", func(t *testing.T) {
|
||||||
|
seats := []DeclaredSeat{{Name: "telegram-sender", Accepts: []string{"send"}}}
|
||||||
|
for _, s := range SeatStreams(seats) {
|
||||||
|
if err := js.EnsureStream(s); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if c := AllOverlaps(seats); len(c) != 0 {
|
||||||
|
t.Fatalf("overlaps the server would refuse: %v", c)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("a module's consumer is accepted and is idempotent", func(t *testing.T) {
|
||||||
|
c, ok := ConsumerFor(Principal{Kind: KindModule, Node: "one", Module: "audit",
|
||||||
|
Consumes: []string{"shop.order.placed", "billing.invoice.sent"}, PasswordHash: "x"})
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("no consumer derived")
|
||||||
|
}
|
||||||
|
if err := js.EnsureConsumer(c); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := js.EnsureConsumer(c); err != nil {
|
||||||
|
t.Fatalf("the second assertion failed: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("a holder's worker is accepted with its queue group", func(t *testing.T) {
|
||||||
|
c, _ := HolderConsumerFor("one", "telegram",
|
||||||
|
DeclaredSeat{Name: "telegram-sender", Accepts: []string{"send"}})
|
||||||
|
if err := js.EnsureConsumer(c); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,547 @@
|
|||||||
|
// Composing the bus's own configuration.
|
||||||
|
//
|
||||||
|
// An account is *composed*, never called for: the controller writes accounts, users and
|
||||||
|
// per-subject permissions into one file the host keeps current, and the server reloads it in
|
||||||
|
// place (novox/hq ADR 0106 — never through a management API; design 25 §4).
|
||||||
|
//
|
||||||
|
// Everything here is pure. Given the principals, it returns the file's text — so the whole of the
|
||||||
|
// mesh's authority model is testable as strings, with no server.
|
||||||
|
//
|
||||||
|
// **Permissions are per subject, so a module's own name is the server's to enforce.** ADR 0042
|
||||||
|
// reserves a module's origin — it publishes only under its own name — and here that is a refusal
|
||||||
|
// rather than something a library promises.
|
||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A Kind is what a principal is, which decides the shape of its authority rather than its
|
||||||
|
// contents: a module's comes from its declaration, a host's from its node, and the controller's
|
||||||
|
// and the enrolment user's are fixed.
|
||||||
|
type Kind string
|
||||||
|
|
||||||
|
const (
|
||||||
|
KindModule Kind = "module"
|
||||||
|
KindNode Kind = "node"
|
||||||
|
KindController Kind = "controller"
|
||||||
|
KindEnrolment Kind = "enrolment"
|
||||||
|
// KindPerson is somebody reaching the mesh's tools from a workstation (design 25 §7). Its
|
||||||
|
// authority is a list of tools and nothing else — not control, not declarations, not builds,
|
||||||
|
// and no ability to answer anything, because a person asks.
|
||||||
|
KindPerson Kind = "person"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Seat is a role on the bus as a principal relates to it: the subjects it accepts, and those it
|
||||||
|
// emits (novox/hq ADR 0118, design 29 §5).
|
||||||
|
type Seat struct {
|
||||||
|
Name string
|
||||||
|
Accepts []string
|
||||||
|
Emits []string
|
||||||
|
Serves []string
|
||||||
|
Versions []string // protocol versions served beside the current one; empty for v1 only
|
||||||
|
}
|
||||||
|
|
||||||
|
// A Principal is one user of the bus. Its permissions are derived from what it declares and
|
||||||
|
// nothing else (novox/hq ADR 0043), over the three namespaces of design 29 §2: its own, the seats
|
||||||
|
// it holds, and the seats it uses.
|
||||||
|
type Principal struct {
|
||||||
|
Kind Kind
|
||||||
|
Node string
|
||||||
|
Module string
|
||||||
|
|
||||||
|
Emits []string
|
||||||
|
Consumes []string
|
||||||
|
Serves []string
|
||||||
|
|
||||||
|
Holds []Seat
|
||||||
|
Uses []Seat
|
||||||
|
// Watches are seats whose events this principal consumes. Separate from Consumes because a
|
||||||
|
// role's event lives under the seat's namespace and not a module's, and this package cannot tell
|
||||||
|
// a seat's name from a module's by looking at it — whoever resolved the declaration can, and
|
||||||
|
// does (novox/hq ADR 0121).
|
||||||
|
//
|
||||||
|
// **Found by a consumer reading nothing.** The catalogue consumes the build machine's outcome;
|
||||||
|
// with that name read as a module's, its subscription pointed at `mesh.mod.mesh-build-machine.…`,
|
||||||
|
// a namespace no such module owns. Every service started and the graph stayed empty.
|
||||||
|
Watches []Seat
|
||||||
|
|
||||||
|
// Invokes are the tools a person may call, as `<module>.<tool>`; a single `*` is every tool,
|
||||||
|
// for an administrator. Only meaningful for KindPerson.
|
||||||
|
//
|
||||||
|
// **A list, not a role.** A person is not a module and holds no seat: nothing is addressed
|
||||||
|
// to them, nothing is delivered to them, and they have no durable consumer to acknowledge.
|
||||||
|
// What they have is permission to ask.
|
||||||
|
Invokes []string
|
||||||
|
|
||||||
|
// PasswordHash is the bcrypt hash the mesh minted. The plaintext is sealed to the principal
|
||||||
|
// and never appears here: this file is written to a node's disk and read by a server, and a
|
||||||
|
// secret that can be read from a configuration file is a secret with a wider blast radius
|
||||||
|
// than the one it protects (novox/hq design 29 §10).
|
||||||
|
PasswordHash string
|
||||||
|
}
|
||||||
|
|
||||||
|
// meshSeatsTheControllerUses are the roles the mesh's own flows submit work to. Named rather than
|
||||||
|
// derived from the seat set: the controller is not a module and declares no `uses`, so its side of a
|
||||||
|
// seat has to be stated, and a list is what makes "which roles does the mesh itself talk to" answerable.
|
||||||
|
var meshSeatsTheControllerUses = []string{"mesh-build-machine"}
|
||||||
|
|
||||||
|
// enrolmentPrefix is the space every enrolling node's user and inbox live under, so the one place the
|
||||||
|
// controller may answer an enrolment is derived from the same constant the user is named from.
|
||||||
|
const enrolmentPrefix = "enrol"
|
||||||
|
|
||||||
|
// safeSubject refuses anything that would change the meaning of a subject rather than sit inside
|
||||||
|
// one. A name carrying a dot would silently widen a permission by adding a token; a name carrying
|
||||||
|
// `>` or `*` would widen it to a wildcard, which is the whole authority model gone.
|
||||||
|
var safeSubject = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)
|
||||||
|
|
||||||
|
// Username is how a principal is named to the server. The node is part of it, so the same module
|
||||||
|
// on two machines holds two users, each sealed to its own — the rule management.go already
|
||||||
|
// applies, kept.
|
||||||
|
func (p Principal) Username() string {
|
||||||
|
switch p.Kind {
|
||||||
|
case KindPerson:
|
||||||
|
return "person." + p.Module
|
||||||
|
case KindModule:
|
||||||
|
return p.Node + "." + p.Module
|
||||||
|
case KindNode:
|
||||||
|
return "node." + p.Node
|
||||||
|
case KindController:
|
||||||
|
return "controller"
|
||||||
|
case KindEnrolment:
|
||||||
|
// Per token, not one shared user. **The inbox is the reason**: with a single `enrolment`
|
||||||
|
// user every machine enrolling at once could read every other's answer, and an answer
|
||||||
|
// carries that node's credentials sealed to it. Design 25 §6 says the inbox a token
|
||||||
|
// derives, and a permission belongs to a user, so the user is per token.
|
||||||
|
//
|
||||||
|
// Named after the node, which **is** the token's id: a token is issued for a node record,
|
||||||
|
// the mesh holds one live claim per record, and the node's name is the one identifier both
|
||||||
|
// sides already have before anything else is agreed. It is also exactly what the other
|
||||||
|
// transport does, where the account is named after the node and the secret is its password.
|
||||||
|
return enrolmentPrefix + "." + p.Node
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// inbox is a principal's own reply space. No user is ever granted a bare `_INBOX.>` (design 25
|
||||||
|
// §4): with one account, inbox privacy is the permission list or it is nothing, so each user's
|
||||||
|
// inbox is derived from its own identity and its permissions name that prefix and no other.
|
||||||
|
func (p Principal) inbox() string { return "_INBOX." + p.Username() + ".>" }
|
||||||
|
|
||||||
|
// Permissions is what a principal may publish and subscribe, and whether it may answer.
|
||||||
|
type Permissions struct {
|
||||||
|
Publish []string
|
||||||
|
Subscribe []string
|
||||||
|
// AllowResponses lets a principal reply to a request it received, on the reply subject that
|
||||||
|
// request carried, once.
|
||||||
|
//
|
||||||
|
// **This is what makes scoped inboxes possible at all**, and design 25 §4 did not say it. If
|
||||||
|
// every user's inbox is private to it, a module serving a tool cannot publish the answer —
|
||||||
|
// the answer goes to the *caller's* inbox, which the responder has no permission for. The two
|
||||||
|
// ways out are granting responders `_INBOX.>`, which is precisely the blanket grant §4
|
||||||
|
// refuses, or this: the server itself permits one reply to the subject of a message the user
|
||||||
|
// actually received, and nothing else. The authority is bounded by having been asked.
|
||||||
|
AllowResponses bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// PermissionsFor derives a principal's authority. Pure, and the only place authority is decided:
|
||||||
|
// a permission that cannot be derived from a declaration is a permission nobody can explain.
|
||||||
|
func PermissionsFor(p Principal) (Permissions, error) {
|
||||||
|
for _, part := range []struct{ what, value string }{
|
||||||
|
{"node", p.Node}, {"module", p.Module},
|
||||||
|
} {
|
||||||
|
if part.value == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !safeSubject.MatchString(part.value) {
|
||||||
|
return Permissions{}, fmt.Errorf(
|
||||||
|
"%q cannot be part of a subject: a permission is a subject pattern, and this would widen it", part.value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var pub, sub []string
|
||||||
|
switch p.Kind {
|
||||||
|
case KindController:
|
||||||
|
// The controller owns the mesh's own traffic and the streams. It is the only writer of
|
||||||
|
// stream definitions (design 25 §3), so it alone reaches the JetStream API.
|
||||||
|
pub = []string{"mesh.control.>", "mesh.node.>", "$JS.API.>"}
|
||||||
|
sub = []string{"mesh.control.>", "$JS.API.>"}
|
||||||
|
|
||||||
|
// Work the mesh's own flows submit to a role, and the outcomes they wait on (ADR 0121). A
|
||||||
|
// build is the one today: the controller asks, and reads the answer from the seat's event
|
||||||
|
// like the catalogue does — which is why no holder needs to publish into anybody's inbox.
|
||||||
|
for _, seat := range meshSeatsTheControllerUses {
|
||||||
|
pub = append(pub, "mesh.seat."+seat+".accept.>")
|
||||||
|
}
|
||||||
|
|
||||||
|
// The two events it reacts to, and its ack subject on the stream they arrive from
|
||||||
|
// (streams.go). **Each named, not a pattern**: `mesh.mod.*.event.>` would make the
|
||||||
|
// controller a subscriber to every event in the mesh, and its permission list would stop
|
||||||
|
// saying what it is for. The ack grant below is scoped per stream because the controller's
|
||||||
|
// consumer name is the same on both and `$JS.ACK.CONTROL.controller.>` does not cover a
|
||||||
|
// delivery from EVENTS — a consumer that cannot ack has every message redelivered for
|
||||||
|
// ever, refused by the list it already has.
|
||||||
|
sub = append(sub, ControllerFollows...)
|
||||||
|
pub = append(pub, "$JS.ACK.EVENTS."+ControllerName+".>")
|
||||||
|
|
||||||
|
// **Where an enrolment's answer goes**, and `allow_responses` does not cover it. That
|
||||||
|
// permits one reply to the reply subject of a message the user received — and a message a
|
||||||
|
// JetStream consumer delivers has had that field claimed for the consumer's own ack address
|
||||||
|
// (design 25 §2), so the address the controller actually answers is the one the request
|
||||||
|
// carried in its payload, which is not a reply subject as the server understands it.
|
||||||
|
//
|
||||||
|
// Verified against a real server before this line existed: the answer was refused with
|
||||||
|
// "Permissions Violation for Publish to _INBOX.enrol.anchor…", and every enrolment on the
|
||||||
|
// mesh would have timed out while the controller logged success.
|
||||||
|
//
|
||||||
|
// **The enrolment inbox space, not a blanket `_INBOX.>`.** Design 25 §4 refuses that, and
|
||||||
|
// this is not it: nothing but an enrolling node ever subscribes under this prefix, each
|
||||||
|
// scoped to its own token's, so the controller publishing here is the mesh answering
|
||||||
|
// enrolments and can reach nothing else.
|
||||||
|
pub = append(pub, "_INBOX."+enrolmentPrefix+".>")
|
||||||
|
|
||||||
|
case KindPerson:
|
||||||
|
// Tools, and nothing else. Every subject a person may publish is a tool call; a person
|
||||||
|
// who could publish an event would be able to claim a module said something.
|
||||||
|
for _, t := range p.Invokes {
|
||||||
|
if t == "*" {
|
||||||
|
pub = append(pub, "mesh.mod.*.tool.>")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
module, tool, ok := strings.Cut(t, ".")
|
||||||
|
if !ok {
|
||||||
|
return Permissions{}, fmt.Errorf(
|
||||||
|
"%q does not name a tool: a person invokes <module>.<tool>, or * for every one", t)
|
||||||
|
}
|
||||||
|
pub = append(pub, "mesh.mod."+module+".tool."+tool)
|
||||||
|
}
|
||||||
|
|
||||||
|
case KindEnrolment:
|
||||||
|
// A leaked token is useless for anything but enrolling: it cannot read a declaration, hear
|
||||||
|
// an event, or subscribe any inbox but the one its own token derives (design 25 §6).
|
||||||
|
//
|
||||||
|
// **The inbox was missing and the handshake could not have completed without it.** An
|
||||||
|
// enrolling node publishes its request and waits on an address it states in the payload;
|
||||||
|
// with nothing to subscribe it waits out its timeout against a mesh that answered. Its own
|
||||||
|
// and no wider: `_INBOX.enrol.<node>.>`, so what is sealed to one machine cannot be read by
|
||||||
|
// another enrolling beside it.
|
||||||
|
if p.Node == "" {
|
||||||
|
// Refused rather than composed into `_INBOX.enrol..>`, which is a subject with an empty
|
||||||
|
// token in it — and worse, one every nameless enrolment user would share. A shared
|
||||||
|
// enrolment inbox is one machine able to read the credentials sealed to another.
|
||||||
|
return Permissions{}, errors.New(
|
||||||
|
"an enrolment user names no node, so its inbox would be shared with every other " +
|
||||||
|
"enrolment: a token is issued for a node record, and that record's name is " +
|
||||||
|
"the token's id")
|
||||||
|
}
|
||||||
|
pub = []string{"mesh.control.enrol"}
|
||||||
|
sub = []string{p.inbox()}
|
||||||
|
|
||||||
|
case KindNode:
|
||||||
|
// A host publishes its own node's control traffic and subscribes its own declaration —
|
||||||
|
// and nothing of any other node's.
|
||||||
|
pub = []string{"mesh.control." + p.Node + ".>"}
|
||||||
|
sub = []string{"mesh.node." + p.Node + ".declare"}
|
||||||
|
|
||||||
|
case KindModule:
|
||||||
|
// 1. Its own namespace: it publishes its events there and serves its tools there. Nothing
|
||||||
|
// else may publish into it, so an event's source is a fact the server enforces rather
|
||||||
|
// than a claim in the body (design 29 §2).
|
||||||
|
own := "mesh.mod." + p.Module
|
||||||
|
for _, e := range p.Emits {
|
||||||
|
pub = append(pub, own+".event."+e)
|
||||||
|
}
|
||||||
|
for _, t := range p.Serves {
|
||||||
|
sub = append(sub, own+".tool."+t)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. What it consumes, by the emitter's own subject — an event is addressed to its
|
||||||
|
// emitter, because the emitter's identity is the meaning (ADR 0118).
|
||||||
|
for _, c := range p.Consumes {
|
||||||
|
subject, err := consumedSubject(c)
|
||||||
|
if err != nil {
|
||||||
|
return Permissions{}, err
|
||||||
|
}
|
||||||
|
sub = append(sub, subject)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2b. Events of a role it watches, under the seat's own namespace. Subscribe only: watching a
|
||||||
|
// role is hearing what it announced, not taking part in it.
|
||||||
|
for _, w := range p.Watches {
|
||||||
|
for _, e := range w.Emits {
|
||||||
|
sub = append(sub, seatSubject(w, "event", e))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Seats it holds: full participation.
|
||||||
|
for _, s := range p.Holds {
|
||||||
|
for _, a := range s.Accepts {
|
||||||
|
sub = append(sub, seatSubject(s, "accept", a))
|
||||||
|
}
|
||||||
|
for _, e := range s.Emits {
|
||||||
|
pub = append(pub, seatSubject(s, "event", e))
|
||||||
|
}
|
||||||
|
for _, t := range s.Serves {
|
||||||
|
sub = append(sub, seatSubject(s, "tool", t))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. Seats it uses: publish only, and only the accepts half. A caller cannot subscribe a
|
||||||
|
// seat's inbound subject and watch other modules' traffic, nor publish its outbound
|
||||||
|
// events and lie about outcomes (design 29 §2).
|
||||||
|
for _, s := range p.Uses {
|
||||||
|
for _, a := range s.Accepts {
|
||||||
|
pub = append(pub, seatSubject(s, "accept", a))
|
||||||
|
}
|
||||||
|
for _, t := range s.Serves {
|
||||||
|
pub = append(pub, seatSubject(s, "tool", t))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if p.Kind == KindPerson {
|
||||||
|
// An inbox to hear answers in, and nothing else. No ack subject: a person has no durable
|
||||||
|
// consumer, because nothing is delivered to a person — they ask and are answered.
|
||||||
|
sub = append(sub, p.inbox())
|
||||||
|
}
|
||||||
|
|
||||||
|
if p.Kind == KindModule || p.Kind == KindNode || p.Kind == KindController {
|
||||||
|
// Its own reply space, and nothing wider.
|
||||||
|
sub = append(sub, p.inbox())
|
||||||
|
|
||||||
|
// Acking a JetStream delivery is a publish to that consumer's own ack address — a
|
||||||
|
// different subject from anything the consumer subscribes. Without it every message a
|
||||||
|
// module received would be redelivered forever, refused by the permission list it already
|
||||||
|
// has (design 25 §4). Scoped to this principal's own consumer name, so it can ack its own
|
||||||
|
// deliveries and no other's.
|
||||||
|
pub = append(pub, "$JS.ACK."+consumerStream(p)+"."+consumerDurable(p)+".>")
|
||||||
|
}
|
||||||
|
|
||||||
|
sort.Strings(pub)
|
||||||
|
sort.Strings(sub)
|
||||||
|
return Permissions{
|
||||||
|
Publish: pub,
|
||||||
|
Subscribe: sub,
|
||||||
|
// Only something that serves is ever answering. A pure consumer is granted nothing here.
|
||||||
|
AllowResponses: p.Kind == KindModule && (len(p.Serves) > 0 || len(p.Holds) > 0) ||
|
||||||
|
p.Kind == KindController,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// seatSubject places a seat's verb under the kind of traffic it is.
|
||||||
|
//
|
||||||
|
// **The kind token is load-bearing, not decoration.** A stream is defined by a subject filter, so
|
||||||
|
// without it a stream over a seat or a module's namespace would capture that namespace's *tool*
|
||||||
|
// traffic too — and a tool call must never be persisted (design 25 §3: tools stay on core NATS).
|
||||||
|
// Found while defining the streams: the first draft of design 29 had one namespace per module
|
||||||
|
// with no kind, which reads well and cannot be filtered.
|
||||||
|
//
|
||||||
|
// A seat serving more than its current protocol version carries the version as a token
|
||||||
|
// (design 29 §8): the seat stays one role, and v1 and v2 run beside each other until nothing is
|
||||||
|
// bound to the old one.
|
||||||
|
func seatSubject(s Seat, kind, verb string) string {
|
||||||
|
return "mesh.seat." + s.Name + "." + kind + "." + verb
|
||||||
|
}
|
||||||
|
|
||||||
|
// consumerStream and consumerDurable are the two halves of a consumer's identity, and they are
|
||||||
|
// two functions because conflating them was a real bug.
|
||||||
|
//
|
||||||
|
// **A durable name may not contain a dot; an ack subject is built from two names that do.** The
|
||||||
|
// server acknowledges on `$JS.ACK.<stream>.<consumer>.…`, so a single string "EVENTS.one_audit"
|
||||||
|
// reads correctly inside the permission and is rejected as a consumer name — *nats: invalid
|
||||||
|
// consumer name*. Caught against a running server, and worth the comment because the shape of
|
||||||
|
// the failure if it had not been is the one design 25 §4 warns about: a consumer that cannot ack
|
||||||
|
// has every message redelivered forever, and its permission list looks right while it happens.
|
||||||
|
//
|
||||||
|
// They are derived here, beside the permission that must match them, because two places deriving
|
||||||
|
// the same name is how a module ends up unable to ack its own deliveries.
|
||||||
|
// consumedSubject is where a consumed event lands, from the local pattern a module declared.
|
||||||
|
//
|
||||||
|
// **The mesh's wildcards become this transport's** (design 29 §1): `*` is one name on both, and `**`
|
||||||
|
// — the rest — is `>` here. A module writes neither transport's spelling, so a manifest stays correct
|
||||||
|
// when the wire changes, which is the whole reason names are local.
|
||||||
|
//
|
||||||
|
// `**` on its own is every event from every module: the emitter is any, the event is anything. An
|
||||||
|
// audit logger wants exactly that and says so in one token.
|
||||||
|
func consumedSubject(pattern string) (string, error) {
|
||||||
|
if pattern == catalogueTheRest {
|
||||||
|
return "mesh.mod.*.event.>", nil
|
||||||
|
}
|
||||||
|
emitter, event, named := strings.Cut(pattern, ".")
|
||||||
|
if !named || emitter == "" || event == "" {
|
||||||
|
return "", fmt.Errorf(
|
||||||
|
"%q does not name an emitter and an event: a consumed event is <emitter>.<event>, or "+
|
||||||
|
"%q for every event", pattern, catalogueTheRest)
|
||||||
|
}
|
||||||
|
if emitter == catalogueTheRest {
|
||||||
|
return "", fmt.Errorf("%q stands for the rest of a name, so it cannot name the emitter", catalogueTheRest)
|
||||||
|
}
|
||||||
|
// Each name is checked before it becomes a subject: a name carrying a dot would add a token and
|
||||||
|
// silently widen the permission, which is the whole reason safeSubject exists.
|
||||||
|
var out []string
|
||||||
|
for _, part := range strings.Split(event, ".") {
|
||||||
|
switch part {
|
||||||
|
case catalogueTheRest:
|
||||||
|
out = append(out, ">")
|
||||||
|
case "*":
|
||||||
|
out = append(out, "*")
|
||||||
|
default:
|
||||||
|
if !safeSubject.MatchString(part) {
|
||||||
|
return "", fmt.Errorf("%q cannot be part of a subject: it would widen the permission", part)
|
||||||
|
}
|
||||||
|
out = append(out, part)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if emitter != "*" && !safeSubject.MatchString(emitter) {
|
||||||
|
return "", fmt.Errorf("%q cannot name an emitter: it would widen the permission", emitter)
|
||||||
|
}
|
||||||
|
return "mesh.mod." + emitter + ".event." + strings.Join(out, "."), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// catalogueTheRest is the mesh's wildcard for "the rest of a name", duplicated from the catalogue
|
||||||
|
// package for the one direction of dependency the build queue's name is duplicated for.
|
||||||
|
const catalogueTheRest = "**"
|
||||||
|
|
||||||
|
func consumerStream(p Principal) string {
|
||||||
|
switch p.Kind {
|
||||||
|
case KindModule:
|
||||||
|
return "EVENTS"
|
||||||
|
case KindNode:
|
||||||
|
return "NODES"
|
||||||
|
case KindController:
|
||||||
|
return "CONTROL"
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func consumerDurable(p Principal) string {
|
||||||
|
switch p.Kind {
|
||||||
|
case KindModule:
|
||||||
|
return p.Node + "_" + p.Module
|
||||||
|
case KindNode:
|
||||||
|
return p.Node
|
||||||
|
case KindController:
|
||||||
|
return "controller"
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// Server is everything the composed file needs that is not a principal.
|
||||||
|
type Server struct {
|
||||||
|
// ClientPort carries TLS itself. There is no plaintext port beside it: a bus reachable
|
||||||
|
// without TLS is one a module can reach without TLS by mistake.
|
||||||
|
ClientPort int
|
||||||
|
MonitoringPort int
|
||||||
|
TLSCert string
|
||||||
|
TLSKey string
|
||||||
|
TLSCA string
|
||||||
|
// StoreDir is a host directory bind, not a named volume — issue 115 is resolved and converted
|
||||||
|
// four modules away from named volumes; the bus's own data is not the place to bring one back.
|
||||||
|
StoreDir string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compose renders the server's whole configuration. The order is stable and the output is
|
||||||
|
// deterministic, because the file's digest is what the module's entrypoint watches to decide
|
||||||
|
// whether to reload: a composer that reordered a map on each run would signal a reload every time
|
||||||
|
// the controller restarted, for a file that had not changed.
|
||||||
|
func Compose(s Server, principals []Principal) (string, error) {
|
||||||
|
sorted := append([]Principal(nil), principals...)
|
||||||
|
sort.Slice(sorted, func(i, j int) bool { return sorted[i].Username() < sorted[j].Username() })
|
||||||
|
|
||||||
|
var b strings.Builder
|
||||||
|
b.WriteString("# Composed by the mesh controller. Do not edit: the next composition overwrites it.\n")
|
||||||
|
b.WriteString("# Accounts and permissions are derived from what each module declares and nothing\n")
|
||||||
|
b.WriteString("# else (novox/hq ADR 0043, design 29 §2).\n\n")
|
||||||
|
|
||||||
|
fmt.Fprintf(&b, "port: %d\n", s.ClientPort)
|
||||||
|
fmt.Fprintf(&b, "http: 127.0.0.1:%d\n\n", s.MonitoringPort)
|
||||||
|
|
||||||
|
// **No `verify`, and it said `verify: true` until this configuration was run.** That setting
|
||||||
|
// makes the server demand a *client* certificate, and nothing in the mesh presents one: a host
|
||||||
|
// pins this server's exact certificate and authenticates with the password the mesh minted
|
||||||
|
// (ADR 0004, design 25 §4), and so does a module's runtime. With it on, every connection in the
|
||||||
|
// mesh is refused at the TLS handshake before any password is looked at, and the error —
|
||||||
|
// "client didn't provide a certificate" — reads as a fault in the client.
|
||||||
|
//
|
||||||
|
// TLS is still required: the block is what requires it, and verify only decides whether client
|
||||||
|
// certificates are checked.
|
||||||
|
b.WriteString("tls {\n")
|
||||||
|
fmt.Fprintf(&b, " cert_file: %q\n", s.TLSCert)
|
||||||
|
fmt.Fprintf(&b, " key_file: %q\n", s.TLSKey)
|
||||||
|
fmt.Fprintf(&b, " ca_file: %q\n", s.TLSCA)
|
||||||
|
b.WriteString("}\n\n")
|
||||||
|
|
||||||
|
b.WriteString("jetstream {\n")
|
||||||
|
fmt.Fprintf(&b, " store_dir: %q\n", s.StoreDir)
|
||||||
|
b.WriteString("}\n\n")
|
||||||
|
|
||||||
|
accounts, err := ComposeAccounts(sorted)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
b.WriteString(accounts)
|
||||||
|
return b.String(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ComposeAccounts is the accounts block alone — every user, and nothing about the server.
|
||||||
|
//
|
||||||
|
// **This is the only part of the configuration the mesh writes, and the split is deliberate.** A
|
||||||
|
// server's ports, its TLS paths and its store directory are properties of the container the module
|
||||||
|
// raises: they live in its image and its mounts, and they change when it does. The controller has no
|
||||||
|
// business knowing them, and a controller that did would have to be kept in step with a Dockerfile
|
||||||
|
// it never sees. What only the mesh knows is *who may connect*, so that is what it writes, and the
|
||||||
|
// module's own configuration includes it.
|
||||||
|
//
|
||||||
|
// Four things checked against a running server before this shape was committed to: a user in an
|
||||||
|
// included file authenticates; an unknown user is refused, so the include is the whole authority
|
||||||
|
// rather than an addition to something; a publish outside a user's grant is refused; and rewriting
|
||||||
|
// this file alone and signalling a reload makes a new user appear **without dropping the connection
|
||||||
|
// the mesh already has** — which is what makes every later account, permission or person change cost
|
||||||
|
// nothing (task 1.2's payoff).
|
||||||
|
func ComposeAccounts(principals []Principal) (string, error) {
|
||||||
|
sorted := append([]Principal(nil), principals...)
|
||||||
|
sort.Slice(sorted, func(i, j int) bool { return sorted[i].Username() < sorted[j].Username() })
|
||||||
|
|
||||||
|
var b strings.Builder
|
||||||
|
b.WriteString("# The mesh's users, composed by the controller. Do not edit: the next\n")
|
||||||
|
b.WriteString("# composition overwrites it. Permissions are derived from what each module\n")
|
||||||
|
b.WriteString("# declares and nothing else (novox/hq ADR 0043, design 29 §2).\n\n")
|
||||||
|
|
||||||
|
// One account for the mesh: accounts in NATS isolate subject spaces entirely, and the mesh is
|
||||||
|
// one space (design 25 §4). The cost of that — that permissions are the only isolation — is
|
||||||
|
// paid in the scoping of every inbox and every ack subject.
|
||||||
|
b.WriteString("accounts {\n MESH {\n users = [\n")
|
||||||
|
for _, p := range sorted {
|
||||||
|
perms, err := PermissionsFor(p)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if p.PasswordHash == "" {
|
||||||
|
return "", fmt.Errorf("%s has no password hash: a user without one is a user anybody is", p.Username())
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&b, " { user: %q, password: %q, permissions: {\n", p.Username(), p.PasswordHash)
|
||||||
|
fmt.Fprintf(&b, " publish: { allow: [%s] }\n", quoted(perms.Publish))
|
||||||
|
fmt.Fprintf(&b, " subscribe: { allow: [%s] }\n", quoted(perms.Subscribe))
|
||||||
|
if perms.AllowResponses {
|
||||||
|
b.WriteString(" allow_responses: { max: 1, ttl: \"1m\" }\n")
|
||||||
|
}
|
||||||
|
b.WriteString(" } }\n")
|
||||||
|
}
|
||||||
|
b.WriteString(" ]\n }\n}\n")
|
||||||
|
return b.String(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func quoted(values []string) string {
|
||||||
|
if len(values) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
out := make([]string, len(values))
|
||||||
|
for i, v := range values {
|
||||||
|
out[i] = fmt.Sprintf("%q", v)
|
||||||
|
}
|
||||||
|
return strings.Join(out, ", ")
|
||||||
|
}
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"flag"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
var update = flag.Bool("update", false, "rewrite the golden composition")
|
||||||
|
|
||||||
|
// The composed file is the mesh's whole authority model, so a change to it should be visible in a
|
||||||
|
// review rather than inferred from a diff of Go. The fixture is also the exact text checked
|
||||||
|
// against the real server's parser (`nats-server -t`), which is what says this syntax is the
|
||||||
|
// server's and not one we invented.
|
||||||
|
func TestTheComposedConfigMatchesTheGolden(t *testing.T) {
|
||||||
|
seat := Seat{Name: "telegram-sender", Accepts: []string{"send"}, Emits: []string{"delivered", "failed"}}
|
||||||
|
got, err := Compose(
|
||||||
|
Server{ClientPort: 4222, MonitoringPort: 8222, StoreDir: "/data",
|
||||||
|
TLSCert: "/tls/tls.crt", TLSKey: "/tls/tls.key", TLSCA: "/tls/ca.crt"},
|
||||||
|
[]Principal{
|
||||||
|
{Kind: KindController, PasswordHash: "$2a$11$cccccccccccccccccccccc"},
|
||||||
|
{Kind: KindEnrolment, Node: "one", PasswordHash: "$2a$11$eeeeeeeeeeeeeeeeeeeeee"},
|
||||||
|
{Kind: KindNode, Node: "one", PasswordHash: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn"},
|
||||||
|
{Kind: KindModule, Node: "one", Module: "telegram", Holds: []Seat{seat},
|
||||||
|
Serves: []string{"status"}, PasswordHash: "$2a$11$tttttttttttttttttttttt"},
|
||||||
|
{Kind: KindModule, Node: "two", Module: "shop", Uses: []Seat{seat},
|
||||||
|
Emits: []string{"order.placed"}, PasswordHash: "$2a$11$ssssssssssssssssssssss"},
|
||||||
|
{Kind: KindModule, Node: "two", Module: "audit",
|
||||||
|
Consumes: []string{"shop.order.placed"}, PasswordHash: "$2a$11$aaaaaaaaaaaaaaaaaaaaaa"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
golden := filepath.Join("testdata", "composed.conf")
|
||||||
|
if *update {
|
||||||
|
if err := os.WriteFile(golden, []byte(got), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
want, err := os.ReadFile(golden)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got != string(want) {
|
||||||
|
t.Errorf("composition changed; re-run with -update and read the diff:\n%s", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,326 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func has(t *testing.T, subjects []string, want string) {
|
||||||
|
t.Helper()
|
||||||
|
for _, s := range subjects {
|
||||||
|
if s == want {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("expected %q among %v", want, subjects)
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasNot(t *testing.T, subjects []string, unwanted string) {
|
||||||
|
t.Helper()
|
||||||
|
for _, s := range subjects {
|
||||||
|
if s == unwanted {
|
||||||
|
t.Fatalf("did not expect %q among %v", unwanted, subjects)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A module's authority comes from its declaration and nothing else (novox/hq ADR 0043).
|
||||||
|
func TestAModulePublishesOnlyWhatItEmits(t *testing.T) {
|
||||||
|
p := Principal{Kind: KindModule, Node: "one", Module: "billing",
|
||||||
|
Emits: []string{"order.placed"}, PasswordHash: "x"}
|
||||||
|
perms, err := PermissionsFor(p)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
has(t, perms.Publish, "mesh.mod.billing.event.order.placed")
|
||||||
|
hasNot(t, perms.Publish, "mesh.mod.billing.>")
|
||||||
|
hasNot(t, perms.Publish, "mesh.mod.shipping.event.order.placed")
|
||||||
|
}
|
||||||
|
|
||||||
|
// The gap AMQP left open — an emitter granted the events exchange whole — is closed by per-subject
|
||||||
|
// permissions. A module cannot publish under another module's name.
|
||||||
|
func TestAModuleCannotPublishUnderAnothersName(t *testing.T) {
|
||||||
|
perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "billing",
|
||||||
|
Emits: []string{"order.placed"}, PasswordHash: "x"})
|
||||||
|
for _, p := range perms.Publish {
|
||||||
|
if strings.HasPrefix(p, "mesh.mod.") && !strings.HasPrefix(p, "mesh.mod.billing.") {
|
||||||
|
t.Fatalf("billing may publish %q, which is not its own namespace", p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A caller of a seat may publish what the seat accepts, and nothing else of it: not its outbound
|
||||||
|
// events, and not a subscription to its inbound queue (design 29 §2).
|
||||||
|
func TestUsingASeatIsPublishOnlyAndInboundOnly(t *testing.T) {
|
||||||
|
seat := Seat{Name: "telegram-sender", Accepts: []string{"send"}, Emits: []string{"delivered", "failed"}}
|
||||||
|
perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "shop",
|
||||||
|
Uses: []Seat{seat}, PasswordHash: "x"})
|
||||||
|
has(t, perms.Publish, "mesh.seat.telegram-sender.accept.send")
|
||||||
|
hasNot(t, perms.Publish, "mesh.seat.telegram-sender.event.delivered")
|
||||||
|
hasNot(t, perms.Subscribe, "mesh.seat.telegram-sender.accept.send")
|
||||||
|
}
|
||||||
|
|
||||||
|
// The holder is the mirror image: it consumes what the seat accepts and publishes what it emits.
|
||||||
|
func TestHoldingASeatIsTheMirrorOfUsingIt(t *testing.T) {
|
||||||
|
seat := Seat{Name: "telegram-sender", Accepts: []string{"send"}, Emits: []string{"delivered", "failed"}}
|
||||||
|
perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "telegram",
|
||||||
|
Holds: []Seat{seat}, PasswordHash: "x"})
|
||||||
|
has(t, perms.Subscribe, "mesh.seat.telegram-sender.accept.send")
|
||||||
|
has(t, perms.Publish, "mesh.seat.telegram-sender.event.delivered")
|
||||||
|
hasNot(t, perms.Publish, "mesh.seat.telegram-sender.accept.send")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Without an ack permission a durable consumer never really consumes: every message it receives is
|
||||||
|
// redelivered forever, refused by the permission list it already has (design 25 §4).
|
||||||
|
func TestAModuleMayAckItsOwnDeliveriesAndNoOthers(t *testing.T) {
|
||||||
|
perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "billing",
|
||||||
|
Consumes: []string{"shop.order.placed"}, PasswordHash: "x"})
|
||||||
|
has(t, perms.Publish, "$JS.ACK.EVENTS.one_billing.>")
|
||||||
|
hasNot(t, perms.Publish, "$JS.ACK.>")
|
||||||
|
hasNot(t, perms.Publish, "$JS.ACK.EVENTS.one_shop.>")
|
||||||
|
}
|
||||||
|
|
||||||
|
// With one account, inbox privacy is the permission list or it is nothing.
|
||||||
|
func TestAnInboxIsScopedToItsOwner(t *testing.T) {
|
||||||
|
perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "billing", PasswordHash: "x"})
|
||||||
|
has(t, perms.Subscribe, "_INBOX.one.billing.>")
|
||||||
|
hasNot(t, perms.Subscribe, "_INBOX.>")
|
||||||
|
hasNot(t, perms.Subscribe, "_INBOX.one.shop.>")
|
||||||
|
}
|
||||||
|
|
||||||
|
// A responder answers on the caller's inbox, which it has no permission for. allow_responses is
|
||||||
|
// what makes a scoped inbox workable at all — the authority is bounded by having been asked.
|
||||||
|
func TestOnlySomethingThatServesMayAnswer(t *testing.T) {
|
||||||
|
serving, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "billing",
|
||||||
|
Serves: []string{"status"}, PasswordHash: "x"})
|
||||||
|
if !serving.AllowResponses {
|
||||||
|
t.Fatal("a module serving a tool cannot answer the caller's inbox")
|
||||||
|
}
|
||||||
|
consumer, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "audit",
|
||||||
|
Consumes: []string{"shop.order.placed"}, PasswordHash: "x"})
|
||||||
|
if consumer.AllowResponses {
|
||||||
|
t.Fatal("a pure consumer was granted the right to answer, which nothing asked it to do")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A host reaches its own node's control traffic and its own declaration, and nothing of any
|
||||||
|
// other node's.
|
||||||
|
func TestAHostIsConfinedToItsOwnNode(t *testing.T) {
|
||||||
|
perms, _ := PermissionsFor(Principal{Kind: KindNode, Node: "one", PasswordHash: "x"})
|
||||||
|
has(t, perms.Publish, "mesh.control.one.>")
|
||||||
|
has(t, perms.Subscribe, "mesh.node.one.declare")
|
||||||
|
hasNot(t, perms.Subscribe, "mesh.node.two.declare")
|
||||||
|
hasNot(t, perms.Subscribe, "mesh.node.>")
|
||||||
|
}
|
||||||
|
|
||||||
|
// A leaked enrolment token is useless for anything but enrolling (design 25 §6).
|
||||||
|
func TestTheEnrolmentUserCanOnlyEnrol(t *testing.T) {
|
||||||
|
perms, err := PermissionsFor(Principal{Kind: KindEnrolment, Node: "anchor", PasswordHash: "x"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(perms.Publish) != 1 || perms.Publish[0] != "mesh.control.enrol" {
|
||||||
|
t.Fatalf("enrolment may publish %v", perms.Publish)
|
||||||
|
}
|
||||||
|
// Its own inbox and nothing else. **Nothing else** is the point: no declaration, no event, and
|
||||||
|
// no other machine's answer — and the inbox itself is needed, because a node that cannot
|
||||||
|
// subscribe one waits out its timeout against a mesh that answered.
|
||||||
|
if len(perms.Subscribe) != 1 || perms.Subscribe[0] != "_INBOX.enrol.anchor.>" {
|
||||||
|
t.Fatalf("enrolment may subscribe %v, which is not its own inbox alone", perms.Subscribe)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An enrolment user that names no node is refused: its inbox would be an empty subject token, and
|
||||||
|
// one that every nameless enrolment user shared — which is one machine reading the credentials
|
||||||
|
// sealed to another.
|
||||||
|
func TestAnEnrolmentUserWithoutANodeIsRefused(t *testing.T) {
|
||||||
|
if _, err := PermissionsFor(Principal{Kind: KindEnrolment, PasswordHash: "x"}); err == nil {
|
||||||
|
t.Fatal("an enrolment user with no node was composed, so its inbox is shared")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A name that would widen a permission is refused rather than quietly stretching one.
|
||||||
|
func TestANameThatWouldWidenAPermissionIsRefused(t *testing.T) {
|
||||||
|
for _, bad := range []string{"bill.ing", "billing.>", "*", "bil>ling"} {
|
||||||
|
if _, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: bad, PasswordHash: "x"}); err == nil {
|
||||||
|
t.Fatalf("%q was accepted as part of a subject", bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The entrypoint reloads on the file's digest changing, so an unchanged mesh must compose an
|
||||||
|
// identical file — otherwise every controller restart signals a reload of the whole bus.
|
||||||
|
func TestComposingTwiceGivesTheSameBytes(t *testing.T) {
|
||||||
|
s := Server{ClientPort: 4222, MonitoringPort: 8222, StoreDir: "/data",
|
||||||
|
TLSCert: "/tls/tls.crt", TLSKey: "/tls/tls.key", TLSCA: "/tls/ca.crt"}
|
||||||
|
ps := []Principal{
|
||||||
|
{Kind: KindModule, Node: "two", Module: "shop", Emits: []string{"order.placed"}, PasswordHash: "b"},
|
||||||
|
{Kind: KindController, PasswordHash: "c"},
|
||||||
|
{Kind: KindModule, Node: "one", Module: "billing", Consumes: []string{"shop.order.placed"}, PasswordHash: "a"},
|
||||||
|
}
|
||||||
|
first, err := Compose(s, ps)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
shuffled := []Principal{ps[2], ps[0], ps[1]}
|
||||||
|
second, err := Compose(s, shuffled)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if first != second {
|
||||||
|
t.Fatal("composition is order-dependent; every controller restart would reload the bus")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A user without a password is a user anybody is.
|
||||||
|
func TestAUserWithoutAPasswordIsRefused(t *testing.T) {
|
||||||
|
_, err := Compose(Server{ClientPort: 4222}, []Principal{{Kind: KindController}})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("composed a user with no password hash")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A person reaches the mesh's tools from a workstation (design 25 §7). Their authority is a list
|
||||||
|
// of tools and nothing else.
|
||||||
|
func TestAPersonMayAskOnlyTheToolsTheyWereGiven(t *testing.T) {
|
||||||
|
perms, err := PermissionsFor(Principal{Kind: KindPerson, Module: "jo",
|
||||||
|
Invokes: []string{"shop.price", "telegram.status"}, PasswordHash: "x"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
has(t, perms.Publish, "mesh.mod.shop.tool.price")
|
||||||
|
has(t, perms.Publish, "mesh.mod.telegram.tool.status")
|
||||||
|
hasNot(t, perms.Publish, "mesh.mod.shop.tool.refund")
|
||||||
|
hasNot(t, perms.Publish, "mesh.mod.*.tool.>")
|
||||||
|
}
|
||||||
|
|
||||||
|
// An administrator gets every tool, which is a different grant and looks like one.
|
||||||
|
func TestAnAdministratorMayAskAnyTool(t *testing.T) {
|
||||||
|
perms, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo",
|
||||||
|
Invokes: []string{"*"}, PasswordHash: "x"})
|
||||||
|
has(t, perms.Publish, "mesh.mod.*.tool.>")
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Nothing but tools.** A person who could publish an event would be able to claim a module
|
||||||
|
// said something; one who could publish control traffic would be a second controller.
|
||||||
|
func TestAPersonReachesNothingButTools(t *testing.T) {
|
||||||
|
perms, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo",
|
||||||
|
Invokes: []string{"*"}, PasswordHash: "x"})
|
||||||
|
for _, p := range perms.Publish {
|
||||||
|
if !strings.Contains(p, ".tool.") {
|
||||||
|
t.Errorf("a person may publish %q, which is not a tool call", p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, s := range perms.Subscribe {
|
||||||
|
if !strings.HasPrefix(s, "_INBOX.person.") {
|
||||||
|
t.Errorf("a person may subscribe %q; only their own inbox should be reachable", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A person has no durable consumer, because nothing is delivered to a person — so no ack
|
||||||
|
// subject, and an ack permission would be authority over something that does not exist.
|
||||||
|
func TestAPersonHasNoAckSubject(t *testing.T) {
|
||||||
|
perms, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo",
|
||||||
|
Invokes: []string{"*"}, PasswordHash: "x"})
|
||||||
|
for _, p := range perms.Publish {
|
||||||
|
if strings.HasPrefix(p, "$JS.ACK") {
|
||||||
|
t.Errorf("a person was granted %q, and has no consumer to acknowledge", p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A person asks and is answered; they never answer. allow_responses would let a person reply to
|
||||||
|
// a request — which, on a bus where anyone may serve a tool, is somebody impersonating a module.
|
||||||
|
func TestAPersonMayNotAnswer(t *testing.T) {
|
||||||
|
perms, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo",
|
||||||
|
Invokes: []string{"*"}, PasswordHash: "x"})
|
||||||
|
if perms.AllowResponses {
|
||||||
|
t.Fatal("a person may answer a request, which is impersonating a module")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Two people do not share an inbox, or one would read the other's answers.
|
||||||
|
func TestTwoPeopleDoNotShareAnInbox(t *testing.T) {
|
||||||
|
a, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo", Invokes: []string{"*"}, PasswordHash: "x"})
|
||||||
|
b, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "sam", Invokes: []string{"*"}, PasswordHash: "x"})
|
||||||
|
if a.Subscribe[0] == b.Subscribe[0] {
|
||||||
|
t.Fatalf("both read %s", a.Subscribe[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A malformed grant is refused rather than widened into something that happens to parse.
|
||||||
|
func TestAToolGrantThatNamesNoToolIsRefused(t *testing.T) {
|
||||||
|
if _, err := PermissionsFor(Principal{Kind: KindPerson, Module: "jo",
|
||||||
|
Invokes: []string{"shop"}, PasswordHash: "x"}); err == nil {
|
||||||
|
t.Fatal("a grant naming a module but no tool was accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The controller can answer an enrolment, and reach no other inbox.
|
||||||
|
//
|
||||||
|
// **`allow_responses` does not cover this and that is the trap.** It permits one reply to the reply
|
||||||
|
// subject of a message the user received — and a message a JetStream consumer delivers has had that
|
||||||
|
// field claimed for the consumer's own ack address, so the address the controller actually answers is
|
||||||
|
// the one the request carried in its payload, which the server does not recognise as a reply subject
|
||||||
|
// at all.
|
||||||
|
//
|
||||||
|
// Found against a real server, after a live test on an *unpermissioned* one had passed: every
|
||||||
|
// enrolment on the mesh would have timed out while the controller logged success.
|
||||||
|
func TestTheControllerCanAnswerAnEnrolmentAndReachNoOtherInbox(t *testing.T) {
|
||||||
|
ctl, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
enrolling, err := PermissionsFor(Principal{Kind: KindEnrolment, Node: "anchor", PasswordHash: "x"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Whatever the enrolling node waits on, the controller must be able to publish to.
|
||||||
|
if len(enrolling.Subscribe) != 1 {
|
||||||
|
t.Fatalf("an enrolling node subscribes %v, and this test knows only how to check one",
|
||||||
|
enrolling.Subscribe)
|
||||||
|
}
|
||||||
|
waitsOn := enrolling.Subscribe[0]
|
||||||
|
if !covers(ctl.Publish, waitsOn) {
|
||||||
|
t.Fatalf("the controller may publish %v, none of which reaches %s — so every enrolment on "+
|
||||||
|
"the mesh times out while the controller logs success", ctl.Publish, waitsOn)
|
||||||
|
}
|
||||||
|
|
||||||
|
// And nothing wider. A node's own inbox and a module's are not the controller's to write into:
|
||||||
|
// that is the blanket grant design 25 §4 refuses.
|
||||||
|
for _, other := range []string{"_INBOX.node.anchor.x", "_INBOX.one.shop.x", "_INBOX.person.ada.x"} {
|
||||||
|
if covers(ctl.Publish, other) {
|
||||||
|
t.Errorf("the controller can publish to %s, which is an inbox privacy the permission "+
|
||||||
|
"list is the only thing protecting", other)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// covers says whether any granted subject pattern admits one concrete subject, with NATS's own
|
||||||
|
// wildcard meanings: `*` is one token, `>` is the rest.
|
||||||
|
func covers(granted []string, subject string) bool {
|
||||||
|
want := strings.Split(subject, ".")
|
||||||
|
for _, pattern := range granted {
|
||||||
|
if admits(strings.Split(pattern, "."), want) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func admits(pattern, subject []string) bool {
|
||||||
|
for i, token := range pattern {
|
||||||
|
if token == ">" {
|
||||||
|
return i < len(subject)
|
||||||
|
}
|
||||||
|
if i >= len(subject) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if token != "*" && token != subject[i] {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return len(pattern) == len(subject)
|
||||||
|
}
|
||||||
@@ -0,0 +1,91 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/envfile"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Whether this mesh's own traffic is on the bus being built.
|
||||||
|
//
|
||||||
|
// **One switch, read in one place** (novox/hq ADR 0116 step 5). Every seam the bus change went
|
||||||
|
// behind ships both implementations, and until the rollout every one of them chooses the bus the
|
||||||
|
// mesh runs on today. This is what the rollout flips, and it is deliberately a single fact rather
|
||||||
|
// than a fact per component: a controller whose outbound is on one bus and whose inbound is on the
|
||||||
|
// other is a mesh that hears nothing, and no test of either half would catch it.
|
||||||
|
|
||||||
|
// NATSVar is where the controller finds the bus being built. Unset is the ordinary case and means
|
||||||
|
// the mesh runs on the bus it has always run on.
|
||||||
|
const NATSVar = "MESH_BUS_NATS"
|
||||||
|
|
||||||
|
// OnNATS is the address of the bus being built, and whether the mesh is on it.
|
||||||
|
//
|
||||||
|
// Read from the node's own settings rather than baked in, for the reason the broker's address is
|
||||||
|
// (novox/hq 04-ISSUES/102): an address recorded once does not follow a node's ports.
|
||||||
|
func OnNATS() (address string, on bool, err error) {
|
||||||
|
address, err = envfile.Placed(NATSVar)
|
||||||
|
if err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
address = strings.TrimSpace(address)
|
||||||
|
if address == "" {
|
||||||
|
return "", false, nil
|
||||||
|
}
|
||||||
|
return address, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// CredentialIn reads the user and password out of a bus address, and the address without them.
|
||||||
|
//
|
||||||
|
// The controller's own credential arrives in its address, the way the old bus's does. Split out so the
|
||||||
|
// controller can record a hash of what it is actually using: its user is created by the installer at a
|
||||||
|
// bootstrap password, before the controller exists to mint one, and a composition that left itself out
|
||||||
|
// would produce a bus the writer cannot connect to.
|
||||||
|
func CredentialIn(address string) (user, password, bare string) {
|
||||||
|
at := strings.LastIndex(address, "@")
|
||||||
|
if at < 0 {
|
||||||
|
return "", "", address
|
||||||
|
}
|
||||||
|
scheme := ""
|
||||||
|
rest := address[:at]
|
||||||
|
if i := strings.Index(rest, "://"); i >= 0 {
|
||||||
|
scheme, rest = rest[:i+3], rest[i+3:]
|
||||||
|
}
|
||||||
|
user, password, _ = strings.Cut(rest, ":")
|
||||||
|
return user, password, scheme + address[at+1:]
|
||||||
|
}
|
||||||
|
|
||||||
|
// BareAddress is a bus address with any credential stripped, for something that only needs to know
|
||||||
|
// whether a server is answering there.
|
||||||
|
func BareAddress(address string) string {
|
||||||
|
_, _, bare := CredentialIn(address)
|
||||||
|
if bare == "" {
|
||||||
|
return address
|
||||||
|
}
|
||||||
|
if strings.Contains(bare, "://") {
|
||||||
|
return bare
|
||||||
|
}
|
||||||
|
return "nats://" + bare
|
||||||
|
}
|
||||||
|
|
||||||
|
// MustBeOneBus refuses a configuration that names both buses for the mesh's own traffic.
|
||||||
|
//
|
||||||
|
// **Both clients ship and that is the point; both being live is not.** The rollout moves every node
|
||||||
|
// at once (ADR 0116 step 5): a mesh half on each is one where a declaration goes out on one bus and
|
||||||
|
// the report comes back on the other, and nothing anywhere says so — every component would log
|
||||||
|
// success. Refused at start, where it can be said in one sentence.
|
||||||
|
func MustBeOneBus(amqp, nats string) error {
|
||||||
|
if strings.TrimSpace(amqp) != "" && strings.TrimSpace(nats) != "" {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"this control plane is told about both buses (%s and %s) and can only be on one. A mesh "+
|
||||||
|
"half on each is one where a declaration goes out on one and the report comes back "+
|
||||||
|
"on the other, and every component reports success while it happens. The rollout "+
|
||||||
|
"moves every node at once: unset %s to stay, or unset %s to move",
|
||||||
|
AMQPVarName, NATSVar, NATSVar, AMQPVarName)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// AMQPVarName is the variable naming the bus the mesh runs on today. Named here rather than
|
||||||
|
// imported from the link package, for the one direction of dependency.
|
||||||
|
const AMQPVarName = "MESH_BROKER_AMQP"
|
||||||
@@ -0,0 +1,60 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Which bus the mesh is on is one fact, and being told about both is refused.
|
||||||
|
//
|
||||||
|
// **Not a warning.** A mesh half on each bus is one where a declaration goes out on one and the
|
||||||
|
// report comes back on the other, and every component reports success while it happens — which is
|
||||||
|
// the exact failure ADR 0074 exists to catch, arriving through configuration instead of through code.
|
||||||
|
func TestBeingToldAboutBothBusesIsRefused(t *testing.T) {
|
||||||
|
err := MustBeOneBus("amqps://broker:5671/", "nats://bus:4222")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a control plane told about both buses was allowed to start")
|
||||||
|
}
|
||||||
|
// The remedy is in the words, because whoever reads this has to choose one and the wrong choice
|
||||||
|
// is a rollout half done.
|
||||||
|
for _, want := range []string{AMQPVarName, NATSVar, "unset"} {
|
||||||
|
if !strings.Contains(err.Error(), want) {
|
||||||
|
t.Errorf("the refusal does not mention %s: %v", want, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// One bus, or none, is ordinary. None is a control plane that publishes nothing and holds records,
|
||||||
|
// which several of its own commands are.
|
||||||
|
func TestOneBusOrNeitherIsAllowed(t *testing.T) {
|
||||||
|
for _, c := range []struct{ what, amqp, nats string }{
|
||||||
|
{"the bus the mesh runs on today", "amqps://broker:5671/", ""},
|
||||||
|
{"the bus being built", "", "nats://bus:4222"},
|
||||||
|
{"neither", "", ""},
|
||||||
|
{"neither, with whitespace for an address", " ", "\t"},
|
||||||
|
} {
|
||||||
|
if err := MustBeOneBus(c.amqp, c.nats); err != nil {
|
||||||
|
t.Errorf("%s was refused: %v", c.what, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The controller's own credential arrives in its address, and has to be readable out of it — its user
|
||||||
|
// is created by the installer at a bootstrap password, before the controller exists to mint one.
|
||||||
|
func TestACredentialIsReadOutOfABusAddress(t *testing.T) {
|
||||||
|
for _, c := range []struct{ in, user, password, bare string }{
|
||||||
|
{"nats://controller:secret@127.0.0.1:4222", "controller", "secret", "nats://127.0.0.1:4222"},
|
||||||
|
{"controller:secret@127.0.0.1:4222", "controller", "secret", "127.0.0.1:4222"},
|
||||||
|
{"nats://127.0.0.1:4222", "", "", "nats://127.0.0.1:4222"},
|
||||||
|
{"127.0.0.1:4222", "", "", "127.0.0.1:4222"},
|
||||||
|
// A password containing an at-sign: split on the last one, or the address becomes part of the
|
||||||
|
// credential and the connection goes somewhere nobody named.
|
||||||
|
{"nats://controller:a@b@127.0.0.1:4222", "controller", "a@b", "nats://127.0.0.1:4222"},
|
||||||
|
} {
|
||||||
|
user, password, bare := CredentialIn(c.in)
|
||||||
|
if user != c.user || password != c.password || bare != c.bare {
|
||||||
|
t.Errorf("%q read as %q/%q at %q; wanted %q/%q at %q",
|
||||||
|
c.in, user, password, bare, c.user, c.password, c.bare)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import "fmt"
|
||||||
|
|
||||||
|
// Bringing the bus's own objects into being, in the one order that works.
|
||||||
|
//
|
||||||
|
// **Asserted on every start rather than created once at genesis.** A stream somebody deleted, a mesh
|
||||||
|
// raised from a restored backup, or a bus whose data directory was replaced all have records and no
|
||||||
|
// objects — and a node whose consumer is missing hears nothing while everything else about it looks
|
||||||
|
// correct. Idempotence is the whole requirement, and the parts are already idempotent; this is the
|
||||||
|
// order they have to be asked in.
|
||||||
|
|
||||||
|
// Raiser is everything asserting the bus's objects needs of a connection to it.
|
||||||
|
type Raiser interface {
|
||||||
|
Asserter
|
||||||
|
Ensurer
|
||||||
|
}
|
||||||
|
|
||||||
|
// Raise asserts the mesh's streams, the controller's own consumers, and one consumer per node.
|
||||||
|
//
|
||||||
|
// **The order is not a preference.** A consumer on a stream that does not exist is refused, and the
|
||||||
|
// refusal names the stream rather than the order — so somebody reading it goes looking for a deleted
|
||||||
|
// stream instead of a reversed pair of lines. Nodes last, because the one a node reads lives on a
|
||||||
|
// stream the mesh's own set defines.
|
||||||
|
func Raise(r Raiser, nodes []string) error {
|
||||||
|
if err := AssertMeshStreams(r); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := AssertMeshConsumers(r); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := AssertNodeConsumers(r, nodes); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// RaiseSeats asserts one work queue per declared seat, and the worker of whoever holds it.
|
||||||
|
//
|
||||||
|
// Separate from Raise because it is answered by a different question: the mesh's own objects exist
|
||||||
|
// because the mesh does, and a seat's exist because a module declaring one was registered. Kept
|
||||||
|
// beside it so the order is visible — a holder's worker needs the seat's stream, and a seat's stream
|
||||||
|
// needs nothing.
|
||||||
|
func RaiseSeats(r Raiser, seats []DeclaredSeat, holders map[string]Holder) error {
|
||||||
|
for _, s := range SeatStreams(seats) {
|
||||||
|
if err := r.EnsureStream(s); err != nil {
|
||||||
|
return fmt.Errorf("asserting the work queue for %s: %w", s.Name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, s := range seats {
|
||||||
|
h, held := holders[s.Name]
|
||||||
|
if !held {
|
||||||
|
// **The stream exists and the consumer does not, on purpose.** Work queues until a
|
||||||
|
// holder appears, so installing the module a week after something started sending to it
|
||||||
|
// flushes the backlog instead of having lost it.
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
c, needed := HolderConsumerFor(h.Node, h.Module, s)
|
||||||
|
if !needed {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := r.EnsureConsumer(c); err != nil {
|
||||||
|
return fmt.Errorf("asserting how %s on %s works %s: %w", h.Module, h.Node, s.Name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Holder is which module on which machine holds a seat.
|
||||||
|
type Holder struct {
|
||||||
|
Node string
|
||||||
|
Module string
|
||||||
|
}
|
||||||
@@ -0,0 +1,196 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Raising the bus's objects against a real server.
|
||||||
|
//
|
||||||
|
// The pure tests above say what is asked for and in what order. Only a server can say whether it
|
||||||
|
// accepts them — and two of these are claims about the server's own behaviour that nothing else
|
||||||
|
// could answer: that asserting twice changes nothing, and that a consumer really is bound to the one
|
||||||
|
// subject its node is allowed to read.
|
||||||
|
//
|
||||||
|
// docker run -d --rm --name t -p 14227:4222 nats:2.10-alpine -js
|
||||||
|
// MESH_TEST_NATS=nats://127.0.0.1:14227 go test ./internal/broker/ -run TestRaising
|
||||||
|
|
||||||
|
func aLiveBus(t *testing.T) *JetStream {
|
||||||
|
t.Helper()
|
||||||
|
url := os.Getenv("MESH_TEST_NATS")
|
||||||
|
if url == "" {
|
||||||
|
t.Skip("MESH_TEST_NATS unset")
|
||||||
|
}
|
||||||
|
js, err := Dial(url)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(js.Close)
|
||||||
|
// **Nothing is deleted here, deliberately.** These objects are the mesh's own and every live
|
||||||
|
// test in every package shares one server: a test that deleted a stream to get a clean slate
|
||||||
|
// took it out from under whatever was running beside it, and the failure landed in the other
|
||||||
|
// test as "stream not found" — which reads as a bug in the code under test. Raise is idempotent
|
||||||
|
// by requirement, so asserting against whatever is already there is both safe and the realistic
|
||||||
|
// case.
|
||||||
|
return js
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every object the mesh's own traffic needs, accepted by a real server, and asserting again changes
|
||||||
|
// nothing — which is the whole requirement, because this runs on every start.
|
||||||
|
func TestRaisingTheBusIsAcceptedAndIdempotent(t *testing.T) {
|
||||||
|
js := aLiveBus(t)
|
||||||
|
|
||||||
|
if err := Raise(js, []string{"anchor", "laptop"}); err != nil {
|
||||||
|
t.Fatalf("a real server refused the mesh's own objects: %v", err)
|
||||||
|
}
|
||||||
|
// Twice, with nothing in between. A start that failed the second time is a controller that
|
||||||
|
// cannot restart.
|
||||||
|
if err := Raise(js, []string{"anchor", "laptop"}); err != nil {
|
||||||
|
t.Fatalf("asserting the bus's objects a second time failed, so a restart would: %v", err)
|
||||||
|
}
|
||||||
|
// And again with a machine that was not there before, which is what enrolling one is.
|
||||||
|
if err := Raise(js, []string{"anchor", "laptop", "workstation"}); err != nil {
|
||||||
|
t.Fatalf("a machine joining an already-raised bus was refused: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, s := range MeshStreams() {
|
||||||
|
if _, err := js.Context().StreamInfo(s.Name); err != nil {
|
||||||
|
t.Errorf("stream %s is not there: %v", s.Name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, c := range MeshConsumers() {
|
||||||
|
if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); err != nil {
|
||||||
|
t.Errorf("the controller's consumer on %s is not there: %v", c.Stream, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, node := range []string{"anchor", "laptop", "workstation"} {
|
||||||
|
info, err := js.Context().ConsumerInfo("NODES", node)
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("%s has no way to hear its declaration: %v", node, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// **Its own subject and no other node's.** A consumer filtered on anything wider is a node
|
||||||
|
// reading another machine's declaration, and its own ack grant would not cover it either.
|
||||||
|
if info.Config.FilterSubject != "mesh.node."+node+".declare" {
|
||||||
|
t.Errorf("%s's consumer reads %q", node, info.Config.FilterSubject)
|
||||||
|
}
|
||||||
|
if info.Config.AckPolicy != nats.AckExplicitPolicy {
|
||||||
|
t.Errorf("%s's consumer acknowledges on delivery, so a declaration it died applying is "+
|
||||||
|
"never sent again", node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The store window needs unlimited redelivery on CONTROL: the bound belongs to the controller, and a
|
||||||
|
// server that dead-lettered first would discard the push the stream exists to protect.
|
||||||
|
func TestTheControlConsumerDoesNotDeadLetterBeforeTheControllerGivesUp(t *testing.T) {
|
||||||
|
js := aLiveBus(t)
|
||||||
|
if err := Raise(js, nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
info, err := js.Context().ConsumerInfo("CONTROL", ControllerName)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if info.Config.MaxDeliver > 0 {
|
||||||
|
t.Fatalf("max-deliver is %d: a push held through a store restart would be dead-lettered "+
|
||||||
|
"before the controller finished deciding about it", info.Config.MaxDeliver)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A role's work queue exists before anybody holds it, against a real server.
|
||||||
|
//
|
||||||
|
// **The queue before the holder is the point** (novox/hq ADR 0121): work queues until somebody arrives
|
||||||
|
// to do it, so assigning a build machine a week after something started asking for builds flushes the
|
||||||
|
// backlog instead of having lost it. A stream created at assignment would make "the holder is not here
|
||||||
|
// yet" mean "your requests are gone".
|
||||||
|
func TestRaisingAMeshRolesWorkQueue(t *testing.T) {
|
||||||
|
js := aLiveBus(t)
|
||||||
|
seats := []DeclaredSeat{{Name: "mesh-build-machine", Accepts: []string{"build"},
|
||||||
|
Emits: []string{"built"}}}
|
||||||
|
t.Cleanup(func() { _ = js.Context().DeleteStream("SEAT_MESH_BUILD_MACHINE") })
|
||||||
|
|
||||||
|
if err := RaiseSeats(js, seats, nil); err != nil {
|
||||||
|
t.Fatalf("a real server refused a role's work queue: %v", err)
|
||||||
|
}
|
||||||
|
info, err := js.Context().StreamInfo("SEAT_MESH_BUILD_MACHINE")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("the role has no work queue: %v", err)
|
||||||
|
}
|
||||||
|
if info.Config.Retention != nats.WorkQueuePolicy {
|
||||||
|
t.Errorf("the queue retains as %v: work a holder took must leave it, or the next holder does "+
|
||||||
|
"it again", info.Config.Retention)
|
||||||
|
}
|
||||||
|
if len(info.Config.Subjects) != 1 || info.Config.Subjects[0] != "mesh.seat.mesh-build-machine.accept.>" {
|
||||||
|
t.Errorf("it carries %v rather than the role's own inbound subjects", info.Config.Subjects)
|
||||||
|
}
|
||||||
|
// Nobody holds it, so there is no worker — and asserting again changes nothing, because this runs
|
||||||
|
// on every start.
|
||||||
|
if err := RaiseSeats(js, seats, nil); err != nil {
|
||||||
|
t.Fatalf("asserting a role's queue a second time failed, so a restart would: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// And once somebody holds it, the worker appears on that same queue.
|
||||||
|
if err := RaiseSeats(js, seats, map[string]Holder{
|
||||||
|
"mesh-build-machine": {Node: "anchor", Module: "builder"},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := js.Context().ConsumerInfo("SEAT_MESH_BUILD_MACHINE",
|
||||||
|
"SEAT_MESH_BUILD_MACHINE_worker"); err != nil {
|
||||||
|
t.Fatalf("the holder got no worker on the role's queue: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A consumer created after the fact still sees what came before it**, which is why the mesh needs no
|
||||||
|
// catch-up at all on this bus (novox/hq 04-ISSUES/050).
|
||||||
|
//
|
||||||
|
// On the bus the mesh runs on today a queue receives only what is published after it is bound, so
|
||||||
|
// everything built before the catalogue existed was announced to nobody — and on a fresh mesh that is
|
||||||
|
// always the foundation, because those are the things the catalogue needed in order to exist. A whole
|
||||||
|
// mechanism was built for it: the catalogue asks, the controller re-publishes.
|
||||||
|
//
|
||||||
|
// A stream is a log and a consumer is a position in it. A consumer created later starts at the
|
||||||
|
// beginning by default, so the builds are simply there. Asked of a real server rather than assumed,
|
||||||
|
// because the whole decision about whether to keep that mechanism rests on it.
|
||||||
|
func TestAConsumerCreatedAfterwardsStillSeesWhatCameBefore(t *testing.T) {
|
||||||
|
js := aLiveBus(t)
|
||||||
|
if err := AssertMeshStreams(js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := js.Context().PurgeStream("EVENTS"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Genesis: things are built before anything is listening.
|
||||||
|
built := []string{"base", "store", "mesh-catalog"}
|
||||||
|
for _, m := range built {
|
||||||
|
if _, err := js.Context().Publish("mesh.seat.mesh-build-machine.event.built",
|
||||||
|
[]byte(`{"module":"`+m+`"}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Now the catalogue is installed and the controller creates its consumer.
|
||||||
|
c, ok := ConsumerFor(Principal{Kind: KindModule, Node: "one", Module: "mesh-catalog",
|
||||||
|
Watches: []Seat{{Name: "mesh-build-machine", Emits: []string{"built"}}}, PasswordHash: "x"})
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("a module that watches a role got no consumer")
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = js.Context().DeleteConsumer(c.Stream, c.Name) })
|
||||||
|
if err := js.EnsureConsumer(c); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
info, err := js.Context().ConsumerInfo(c.Stream, c.Name)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if info.NumPending != uint64(len(built)) {
|
||||||
|
t.Fatalf("a consumer created after %d builds has %d waiting for it — if this is 0 the mesh "+
|
||||||
|
"does need a catch-up after all, and the reasoning for deleting it is wrong",
|
||||||
|
len(built), info.NumPending)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,142 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Whether a mesh could move its bus, and what is missing if not.
|
||||||
|
//
|
||||||
|
// **Asked before anything moves, and answerable from records alone.** The rollout moves every node at
|
||||||
|
// once (novox/hq ADR 0116 step 5), so there is no partial state to inspect afterwards and no half to
|
||||||
|
// roll back: either the mesh was ready or it was not. That makes a readiness question the most
|
||||||
|
// valuable thing here — it costs nothing, it can be asked of a running mesh any number of times, and
|
||||||
|
// every answer is a thing somebody can go and fix.
|
||||||
|
//
|
||||||
|
// Deliberately pure. It is handed what the mesh knows and returns sentences; nothing here connects to
|
||||||
|
// anything, so it can be asked on a workstation about a mesh it has never reached.
|
||||||
|
|
||||||
|
// Readiness is what the mesh knows about its own ability to move.
|
||||||
|
type Readiness struct {
|
||||||
|
// TheBus is the address the mesh's own traffic would move to, empty when nothing names one.
|
||||||
|
TheBus string
|
||||||
|
// ServerStanding is whether a bus is reachable at that address, as somebody checked.
|
||||||
|
ServerStanding bool
|
||||||
|
// Holder is the node running the module that holds the bus seat, empty when nothing does.
|
||||||
|
Holder string
|
||||||
|
// AccountsComposed is whether that node has been sent the composed user list.
|
||||||
|
AccountsComposed bool
|
||||||
|
// Nodes is every machine the mesh knows.
|
||||||
|
Nodes []string
|
||||||
|
// Credentialled is which of them has a credential for the new bus.
|
||||||
|
Credentialled map[string]bool
|
||||||
|
// Modules is every assigned module, as `<node>/<module>`.
|
||||||
|
Modules []string
|
||||||
|
// ModuleCredentialled is which of those has one.
|
||||||
|
ModuleCredentialled map[string]bool
|
||||||
|
// StillOnTheOldBus is whether anything of the mesh's own still needs the bus it is leaving —
|
||||||
|
// which is not a reason to stop, because that broker stays as an ordinary provider of `amqp`
|
||||||
|
// (ADR 0119). Recorded so nobody reads the move as a retirement.
|
||||||
|
OldBusHasOtherClients bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// NotReady is every reason this mesh cannot move its bus yet, in the order somebody would fix them.
|
||||||
|
//
|
||||||
|
// Empty means ready. **Each entry names one thing and what to do about it**, because a readiness check
|
||||||
|
// that says "not ready" is a check nobody can act on — and this is read at the point where the next
|
||||||
|
// step is irreversible.
|
||||||
|
func NotReady(r Readiness) []string {
|
||||||
|
var why []string
|
||||||
|
|
||||||
|
if strings.TrimSpace(r.TheBus) == "" {
|
||||||
|
why = append(why, "nothing names the bus to move to: set "+NATSVar+" on the control node "+
|
||||||
|
"to the address the new server answers on")
|
||||||
|
}
|
||||||
|
if !r.ServerStanding {
|
||||||
|
why = append(why, "no bus is answering at that address. Step 2 of the change raises it beside "+
|
||||||
|
"the one the mesh is on, carrying nothing — assign the module that holds "+
|
||||||
|
"mesh-broker and push the machine that runs it")
|
||||||
|
}
|
||||||
|
if r.Holder == "" {
|
||||||
|
why = append(why, "no machine holds mesh-broker, so nothing would compose the bus's user "+
|
||||||
|
"list. Assign the module that claims it")
|
||||||
|
} else if !r.AccountsComposed {
|
||||||
|
why = append(why, fmt.Sprintf(
|
||||||
|
"%s holds mesh-broker and has not been sent the composed user list, so the bus would "+
|
||||||
|
"refuse every connection. `push %s`", r.Holder, r.Holder))
|
||||||
|
}
|
||||||
|
|
||||||
|
// A node with no credential cannot come back after the move, and a node that cannot come back is
|
||||||
|
// a machine the mesh has lost until somebody goes to it.
|
||||||
|
var missing []string
|
||||||
|
for _, n := range r.Nodes {
|
||||||
|
if !r.Credentialled[n] {
|
||||||
|
missing = append(missing, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(missing)
|
||||||
|
if len(missing) > 0 {
|
||||||
|
why = append(why, fmt.Sprintf(
|
||||||
|
"%d machine(s) have no credential for the new bus and would not come back: %s. Each needs "+
|
||||||
|
"one minted before the move, not after — after, there is no bus to ask over",
|
||||||
|
len(missing), strings.Join(missing, ", ")))
|
||||||
|
}
|
||||||
|
|
||||||
|
// A module without one keeps running and stops being reachable, which is a smaller fault and still
|
||||||
|
// one somebody should choose rather than discover.
|
||||||
|
var quiet []string
|
||||||
|
for _, m := range r.Modules {
|
||||||
|
if !r.ModuleCredentialled[m] {
|
||||||
|
quiet = append(quiet, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(quiet)
|
||||||
|
if len(quiet) > 0 {
|
||||||
|
why = append(why, fmt.Sprintf(
|
||||||
|
"%d module(s) have no credential for the new bus: %s. Each keeps serving and stops "+
|
||||||
|
"answering tools and hearing events until it is issued one",
|
||||||
|
len(quiet), strings.Join(quiet, ", ")))
|
||||||
|
}
|
||||||
|
|
||||||
|
return why
|
||||||
|
}
|
||||||
|
|
||||||
|
// WhatMoves is what the rollout would do, in order, for somebody reading before they commit.
|
||||||
|
//
|
||||||
|
// **Written out rather than summarised.** This is the one step with nothing to inspect afterwards, so
|
||||||
|
// the last useful moment to disagree with it is while reading this.
|
||||||
|
func WhatMoves(r Readiness) []string {
|
||||||
|
out := []string{
|
||||||
|
fmt.Sprintf("compose the bus's user list and send it to %s", holderOr(r.Holder)),
|
||||||
|
fmt.Sprintf("move this control plane to %s, and confirm it is heard", busOr(r.TheBus)),
|
||||||
|
}
|
||||||
|
nodes := append([]string(nil), r.Nodes...)
|
||||||
|
sort.Strings(nodes)
|
||||||
|
for _, n := range nodes {
|
||||||
|
out = append(out, fmt.Sprintf("move %s, and confirm it reports", n))
|
||||||
|
}
|
||||||
|
if len(r.Modules) > 0 {
|
||||||
|
out = append(out, fmt.Sprintf("move %d module runtime(s), and confirm each answers",
|
||||||
|
len(r.Modules)))
|
||||||
|
}
|
||||||
|
if r.OldBusHasOtherClients {
|
||||||
|
out = append(out, "leave the old broker running: it stays an ordinary provider of `amqp` for "+
|
||||||
|
"whatever else uses it (ADR 0119), and this move is not its retirement")
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func holderOr(node string) string {
|
||||||
|
if node == "" {
|
||||||
|
return "whichever machine holds mesh-broker"
|
||||||
|
}
|
||||||
|
return node
|
||||||
|
}
|
||||||
|
|
||||||
|
func busOr(address string) string {
|
||||||
|
if address == "" {
|
||||||
|
return "the new bus"
|
||||||
|
}
|
||||||
|
return address
|
||||||
|
}
|
||||||
@@ -0,0 +1,97 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Whether a mesh could move its bus.
|
||||||
|
//
|
||||||
|
// Every case here is a way of moving that leaves something behind, and the one that matters most is a
|
||||||
|
// machine with no credential: after the move there is no bus to ask it over, so it is lost until
|
||||||
|
// somebody walks to it.
|
||||||
|
|
||||||
|
func aMeshReadyToMove() Readiness {
|
||||||
|
return Readiness{
|
||||||
|
TheBus: "nats://127.0.0.1:5671", ServerStanding: true,
|
||||||
|
Holder: "anchor", AccountsComposed: true,
|
||||||
|
Nodes: []string{"anchor", "laptop"},
|
||||||
|
Credentialled: map[string]bool{"anchor": true, "laptop": true},
|
||||||
|
Modules: []string{"anchor/gitea"},
|
||||||
|
ModuleCredentialled: map[string]bool{"anchor/gitea": true},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAMeshWithEverythingInPlaceIsReady(t *testing.T) {
|
||||||
|
if why := NotReady(aMeshReadyToMove()); len(why) != 0 {
|
||||||
|
t.Fatalf("a mesh with everything in place was refused: %v", why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A machine with no credential is the one that must stop this.** It keeps running and cannot come
|
||||||
|
// back, and there is no bus left to tell it anything over — so the remedy has to happen before, and
|
||||||
|
// the message says so.
|
||||||
|
func TestAMachineWithNoCredentialStopsTheMove(t *testing.T) {
|
||||||
|
r := aMeshReadyToMove()
|
||||||
|
r.Credentialled = map[string]bool{"anchor": true}
|
||||||
|
|
||||||
|
why := NotReady(r)
|
||||||
|
if len(why) == 0 {
|
||||||
|
t.Fatal("a machine that could not come back did not stop the move")
|
||||||
|
}
|
||||||
|
said := strings.Join(why, "\n")
|
||||||
|
if !strings.Contains(said, "laptop") {
|
||||||
|
t.Errorf("the refusal does not name the machine: %s", said)
|
||||||
|
}
|
||||||
|
if !strings.Contains(said, "before the move") {
|
||||||
|
t.Errorf("the refusal does not say the remedy comes first: %s", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A bus nobody has raised, a seat nobody holds, and a user list nobody has been sent: each stops it,
|
||||||
|
// and each names its own next step, because "not ready" that cannot be acted on is not an answer.
|
||||||
|
func TestEachThingMissingNamesItsOwnRemedy(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
what string
|
||||||
|
break_ func(*Readiness)
|
||||||
|
says string
|
||||||
|
}{
|
||||||
|
{"no address", func(r *Readiness) { r.TheBus = "" }, NATSVar},
|
||||||
|
{"no server", func(r *Readiness) { r.ServerStanding = false }, "carrying nothing"},
|
||||||
|
{"no holder", func(r *Readiness) { r.Holder = "" }, "mesh-broker"},
|
||||||
|
{"no user list", func(r *Readiness) { r.AccountsComposed = false }, "push anchor"},
|
||||||
|
{"a module with none", func(r *Readiness) {
|
||||||
|
r.ModuleCredentialled = map[string]bool{}
|
||||||
|
}, "anchor/gitea"},
|
||||||
|
} {
|
||||||
|
r := aMeshReadyToMove()
|
||||||
|
c.break_(&r)
|
||||||
|
why := NotReady(r)
|
||||||
|
if len(why) == 0 {
|
||||||
|
t.Errorf("%s did not stop the move", c.what)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !strings.Contains(strings.Join(why, "\n"), c.says) {
|
||||||
|
t.Errorf("%s: the refusal does not mention %q: %v", c.what, c.says, why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// What the move would do is written out rather than summarised, because this is the one step with
|
||||||
|
// nothing to inspect afterwards — so reading it is the last chance to disagree.
|
||||||
|
func TestWhatMovesNamesEveryMachineAndSaysTheOldBrokerStays(t *testing.T) {
|
||||||
|
r := aMeshReadyToMove()
|
||||||
|
r.OldBusHasOtherClients = true
|
||||||
|
steps := strings.Join(WhatMoves(r), "\n")
|
||||||
|
|
||||||
|
for _, want := range []string{"anchor", "laptop", "user list", "module runtime"} {
|
||||||
|
if !strings.Contains(steps, want) {
|
||||||
|
t.Errorf("the plan does not mention %q:\n%s", want, steps)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Said explicitly, so nobody reads the move as switching the old broker off — it stays serving
|
||||||
|
// whatever else uses it, and that is a decision already taken.
|
||||||
|
if !strings.Contains(steps, "not its retirement") {
|
||||||
|
t.Errorf("the plan does not say the old broker stays:\n%s", steps)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,238 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The mesh's own streams.
|
||||||
|
//
|
||||||
|
// **These four and no more** (novox/hq ADR 0116 task 1.4, as revised by ADR 0118). An earlier
|
||||||
|
// reading had the controller create *every* stream at genesis, from a fixed set. That is only the
|
||||||
|
// mesh's own half: a seat's streams are created when the module declaring it is registered, and a
|
||||||
|
// module's durable consumers when it is assigned — neither of which has happened at genesis. What
|
||||||
|
// is here is the foundation, which exists before any module does.
|
||||||
|
//
|
||||||
|
// The controller is the only writer of stream definitions (design 25 §3). A module declares
|
||||||
|
// nothing about them and cannot reach the JetStream API to make one.
|
||||||
|
|
||||||
|
// Retention is how a stream decides what to keep, which is the whole of what distinguishes the
|
||||||
|
// mesh's four relationships on the wire (design 29 §4).
|
||||||
|
type Retention string
|
||||||
|
|
||||||
|
const (
|
||||||
|
// RetentionWorkQueue: a message is removed once a consumer acknowledges it. Exactly one
|
||||||
|
// worker does the work, and a worker that dies has its message redelivered.
|
||||||
|
RetentionWorkQueue Retention = "workqueue"
|
||||||
|
// RetentionLastPerSubject: only the newest message on each subject survives. This is the
|
||||||
|
// state shape — a node that was away gets exactly the current declaration and nothing older.
|
||||||
|
RetentionLastPerSubject Retention = "last_per_subject"
|
||||||
|
// RetentionLimits: kept until it ages or the stream fills. Events, where a subscriber that
|
||||||
|
// was down catches up and nobody is obliged to act.
|
||||||
|
RetentionLimits Retention = "limits"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A Stream is one of the mesh's own, as the controller asserts it.
|
||||||
|
type Stream struct {
|
||||||
|
Name string
|
||||||
|
Subjects []string
|
||||||
|
Retention Retention
|
||||||
|
// MaxAge in seconds, zero for unbounded. Per stream — JetStream has no per-subject age,
|
||||||
|
// which is why differing retention between modules would mean a stream each.
|
||||||
|
MaxAge int
|
||||||
|
// MaxMsgsPerSubject caps each subject independently, so one noisy emitter cannot push
|
||||||
|
// another's events out of a shared stream. Verified: with a cap of 3, ten messages on one
|
||||||
|
// subject and one on another leave four in the stream, not three.
|
||||||
|
MaxMsgsPerSubject int
|
||||||
|
// Why is carried into the assertion so an operator reading the server's own state finds the
|
||||||
|
// reason there, rather than only in a repository they may not have.
|
||||||
|
Why string
|
||||||
|
}
|
||||||
|
|
||||||
|
// MeshStreams is the foundation set, in the order a person reads it.
|
||||||
|
//
|
||||||
|
// **CONTROL names its subjects rather than taking `mesh.control.>`**, because heartbeats live
|
||||||
|
// under that prefix and must not be persisted: a lost heartbeat is the next heartbeat, and a
|
||||||
|
// stream of them is a stream of the least valuable messages the mesh sends, competing for the
|
||||||
|
// same retention as the ones that matter.
|
||||||
|
//
|
||||||
|
// **EVENTS filters on the `event` token**, which is the reason that token exists. A module's
|
||||||
|
// namespace carries both its events and its tool calls; a filter of `mesh.mod.*.>` would persist
|
||||||
|
// every tool invocation in the mesh, and a tool call must never be persisted (design 25 §3 keeps
|
||||||
|
// tools on core NATS, where a lost call is a timeout the caller already handles).
|
||||||
|
func MeshStreams() []Stream {
|
||||||
|
return []Stream{
|
||||||
|
{
|
||||||
|
Name: "CONTROL",
|
||||||
|
// A build's outcome is no longer here: it is the build-machine seat's own event, so one
|
||||||
|
// publish reaches whoever asked, the controller and the catalogue (novox/hq ADR 0121).
|
||||||
|
Subjects: []string{"mesh.control.*.report", "mesh.control.enrol"},
|
||||||
|
Retention: RetentionWorkQueue,
|
||||||
|
Why: "the store-window guarantee (ADR 0083): the controller naks with a delay while its " +
|
||||||
|
"store is away and the message is redelivered; nothing is dropped",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Name: "NODES",
|
||||||
|
Subjects: []string{"mesh.node.*.declare"},
|
||||||
|
Retention: RetentionLastPerSubject,
|
||||||
|
Why: "one declaration per node, always the newest; a node that sees sequence n refuses " +
|
||||||
|
"n-1 by construction (issue 107)",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Name: "EVENTS",
|
||||||
|
// A seat's own events ride here too: they are 1:many like any event, and the
|
||||||
|
// `event` token keeps them clear of both the seat's work queue (`accept`) and its
|
||||||
|
// tools (`tool`), which must not be persisted.
|
||||||
|
Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"},
|
||||||
|
Retention: RetentionLimits,
|
||||||
|
MaxAge: 7 * 24 * 60 * 60,
|
||||||
|
MaxMsgsPerSubject: 10000,
|
||||||
|
Why: "a subscriber that was down catches up; tool traffic under the same prefix is " +
|
||||||
|
"excluded by the event token; per-subject caps keep a noisy emitter from " +
|
||||||
|
"evicting a quiet one without splitting the stream",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An Asserter is the part of a JetStream connection stream assertion needs. Narrow on purpose: it
|
||||||
|
// keeps this testable without a server, and keeps the client library out of everything that only
|
||||||
|
// wants to know what the streams are.
|
||||||
|
type Asserter interface {
|
||||||
|
// EnsureStream creates the stream if absent and updates it to match if present. It must be
|
||||||
|
// idempotent: the controller asserts on every start, not only at genesis.
|
||||||
|
EnsureStream(s Stream) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// AssertMeshStreams brings the foundation set into being, in order, and says which one failed
|
||||||
|
// rather than that something did.
|
||||||
|
//
|
||||||
|
// Asserted on every start rather than created once at genesis, because a stream that was deleted,
|
||||||
|
// or a mesh raised from a restored backup, must converge rather than run without the guarantee
|
||||||
|
// its messages assume. Idempotence is the whole requirement.
|
||||||
|
func AssertMeshStreams(a Asserter) error {
|
||||||
|
for _, s := range MeshStreams() {
|
||||||
|
if err := a.EnsureStream(s); err != nil {
|
||||||
|
return fmt.Errorf("asserting stream %s: %w", s.Name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Overlaps reports subject filters claimed by more than one stream.
|
||||||
|
//
|
||||||
|
// **Corrected against the server**: an earlier version of this comment said NATS accepts
|
||||||
|
// overlapping streams and stores the message twice. It does not — it refuses the second stream
|
||||||
|
// with "subjects overlap with an existing stream" (verified against nats-server 2.10). The check
|
||||||
|
// still earns its place, for a different reason: the server's refusal arrives when the controller
|
||||||
|
// is applying, naming one stream, at a moment when the mesh is half-configured. This one arrives
|
||||||
|
// where the set is written, names both, and cannot reach a running mesh.
|
||||||
|
//
|
||||||
|
// It also decides a design question. Because overlap is refused rather than merged, a shared
|
||||||
|
// EVENTS stream and a per-module stream cannot coexist — the module's would be refused — so
|
||||||
|
// "one stream for most, its own for a module that wants different retention" is not an option
|
||||||
|
// the server allows. It is all of one or all of the other.
|
||||||
|
func Overlaps() []string {
|
||||||
|
seen := map[string]string{}
|
||||||
|
var clashes []string
|
||||||
|
for _, s := range MeshStreams() {
|
||||||
|
for _, subject := range s.Subjects {
|
||||||
|
if first, ok := seen[subject]; ok {
|
||||||
|
clashes = append(clashes, fmt.Sprintf("%s and %s both claim %s", first, s.Name, subject))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[subject] = s.Name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(clashes)
|
||||||
|
return clashes
|
||||||
|
}
|
||||||
|
|
||||||
|
// The mesh's own consumers.
|
||||||
|
//
|
||||||
|
// A seat's streams and a module's consumers are derived from declarations (derived.go). These two
|
||||||
|
// are not: **the controller is not a module and files no manifest**, so its authority and its
|
||||||
|
// subscriptions cannot come from a declaration that does not exist. They are named here, where the
|
||||||
|
// mesh's own streams are named, and narrowly — a controller subscribing `mesh.mod.*.event.>` would
|
||||||
|
// hear every event in the mesh, which it has no business doing and which would make its permission
|
||||||
|
// list stop explaining anything.
|
||||||
|
|
||||||
|
// ControllerName is the controller's durable consumer on each stream it reads, and the name its
|
||||||
|
// ack subject is derived from (nats.go: `$JS.ACK.<stream>.controller.>`).
|
||||||
|
const ControllerName = "controller"
|
||||||
|
|
||||||
|
// ControllerFollows are the events the controller reacts to: the catalogue saying a module's
|
||||||
|
// current version moved, and a catalogue that has just started saying it may have missed builds.
|
||||||
|
//
|
||||||
|
// **Derived the same way a module's subscription is**, from the emitter and the bare local event
|
||||||
|
// name, rather than written out. They were written out while the catalogue still spelled its events
|
||||||
|
// as the old bus's routing keys, and the moment those were converted (novox/hq 04-ISSUES/127) a
|
||||||
|
// hard-coded pair became a controller listening to a subject nothing publishes — the same fault, from
|
||||||
|
// the other side. Deriving them means the conversion could not leave these behind.
|
||||||
|
var ControllerFollows = []string{
|
||||||
|
moduleEventSubject("mesh-catalog", "upgraded"),
|
||||||
|
moduleEventSubject("mesh-catalog", "catching-up"),
|
||||||
|
// A build's outcome, which is the build-machine role's own event now (ADR 0121) rather than a
|
||||||
|
// message on the control branch. Same three audiences, one publish: whoever asked, this, and the
|
||||||
|
// catalogue.
|
||||||
|
seatEventSubject("mesh-build-machine", "built"),
|
||||||
|
}
|
||||||
|
|
||||||
|
// moduleEventSubject is where one module's event lands. The same derivation PermissionsFor uses, so
|
||||||
|
// what the controller subscribes and what the emitter is permitted to publish cannot drift apart.
|
||||||
|
func moduleEventSubject(module, event string) string {
|
||||||
|
return "mesh.mod." + module + ".event." + event
|
||||||
|
}
|
||||||
|
|
||||||
|
// seatEventSubject is where a role's own event lands, derived the same way a holder's permission is.
|
||||||
|
func seatEventSubject(seat, verb string) string {
|
||||||
|
return "mesh.seat." + seat + ".event." + verb
|
||||||
|
}
|
||||||
|
|
||||||
|
// MeshConsumers is what the controller consumes, in the order a person reads it.
|
||||||
|
//
|
||||||
|
// **Unlimited redelivery on CONTROL, deliberately.** The store window's bound is the controller's,
|
||||||
|
// not the server's (window.go): a message is held with a nak-and-delay until the controller either
|
||||||
|
// takes it or gives up and says so. A max-deliver here would dead-letter a push that was being
|
||||||
|
// held through a store restart — the exact message the stream exists to protect — some minutes
|
||||||
|
// before the controller had finished deciding about it.
|
||||||
|
func MeshConsumers() []Consumer {
|
||||||
|
return []Consumer{
|
||||||
|
{
|
||||||
|
Name: ControllerName,
|
||||||
|
Stream: "CONTROL",
|
||||||
|
Push: true,
|
||||||
|
AckWaitSeconds: 30,
|
||||||
|
Why: "the controller is the single consumer of what nodes say; explicit ack and no " +
|
||||||
|
"max-deliver, because the store window's bound is the controller's own",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Name: ControllerName,
|
||||||
|
Stream: "EVENTS",
|
||||||
|
Filters: ControllerFollows,
|
||||||
|
Push: true,
|
||||||
|
AckWaitSeconds: 30,
|
||||||
|
MaxDeliver: 5,
|
||||||
|
Why: "the two events the mesh's own controller reacts to; after max-deliver it " +
|
||||||
|
"dead-letters, because an announcement it cannot act on will not become actionable",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensurer is the part of a JetStream connection consumer assertion needs, narrow for the reason
|
||||||
|
// Asserter is.
|
||||||
|
type Ensurer interface {
|
||||||
|
EnsureConsumer(c Consumer) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// AssertMeshConsumers brings the controller's own consumers into being, and says which one failed.
|
||||||
|
//
|
||||||
|
// After the streams, necessarily: a consumer on a stream that does not exist is refused, and the
|
||||||
|
// refusal names the stream rather than the order.
|
||||||
|
func AssertMeshConsumers(e Ensurer) error {
|
||||||
|
for _, c := range MeshConsumers() {
|
||||||
|
if err := e.EnsureConsumer(c); err != nil {
|
||||||
|
return fmt.Errorf("asserting consumer %s on %s: %w", c.Name, c.Stream, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,235 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
type recorder struct {
|
||||||
|
seen []Stream
|
||||||
|
fail string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *recorder) EnsureStream(s Stream) error {
|
||||||
|
if s.Name == r.fail {
|
||||||
|
return errors.New("refused")
|
||||||
|
}
|
||||||
|
r.seen = append(r.seen, s)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// The controller asserts on every start, not only at genesis: a stream that was deleted, or a mesh
|
||||||
|
// raised from a backup, must converge rather than run without the guarantee its messages assume.
|
||||||
|
func TestAssertingTwiceIsTheSameAsOnce(t *testing.T) {
|
||||||
|
a, b := &recorder{}, &recorder{}
|
||||||
|
if err := AssertMeshStreams(a); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := AssertMeshStreams(a); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := AssertMeshStreams(b); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(a.seen) != 2*len(b.seen) {
|
||||||
|
t.Fatalf("asserted %d then %d; assertion is not repeatable", len(a.seen), len(b.seen))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAFailedAssertionNamesItsStream(t *testing.T) {
|
||||||
|
err := AssertMeshStreams(&recorder{fail: "NODES"})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "NODES") {
|
||||||
|
t.Fatalf("got %v, which does not say which stream failed", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Two streams matching one subject is accepted by NATS and stores the message twice under two
|
||||||
|
// retentions. Nothing reports that, so it is refused where the set is written.
|
||||||
|
func TestNoTwoStreamsClaimTheSameSubject(t *testing.T) {
|
||||||
|
if clashes := Overlaps(); len(clashes) != 0 {
|
||||||
|
t.Fatalf("overlapping subject filters: %v", clashes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A heartbeat under mesh.control.> must not be persisted: a lost one is the next one, and a
|
||||||
|
// stream of them competes for retention with the messages that matter.
|
||||||
|
func TestHeartbeatsAreNotInTheControlStream(t *testing.T) {
|
||||||
|
for _, s := range MeshStreams() {
|
||||||
|
for _, subject := range s.Subjects {
|
||||||
|
if subject == "mesh.control.>" || strings.Contains(subject, "alive") {
|
||||||
|
t.Fatalf("stream %s claims %q, which captures heartbeats", s.Name, subject)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The reason the kind token exists: a filter over a module's whole namespace would persist every
|
||||||
|
// tool call in the mesh.
|
||||||
|
func TestTheEventsStreamDoesNotCaptureToolCalls(t *testing.T) {
|
||||||
|
var events Stream
|
||||||
|
for _, s := range MeshStreams() {
|
||||||
|
if s.Name == "EVENTS" {
|
||||||
|
events = s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Nothing a tool call rides may match any of the filters — a module's or a seat's.
|
||||||
|
for _, tool := range []string{
|
||||||
|
"mesh.mod.billing.tool.status",
|
||||||
|
"mesh.seat.telegram-sender.tool.status",
|
||||||
|
"mesh.seat.telegram-sender.accept.send", // work, not an event: its own stream
|
||||||
|
} {
|
||||||
|
for _, f := range events.Subjects {
|
||||||
|
if subjectMatches(f, tool) {
|
||||||
|
t.Fatalf("%q matches the events filter %q, so it would be persisted here", tool, f)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// And both kinds of event do match.
|
||||||
|
for _, event := range []string{
|
||||||
|
"mesh.mod.billing.event.order.placed",
|
||||||
|
"mesh.seat.telegram-sender.event.delivered",
|
||||||
|
} {
|
||||||
|
matched := false
|
||||||
|
for _, f := range events.Subjects {
|
||||||
|
if subjectMatches(f, event) {
|
||||||
|
matched = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !matched {
|
||||||
|
t.Fatalf("%q matches no events filter, so nothing would keep it", event)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// subjectMatches is NATS subject matching, enough for these filters: `*` is one token, `>` is the
|
||||||
|
// rest.
|
||||||
|
func subjectMatches(filter, subject string) bool {
|
||||||
|
f, s := strings.Split(filter, "."), strings.Split(subject, ".")
|
||||||
|
for i, tok := range f {
|
||||||
|
if tok == ">" {
|
||||||
|
return i <= len(s)
|
||||||
|
}
|
||||||
|
if i >= len(s) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if tok != "*" && tok != s[i] {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return len(f) == len(s)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Each relationship's retention is the thing that makes it what it is (design 29 §4).
|
||||||
|
func TestEachStreamCarriesTheRetentionItsShapeNeeds(t *testing.T) {
|
||||||
|
want := map[string]Retention{
|
||||||
|
"CONTROL": RetentionWorkQueue,
|
||||||
|
"NODES": RetentionLastPerSubject,
|
||||||
|
"EVENTS": RetentionLimits,
|
||||||
|
}
|
||||||
|
got := map[string]Retention{}
|
||||||
|
for _, s := range MeshStreams() {
|
||||||
|
got[s.Name] = s.Retention
|
||||||
|
if s.Why == "" {
|
||||||
|
t.Errorf("stream %s says no reason it exists", s.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(got) != len(want) {
|
||||||
|
t.Fatalf("the foundation set is %v", got)
|
||||||
|
}
|
||||||
|
for name, r := range want {
|
||||||
|
if got[name] != r {
|
||||||
|
t.Errorf("%s retains as %q, expected %q", name, got[name], r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The order the bus's objects are asserted in, because getting it wrong is a refusal that names the
|
||||||
|
// wrong thing: a consumer on a stream that does not exist is refused naming the *stream*, so
|
||||||
|
// somebody reading it goes looking for a deletion instead of a reversed pair of lines.
|
||||||
|
func TestTheBusesObjectsAreAssertedStreamsBeforeConsumers(t *testing.T) {
|
||||||
|
r := &recording{}
|
||||||
|
if err := Raise(r, []string{"anchor", "laptop"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every stream before every consumer.
|
||||||
|
firstConsumer := -1
|
||||||
|
for i, step := range r.steps {
|
||||||
|
if strings.HasPrefix(step, "consumer ") && firstConsumer < 0 {
|
||||||
|
firstConsumer = i
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(step, "stream ") && firstConsumer >= 0 {
|
||||||
|
t.Fatalf("a stream was asserted after a consumer: %v", r.steps)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if firstConsumer < 0 {
|
||||||
|
t.Fatalf("no consumer was asserted: %v", r.steps)
|
||||||
|
}
|
||||||
|
|
||||||
|
// And every node got one, named after it — without which that node hears nothing while
|
||||||
|
// everything else about it looks correct.
|
||||||
|
for _, node := range []string{"anchor", "laptop"} {
|
||||||
|
if !containsStep(r.steps, "consumer NODES/"+node) {
|
||||||
|
t.Errorf("%s was given no way to hear its declaration: %v", node, r.steps)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// And the controller its own, on both streams it reads.
|
||||||
|
for _, want := range []string{"consumer CONTROL/controller", "consumer EVENTS/controller"} {
|
||||||
|
if !containsStep(r.steps, want) {
|
||||||
|
t.Errorf("the controller is missing %s: %v", want, r.steps)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A seat's work queue is asserted whether or not anybody holds it; the holder's worker only when
|
||||||
|
// somebody does. **The stream without the consumer is the point**: work queues until a holder
|
||||||
|
// appears, so installing the module later flushes the backlog instead of having lost it.
|
||||||
|
func TestASeatsQueueExistsBeforeItsHolderDoes(t *testing.T) {
|
||||||
|
seats := []DeclaredSeat{{Name: "telegram-sender", Accepts: []string{"send"}}}
|
||||||
|
|
||||||
|
unheld := &recording{}
|
||||||
|
if err := RaiseSeats(unheld, seats, nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !containsStep(unheld.steps, "stream SEAT_TELEGRAM_SENDER") {
|
||||||
|
t.Fatalf("a declared seat got no work queue: %v", unheld.steps)
|
||||||
|
}
|
||||||
|
for _, step := range unheld.steps {
|
||||||
|
if strings.HasPrefix(step, "consumer ") {
|
||||||
|
t.Fatalf("a seat nobody holds got a worker: %v", unheld.steps)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
held := &recording{}
|
||||||
|
if err := RaiseSeats(held, seats, map[string]Holder{
|
||||||
|
"telegram-sender": {Node: "anchor", Module: "telegram"},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !containsStep(held.steps, "consumer SEAT_TELEGRAM_SENDER/SEAT_TELEGRAM_SENDER_worker") {
|
||||||
|
t.Fatalf("the seat's holder got no worker: %v", held.steps)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// recording is a connection to the bus that writes down what it was asked for.
|
||||||
|
type recording struct{ steps []string }
|
||||||
|
|
||||||
|
func (r *recording) EnsureStream(s Stream) error {
|
||||||
|
r.steps = append(r.steps, "stream "+s.Name)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *recording) EnsureConsumer(c Consumer) error {
|
||||||
|
r.steps = append(r.steps, "consumer "+c.Stream+"/"+c.Name)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func containsStep(steps []string, want string) bool {
|
||||||
|
for _, s := range steps {
|
||||||
|
if s == want {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
+53
@@ -0,0 +1,53 @@
|
|||||||
|
# Composed by the mesh controller. Do not edit: the next composition overwrites it.
|
||||||
|
# Accounts and permissions are derived from what each module declares and nothing
|
||||||
|
# else (novox/hq ADR 0043, design 29 §2).
|
||||||
|
|
||||||
|
port: 4222
|
||||||
|
http: 127.0.0.1:8222
|
||||||
|
|
||||||
|
tls {
|
||||||
|
cert_file: "/tls/tls.crt"
|
||||||
|
key_file: "/tls/tls.key"
|
||||||
|
ca_file: "/tls/ca.crt"
|
||||||
|
}
|
||||||
|
|
||||||
|
jetstream {
|
||||||
|
store_dir: "/data"
|
||||||
|
}
|
||||||
|
|
||||||
|
# The mesh's users, composed by the controller. Do not edit: the next
|
||||||
|
# composition overwrites it. Permissions are derived from what each module
|
||||||
|
# declares and nothing else (novox/hq ADR 0043, design 29 §2).
|
||||||
|
|
||||||
|
accounts {
|
||||||
|
MESH {
|
||||||
|
users = [
|
||||||
|
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
||||||
|
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.control.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>"] }
|
||||||
|
subscribe: { allow: ["$JS.API.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built"] }
|
||||||
|
allow_responses: { max: 1, ttl: "1m" }
|
||||||
|
} }
|
||||||
|
{ user: "enrol.one", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: {
|
||||||
|
publish: { allow: ["mesh.control.enrol"] }
|
||||||
|
subscribe: { allow: ["_INBOX.enrol.one.>"] }
|
||||||
|
} }
|
||||||
|
{ user: "node.one", password: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn", permissions: {
|
||||||
|
publish: { allow: ["$JS.ACK.NODES.one.>", "mesh.control.one.>"] }
|
||||||
|
subscribe: { allow: ["_INBOX.node.one.>", "mesh.node.one.declare"] }
|
||||||
|
} }
|
||||||
|
{ user: "one.telegram", password: "$2a$11$tttttttttttttttttttttt", permissions: {
|
||||||
|
publish: { allow: ["$JS.ACK.EVENTS.one_telegram.>", "mesh.seat.telegram-sender.event.delivered", "mesh.seat.telegram-sender.event.failed"] }
|
||||||
|
subscribe: { allow: ["_INBOX.one.telegram.>", "mesh.mod.telegram.tool.status", "mesh.seat.telegram-sender.accept.send"] }
|
||||||
|
allow_responses: { max: 1, ttl: "1m" }
|
||||||
|
} }
|
||||||
|
{ user: "two.audit", password: "$2a$11$aaaaaaaaaaaaaaaaaaaaaa", permissions: {
|
||||||
|
publish: { allow: ["$JS.ACK.EVENTS.two_audit.>"] }
|
||||||
|
subscribe: { allow: ["_INBOX.two.audit.>", "mesh.mod.shop.event.order.placed"] }
|
||||||
|
} }
|
||||||
|
{ user: "two.shop", password: "$2a$11$ssssssssssssssssssssss", permissions: {
|
||||||
|
publish: { allow: ["$JS.ACK.EVENTS.two_shop.>", "mesh.mod.shop.event.order.placed", "mesh.seat.telegram-sender.accept.send"] }
|
||||||
|
subscribe: { allow: ["_INBOX.two.shop.>"] }
|
||||||
|
} }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,127 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Every user the composed file should contain, derived from what the mesh knows.
|
||||||
|
//
|
||||||
|
// **The list is derived, never kept.** A stored user list would be a second account of who may
|
||||||
|
// reach the bus, able to disagree with the records it came from — and the disagreement would be
|
||||||
|
// invisible, because both would look internally consistent. So this is a pure function of the
|
||||||
|
// mesh's records, run again every time the file is written.
|
||||||
|
//
|
||||||
|
// Records are mirrored into this package's own types rather than imported from the catalogue, for
|
||||||
|
// the reason DeclaredSeat is: composing authority is a different job from parsing a manifest, and
|
||||||
|
// this package stays free of the other's types so a change to a manifest field cannot quietly widen
|
||||||
|
// a permission.
|
||||||
|
|
||||||
|
// Declared is one module on one node, as composing its authority needs it.
|
||||||
|
type Declared struct {
|
||||||
|
Module string
|
||||||
|
Emits []string
|
||||||
|
Consumes []string
|
||||||
|
Serves []string
|
||||||
|
// Holds are the seats this module claims, with the protocol each seat declares. A seat the
|
||||||
|
// mesh defines for itself declares no protocol, so holding one grants nothing on the bus —
|
||||||
|
// which is right: those seats are about who does a job, not about who may say what.
|
||||||
|
Holds []Seat
|
||||||
|
// Uses are the seats this module sends to.
|
||||||
|
Uses []Seat
|
||||||
|
// Watches are the seats whose events it consumes.
|
||||||
|
Watches []Seat
|
||||||
|
}
|
||||||
|
|
||||||
|
// Records is what composing a user list needs to know about the mesh, and nothing more.
|
||||||
|
type Records struct {
|
||||||
|
// Nodes is every machine the mesh knows. Each gets a host user.
|
||||||
|
Nodes []string
|
||||||
|
// Assigned is the modules on each node, as they declare themselves.
|
||||||
|
Assigned map[string][]Declared
|
||||||
|
// Enrolling is every node with a live token — one enrolment user each, because the inbox an
|
||||||
|
// answer goes to is scoped to the token and a shared one is one machine reading another's
|
||||||
|
// sealed credentials (design 25 §6).
|
||||||
|
Enrolling []string
|
||||||
|
// People is each person's name against the tools they may invoke, `*` for an administrator.
|
||||||
|
People map[string][]string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Users is every user the composed file should contain, in the order it will be written.
|
||||||
|
//
|
||||||
|
// The controller is always first and always present: a mesh whose own controller is not in the file
|
||||||
|
// is a mesh that cannot be told anything, and there is no state of the records in which that is
|
||||||
|
// correct.
|
||||||
|
func Users(r Records) ([]Principal, error) {
|
||||||
|
out := []Principal{{Kind: KindController}}
|
||||||
|
|
||||||
|
for _, node := range sortedCopy(r.Nodes) {
|
||||||
|
out = append(out, Principal{Kind: KindNode, Node: node})
|
||||||
|
for _, d := range r.Assigned[node] {
|
||||||
|
out = append(out, Principal{
|
||||||
|
Kind: KindModule, Node: node, Module: d.Module,
|
||||||
|
Emits: d.Emits, Consumes: d.Consumes, Serves: d.Serves,
|
||||||
|
Holds: d.Holds, Uses: d.Uses, Watches: d.Watches,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, node := range sortedCopy(r.Enrolling) {
|
||||||
|
out = append(out, Principal{Kind: KindEnrolment, Node: node})
|
||||||
|
}
|
||||||
|
for _, person := range sortedNames(r.People) {
|
||||||
|
out = append(out, Principal{Kind: KindPerson, Module: person, Invokes: r.People[person]})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Refused here rather than discovered by the server. Two users with one name is a file the
|
||||||
|
// server reads as one of them, and which one depends on the order — so a module assigned to a
|
||||||
|
// node twice, or a person named after nothing, is a composition that must not be written.
|
||||||
|
seen := map[string]string{}
|
||||||
|
for _, p := range out {
|
||||||
|
name := p.Username()
|
||||||
|
if name == "" || name == "." {
|
||||||
|
return nil, fmt.Errorf("a %s user has no name, so nothing could authenticate as it", p.Kind)
|
||||||
|
}
|
||||||
|
if first, already := seen[name]; already {
|
||||||
|
return nil, fmt.Errorf(
|
||||||
|
"two users would be called %q (a %s and a %s): the server would read the file as "+
|
||||||
|
"one of them, and which one depends on the order", name, first, p.Kind)
|
||||||
|
}
|
||||||
|
seen[name] = string(p.Kind)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithPasswords fills each user's hash from what the mesh minted, and says which users have none.
|
||||||
|
//
|
||||||
|
// **Separated from Users because they fail differently.** A user missing from the records is a bug
|
||||||
|
// in deriving them; a user with no password is a step that has not happened yet — a module assigned
|
||||||
|
// but never given a credential, a node enrolled before this existed. The second is ordinary and its
|
||||||
|
// remedy is to mint one, so it is named rather than returned as an error, and the caller decides
|
||||||
|
// whether a partial composition is worth writing.
|
||||||
|
func WithPasswords(principals []Principal, hashes map[string]string) (filled []Principal, missing []string) {
|
||||||
|
for _, p := range principals {
|
||||||
|
hash, ok := hashes[p.Username()]
|
||||||
|
if !ok || hash == "" {
|
||||||
|
missing = append(missing, p.Username())
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
p.PasswordHash = hash
|
||||||
|
filled = append(filled, p)
|
||||||
|
}
|
||||||
|
return filled, missing
|
||||||
|
}
|
||||||
|
|
||||||
|
func sortedCopy(in []string) []string {
|
||||||
|
out := append([]string(nil), in...)
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func sortedNames(in map[string][]string) []string {
|
||||||
|
out := make([]string, 0, len(in))
|
||||||
|
for k := range in {
|
||||||
|
out = append(out, k)
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,241 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Deriving the bus's user list from the mesh's records.
|
||||||
|
//
|
||||||
|
// Every test here is about a way the list could be wrong that the server would not tell anybody
|
||||||
|
// about: a user missing, a user named twice, a user with authority it did not declare.
|
||||||
|
|
||||||
|
func someRecords() Records {
|
||||||
|
return Records{
|
||||||
|
Nodes: []string{"two", "one"},
|
||||||
|
Assigned: map[string][]Declared{
|
||||||
|
"one": {{Module: "telegram", Serves: []string{"status"}}},
|
||||||
|
"two": {{Module: "shop", Emits: []string{"order.placed"}}},
|
||||||
|
},
|
||||||
|
Enrolling: []string{"three"},
|
||||||
|
People: map[string][]string{"ada": {"mesh-catalog.catalog_tools"}},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func namesOf(t *testing.T, r Records) []string {
|
||||||
|
t.Helper()
|
||||||
|
users, err := Users(r)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
out := make([]string, 0, len(users))
|
||||||
|
for _, u := range users {
|
||||||
|
out = append(out, u.Username())
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// The controller is always there. A mesh whose own controller is not in the file is a mesh that
|
||||||
|
// cannot be told anything, and there is no state of the records in which that is correct.
|
||||||
|
func TestTheControllerIsAlwaysInTheList(t *testing.T) {
|
||||||
|
for _, r := range []Records{{}, someRecords()} {
|
||||||
|
names := namesOf(t, r)
|
||||||
|
if len(names) == 0 || names[0] != "controller" {
|
||||||
|
t.Fatalf("the controller is not first in %v", names)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// One user per node, one per module per node, one per live token and one per person — and nothing
|
||||||
|
// else, because a user nobody derived is a user nobody can explain.
|
||||||
|
func TestEveryRecordBecomesExactlyOneUser(t *testing.T) {
|
||||||
|
names := namesOf(t, someRecords())
|
||||||
|
want := []string{
|
||||||
|
"controller",
|
||||||
|
"node.one", "one.telegram",
|
||||||
|
"node.two", "two.shop",
|
||||||
|
"enrol.three",
|
||||||
|
"person.ada",
|
||||||
|
}
|
||||||
|
if strings.Join(names, ",") != strings.Join(want, ",") {
|
||||||
|
t.Fatalf("derived %v\n want %v", names, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Two users with one name is a file the server reads as one of them, and which one depends on the
|
||||||
|
// order. Refused here, where both can be named, rather than left to be whichever the server picked.
|
||||||
|
func TestTwoUsersWithOneNameAreRefused(t *testing.T) {
|
||||||
|
r := someRecords()
|
||||||
|
r.Assigned["one"] = append(r.Assigned["one"], Declared{Module: "telegram"})
|
||||||
|
_, err := Users(r)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a module assigned twice to one node composed two users with one name")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "one.telegram") {
|
||||||
|
t.Fatalf("the refusal does not name the user: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A module's authority is what it declared and nothing more, carried through the derivation intact —
|
||||||
|
// because this is the step where a mistake would grant something no manifest asked for.
|
||||||
|
func TestAModulesAuthorityIsWhatItDeclared(t *testing.T) {
|
||||||
|
seat := Seat{Name: "telegram-sender", Accepts: []string{"send"}, Emits: []string{"delivered"}}
|
||||||
|
users, err := Users(Records{
|
||||||
|
Nodes: []string{"one"},
|
||||||
|
Assigned: map[string][]Declared{"one": {{
|
||||||
|
Module: "shop", Emits: []string{"order.placed"}, Uses: []Seat{seat},
|
||||||
|
}}},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
perms, err := PermissionsFor(users[len(users)-1])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
has(t, perms.Publish, "mesh.mod.shop.event.order.placed")
|
||||||
|
has(t, perms.Publish, "mesh.seat.telegram-sender.accept.send")
|
||||||
|
// A seat it uses, not one it holds: it may submit work and may not publish the seat's own
|
||||||
|
// events, or it could lie about outcomes on a role somebody else fills.
|
||||||
|
hasNot(t, perms.Publish, "mesh.seat.telegram-sender.event.delivered")
|
||||||
|
hasNot(t, perms.Subscribe, "mesh.seat.telegram-sender.accept.send")
|
||||||
|
}
|
||||||
|
|
||||||
|
// A user the mesh has never minted a password for is named rather than silently dropped or
|
||||||
|
// composed as a user anybody is. It is an ordinary situation — a module assigned a moment ago — and
|
||||||
|
// the remedy is to mint one, so the caller decides whether to write a partial file.
|
||||||
|
func TestAUserWithNoPasswordIsNamedRatherThanWritten(t *testing.T) {
|
||||||
|
users, err := Users(someRecords())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
filled, missing := WithPasswords(users, map[string]string{
|
||||||
|
"controller": "$2a$hash", "node.one": "$2a$hash",
|
||||||
|
})
|
||||||
|
if len(filled) != 2 {
|
||||||
|
t.Fatalf("composed %d users from two hashes", len(filled))
|
||||||
|
}
|
||||||
|
if len(missing) != len(users)-2 {
|
||||||
|
t.Fatalf("%d users are missing a password, of %d: %v", len(missing), len(users), missing)
|
||||||
|
}
|
||||||
|
for _, p := range filled {
|
||||||
|
if p.PasswordHash == "" {
|
||||||
|
t.Fatalf("%s was kept with no password, which is a user anybody is", p.Username())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// And the whole thing composes: records in, a file the server would read out.
|
||||||
|
func TestRecordsComposeIntoAFile(t *testing.T) {
|
||||||
|
users, err := Users(someRecords())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
hashes := map[string]string{}
|
||||||
|
for _, u := range users {
|
||||||
|
hashes[u.Username()] = "$2a$11$" + strings.Repeat("x", 22)
|
||||||
|
}
|
||||||
|
filled, missing := WithPasswords(users, hashes)
|
||||||
|
if len(missing) != 0 {
|
||||||
|
t.Fatalf("users with no password: %v", missing)
|
||||||
|
}
|
||||||
|
got, err := Compose(Server{ClientPort: 4222, MonitoringPort: 8222, StoreDir: "/data",
|
||||||
|
TLSCert: "/tls/tls.crt", TLSKey: "/tls/tls.key", TLSCA: "/tls/ca.crt"}, filled)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, want := range []string{
|
||||||
|
`user: "controller"`, `user: "node.one"`, `user: "one.telegram"`,
|
||||||
|
`user: "enrol.three"`, `user: "person.ada"`,
|
||||||
|
`"_INBOX.enrol.three.>"`, `"mesh.mod.mesh-catalog.tool.catalog_tools"`,
|
||||||
|
} {
|
||||||
|
if !strings.Contains(got, want) {
|
||||||
|
t.Errorf("the composed file does not contain %s", want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The accounts block alone is what the mesh writes, and it holds nothing about the server.
|
||||||
|
//
|
||||||
|
// **The split is the whole design decision** (ComposeAccounts): ports, TLS paths and a store
|
||||||
|
// directory are properties of the container the module raises, and a controller that wrote them
|
||||||
|
// would have to be kept in step with a Dockerfile it never sees. So this test says what must not be
|
||||||
|
// in the file as plainly as what must.
|
||||||
|
func TestWhatTheMeshWritesIsUsersAndNothingAboutTheServer(t *testing.T) {
|
||||||
|
users, err := Users(someRecords())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
hashes := map[string]string{}
|
||||||
|
for _, u := range users {
|
||||||
|
hashes[u.Username()] = "$2a$11$" + strings.Repeat("x", 22)
|
||||||
|
}
|
||||||
|
filled, missing := WithPasswords(users, hashes)
|
||||||
|
if len(missing) != 0 {
|
||||||
|
t.Fatalf("users with no password: %v", missing)
|
||||||
|
}
|
||||||
|
got, err := ComposeAccounts(filled)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, want := range []string{"accounts {", `user: "controller"`, `user: "one.telegram"`} {
|
||||||
|
if !strings.Contains(got, want) {
|
||||||
|
t.Errorf("the accounts file does not contain %s", want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// None of the server's own settings. Each of these in the mesh's file is a value the controller
|
||||||
|
// would then own, and the module could no longer change its own image without the mesh agreeing.
|
||||||
|
for _, absent := range []string{"port:", "http:", "jetstream", "tls {", "store_dir", "cert_file"} {
|
||||||
|
if strings.Contains(got, absent) {
|
||||||
|
t.Errorf("the accounts file contains %q, which belongs to the module that raises the "+
|
||||||
|
"server, not to the mesh", absent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A user with no password is refused here too, not only by the whole-file composition: this is the
|
||||||
|
// function the controller actually calls, and a user without a password is a user anybody is.
|
||||||
|
func TestTheAccountsFileRefusesAUserWithNoPassword(t *testing.T) {
|
||||||
|
if _, err := ComposeAccounts([]Principal{{Kind: KindController}}); err == nil {
|
||||||
|
t.Fatal("a user with no password hash was written")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A user list is composed for a bus the mesh has not moved onto yet**, and that is the whole of
|
||||||
|
// step 2 (novox/hq ADR 0116): the server stands in the mesh carrying nothing, on its own ports, while
|
||||||
|
// every node is still on the bus it was on.
|
||||||
|
//
|
||||||
|
// Pinned because the first version of the composing step got it backwards — it wrote the list only
|
||||||
|
// once the controller was already on the new bus, which is a step that cannot be taken: the module
|
||||||
|
// comes up, finds no accounts file, and waits for one the controller had decided not to write.
|
||||||
|
func TestAUserListIsComposedBeforeAnythingMovesOntoTheBus(t *testing.T) {
|
||||||
|
// Exactly the records of a mesh mid-change: everything running, nothing on the new bus.
|
||||||
|
users, err := Users(Records{
|
||||||
|
Nodes: []string{"anchor"},
|
||||||
|
Assigned: map[string][]Declared{"anchor": {{Module: "nats"}}},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
hashes := map[string]string{}
|
||||||
|
for _, u := range users {
|
||||||
|
hashes[u.Username()] = "$2a$11$" + strings.Repeat("x", 22)
|
||||||
|
}
|
||||||
|
filled, missing := WithPasswords(users, hashes)
|
||||||
|
if len(missing) != 0 {
|
||||||
|
t.Fatalf("users with no credential: %v", missing)
|
||||||
|
}
|
||||||
|
accounts, err := ComposeAccounts(filled)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// The controller's own user above all: a file without it is a bus its writer cannot connect to,
|
||||||
|
// which is what the server would be left holding the moment it starts.
|
||||||
|
if !strings.Contains(accounts, `user: "controller"`) {
|
||||||
|
t.Fatalf("the composed list does not contain the controller:\n%s", accounts)
|
||||||
|
}
|
||||||
|
if !strings.Contains(accounts, `user: "node.anchor"`) {
|
||||||
|
t.Errorf("the composed list does not contain the machine running the bus")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,101 @@
|
|||||||
|
package catalogue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The mesh's bus is one per mesh, read from the catalogue beside this checkout.
|
||||||
|
//
|
||||||
|
// **This is step 2's claim, and it is checked here rather than in a bed** (novox/hq ADR 0116):
|
||||||
|
// adoption puts the NATS server into the `mesh-broker` seat on a mesh that is already running,
|
||||||
|
// and the property that matters is that a second one anywhere is refused *when it is assigned*,
|
||||||
|
// not discovered later as two servers holding different halves of the mesh's traffic. A second
|
||||||
|
// bus is not a degraded mesh; it is two meshes that both believe they are the one.
|
||||||
|
func TestASecondMeshBusAnywhereIsRefusedByName(t *testing.T) {
|
||||||
|
nats := catalogueManifest(t, "nats")
|
||||||
|
|
||||||
|
if _, err := Resolve(shelf(nats), []string{"nats"}, workstation(), World{}); err != nil {
|
||||||
|
t.Fatalf("the bus alone does not resolve: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
elsewhere := World{Held: []Held{{Claim: "mesh-broker", Scope: ScopeMesh,
|
||||||
|
Node: "anchor", Module: "nats"}}}
|
||||||
|
other := workstation()
|
||||||
|
other.Name = "laptop"
|
||||||
|
_, err := Resolve(shelf(nats), []string{"nats"}, other, elsewhere)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a second bus was accepted on another machine")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "mesh-broker") || !strings.Contains(err.Error(), "one per mesh") {
|
||||||
|
t.Fatalf("refused without naming the seat: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The seat is the server's role, not the product's name (novox/hq ADR 0079). A different
|
||||||
|
// implementation of the bus claims the same seat, and the mesh refuses it for the same reason —
|
||||||
|
// which is the property that lets the bus be replaced at all.
|
||||||
|
func TestTheSeatRefusesADifferentBusToo(t *testing.T) {
|
||||||
|
nats := catalogueManifest(t, "nats")
|
||||||
|
held := World{Held: []Held{{Claim: "mesh-broker", Scope: ScopeMesh,
|
||||||
|
Node: "anchor", Module: "some-other-broker"}}}
|
||||||
|
other := workstation()
|
||||||
|
other.Name = "laptop"
|
||||||
|
if _, err := Resolve(shelf(nats), []string{"nats"}, other, held); err == nil {
|
||||||
|
t.Fatal("the seat admitted a second holder because the module's name differed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The old broker no longer claims the seat: it is an ordinary provider of `amqp`
|
||||||
|
// (novox/hq ADR 0119), so it can sit on the same mesh as the bus without contending for it.
|
||||||
|
func TestTheAmqpBrokerDoesNotContendForTheSeat(t *testing.T) {
|
||||||
|
lavinmq := catalogueManifest(t, "lavinmq")
|
||||||
|
for _, c := range lavinmq.Claims {
|
||||||
|
if c.Name == "mesh-broker" {
|
||||||
|
t.Fatal("the amqp broker still claims mesh-broker; it is a provider, not foundation")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
busHeld := World{Held: []Held{{Claim: "mesh-broker", Scope: ScopeMesh,
|
||||||
|
Node: "anchor", Module: "nats"}}}
|
||||||
|
other := workstation()
|
||||||
|
other.Name = "laptop"
|
||||||
|
if _, err := Resolve(shelf(lavinmq), []string{"lavinmq"}, other, busHeld); err != nil {
|
||||||
|
t.Fatalf("the amqp broker was refused beside the mesh bus: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A seat and the interface it delivers are different names, and renaming one must not rename
|
||||||
|
// the other** (novox/hq ADR 0118). This nearly went wrong: the seats were renamed to the `mesh-*`
|
||||||
|
// prefix, and a blanket search-and-replace also renamed `npm-package-registry` and `git` where
|
||||||
|
// they are *provisions* — which a consumer requires and a provider offers. The tests failed with
|
||||||
|
// "the package registry is served on <nil>", which does not say "you renamed an interface".
|
||||||
|
func TestRenamingASeatDidNotRenameTheInterfaceItDelivers(t *testing.T) {
|
||||||
|
for _, pair := range []struct{ seat, delivers string }{
|
||||||
|
{"git", "git"},
|
||||||
|
{"npm-package-registry", "npm-package-registry"},
|
||||||
|
{"the-artifact-store", "artifact-store"},
|
||||||
|
{"mesh-store", "postgres-database"},
|
||||||
|
{"mesh-broker", "mesh-bus"},
|
||||||
|
} {
|
||||||
|
s, known := SeatNamed(pair.seat)
|
||||||
|
if !known {
|
||||||
|
t.Fatalf("%q is not a seat", pair.seat)
|
||||||
|
}
|
||||||
|
if s.Delivers != pair.delivers {
|
||||||
|
t.Errorf("the %s seat delivers %q, expected %q — renaming the seat moved the "+
|
||||||
|
"interface with it, and every consumer requiring it would stop resolving",
|
||||||
|
pair.seat, s.Delivers, pair.delivers)
|
||||||
|
}
|
||||||
|
// **Three of these deliberately share a name with what they deliver**, and that is not an
|
||||||
|
// incomplete rename. Renaming a seat that delivers a provision cascades to every consumer
|
||||||
|
// requiring it, with a mesh-wide window where a holder stops resolving mid-flight — so the
|
||||||
|
// trunk deferred exactly those three (novox/hq ADR 0121) while renaming the node-scoped ones.
|
||||||
|
// What this test is for is the other direction: that renaming a seat never moves the
|
||||||
|
// interface, which once produced "the package registry is served on <nil>".
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A manifest written against an old seat name is told what it became rather than refused as
|
||||||
|
// unknown. **That map is the controller's store now, not this package** (novox/hq ADR 0122): a
|
||||||
|
// rename is a row, so the courtesy survives a rename nobody recompiled for. Checked where the
|
||||||
|
// table is read, not here, where there is no longer a hardcoded list to check against.
|
||||||
@@ -97,6 +97,24 @@ type Rendering struct {
|
|||||||
// compose it a second time.
|
// compose it a second time.
|
||||||
Suffix string
|
Suffix string
|
||||||
|
|
||||||
|
// BusUsers is the mesh's composed user list, for the module holding `mesh-broker`. Empty on
|
||||||
|
// every other node, and on this one until the controller has composed it.
|
||||||
|
//
|
||||||
|
// **Only the users, never the server's own settings**: those are the module's, in its image and
|
||||||
|
// its mounts (Manifest.BusUsers).
|
||||||
|
BusUsers string
|
||||||
|
|
||||||
|
// MeshRange is the private network's CIDR (the range node addresses are allocated from), for a
|
||||||
|
// module that must name the whole mesh rather than one machine — an intrusion filter that must
|
||||||
|
// never ban a tunnel peer, say. A per-mesh value the module cannot know, so it is carried here
|
||||||
|
// and offered as ${machine:mesh-range}, the same way one machine's address is.
|
||||||
|
MeshRange string
|
||||||
|
|
||||||
|
// Accounts is each machine's operator account, by the same internal name Names uses (novox/hq
|
||||||
|
// to-be 29). What an ssh Host block's `User` line is composed from; empty for a machine no
|
||||||
|
// operator account is known on.
|
||||||
|
Accounts map[string]string
|
||||||
|
|
||||||
// Kept is every operator-sealed secret in the mesh, for a module that `keeps` them. Nil when
|
// Kept is every operator-sealed secret in the mesh, for a module that `keeps` them. Nil when
|
||||||
// nothing on this node keeps them, or the mesh has no operator key.
|
// nothing on this node keeps them, or the mesh has no operator key.
|
||||||
Kept *KeptExport
|
Kept *KeptExport
|
||||||
@@ -339,6 +357,12 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri
|
|||||||
"content": filtering, "mode": "0600",
|
"content": filtering, "mode": "0600",
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
// The node's fail2ban jails, composed from every module it runs (novox/hq to-be 31), written
|
||||||
|
// where the intrusion-prevention holder owns them. Like the rule set above: gathered from all
|
||||||
|
// modules, written by the one that holds the role.
|
||||||
|
if j := m.Jailing; j != nil {
|
||||||
|
first = append(first, jailsInto(r.Modules, j)...)
|
||||||
|
}
|
||||||
if c := m.Certificate; c != nil {
|
if c := m.Certificate; c != nil {
|
||||||
if with.Certificate == "" {
|
if with.Certificate == "" {
|
||||||
// Asked for and not issued. Refused rather than skipped: a module that serves TLS
|
// Asked for and not issued. Refused rather than skipped: a module that serves TLS
|
||||||
@@ -359,6 +383,35 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if m.BusUsers != "" {
|
||||||
|
// **The claim authorises it, not the field.** This file holds every user's password
|
||||||
|
// hash, so a module that could ask for it could read every credential on the bus.
|
||||||
|
// Checked from this manifest alone, which is the cheapest check there is: whether some
|
||||||
|
// other module also claims the seat is resolution's business elsewhere, and one holder
|
||||||
|
// mesh-wide is already guaranteed.
|
||||||
|
if !m.ClaimsSeat("mesh-broker") {
|
||||||
|
return nil, fmt.Errorf(
|
||||||
|
"%s asks for the mesh's user list and does not claim mesh-broker. That file "+
|
||||||
|
"holds every user's password hash, so the seat is what authorises it",
|
||||||
|
m.Module)
|
||||||
|
}
|
||||||
|
if with.BusUsers == "" {
|
||||||
|
// Asked for and not composed. Refused rather than skipped, for the reason a
|
||||||
|
// certificate is: a bus with no user list refuses every connection in the mesh, and
|
||||||
|
// an empty file would look like a configuration problem on the machine.
|
||||||
|
return nil, fmt.Errorf(
|
||||||
|
"%s holds mesh-broker and the mesh composed no user list, so the bus would "+
|
||||||
|
"refuse every connection", m.Module)
|
||||||
|
}
|
||||||
|
first = append(first, map[string]any{
|
||||||
|
"id": BusUsersID(), "type": "file", "path": m.BusUsers,
|
||||||
|
"content": with.BusUsers,
|
||||||
|
// Readable by the server and nothing else. Hashes rather than passwords, so this is
|
||||||
|
// not a set of working credentials — but a list of every user in the mesh is worth
|
||||||
|
// keeping to the one process that needs it.
|
||||||
|
"mode": "0600",
|
||||||
|
})
|
||||||
|
}
|
||||||
for _, name := range sortedKeys(m.OwnSecrets) {
|
for _, name := range sortedKeys(m.OwnSecrets) {
|
||||||
sealed := with.Needed[m.Module][name]
|
sealed := with.Needed[m.Module][name]
|
||||||
if sealed == "" {
|
if sealed == "" {
|
||||||
@@ -545,7 +598,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri
|
|||||||
// (novox/hq ADR 0112) — resolved once per module, named by ${dir:…} from any resource.
|
// (novox/hq ADR 0112) — resolved once per module, named by ${dir:…} from any resource.
|
||||||
dirs := dirsFor(m, with)
|
dirs := dirsFor(m, with)
|
||||||
// And the machine underneath, which no binding of its own can tell it.
|
// And the machine underneath, which no binding of its own can tell it.
|
||||||
thisMachine := machineFacts(r, with.Names)
|
thisMachine := machineFacts(r, with.Names, with.MeshRange)
|
||||||
|
|
||||||
// Which of this module's files carry a secret, for the rule that a container may not read
|
// Which of this module's files carry a secret, for the rule that a container may not read
|
||||||
// one of them as its environment without saying so (ADR 0086, issue 041).
|
// one of them as its environment without saying so (ADR 0086, issue 041).
|
||||||
@@ -644,7 +697,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri
|
|||||||
// plane's; making a name resolve is the module's software. Emitted as ordinary files under
|
// plane's; making a name resolve is the module's software. Emitted as ordinary files under
|
||||||
// this module's name, so they are applied, reported and removed exactly as anything else
|
// this module's name, so they are applied, reported and removed exactly as anything else
|
||||||
// it declares.
|
// it declares.
|
||||||
given, err := FactsInto(m, r, with.Names, with.Machines, with.Suffix)
|
given, err := FactsInto(m, r, with.Names, with.Machines, with.Accounts, with.Suffix)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,80 @@
|
|||||||
|
package catalogue
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
// The mesh's user list reaches the module holding the bus, and nothing else.
|
||||||
|
//
|
||||||
|
// Three refusals and one delivery, because each of the refusals would be silent in a different way:
|
||||||
|
// a module that asked and was given it could read every credential on the bus; a bus given an empty
|
||||||
|
// file refuses every connection in the mesh and looks like a machine problem; and a bus that never
|
||||||
|
// asked gets nothing rather than a file it does not read.
|
||||||
|
func TestTheMeshsUserListGoesOnlyToTheModuleHoldingTheBus(t *testing.T) {
|
||||||
|
theBus := func() Manifest {
|
||||||
|
return Manifest{
|
||||||
|
Module: "nats", Version: "1",
|
||||||
|
Claims: []Claim{{Name: "mesh-broker", Scope: ScopeMesh}},
|
||||||
|
BusUsers: "/var/lib/nats-module/conf/accounts.conf",
|
||||||
|
Resources: []map[string]any{},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
on := func(t *testing.T, m Manifest, with Rendering) ([]map[string]any, error) {
|
||||||
|
t.Helper()
|
||||||
|
return Resolution{Node: "anchor", Modules: []Manifest{m}}.Declaration(with)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("the holder is given it", func(t *testing.T) {
|
||||||
|
resources, err := on(t, theBus(), Rendering{BusUsers: "accounts { MESH { users = [] } }"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Prefixed with the module it came from, like every resource: two modules may reasonably
|
||||||
|
// both call something "config", and without the prefix the second would silently replace
|
||||||
|
// the first.
|
||||||
|
var found map[string]any
|
||||||
|
for _, r := range resources {
|
||||||
|
if r["id"] == "nats."+BusUsersID() {
|
||||||
|
found = r
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if found == nil {
|
||||||
|
t.Fatalf("the bus was given no user list: %+v", resources)
|
||||||
|
}
|
||||||
|
if found["path"] != "/var/lib/nats-module/conf/accounts.conf" {
|
||||||
|
t.Errorf("written to %v rather than where the module asked", found["path"])
|
||||||
|
}
|
||||||
|
if found["mode"] != "0600" {
|
||||||
|
t.Errorf("mode %v: a list of every user in the mesh belongs to the one process that "+
|
||||||
|
"needs it", found["mode"])
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("a module that does not claim the seat is refused", func(t *testing.T) {
|
||||||
|
m := theBus()
|
||||||
|
m.Claims = nil
|
||||||
|
if _, err := on(t, m, Rendering{BusUsers: "accounts {}"}); err == nil {
|
||||||
|
t.Fatal("a module that claims nothing was handed every user's password hash")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("the holder with nothing composed is refused", func(t *testing.T) {
|
||||||
|
if _, err := on(t, theBus(), Rendering{}); err == nil {
|
||||||
|
t.Fatal("the bus was given an empty user list, so it would refuse every connection in " +
|
||||||
|
"the mesh and look like a machine problem")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("a module that did not ask gets nothing", func(t *testing.T) {
|
||||||
|
m := theBus()
|
||||||
|
m.BusUsers = ""
|
||||||
|
resources, err := on(t, m, Rendering{BusUsers: "accounts {}"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, r := range resources {
|
||||||
|
if r["id"] == "nats."+BusUsersID() {
|
||||||
|
t.Fatal("a module that asked for no user list was given one")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,161 @@
|
|||||||
|
package catalogue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What a module may call an event, and what a consumer may ask for.
|
||||||
|
//
|
||||||
|
// A module names an event **locally**: `order.placed`, not a subject and not a routing key
|
||||||
|
// (design 29 §1). A consumer names the emitter and the event: `billing.order.placed`. The mesh
|
||||||
|
// derives the subject from those, so reorganising the subject space leaves every manifest correct.
|
||||||
|
//
|
||||||
|
// **Nothing checked this until every manifest in the catalogue was wrong the same way**
|
||||||
|
// (novox/hq 04-ISSUES/127). All thirty-seven kept the old bus's routing key —
|
||||||
|
// `module.<module>.<verb>` — which the derivation read as "a module called `module`", so every
|
||||||
|
// cross-module subscription in the mesh pointed at a namespace nobody publishes to. Nothing failed:
|
||||||
|
// the services started and none of them reacted. The documentation on these fields taught the old
|
||||||
|
// form too, which is why the drift was uniform rather than scattered.
|
||||||
|
|
||||||
|
// eventName is one name in a local event: lower-case, and no wildcard.
|
||||||
|
var eventName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`)
|
||||||
|
|
||||||
|
// The wildcards a consumer may use, spelled the mesh's way and derived to whatever the transport
|
||||||
|
// spells them as.
|
||||||
|
//
|
||||||
|
// **A manifest holds no transport token**, which is the whole point of naming locally: the bus the
|
||||||
|
// mesh runs on today spells these `*` and `#`, and the one being built spells them `*` and `>`. A
|
||||||
|
// manifest that said either would be a manifest that stopped being true when the wire changed.
|
||||||
|
const (
|
||||||
|
// OneName stands for exactly one name.
|
||||||
|
OneName = "*"
|
||||||
|
// TheRest stands for one or more names, and may only come last.
|
||||||
|
TheRest = "**"
|
||||||
|
)
|
||||||
|
|
||||||
|
// EventProblems is what is wrong with a manifest's events.
|
||||||
|
//
|
||||||
|
// Refused at registration, because the alternative is a module that installs, starts, connects and
|
||||||
|
// reacts to nothing — and every log line says it is fine.
|
||||||
|
func EventProblems(m Manifest) []string {
|
||||||
|
var problems []string
|
||||||
|
|
||||||
|
for _, e := range m.Emits {
|
||||||
|
if was, stale := staleEventForm(e, m.Module); stale {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s emits %q, which is the old bus's routing key. An event is named locally now, so "+
|
||||||
|
"write %q — the mesh derives the subject (novox/hq design 29 §1)",
|
||||||
|
m.Module, was, strings.TrimPrefix(was, "module."+m.Module+".")))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(e, "module.") {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s emits %q: `module.` is reserved, because it is how the old bus spelled a "+
|
||||||
|
"routing key and an event named that way derives into a namespace nobody owns",
|
||||||
|
m.Module, e))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := localName(e); err != nil {
|
||||||
|
problems = append(problems, fmt.Sprintf("%s emits %q: %v", m.Module, e, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// **Its own name, never another's.** The bus enforces that a namespace belongs to the module
|
||||||
|
// it is named for, so an event named for somebody else cannot be published at all. If the
|
||||||
|
// event is about a role rather than about this module, it belongs on the seat: a name that
|
||||||
|
// is stable across whoever fills it (04-ISSUES/127).
|
||||||
|
if first, _, split := strings.Cut(e, "."); split && isAModuleNameOtherThan(first, m.Module) {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s emits %q, which reads as another module's event. A module publishes under its "+
|
||||||
|
"own name only. If this is about a role rather than about %s, declare it on that "+
|
||||||
|
"seat, where the name survives the holder changing",
|
||||||
|
m.Module, e, m.Module))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, c := range m.Consumes {
|
||||||
|
if strings.HasPrefix(c, "module.") {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s consumes %q, which is the old bus's pattern. A consumed event names its emitter "+
|
||||||
|
"and the event: write %q", m.Module, c, strings.TrimPrefix(c, "module.")))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if c == "#" {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s consumes %q, which is the old bus's wildcard for everything. Write %q",
|
||||||
|
m.Module, c, TheRest))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := consumePattern(c); err != nil {
|
||||||
|
problems = append(problems, fmt.Sprintf("%s consumes %q: %v", m.Module, c, err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return problems
|
||||||
|
}
|
||||||
|
|
||||||
|
// staleEventForm says an emitted name is this module's own old routing key, and what it was.
|
||||||
|
func staleEventForm(event, module string) (string, bool) {
|
||||||
|
return event, module != "" && strings.HasPrefix(event, "module."+module+".")
|
||||||
|
}
|
||||||
|
|
||||||
|
// isAModuleNameOtherThan says a first token names some module of this mesh that is not this one.
|
||||||
|
//
|
||||||
|
// Only the mesh's own seats and the catalogue could answer this properly, and neither is reachable
|
||||||
|
// from a parser given one manifest. So this catches the case that actually happened — a name that
|
||||||
|
// is a *provision* the mesh defines, which is where "another module's event" comes from in practice
|
||||||
|
// — and the whole-catalogue check catches the rest.
|
||||||
|
func isAModuleNameOtherThan(first, module string) bool {
|
||||||
|
if first == module || first == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if _, isASeat := SeatNamed(first); isASeat {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if _, isASeat := SeatDelivering(first); isASeat {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// localName checks one event name: dot-separated names, no wildcards, nothing else.
|
||||||
|
func localName(event string) error {
|
||||||
|
if event == "" {
|
||||||
|
return fmt.Errorf("an event needs a name")
|
||||||
|
}
|
||||||
|
for _, part := range strings.Split(event, ".") {
|
||||||
|
if part == OneName || part == TheRest {
|
||||||
|
return fmt.Errorf("an emitted event names one event, so it carries no wildcard")
|
||||||
|
}
|
||||||
|
if !eventName.MatchString(part) {
|
||||||
|
return fmt.Errorf("%q is not a usable name: lower-case letters, digits and dashes", part)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// consumePattern checks a consumed pattern: the emitter, then the event, with wildcards.
|
||||||
|
func consumePattern(pattern string) error {
|
||||||
|
if pattern == "" {
|
||||||
|
return fmt.Errorf("a consumed event needs an emitter and an event")
|
||||||
|
}
|
||||||
|
parts := strings.Split(pattern, ".")
|
||||||
|
for i, part := range parts {
|
||||||
|
switch {
|
||||||
|
case part == TheRest:
|
||||||
|
if i != len(parts)-1 {
|
||||||
|
return fmt.Errorf("%q stands for the rest of a name, so nothing may follow it", TheRest)
|
||||||
|
}
|
||||||
|
case part == OneName:
|
||||||
|
case !eventName.MatchString(part):
|
||||||
|
return fmt.Errorf("%q is not a usable name: lower-case letters, digits and dashes", part)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// `**` alone is every event from every module, which the audit logger wants and says plainly.
|
||||||
|
if len(parts) == 1 && parts[0] != TheRest {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"%q names an emitter and no event. Write <emitter>.<event>, or %q for every event",
|
||||||
|
pattern, TheRest)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -1,184 +0,0 @@
|
|||||||
package catalogue
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
// What only the mesh knows, written where a module asks for it.
|
|
||||||
//
|
|
||||||
// **The graph is the control plane's; using it is the module's.** The mesh knows which machines
|
|
||||||
// exist, what they are called, and where they are. Turning that into a name that resolves is
|
|
||||||
// somebody's software, and which software is a choice the mesh should not be making.
|
|
||||||
//
|
|
||||||
// This replaced three modules — names, a resolver's data, and the private network's own
|
|
||||||
// configuration — that existed only because computed output needed somewhere to live. They ran no
|
|
||||||
// software and could not be swapped for anything, which is the test of whether something is a
|
|
||||||
// module at all (novox/hq ADR 0040).
|
|
||||||
|
|
||||||
const (
|
|
||||||
// FactNodeNames is every machine's name and address, as a hosts file.
|
|
||||||
//
|
|
||||||
// Exact names only: `homer` and `homer.internal` resolve to homer. Anything *under* a machine
|
|
||||||
// is a wildcard, which a hosts file cannot express — that is FactNodeZones.
|
|
||||||
FactNodeNames = "node-names"
|
|
||||||
|
|
||||||
// FactNodeZones is every machine as a wildcard: `*.homer.internal` is homer.
|
|
||||||
//
|
|
||||||
// Written in the form a resolver reads. A machine's own name and everything under it are one
|
|
||||||
// fact — if homer is at an address, so is anything homer serves.
|
|
||||||
FactNodeZones = "node-zones"
|
|
||||||
)
|
|
||||||
|
|
||||||
// facts is every fact the mesh computes, and what writes it.
|
|
||||||
//
|
|
||||||
// **A closed list.** A module asking for a fact the mesh does not have is asking for a file nobody
|
|
||||||
// will write, and finding that out on a machine — as a daemon that starts, reads nothing, and
|
|
||||||
// answers no queries — is worse than being told where the manifest is.
|
|
||||||
// A fact is written from the names it is about. `every` is every name the mesh serves — machines
|
|
||||||
// and the names it was told to route; `machines` is only the machines. A fact takes the set it is
|
|
||||||
// true of, and the two must not be confused (novox/hq 04-ISSUES/111).
|
|
||||||
var facts = map[string]func(r Resolution, every, machines map[string]string, suffix string) string{
|
|
||||||
FactNodeNames: func(r Resolution, every, _ map[string]string, suffix string) string {
|
|
||||||
return nodeNames(r, every, suffix)
|
|
||||||
},
|
|
||||||
FactNodeZones: func(r Resolution, _, machines map[string]string, suffix string) string {
|
|
||||||
return nodeZones(r, machines, suffix)
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
// FactsInto renders the facts a module asked for, as files it will be given.
|
|
||||||
//
|
|
||||||
// The module owns everything after the file exists: loading it, restarting on it, what a resolver
|
|
||||||
// does with it. This only puts it there.
|
|
||||||
func FactsInto(m Manifest, r Resolution, addresses, machines map[string]string, suffix string) ([]map[string]any, error) {
|
|
||||||
if len(m.Facts) == 0 {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
names := make([]string, 0, len(m.Facts))
|
|
||||||
for name := range m.Facts {
|
|
||||||
names = append(names, name)
|
|
||||||
}
|
|
||||||
sort.Strings(names)
|
|
||||||
|
|
||||||
out := make([]map[string]any, 0, len(names))
|
|
||||||
for _, name := range names {
|
|
||||||
write, known := facts[name]
|
|
||||||
if !known {
|
|
||||||
return nil, fmt.Errorf(
|
|
||||||
"%s asks the mesh for %q, which it does not compute. It has %s",
|
|
||||||
m.Module, name, spokenFacts())
|
|
||||||
}
|
|
||||||
path := m.Facts[name]
|
|
||||||
if !strings.HasPrefix(path, "/") {
|
|
||||||
return nil, fmt.Errorf(
|
|
||||||
"%s asks for %q at %q, which is not an absolute path", m.Module, name, path)
|
|
||||||
}
|
|
||||||
out = append(out, map[string]any{
|
|
||||||
"id": "fact-" + name, "type": "file", "path": path, "mode": "0644",
|
|
||||||
"content": write(r, addresses, machines, suffix),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// spokenFacts lists them, so a refusal says what would have worked.
|
|
||||||
func spokenFacts() string {
|
|
||||||
names := make([]string, 0, len(facts))
|
|
||||||
for name := range facts {
|
|
||||||
names = append(names, name)
|
|
||||||
}
|
|
||||||
sort.Strings(names)
|
|
||||||
return strings.Join(names, ", ")
|
|
||||||
}
|
|
||||||
|
|
||||||
// nodeNames is every machine's name and address, as a hosts file.
|
|
||||||
//
|
|
||||||
// **A machine with no address is left out.** The mesh has a record for it — somebody added it —
|
|
||||||
// and does not yet know where it is, which is the ordinary state between adding a machine and it
|
|
||||||
// joining. Writing the name anyway would give a name that resolves to nothing, and a connection to
|
|
||||||
// that hangs; leaving it out fails at once and says the name is unknown.
|
|
||||||
func nodeNames(r Resolution, addresses map[string]string, suffix string) string {
|
|
||||||
var b strings.Builder
|
|
||||||
b.WriteString("# Generated by the mesh. Do not edit — this file is replaced whenever a machine\n")
|
|
||||||
b.WriteString("# joins or leaves, and an edit would survive until then and vanish.\n\n")
|
|
||||||
// The floor every Linux expects, and which removing would break things that have nothing to do
|
|
||||||
// with the mesh.
|
|
||||||
b.WriteString("127.0.0.1\tlocalhost\n")
|
|
||||||
b.WriteString("::1\t\tlocalhost ip6-localhost ip6-loopback\n")
|
|
||||||
if r.Node != "" {
|
|
||||||
fmt.Fprintf(&b, "127.0.1.1\t%s\n", r.Node)
|
|
||||||
}
|
|
||||||
b.WriteString("\n")
|
|
||||||
for _, name := range sortedNames(addresses) {
|
|
||||||
at := addresses[name]
|
|
||||||
internal, bare := meshName(name, suffix)
|
|
||||||
// Its mesh name resolves to its address on the private network rather than to loopback,
|
|
||||||
// so a service binding the name it was given stays reachable from everywhere else.
|
|
||||||
fmt.Fprintf(&b, "%s\t%s\t%s", at, internal, bare)
|
|
||||||
if bare == r.Node {
|
|
||||||
b.WriteString("\t# this machine")
|
|
||||||
}
|
|
||||||
b.WriteString("\n")
|
|
||||||
}
|
|
||||||
return b.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// nodeZones is every machine as a wildcard, in the form a resolver reads.
|
|
||||||
//
|
|
||||||
// `*.homer.internal` is homer, which is the whole rule: if homer is at an address, so is anything
|
|
||||||
// homer serves. A module wanting this runs the resolver; the mesh only says what is true.
|
|
||||||
//
|
|
||||||
// **And the suffix itself, as a local domain.** A resolver that forwards what it cannot answer
|
|
||||||
// would otherwise send a mesh name it does not know — a machine that left, a typo — to a public
|
|
||||||
// resolver, which is a leak of the mesh's names for no answer. `local=` keeps everything under the
|
|
||||||
// suffix here: answered from the lines below or refused. Written in this file rather than in the
|
|
||||||
// resolver's own configuration because the suffix is the mesh's choice (the operator may have
|
|
||||||
// picked another) and this file is the one place the mesh writes what it chose.
|
|
||||||
func nodeZones(_ Resolution, addresses map[string]string, suffix string) string {
|
|
||||||
var b strings.Builder
|
|
||||||
b.WriteString("# Generated by the mesh. Do not edit — this file is replaced whenever a machine\n")
|
|
||||||
b.WriteString("# joins or leaves, and an edit would survive until then and vanish.\n\n")
|
|
||||||
fmt.Fprintf(&b, "local=/%s/\n", strings.TrimPrefix(suffixOr(suffix), "."))
|
|
||||||
for _, name := range sortedNames(addresses) {
|
|
||||||
internal, _ := meshName(name, suffix)
|
|
||||||
fmt.Fprintf(&b, "address=/%s/%s\n", internal, addresses[name])
|
|
||||||
}
|
|
||||||
return b.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// meshName is a machine's internal name and its bare one, from either. The control plane keys
|
|
||||||
// the names it hands a resolution by the internal name (`homer.internal`), the same map a
|
|
||||||
// container gets as its hosts; a caller that keys by the bare name gets the same answer. The
|
|
||||||
// suffix is the one the control plane composed those names with, handed down rather than written
|
|
||||||
// here a second time — the alternative was `homer.internal.internal` on every machine.
|
|
||||||
func meshName(name, suffix string) (internal, bare string) {
|
|
||||||
dotted := "." + strings.TrimPrefix(suffixOr(suffix), ".")
|
|
||||||
if strings.HasSuffix(name, dotted) {
|
|
||||||
return name, strings.TrimSuffix(name, dotted)
|
|
||||||
}
|
|
||||||
return name + dotted, name
|
|
||||||
}
|
|
||||||
|
|
||||||
// suffixOr is the suffix given, or the one the mesh composes names with when none was handed down.
|
|
||||||
// The one place the default is written in this file, so a fact and a name cannot disagree about it.
|
|
||||||
func suffixOr(suffix string) string {
|
|
||||||
if suffix == "" {
|
|
||||||
return "internal"
|
|
||||||
}
|
|
||||||
return suffix
|
|
||||||
}
|
|
||||||
|
|
||||||
func sortedNames(addresses map[string]string) []string {
|
|
||||||
out := make([]string, 0, len(addresses))
|
|
||||||
for name, at := range addresses {
|
|
||||||
// See nodeNames: a machine the mesh cannot place is left out rather than named at nothing.
|
|
||||||
if at == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
out = append(out, name)
|
|
||||||
}
|
|
||||||
sort.Strings(out)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
@@ -1,188 +0,0 @@
|
|||||||
package catalogue
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Keyed by the internal name, as the control plane hands them (issue 079).
|
|
||||||
var threeMachines = map[string]string{"homer.internal": "10.42.0.1", "marge.internal": "10.42.0.2", "bart.internal": ""}
|
|
||||||
|
|
||||||
// **`*.homer.internal` is homer. That is the whole rule.** And the suffix itself is local: a
|
|
||||||
// resolver that forwards what it cannot answer must not send a mesh name it does not know — a
|
|
||||||
// machine that left, a typo — to a public resolver (hal dnsmasq-app conversion, novox/hq
|
|
||||||
// 08-connectivity).
|
|
||||||
func TestEveryMachineIsAWildcardUnderItsOwnName(t *testing.T) {
|
|
||||||
out := nodeZones(Resolution{Node: "homer"}, threeMachines, "")
|
|
||||||
for _, want := range []string{
|
|
||||||
"local=/internal/",
|
|
||||||
"address=/homer.internal/10.42.0.1",
|
|
||||||
"address=/marge.internal/10.42.0.2",
|
|
||||||
} {
|
|
||||||
if !strings.Contains(out, want) {
|
|
||||||
t.Fatalf("missing %q:\n%s", want, out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A machine the mesh has a record for and cannot place is left out of both.
|
|
||||||
//
|
|
||||||
// **Not an oversight — the alternative is worse.** A name written with no address resolves to
|
|
||||||
// nothing, and a connection to that hangs. Leaving it out fails at once and says the name is
|
|
||||||
// unknown, which is a thing somebody can act on.
|
|
||||||
func TestAMachineWithNoAddressIsNotNamed(t *testing.T) {
|
|
||||||
for _, out := range []string{
|
|
||||||
nodeNames(Resolution{Node: "homer"}, threeMachines, ""),
|
|
||||||
nodeZones(Resolution{Node: "homer"}, threeMachines, ""),
|
|
||||||
} {
|
|
||||||
if strings.Contains(out, "bart") {
|
|
||||||
t.Fatalf("a machine with no address was named, so its name resolves to nothing:\n%s", out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A machine's own mesh name points at its address on the private network, not at loopback — or a
|
|
||||||
// service binding the name it was given is unreachable from everywhere else.
|
|
||||||
func TestAMachinesOwnNameIsItsMeshAddress(t *testing.T) {
|
|
||||||
out := nodeNames(Resolution{Node: "homer"}, threeMachines, "")
|
|
||||||
var line string
|
|
||||||
for _, l := range strings.Split(out, "\n") {
|
|
||||||
if strings.Contains(l, "homer.internal") {
|
|
||||||
line = l
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !strings.HasPrefix(line, "10.42.0.1") {
|
|
||||||
t.Fatalf("a machine's own mesh name is not its mesh address: %q", line)
|
|
||||||
}
|
|
||||||
// And the loopback floor is still there, or things with nothing to do with the mesh break.
|
|
||||||
if !strings.Contains(out, "127.0.0.1\tlocalhost") {
|
|
||||||
t.Fatalf("the loopback floor was removed:\n%s", out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A module says where it wants a fact, and is given a file.
|
|
||||||
func TestAModuleIsGivenTheFactsItAskedFor(t *testing.T) {
|
|
||||||
m := Manifest{Module: "dnsmasq", Facts: map[string]string{FactNodeZones: "/etc/mesh/zones.conf"}}
|
|
||||||
given, err := FactsInto(m, Resolution{Node: "homer"}, threeMachines, threeMachines, "")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(given) != 1 {
|
|
||||||
t.Fatalf("expected one file, got %d", len(given))
|
|
||||||
}
|
|
||||||
if given[0]["path"] != "/etc/mesh/zones.conf" || given[0]["type"] != "file" {
|
|
||||||
t.Fatalf("not written where it was asked for: %v", given[0])
|
|
||||||
}
|
|
||||||
if !strings.Contains(given[0]["content"].(string), "homer.internal") {
|
|
||||||
t.Fatalf("the file does not hold the fact: %v", given[0]["content"])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **Asking for a fact the mesh does not have is refused here, not on a machine.** A daemon that
|
|
||||||
// starts, reads a file nobody wrote, and answers no queries is a much worse way to find out.
|
|
||||||
func TestAskingForAFactTheMeshDoesNotHaveIsRefused(t *testing.T) {
|
|
||||||
m := Manifest{Module: "dnsmasq", Facts: map[string]string{"the-weather": "/etc/weather"}}
|
|
||||||
_, err := FactsInto(m, Resolution{}, nil, nil, "")
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("a module asked for something nobody computes and was given nothing, silently")
|
|
||||||
}
|
|
||||||
for _, known := range []string{FactNodeNames, FactNodeZones} {
|
|
||||||
if !strings.Contains(err.Error(), known) {
|
|
||||||
t.Fatalf("the refusal does not say what would have worked: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// And a relative path is refused, or a module decides where the mesh writes on a machine.
|
|
||||||
func TestAFactMustBeAskedForAtAnAbsolutePath(t *testing.T) {
|
|
||||||
m := Manifest{Module: "dnsmasq", Facts: map[string]string{FactNodeNames: "etc/hosts"}}
|
|
||||||
if _, err := FactsInto(m, Resolution{}, nil, nil, ""); err == nil {
|
|
||||||
t.Fatal("a relative path was accepted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **The names the control plane hands a resolution are already internal names** — `homer.internal`,
|
|
||||||
// the same map every container gets as its hosts. Appending the suffix again wrote
|
|
||||||
// `homer.internal.internal` into every hosts file and every resolver's zones, and the large mesh
|
|
||||||
// bed's name test was the first to read it back. Either key gives the same files.
|
|
||||||
func TestNamesKeyedByInternalNameAreNotSuffixedTwice(t *testing.T) {
|
|
||||||
internal := map[string]string{"homer.internal": "10.42.0.1", "marge.internal": "10.42.0.2"}
|
|
||||||
bare := map[string]string{"homer": "10.42.0.1", "marge": "10.42.0.2"}
|
|
||||||
if a, b := nodeZones(Resolution{Node: "homer"}, internal, ""), nodeZones(Resolution{Node: "homer"}, bare, ""); a != b {
|
|
||||||
t.Fatalf("the zones differ by how the names were keyed:\n%s\n---\n%s", a, b)
|
|
||||||
}
|
|
||||||
if a, b := nodeNames(Resolution{Node: "homer"}, internal, ""), nodeNames(Resolution{Node: "homer"}, bare, ""); a != b {
|
|
||||||
t.Fatalf("the hosts differ by how the names were keyed:\n%s\n---\n%s", a, b)
|
|
||||||
}
|
|
||||||
zones := nodeZones(Resolution{Node: "homer"}, internal, "")
|
|
||||||
if strings.Contains(zones, "internal.internal") || !strings.Contains(zones, "address=/homer.internal/10.42.0.1") {
|
|
||||||
t.Fatalf("the zones carry a doubled suffix or miss the name:\n%s", zones)
|
|
||||||
}
|
|
||||||
hosts := nodeNames(Resolution{Node: "homer"}, internal, "")
|
|
||||||
if !strings.Contains(hosts, "10.42.0.1\thomer.internal\thomer\t# this machine") {
|
|
||||||
t.Fatalf("the hosts line for the machine itself is not name, bare name and the mark:\n%s", hosts)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The suffix the control plane composed the names with is the one the facts write — an operator
|
|
||||||
// who chose another does not get `.internal` appended to it.
|
|
||||||
func TestTheFactsWriteTheSuffixTheNamesWereComposedWith(t *testing.T) {
|
|
||||||
names := map[string]string{"homer.lan": "10.42.0.1"}
|
|
||||||
zones := nodeZones(Resolution{Node: "homer"}, names, "lan")
|
|
||||||
if !strings.Contains(zones, "address=/homer.lan/10.42.0.1") || strings.Contains(zones, "internal") {
|
|
||||||
t.Fatalf("the zones do not carry the operator's suffix as given:\n%s", zones)
|
|
||||||
}
|
|
||||||
if !strings.Contains(zones, "local=/lan/") {
|
|
||||||
t.Fatalf("the local domain is not the operator's suffix, so its names would leak upstream:\n%s", zones)
|
|
||||||
}
|
|
||||||
hosts := nodeNames(Resolution{Node: "homer"}, names, "lan")
|
|
||||||
if !strings.Contains(hosts, "10.42.0.1\thomer.lan\thomer\t# this machine") {
|
|
||||||
t.Fatalf("the hosts line does not carry the operator's suffix as given:\n%s", hosts)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// novox/hq 04-ISSUES/111: the map the control plane hands a resolution holds every name the mesh
|
|
||||||
// serves — the machines, and the names it was told to route to whichever machine serves them. A
|
|
||||||
// container's hosts wants all of it. A resolver's zones want only the machines: told the mesh's
|
|
||||||
// suffix is its own, it answers authoritatively for everything under it and forwards nothing, so a
|
|
||||||
// routed name written there with the suffix appended is a name nobody will ever ask for, standing
|
|
||||||
// beside the machines and looking as real.
|
|
||||||
func TestTheResolverIsToldTheMachinesAndNotTheNamesTheMeshMerelyServes(t *testing.T) {
|
|
||||||
machines := map[string]string{"homer.internal": "10.42.0.1", "marge.internal": "10.42.0.2"}
|
|
||||||
every := map[string]string{
|
|
||||||
"homer.internal": "10.42.0.1", "marge.internal": "10.42.0.2",
|
|
||||||
"drive.example.test": "10.42.0.1", "git.example.test": "10.42.0.2",
|
|
||||||
}
|
|
||||||
m := Manifest{Module: "resolver", Facts: map[string]string{
|
|
||||||
FactNodeZones: "/etc/zones.conf", FactNodeNames: "/etc/hosts",
|
|
||||||
}}
|
|
||||||
given, err := FactsInto(m, Resolution{Node: "homer"}, every, machines, "")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
by := map[string]string{}
|
|
||||||
for _, f := range given {
|
|
||||||
by[f["path"].(string)] = f["content"].(string)
|
|
||||||
}
|
|
||||||
|
|
||||||
zones := by["/etc/zones.conf"]
|
|
||||||
for _, machine := range []string{"address=/homer.internal/10.42.0.1", "address=/marge.internal/10.42.0.2"} {
|
|
||||||
if !strings.Contains(zones, machine) {
|
|
||||||
t.Fatalf("the resolver was not told %q:\n%s", machine, zones)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, served := range []string{"drive.example.test", "git.example.test"} {
|
|
||||||
if strings.Contains(zones, served) {
|
|
||||||
t.Fatalf("the resolver was told %q, a name the mesh serves rather than a machine:\n%s", served, zones)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// And the hosts file is the other way about: every name, so a container reaching a routed name
|
|
||||||
// finds the machine serving it.
|
|
||||||
hosts := by["/etc/hosts"]
|
|
||||||
for _, name := range []string{"homer.internal", "drive.example.test", "git.example.test"} {
|
|
||||||
if !strings.Contains(hosts, name) {
|
|
||||||
t.Fatalf("a container would not resolve %q from its hosts:\n%s", name, hosts)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,106 @@
|
|||||||
|
package catalogue_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/overlay"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq issue 128, held where the host will see it: the shipped networking module's names,
|
||||||
|
// through the whole composition, and not only through FactsInto.
|
||||||
|
//
|
||||||
|
// Composition prefixes a fact's id with the module that asked for it and passes everything else
|
||||||
|
// through; a step that dropped `into` on the way would send the region as a whole file, and the
|
||||||
|
// host would write the machine's hosts file over again with every unit test above still green.
|
||||||
|
|
||||||
|
// onTheNetwork stands in for the overlay's generator: the node is part of the private network,
|
||||||
|
// and what the generator writes is not what is under test here.
|
||||||
|
type onTheNetwork struct{}
|
||||||
|
|
||||||
|
func (onTheNetwork) Resources(string) ([]map[string]any, bool, error) {
|
||||||
|
return []map[string]any{{"id": "overlay-config", "type": "file",
|
||||||
|
"path": "/etc/wireguard/mesh0.conf", "mode": "0600", "content": "[Interface]\n"}}, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheHostsRegionArrivesAsTheHostWillReadIt(t *testing.T) {
|
||||||
|
shelf := provided(t)
|
||||||
|
// A resolver restarting on the names another module put on the machine, and one resource it
|
||||||
|
// only runs at start — neither of which composition has any business changing.
|
||||||
|
resolver, err := catalogue.ParseManifest([]byte(`{
|
||||||
|
"module": "resolver", "version": "1", "requires": ["mesh-addressing"],
|
||||||
|
"resources": [
|
||||||
|
{"id": "seed", "type": "file", "path": "/etc/resolver/seed", "mode": "0644",
|
||||||
|
"content": "seed\n", "at": "start"},
|
||||||
|
{"id": "daemon", "type": "service", "unit": "resolver.service", "state": "running",
|
||||||
|
"restart-on": ["seed", "mesh-wireguard.fact-node-names"]}
|
||||||
|
]}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
shelf[resolver.Module] = resolver
|
||||||
|
|
||||||
|
got, err := catalogue.Resolve(shelf, []string{overlay.Domain, "resolver"},
|
||||||
|
catalogue.Node{Name: "homer", At: "homer.internal"}, catalogue.World{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
names := map[string]string{"homer.internal": "10.42.0.1", "marge.internal": "10.42.0.2"}
|
||||||
|
out, err := got.Declaration(catalogue.Rendering{
|
||||||
|
Names: names, Machines: names, Suffix: "internal",
|
||||||
|
Generators: map[string]catalogue.Generator{overlay.Name: onTheNetwork{}},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ids := map[string]map[string]any{}
|
||||||
|
for _, r := range out {
|
||||||
|
ids[r["id"].(string)] = r
|
||||||
|
}
|
||||||
|
|
||||||
|
hosts := ids[overlay.Name+".fact-node-names"]
|
||||||
|
if hosts == nil {
|
||||||
|
t.Fatalf("no names reached the machine; the declaration has %v", keys(ids))
|
||||||
|
}
|
||||||
|
if hosts["path"] != "/etc/hosts" || hosts["into"] != "block" {
|
||||||
|
t.Fatalf("the hosts file is not written into as a region: %v", hosts)
|
||||||
|
}
|
||||||
|
content := hosts["content"].(string)
|
||||||
|
if !strings.Contains(content, "10.42.0.1\thomer.internal\thomer\t# this machine\n") {
|
||||||
|
t.Errorf("the region does not name the machine:\n%s", content)
|
||||||
|
}
|
||||||
|
for _, floor := range []string{"Generated by the mesh", "localhost", "127.0.1.1"} {
|
||||||
|
if strings.Contains(content, floor) {
|
||||||
|
t.Errorf("the region carries %q, which is the machine's:\n%s", floor, content)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The resolver's reference to it still names a resource the host will be sent.
|
||||||
|
daemon := ids["resolver.daemon"]
|
||||||
|
if daemon == nil {
|
||||||
|
t.Fatalf("the resolver's service was not composed: %v", keys(ids))
|
||||||
|
}
|
||||||
|
for _, named := range daemon["restart-on"].([]any) {
|
||||||
|
if ids[named.(string)] == nil {
|
||||||
|
t.Errorf("the resolver restarts on %v, which is nothing the host is sent", named)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(daemon["restart-on"], []any{"resolver.seed", overlay.Name + ".fact-node-names"}) {
|
||||||
|
t.Errorf("restart-on is %v", daemon["restart-on"])
|
||||||
|
}
|
||||||
|
|
||||||
|
// And a resource's own `at` passes through as the manifest wrote it.
|
||||||
|
if seed := ids["resolver.seed"]; seed == nil || seed["at"] != "start" {
|
||||||
|
t.Errorf("a resource's at did not survive composition: %v", seed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func keys(m map[string]map[string]any) []string {
|
||||||
|
out := make([]string, 0, len(m))
|
||||||
|
for k := range m {
|
||||||
|
out = append(out, k)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
package catalogue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A node's fail2ban jails, composed from the modules it runs (novox/hq to-be 31).
|
||||||
|
//
|
||||||
|
// **The same shape as the firewall.** Every module's `listens` become the node's rule set; every
|
||||||
|
// module's `jails` become the node's fail2ban config. A module that runs an authenticating service
|
||||||
|
// declares what a break-in on it looks like and how to ban it, naming no node and no path (ADR
|
||||||
|
// 0112); the intrusion-prevention holder — the one module with `jailing` — gathers them and writes
|
||||||
|
// them where it owns. A node not running a module has none of its jails.
|
||||||
|
|
||||||
|
// jailsInto composes every jail declared by the modules on a node into the files the holder writes:
|
||||||
|
// one jail file (all stanzas, so the fail2ban service restarts on a single resource) and one filter
|
||||||
|
// file per jail (its failregex, which fail2ban references by the jail's name).
|
||||||
|
//
|
||||||
|
// Owned by the holder, because the directory is: two modules writing into one fail2ban is the
|
||||||
|
// collision the holder model exists to prevent. Empty when nothing declares a jail — then the file
|
||||||
|
// is written empty rather than absent, so removing the last jail is an ordinary change the service
|
||||||
|
// restarts on rather than a file that vanishes.
|
||||||
|
func jailsInto(modules []Manifest, j *Jailing) []map[string]any {
|
||||||
|
type declared struct {
|
||||||
|
module string
|
||||||
|
jail Jail
|
||||||
|
}
|
||||||
|
var jails []declared
|
||||||
|
for _, m := range modules {
|
||||||
|
for _, jail := range m.Jails {
|
||||||
|
jails = append(jails, declared{m.Module, jail})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A stable order the host applies as given (ADR 0005), and so the same set composes byte for
|
||||||
|
// byte every time rather than differing by map iteration.
|
||||||
|
sort.Slice(jails, func(a, b int) bool { return jails[a].jail.Name < jails[b].jail.Name })
|
||||||
|
|
||||||
|
var composed strings.Builder
|
||||||
|
composed.WriteString("# The mesh's jails, composed from the modules this node runs. Do not edit —\n")
|
||||||
|
composed.WriteString("# replaced whenever the node's modules change (novox/hq to-be 31).\n")
|
||||||
|
|
||||||
|
out := make([]map[string]any, 0, len(jails)+1)
|
||||||
|
for _, d := range jails {
|
||||||
|
fmt.Fprintf(&composed, "\n# from %s\n[%s]\nenabled = true\nfilter = %s\n%s\n",
|
||||||
|
d.module, d.jail.Name, d.jail.Name, strings.TrimRight(d.jail.Jail, "\n"))
|
||||||
|
// The filter is a file of its own, named as the jail's filter= references it.
|
||||||
|
out = append(out, map[string]any{
|
||||||
|
"id": "filter-" + d.jail.Name,
|
||||||
|
"type": "file", "path": strings.TrimRight(j.FilterInto, "/") + "/" + d.jail.Name + ".conf",
|
||||||
|
"mode": "0644",
|
||||||
|
"content": "# Generated by the mesh (from module " + d.module + "). Do not edit.\n" +
|
||||||
|
"[Definition]\nfailregex = " + d.jail.Failregex + "\n",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
// The one jail file, first, with the fixed id the fail2ban service names in its restart-on.
|
||||||
|
return append([]map[string]any{{
|
||||||
|
"id": ComposedJailsID(), "type": "file", "path": j.Into, "mode": "0644",
|
||||||
|
"content": composed.String(),
|
||||||
|
}}, out...)
|
||||||
|
}
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
package catalogue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A node's fail2ban jails are composed from the modules it runs (novox/hq to-be 31): the holder
|
||||||
|
// (jailing) gathers every module's declared jail into one jail file and a filter file per jail.
|
||||||
|
func TestJailsAreComposedFromTheNodesModules(t *testing.T) {
|
||||||
|
modules := []Manifest{
|
||||||
|
{Module: "fail2ban", Jailing: &Jailing{Into: "/etc/fail2ban/jail.d/mesh-composed.conf", FilterInto: "/etc/fail2ban/filter.d"}},
|
||||||
|
{Module: "postgres", Jails: []Jail{{Name: "postgres-auth", Failregex: "auth failed from <HOST>", Jail: "port = 5432\nmaxretry = 5"}}},
|
||||||
|
}
|
||||||
|
files := jailsInto(modules, modules[0].Jailing)
|
||||||
|
|
||||||
|
by := map[string]map[string]any{}
|
||||||
|
for _, f := range files {
|
||||||
|
by[f["id"].(string)] = f
|
||||||
|
}
|
||||||
|
jail := by[ComposedJailsID()]
|
||||||
|
if jail == nil || jail["path"] != "/etc/fail2ban/jail.d/mesh-composed.conf" {
|
||||||
|
t.Fatalf("the composed jail file was not written: %v", jail)
|
||||||
|
}
|
||||||
|
body := jail["content"].(string)
|
||||||
|
if !strings.Contains(body, "[postgres-auth]") || !strings.Contains(body, "filter = postgres-auth") ||
|
||||||
|
!strings.Contains(body, "port = 5432") {
|
||||||
|
t.Fatalf("the postgres jail stanza was not composed in:\n%s", body)
|
||||||
|
}
|
||||||
|
filter := by["filter-postgres-auth"]
|
||||||
|
if filter == nil || filter["path"] != "/etc/fail2ban/filter.d/postgres-auth.conf" {
|
||||||
|
t.Fatalf("the jail's filter file was not written: %v", filter)
|
||||||
|
}
|
||||||
|
if !strings.Contains(filter["content"].(string), "failregex = auth failed from <HOST>") {
|
||||||
|
t.Fatalf("the failregex was not written: %v", filter["content"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A holder whose node runs no jail-declaring module still gets the file, empty — so removing the
|
||||||
|
// last jail is a change the service restarts on, not a file that vanishes.
|
||||||
|
func TestTheComposedJailFileIsWrittenEvenWhenEmpty(t *testing.T) {
|
||||||
|
files := jailsInto([]Manifest{{Module: "fail2ban"}}, &Jailing{Into: "/x", FilterInto: "/f"})
|
||||||
|
if len(files) != 1 || files[0]["id"] != ComposedJailsID() {
|
||||||
|
t.Fatalf("the empty composed jail file was not written alone: %v", files)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -57,7 +57,7 @@ func machineUsed(content string) []string {
|
|||||||
// the hosts file and the resolver's wildcards are written from, so a file naming the machine's
|
// the hosts file and the resolver's wildcards are written from, so a file naming the machine's
|
||||||
// address and the file every other machine reaches it by cannot disagree. Absent, like `at`, when
|
// address and the file every other machine reaches it by cannot disagree. Absent, like `at`, when
|
||||||
// the machine is off the network or the mesh has not placed it.
|
// the machine is off the network or the mesh has not placed it.
|
||||||
func machineFacts(r Resolution, names map[string]string) map[string]string {
|
func machineFacts(r Resolution, names map[string]string, meshRange string) map[string]string {
|
||||||
out := map[string]string{"name": r.Node}
|
out := map[string]string{"name": r.Node}
|
||||||
if r.At != "" {
|
if r.At != "" {
|
||||||
out["at"] = r.At
|
out["at"] = r.At
|
||||||
@@ -65,32 +65,59 @@ func machineFacts(r Resolution, names map[string]string) map[string]string {
|
|||||||
out["address"] = address
|
out["address"] = address
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// The private network's whole range — a mesh-wide fact, not this machine's, but named here
|
||||||
|
// because a module cannot know it and sometimes must (an intrusion filter that must never ban a
|
||||||
|
// tunnel peer). Absent when the mesh has no range to give.
|
||||||
|
if meshRange != "" {
|
||||||
|
out["mesh-range"] = meshRange
|
||||||
|
}
|
||||||
|
// The operator's login on this machine and where its home is (novox/hq to-be 29), so a module
|
||||||
|
// that writes operator config names the account and its home rather than a value it cannot know.
|
||||||
|
// Absent when no operator account is known — a headless box a person never logs into.
|
||||||
|
if r.Account != "" {
|
||||||
|
out["account"] = r.Account
|
||||||
|
out["account-home"] = accountHomeOf(r.Account, r.AccountHome)
|
||||||
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// accountHomeOf is where an account's home is: what was stored, or the derived default — /root for
|
||||||
|
// root, /home/<account> otherwise. The one place the default is written, so a fact and the store
|
||||||
|
// cannot disagree about it.
|
||||||
|
func accountHomeOf(account, home string) string {
|
||||||
|
if home != "" {
|
||||||
|
return home
|
||||||
|
}
|
||||||
|
if account == "root" {
|
||||||
|
return "/root"
|
||||||
|
}
|
||||||
|
return "/home/" + account
|
||||||
|
}
|
||||||
|
|
||||||
// machineInto replaces a file's ${machine:…} placeholders with what the mesh knows about the
|
// machineInto replaces a file's ${machine:…} placeholders with what the mesh knows about the
|
||||||
// machine the module was assigned to.
|
// machine the module was assigned to.
|
||||||
//
|
//
|
||||||
// A key the mesh does not hold is refused, for the same reason a binding's is: left alone, the
|
// A key the mesh does not hold is refused, for the same reason a binding's is: left alone, the
|
||||||
// literal would be written into a configuration file and read as a value.
|
// literal would be written into a configuration file and read as a value.
|
||||||
func machineInto(resource map[string]any, facts map[string]string, module string) error {
|
func machineInto(resource map[string]any, facts map[string]string, module string) error {
|
||||||
if fmt.Sprint(resource["type"]) != "file" {
|
// Content, and now the path and owner too: a module that writes into a person's home names it
|
||||||
return nil
|
// with ${machine:account-home} and ${machine:account}, which it cannot know until assigned
|
||||||
}
|
// (novox/hq to-be 29), the same reason its content names ${machine:address}.
|
||||||
content, ok := resource["content"].(string)
|
for _, field := range []string{"path", "owner", "content"} {
|
||||||
if !ok {
|
s, ok := resource[field].(string)
|
||||||
return nil
|
if !ok {
|
||||||
}
|
continue
|
||||||
for _, key := range machineUsed(content) {
|
}
|
||||||
value, has := facts[key]
|
for _, key := range machineUsed(s) {
|
||||||
if !has {
|
value, has := facts[key]
|
||||||
return fmt.Errorf(
|
if !has {
|
||||||
"%s has a file that says ${machine:%s}, and this machine says %s",
|
return fmt.Errorf(
|
||||||
module, key, orNothing(namesOfFacts(facts)))
|
"%s has a %s that says ${machine:%s}, and this machine says %s",
|
||||||
|
module, field, key, orNothing(namesOfFacts(facts)))
|
||||||
|
}
|
||||||
|
s = strings.ReplaceAll(s, fmt.Sprintf("${machine:%s}", key), value)
|
||||||
|
resource[field] = s
|
||||||
}
|
}
|
||||||
resource["content"] = strings.ReplaceAll(
|
|
||||||
content, fmt.Sprintf("${machine:%s}", key), value)
|
|
||||||
content = resource["content"].(string)
|
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -103,3 +103,26 @@ func TestAModuleNamesTheAddressBehindItsMachinesName(t *testing.T) {
|
|||||||
t.Fatalf("a machine off the network was given an address, or refused for another reason: %v", err)
|
t.Fatalf("a machine off the network was given an address, or refused for another reason: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The mesh's private range is offered as ${machine:mesh-range}, so a module names it rather than
|
||||||
|
// hardcoding a value it cannot know (novox/hq ADR 0112) — the fail2ban ignoreip is the case.
|
||||||
|
func TestAModuleNamesTheMeshRange(t *testing.T) {
|
||||||
|
facts := machineFacts(Resolution{Node: "anchor", At: "anchor.internal"},
|
||||||
|
map[string]string{"anchor.internal": "10.10.0.1"}, "10.10.0.0/24")
|
||||||
|
if facts["mesh-range"] != "10.10.0.0/24" {
|
||||||
|
t.Fatalf("the mesh range is not a machine fact: %v", facts)
|
||||||
|
}
|
||||||
|
res := map[string]any{"type": "file", "id": "jail", "content": "ignoreip = 127.0.0.1/8 ${machine:mesh-range}\n"}
|
||||||
|
if err := machineInto(res, facts, "fail2ban"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := res["content"].(string); !strings.Contains(got, "10.10.0.0/24") || strings.Contains(got, "${machine:") {
|
||||||
|
t.Fatalf("the mesh range was not written in: %q", got)
|
||||||
|
}
|
||||||
|
// A mesh with no range gives no such fact, and a file that names it is refused rather than
|
||||||
|
// left with a literal placeholder in it.
|
||||||
|
none := machineFacts(Resolution{Node: "anchor"}, nil, "")
|
||||||
|
if _, has := none["mesh-range"]; has {
|
||||||
|
t.Fatal("a mesh with no range still offered one")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+141
-19
@@ -176,15 +176,29 @@ type Manifest struct {
|
|||||||
// Requires are names that must be provided by something assigned to the same node.
|
// Requires are names that must be provided by something assigned to the same node.
|
||||||
Requires []string `json:"requires,omitempty"`
|
Requires []string `json:"requires,omitempty"`
|
||||||
|
|
||||||
// Emits are the event types this module publishes onto the broker — dotted topic keys, e.g.
|
// Emits are the events this module publishes, named **locally**: `order.placed`, not a subject
|
||||||
// "module.umami.site.created". Declared so the mesh knows the event graph; events are
|
// and not a routing key. The mesh derives where it lands (design 29 §1), so reorganising the
|
||||||
// provisioning's lighter sibling — 1:many and broadcast, no credential (novox/hq ADR 0041).
|
// subject space leaves this manifest correct. Events are provisioning's lighter sibling — 1:many
|
||||||
|
// and broadcast, no credential (novox/hq ADR 0041).
|
||||||
|
//
|
||||||
|
// A module publishes under its own name only. If the event is about a *role* rather than about
|
||||||
|
// this module, it belongs on that seat, where the name outlives whoever holds it.
|
||||||
|
//
|
||||||
|
// **This said "dotted topic keys, e.g. module.umami.site.created" until 04-ISSUES/127**, which
|
||||||
|
// is the old bus's routing key, and is why every manifest in the catalogue had the same mistake:
|
||||||
|
// nobody was guessing, everybody followed this comment.
|
||||||
Emits []string `json:"emits,omitempty"`
|
Emits []string `json:"emits,omitempty"`
|
||||||
|
|
||||||
// Consumes are the event patterns this module subscribes to — topic patterns over module,
|
// Consumes are the events this module reacts to, each naming its emitter and the event:
|
||||||
// mesh and node events alike, e.g. "node.*.joined" or "#" (the audit logger). The runtime
|
// `billing.order.placed`. `*` stands for one name and `**` for the rest, so `*.download.completed`
|
||||||
// wires the subscription; the module ships the handler. A Consumes for an event nothing on
|
// is that event from any module and `**` is every event in the mesh.
|
||||||
// the mesh Emits is a dangling edge.
|
//
|
||||||
|
// Spelled the mesh's way rather than the wire's, for the reason Emits is: the bus the mesh runs
|
||||||
|
// on today spells these `*` and `#`, the one being built spells them `*` and `>`, and a manifest
|
||||||
|
// naming either would stop being true when the wire changed.
|
||||||
|
//
|
||||||
|
// The runtime wires the subscription; the module ships the handler. A Consumes for an event
|
||||||
|
// nothing on the mesh Emits is a dangling edge.
|
||||||
Consumes []string `json:"consumes,omitempty"`
|
Consumes []string `json:"consumes,omitempty"`
|
||||||
|
|
||||||
// Claims are singular resources. Two modules claiming one thing within a scope cannot both
|
// Claims are singular resources. Two modules claiming one thing within a scope cannot both
|
||||||
@@ -192,6 +206,34 @@ type Manifest struct {
|
|||||||
// that every new module would force its predecessors to update.
|
// that every new module would force its predecessors to update.
|
||||||
Claims []Claim `json:"claims,omitempty"`
|
Claims []Claim `json:"claims,omitempty"`
|
||||||
|
|
||||||
|
// DefinesSeats are the seats this module defines for itself, with their protocols
|
||||||
|
// (novox/hq ADR 0121, ADR 0129). The control plane defines the system seats — `mesh-*` and
|
||||||
|
// `node-*` — and a module may define its own, named outside that namespace, to coordinate its
|
||||||
|
// own instances: the mesh enforces one-holder-per-scope for it without knowing what it means. A
|
||||||
|
// module's declared seat is the only non-system name it may then claim; a claim to a name
|
||||||
|
// neither the mesh nor the module defines is refused.
|
||||||
|
//
|
||||||
|
// **Two lines of work built this at once**, one calling it `Seats` with a protocol and one
|
||||||
|
// `DefinesSeats` without. Same key in the file, so no manifest is affected: this is the trunk's
|
||||||
|
// name with the richer type, because what a role accepts, emits and serves is what lets the mesh
|
||||||
|
// check that a holder answers what its seat promises.
|
||||||
|
DefinesSeats []SeatDeclaration `json:"seats,omitempty"`
|
||||||
|
|
||||||
|
// Uses are seats this module sends to. It names the *seat*, never the module holding it, so
|
||||||
|
// the implementation can be replaced under it and no caller changes. A caller gets publish
|
||||||
|
// on that seat's inbound subjects and nothing else — not its outbound events, and not a
|
||||||
|
// subscription to the queue it writes to (design 29 §2).
|
||||||
|
Uses []string `json:"uses,omitempty"`
|
||||||
|
|
||||||
|
// Tools are the tools this module answers — request and reply, awaited.
|
||||||
|
//
|
||||||
|
// **New, and not `serves`**, which this manifest already uses for the facts a consumer needs
|
||||||
|
// in order to reach a provision. Two meanings under one key would be a footgun in the one
|
||||||
|
// file a module author reads most. Until now a module's tools were known only at runtime,
|
||||||
|
// from MESH_TOOL_MODULES in its image; declaring them is what lets the mesh check that a
|
||||||
|
// module claiming a seat answers what that seat's protocol promises (novox/hq ADR 0118).
|
||||||
|
Tools []string `json:"tools,omitempty"`
|
||||||
|
|
||||||
// Capabilities the machine must have. A different field from Requires because the remedy
|
// Capabilities the machine must have. A different field from Requires because the remedy
|
||||||
// differs: a missing module can be assigned, and a missing capability means the wrong
|
// differs: a missing module can be assigned, and a missing capability means the wrong
|
||||||
// machine.
|
// machine.
|
||||||
@@ -358,6 +400,14 @@ type Manifest struct {
|
|||||||
// that could only see its own ports would write a rule set that closed everything else.
|
// that could only see its own ports would write a rule set that closed everything else.
|
||||||
Filtering *Filtering `json:"filtering,omitempty"`
|
Filtering *Filtering `json:"filtering,omitempty"`
|
||||||
|
|
||||||
|
// Jails are the fail2ban jails this module declares for its own service (novox/hq to-be 31).
|
||||||
|
// Written into whichever node runs the module, the same way `listens` become that node's rules.
|
||||||
|
Jails []Jail `json:"jails,omitempty"`
|
||||||
|
|
||||||
|
// Jailing marks the module that composes the node's fail2ban jails — the intrusion-prevention
|
||||||
|
// holder. Like Filtering: one module per node gathers what every module declared and writes it.
|
||||||
|
Jailing *Jailing `json:"jailing,omitempty"`
|
||||||
|
|
||||||
// Guards are ports of this module's the mesh refuses on an adopted node except from the
|
// Guards are ports of this module's the mesh refuses on an adopted node except from the
|
||||||
// private network and from the machine itself (novox/hq ADR 0100) — the store's port and the
|
// private network and from the machine itself (novox/hq ADR 0100) — the store's port and the
|
||||||
// broker's management port. The ports the software uses; the mesh guards where the machine
|
// broker's management port. The ports the software uses; the mesh guards where the machine
|
||||||
@@ -366,24 +416,29 @@ type Manifest struct {
|
|||||||
// firewall does. Ignored on a converged node, whose derived filter already closes them.
|
// firewall does. Ignored on a converged node, whose derived filter already closes them.
|
||||||
Guards []int `json:"guards,omitempty"`
|
Guards []int `json:"guards,omitempty"`
|
||||||
|
|
||||||
// Facts are things only the mesh knows, written where this module asks for them.
|
// Facts are things only the mesh knows, written where this module asks for them — in the
|
||||||
|
// module's own format.
|
||||||
//
|
//
|
||||||
// **The graph is the control plane's; how a machine uses it is the module's.** The mesh knows
|
// **The graph is the control plane's; the format is the module's.** The mesh knows which
|
||||||
// which machines exist, what they are called and where they are. Making a name resolve, or a
|
// machines exist, what they are called and where they are. Turning that into a name that
|
||||||
// peer reachable, is somebody's software — dnsmasq, a resolver, a VPN — and the mesh has no
|
// resolves, a peer that is reachable, a host a client trusts, is somebody's software — dnsmasq,
|
||||||
// business shipping one, choosing which, or knowing its configuration language.
|
// a resolver, a VPN, ssh — in its own configuration language, and the mesh has no business
|
||||||
|
// knowing it. So a module gives a path and a template; the mesh renders the roster through it
|
||||||
|
// and owns nothing of what the file says.
|
||||||
//
|
//
|
||||||
// So a module says *put the node names here* and owns everything after that. The same shape as
|
// This used to be a closed list of fact names, each formatted in Go in the control plane, so a
|
||||||
// `filtering`, generalised: a fact, and a path.
|
// new consumer meant a new formatter here in the consumer's language. Now the data is the mesh's
|
||||||
|
// and the format is the module's: the two built-in cases — the network module's `/etc/hosts` and
|
||||||
|
// dnsmasq's zones — render through the same template path any module uses, and no format lives
|
||||||
|
// in the control plane at all. See RosterFile for what a template sees.
|
||||||
//
|
//
|
||||||
// It replaces three modules that existed only because computed output needed somewhere to
|
// It replaces three modules that existed only because computed output needed somewhere to
|
||||||
// live — they ran no software, could not be swapped for anything, and appeared in the graph as
|
// live — they ran no software, could not be swapped for anything, and appeared in the graph as
|
||||||
// modules while being a data channel wearing a costume.
|
// modules while being a data channel wearing a costume.
|
||||||
//
|
//
|
||||||
// Keyed by fact name; the names are a closed list, because a module asking for one the mesh
|
// Keyed by a name the module chooses, which is the rendered file's id (`fact-<name>`) — what a
|
||||||
// does not compute is asking for something nobody will write, and finding that out on a machine
|
// `restart-on` names to restart when the roster changes.
|
||||||
// is worse than being told here.
|
Facts map[string]RosterFile `json:"facts,omitempty"`
|
||||||
Facts map[string]string `json:"facts,omitempty"`
|
|
||||||
|
|
||||||
// Certificate is where this module wants a certificate for its machine's name inside the
|
// Certificate is where this module wants a certificate for its machine's name inside the
|
||||||
// mesh, and where the key that goes with it can be found.
|
// mesh, and where the key that goes with it can be found.
|
||||||
@@ -403,6 +458,20 @@ type Manifest struct {
|
|||||||
// A directory rather than one document for the same reason as above: each value is sealed
|
// A directory rather than one document for the same reason as above: each value is sealed
|
||||||
// separately and the mesh cannot open any of them to build a list.
|
// separately and the mesh cannot open any of them to build a list.
|
||||||
Grants map[string]string `json:"grants,omitempty"`
|
Grants map[string]string `json:"grants,omitempty"`
|
||||||
|
|
||||||
|
// BusUsers is where this module wants the mesh's user list written, and it is only ever
|
||||||
|
// answered for the module holding `mesh-broker`.
|
||||||
|
//
|
||||||
|
// **The mesh writes who may connect; the module owns everything else about its server**
|
||||||
|
// (novox/hq design 25 §4, task 1.7). Ports, TLS paths and a store directory live in this
|
||||||
|
// module's image and its mounts and change when it does, so the module's own configuration
|
||||||
|
// carries them and includes this file. A controller that wrote the whole configuration would
|
||||||
|
// have to be kept in step with a Dockerfile it never sees.
|
||||||
|
//
|
||||||
|
// **Asking for it is not enough to receive it.** This file holds every user's password hash, so
|
||||||
|
// a module that could ask for it could read every credential on the bus — and the claim on
|
||||||
|
// `mesh-broker` is what authorises it, checked from this manifest alone.
|
||||||
|
BusUsers string `json:"bus-users,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Build says how to produce this module's artifacts from its source.
|
// Build says how to produce this module's artifacts from its source.
|
||||||
@@ -602,6 +671,36 @@ func (l Listening) At() string {
|
|||||||
return l.Protocol
|
return l.Protocol
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Jail is a fail2ban jail a module declares for its own service (novox/hq to-be 31).
|
||||||
|
//
|
||||||
|
// **The module names no node and no path** (ADR 0112): it says what a break-in on its service looks
|
||||||
|
// like — the failregex — and the jail's own keys (the port it watches, where it logs, how many
|
||||||
|
// tries, how long to ban). The mesh writes it into whichever node's fail2ban runs the module, the
|
||||||
|
// same way a module's `listens` become that node's firewall rules. A node not running the module
|
||||||
|
// has no such jail.
|
||||||
|
type Jail struct {
|
||||||
|
// Name is the jail and its filter, e.g. "postgres-auth". One holder of the name per node.
|
||||||
|
Name string `json:"name"`
|
||||||
|
// Failregex is what a failed authentication looks like in the service's log — the filter.
|
||||||
|
Failregex string `json:"failregex"`
|
||||||
|
// Jail is the body of the jail's stanza: the keys under [<name>] the module knows and the mesh
|
||||||
|
// does not — the port it watches, its logpath and backend, maxretry, bantime.
|
||||||
|
Jail string `json:"jail"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Jailing says a module composes the node's fail2ban jails — the intrusion-prevention holder. Like
|
||||||
|
// Filtering for the firewall: one module gathers what every other module declared and writes it
|
||||||
|
// where it owns. Into is the one jail file the stanzas are composed into (so the fail2ban service
|
||||||
|
// can restart on a single resource); FilterInto is the directory each jail's filter file goes in.
|
||||||
|
type Jailing struct {
|
||||||
|
Into string `json:"into"`
|
||||||
|
FilterInto string `json:"filter-into"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ComposedJailsID is the single jail file the mesh composes every declared jail into, so the
|
||||||
|
// fail2ban service names one resource in its restart-on and a jail added or removed reaches it.
|
||||||
|
func ComposedJailsID() string { return "composed-jails" }
|
||||||
|
|
||||||
// Filtering says where a module wants the computed rule set.
|
// Filtering says where a module wants the computed rule set.
|
||||||
type Filtering struct {
|
type Filtering struct {
|
||||||
// Into is the path to write it to. Whatever loads it is this module's own business — an
|
// Into is the path to write it to. Whatever loads it is this module's own business — an
|
||||||
@@ -621,7 +720,22 @@ type Certificate struct {
|
|||||||
|
|
||||||
// CertificateID and AuthorityID are the resource identities of what the mesh issued.
|
// CertificateID and AuthorityID are the resource identities of what the mesh issued.
|
||||||
func CertificateID() string { return "certificate" }
|
func CertificateID() string { return "certificate" }
|
||||||
func AuthorityID() string { return "certificate-authority" }
|
|
||||||
|
// ClaimsSeat says whether this manifest claims one named seat.
|
||||||
|
func (m Manifest) ClaimsSeat(seat string) bool {
|
||||||
|
for _, c := range m.Claims {
|
||||||
|
if c.Name == seat {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// BusUsersID names the mesh's composed user list, so it is the same resource across every
|
||||||
|
// declaration and a change to it is an update rather than a second file beside the old one — which
|
||||||
|
// on a bus reading a directory would be two account lists, and the server would take both.
|
||||||
|
func BusUsersID() string { return "bus-users" }
|
||||||
|
func AuthorityID() string { return "certificate-authority" }
|
||||||
|
|
||||||
// FilteringID names the computed rule set, so it is the same resource across every declaration
|
// FilteringID names the computed rule set, so it is the same resource across every declaration
|
||||||
// and a change to it is an update rather than an addition beside the old one.
|
// and a change to it is an update rather than an addition beside the old one.
|
||||||
@@ -950,6 +1064,10 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
|||||||
problems = append(problems, fmt.Sprintf("%s requires itself", m.Module))
|
problems = append(problems, fmt.Sprintf("%s requires itself", m.Module))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// What it may call an event, and what it may ask to hear (events.go). Checked here because a
|
||||||
|
// module whose event names are wrong installs, starts, connects and reacts to nothing, with
|
||||||
|
// every log line saying it is fine (novox/hq 04-ISSUES/127).
|
||||||
|
problems = append(problems, EventProblems(m)...)
|
||||||
wellFormed := true
|
wellFormed := true
|
||||||
for _, c := range m.Claims {
|
for _, c := range m.Claims {
|
||||||
if !name.MatchString(c.Name) {
|
if !name.MatchString(c.Name) {
|
||||||
@@ -970,6 +1088,10 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
|||||||
if wellFormed {
|
if wellFormed {
|
||||||
problems = append(problems, claimProblems(m)...)
|
problems = append(problems, claimProblems(m)...)
|
||||||
}
|
}
|
||||||
|
// What one manifest can be judged on: a declaration's shape, its scope, and the reserved
|
||||||
|
// prefix. Whether a seat anybody names exists, and whether a holder answers for it, are
|
||||||
|
// facts about the catalogue and are checked at registration (CatalogueProblems).
|
||||||
|
problems = append(problems, declaredSeatProblems(m)...)
|
||||||
if m.Computed != "" && len(m.Resources) > 0 {
|
if m.Computed != "" && len(m.Resources) > 0 {
|
||||||
// One or the other. A module that both ships files and has them computed would leave
|
// One or the other. A module that both ships files and has them computed would leave
|
||||||
// nobody able to say where a given file came from.
|
// nobody able to say where a given file came from.
|
||||||
|
|||||||
@@ -0,0 +1,121 @@
|
|||||||
|
package catalogue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **A manifest holds no subject** (novox/hq design 29 §1).
|
||||||
|
//
|
||||||
|
// A module names its events, tools and seats locally, and the mesh derives where they land. The
|
||||||
|
// property that buys: reorganise the subject space and every manifest in the catalogue is still
|
||||||
|
// correct. It holds today by construction — nothing reads a subject from a manifest — and a rule
|
||||||
|
// held by construction is one a later field breaks quietly, with the symptom appearing as a
|
||||||
|
// permission that does not match a subject rather than as a manifest that was wrong.
|
||||||
|
func TestNoManifestContainsASubject(t *testing.T) {
|
||||||
|
root := filepath.Join("..", "..", "..", "mesh-catalog", "modules")
|
||||||
|
entries, err := os.ReadDir(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Skipf("catalogue sibling not present: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Anything in the mesh's own subject space, and anything shaped like a wire address.
|
||||||
|
subject := regexp.MustCompile(`^(mesh|\$JS)\.[a-zA-Z0-9_*>.-]+$`)
|
||||||
|
|
||||||
|
var found []string
|
||||||
|
var walk func(module string, path string, v any)
|
||||||
|
walk = func(module, path string, v any) {
|
||||||
|
switch t := v.(type) {
|
||||||
|
case string:
|
||||||
|
if subject.MatchString(t) {
|
||||||
|
found = append(found, module+" "+path+" = "+t)
|
||||||
|
}
|
||||||
|
case map[string]any:
|
||||||
|
for k, inner := range t {
|
||||||
|
walk(module, path+"."+k, inner)
|
||||||
|
}
|
||||||
|
case []any:
|
||||||
|
for _, inner := range t {
|
||||||
|
walk(module, path+"[]", inner)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
checked := 0
|
||||||
|
for _, e := range entries {
|
||||||
|
if !e.IsDir() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(filepath.Join(root, e.Name(), "module.json"))
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var m any
|
||||||
|
if err := json.Unmarshal(raw, &m); err != nil {
|
||||||
|
t.Errorf("%s: %v", e.Name(), err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
checked++
|
||||||
|
walk(e.Name(), "", m)
|
||||||
|
}
|
||||||
|
if checked == 0 {
|
||||||
|
t.Skip("no manifests read")
|
||||||
|
}
|
||||||
|
if len(found) > 0 {
|
||||||
|
t.Errorf("a manifest names a subject, so reorganising the subject space would mean "+
|
||||||
|
"editing the catalogue:\n %s", strings.Join(found, "\n "))
|
||||||
|
}
|
||||||
|
t.Logf("%d manifests hold no subject", checked)
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Every module's event names are what design 29 says, across the whole catalogue.**
|
||||||
|
//
|
||||||
|
// The rule above holds by construction and turned out to be weaker than it reads: a manifest holds
|
||||||
|
// no subject, and every manifest in the catalogue still held the old bus's routing key, which
|
||||||
|
// derives into a namespace nobody owns (novox/hq 04-ISSUES/127). Nothing failed — the services
|
||||||
|
// started and none of them reacted. This is the check that was missing.
|
||||||
|
func TestEveryManifestsEventNamesAreLocal(t *testing.T) {
|
||||||
|
manifests := theCatalogue(t)
|
||||||
|
|
||||||
|
var problems []string
|
||||||
|
for _, m := range manifests {
|
||||||
|
problems = append(problems, EventProblems(m)...)
|
||||||
|
}
|
||||||
|
if len(problems) > 0 {
|
||||||
|
t.Fatalf("the catalogue holds %d event name(s) the mesh would derive wrongly:\n %s",
|
||||||
|
len(problems), strings.Join(problems, "\n "))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// theCatalogue is every manifest beside this checkout, parsed the way registration parses one.
|
||||||
|
func theCatalogue(t *testing.T) []Manifest {
|
||||||
|
t.Helper()
|
||||||
|
root := filepath.Join("..", "..", "..", "mesh-catalog", "modules")
|
||||||
|
entries, err := os.ReadDir(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Skipf("catalogue sibling not present: %v", err)
|
||||||
|
}
|
||||||
|
var out []Manifest
|
||||||
|
for _, e := range entries {
|
||||||
|
if !e.IsDir() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(filepath.Join(root, e.Name(), "module.json"))
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var m Manifest
|
||||||
|
if err := json.Unmarshal(raw, &m); err != nil {
|
||||||
|
t.Fatalf("%s: %v", e.Name(), err)
|
||||||
|
}
|
||||||
|
out = append(out, m)
|
||||||
|
}
|
||||||
|
if len(out) == 0 {
|
||||||
|
t.Skip("no manifests found beside this checkout")
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -54,7 +54,7 @@ func TestTheShippedNetworkingModulesResolveOnTheirOwn(t *testing.T) {
|
|||||||
// network is what gives a machine a name, so the provider asks for the node-names fact and
|
// network is what gives a machine a name, so the provider asks for the node-names fact and
|
||||||
// there is nothing else to bring in. A module that ran nothing used to be here.
|
// there is nothing else to bring in. A module that ran nothing used to be here.
|
||||||
for _, m := range got.Modules {
|
for _, m := range got.Modules {
|
||||||
if m.Module == overlay.Name && m.Facts["node-names"] == "" {
|
if m.Module == overlay.Name && m.Facts["node-names"].Path == "" {
|
||||||
t.Fatalf("the network's provider does not ask for the names: %+v", m.Facts)
|
t.Fatalf("the network's provider does not ask for the names: %+v", m.Facts)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -28,6 +28,10 @@ type Node struct {
|
|||||||
// (novox/hq ADR 0066). A route contribution carries only a label — the subdomain — and the mesh
|
// (novox/hq ADR 0066). A route contribution carries only a label — the subdomain — and the mesh
|
||||||
// joins <label>.<public-domain> to make the name it grants, interpreting neither half.
|
// joins <label>.<public-domain> to make the name it grants, interpreting neither half.
|
||||||
PublicDomain string
|
PublicDomain string
|
||||||
|
// Account is the operator's login on this machine, AccountHome where its home is (novox/hq
|
||||||
|
// to-be 29). What a home-scoped file is owned by and what ${machine:account} resolves to.
|
||||||
|
Account string
|
||||||
|
AccountHome string
|
||||||
}
|
}
|
||||||
|
|
||||||
// World is what the rest of the mesh already has.
|
// World is what the rest of the mesh already has.
|
||||||
@@ -114,6 +118,11 @@ type Resolution struct {
|
|||||||
// (novox/hq ADR 0066). Carried from the node so that composing <label>.<public-domain> for a
|
// (novox/hq ADR 0066). Carried from the node so that composing <label>.<public-domain> for a
|
||||||
// route contribution needs no store lookup here — the join is a fact about this one machine.
|
// route contribution needs no store lookup here — the join is a fact about this one machine.
|
||||||
PublicDomain string
|
PublicDomain string
|
||||||
|
// Account and AccountHome are the operator's login on this machine and where its home is
|
||||||
|
// (novox/hq to-be 29), carried from the node so a home-scoped file's owner and path resolve
|
||||||
|
// here without a store lookup.
|
||||||
|
Account string
|
||||||
|
AccountHome string
|
||||||
|
|
||||||
// Modules in the order they were resolved: assigned first, then what they pulled in.
|
// Modules in the order they were resolved: assigned first, then what they pulled in.
|
||||||
Modules []Manifest
|
Modules []Manifest
|
||||||
@@ -541,6 +550,7 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
|
|||||||
}
|
}
|
||||||
|
|
||||||
resolution := Resolution{Node: node.Name, At: node.At, PublicDomain: node.PublicDomain,
|
resolution := Resolution{Node: node.Name, At: node.At, PublicDomain: node.PublicDomain,
|
||||||
|
Account: node.Account, AccountHome: node.AccountHome,
|
||||||
Because: because, Needs: needs, Unhostable: unhostable}
|
Because: because, Needs: needs, Unhostable: unhostable}
|
||||||
for _, n := range providersFirst(order, catalogue) {
|
for _, n := range providersFirst(order, catalogue) {
|
||||||
resolution.Modules = append(resolution.Modules, catalogue[n])
|
resolution.Modules = append(resolution.Modules, catalogue[n])
|
||||||
|
|||||||
@@ -50,7 +50,7 @@ func TestTheResolverForwardsToFixedUpstreamsAndNeverReadsResolvConf(t *testing.T
|
|||||||
"\nno-resolv\n", "\nserver=1.1.1.1\n", "\nserver=8.8.8.8\n",
|
"\nno-resolv\n", "\nserver=1.1.1.1\n", "\nserver=8.8.8.8\n",
|
||||||
"\nlisten-address=127.0.0.1\n", "\ninterface=mesh0\n", "\nbind-dynamic\n",
|
"\nlisten-address=127.0.0.1\n", "\ninterface=mesh0\n", "\nbind-dynamic\n",
|
||||||
"\ndomain-needed\n", "\nbogus-priv\n",
|
"\ndomain-needed\n", "\nbogus-priv\n",
|
||||||
"\nconf-file=" + m.Facts[FactNodeZones] + "\n",
|
"\nconf-file=" + m.Facts["node-zones"].Path + "\n",
|
||||||
} {
|
} {
|
||||||
if !strings.Contains(config, want) {
|
if !strings.Contains(config, want) {
|
||||||
t.Errorf("the resolver's configuration lacks %q:\n%s", strings.TrimSpace(want), config)
|
t.Errorf("the resolver's configuration lacks %q:\n%s", strings.TrimSpace(want), config)
|
||||||
@@ -170,7 +170,7 @@ func TestTwoThingsDecidingWhatAMachineAsksAreRefused(t *testing.T) {
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("resolv-conf and resolved-split-dns were both assigned to one machine")
|
t.Fatal("resolv-conf and resolved-split-dns were both assigned to one machine")
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "the-resolver-configuration") {
|
if !strings.Contains(err.Error(), "node-resolver-config") {
|
||||||
t.Fatalf("the refusal does not say what was claimed: %v", err)
|
t.Fatalf("the refusal does not say what was claimed: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,210 @@
|
|||||||
|
package catalogue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"text/template"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What only the mesh knows, written where a module asks for it — in the module's own format.
|
||||||
|
//
|
||||||
|
// **The graph is the control plane's; the format is the module's.** The mesh knows which machines
|
||||||
|
// exist, what they are called and where they are. Turning that into a hosts file, a resolver's
|
||||||
|
// zones, an ssh known_hosts is somebody's configuration language, and the mesh has no business
|
||||||
|
// knowing it. So the mesh hands the roster to a template the module wrote and renders it; it never
|
||||||
|
// learns what the file means.
|
||||||
|
//
|
||||||
|
// This used to be a closed list of fact names, each with its format written in Go here — a hosts
|
||||||
|
// file, a resolver's zones. Every new consumer meant a new formatter in the control plane, in the
|
||||||
|
// consumer's configuration language. Now the data is the mesh's and the format is a template the
|
||||||
|
// module ships: the two built-in cases (the network module's `/etc/hosts`, dnsmasq's zones) render
|
||||||
|
// the same way any module's would, and the control plane holds no format at all.
|
||||||
|
//
|
||||||
|
// It replaced three modules that existed only because computed output needed somewhere to live —
|
||||||
|
// they ran no software, could not be swapped for anything, and appeared in the graph as modules
|
||||||
|
// while being a data channel wearing a costume (novox/hq ADR 0040).
|
||||||
|
|
||||||
|
// A RosterFile is a file the mesh renders from the roster of machines, in the format the module
|
||||||
|
// gives as a Go text/template. The template sees a rosterView: `.Node` (this machine's bare name),
|
||||||
|
// `.Suffix` (what its mesh name ends in), and two sets of `{Name, FQDN, Address}` — `.Names`, every
|
||||||
|
// name the mesh serves, and `.Machines`, only the nodes of the mesh. Which set a template ranges is
|
||||||
|
// how the hq issue 111 distinction is drawn: a container's hosts wants every name; a resolver told
|
||||||
|
// the suffix is its own wants only the machines.
|
||||||
|
type RosterFile struct {
|
||||||
|
// Path is where on the machine the rendered file goes. Absolute, or it is refused here rather
|
||||||
|
// than discovered as a daemon that reads nothing.
|
||||||
|
Path string `json:"path"`
|
||||||
|
// Template is the module's format, a Go text/template over the rosterView. It is the module's,
|
||||||
|
// not the mesh's: the mesh renders it and does not read it.
|
||||||
|
Template string `json:"template"`
|
||||||
|
// Shared is whether the file the fact goes to belongs to the machine rather than the mesh. When
|
||||||
|
// it does, the mesh owns only a marked region of it and keeps the rest byte for byte (novox/hq
|
||||||
|
// issue 128) — a hosts file is shared, since the distribution's `localhost`, the operator's own
|
||||||
|
// lines and other tools' blocks live there too; a resolver's zones file is not, the mesh owns it
|
||||||
|
// whole. A property of the fact, not of the path: the format determines whether the file is
|
||||||
|
// wholly the mesh's, not where a module happened to ask for it.
|
||||||
|
Shared bool `json:"shared,omitempty"`
|
||||||
|
// Home places the file under this node's operator-account home and chowns it to that account,
|
||||||
|
// rather than at an absolute system path (novox/hq to-be 29). Then Path is home-relative
|
||||||
|
// (`.ssh/config.d/mesh`), resolved against the account's home on the node it is composed for; a
|
||||||
|
// node with no operator account gets no such file. This is how the ssh-client config — every
|
||||||
|
// other node's Host block — is written into a person's home rather than into /etc.
|
||||||
|
Home bool `json:"home,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// rosterView is what a RosterFile's template sees. A closed shape — a template referencing a field
|
||||||
|
// the mesh does not compute fails to render here, not on a machine.
|
||||||
|
type rosterView struct {
|
||||||
|
Node string
|
||||||
|
Suffix string
|
||||||
|
Names []rosterEntry
|
||||||
|
Machines []rosterEntry
|
||||||
|
}
|
||||||
|
|
||||||
|
// rosterEntry is one machine as a template sees it: its bare name, its full mesh name, its address,
|
||||||
|
// and the operator account to log into it as (novox/hq to-be 29) — empty when none is known, so an
|
||||||
|
// ssh Host block template can omit the User line for a machine nobody has an account on.
|
||||||
|
type rosterEntry struct {
|
||||||
|
Name string
|
||||||
|
FQDN string
|
||||||
|
Address string
|
||||||
|
Account string
|
||||||
|
}
|
||||||
|
|
||||||
|
// FactsInto renders the roster files a module asked for, as files it will be given.
|
||||||
|
//
|
||||||
|
// The module owns everything after the file exists: loading it, restarting on it, what a resolver
|
||||||
|
// or a client does with it. This only puts it there. `every` is every name the mesh serves;
|
||||||
|
// `machines` is only the machines — the two must not be confused (novox/hq 04-ISSUES/111), so both
|
||||||
|
// are given and the template chooses.
|
||||||
|
func FactsInto(m Manifest, r Resolution, every, machines, accounts map[string]string, suffix string) ([]map[string]any, error) {
|
||||||
|
if len(m.Facts) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(m.Facts))
|
||||||
|
for name := range m.Facts {
|
||||||
|
names = append(names, name)
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
|
||||||
|
view := rosterView{
|
||||||
|
Node: r.Node,
|
||||||
|
Suffix: strings.TrimPrefix(suffixOr(suffix), "."),
|
||||||
|
Names: entriesFrom(every, accounts, suffix),
|
||||||
|
Machines: entriesFrom(machines, accounts, suffix),
|
||||||
|
}
|
||||||
|
|
||||||
|
out := make([]map[string]any, 0, len(names))
|
||||||
|
for _, name := range names {
|
||||||
|
fact := m.Facts[name]
|
||||||
|
content, err := renderRoster(fact.Template, view)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%s cannot render %q: %w", m.Module, name, err)
|
||||||
|
}
|
||||||
|
// Where the file goes: under the operator's home and chowned to it (a home fact), or at the
|
||||||
|
// absolute system path it names. A home fact on a machine with no operator account cannot be
|
||||||
|
// placed, and is left out rather than written to nowhere (novox/hq to-be 29).
|
||||||
|
path := fact.Path
|
||||||
|
var owner string
|
||||||
|
if fact.Home {
|
||||||
|
if r.Account == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
path = accountHomeOf(r.Account, r.AccountHome) + "/" + strings.TrimLeft(fact.Path, "/")
|
||||||
|
owner = r.Account
|
||||||
|
} else if !strings.HasPrefix(fact.Path, "/") {
|
||||||
|
return nil, fmt.Errorf(
|
||||||
|
"%s asks for %q at %q, which is not an absolute path", m.Module, name, fact.Path)
|
||||||
|
}
|
||||||
|
file := map[string]any{
|
||||||
|
"id": "fact-" + name, "type": "file", "path": path, "mode": "0644",
|
||||||
|
"content": content,
|
||||||
|
}
|
||||||
|
if owner != "" {
|
||||||
|
file["owner"] = owner
|
||||||
|
}
|
||||||
|
if fact.Shared {
|
||||||
|
// The host owns only the lines between `# BEGIN mesh <id>` and `# END mesh <id>` and
|
||||||
|
// keeps the rest of the file byte for byte; undeclared, the region goes and nothing else
|
||||||
|
// does (novox/hq issue 128). Every node on the private network receives this, so every
|
||||||
|
// node's host — the controller's own machine included — must be block-aware before a
|
||||||
|
// controller emitting it is rolled out: the order ADR 0102 set for `into: json`.
|
||||||
|
file["into"] = "block"
|
||||||
|
}
|
||||||
|
out = append(out, file)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// renderRoster runs a module's template over the roster. A template that will not parse, or reads
|
||||||
|
// a field the mesh does not have, is an error here — where the manifest is — rather than an empty
|
||||||
|
// file on a machine.
|
||||||
|
func renderRoster(tmpl string, view rosterView) (string, error) {
|
||||||
|
t, err := template.New("roster").Option("missingkey=error").Parse(tmpl)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
var b bytes.Buffer
|
||||||
|
if err := t.Execute(&b, view); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return b.String(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// entriesFrom is a name→address map as sorted roster entries.
|
||||||
|
//
|
||||||
|
// **A machine with no address is left out.** The mesh has a record for it — somebody added it —
|
||||||
|
// and does not yet know where it is, which is the ordinary state between adding a machine and it
|
||||||
|
// joining. Writing the name anyway would give a name that resolves to nothing, and a connection to
|
||||||
|
// that hangs; leaving it out fails at once and says the name is unknown.
|
||||||
|
func entriesFrom(addresses, accounts map[string]string, suffix string) []rosterEntry {
|
||||||
|
out := make([]rosterEntry, 0, len(addresses))
|
||||||
|
for _, name := range sortedNames(addresses) {
|
||||||
|
internal, bare := meshName(name, suffix)
|
||||||
|
// The account is looked up by whichever key the caller keys accounts on — the internal name
|
||||||
|
// or the bare one — so a template gets the right login however the maps were built.
|
||||||
|
account := accounts[name]
|
||||||
|
if account == "" {
|
||||||
|
account = accounts[bare]
|
||||||
|
}
|
||||||
|
out = append(out, rosterEntry{Name: bare, FQDN: internal, Address: addresses[name], Account: account})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// meshName is a machine's internal name and its bare one, from either. The control plane keys
|
||||||
|
// the names it hands a resolution by the internal name (`homer.internal`), the same map a
|
||||||
|
// container gets as its hosts; a caller that keys by the bare name gets the same answer. The
|
||||||
|
// suffix is the one the control plane composed those names with, handed down rather than written
|
||||||
|
// here a second time — the alternative was `homer.internal.internal` on every machine.
|
||||||
|
func meshName(name, suffix string) (internal, bare string) {
|
||||||
|
dotted := "." + strings.TrimPrefix(suffixOr(suffix), ".")
|
||||||
|
if strings.HasSuffix(name, dotted) {
|
||||||
|
return name, strings.TrimSuffix(name, dotted)
|
||||||
|
}
|
||||||
|
return name + dotted, name
|
||||||
|
}
|
||||||
|
|
||||||
|
// suffixOr is the suffix given, or the one the mesh composes names with when none was handed down.
|
||||||
|
// The one place the default is written, so a fact and a name cannot disagree about it.
|
||||||
|
func suffixOr(suffix string) string {
|
||||||
|
if suffix == "" {
|
||||||
|
return "internal"
|
||||||
|
}
|
||||||
|
return suffix
|
||||||
|
}
|
||||||
|
|
||||||
|
func sortedNames(addresses map[string]string) []string {
|
||||||
|
out := make([]string, 0, len(addresses))
|
||||||
|
for name, at := range addresses {
|
||||||
|
// A machine the mesh cannot place is left out rather than named at nothing.
|
||||||
|
if at == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, name)
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,260 @@
|
|||||||
|
package catalogue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Keyed by the internal name, as the control plane hands them (issue 079). bart has no address —
|
||||||
|
// the ordinary state between adding a machine and it joining.
|
||||||
|
var threeMachines = map[string]string{"homer.internal": "10.42.0.1", "marge.internal": "10.42.0.2", "bart.internal": ""}
|
||||||
|
|
||||||
|
// A module says where it wants a roster file and in what format, and is given the rendered file.
|
||||||
|
func TestAModuleIsGivenTheFileItAskedFor(t *testing.T) {
|
||||||
|
m := Manifest{Module: "resolver", Facts: map[string]RosterFile{
|
||||||
|
"zones": {Path: "/etc/mesh/zones.conf", Template: "{{range .Machines}}address=/{{.FQDN}}/{{.Address}}\n{{end}}"},
|
||||||
|
}}
|
||||||
|
given, err := FactsInto(m, Resolution{Node: "homer"}, threeMachines, threeMachines, nil, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(given) != 1 {
|
||||||
|
t.Fatalf("expected one file, got %d", len(given))
|
||||||
|
}
|
||||||
|
if given[0]["path"] != "/etc/mesh/zones.conf" || given[0]["type"] != "file" {
|
||||||
|
t.Fatalf("not written where it was asked for: %v", given[0])
|
||||||
|
}
|
||||||
|
// The id is fact-<name>, which is what a restart-on names when the roster changes.
|
||||||
|
if given[0]["id"] != "fact-zones" {
|
||||||
|
t.Fatalf("the file's id is not fact-<name>, so a restart-on cannot find it: %v", given[0]["id"])
|
||||||
|
}
|
||||||
|
if !strings.Contains(given[0]["content"].(string), "address=/homer.internal/10.42.0.1") {
|
||||||
|
t.Fatalf("the file does not hold the fact: %v", given[0]["content"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A shared fact is written into a region of the machine's file, not over it** (novox/hq issue
|
||||||
|
// 128). A hosts file is the machine's — its localhost, the operator's lines, other tools' blocks —
|
||||||
|
// so the mesh owns only a marked region (`into: block`); a resolver's zones file is the mesh's
|
||||||
|
// whole, and carries no `into`.
|
||||||
|
func TestASharedFactIsWrittenIntoARegion(t *testing.T) {
|
||||||
|
roster := map[string]string{"homer.internal": "10.42.0.1"}
|
||||||
|
m := Manifest{Module: "net", Facts: map[string]RosterFile{
|
||||||
|
"node-names": {Path: "/etc/hosts", Template: "{{range .Names}}{{.FQDN}}\n{{end}}", Shared: true},
|
||||||
|
"node-zones": {Path: "/etc/zones", Template: "{{range .Machines}}{{.FQDN}}\n{{end}}"},
|
||||||
|
}}
|
||||||
|
given, err := FactsInto(m, Resolution{Node: "homer"}, roster, roster, nil, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
by := map[string]map[string]any{}
|
||||||
|
for _, f := range given {
|
||||||
|
by[f["path"].(string)] = f
|
||||||
|
}
|
||||||
|
if by["/etc/hosts"]["into"] != "block" {
|
||||||
|
t.Fatalf("a shared fact is not written into a region, so the mesh writes the file whole: %v", by["/etc/hosts"])
|
||||||
|
}
|
||||||
|
if _, has := by["/etc/zones"]["into"]; has {
|
||||||
|
t.Fatalf("an unshared fact was written into a region, so the mesh does not own its own file whole: %v", by["/etc/zones"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The format is the module's — the mesh renders whatever template it gives.** The same roster
|
||||||
|
// through two templates is two entirely different files, and the control plane reads neither.
|
||||||
|
func TestTheFormatIsTheModulesOwn(t *testing.T) {
|
||||||
|
roster := map[string]string{"homer.internal": "10.42.0.1"}
|
||||||
|
hostsish := Manifest{Module: "a", Facts: map[string]RosterFile{
|
||||||
|
"f": {Path: "/f", Template: "{{range .Names}}{{.Address}}\t{{.Name}}\n{{end}}"}}}
|
||||||
|
sshish := Manifest{Module: "b", Facts: map[string]RosterFile{
|
||||||
|
"f": {Path: "/f", Template: "{{range .Names}}Host {{.Name}}\n HostName {{.FQDN}}\n{{end}}"}}}
|
||||||
|
|
||||||
|
h, err := FactsInto(hostsish, Resolution{Node: "homer"}, roster, roster, nil, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
s, err := FactsInto(sshish, Resolution{Node: "homer"}, roster, roster, nil, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if h[0]["content"] != "10.42.0.1\thomer\n" {
|
||||||
|
t.Fatalf("the hosts-shaped template did not render its format: %q", h[0]["content"])
|
||||||
|
}
|
||||||
|
if s[0]["content"] != "Host homer\n HostName homer.internal\n" {
|
||||||
|
t.Fatalf("the ssh-shaped template did not render its format: %q", s[0]["content"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A template that will not parse is refused here, not on a machine.** A daemon that starts, reads
|
||||||
|
// a file the mesh could not render, and answers nothing is a much worse way to find out.
|
||||||
|
func TestABrokenTemplateIsRefusedHere(t *testing.T) {
|
||||||
|
m := Manifest{Module: "resolver", Facts: map[string]RosterFile{
|
||||||
|
"zones": {Path: "/etc/zones", Template: "{{range .Machines}}oops"}}}
|
||||||
|
_, err := FactsInto(m, Resolution{}, nil, nil, nil, "")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a template that does not parse was accepted, so the machine gets an empty file")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "resolver") || !strings.Contains(err.Error(), "zones") {
|
||||||
|
t.Fatalf("the refusal does not say whose template, or which: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A template reading something the mesh does not compute is refused, not rendered empty. The roster
|
||||||
|
// is a closed shape; asking it for the weather fails where the manifest is.
|
||||||
|
func TestATemplateReadingWhatTheMeshDoesNotHaveIsRefused(t *testing.T) {
|
||||||
|
m := Manifest{Module: "resolver", Facts: map[string]RosterFile{
|
||||||
|
"zones": {Path: "/etc/zones", Template: "{{.Weather}}"}}}
|
||||||
|
if _, err := FactsInto(m, Resolution{}, nil, nil, nil, ""); err == nil {
|
||||||
|
t.Fatal("a template read a field nobody computes and rendered anyway, silently")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A relative path is refused, or a module decides where the mesh writes on a machine.
|
||||||
|
func TestAFactMustBeAskedForAtAnAbsolutePath(t *testing.T) {
|
||||||
|
m := Manifest{Module: "resolver", Facts: map[string]RosterFile{
|
||||||
|
"hosts": {Path: "etc/hosts", Template: "x"}}}
|
||||||
|
if _, err := FactsInto(m, Resolution{}, nil, nil, nil, ""); err == nil {
|
||||||
|
t.Fatal("a relative path was accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A machine the mesh has a record for and cannot place is left out of the roster.
|
||||||
|
//
|
||||||
|
// **Not an oversight — the alternative is worse.** A name written with no address resolves to
|
||||||
|
// nothing, and a connection to that hangs. Leaving it out fails at once and says the name is
|
||||||
|
// unknown, which is a thing somebody can act on.
|
||||||
|
func TestAMachineWithNoAddressIsNotInTheRoster(t *testing.T) {
|
||||||
|
m := Manifest{Module: "a", Facts: map[string]RosterFile{
|
||||||
|
"f": {Path: "/f", Template: "{{range .Machines}}{{.Name}}\n{{end}}"}}}
|
||||||
|
given, err := FactsInto(m, Resolution{Node: "homer"}, threeMachines, threeMachines, nil, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if strings.Contains(given[0]["content"].(string), "bart") {
|
||||||
|
t.Fatalf("a machine with no address was in the roster, so its name resolves to nothing:\n%s", given[0]["content"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The names the control plane hands a resolution are already internal names** — `homer.internal`,
|
||||||
|
// the same map every container gets as its hosts. A roster entry's FQDN is that name, not it with
|
||||||
|
// the suffix appended a second time; either key gives the same entries.
|
||||||
|
func TestNamesAreNotSuffixedTwice(t *testing.T) {
|
||||||
|
internal := map[string]string{"homer.internal": "10.42.0.1"}
|
||||||
|
bare := map[string]string{"homer": "10.42.0.1"}
|
||||||
|
tmpl := RosterFile{Path: "/f", Template: "{{range .Machines}}{{.FQDN}} {{.Name}}\n{{end}}"}
|
||||||
|
|
||||||
|
fromInternal, err := FactsInto(Manifest{Module: "a", Facts: map[string]RosterFile{"f": tmpl}}, Resolution{Node: "homer"}, internal, internal, nil, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
fromBare, err := FactsInto(Manifest{Module: "a", Facts: map[string]RosterFile{"f": tmpl}}, Resolution{Node: "homer"}, bare, bare, nil, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if fromInternal[0]["content"] != fromBare[0]["content"] {
|
||||||
|
t.Fatalf("the roster differs by how the names were keyed:\n%q\n%q", fromInternal[0]["content"], fromBare[0]["content"])
|
||||||
|
}
|
||||||
|
got := fromInternal[0]["content"].(string)
|
||||||
|
if strings.Contains(got, "internal.internal") || !strings.Contains(got, "homer.internal homer") {
|
||||||
|
t.Fatalf("the entry carries a doubled suffix or the wrong bare name:\n%s", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The suffix the control plane composed the names with is the one a template sees — an operator who
|
||||||
|
// chose another does not get `.internal`. `.Suffix` is the bare form, and FQDNs carry it.
|
||||||
|
func TestTheSuffixIsCarriedAsComposed(t *testing.T) {
|
||||||
|
names := map[string]string{"homer.lan": "10.42.0.1"}
|
||||||
|
m := Manifest{Module: "a", Facts: map[string]RosterFile{
|
||||||
|
"f": {Path: "/f", Template: "local=/{{.Suffix}}/\n{{range .Machines}}{{.FQDN}}\n{{end}}"}}}
|
||||||
|
given, err := FactsInto(m, Resolution{Node: "homer"}, names, names, nil, "lan")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got := given[0]["content"].(string)
|
||||||
|
if !strings.Contains(got, "local=/lan/") || strings.Contains(got, "internal") {
|
||||||
|
t.Fatalf("the operator's suffix was not carried, so its names would be wrong:\n%s", got)
|
||||||
|
}
|
||||||
|
if !strings.Contains(got, "homer.lan") {
|
||||||
|
t.Fatalf("the FQDN does not carry the operator's suffix:\n%s", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq 04-ISSUES/111: a template is given both sets and chooses. `.Names` is every name the mesh
|
||||||
|
// serves — the machines and the names it was told to route; `.Machines` is only the machines. A
|
||||||
|
// container's hosts wants every name so a routed name resolves to the machine serving it; a resolver
|
||||||
|
// told the suffix is its own wants only the machines, or a routed name written there with the suffix
|
||||||
|
// is a name nobody will ever ask for, standing beside the machines and looking as real.
|
||||||
|
func TestATemplateChoosesMachinesOrEveryName(t *testing.T) {
|
||||||
|
machines := map[string]string{"homer.internal": "10.42.0.1", "marge.internal": "10.42.0.2"}
|
||||||
|
every := map[string]string{
|
||||||
|
"homer.internal": "10.42.0.1", "marge.internal": "10.42.0.2",
|
||||||
|
"drive.example.test": "10.42.0.1", "git.example.test": "10.42.0.2",
|
||||||
|
}
|
||||||
|
m := Manifest{Module: "resolver", Facts: map[string]RosterFile{
|
||||||
|
"zones": {Path: "/etc/zones", Template: "{{range .Machines}}{{.FQDN}}\n{{end}}"},
|
||||||
|
"hosts": {Path: "/etc/hosts", Template: "{{range .Names}}{{.FQDN}}\n{{end}}"},
|
||||||
|
}}
|
||||||
|
given, err := FactsInto(m, Resolution{Node: "homer"}, every, machines, nil, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
by := map[string]string{}
|
||||||
|
for _, f := range given {
|
||||||
|
by[f["path"].(string)] = f["content"].(string)
|
||||||
|
}
|
||||||
|
|
||||||
|
zones := by["/etc/zones"]
|
||||||
|
for _, served := range []string{"drive.example.test", "git.example.test"} {
|
||||||
|
if strings.Contains(zones, served) {
|
||||||
|
t.Fatalf("a template over .Machines saw %q, a name the mesh serves rather than a machine:\n%s", served, zones)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !strings.Contains(zones, "homer.internal") {
|
||||||
|
t.Fatalf("a template over .Machines did not see the machines:\n%s", zones)
|
||||||
|
}
|
||||||
|
|
||||||
|
hosts := by["/etc/hosts"]
|
||||||
|
for _, name := range []string{"homer.internal", "drive.example.test", "git.example.test"} {
|
||||||
|
if !strings.Contains(hosts, name) {
|
||||||
|
t.Fatalf("a template over .Names did not see %q, so a container would not resolve it:\n%s", name, hosts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A home fact is placed under the operator account's home and chowned to it, and its template sees
|
||||||
|
// each node's account (novox/hq to-be 29) — the ssh-client config is the case.
|
||||||
|
func TestAHomeFactIsPlacedUnderTheAccountsHomeAndOwnedByIt(t *testing.T) {
|
||||||
|
names := map[string]string{"homer.internal": "10.42.0.1", "marge.internal": "10.42.0.2"}
|
||||||
|
accounts := map[string]string{"homer": "jo", "marge": "jo"}
|
||||||
|
m := Manifest{Module: "ssh-client", Facts: map[string]RosterFile{
|
||||||
|
"ssh-config": {Path: ".ssh/config.d/mesh", Home: true,
|
||||||
|
Template: "{{range .Names}}Host {{.Name}}\n HostName {{.FQDN}}\n User {{.Account}}\n{{end}}"}}}
|
||||||
|
given, err := FactsInto(m, Resolution{Node: "homer", Account: "jo"}, names, names, accounts, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
f := given[0]
|
||||||
|
if f["path"] != "/home/jo/.ssh/config.d/mesh" {
|
||||||
|
t.Fatalf("the home fact was not placed under the account's home: %v", f["path"])
|
||||||
|
}
|
||||||
|
if f["owner"] != "jo" {
|
||||||
|
t.Fatalf("the home fact is not owned by the account: %v", f["owner"])
|
||||||
|
}
|
||||||
|
if !strings.Contains(f["content"].(string), "Host marge\n HostName marge.internal\n User jo") {
|
||||||
|
t.Fatalf("the config does not name the peer's account:\n%s", f["content"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A machine with no operator account gets no home fact — it cannot be placed, so it is left out
|
||||||
|
// rather than written to nowhere.
|
||||||
|
func TestAHomeFactIsSkippedWhereThereIsNoAccount(t *testing.T) {
|
||||||
|
names := map[string]string{"homer.internal": "10.42.0.1"}
|
||||||
|
m := Manifest{Module: "ssh-client", Facts: map[string]RosterFile{
|
||||||
|
"ssh-config": {Path: ".ssh/config", Home: true, Template: "x"}}}
|
||||||
|
given, err := FactsInto(m, Resolution{Node: "homer"}, names, names, nil, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(given) != 0 {
|
||||||
|
t.Fatalf("a home fact was placed on a machine with no operator account: %v", given)
|
||||||
|
}
|
||||||
|
}
|
||||||
+162
-28
@@ -27,40 +27,123 @@ type Seat struct {
|
|||||||
// provision may only be held by a module providing it at the seat's scope, and its holder is
|
// provision may only be held by a module providing it at the seat's scope, and its holder is
|
||||||
// what a requirement for that provision resolves to when several modules provide it.
|
// what a requirement for that provision resolves to when several modules provide it.
|
||||||
Delivers string
|
Delivers string
|
||||||
|
// Accepts, Emits and Serves are the protocol of the role, as local verbs — the same three a
|
||||||
|
// module declares for a seat of its own (novox/hq ADR 0118), and empty for most of these: a seat
|
||||||
|
// is usually about who does a job and not about what may be said to them.
|
||||||
|
//
|
||||||
|
// **Named here so the mesh has no role it cannot describe** (ADR 0121). Without them a build
|
||||||
|
// machine had three audiences for one outcome and nothing derived a grant for any of them, and an
|
||||||
|
// event about a role had nowhere to live but the namespace of whichever module held that role
|
||||||
|
// today — which the bus refuses, because a namespace belongs to who it is named for.
|
||||||
|
Accepts []string
|
||||||
|
Emits []string
|
||||||
|
Serves []string
|
||||||
// Decision is the record that made it a seat.
|
// Decision is the record that made it a seat.
|
||||||
Decision string
|
Decision string
|
||||||
}
|
}
|
||||||
|
|
||||||
// seats is the whole set, in the order a person reads it: the mesh's own, then a node's.
|
// defaultSeats is the set the mesh ships with — the seed for the control plane's seat table and the
|
||||||
var seats = []Seat{
|
// fallback when it has none (novox/hq ADR 0122). It is the one place the closed set 0110 defines is
|
||||||
|
// written; the store's table is seeded from it and thereafter is the live, editable copy.
|
||||||
|
//
|
||||||
|
// In the order a person reads it: the mesh's own, then a node's.
|
||||||
|
var defaultSeats = []Seat{
|
||||||
{Name: "mesh-controller", Scope: ScopeMesh, Decision: "novox/hq ADR 0079"},
|
{Name: "mesh-controller", Scope: ScopeMesh, Decision: "novox/hq ADR 0079"},
|
||||||
{Name: "mesh-store", Scope: ScopeMesh, Delivers: "postgres-database", Decision: "novox/hq ADR 0079"},
|
{Name: "mesh-store", Scope: ScopeMesh, Delivers: "postgres-database", Decision: "novox/hq ADR 0079"},
|
||||||
{Name: "mesh-broker", Scope: ScopeMesh, Delivers: "amqp", Decision: "novox/hq ADR 0079"},
|
// **Delivers the mesh's own bus, not `amqp`.** Those were the same word until
|
||||||
|
// ADR 0127 separated them: `amqp` is a backing service a module may require, and this seat is
|
||||||
|
// the mesh's own transport. ADR 0128 then made that connection something a module requires
|
||||||
|
// rather than receives ambiently — 23 of the catalogue's modules never speak, and an ambient
|
||||||
|
// connection would mint a credential for each.
|
||||||
|
{Name: "mesh-broker", Scope: ScopeMesh, Delivers: "mesh-bus", Decision: "novox/hq ADR 0079"},
|
||||||
{Name: "the-artifact-store", Scope: ScopeMesh, Delivers: "artifact-store", Decision: "novox/hq ADR 0075"},
|
{Name: "the-artifact-store", Scope: ScopeMesh, Delivers: "artifact-store", Decision: "novox/hq ADR 0075"},
|
||||||
{Name: "the-catalogue", Scope: ScopeMesh, Decision: "novox/hq ADR 0110"},
|
{Name: "mesh-catalog", Scope: ScopeMesh, Decision: "novox/hq ADR 0121"},
|
||||||
|
// Deferred renames (novox/hq ADR 0121): these deliver a provision, so renaming them is a
|
||||||
|
// delivering-seat migration with a mesh-wide cascade if a holder stops resolving mid-flight.
|
||||||
|
// They keep their names until that migration is done deliberately, apart from the node-* pass.
|
||||||
{Name: "npm-package-registry", Scope: ScopeMesh, Delivers: "npm-package-registry", Decision: "novox/hq ADR 0109"},
|
{Name: "npm-package-registry", Scope: ScopeMesh, Delivers: "npm-package-registry", Decision: "novox/hq ADR 0109"},
|
||||||
{Name: "git", Scope: ScopeMesh, Delivers: "git", Decision: "novox/hq ADR 0111"},
|
{Name: "git", Scope: ScopeMesh, Delivers: "git", Decision: "novox/hq ADR 0111"},
|
||||||
{Name: "the-build-machine", Scope: ScopeNode, Decision: "novox/hq ADR 0110"},
|
// A build is work submitted to this role and its outcome is the role's own event (ADR 0129).
|
||||||
{Name: "the-dns-port", Scope: ScopeNode, Decision: "novox/hq ADR 0110"},
|
// One publish reaches whoever asked, the controller that records it, and the catalogue that
|
||||||
{Name: "the-intrusion-prevention", Scope: ScopeNode, Decision: "novox/hq ADR 0110"},
|
// places it in the graph — what the old bus's shared exchange did for free.
|
||||||
{Name: "the-packet-filter", Scope: ScopeNode, Decision: "novox/hq ADR 0110"},
|
{Name: "mesh-build-machine", Scope: ScopeMesh,
|
||||||
|
Accepts: []string{"build"}, Emits: []string{"built"}, Decision: "novox/hq ADR 0121"},
|
||||||
|
{Name: "node-dns-resolver", Scope: ScopeNode, Decision: "novox/hq ADR 0121"},
|
||||||
|
{Name: "node-intrusion-prevention", Scope: ScopeNode, Decision: "novox/hq ADR 0121"},
|
||||||
|
{Name: "node-packet-filter", Scope: ScopeNode, Decision: "novox/hq ADR 0121"},
|
||||||
|
// Deferred (novox/hq ADR 0121): renaming to mesh-private-network is a scope + server/client
|
||||||
|
// model change, not a rename, so it stays until that is built.
|
||||||
{Name: "the-private-network", Scope: ScopeNode, Decision: "novox/hq ADR 0110"},
|
{Name: "the-private-network", Scope: ScopeNode, Decision: "novox/hq ADR 0110"},
|
||||||
{Name: "the-resolver-configuration", Scope: ScopeNode, Decision: "novox/hq ADR 0110"},
|
{Name: "node-resolver-config", Scope: ScopeNode, Decision: "novox/hq ADR 0121"},
|
||||||
{Name: "the-showcase", Scope: ScopeNode, Decision: "novox/hq ADR 0110"},
|
// The program that manages the machine's own network. It delivers nothing: its holder only
|
||||||
|
// keeps the manager and the mesh from contradicting each other — the resolver file left to the
|
||||||
|
// mesh, the private network's interface left alone — and never declares a link, an address or
|
||||||
|
// a wireless network, because the link is the only channel a fix could arrive on. A seat
|
||||||
|
// rather than a condition in the resolver's module, so a machine running two managers is
|
||||||
|
// refused at assignment instead of found by the resolver being rewritten (novox/hq ADR 0117).
|
||||||
|
{Name: "node-uplink", Scope: ScopeNode, Decision: "novox/hq ADR 0117"},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A system seat name is the control plane's namespace: `mesh-*` for a mesh-wide role, `node-*` for
|
||||||
|
// a per-node one (novox/hq ADR 0121). A claim to a system name the mesh does not define is refused;
|
||||||
|
// any other name is a module's own to define and claim. Some of the mesh's own seats predate this
|
||||||
|
// convention and are not yet renamed (git, npm-package-registry, the-artifact-store,
|
||||||
|
// the-private-network) — those are in the set, so they resolve by name, not by prefix.
|
||||||
|
func isSystemSeatName(name string) bool {
|
||||||
|
return strings.HasPrefix(name, "mesh-") || strings.HasPrefix(name, "node-")
|
||||||
|
}
|
||||||
|
|
||||||
|
// seats is the working set the lookups read. It starts as the compiled defaults and is replaced by
|
||||||
|
// what the control plane loaded from its store (novox/hq ADR 0122), so a change to the set is a
|
||||||
|
// change to data, not to this code.
|
||||||
|
var seats = defaultSeats
|
||||||
|
|
||||||
|
// DefaultSeats is the set the mesh ships with, for seeding the store's seat table.
|
||||||
|
func DefaultSeats() []Seat { return append([]Seat(nil), defaultSeats...) }
|
||||||
|
|
||||||
|
// UseSeats replaces the working set with the one the control plane read from its store.
|
||||||
|
//
|
||||||
|
// **Empty is ignored on purpose.** A store that has not been seeded yet — or one that could not be
|
||||||
|
// read — must leave the compiled defaults in force rather than emptying the set: an empty set would
|
||||||
|
// refuse every claim and could stop the control plane composing at all, which is a far worse failure
|
||||||
|
// than running on the set the binary shipped with. So the store can only ever *replace* the set with
|
||||||
|
// a non-empty one, never erase it.
|
||||||
|
func UseSeats(s []Seat) {
|
||||||
|
if len(s) > 0 {
|
||||||
|
seats = s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// aliases maps a seat's former names to its current canonical name (novox/hq ADR 0122). Loaded from
|
||||||
|
// the store alongside the set, so a reference to a name a seat used to have — a manifest's claim, a
|
||||||
|
// held record — still resolves to it after a rename, and nothing downstream has to change.
|
||||||
|
var aliases = map[string]string{}
|
||||||
|
|
||||||
|
// UseAliases replaces the former-name map with the one the control plane read from its store. Empty
|
||||||
|
// is fine and ordinary: a mesh whose seats have never been renamed has no aliases.
|
||||||
|
func UseAliases(m map[string]string) { aliases = m }
|
||||||
|
|
||||||
// Seats is every seat the mesh defines, in reading order.
|
// Seats is every seat the mesh defines, in reading order.
|
||||||
func Seats() []Seat {
|
func Seats() []Seat {
|
||||||
return append([]Seat(nil), seats...)
|
return append([]Seat(nil), seats...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// SeatNamed is the seat a claim names, if the mesh defines one.
|
// SeatNamed is the seat a name refers to, whether that is its current name or one it used to have
|
||||||
|
// (novox/hq ADR 0122). A former name resolves to the seat's canonical row, so a rename breaks no
|
||||||
|
// reference to the old name.
|
||||||
func SeatNamed(name string) (Seat, bool) {
|
func SeatNamed(name string) (Seat, bool) {
|
||||||
for _, s := range seats {
|
for _, s := range seats {
|
||||||
if s.Name == name {
|
if s.Name == name {
|
||||||
return s, true
|
return s, true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if canonical, aliased := aliases[name]; aliased {
|
||||||
|
for _, s := range seats {
|
||||||
|
if s.Name == canonical {
|
||||||
|
return s, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
return Seat{}, false
|
return Seat{}, false
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -77,30 +160,63 @@ func SeatDelivering(provision string) (Seat, bool) {
|
|||||||
return Seat{}, false
|
return Seat{}, false
|
||||||
}
|
}
|
||||||
|
|
||||||
// claimProblems is what is wrong with a manifest's claims against the set.
|
// claimProblems is what is wrong with a manifest's claims and the seats it defines.
|
||||||
//
|
//
|
||||||
// Three refusals, each naming the seat: a seat the mesh does not define, a seat claimed at another
|
// A claim is one of three things (novox/hq ADR 0121): a **system seat** the control plane defines —
|
||||||
// scope, and a seat that delivers a provision claimed by a module that does not provide it — which
|
// checked for scope and, if it delivers a provision, that the claimant provides it; a **system name
|
||||||
// would make the module the mesh's answer for something it cannot answer.
|
// the mesh does not define** (`mesh-*`/`node-*`) — refused, because that namespace is the control
|
||||||
|
// plane's; or a **module-defined seat** — valid only when this manifest also declares it, since a
|
||||||
|
// module may coordinate its own instances through a seat of its own but may not invent one by
|
||||||
|
// claiming it. A module's own seat declaration may not sit in the system namespace or shadow a
|
||||||
|
// system seat.
|
||||||
func claimProblems(m Manifest) []string {
|
func claimProblems(m Manifest) []string {
|
||||||
var problems []string
|
var problems []string
|
||||||
for _, c := range m.Claims {
|
|
||||||
seat, known := SeatNamed(c.Name)
|
defined := map[string]SeatDeclaration{}
|
||||||
if !known {
|
for _, d := range m.DefinesSeats {
|
||||||
|
if _, isSystem := SeatNamed(d.Name); isSystem || isSystemSeatName(d.Name) {
|
||||||
problems = append(problems, fmt.Sprintf(
|
problems = append(problems, fmt.Sprintf(
|
||||||
"%s claims %q, which is not a seat this mesh defines (novox/hq ADR 0110) — "+
|
"%s defines a seat %q in the mesh's own namespace; a module's seat is named outside "+
|
||||||
"the seats are: %s", m.Module, c.Name, seatNames()))
|
"mesh-*/node-* (novox/hq ADR 0121)", m.Module, d.Name))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if c.At() != seat.Scope {
|
defined[d.Name] = d
|
||||||
problems = append(problems, fmt.Sprintf(
|
}
|
||||||
"%s claims %s at scope %q, and %s is a %s seat",
|
|
||||||
m.Module, c.Name, c.At(), c.Name, seat.Scope))
|
for _, c := range m.Claims {
|
||||||
|
if seat, known := SeatNamed(c.Name); known {
|
||||||
|
if c.At() != seat.Scope {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s claims %s at scope %q, and %s is a %s seat",
|
||||||
|
m.Module, c.Name, c.At(), c.Name, seat.Scope))
|
||||||
|
}
|
||||||
|
if seat.Delivers != "" && !providesAt(m, seat.Delivers, seat.Scope) {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s claims %s, whose holder answers for %q, and %s does not provide %q at %s scope",
|
||||||
|
m.Module, c.Name, seat.Delivers, m.Module, seat.Delivers, seat.Scope))
|
||||||
|
}
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
if seat.Delivers != "" && !providesAt(m, seat.Delivers, seat.Scope) {
|
if isSystemSeatName(c.Name) {
|
||||||
problems = append(problems, fmt.Sprintf(
|
problems = append(problems, fmt.Sprintf(
|
||||||
"%s claims %s, whose holder answers for %q, and %s does not provide %q at %s scope",
|
"%s claims %q, which is a seat in the mesh's own namespace (mesh-*/node-*) that it "+
|
||||||
m.Module, c.Name, seat.Delivers, m.Module, seat.Delivers, seat.Scope))
|
"does not define (novox/hq ADR 0121) — the seats are: %s", m.Module, c.Name, seatNames()))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
d, ours := defined[c.Name]
|
||||||
|
if !ours {
|
||||||
|
// **A claim on a seat this manifest does not declare is not the parser's to judge.**
|
||||||
|
// A module may hold a seat another module declared — that is why ADR 0126 has callers
|
||||||
|
// name the seat and not its provider, so an implementation can be replaced without
|
||||||
|
// touching a caller. Whether the seat exists is a fact about the whole catalogue, so
|
||||||
|
// the refusal is at registration, where every declaration is in view
|
||||||
|
// (`CatalogueProblems`: "which no module declares and the mesh does not define").
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if c.At() != d.At() {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s claims its own seat %s at scope %q, having declared it at %q",
|
||||||
|
m.Module, c.Name, c.At(), d.At()))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return problems
|
return problems
|
||||||
@@ -136,7 +252,10 @@ func HolderAmong(provision string, providers []Provider, held []Held) (Provider,
|
|||||||
return Provider{}, false
|
return Provider{}, false
|
||||||
}
|
}
|
||||||
for _, h := range held {
|
for _, h := range held {
|
||||||
if h.Claim != seat.Name || h.Scope != seat.Scope {
|
// Resolve the held claim to a seat rather than comparing names, so a record naming a seat's
|
||||||
|
// former name still matches it after a rename (novox/hq ADR 0122).
|
||||||
|
hs, ok := SeatNamed(h.Claim)
|
||||||
|
if !ok || hs.Name != seat.Name || h.Scope != seat.Scope {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
for _, p := range providers {
|
for _, p := range providers {
|
||||||
@@ -147,3 +266,18 @@ func HolderAmong(provision string, providers []Provider, held []Held) (Provider,
|
|||||||
}
|
}
|
||||||
return Provider{}, false
|
return Provider{}, false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SeatsWithAProtocol are the mesh's own seats that say something about what may be said to them or by
|
||||||
|
// them, which is the set the bus derives streams, consumers and permissions from.
|
||||||
|
//
|
||||||
|
// Most of the set is not here, and that is the ordinary case: a seat saying only who does a job grants
|
||||||
|
// nothing on the bus and needs no queue.
|
||||||
|
func SeatsWithAProtocol() []Seat {
|
||||||
|
var out []Seat
|
||||||
|
for _, s := range seats {
|
||||||
|
if len(s.Accepts) > 0 || len(s.Emits) > 0 || len(s.Serves) > 0 {
|
||||||
|
out = append(out, s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,239 @@
|
|||||||
|
package catalogue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Seats a module declares of its own (novox/hq ADR 0118).
|
||||||
|
//
|
||||||
|
// The set of seats a mesh has is **derived**: the mesh's own, in seats.go, plus those declared by
|
||||||
|
// every module it has registered. Still closed — a seat named nowhere is refused — but computed
|
||||||
|
// from the catalogue rather than written in the controller, which is what ADR 0110 actually
|
||||||
|
// needed and a hand-maintained table could not keep. Its own evidence: the enumeration done by
|
||||||
|
// hand while that record was written reported eleven claims where there were thirteen.
|
||||||
|
//
|
||||||
|
// **What can be checked from one manifest and what cannot.** A declaration's shape, its scope,
|
||||||
|
// and the reserved prefix are facts about the manifest in front of you. Whether a seat anybody
|
||||||
|
// names actually exists, whether two modules declared the same one, and whether a holder
|
||||||
|
// satisfies the protocol are facts about the *catalogue* — so they are checked at registration,
|
||||||
|
// by CatalogueProblems, which is the last moment the mesh can still say no.
|
||||||
|
|
||||||
|
// meshSeatPrefix is reserved to the mesh. The prefix *is* the reservation rule: no list of
|
||||||
|
// reserved names to maintain, no way for the mesh's own namespace to be colonised by a manifest,
|
||||||
|
// and nothing to keep in step when a mesh seat is added.
|
||||||
|
const meshSeatPrefix = "mesh-"
|
||||||
|
|
||||||
|
// A SeatDeclaration is a role a module offers on the bus: what may be sent to it, what it says,
|
||||||
|
// and what it answers. A caller declares that it uses the *seat*, never the module, so the
|
||||||
|
// implementation can be replaced under it.
|
||||||
|
type SeatDeclaration struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Scope string `json:"scope,omitempty"`
|
||||||
|
|
||||||
|
// Accepts are the verbs others may submit work on. Each becomes a work-queue subject, and
|
||||||
|
// the holder is the only consumer — so exactly one worker does the job, by construction
|
||||||
|
// rather than by how carefully somebody wrote a subscribe call.
|
||||||
|
Accepts []string `json:"accepts,omitempty"`
|
||||||
|
// Emits are the verbs the holder publishes: 1:many, nobody obliged to act.
|
||||||
|
Emits []string `json:"emits,omitempty"`
|
||||||
|
// Serves are the verbs the holder answers: request and reply, awaited.
|
||||||
|
Serves []string `json:"serves,omitempty"`
|
||||||
|
|
||||||
|
// RetainSeconds is how long the inbound backlog survives with no holder, zero for the
|
||||||
|
// mesh's default. Retention belongs to whoever owns the namespace (design 29 §3) — a seat
|
||||||
|
// owns its own, which is why a seat is also the answer for a module that needs retention
|
||||||
|
// its events cannot have.
|
||||||
|
RetainSeconds int `json:"retain-seconds,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// At is this declaration's scope, with the default applied. Mesh by default, because a seat
|
||||||
|
// declared by a module is nearly always "there is one of these in the mesh" — a per-node worker
|
||||||
|
// is the deliberate case, and says so.
|
||||||
|
func (s SeatDeclaration) At() string {
|
||||||
|
if s.Scope == "" {
|
||||||
|
return ScopeMesh
|
||||||
|
}
|
||||||
|
return s.Scope
|
||||||
|
}
|
||||||
|
|
||||||
|
// verbs is everything the protocol names, for the checks that do not care which half.
|
||||||
|
func (s SeatDeclaration) verbs() []string {
|
||||||
|
out := append([]string{}, s.Accepts...)
|
||||||
|
out = append(out, s.Emits...)
|
||||||
|
return append(out, s.Serves...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// declaredSeatProblems is what one manifest can be judged on alone.
|
||||||
|
func declaredSeatProblems(m Manifest) []string {
|
||||||
|
var problems []string
|
||||||
|
seen := map[string]bool{}
|
||||||
|
|
||||||
|
for _, s := range m.DefinesSeats {
|
||||||
|
switch {
|
||||||
|
case s.Name == "":
|
||||||
|
problems = append(problems, fmt.Sprintf("%s declares a seat with no name", m.Module))
|
||||||
|
continue
|
||||||
|
case !name.MatchString(s.Name):
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s declares a seat named %q, which is not a usable name", m.Module, s.Name))
|
||||||
|
continue
|
||||||
|
case strings.HasPrefix(s.Name, meshSeatPrefix):
|
||||||
|
// The mesh's own code dereferences its seats by name — the resolver *is* the thing
|
||||||
|
// that finds the store — so the prefix is not a convention, it is a namespace.
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s declares a seat named %q; %q is reserved to the mesh, which defines its own "+
|
||||||
|
"seats (novox/hq ADR 0118)", m.Module, s.Name, meshSeatPrefix+"*"))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if seen[s.Name] {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s declares the seat %q twice", m.Module, s.Name))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[s.Name] = true
|
||||||
|
|
||||||
|
if _, isMesh := SeatNamed(s.Name); isMesh {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s declares %q, which is a seat the mesh already defines", m.Module, s.Name))
|
||||||
|
}
|
||||||
|
switch s.At() {
|
||||||
|
case ScopeNode, ScopeSite, ScopeMesh:
|
||||||
|
default:
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s declares seat %s at scope %q; a seat is held per node, per site or per mesh",
|
||||||
|
m.Module, s.Name, s.Scope))
|
||||||
|
}
|
||||||
|
// A seat with an empty protocol is allowed, and is the mesh saying what a machine is:
|
||||||
|
// which module is this node's packet filter, or its showcase. Design 26 calls it a seat
|
||||||
|
// that delivers nothing, and that is most of the node-scoped ones. ADR 0126's "a declared
|
||||||
|
// seat carries a protocol" governs what a holder must satisfy, not that every seat offers
|
||||||
|
// something — a marker seat's protocol is satisfied by holding it. Nothing can reach this
|
||||||
|
// state by accident: a mistyped field name is refused by the parser above, so an empty
|
||||||
|
// protocol was written as one.
|
||||||
|
for _, v := range s.verbs() {
|
||||||
|
if !name.MatchString(v) {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s declares %s.%s, which is not a usable verb", m.Module, s.Name, v))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, u := range m.Uses {
|
||||||
|
if !name.MatchString(u) {
|
||||||
|
problems = append(problems, fmt.Sprintf("%s uses %q, which is not a usable seat name", m.Module, u))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return problems
|
||||||
|
}
|
||||||
|
|
||||||
|
// A Shelf is every manifest the mesh has registered, by module name.
|
||||||
|
type Shelf map[string]Manifest
|
||||||
|
|
||||||
|
// CatalogueProblems are the rules no single manifest can be judged against.
|
||||||
|
//
|
||||||
|
// Run at registration, which is the last moment the mesh can still refuse: after it, a caller is
|
||||||
|
// bound to a seat and a refusal is an outage rather than a conversation.
|
||||||
|
func CatalogueProblems(shelf Shelf) []string {
|
||||||
|
var problems []string
|
||||||
|
|
||||||
|
// Who declares what, and who declared it first.
|
||||||
|
declaredBy := map[string]string{}
|
||||||
|
declared := map[string]SeatDeclaration{}
|
||||||
|
for _, module := range shelfOrder(shelf) {
|
||||||
|
for _, s := range shelf[module].DefinesSeats {
|
||||||
|
if s.Name == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if first, taken := declaredBy[s.Name]; taken {
|
||||||
|
// The second loses. A seat name meaning two different protocols is the failure
|
||||||
|
// nobody could diagnose afterwards — a caller would bind to whichever happened
|
||||||
|
// to register first, and the symptom would appear in the other module.
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s declares the seat %q, which %s already declares; a seat name means one "+
|
||||||
|
"protocol", module, s.Name, first))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
declaredBy[s.Name] = module
|
||||||
|
declared[s.Name] = s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
exists := func(seat string) bool {
|
||||||
|
if _, isMesh := SeatNamed(seat); isMesh {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
_, ok := declaredBy[seat]
|
||||||
|
return ok
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, module := range shelfOrder(shelf) {
|
||||||
|
m := shelf[module]
|
||||||
|
|
||||||
|
// A `uses` naming nothing is where ADR 0110's guarantee lands under a derived set: the
|
||||||
|
// same refusal, at the same moment, from a set nobody maintains by hand.
|
||||||
|
for _, u := range m.Uses {
|
||||||
|
if !exists(u) {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s uses the seat %q, which no module declares and the mesh does not define",
|
||||||
|
module, u))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, c := range m.Claims {
|
||||||
|
if !exists(c.Name) {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s claims the seat %q, which no module declares and the mesh does not define",
|
||||||
|
module, c.Name))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
s, isModuleSeat := declared[c.Name]
|
||||||
|
if !isModuleSeat {
|
||||||
|
continue // a mesh seat: already judged by claimProblems
|
||||||
|
}
|
||||||
|
if c.At() != s.At() {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s claims %s at scope %q, and %s declares it at %s",
|
||||||
|
module, c.Name, c.At(), declaredBy[c.Name], s.At()))
|
||||||
|
}
|
||||||
|
// A holder that does not answer what the seat promises is a caller's timeout, found
|
||||||
|
// at assignment instead.
|
||||||
|
if missing := unserved(m, s); len(missing) > 0 {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s claims %s but does not serve %s, which that seat's protocol promises",
|
||||||
|
module, c.Name, strings.Join(missing, ", ")))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(problems)
|
||||||
|
return problems
|
||||||
|
}
|
||||||
|
|
||||||
|
// unserved is what a seat's protocol promises and the claimant does not answer. Only the tools
|
||||||
|
// are checked: `accepts` and `emits` are wired by the runtime from the declaration, while a tool
|
||||||
|
// is code the module either has or has not written.
|
||||||
|
func unserved(m Manifest, s SeatDeclaration) []string {
|
||||||
|
has := map[string]bool{}
|
||||||
|
for _, t := range m.Tools {
|
||||||
|
has[t] = true
|
||||||
|
}
|
||||||
|
var missing []string
|
||||||
|
for _, t := range s.Serves {
|
||||||
|
if !has[t] {
|
||||||
|
missing = append(missing, t)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return missing
|
||||||
|
}
|
||||||
|
|
||||||
|
// shelfOrder is the catalogue in a stable order, so two runs report the same problems in the same
|
||||||
|
// sequence — a refusal that reorders itself is a refusal nobody can diff.
|
||||||
|
func shelfOrder(shelf Shelf) []string {
|
||||||
|
out := make([]string, 0, len(shelf))
|
||||||
|
for k := range shelf {
|
||||||
|
out = append(out, k)
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
package catalogue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func problemsFor(t *testing.T, shelf Shelf) string {
|
||||||
|
t.Helper()
|
||||||
|
return strings.Join(CatalogueProblems(shelf), "; ")
|
||||||
|
}
|
||||||
|
|
||||||
|
func telegram() Manifest {
|
||||||
|
return Manifest{Module: "telegram", Tools: []string{"status"}, DefinesSeats: []SeatDeclaration{{
|
||||||
|
Name: "telegram-sender", Scope: ScopeMesh,
|
||||||
|
Accepts: []string{"send"}, Emits: []string{"delivered", "failed"}, Serves: []string{"status"},
|
||||||
|
}}, Claims: []Claim{{Name: "telegram-sender", Scope: ScopeMesh}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The whole point: a module contributes a capability without the mesh being changed.
|
||||||
|
func TestAModuleDeclaresItsOwnSeatAndHoldsIt(t *testing.T) {
|
||||||
|
shop := Manifest{Module: "shop", Uses: []string{"telegram-sender"}}
|
||||||
|
if got := problemsFor(t, Shelf{"telegram": telegram(), "shop": shop}); got != "" {
|
||||||
|
t.Fatalf("a declared seat and its caller were refused: %s", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The prefix is the reservation rule, so there is no list to maintain and none to drift.
|
||||||
|
func TestAModuleCannotDeclareAMeshSeat(t *testing.T) {
|
||||||
|
for _, n := range []string{"mesh-broker", "mesh-anything", "mesh-store"} {
|
||||||
|
m := Manifest{Module: "impostor", DefinesSeats: []SeatDeclaration{{Name: n, Accepts: []string{"x"}}}}
|
||||||
|
got := strings.Join(declaredSeatProblems(m), "; ")
|
||||||
|
if !strings.Contains(got, "reserved to the mesh") {
|
||||||
|
t.Fatalf("%q was accepted as a module's seat: %q", n, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A seat name meaning two protocols is the failure nobody could diagnose afterwards.
|
||||||
|
func TestTwoModulesCannotDeclareTheSameSeat(t *testing.T) {
|
||||||
|
other := Manifest{Module: "aardvark", DefinesSeats: []SeatDeclaration{{
|
||||||
|
Name: "telegram-sender", Scope: ScopeMesh, Accepts: []string{"something-else"}}}}
|
||||||
|
got := problemsFor(t, Shelf{"telegram": telegram(), "aardvark": other})
|
||||||
|
if !strings.Contains(got, "already declares") {
|
||||||
|
t.Fatalf("both declarations stood: %s", got)
|
||||||
|
}
|
||||||
|
// The first declarer keeps it; only the second is refused.
|
||||||
|
if strings.Count(got, "already declares") != 1 {
|
||||||
|
t.Fatalf("expected exactly one refusal: %s", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Where ADR 0110's guarantee lands under a derived set: a typo is refused, not resolved to
|
||||||
|
// nothing at runtime.
|
||||||
|
func TestUsingASeatNobodyDeclaresIsRefused(t *testing.T) {
|
||||||
|
shop := Manifest{Module: "shop", Uses: []string{"telegram-sendr"}}
|
||||||
|
got := problemsFor(t, Shelf{"telegram": telegram(), "shop": shop})
|
||||||
|
if !strings.Contains(got, "telegram-sendr") || !strings.Contains(got, "no module declares") {
|
||||||
|
t.Fatalf("a misspelled seat was accepted: %s", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A holder that does not answer what the seat promises is a caller's timeout, found here instead.
|
||||||
|
func TestAHolderMustServeWhatItsSeatPromises(t *testing.T) {
|
||||||
|
m := telegram()
|
||||||
|
m.Tools = nil // declares the seat, serves none of it
|
||||||
|
got := problemsFor(t, Shelf{"telegram": m})
|
||||||
|
if !strings.Contains(got, "does not serve status") {
|
||||||
|
t.Fatalf("a holder was accepted that answers nothing its seat promises: %s", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A seat with no protocol is a marker: which module is this node's showcase, or its packet filter.
|
||||||
|
// Most node-scoped seats are markers, so refusing one would refuse the majority of the set.
|
||||||
|
func TestASeatWithoutAProtocolIsAMarkerNotAMistake(t *testing.T) {
|
||||||
|
m := Manifest{Module: "vague", DefinesSeats: []SeatDeclaration{{Name: "something", Scope: ScopeNode}}}
|
||||||
|
if got := strings.Join(declaredSeatProblems(m), "; "); got != "" {
|
||||||
|
t.Fatalf("a marker seat was refused: %s", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A claim at the wrong scope is a different seat than the one declared.
|
||||||
|
func TestAClaimMustMatchTheDeclaredScope(t *testing.T) {
|
||||||
|
m := telegram()
|
||||||
|
m.Claims = []Claim{{Name: "telegram-sender", Scope: ScopeNode}}
|
||||||
|
got := problemsFor(t, Shelf{"telegram": m})
|
||||||
|
if !strings.Contains(got, "scope") {
|
||||||
|
t.Fatalf("a claim at the wrong scope was accepted: %s", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The mesh's own seats still work, and are not shadowed by the derived half.
|
||||||
|
func TestTheMeshsOwnSeatsAreStillClaimable(t *testing.T) {
|
||||||
|
m := Manifest{Module: "nats", Claims: []Claim{{Name: "mesh-broker", Scope: ScopeMesh}}}
|
||||||
|
if got := problemsFor(t, Shelf{"nats": m}); got != "" {
|
||||||
|
t.Fatalf("a mesh seat was refused by the derived check: %s", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A refusal that reorders itself between runs is a refusal nobody can diff.
|
||||||
|
func TestTheProblemsAreStable(t *testing.T) {
|
||||||
|
shelf := Shelf{"telegram": telegram(), "shop": {Module: "shop", Uses: []string{"nope"}},
|
||||||
|
"other": {Module: "other", Uses: []string{"also-nope"}}}
|
||||||
|
first, second := problemsFor(t, shelf), problemsFor(t, shelf)
|
||||||
|
if first != second {
|
||||||
|
t.Fatalf("unstable:\n%s\n%s", first, second)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -50,21 +50,91 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0117: a machine's uplink is a seat, held per machine, and delivers nothing.
|
||||||
|
//
|
||||||
|
// **Nothing, because nothing may be required of it.** A holder only keeps its network manager from
|
||||||
|
// contradicting the mesh; a requirement resolving to it would make the manager the mesh's answer
|
||||||
|
// for something, and the manager's link is the one thing the mesh must never be able to break.
|
||||||
|
func TestTheUplinkIsANodeSeatThatDeliversNothing(t *testing.T) {
|
||||||
|
seat, known := SeatNamed("node-uplink")
|
||||||
|
if !known {
|
||||||
|
t.Fatalf("the uplink is not a seat; the seats are: %s", seatNames())
|
||||||
|
}
|
||||||
|
if seat.Scope != ScopeNode || seat.Delivers != "" || seat.Decision != "novox/hq ADR 0117" {
|
||||||
|
t.Fatalf("the uplink is %+v, not a node seat delivering nothing by ADR 0117", seat)
|
||||||
|
}
|
||||||
|
// And a manager's module can hold it without providing anything.
|
||||||
|
raw := []byte(`{"module":"networkmanager","version":"1","claims":[{"name":"node-uplink","scope":"node"}]}`)
|
||||||
|
if _, err := ParseManifest(raw); err != nil {
|
||||||
|
t.Fatalf("a network manager's module could not hold the uplink: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func claimed(claims string) []byte {
|
func claimed(claims string) []byte {
|
||||||
return []byte(`{"module":"thing","version":"1","provides":[{"name":"npm-package-registry","scope":"mesh"}],"claims":` + claims + `}`)
|
return []byte(`{"module":"thing","version":"1","provides":[{"name":"npm-package-registry","scope":"mesh"}],"claims":` + claims + `}`)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAClaimOnASeatTheMeshDoesNotDefineIsRefused(t *testing.T) {
|
// **The refusal moved, it did not go** (novox/hq ADR 0118, superseding 0110). A module may now
|
||||||
_, err := ParseManifest(claimed(`[{"name":"the-anything","scope":"node"}]`))
|
// declare its own seats, so whether a claimed seat exists is a fact about the *catalogue* and not
|
||||||
if err == nil {
|
// about the manifest in front of the parser: a claim on a seat another registered module declares
|
||||||
t.Fatal("a module invented a seat by claiming it")
|
// is perfectly good, and the parser cannot tell the two cases apart. So the parser accepts it and
|
||||||
|
// registration refuses it — the same guarantee, at the same moment work would otherwise start,
|
||||||
|
// from a set nobody maintains by hand.
|
||||||
|
func TestAClaimOnASeatNobodyDeclaresIsRefusedAtRegistration(t *testing.T) {
|
||||||
|
m, err := ParseManifest(claimed(`[{"name":"the-anything","scope":"node"}]`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("the parser judged a claim it cannot judge alone: %v", err)
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "the-anything") || !strings.Contains(err.Error(), "not a seat") {
|
|
||||||
t.Fatalf("the refusal does not say the seat is unknown: %v", err)
|
problems := CatalogueProblems(Shelf{m.Module: m})
|
||||||
|
if len(problems) == 0 {
|
||||||
|
t.Fatal("a module invented a seat by claiming it, and registration allowed it")
|
||||||
}
|
}
|
||||||
// And it says what the seats are, because "no" without the list sends somebody reading code.
|
joined := strings.Join(problems, "; ")
|
||||||
if !strings.Contains(err.Error(), "the-packet-filter") {
|
if !strings.Contains(joined, "the-anything") || !strings.Contains(joined, "no module declares") {
|
||||||
t.Fatalf("the refusal does not list the seats: %v", err)
|
t.Fatalf("the refusal does not say the seat is nobody's: %v", problems)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// And the same claim is fine once something declares that seat, which is the case the parser
|
||||||
|
// could not have distinguished.
|
||||||
|
func TestAClaimOnASeatAnotherModuleDeclaresIsAccepted(t *testing.T) {
|
||||||
|
claimant, err := ParseManifest(claimed(`[{"name":"the-anything","scope":"node"}]`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
declarer := Manifest{Module: "someone", DefinesSeats: []SeatDeclaration{
|
||||||
|
{Name: "the-anything", Scope: ScopeNode, Accepts: []string{"work"}},
|
||||||
|
}}
|
||||||
|
if problems := CatalogueProblems(Shelf{claimant.Module: claimant, "someone": declarer}); len(problems) != 0 {
|
||||||
|
t.Fatalf("a claim on a declared seat was refused: %v", problems)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A module may define its own seat and claim it — the mesh enforces exclusivity without knowing
|
||||||
|
// what it means (novox/hq ADR 0121). But it may not define one in the mesh's own namespace.
|
||||||
|
func TestAModuleDefinesAndClaimsItsOwnSeat(t *testing.T) {
|
||||||
|
ok := []byte(`{"module":"showcase","version":"1","seats":[{"name":"the-showcase","scope":"node"}],` +
|
||||||
|
`"claims":[{"name":"the-showcase","scope":"node"}]}`)
|
||||||
|
if _, err := ParseManifest(ok); err != nil {
|
||||||
|
t.Fatalf("a module could not define and claim its own seat: %v", err)
|
||||||
|
}
|
||||||
|
// Claiming a name nobody defines is still refused — but **at registration, not here**: with
|
||||||
|
// seats declared by modules, a claim on a seat *another* module declares is good, and the
|
||||||
|
// parser cannot tell that from an invented name. See
|
||||||
|
// TestAClaimOnASeatNobodyDeclaresIsRefusedAtRegistration.
|
||||||
|
claimant, err := ParseManifest(claimed(`[{"name":"the-anything","scope":"node"}]`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("the parser judged a claim it cannot judge alone: %v", err)
|
||||||
|
}
|
||||||
|
if len(CatalogueProblems(Shelf{claimant.Module: claimant})) == 0 {
|
||||||
|
t.Fatal("a module claimed a seat nobody defines")
|
||||||
|
}
|
||||||
|
// A module may not carve its seat out of the mesh's own namespace.
|
||||||
|
bad := []byte(`{"module":"x","version":"1","seats":[{"name":"node-mine","scope":"node"}],` +
|
||||||
|
`"claims":[{"name":"node-mine","scope":"node"}]}`)
|
||||||
|
if _, err := ParseManifest(bad); err == nil || !strings.Contains(err.Error(), "own namespace") {
|
||||||
|
t.Fatalf("a module defined a seat in the mesh's namespace and was not refused: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -206,3 +276,43 @@ func TestTheHolderIsTheModuleNotTheMachine(t *testing.T) {
|
|||||||
t.Fatalf("the holder was not told apart from a neighbour: %+v", holder)
|
t.Fatalf("the holder was not told apart from a neighbour: %+v", holder)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The working set is loaded from the store, and an empty load never erases it (novox/hq ADR 0122).
|
||||||
|
func TestUseSeatsReplacesTheSetButNeverEmptiesIt(t *testing.T) {
|
||||||
|
before := Seats()
|
||||||
|
defer UseSeats(DefaultSeats()) // restore for other tests, whatever this leaves it as
|
||||||
|
|
||||||
|
// An empty load (store not seeded, or unreadable) leaves the compiled defaults in force.
|
||||||
|
UseSeats(nil)
|
||||||
|
if len(Seats()) != len(before) {
|
||||||
|
t.Fatalf("an empty load changed the set from %d to %d seats", len(before), len(Seats()))
|
||||||
|
}
|
||||||
|
// A non-empty load replaces it — this is how a rename in the store reaches the lookups.
|
||||||
|
UseSeats([]Seat{{Name: "node-firewall", Scope: ScopeNode, Decision: "novox/hq ADR 0122"}})
|
||||||
|
if _, known := SeatNamed("node-firewall"); !known {
|
||||||
|
t.Fatal("the loaded set did not replace the working set")
|
||||||
|
}
|
||||||
|
if len(Seats()) != 1 {
|
||||||
|
t.Fatalf("the working set is %d seats, not the one that was loaded", len(Seats()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A former name resolves to the seat it was renamed from (novox/hq ADR 0122), so a manifest's claim
|
||||||
|
// and a held record naming the old name break nothing after a rename.
|
||||||
|
func TestAFormerNameResolvesAfterARename(t *testing.T) {
|
||||||
|
defer func() { UseSeats(DefaultSeats()); UseAliases(nil) }()
|
||||||
|
UseSeats([]Seat{{Name: "git", Scope: ScopeMesh, Delivers: "git", Decision: "novox/hq ADR 0121"}})
|
||||||
|
UseAliases(map[string]string{"git": "git"})
|
||||||
|
|
||||||
|
// The old name resolves to the renamed seat.
|
||||||
|
if s, ok := SeatNamed("git"); !ok || s.Name != "git" {
|
||||||
|
t.Fatalf("the former name did not resolve to the renamed seat: %+v ok=%v", s, ok)
|
||||||
|
}
|
||||||
|
// And a holder recorded under the old name is still found for the provision the seat delivers.
|
||||||
|
providers := []Provider{{Node: "anchor", At: "anchor.internal", Module: "gitea"}}
|
||||||
|
held := []Held{{Claim: "git", Scope: ScopeMesh, Node: "anchor", Module: "gitea"}}
|
||||||
|
holder, found := HolderAmong("git", providers, held)
|
||||||
|
if !found || holder.Module != "gitea" {
|
||||||
|
t.Fatalf("the holder recorded under the former name was not matched: %+v found=%v", holder, found)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,46 @@
|
|||||||
|
package catalogue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The ssh-client module, composed as a machine receives it (novox/hq to-be 29): every other node's
|
||||||
|
// Host block written into a marked region of the operator's ~/.ssh/config, owned by the account,
|
||||||
|
// with ~/.ssh created 0700 — the operator's own config kept.
|
||||||
|
func TestSSHClientOwnsTheOperatorsSSHConfig(t *testing.T) {
|
||||||
|
shelf := shelf(catalogueManifest(t, "ssh-client"))
|
||||||
|
got, err := Resolve(shelf, []string{"ssh-client"},
|
||||||
|
Node{Name: "homer", At: "homer.internal", Account: "jo"}, World{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
names := map[string]string{"homer.internal": "10.10.0.1", "marge.internal": "10.10.0.2"}
|
||||||
|
out, err := got.Declaration(Rendering{
|
||||||
|
Names: names, Machines: names, Accounts: map[string]string{"homer": "jo", "marge": "jo"},
|
||||||
|
Suffix: "internal",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
by := map[string]map[string]any{}
|
||||||
|
for _, r := range out {
|
||||||
|
by[r["id"].(string)] = r
|
||||||
|
}
|
||||||
|
|
||||||
|
dir := by["ssh-client.ssh-dir"]
|
||||||
|
if dir == nil || dir["path"] != "/home/jo/.ssh" || dir["owner"] != "jo" || dir["mode"] != "0700" {
|
||||||
|
t.Fatalf("~/.ssh is not created 0700 owned by the account: %v", dir)
|
||||||
|
}
|
||||||
|
cfg := by["ssh-client.fact-ssh-config"]
|
||||||
|
if cfg == nil || cfg["path"] != "/home/jo/.ssh/config" || cfg["owner"] != "jo" || cfg["into"] != "block" {
|
||||||
|
t.Fatalf("the ssh config is not written into the operator's ~/.ssh/config as a region: %v", cfg)
|
||||||
|
}
|
||||||
|
body := cfg["content"].(string)
|
||||||
|
if !strings.Contains(body, "Host marge marge.internal") || !strings.Contains(body, "User jo") {
|
||||||
|
t.Fatalf("the config does not name the peer node and its account:\n%s", body)
|
||||||
|
}
|
||||||
|
if strings.Contains(body, "Host homer ") {
|
||||||
|
t.Fatalf("the config names the machine itself, not only its peers:\n%s", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,178 @@
|
|||||||
|
package inventory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What the bus's user list is derived from, read out of the mesh's records.
|
||||||
|
//
|
||||||
|
// The deriving itself is pure and lives in the broker package; this is the reading, and it is kept
|
||||||
|
// apart for the reason that package keeps its own types: a permission must be a function of what a
|
||||||
|
// module declared, and a query that decided anything would be a second place authority came from.
|
||||||
|
|
||||||
|
// BusRecords is every fact the composer needs about who may reach the bus.
|
||||||
|
//
|
||||||
|
// **A module's authority comes from the manifest, not from the assignment.** The assignment says
|
||||||
|
// *where* it runs; what it may say is in what it declared, so the two are read together and the
|
||||||
|
// manifest is the one that decides.
|
||||||
|
func (i *Inventory) BusRecords(ctx context.Context) (broker.Records, error) {
|
||||||
|
nodes, err := i.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return broker.Records{}, fmt.Errorf("cannot read the mesh's machines: %w", err)
|
||||||
|
}
|
||||||
|
declared, err := i.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return broker.Records{}, fmt.Errorf("cannot read the catalogue: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every seat any module declares, by name, so a module's claim can be resolved to the protocol
|
||||||
|
// that seat promises. **Across the whole catalogue, not one manifest**: a seat is declared by
|
||||||
|
// one module and held by another, which is the whole reason a seat exists (ADR 0118).
|
||||||
|
seats := map[string]catalogue.SeatDeclaration{}
|
||||||
|
for _, m := range declared {
|
||||||
|
for _, s := range m.DefinesSeats {
|
||||||
|
seats[s.Name] = s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// And the mesh's own, which carry protocol too (novox/hq ADR 0121). Added after the modules'
|
||||||
|
// rather than before, because a `mesh-*` name is the mesh's and registration refuses a module
|
||||||
|
// declaring one — so this cannot be shadowed, and if it ever were, the mesh's own would win.
|
||||||
|
for _, own := range catalogue.SeatsWithAProtocol() {
|
||||||
|
seats[own.Name] = catalogue.SeatDeclaration{
|
||||||
|
Name: own.Name, Scope: own.Scope,
|
||||||
|
Accepts: own.Accepts, Emits: own.Emits, Serves: own.Serves,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
out := broker.Records{Assigned: map[string][]broker.Declared{}, People: map[string][]string{}}
|
||||||
|
for _, n := range nodes {
|
||||||
|
out.Nodes = append(out.Nodes, n.Name)
|
||||||
|
modules, err := i.Assigned(ctx, n.Name)
|
||||||
|
if err != nil {
|
||||||
|
return broker.Records{}, fmt.Errorf("cannot read what %s runs: %w", n.Name, err)
|
||||||
|
}
|
||||||
|
for _, module := range modules {
|
||||||
|
m, known := declared[module]
|
||||||
|
if !known {
|
||||||
|
// Assigned and not in the catalogue. Said rather than composed with no authority:
|
||||||
|
// a user with an empty permission list is a module that starts, connects, and is
|
||||||
|
// refused by the server on its first publish — an authorisation error that says
|
||||||
|
// nothing about a missing manifest.
|
||||||
|
//
|
||||||
|
// **The catalogue refuses to forget an assigned module, so this is the second line
|
||||||
|
// and not the first.** It earns its place there anyway: relying on another
|
||||||
|
// package's invariant is how a rule ends up enforced by nothing.
|
||||||
|
return broker.Records{}, fmt.Errorf(
|
||||||
|
"%s is assigned to %s and is not in the catalogue, so what it may say cannot "+
|
||||||
|
"be derived", module, n.Name)
|
||||||
|
}
|
||||||
|
out.Assigned[n.Name] = append(out.Assigned[n.Name], declaredFor(m, seats))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
enrolling, err := i.NodesWithALiveToken(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return broker.Records{}, err
|
||||||
|
}
|
||||||
|
out.Enrolling = enrolling
|
||||||
|
|
||||||
|
people, err := i.People(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return broker.Records{}, err
|
||||||
|
}
|
||||||
|
for _, p := range people {
|
||||||
|
out.People[p.Name] = p.Invokes
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// declaredFor is one module's manifest as the composer needs it: what it says about itself, and the
|
||||||
|
// protocol of every seat it holds or uses.
|
||||||
|
func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaration) broker.Declared {
|
||||||
|
// A consumed name is a module's event unless it names a seat, and only somebody holding the seat
|
||||||
|
// set can tell (novox/hq ADR 0121). Split here, because the composer cannot look at a name and
|
||||||
|
// know — and a role's event read as a module's is a subscription to a namespace nobody owns.
|
||||||
|
var fromModules []string
|
||||||
|
var watches []broker.Seat
|
||||||
|
for _, c := range m.Consumes {
|
||||||
|
emitter, event, named := strings.Cut(c, ".")
|
||||||
|
if named {
|
||||||
|
if s, isASeat := seats[emitter]; isASeat {
|
||||||
|
watches = append(watches, broker.Seat{Name: s.Name, Emits: []string{event}})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fromModules = append(fromModules, c)
|
||||||
|
}
|
||||||
|
|
||||||
|
d := broker.Declared{
|
||||||
|
Module: m.Module,
|
||||||
|
Emits: m.Emits,
|
||||||
|
Consumes: fromModules,
|
||||||
|
Watches: watches,
|
||||||
|
// The tools it answers, which is `tools` and not `serves`: the manifest's `serves` is the
|
||||||
|
// facts a consumer needs to reach a provision, a different meaning under a similar word.
|
||||||
|
Serves: m.Tools,
|
||||||
|
}
|
||||||
|
for _, c := range m.Claims {
|
||||||
|
// Every seat with a protocol, the mesh's own included. One that says only who does a job is
|
||||||
|
// not here and grants nothing, which is most of them.
|
||||||
|
if s, hasAProtocol := seats[c.Name]; hasAProtocol {
|
||||||
|
d.Holds = append(d.Holds, asSeat(s))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, name := range m.Uses {
|
||||||
|
if s, declaredSomewhere := seats[name]; declaredSomewhere {
|
||||||
|
d.Uses = append(d.Uses, asSeat(s))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
|
||||||
|
func asSeat(s catalogue.SeatDeclaration) broker.Seat {
|
||||||
|
return broker.Seat{Name: s.Name, Accepts: s.Accepts, Emits: s.Emits, Serves: s.Serves}
|
||||||
|
}
|
||||||
|
|
||||||
|
// MeshSeats are the mesh's own seats that carry a protocol, as the bus needs them: what to make a work
|
||||||
|
// queue for, and whose holder gets a worker on it (novox/hq ADR 0121).
|
||||||
|
func MeshSeats() []broker.DeclaredSeat {
|
||||||
|
var out []broker.DeclaredSeat
|
||||||
|
for _, s := range catalogue.SeatsWithAProtocol() {
|
||||||
|
out = append(out, broker.DeclaredSeat{
|
||||||
|
Name: s.Name, Accepts: s.Accepts, Emits: s.Emits, Serves: s.Serves,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// NodesWithALiveToken is every machine holding a token that could still be presented — issued, not
|
||||||
|
// expired, not redeemed.
|
||||||
|
//
|
||||||
|
// **One enrolment user per such token** (design 25 §6): the inbox an answer goes to is scoped to the
|
||||||
|
// token, because an answer carries that machine's credentials sealed to it and a shared inbox is one
|
||||||
|
// machine able to read another's.
|
||||||
|
func (i *Inventory) NodesWithALiveToken(ctx context.Context) ([]string, error) {
|
||||||
|
rows, err := i.store.Pool().Query(ctx,
|
||||||
|
`select distinct n.name
|
||||||
|
from enrolment_token t join node n on n.id = t.node
|
||||||
|
where t.redeemed is null and t.expires > now()
|
||||||
|
order by n.name`)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot read which machines hold a live token: %w", err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var out []string
|
||||||
|
for rows.Next() {
|
||||||
|
var name string
|
||||||
|
if err := rows.Scan(&name); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out = append(out, name)
|
||||||
|
}
|
||||||
|
return out, rows.Err()
|
||||||
|
}
|
||||||
@@ -0,0 +1,164 @@
|
|||||||
|
package inventory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Reading the bus's user list out of the mesh's records, against a real store.
|
||||||
|
//
|
||||||
|
// What each of these is about is a user that would be **missing or wrong in a way nothing reports**:
|
||||||
|
// the server reads whatever file it is given, and a module whose user is absent fails on its first
|
||||||
|
// publish with an authorisation error that says nothing about a missing assignment.
|
||||||
|
|
||||||
|
func aMeshWith(t *testing.T, manifests ...catalogue.Manifest) (*Inventory, context.Context) {
|
||||||
|
t.Helper()
|
||||||
|
inv := ForTest(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
for _, m := range manifests {
|
||||||
|
if err := inv.RegisterModule(ctx, m, Source{Repository: "/r"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return inv, ctx
|
||||||
|
}
|
||||||
|
|
||||||
|
func theSeatDeclarer() catalogue.Manifest {
|
||||||
|
return catalogue.Manifest{
|
||||||
|
Module: "telegram", Version: "1",
|
||||||
|
DefinesSeats: []catalogue.SeatDeclaration{{
|
||||||
|
Name: "telegram-sender", Accepts: []string{"send"}, Emits: []string{"delivered"},
|
||||||
|
}},
|
||||||
|
Claims: []catalogue.Claim{{Name: "telegram-sender", Scope: catalogue.ScopeMesh}},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A module assigned to a machine becomes a user with the authority its manifest declared — and the
|
||||||
|
// protocol of a seat declared by a *different* module, which is the whole reason a seat exists.
|
||||||
|
func TestAnAssignedModuleBecomesAUserWithWhatItDeclared(t *testing.T) {
|
||||||
|
shop := catalogue.Manifest{
|
||||||
|
Module: "shop", Version: "1",
|
||||||
|
Emits: []string{"order.placed"}, Tools: []string{"price"},
|
||||||
|
Uses: []string{"telegram-sender"},
|
||||||
|
}
|
||||||
|
inv, ctx := aMeshWith(t, theSeatDeclarer(), shop)
|
||||||
|
if _, err := inv.AddNode(ctx, "one"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := inv.Assign(ctx, "one", "shop"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
records, err := inv.BusRecords(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
on := records.Assigned["one"]
|
||||||
|
if len(on) != 1 || on[0].Module != "shop" {
|
||||||
|
t.Fatalf("the machine's modules read as %+v", on)
|
||||||
|
}
|
||||||
|
if len(on[0].Uses) != 1 || on[0].Uses[0].Accepts[0] != "send" {
|
||||||
|
t.Fatalf("the seat it uses carries no protocol: %+v — so it would be granted nothing on a "+
|
||||||
|
"seat it was assigned to send to", on[0].Uses)
|
||||||
|
}
|
||||||
|
if len(on[0].Serves) != 1 || on[0].Serves[0] != "price" {
|
||||||
|
t.Fatalf("its tools read as %v, and a module that cannot subscribe its own tool subject "+
|
||||||
|
"serves nothing", on[0].Serves)
|
||||||
|
}
|
||||||
|
|
||||||
|
// And it derives into a user the server would accept.
|
||||||
|
users, err := broker.Users(records)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var found bool
|
||||||
|
for _, u := range users {
|
||||||
|
if u.Username() != "one.shop" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
found = true
|
||||||
|
perms, err := broker.PermissionsFor(u)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !granted(perms.Publish, "mesh.mod.shop.event.order.placed") ||
|
||||||
|
!granted(perms.Publish, "mesh.seat.telegram-sender.accept.send") ||
|
||||||
|
!granted(perms.Subscribe, "mesh.mod.shop.tool.price") {
|
||||||
|
t.Fatalf("one.shop's authority is not what it declared: %+v", perms)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatal("no user was derived for the assigned module")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A machine holding a live token gets an enrolment user; one whose token is spent or expired does
|
||||||
|
// not. **An enrolment user outliving its token is a right to join that nobody issued.**
|
||||||
|
func TestOnlyAMachineWithALiveTokenHasAnEnrolmentUser(t *testing.T) {
|
||||||
|
inv, ctx := aMeshWith(t)
|
||||||
|
for _, name := range []string{"live", "expired", "none"} {
|
||||||
|
if _, err := inv.AddNode(ctx, name); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := inv.IssueToken(ctx, "live", time.Hour); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Briefly, then waited out: a token with no lifetime is refused at issue, which is the right
|
||||||
|
// refusal and leaves this as the way to have an expired one.
|
||||||
|
if _, err := inv.IssueToken(ctx, "expired", 10*time.Millisecond); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
|
||||||
|
records, err := inv.BusRecords(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if strings.Join(records.Enrolling, ",") != "live" {
|
||||||
|
t.Fatalf("machines with a live token read as %v", records.Enrolling)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A module assigned and absent from the catalogue is refused rather than composed with no authority.
|
||||||
|
//
|
||||||
|
// **The catalogue refuses to forget an assigned module, so this is the second line and not the
|
||||||
|
// first** — and it earns its place there: relying on another package's invariant is how a rule ends
|
||||||
|
// up enforced by nothing. Checked against the derivation directly, because the situation cannot be
|
||||||
|
// reached through the store.
|
||||||
|
func TestAnAssignmentWithNoManifestDerivesNoAuthority(t *testing.T) {
|
||||||
|
// What BusRecords would have produced had it composed a ghost: a module with nothing declared.
|
||||||
|
users, err := broker.Users(broker.Records{
|
||||||
|
Nodes: []string{"one"},
|
||||||
|
Assigned: map[string][]broker.Declared{"one": {{Module: "ghost"}}},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
perms, err := broker.PermissionsFor(users[len(users)-1])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Its inbox and its ack subject, and nothing it could say. That is a module which starts,
|
||||||
|
// connects, and is refused by the server on its first publish — an authorisation error that
|
||||||
|
// says nothing about a missing manifest, which is why BusRecords names it instead.
|
||||||
|
for _, p := range perms.Publish {
|
||||||
|
if strings.HasPrefix(p, "mesh.mod.ghost.event.") {
|
||||||
|
t.Fatalf("a module with no manifest was granted %s", p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func granted(all []string, one string) bool {
|
||||||
|
for _, s := range all {
|
||||||
|
if s == one {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -0,0 +1,232 @@
|
|||||||
|
package inventory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/base64"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
|
"golang.org/x/crypto/bcrypt"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The bus's own users, as records.
|
||||||
|
//
|
||||||
|
// **Only the credential is kept here.** A user's *authority* is derived from what its module
|
||||||
|
// declares, every time the file is written (novox/hq ADR 0043) — a stored copy of a permission list
|
||||||
|
// would be a second account of a user's authority, able to disagree with the first, and the
|
||||||
|
// disagreement would be invisible until somebody compared a composed file with a manifest.
|
||||||
|
//
|
||||||
|
// What cannot be derived is the password, and on the bus being built it has to outlive its own
|
||||||
|
// minting: the whole user list is one file, rewritten whenever any of it changes, so a person's
|
||||||
|
// access change would blank every module's password if the mesh kept nothing (design 25 §4, and the
|
||||||
|
// migration beside this).
|
||||||
|
|
||||||
|
// BusUser is one user of the bus, as the mesh records it.
|
||||||
|
type BusUser struct {
|
||||||
|
Username string
|
||||||
|
Kind string
|
||||||
|
Node string
|
||||||
|
Module string
|
||||||
|
// PasswordHash is what the composed file carries. The plaintext is returned once, by Mint, and
|
||||||
|
// then exists only where it was sealed.
|
||||||
|
PasswordHash string
|
||||||
|
}
|
||||||
|
|
||||||
|
// The kinds of bus user the mesh records. The same words the composer uses, so a row and a
|
||||||
|
// principal do not need a translation table between them.
|
||||||
|
const (
|
||||||
|
BusController = "controller"
|
||||||
|
BusNode = "node"
|
||||||
|
BusModule = "module"
|
||||||
|
BusEnrolment = "enrolment"
|
||||||
|
BusPerson = "person"
|
||||||
|
)
|
||||||
|
|
||||||
|
// MintBusPassword makes a bus password and records its hash under a username, replacing whatever was
|
||||||
|
// there, and returns the plaintext **once**.
|
||||||
|
//
|
||||||
|
// **Once is the whole contract.** The caller seals it to whoever will use it — into an enrolment
|
||||||
|
// reply, into a module's sealed environment — and the mesh keeps only the hash, so a credential is
|
||||||
|
// never recoverable from the store. A caller that loses it must mint again, which is a rotation and
|
||||||
|
// is meant to feel like one.
|
||||||
|
func (i *Inventory) MintBusPassword(ctx context.Context, u BusUser) (string, error) {
|
||||||
|
if u.Username == "" || u.Kind == "" {
|
||||||
|
return "", errors.New("a bus user needs a username and a kind")
|
||||||
|
}
|
||||||
|
raw := make([]byte, 32)
|
||||||
|
if _, err := rand.Read(raw); err != nil {
|
||||||
|
return "", fmt.Errorf("cannot generate a bus password: %w", err)
|
||||||
|
}
|
||||||
|
password := base64.RawURLEncoding.EncodeToString(raw)
|
||||||
|
|
||||||
|
// The cost the server will pay on every connection. Left at the library's default rather than
|
||||||
|
// raised: a node reconnecting after a network blip pays it, and the mesh's own links reconnect
|
||||||
|
// far more often than a person logs in anywhere.
|
||||||
|
hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("cannot hash a bus password: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := i.store.Pool().Exec(ctx,
|
||||||
|
`insert into bus_user (username, kind, node, module, password_hash)
|
||||||
|
values ($1, $2, $3, $4, $5)
|
||||||
|
on conflict (username) do update
|
||||||
|
set kind = excluded.kind, node = excluded.node, module = excluded.module,
|
||||||
|
password_hash = excluded.password_hash, minted_at = now()`,
|
||||||
|
u.Username, u.Kind, u.Node, u.Module, string(hash)); err != nil {
|
||||||
|
return "", fmt.Errorf("cannot record the bus user %s: %w", u.Username, err)
|
||||||
|
}
|
||||||
|
return password, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// BusUsers is every user the composed file should contain, by username.
|
||||||
|
//
|
||||||
|
// Returned as a map because the composer asks by username: the principals are derived from records
|
||||||
|
// elsewhere, and this is only what each one's password is. A principal with no row here has no
|
||||||
|
// password, and the composer refuses it rather than writing a user anybody is.
|
||||||
|
func (i *Inventory) BusUsers(ctx context.Context) (map[string]BusUser, error) {
|
||||||
|
rows, err := i.store.Pool().Query(ctx,
|
||||||
|
`select username, kind, node, module, password_hash from bus_user order by username`)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
out := map[string]BusUser{}
|
||||||
|
for rows.Next() {
|
||||||
|
var u BusUser
|
||||||
|
if err := rows.Scan(&u.Username, &u.Kind, &u.Node, &u.Module, &u.PasswordHash); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out[u.Username] = u
|
||||||
|
}
|
||||||
|
return out, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// BusUserHash is one user's hash, or false when the mesh has never minted one for it.
|
||||||
|
func (i *Inventory) BusUserHash(ctx context.Context, username string) (string, bool, error) {
|
||||||
|
var hash string
|
||||||
|
err := i.store.Pool().QueryRow(ctx,
|
||||||
|
`select password_hash from bus_user where username = $1`, username).Scan(&hash)
|
||||||
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
return "", false, nil
|
||||||
|
}
|
||||||
|
return hash, err == nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// ForgetBusUser removes one user, so the next composition does not contain it.
|
||||||
|
//
|
||||||
|
// **Removal is what makes revocation real here.** On a bus with a management call, deleting an
|
||||||
|
// account ends its connections; here the credential stops working when the file no longer names it,
|
||||||
|
// which is the next composition — so forgetting the row and composing are one act, and a caller
|
||||||
|
// that does the first without the second has revoked nothing.
|
||||||
|
func (i *Inventory) ForgetBusUser(ctx context.Context, username string) error {
|
||||||
|
_, err := i.store.Pool().Exec(ctx, `delete from bus_user where username = $1`, username)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// ForgetBusUsersOf removes every user belonging to one node — its host's, and every module assigned
|
||||||
|
// to it. What a forgotten node leaves behind on the bus is otherwise a set of credentials for a
|
||||||
|
// machine the mesh no longer knows.
|
||||||
|
func (i *Inventory) ForgetBusUsersOf(ctx context.Context, node string) error {
|
||||||
|
if node == "" {
|
||||||
|
return errors.New("forgetting the bus users of no node would forget every user that has none")
|
||||||
|
}
|
||||||
|
_, err := i.store.Pool().Exec(ctx, `delete from bus_user where node = $1`, node)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// SeedBusUser records a hash of a credential the mesh did not mint, so a composition contains it.
|
||||||
|
//
|
||||||
|
// **Genesis is the reason this exists.** The controller's own user is created before the controller
|
||||||
|
// runs — by the installer, at a well-known bootstrap password, the way the store's and the old bus's
|
||||||
|
// are (`postgres:bootstrap`, `guest:guest`). Nothing minted it, so nothing recorded a hash for it, and
|
||||||
|
// the controller's first composition would leave itself out of the very file it was writing: a bus
|
||||||
|
// nothing can connect to, produced by the thing connected to it.
|
||||||
|
//
|
||||||
|
// Idempotent, and it does not overwrite. A credential the mesh *did* mint is the one that counts, so
|
||||||
|
// once there is a row this does nothing — otherwise a restart would put the bootstrap password back
|
||||||
|
// over a rotated one.
|
||||||
|
func (i *Inventory) SeedBusUser(ctx context.Context, u BusUser, password string) error {
|
||||||
|
if u.Username == "" || u.Kind == "" || password == "" {
|
||||||
|
return errors.New("a bus user needs a username, a kind and the credential it is using")
|
||||||
|
}
|
||||||
|
hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot hash a bus password: %w", err)
|
||||||
|
}
|
||||||
|
_, err = i.store.Pool().Exec(ctx,
|
||||||
|
`insert into bus_user (username, kind, node, module, password_hash)
|
||||||
|
values ($1, $2, $3, $4, $5)
|
||||||
|
on conflict (username) do nothing`,
|
||||||
|
u.Username, u.Kind, u.Node, u.Module, string(hash))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// A person who may call the mesh's tools (novox/hq design 25 §7).
|
||||||
|
//
|
||||||
|
// **Their authority is a list of tools and nothing else.** Not a module: they hold no seat, nothing is
|
||||||
|
// addressed to them, nothing is delivered to them, and they have no consumer to acknowledge. What
|
||||||
|
// they have is permission to ask.
|
||||||
|
|
||||||
|
// Person is somebody who may reach the mesh's tools.
|
||||||
|
type Person struct {
|
||||||
|
Name string
|
||||||
|
// Invokes are the tools they may call, each `<module>.<tool>`, or the single entry `*` for an
|
||||||
|
// administrator.
|
||||||
|
Invokes []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordPerson adds somebody, or changes what they may call.
|
||||||
|
//
|
||||||
|
// Replacing rather than merging: what a person may call is stated in full, so a change that meant to
|
||||||
|
// remove a tool does remove it. A list that could only grow is a permission nobody can take back.
|
||||||
|
func (i *Inventory) RecordPerson(ctx context.Context, p Person) error {
|
||||||
|
if p.Name == "" {
|
||||||
|
return errors.New("a person needs a name: it becomes their user on the bus")
|
||||||
|
}
|
||||||
|
if len(p.Invokes) == 0 {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"%s may call nothing, so there is no reason for them to reach the mesh. Name the tools, "+
|
||||||
|
"or `*` for an administrator", p.Name)
|
||||||
|
}
|
||||||
|
_, err := i.store.Pool().Exec(ctx,
|
||||||
|
`insert into person (name, invokes) values ($1, $2)
|
||||||
|
on conflict (name) do update set invokes = excluded.invokes`,
|
||||||
|
p.Name, p.Invokes)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// People is everybody who may reach the mesh's tools.
|
||||||
|
func (i *Inventory) People(ctx context.Context) ([]Person, error) {
|
||||||
|
rows, err := i.store.Pool().Query(ctx, `select name, invokes from person order by name`)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var out []Person
|
||||||
|
for rows.Next() {
|
||||||
|
var p Person
|
||||||
|
if err := rows.Scan(&p.Name, &p.Invokes); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out = append(out, p)
|
||||||
|
}
|
||||||
|
return out, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// ForgetPerson removes somebody and the credential they were given.
|
||||||
|
//
|
||||||
|
// **Both, or neither is a revocation.** A person's row gone and their bus user left behind is a
|
||||||
|
// credential that still works and that nothing derives, which is the worst of both: it keeps working
|
||||||
|
// and nobody can explain why.
|
||||||
|
func (i *Inventory) ForgetPerson(ctx context.Context, name string) error {
|
||||||
|
if name == "" {
|
||||||
|
return errors.New("forgetting nobody would forget everybody")
|
||||||
|
}
|
||||||
|
if _, err := i.store.Pool().Exec(ctx, `delete from person where name = $1`, name); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return i.ForgetBusUser(ctx, "person."+name)
|
||||||
|
}
|
||||||
@@ -0,0 +1,160 @@
|
|||||||
|
package inventory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"golang.org/x/crypto/bcrypt"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The bus's users as records — against a real store, because what is being checked is that the
|
||||||
|
// column exists, the upsert behaves, and a plaintext is returned exactly once.
|
||||||
|
|
||||||
|
func aBusUser(module string) BusUser {
|
||||||
|
return BusUser{Username: "one." + module, Kind: BusModule, Node: "one", Module: module}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The plaintext comes back once and the store keeps only a hash that verifies against it. **A
|
||||||
|
// credential recoverable from the mesh's store is one whose blast radius is the store's**, so what
|
||||||
|
// is asserted is that the password is not in there.
|
||||||
|
func TestABusPasswordIsReturnedOnceAndOnlyItsHashIsKept(t *testing.T) {
|
||||||
|
inv := ForTest(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
password, err := inv.MintBusPassword(ctx, aBusUser("shop"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if password == "" {
|
||||||
|
t.Fatal("no password came back, so nothing can be sealed to the module")
|
||||||
|
}
|
||||||
|
|
||||||
|
hash, known, err := inv.BusUserHash(ctx, "one.shop")
|
||||||
|
if err != nil || !known {
|
||||||
|
t.Fatalf("the user was not recorded: %v %v", known, err)
|
||||||
|
}
|
||||||
|
if hash == password {
|
||||||
|
t.Fatal("the store holds the password itself")
|
||||||
|
}
|
||||||
|
if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(password)); err != nil {
|
||||||
|
t.Fatalf("the recorded hash does not verify the password it was made from: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Minting again replaces what was there rather than failing or adding a second row: that is a
|
||||||
|
// rotation, and the old credential stops working at the next composition.
|
||||||
|
func TestMintingAgainRotatesRatherThanAddsAUser(t *testing.T) {
|
||||||
|
inv := ForTest(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
first, err := inv.MintBusPassword(ctx, aBusUser("shop"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
second, err := inv.MintBusPassword(ctx, aBusUser("shop"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if first == second {
|
||||||
|
t.Fatal("minting twice produced the same password")
|
||||||
|
}
|
||||||
|
users, err := inv.BusUsers(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(users) != 1 {
|
||||||
|
t.Fatalf("%d users after two mints for one name", len(users))
|
||||||
|
}
|
||||||
|
hash := users["one.shop"].PasswordHash
|
||||||
|
if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(second)); err != nil {
|
||||||
|
t.Fatal("the kept hash is not the newest password's")
|
||||||
|
}
|
||||||
|
if bcrypt.CompareHashAndPassword([]byte(hash), []byte(first)) == nil {
|
||||||
|
t.Fatal("the previous password still verifies, so a rotation revoked nothing")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Forgetting a node takes every credential that belonged to it — its host's and every module
|
||||||
|
// assigned to it. What a forgotten node leaves behind otherwise is a working set of credentials for
|
||||||
|
// a machine the mesh no longer knows.
|
||||||
|
func TestForgettingANodeTakesItsBusUsersWithIt(t *testing.T) {
|
||||||
|
inv := ForTest(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
for _, u := range []BusUser{
|
||||||
|
{Username: "node.one", Kind: BusNode, Node: "one"},
|
||||||
|
aBusUser("shop"),
|
||||||
|
{Username: "node.two", Kind: BusNode, Node: "two"},
|
||||||
|
{Username: "controller", Kind: BusController},
|
||||||
|
} {
|
||||||
|
if _, err := inv.MintBusPassword(ctx, u); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := inv.ForgetBusUsersOf(ctx, "one"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
users, err := inv.BusUsers(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, still := users["node.one"]; still {
|
||||||
|
t.Fatal("a forgotten node's host credential still works")
|
||||||
|
}
|
||||||
|
if _, still := users["one.shop"]; still {
|
||||||
|
t.Fatal("a module on a forgotten node still has a credential")
|
||||||
|
}
|
||||||
|
// And nothing else went with it: the controller has no node, and another machine's user is
|
||||||
|
// another machine's.
|
||||||
|
for _, kept := range []string{"node.two", "controller"} {
|
||||||
|
if _, ok := users[kept]; !ok {
|
||||||
|
t.Fatalf("%s was removed with another node's users", kept)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Forgetting the users of no node would forget every user that has none — the controller and every
|
||||||
|
// person — so it is refused rather than run.
|
||||||
|
func TestForgettingTheUsersOfNoNodeIsRefused(t *testing.T) {
|
||||||
|
inv := ForTest(t)
|
||||||
|
if err := inv.ForgetBusUsersOf(context.Background(), ""); err == nil {
|
||||||
|
t.Fatal("forgetting the bus users of no node was allowed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The controller's own user is created by the installer, so the mesh has to be able to record a
|
||||||
|
// credential it did not mint — or the first composition leaves the writer out of the file it writes.
|
||||||
|
func TestACredentialTheMeshDidNotMintIsRecordedOnceAndNotOverwritten(t *testing.T) {
|
||||||
|
inv := ForTest(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
if err := inv.SeedBusUser(ctx, BusUser{Username: "controller", Kind: BusController},
|
||||||
|
"bootstrap"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
hash, known, err := inv.BusUserHash(ctx, "controller")
|
||||||
|
if err != nil || !known {
|
||||||
|
t.Fatalf("the credential was not recorded: %v %v", known, err)
|
||||||
|
}
|
||||||
|
if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte("bootstrap")); err != nil {
|
||||||
|
t.Fatalf("what was recorded does not verify the credential given: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Minted since, then seeded again — which is what a restart does. The rotation must stand, or
|
||||||
|
// every restart would put the bootstrap password back over it.
|
||||||
|
minted, err := inv.MintBusPassword(ctx, BusUser{Username: "controller", Kind: BusController})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.SeedBusUser(ctx, BusUser{Username: "controller", Kind: BusController},
|
||||||
|
"bootstrap"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
hash, _, err = inv.BusUserHash(ctx, "controller")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(minted)); err != nil {
|
||||||
|
t.Fatal("a restart put the bootstrap credential back over a rotated one")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
-- The seats are data the control plane owns, not a slice compiled into it (novox/hq ADR 0122).
|
||||||
|
--
|
||||||
|
-- Until this, the closed set 0110 defines lived only as a Go slice, referenced by name everywhere,
|
||||||
|
-- so renaming a seat or adding one meant a controller rebuild and a mesh-wide, freeze-prone deploy.
|
||||||
|
-- The set is now a table: one row per seat, seeded from the binary's defaults the first time the
|
||||||
|
-- control plane comes up, and thereafter the live copy the control plane reads and an operator can
|
||||||
|
-- change. A rename becomes an update here rather than a release.
|
||||||
|
--
|
||||||
|
-- The name is the key for now, because claims and held records still reference a seat by name; the
|
||||||
|
-- move to a stable id that a rename does not touch is the next step (ADR 0122). `delivers` is empty
|
||||||
|
-- for a seat that answers for no provision, matching the compiled default.
|
||||||
|
create table seat (
|
||||||
|
name text primary key,
|
||||||
|
scope text not null,
|
||||||
|
delivers text not null default '',
|
||||||
|
decided text not null
|
||||||
|
);
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
-- A seat keeps its former names, so a rename breaks nothing (novox/hq ADR 0122).
|
||||||
|
--
|
||||||
|
-- Phase 1 made the seat set data, but a rename still broke every reference to the old name — a
|
||||||
|
-- manifest's claim, a held record, the git-seat lookup — because they name the seat and the name
|
||||||
|
-- had changed. This is the stable identity ADR 0122 asked for, realised the simple way: a seat's
|
||||||
|
-- canonical name changes, and its old name becomes an alias that resolves to it forever. Nothing
|
||||||
|
-- downstream has to change — a manifest goes on claiming the old name, the build machine goes on
|
||||||
|
-- validating it — and a rename is one operation: set the new name, remember the old.
|
||||||
|
create table seat_alias (
|
||||||
|
alias text primary key, -- a former name of a seat
|
||||||
|
seat text not null -- the seat's current canonical name it resolves to
|
||||||
|
);
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
-- A node has an operator account: the human login on it (novox/hq to-be 29).
|
||||||
|
--
|
||||||
|
-- The mesh modelled the machine but not the person on it — `jochens` on novox, `ace` on ace,
|
||||||
|
-- `jochen` on shanks and g14. That name decides who a file under a home is owned by and which
|
||||||
|
-- account `ssh <node>` logs in as; it was silently lost when the predecessor's per-node `user:`
|
||||||
|
-- was not carried over, and `ssh ace` failed to `ace` because nothing here said so.
|
||||||
|
--
|
||||||
|
-- Empty rather than null and defaulted, because "no operator account known yet" is a real state
|
||||||
|
-- (a freshly enrolled machine, a headless box). The home is stored too rather than always assumed
|
||||||
|
-- to be /home/<account>, because root's is /root and a machine may put a home elsewhere; empty
|
||||||
|
-- means "derive it" (/root for root, /home/<account> otherwise), so the common case needs no entry.
|
||||||
|
alter table node add column account text not null default '';
|
||||||
|
alter table node add column account_home text not null default '';
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
-- Every bus user's password hash, because the file has to be written again.
|
||||||
|
--
|
||||||
|
-- novox/hq design 25 §4, task 1.7. On the bus the mesh runs on today an account is created by a
|
||||||
|
-- management call: the mesh mints a password, hands it over, seals the plaintext to whoever will
|
||||||
|
-- use it, and keeps nothing. That works because the broker remembers.
|
||||||
|
--
|
||||||
|
-- The bus being built has no management call — its users are a file the controller composes, and
|
||||||
|
-- **the whole file is written every time any of it changes**. So the first person's access change
|
||||||
|
-- would silently blank every module's password. The hash has to outlive its own minting, which is
|
||||||
|
-- state the mesh did not need before and does now.
|
||||||
|
--
|
||||||
|
-- Keyed by username, because the username is exactly what the composed file needs and what a
|
||||||
|
-- principal derives from its own identity. Nothing else about the user is here: **permissions are
|
||||||
|
-- not stored.** They are derived from what each module declares, every time the file is written
|
||||||
|
-- (ADR 0043) — a stored copy would be a second account of a user's authority, able to disagree
|
||||||
|
-- with the first, and the disagreement would be invisible until somebody compared a file with a
|
||||||
|
-- manifest.
|
||||||
|
--
|
||||||
|
-- The hash and not the password. A file on a node's disk holds the hash, and so does this: a
|
||||||
|
-- credential recoverable from the mesh's store is one whose blast radius is the store's.
|
||||||
|
create table bus_user (
|
||||||
|
username text primary key,
|
||||||
|
-- kind and what it names, so a user whose subject is gone can be found and removed: a module
|
||||||
|
-- unassigned, a node forgotten, a token spent. Recorded rather than parsed back out of the
|
||||||
|
-- username, because a name is for the server and a parser over it would be a second grammar.
|
||||||
|
kind text not null,
|
||||||
|
node text not null default '',
|
||||||
|
module text not null default '',
|
||||||
|
password_hash text not null,
|
||||||
|
minted_at timestamptz not null default now()
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Finding every user of one kind, and every user belonging to one node — which is what removing a
|
||||||
|
-- node, or composing after an assignment, asks.
|
||||||
|
create index bus_user_kind on bus_user (kind);
|
||||||
|
create index bus_user_node on bus_user (node) where node <> '';
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
-- A person who may call the mesh's tools from a workstation.
|
||||||
|
--
|
||||||
|
-- novox/hq design 25 §7. Everything else that reaches the bus is a machine or a module running on
|
||||||
|
-- one; this is the exception the mesh has always had informally — somebody at a terminal — and never
|
||||||
|
-- recorded. Until now "the operator" meant whoever held the keys, which is a role and not a record,
|
||||||
|
-- so nothing could say who may call what.
|
||||||
|
--
|
||||||
|
-- **The authority is a list of tools and nothing else.** A person is not a module: they hold no seat,
|
||||||
|
-- nothing is addressed to them, nothing is delivered to them, and they have no consumer to
|
||||||
|
-- acknowledge. What they have is permission to ask. That is why there is no scope column and no node
|
||||||
|
-- column — a person is not on a machine.
|
||||||
|
create table person (
|
||||||
|
name text primary key,
|
||||||
|
-- The tools this person may invoke, each `<module>.<tool>`, or the single entry `*` for an
|
||||||
|
-- administrator. Stored as given: the permission is derived from it at every composition, so a
|
||||||
|
-- normalised form here would be a second opinion about authority (novox/hq ADR 0043).
|
||||||
|
invokes text[] not null default '{}',
|
||||||
|
created timestamptz not null default now()
|
||||||
|
);
|
||||||
@@ -55,6 +55,29 @@ type Node struct {
|
|||||||
// AdoptedSince is when it last became so; zero for a converged node.
|
// AdoptedSince is when it last became so; zero for a converged node.
|
||||||
Adopted bool
|
Adopted bool
|
||||||
AdoptedSince time.Time
|
AdoptedSince time.Time
|
||||||
|
|
||||||
|
// Account is the operator's login on this machine — `jochens` on novox, `ace` on ace (novox/hq
|
||||||
|
// to-be 29). Empty when none is known yet. AccountHome is where that account's home is; empty
|
||||||
|
// means derive it (/root for root, /home/<account> otherwise), so the common case needs no
|
||||||
|
// entry. What decides who a file under a home is owned by, and which account `ssh <node>` uses.
|
||||||
|
Account string
|
||||||
|
AccountHome string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Home is the account's home directory, derived when not stored: /root for root, /home/<account>
|
||||||
|
// otherwise. Empty only when there is no account at all.
|
||||||
|
func (n Node) Home() string {
|
||||||
|
if n.AccountHome != "" {
|
||||||
|
return n.AccountHome
|
||||||
|
}
|
||||||
|
switch n.Account {
|
||||||
|
case "":
|
||||||
|
return ""
|
||||||
|
case "root":
|
||||||
|
return "/root"
|
||||||
|
default:
|
||||||
|
return "/home/" + n.Account
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Silent is how long since this node was last heard from, and whether it ever was.
|
// Silent is how long since this node was last heard from, and whether it ever was.
|
||||||
@@ -112,12 +135,13 @@ func (i *Inventory) AddNodeAs(ctx context.Context, name string, adopted bool) (N
|
|||||||
|
|
||||||
// nodeColumns and scanNode are the one reading of a node row, so every way of finding a node
|
// nodeColumns and scanNode are the one reading of a node row, so every way of finding a node
|
||||||
// says whether it is adopted.
|
// says whether it is adopted.
|
||||||
const nodeColumns = `id, name, created, last_seen, adopted, adopted_since`
|
const nodeColumns = `id, name, created, last_seen, adopted, adopted_since, account, account_home`
|
||||||
|
|
||||||
func scanNode(row pgx.Row) (Node, error) {
|
func scanNode(row pgx.Row) (Node, error) {
|
||||||
var n Node
|
var n Node
|
||||||
var seen, since *time.Time
|
var seen, since *time.Time
|
||||||
if err := row.Scan(&n.ID, &n.Name, &n.Created, &seen, &n.Adopted, &since); err != nil {
|
if err := row.Scan(&n.ID, &n.Name, &n.Created, &seen, &n.Adopted, &since,
|
||||||
|
&n.Account, &n.AccountHome); err != nil {
|
||||||
return Node{}, err
|
return Node{}, err
|
||||||
}
|
}
|
||||||
if seen != nil {
|
if seen != nil {
|
||||||
@@ -129,6 +153,21 @@ func scanNode(row pgx.Row) (Node, error) {
|
|||||||
return n, nil
|
return n, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetAccount records the operator account on a node — its human login — and optionally where that
|
||||||
|
// account's home is (novox/hq to-be 29). An empty home means the mesh derives it. Clearing the
|
||||||
|
// account (empty name) is allowed: a machine may stop having a known operator.
|
||||||
|
func (i *Inventory) SetAccount(ctx context.Context, node, account, home string) error {
|
||||||
|
tag, err := i.store.Pool().Exec(ctx,
|
||||||
|
`update node set account = $1, account_home = $2 where name = $3`, account, home, node)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if tag.RowsAffected() == 0 {
|
||||||
|
return fmt.Errorf("%w: %s", ErrNoSuchNode, node)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// Nodes are every node record, oldest first.
|
// Nodes are every node record, oldest first.
|
||||||
func (i *Inventory) Nodes(ctx context.Context) ([]Node, error) {
|
func (i *Inventory) Nodes(ctx context.Context) ([]Node, error) {
|
||||||
rows, err := i.store.Pool().Query(ctx,
|
rows, err := i.store.Pool().Query(ctx,
|
||||||
@@ -782,6 +821,25 @@ func (i *Inventory) RecordSent(ctx context.Context, node, digest string) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Outstanding is the digest of the declaration a machine was last sent, by its name, and empty
|
||||||
|
// for one that has never been sent anything.
|
||||||
|
//
|
||||||
|
// **By name rather than by id**, because the caller is the serving loop and what a node puts in a
|
||||||
|
// report is its name. Asked of one machine rather than read from Waiting's sweep, because it is
|
||||||
|
// asked per message: a report names the declaration it is about, and a report about one the mesh
|
||||||
|
// has already moved past is not acted on (design 25 §3).
|
||||||
|
func (i *Inventory) Outstanding(ctx context.Context, name string) (string, error) {
|
||||||
|
var sent string
|
||||||
|
err := i.store.Pool().QueryRow(ctx,
|
||||||
|
`select coalesce(sent, '') from node where name = $1`, name).Scan(&sent)
|
||||||
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
// Not an error worth carrying up: a report from a machine the mesh has no record of has
|
||||||
|
// nothing to be stale against, and whatever is wrong with it is the listener's to say.
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
return sent, err
|
||||||
|
}
|
||||||
|
|
||||||
// Waiting is every machine whose declaration has changed since it was last sent one.
|
// Waiting is every machine whose declaration has changed since it was last sent one.
|
||||||
//
|
//
|
||||||
// The caller works out what each machine should be now, because only it can — resolution is the
|
// The caller works out what each machine should be now, because only it can — resolution is the
|
||||||
|
|||||||
@@ -0,0 +1,120 @@
|
|||||||
|
package inventory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Somebody who may call the mesh's tools, and what the bus makes of them.
|
||||||
|
|
||||||
|
// A person's authority is a list of tools, and it becomes exactly that on the bus — nothing on
|
||||||
|
// control, nothing on nodes, nothing they could publish as a module.
|
||||||
|
func TestAPersonMayCallToolsAndNothingElse(t *testing.T) {
|
||||||
|
inv := ForTest(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
if err := inv.RecordPerson(ctx, Person{Name: "ada",
|
||||||
|
Invokes: []string{"mesh-catalog.catalog_tools"}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
records, err := inv.BusRecords(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := records.People["ada"]; len(got) != 1 || got[0] != "mesh-catalog.catalog_tools" {
|
||||||
|
t.Fatalf("ada may call %v", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
users, err := broker.Users(records)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var found bool
|
||||||
|
for _, u := range users {
|
||||||
|
if u.Username() != "person.ada" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
found = true
|
||||||
|
perms, err := broker.PermissionsFor(u)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(perms.Publish) != 1 || perms.Publish[0] != "mesh.mod.mesh-catalog.tool.catalog_tools" {
|
||||||
|
t.Errorf("ada may publish %v, which should be the one tool and nothing else", perms.Publish)
|
||||||
|
}
|
||||||
|
for _, s := range perms.Publish {
|
||||||
|
if strings.HasPrefix(s, "mesh.control") || strings.HasPrefix(s, "mesh.node") ||
|
||||||
|
strings.Contains(s, ".event.") {
|
||||||
|
t.Errorf("a person may publish %s — an event would let them claim a module said "+
|
||||||
|
"something, and control is not theirs", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if perms.AllowResponses {
|
||||||
|
t.Error("a person may answer a request, which is impersonating a module on a bus where " +
|
||||||
|
"anyone may serve a tool")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatal("no bus user was derived for a recorded person")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stating what somebody may call replaces what was there. A list that could only grow is a permission
|
||||||
|
// nobody can take back.
|
||||||
|
func TestChangingWhatAPersonMayCallRemovesWhatIsNotNamed(t *testing.T) {
|
||||||
|
inv := ForTest(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
if err := inv.RecordPerson(ctx, Person{Name: "ada", Invokes: []string{"a.one", "b.two"}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.RecordPerson(ctx, Person{Name: "ada", Invokes: []string{"a.one"}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
people, err := inv.People(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(people) != 1 || len(people[0].Invokes) != 1 || people[0].Invokes[0] != "a.one" {
|
||||||
|
t.Fatalf("ada may call %v; the removed tool is still there", people)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Somebody who may call nothing is refused: there is no reason for them to reach the mesh, and an
|
||||||
|
// empty list is more likely a mistake than an intention.
|
||||||
|
func TestSomebodyWhoMayCallNothingIsRefused(t *testing.T) {
|
||||||
|
inv := ForTest(t)
|
||||||
|
if err := inv.RecordPerson(context.Background(), Person{Name: "ada"}); err == nil {
|
||||||
|
t.Fatal("somebody who may call nothing was recorded")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Forgetting somebody takes their credential with them. **Both, or it is not a revocation**: a
|
||||||
|
// person's row gone and their bus user left behind is a credential that still works and that nothing
|
||||||
|
// derives.
|
||||||
|
func TestForgettingAPersonTakesTheirCredential(t *testing.T) {
|
||||||
|
inv := ForTest(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
if err := inv.RecordPerson(ctx, Person{Name: "ada", Invokes: []string{"a.one"}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := inv.MintBusPassword(ctx, BusUser{Username: "person.ada", Kind: BusPerson}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := inv.ForgetPerson(ctx, "ada"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, known, err := inv.BusUserHash(ctx, "person.ada"); err != nil || known {
|
||||||
|
t.Fatalf("a forgotten person's credential still works: %v %v", known, err)
|
||||||
|
}
|
||||||
|
records, err := inv.BusRecords(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, still := records.People["ada"]; still {
|
||||||
|
t.Fatal("a forgotten person is still composed into the bus")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
package inventory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The seats the mesh has, as data (novox/hq ADR 0122).
|
||||||
|
//
|
||||||
|
// The set the control plane reads is a table here, not a slice compiled into it. It is seeded from
|
||||||
|
// the binary's defaults the first time the mesh comes up (SeedSeats), and thereafter it is the live
|
||||||
|
// copy: a rename or an added seat is a write here, and the control plane loads it at startup rather
|
||||||
|
// than being rebuilt for it.
|
||||||
|
|
||||||
|
// Seats is every seat the mesh defines, read from the store.
|
||||||
|
func (i *Inventory) Seats(ctx context.Context) ([]catalogue.Seat, error) {
|
||||||
|
rows, err := i.store.Pool().Query(ctx,
|
||||||
|
`select name, scope, delivers, decided from seat order by name`)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
var seats []catalogue.Seat
|
||||||
|
for rows.Next() {
|
||||||
|
var s catalogue.Seat
|
||||||
|
if err := rows.Scan(&s.Name, &s.Scope, &s.Delivers, &s.Decision); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
seats = append(seats, s)
|
||||||
|
}
|
||||||
|
return seats, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// SeedSeats writes the mesh's default set into the table where it is not already present.
|
||||||
|
//
|
||||||
|
// **Idempotent, and never overwriting.** Run every time the control plane migrates, it fills an
|
||||||
|
// empty table on first boot and adds a seat a new release ships — but it leaves a row already there
|
||||||
|
// exactly as it is, so an operator's rename in the table is not undone by the next deploy putting
|
||||||
|
// the old name back. What a release removes from the defaults is not deleted here either; retiring a
|
||||||
|
// seat is its own decision, not a silent consequence of it dropping out of the binary.
|
||||||
|
func (i *Inventory) SeedSeats(ctx context.Context, defaults []catalogue.Seat) (int, error) {
|
||||||
|
var added int
|
||||||
|
for _, s := range defaults {
|
||||||
|
tag, err := i.store.Pool().Exec(ctx,
|
||||||
|
`insert into seat (name, scope, delivers, decided) values ($1, $2, $3, $4)
|
||||||
|
on conflict (name) do nothing`,
|
||||||
|
s.Name, s.Scope, s.Delivers, s.Decision)
|
||||||
|
if err != nil {
|
||||||
|
return added, err
|
||||||
|
}
|
||||||
|
added += int(tag.RowsAffected())
|
||||||
|
}
|
||||||
|
return added, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Aliases is every former seat name and the seat it now resolves to (novox/hq ADR 0122).
|
||||||
|
func (i *Inventory) Aliases(ctx context.Context) (map[string]string, error) {
|
||||||
|
rows, err := i.store.Pool().Query(ctx, `select alias, seat from seat_alias`)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
aliases := map[string]string{}
|
||||||
|
for rows.Next() {
|
||||||
|
var alias, seat string
|
||||||
|
if err := rows.Scan(&alias, &seat); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
aliases[alias] = seat
|
||||||
|
}
|
||||||
|
return aliases, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// RenameSeat gives a seat a new name and keeps the old one as an alias (novox/hq ADR 0122).
|
||||||
|
//
|
||||||
|
// **This is the whole of a rename.** The seat's canonical name becomes `to`; `from` is remembered as
|
||||||
|
// an alias so every reference to it — a manifest's claim, a held record, the build machine's
|
||||||
|
// embedded set — goes on resolving to the same seat, unchanged. Nothing is rebuilt and nothing
|
||||||
|
// freezes. Any alias that pointed to `from` is repointed to `to`, so a chain of renames does not
|
||||||
|
// leave an older name resolving to a name that no longer exists.
|
||||||
|
func (i *Inventory) RenameSeat(ctx context.Context, from, to string) error {
|
||||||
|
if from == to {
|
||||||
|
return fmt.Errorf("a seat is renamed to a different name; %q is already its name", to)
|
||||||
|
}
|
||||||
|
tag, err := i.store.Pool().Exec(ctx, `update seat set name = $1 where name = $2`, to, from)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if tag.RowsAffected() == 0 {
|
||||||
|
return fmt.Errorf("no seat named %q to rename", from)
|
||||||
|
}
|
||||||
|
// The old name resolves to the new one; and any name that resolved to the old one now resolves
|
||||||
|
// to the new one, so no alias is left pointing at a name that is gone.
|
||||||
|
if _, err := i.store.Pool().Exec(ctx,
|
||||||
|
`insert into seat_alias (alias, seat) values ($1, $2)
|
||||||
|
on conflict (alias) do update set seat = excluded.seat`, from, to); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := i.store.Pool().Exec(ctx,
|
||||||
|
`update seat_alias set seat = $1 where seat = $2`, to, from); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,115 @@
|
|||||||
|
package inventory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The seat set is data the control plane owns (novox/hq ADR 0122): seeded from the binary's
|
||||||
|
// defaults, read back as the working set, and thereafter an operator's to change without a rebuild.
|
||||||
|
|
||||||
|
func TestSeatsAreSeededFromTheDefaultsAndReadBack(t *testing.T) {
|
||||||
|
inv := ForTest(t)
|
||||||
|
defaults := catalogue.DefaultSeats()
|
||||||
|
|
||||||
|
added, err := inv.SeedSeats(t.Context(), defaults)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if added != len(defaults) {
|
||||||
|
t.Fatalf("seeded %d of %d seats", added, len(defaults))
|
||||||
|
}
|
||||||
|
got, err := inv.Seats(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(got) != len(defaults) {
|
||||||
|
t.Fatalf("read back %d seats, seeded %d", len(got), len(defaults))
|
||||||
|
}
|
||||||
|
// The set round-trips: name, scope and what it delivers survive the store.
|
||||||
|
by := map[string]catalogue.Seat{}
|
||||||
|
for _, s := range got {
|
||||||
|
by[s.Name] = s
|
||||||
|
}
|
||||||
|
for _, d := range defaults {
|
||||||
|
if by[d.Name].Scope != d.Scope || by[d.Name].Delivers != d.Delivers {
|
||||||
|
t.Errorf("%s came back as %+v, seeded %+v", d.Name, by[d.Name], d)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Re-seeding an already-seeded set adds nothing and changes nothing — every deploy runs SeedSeats,
|
||||||
|
// and a mesh already holding the set must be left exactly as it is (an operator's edit to a row
|
||||||
|
// included). A row's fields are not overwritten: on conflict the insert does nothing.
|
||||||
|
//
|
||||||
|
// (Phase 1 keys the table by name, so a seat *renamed* in the table would have its old name
|
||||||
|
// re-seeded — the move to a stable id a rename does not touch is the next step, ADR 0122. This test
|
||||||
|
// asserts only the property that holds now: an unchanged set re-seeds to a no-op.)
|
||||||
|
func TestReSeedingAnUnchangedSetIsANoOp(t *testing.T) {
|
||||||
|
inv := ForTest(t)
|
||||||
|
defaults := catalogue.DefaultSeats()
|
||||||
|
if _, err := inv.SeedSeats(t.Context(), defaults); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// An operator changes a row's scope in the table — the point of it being data.
|
||||||
|
if _, err := inv.store.Pool().Exec(t.Context(),
|
||||||
|
`update seat set scope = 'mesh' where name = 'node-uplink'`); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
added, err := inv.SeedSeats(t.Context(), defaults)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if added != 0 {
|
||||||
|
t.Fatalf("re-seeding an already-present set added %d rows", added)
|
||||||
|
}
|
||||||
|
got, err := inv.Seats(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, s := range got {
|
||||||
|
if s.Name == "node-uplink" && s.Scope != "mesh" {
|
||||||
|
t.Fatalf("re-seeding overwrote the operator's change: node-uplink scope is %q", s.Scope)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A rename is one operation: the seat gets the new name, the old name becomes an alias that still
|
||||||
|
// resolves to it (novox/hq ADR 0122).
|
||||||
|
func TestRenameSeatKeepsTheFormerNameAsAnAlias(t *testing.T) {
|
||||||
|
inv := ForTest(t)
|
||||||
|
if _, err := inv.SeedSeats(t.Context(), catalogue.DefaultSeats()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.RenameSeat(t.Context(), "node-packet-filter", "node-firewall"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
seats, err := inv.Seats(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
names := map[string]bool{}
|
||||||
|
for _, s := range seats {
|
||||||
|
names[s.Name] = true
|
||||||
|
}
|
||||||
|
if !names["node-firewall"] || names["node-packet-filter"] {
|
||||||
|
t.Fatalf("the seat was not renamed in place: %v", names)
|
||||||
|
}
|
||||||
|
aliases, err := inv.Aliases(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if aliases["node-packet-filter"] != "node-firewall" {
|
||||||
|
t.Fatalf("the former name is not an alias of the new one: %v", aliases)
|
||||||
|
}
|
||||||
|
// Renaming what has no seat is refused; renaming to the same name is refused.
|
||||||
|
if err := inv.RenameSeat(t.Context(), "no-such-seat", "x"); err == nil {
|
||||||
|
t.Fatal("renaming a seat that does not exist was accepted")
|
||||||
|
}
|
||||||
|
if err := inv.RenameSeat(t.Context(), "node-firewall", "node-firewall"); err == nil {
|
||||||
|
t.Fatal("renaming a seat to its own name was accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -33,6 +33,9 @@ type Tunnel struct {
|
|||||||
// Port is the port the found interface listened on — one the hosting provider already lets
|
// Port is the port the found interface listened on — one the hosting provider already lets
|
||||||
// through, which is why it is worth taking.
|
// through, which is why it is worth taking.
|
||||||
Port int `json:"port"`
|
Port int `json:"port"`
|
||||||
|
// MTU is the found interface's, when it set one; the mesh's interface takes it over so a
|
||||||
|
// tuned path does not silently regress to the default (novox/hq: a taken tunnel carries its MTU).
|
||||||
|
MTU int `json:"mtu,omitempty"`
|
||||||
// Address is the interface's own address with its prefix length, 192.0.2.1/24; Range is the
|
// Address is the interface's own address with its prefix length, 192.0.2.1/24; Range is the
|
||||||
// network that prefix names, 192.0.2.0/24.
|
// network that prefix names, 192.0.2.0/24.
|
||||||
Address string `json:"address"`
|
Address string `json:"address"`
|
||||||
|
|||||||
@@ -82,6 +82,16 @@ type BuildResult struct {
|
|||||||
// about the source.
|
// about the source.
|
||||||
On string `json:"on"`
|
On string `json:"on"`
|
||||||
|
|
||||||
|
// Module is what was built, read out of the manifest — the only place it is authoritative, since
|
||||||
|
// a request names a repository and a path.
|
||||||
|
//
|
||||||
|
// **Here because one message now reaches three audiences** (novox/hq ADR 0121). On the bus the
|
||||||
|
// mesh runs on today the answer and the announcement were two publishes to two topologies, so a
|
||||||
|
// result needed no module name and the announcement carried one. On the bus being built the
|
||||||
|
// outcome is the role's own event, and the catalogue reading it needs to know what was built.
|
||||||
|
// Empty on a failed build, which produced no module version.
|
||||||
|
Module string `json:"module,omitempty"`
|
||||||
|
|
||||||
// Commit is what was actually built. The mesh records it, which is what makes "is this
|
// Commit is what was actually built. The mesh records it, which is what makes "is this
|
||||||
// current?" answerable without building again.
|
// current?" answerable without building again.
|
||||||
Commit string `json:"commit,omitempty"`
|
Commit string `json:"commit,omitempty"`
|
||||||
|
|||||||
@@ -0,0 +1,108 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Asking a role to build something, and being told what came of it.
|
||||||
|
//
|
||||||
|
// A build is work submitted to a role, not a message to a machine (novox/hq ADR 0121). The
|
||||||
|
// build-machine seat accepts a build and emits an outcome, so the same publish that answers whoever
|
||||||
|
// asked also reaches the controller that records it and the catalogue that places it in the module
|
||||||
|
// graph — and no build machine needs permission to publish into anybody's inbox.
|
||||||
|
//
|
||||||
|
// **This is the one flow whose shape differs from every other**, which is why it has its own seam
|
||||||
|
// rather than living in `Bus`. Everything else the controller sends is either an event nobody must
|
||||||
|
// act on or a declaration a node reconciles toward; a build is a request that takes minutes and has
|
||||||
|
// exactly one answer. Too long for request/reply, too particular to be an event.
|
||||||
|
|
||||||
|
// TheBuildMachine is the role a build is submitted to.
|
||||||
|
const TheBuildMachine = "mesh-build-machine"
|
||||||
|
|
||||||
|
// BuildWork is where a build request lands, and BuildOutcome is where its result does. Derived from
|
||||||
|
// the seat, so both sides name the role and neither names the other.
|
||||||
|
func BuildWork() string { return "mesh.seat." + TheBuildMachine + ".accept.build" }
|
||||||
|
func BuildOutcome() string { return "mesh.seat." + TheBuildMachine + ".event.built" }
|
||||||
|
|
||||||
|
// KeyRoleBuilt is the build outcome under the role's name, on the bus the mesh runs on today.
|
||||||
|
//
|
||||||
|
// The same event as KeyModuleBuilt and published beside it, because a catalogue installed before this
|
||||||
|
// change listens for the module's name and one installed after listens for the role's. Both, until
|
||||||
|
// this bus retires: a rename needs publisher and subscriber to change together, and a deployment
|
||||||
|
// cannot promise which arrives first.
|
||||||
|
const KeyRoleBuilt = "built"
|
||||||
|
|
||||||
|
// Builders is how work reaches a build machine and how the outcome comes back.
|
||||||
|
type Builders interface {
|
||||||
|
// Submit asks for one build and waits for its outcome.
|
||||||
|
//
|
||||||
|
// The wait is long by nature. A build clones, pulls a base image and runs a container build, so
|
||||||
|
// a timeout here says "nothing is doing builds" rather than "this build is slow" — and the two
|
||||||
|
// need different remedies, which is why the message distinguishes them.
|
||||||
|
Submit(ctx context.Context, request BuildRequest, wait time.Duration) (BuildResult, error)
|
||||||
|
|
||||||
|
// Close lets go of whatever was dialled.
|
||||||
|
Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
// BuildMachine is a machine taking work from the role it holds.
|
||||||
|
type BuildMachine interface {
|
||||||
|
// Take hands each request to do until the context ends, and says why it stopped.
|
||||||
|
Take(ctx context.Context, do func(context.Context, Build)) error
|
||||||
|
Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build is one request a machine has been handed.
|
||||||
|
type Build interface {
|
||||||
|
// Request is what to build.
|
||||||
|
Request() BuildRequest
|
||||||
|
|
||||||
|
// Announce publishes the outcome as the role's own event.
|
||||||
|
//
|
||||||
|
// One publish, three audiences: whoever asked matches it by the id their request carried, the
|
||||||
|
// controller records it, and the catalogue places it. On the bus the mesh runs on today that
|
||||||
|
// fan-out came from a shared exchange; here the mesh derived the subject.
|
||||||
|
Announce(ctx context.Context, result BuildResult) error
|
||||||
|
|
||||||
|
// Done settles the request. Called only after the outcome is away, so a machine that dies
|
||||||
|
// before announcing leaves the work for another rather than losing it.
|
||||||
|
Done() error
|
||||||
|
|
||||||
|
// Hold hands the work back for another attempt after the delay.
|
||||||
|
Hold(after time.Duration) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitingFor is the message a caller gets when nothing answered. Its own function because both
|
||||||
|
// transports say it, and saying it differently in two places is how one of them ends up vague.
|
||||||
|
func waitingFor(wait time.Duration) error {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"no build machine answered within %s. Either nothing holds %s — in which case the work is "+
|
||||||
|
"queued and will be done when something does — or a build is taking longer than this",
|
||||||
|
wait, TheBuildMachine)
|
||||||
|
}
|
||||||
|
|
||||||
|
// theOutcomeOf reads a result and says whether it is the answer to this request.
|
||||||
|
func theOutcomeOf(body []byte, id string) (BuildResult, bool, error) {
|
||||||
|
var result BuildResult
|
||||||
|
if err := json.Unmarshal(body, &result); err != nil {
|
||||||
|
return BuildResult{}, false, fmt.Errorf("a build machine answered with something unreadable: %w", err)
|
||||||
|
}
|
||||||
|
// Somebody else's build. Skipped rather than returned, because returning it would attribute one
|
||||||
|
// build's outcome to another's.
|
||||||
|
return result, result.ID == id, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ModuleOf reads the module's name out of a manifest a build produced, which is the only place it is
|
||||||
|
// authoritative — a request named a repository and a path, not a module.
|
||||||
|
func ModuleOf(manifest json.RawMessage) string {
|
||||||
|
var named struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(manifest, &named); err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return named.Module
|
||||||
|
}
|
||||||
@@ -0,0 +1,202 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
amqp "github.com/rabbitmq/amqp091-go"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The build flow on the bus the mesh runs on today.
|
||||||
|
//
|
||||||
|
// Moved behind the seam rather than changed. The queue, the reply binding and the correlation are
|
||||||
|
// what they were, because the mesh is running on this.
|
||||||
|
|
||||||
|
// currentBuilds asks for builds over a channel.
|
||||||
|
type currentBuilds struct{ channel *amqp.Channel }
|
||||||
|
|
||||||
|
// BuildsOverCurrent is the asking side on the bus the mesh has.
|
||||||
|
func BuildsOverCurrent(channel *amqp.Channel) Builders { return currentBuilds{channel: channel} }
|
||||||
|
|
||||||
|
func (b currentBuilds) Close() {}
|
||||||
|
|
||||||
|
func (b currentBuilds) Submit(ctx context.Context, request BuildRequest,
|
||||||
|
wait time.Duration) (BuildResult, error) {
|
||||||
|
|
||||||
|
// Its own queue for the answer, declared before the ask. Consuming from the shared exchange
|
||||||
|
// would mean competing with the controller's own consumer for a message meant for this caller.
|
||||||
|
replies, err := b.channel.QueueDeclare(ReplyQueue(request.ID), false, true, true, false, nil)
|
||||||
|
if err != nil {
|
||||||
|
return BuildResult{}, err
|
||||||
|
}
|
||||||
|
// **A builder never publishes to the default exchange**, because permission there is per
|
||||||
|
// exchange and not per queue — a builder allowed to use it could publish into any node's queue,
|
||||||
|
// which is the privilege a build machine most obviously should not have. The cost is that every
|
||||||
|
// asker sees every result, which is why the correlation is checked below rather than assumed.
|
||||||
|
if err := b.channel.QueueBind(replies.Name, KeyBuilt, Exchange, false, nil); err != nil {
|
||||||
|
return BuildResult{}, err
|
||||||
|
}
|
||||||
|
answers, err := b.channel.ConsumeWithContext(ctx, replies.Name, "", true, true, false, false, nil)
|
||||||
|
if err != nil {
|
||||||
|
return BuildResult{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
body, err := json.Marshal(request)
|
||||||
|
if err != nil {
|
||||||
|
return BuildResult{}, err
|
||||||
|
}
|
||||||
|
if err := b.channel.PublishWithContext(ctx, "", BuildQueue, false, false, amqp.Publishing{
|
||||||
|
ContentType: "application/json",
|
||||||
|
DeliveryMode: amqp.Persistent,
|
||||||
|
CorrelationId: request.ID,
|
||||||
|
ReplyTo: replies.Name,
|
||||||
|
Body: body,
|
||||||
|
}); err != nil {
|
||||||
|
return BuildResult{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
waiting, cancel := context.WithTimeout(ctx, wait)
|
||||||
|
defer cancel()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-waiting.Done():
|
||||||
|
return BuildResult{}, waitingFor(wait)
|
||||||
|
case delivery, ok := <-answers:
|
||||||
|
if !ok {
|
||||||
|
return BuildResult{}, errors.New("the connection closed while waiting for a build")
|
||||||
|
}
|
||||||
|
result, mine, err := theOutcomeOf(delivery.Body, request.ID)
|
||||||
|
if err != nil {
|
||||||
|
return BuildResult{}, err
|
||||||
|
}
|
||||||
|
if mine {
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- the machine's side ---------------------------------------------------------------------
|
||||||
|
|
||||||
|
type currentMachine struct {
|
||||||
|
conn *amqp.Connection
|
||||||
|
channel *amqp.Channel
|
||||||
|
on string
|
||||||
|
}
|
||||||
|
|
||||||
|
// MachineOverCurrent takes build work over a channel.
|
||||||
|
func MachineOverCurrent(conn *amqp.Connection, channel *amqp.Channel, on string) BuildMachine {
|
||||||
|
return ¤tMachine{conn: conn, channel: channel, on: on}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *currentMachine) Close() {}
|
||||||
|
|
||||||
|
func (m *currentMachine) Take(ctx context.Context, do func(context.Context, Build)) error {
|
||||||
|
if _, err := m.channel.QueueDeclare(BuildQueue, true, false, false, false, nil); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// One at a time. A machine that took five requests at once would run five container builds
|
||||||
|
// against one runtime and finish all of them slower than it would have finished the first — and
|
||||||
|
// the queue is what shares work between machines, so nothing is lost by it.
|
||||||
|
if err := m.channel.Qos(1, 0, false); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Not auto-acknowledged: a request acknowledged on arrival is a build that vanishes if this
|
||||||
|
// process dies mid-way, with nobody waiting on it ever hearing why.
|
||||||
|
requests, err := m.channel.ConsumeWithContext(ctx, BuildQueue, "mesh-builder",
|
||||||
|
false, false, false, false, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil
|
||||||
|
case delivery, ok := <-requests:
|
||||||
|
if !ok {
|
||||||
|
return errors.New("the broker closed the connection")
|
||||||
|
}
|
||||||
|
var request BuildRequest
|
||||||
|
if err := json.Unmarshal(delivery.Body, &request); err != nil {
|
||||||
|
// Unreadable: rejected rather than retried, because the next attempt reads the same
|
||||||
|
// bytes. Nobody waiting hears an answer, which is correct — there was no request.
|
||||||
|
_ = delivery.Reject(false)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
do(ctx, ¤tBuild{request: request, delivery: delivery, on: m.on, channel: m.channel})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type currentBuild struct {
|
||||||
|
request BuildRequest
|
||||||
|
delivery amqp.Delivery
|
||||||
|
on string
|
||||||
|
channel *amqp.Channel
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *currentBuild) Request() BuildRequest { return b.request }
|
||||||
|
|
||||||
|
// Announce answers and announces, which on this bus are two publishes to two exchanges.
|
||||||
|
//
|
||||||
|
// The reply goes to whoever asked, correlated to their request; the announcement says to the whole
|
||||||
|
// mesh that a module now exists at a commit (novox/hq ADR 0072). Only a successful build is
|
||||||
|
// announced: a failed one produced no module version, and announcing one would put something in the
|
||||||
|
// graph that was never made.
|
||||||
|
func (b *currentBuild) Announce(ctx context.Context, result BuildResult) error {
|
||||||
|
body, err := json.Marshal(result)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := b.channel.PublishWithContext(ctx, Exchange, KeyBuilt, false, false, amqp.Publishing{
|
||||||
|
ContentType: "application/json",
|
||||||
|
CorrelationId: result.ID,
|
||||||
|
Body: body,
|
||||||
|
}); err != nil {
|
||||||
|
return fmt.Errorf("cannot answer a build request: %w", err)
|
||||||
|
}
|
||||||
|
if result.Failed != "" || result.Commit == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Announced under both names on this bus, for exactly as long as this bus lives.**
|
||||||
|
//
|
||||||
|
// A build's outcome belongs to the role now (novox/hq ADR 0121), so a catalogue built from the
|
||||||
|
// current manifests listens for the role's name. A catalogue that is *already running* listens for
|
||||||
|
// the module's, because that is what it was told when it was installed. A rename on a live bus
|
||||||
|
// needs the publisher and the subscriber to change together, and a merge cannot promise that: one
|
||||||
|
// of them is deployed first, and in that window the graph silently stops being updated — which is
|
||||||
|
// the failure this whole change was cleaning up after.
|
||||||
|
//
|
||||||
|
// So both, and the order stops mattering. The module's own name goes with the bus, in step 5's
|
||||||
|
// retirement list; nothing has ever run on the bus being built, so there is no legacy name there
|
||||||
|
// and this doubling has no counterpart.
|
||||||
|
announced := announcementOf(result)
|
||||||
|
if err := EmitEvent(ctx, OverCurrent{Channel: b.channel}, KeyModuleBuilt, "builder", b.on,
|
||||||
|
announced); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return EmitEvent(ctx, OverCurrent{Channel: b.channel}, KeyRoleBuilt, TheBuildMachine, b.on,
|
||||||
|
announced)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *currentBuild) Done() error { return b.delivery.Ack(false) }
|
||||||
|
|
||||||
|
func (b *currentBuild) Hold(time.Duration) error {
|
||||||
|
// No delayed redelivery on this bus: handed back at once, which is what it has always done.
|
||||||
|
return b.delivery.Nack(false, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
// announcementOf is what the mesh is told about a finished build. One function, so the two
|
||||||
|
// transports cannot describe the same build differently.
|
||||||
|
func announcementOf(result BuildResult) map[string]any {
|
||||||
|
return map[string]any{
|
||||||
|
"module": ModuleOf(result.Manifest), "commit": result.Commit,
|
||||||
|
"repository": result.Repository, "path": result.Path, "ref": result.Ref,
|
||||||
|
"manifest": json.RawMessage(result.Manifest), "against": result.Against,
|
||||||
|
"made": result.Made,
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **The old bus announces a build under both names, and that is not belt-and-braces.**
|
||||||
|
//
|
||||||
|
// A build's outcome belongs to the role now, so a catalogue built from the current manifests listens
|
||||||
|
// for the role's name — and a catalogue already running listens for the module's, because that is what
|
||||||
|
// it was told when it was installed. A rename on a live bus needs publisher and subscriber to change
|
||||||
|
// together, which a deployment cannot promise: one arrives first, and in that window the module graph
|
||||||
|
// silently stops being updated.
|
||||||
|
//
|
||||||
|
// Caught by asking what merging this would do to the mesh that is actually running, which is the only
|
||||||
|
// place the question could have been asked — the tests were green and both buses were self-consistent.
|
||||||
|
func TestTheOldBusAnnouncesABuildUnderBothNames(t *testing.T) {
|
||||||
|
// The routing key a catalogue installed before the change is bound to.
|
||||||
|
if KeyModuleBuilt != "module.builder.built" {
|
||||||
|
t.Fatalf("the module's own name is %q; a catalogue already running is bound to the old one",
|
||||||
|
KeyModuleBuilt)
|
||||||
|
}
|
||||||
|
// And the local name a catalogue built from the current manifests declares, which the old bus's
|
||||||
|
// client turns into `module.<role>.built`.
|
||||||
|
if KeyRoleBuilt != "built" {
|
||||||
|
t.Fatalf("the role's event is %q, and a holder emits its verbs bare", KeyRoleBuilt)
|
||||||
|
}
|
||||||
|
if TheBuildMachine != "mesh-build-machine" {
|
||||||
|
t.Fatalf("the role is %q", TheBuildMachine)
|
||||||
|
}
|
||||||
|
// The two must differ, or one publish would serve both and this doubling would be pointless.
|
||||||
|
if strings.HasSuffix(KeyModuleBuilt, "."+TheBuildMachine+"."+KeyRoleBuilt) {
|
||||||
|
t.Fatal("the two names are the same, so nothing was renamed and this is dead weight")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,206 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The build flow on the bus being built.
|
||||||
|
//
|
||||||
|
// **One publish where the old bus needed two** (novox/hq ADR 0121). There, the answer went to a
|
||||||
|
// reply queue and the announcement to an events exchange, because the two audiences were reached by
|
||||||
|
// two topologies. Here the outcome is the role's own event: whoever asked matches it by the id their
|
||||||
|
// request carried, the controller records it, the catalogue places it in the graph. So a build
|
||||||
|
// machine publishes once and needs permission for nothing but its own role's subjects — no reply
|
||||||
|
// queue to declare, and no grant over anybody's inbox.
|
||||||
|
|
||||||
|
// natsBuilds asks for builds over a connection.
|
||||||
|
type natsBuilds struct {
|
||||||
|
js *broker.JetStream
|
||||||
|
owned bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// BuildsOverNATS is the asking side on the bus being built. It dials, because the command that asks
|
||||||
|
// for a build is a one-shot and holds nothing else.
|
||||||
|
func BuildsOverNATS(address string) (Builders, error) {
|
||||||
|
js, err := broker.Dial(address)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot reach the bus at %s to ask for a build: %w", address, err)
|
||||||
|
}
|
||||||
|
return &natsBuilds{js: js, owned: true}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *natsBuilds) Close() {
|
||||||
|
if b.owned && b.js != nil {
|
||||||
|
b.js.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *natsBuilds) Submit(ctx context.Context, request BuildRequest,
|
||||||
|
wait time.Duration) (BuildResult, error) {
|
||||||
|
|
||||||
|
// Subscribed before the ask, so an outcome cannot arrive before there is anywhere for it to
|
||||||
|
// land. Core, not the stream: the asker is waiting now, and the durable copy of this outcome is
|
||||||
|
// the same event on EVENTS, which the controller records.
|
||||||
|
outcomes, err := b.js.Conn().SubscribeSync(BuildOutcome())
|
||||||
|
if err != nil {
|
||||||
|
return BuildResult{}, fmt.Errorf("cannot listen for a build's outcome: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = outcomes.Unsubscribe() }()
|
||||||
|
if err := b.js.Conn().Flush(); err != nil {
|
||||||
|
return BuildResult{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
body, err := json.Marshal(request)
|
||||||
|
if err != nil {
|
||||||
|
return BuildResult{}, err
|
||||||
|
}
|
||||||
|
// Into the role's work queue and awaited: work the bus never accepted must fail here rather than
|
||||||
|
// be assumed, because nothing else will ever say so.
|
||||||
|
publish, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if _, err := b.js.Context().Publish(BuildWork(), body, nats.Context(publish)); err != nil {
|
||||||
|
return BuildResult{}, fmt.Errorf("cannot submit a build: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
waiting, cancelWait := context.WithTimeout(ctx, wait)
|
||||||
|
defer cancelWait()
|
||||||
|
for {
|
||||||
|
msg, err := outcomes.NextMsgWithContext(waiting)
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, context.DeadlineExceeded):
|
||||||
|
return BuildResult{}, waitingFor(wait)
|
||||||
|
case errors.Is(err, context.Canceled):
|
||||||
|
return BuildResult{}, ctx.Err()
|
||||||
|
case err != nil:
|
||||||
|
return BuildResult{}, fmt.Errorf("waiting for a build's outcome: %w", err)
|
||||||
|
}
|
||||||
|
result, mine, err := theOutcomeOf(msg.Data, request.ID)
|
||||||
|
if err != nil {
|
||||||
|
return BuildResult{}, err
|
||||||
|
}
|
||||||
|
if mine {
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- the machine's side ---------------------------------------------------------------------
|
||||||
|
|
||||||
|
type natsMachine struct {
|
||||||
|
js *broker.JetStream
|
||||||
|
on string
|
||||||
|
seat string
|
||||||
|
sub *nats.Subscription
|
||||||
|
}
|
||||||
|
|
||||||
|
// MachineOverNATS takes build work from the role this machine holds.
|
||||||
|
func MachineOverNATS(js *broker.JetStream, on string) BuildMachine {
|
||||||
|
return &natsMachine{js: js, on: on, seat: TheBuildMachine}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *natsMachine) Close() {
|
||||||
|
if m.sub != nil {
|
||||||
|
_ = m.sub.Unsubscribe()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Take binds to the role's worker and hands each request over, one at a time.
|
||||||
|
//
|
||||||
|
// **Bound, never created.** The work queue and the worker on it are the controller's to define
|
||||||
|
// (design 25 §3), and a build machine reaches no part of the JetStream API — so a missing one is said
|
||||||
|
// as the mesh's to answer rather than quietly created with whatever this client defaults to.
|
||||||
|
func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build)) error {
|
||||||
|
worker, found := broker.HolderConsumerFor(m.on, "builder",
|
||||||
|
broker.DeclaredSeat{Name: m.seat, Accepts: []string{"build"}})
|
||||||
|
if !found {
|
||||||
|
return fmt.Errorf("%s accepts no work, so there is nothing for this machine to take", m.seat)
|
||||||
|
}
|
||||||
|
|
||||||
|
// One at a time, which the consumer's own ack-pending limit enforces rather than a prefetch
|
||||||
|
// setting: a machine that took five requests at once would run five container builds against one
|
||||||
|
// runtime and finish all of them slower than the first.
|
||||||
|
work := make(chan *nats.Msg, 1)
|
||||||
|
// **The consumer's own filter, not the one subject this machine cares about.** The client checks
|
||||||
|
// what is asked for against the consumer's filter and refuses anything that is not the same —
|
||||||
|
// "subject does not match consumer" — so subscribing `…accept.build` against a consumer filtered
|
||||||
|
// on `…accept.>` is rejected even though it is narrower. Learned twice now, on two different
|
||||||
|
// consumers, which is why it is written down here.
|
||||||
|
filter := worker.Filters[0]
|
||||||
|
sub, err := m.js.Context().ChanQueueSubscribe(filter, worker.Queue, work,
|
||||||
|
nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"this machine cannot take work from %s: %w. The mesh creates that queue and this "+
|
||||||
|
"machine's worker on it, and a build machine may not create one itself — so this is "+
|
||||||
|
"the mesh's to answer, not this machine's", m.seat, err)
|
||||||
|
}
|
||||||
|
m.sub = sub
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil
|
||||||
|
case msg, ok := <-work:
|
||||||
|
if !ok {
|
||||||
|
return errors.New("the bus stopped delivering build work")
|
||||||
|
}
|
||||||
|
var request BuildRequest
|
||||||
|
if err := json.Unmarshal(msg.Data, &request); err != nil {
|
||||||
|
// Unreadable: terminated rather than retried, because the next attempt reads the same
|
||||||
|
// bytes. Nobody waiting hears an answer, which is right — there was no request.
|
||||||
|
_ = msg.Term()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
do(ctx, &natsBuild{request: request, msg: msg, on: m.on, js: m.js})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type natsBuild struct {
|
||||||
|
request BuildRequest
|
||||||
|
msg *nats.Msg
|
||||||
|
on string
|
||||||
|
js *broker.JetStream
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *natsBuild) Request() BuildRequest { return b.request }
|
||||||
|
|
||||||
|
// Announce publishes the outcome as the role's own event, once, for all three audiences.
|
||||||
|
//
|
||||||
|
// Into the stream, so a controller that was restarting still records it and a catalogue that was
|
||||||
|
// down still catches up. The asker is listening on core for the same subject and gets it either way:
|
||||||
|
// a stream delivers to its durable consumers and the plain subscribers both.
|
||||||
|
func (b *natsBuild) Announce(ctx context.Context, result BuildResult) error {
|
||||||
|
// One body, three readers. Whoever asked matches the id; the controller records it; the catalogue
|
||||||
|
// needs to know what was built, which only the manifest says — so the result carries it rather
|
||||||
|
// than a second message carrying a second shape.
|
||||||
|
//
|
||||||
|
// **A failed build names no module**, because it produced no module version and the catalogue
|
||||||
|
// would otherwise put something in the graph that was never made. The asker still gets its
|
||||||
|
// answer: a failure is the answer.
|
||||||
|
if result.Failed == "" && result.Commit != "" {
|
||||||
|
result.Module = ModuleOf(result.Manifest)
|
||||||
|
}
|
||||||
|
body, err := json.Marshal(result)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
publish, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if _, err := b.js.Context().Publish(BuildOutcome(), body, nats.Context(publish)); err != nil {
|
||||||
|
return fmt.Errorf("cannot announce a build's outcome: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *natsBuild) Done() error { return b.msg.Ack() }
|
||||||
|
|
||||||
|
func (b *natsBuild) Hold(after time.Duration) error { return b.msg.NakWithDelay(after) }
|
||||||
@@ -0,0 +1,215 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A build, end to end, against a real server.
|
||||||
|
//
|
||||||
|
// The claim worth checking is the one ADR 0121 rests on: **one publish reaches three audiences**.
|
||||||
|
// Whoever asked matches the outcome by the id their request carried; the controller records it; the
|
||||||
|
// catalogue places it. On the old bus that fan-out came from a shared exchange, and it would be easy
|
||||||
|
// to write a version where only the asker hears it and nobody notices for weeks.
|
||||||
|
//
|
||||||
|
// docker run -d --rm --name t -p 14230:4222 nats:2.10-alpine -js
|
||||||
|
// MESH_TEST_NATS=nats://127.0.0.1:14230 go test ./internal/link/ -run TestNatsABuild
|
||||||
|
|
||||||
|
func aBusWithTheBuildRole(t *testing.T) *broker.JetStream {
|
||||||
|
t.Helper()
|
||||||
|
url := os.Getenv("MESH_TEST_NATS")
|
||||||
|
if url == "" {
|
||||||
|
t.Skip("MESH_TEST_NATS unset")
|
||||||
|
}
|
||||||
|
js, err := broker.Dial(url)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(js.Close)
|
||||||
|
|
||||||
|
seats := []broker.DeclaredSeat{{Name: TheBuildMachine, Accepts: []string{"build"},
|
||||||
|
Emits: []string{"built"}}}
|
||||||
|
if err := broker.AssertMeshStreams(js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := broker.RaiseSeats(js, seats, map[string]broker.Holder{
|
||||||
|
TheBuildMachine: {Node: "anchor", Module: "builder"},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
clean := func() {
|
||||||
|
_ = js.Context().DeleteStream("SEAT_MESH_BUILD_MACHINE")
|
||||||
|
for _, s := range broker.MeshStreams() {
|
||||||
|
_ = js.Context().PurgeStream(s.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Cleanup(clean)
|
||||||
|
return js
|
||||||
|
}
|
||||||
|
|
||||||
|
// The whole round trip: asked, taken, built, and the outcome heard by the asker and by a consumer of
|
||||||
|
// the role's event who never asked for anything.
|
||||||
|
func TestNatsABuildIsTakenAndItsOutcomeReachesEverybody(t *testing.T) {
|
||||||
|
js := aBusWithTheBuildRole(t)
|
||||||
|
ctx, stop := context.WithCancel(context.Background())
|
||||||
|
defer stop()
|
||||||
|
|
||||||
|
// A third party on the role's event — what the catalogue is. Subscribed first, so nothing is
|
||||||
|
// missed.
|
||||||
|
heard := make(chan BuildResult, 4)
|
||||||
|
watching, err := js.Conn().Subscribe(BuildOutcome(), func(msg *nats.Msg) {
|
||||||
|
var r BuildResult
|
||||||
|
if json.Unmarshal(msg.Data, &r) == nil {
|
||||||
|
heard <- r
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() { _ = watching.Unsubscribe() }()
|
||||||
|
_ = js.Conn().Flush()
|
||||||
|
|
||||||
|
// A build machine holding the role.
|
||||||
|
machine := MachineOverNATS(js, "anchor")
|
||||||
|
defer machine.Close()
|
||||||
|
failed := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
failed <- machine.Take(ctx, func(ctx context.Context, work Build) {
|
||||||
|
r := work.Request()
|
||||||
|
_ = work.Announce(ctx, BuildResult{
|
||||||
|
ID: r.ID, Repository: r.Repository, On: "anchor", Commit: "abc1234",
|
||||||
|
Manifest: json.RawMessage(`{"module":"shop"}`),
|
||||||
|
})
|
||||||
|
_ = work.Done()
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
|
||||||
|
ask := &natsBuilds{js: js}
|
||||||
|
result, err := ask.Submit(ctx, BuildRequest{ID: "b-1", Repository: "/r"}, 15*time.Second)
|
||||||
|
if err != nil {
|
||||||
|
select {
|
||||||
|
case why := <-failed:
|
||||||
|
t.Fatalf("the machine could not take work: %v", why)
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
t.Fatalf("the asker never got an outcome: %v", err)
|
||||||
|
}
|
||||||
|
if result.ID != "b-1" || result.Commit != "abc1234" {
|
||||||
|
t.Fatalf("the asker got %+v", result)
|
||||||
|
}
|
||||||
|
// Named in the outcome, because only the manifest says what was built and the catalogue reading
|
||||||
|
// this event needs to know.
|
||||||
|
if result.Module != "shop" {
|
||||||
|
t.Errorf("the outcome names module %q, so a catalogue reading it cannot place the build",
|
||||||
|
result.Module)
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case also := <-heard:
|
||||||
|
if also.ID != "b-1" {
|
||||||
|
t.Fatalf("a third party heard %+v", also)
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("nobody but the asker heard the outcome, so the catalogue would never place the build")
|
||||||
|
}
|
||||||
|
|
||||||
|
// And the work left the queue: a request a machine took and settled must not be given to another.
|
||||||
|
deadline := time.Now().Add(5 * time.Second)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
info, err := js.Context().StreamInfo("SEAT_MESH_BUILD_MACHINE")
|
||||||
|
if err == nil && info.State.Msgs == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
|
}
|
||||||
|
t.Fatal("the request is still queued after being settled, so another machine would build it again")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Work submitted with no machine holding the role waits rather than failing, and is done when one
|
||||||
|
// arrives. **That is what a queue is for**, and the alternative — refusing because nobody is there
|
||||||
|
// yet — would make installing a build machine an ordering problem.
|
||||||
|
func TestNatsABuildWaitsForAMachineRatherThanFailing(t *testing.T) {
|
||||||
|
js := aBusWithTheBuildRole(t)
|
||||||
|
|
||||||
|
body, _ := json.Marshal(BuildRequest{ID: "b-2", Repository: "/r"})
|
||||||
|
if _, err := js.Context().Publish(BuildWork(), body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
info, err := js.Context().StreamInfo("SEAT_MESH_BUILD_MACHINE")
|
||||||
|
if err != nil || info.State.Msgs != 1 {
|
||||||
|
t.Fatalf("the work did not queue: %+v %v", info, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Now a machine arrives and finds it waiting.
|
||||||
|
ctx, stop := context.WithCancel(context.Background())
|
||||||
|
defer stop()
|
||||||
|
took := make(chan string, 1)
|
||||||
|
machine := MachineOverNATS(js, "anchor")
|
||||||
|
defer machine.Close()
|
||||||
|
go func() {
|
||||||
|
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
|
||||||
|
took <- work.Request().ID
|
||||||
|
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
|
||||||
|
_ = work.Done()
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case id := <-took:
|
||||||
|
if id != "b-2" {
|
||||||
|
t.Fatalf("the machine took %q", id)
|
||||||
|
}
|
||||||
|
case <-time.After(10 * time.Second):
|
||||||
|
t.Fatal("a machine that arrived after the work did never got it, so the backlog was lost")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A machine that dies before saying anything leaves the work for another.
|
||||||
|
func TestNatsWorkAMachineDidNotAnswerGoesBackToTheQueue(t *testing.T) {
|
||||||
|
js := aBusWithTheBuildRole(t)
|
||||||
|
ctx, stop := context.WithCancel(context.Background())
|
||||||
|
defer stop()
|
||||||
|
|
||||||
|
body, _ := json.Marshal(BuildRequest{ID: "b-3", Repository: "/r"})
|
||||||
|
if _, err := js.Context().Publish(BuildWork(), body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var once sync.Once
|
||||||
|
handed := make(chan struct{}, 2)
|
||||||
|
machine := MachineOverNATS(js, "anchor")
|
||||||
|
defer machine.Close()
|
||||||
|
go func() {
|
||||||
|
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
|
||||||
|
handed <- struct{}{}
|
||||||
|
// The first time, hand it straight back — a machine that stopped mid-build.
|
||||||
|
var settled bool
|
||||||
|
once.Do(func() { _ = work.Hold(200 * time.Millisecond); settled = true })
|
||||||
|
if !settled {
|
||||||
|
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor"})
|
||||||
|
_ = work.Done()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
|
||||||
|
for i := 0; i < 2; i++ {
|
||||||
|
select {
|
||||||
|
case <-handed:
|
||||||
|
case <-time.After(10 * time.Second):
|
||||||
|
t.Fatalf("the work was handed over %d time(s); unanswered work must come back", i)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func quietLog() *log.Logger { return log.New(io.Discard, "", 0) }
|
||||||
|
|
||||||
|
var _ = quietLog
|
||||||
@@ -0,0 +1,194 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
amqp "github.com/rabbitmq/amqp091-go"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Bus is what the controller needs of the mesh's bus, **in the mesh's own words rather than a
|
||||||
|
// transport's** (novox/hq ADR 0116 step 3).
|
||||||
|
//
|
||||||
|
// Until now every one of these functions took the transport's own channel type, so the transport
|
||||||
|
// reached every caller and changing it meant touching all of them. The seam is small — the
|
||||||
|
// controller sends exactly two kinds of message that expect no answer, and asks two kinds of
|
||||||
|
// question — which is why the bus can be replaced at all.
|
||||||
|
//
|
||||||
|
// Two implementations live below, and both ship until the rollout (ADR 0116: nothing moves a
|
||||||
|
// node's bus before step 5). Both shipping is what makes them comparable — the same caller, the
|
||||||
|
// same arguments, and one conformance fixture holding them to one envelope.
|
||||||
|
type Bus interface {
|
||||||
|
// PublishEvent announces something that happened, under the emitter's own name. 1:many, and
|
||||||
|
// nobody is obliged to act (ADR 0041).
|
||||||
|
PublishEvent(ctx context.Context, key, source, node string, body []byte) error
|
||||||
|
|
||||||
|
// PublishDeclaration delivers one node what it should be. Addressed to that node alone: a
|
||||||
|
// declaration is not an event, and replaying yesterday's is actively harmful
|
||||||
|
// (design 29 §4, the *state* shape).
|
||||||
|
PublishDeclaration(ctx context.Context, node string, body []byte) error
|
||||||
|
|
||||||
|
// AskTool sends one question to a module's tool and awaits one answer. A tool nobody serves
|
||||||
|
// must say so **at once** rather than after the whole wait: the difference between "that
|
||||||
|
// module is down" and "that tool is slow" is the first thing a person asking wants.
|
||||||
|
AskTool(ctx context.Context, module, tool string, args []byte, timeout time.Duration) ([]byte, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- The bus the mesh runs on today -----------------------------------------------------
|
||||||
|
|
||||||
|
// OverCurrent is the bus the mesh runs on today, until the rollout.
|
||||||
|
type OverCurrent struct{ Channel *amqp.Channel }
|
||||||
|
|
||||||
|
func (b OverCurrent) PublishEvent(ctx context.Context, key, source, node string, body []byte) error {
|
||||||
|
id, err := eventID()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return b.Channel.PublishWithContext(ctx, EventsExchange, key, false, false, amqp.Publishing{
|
||||||
|
ContentType: "application/json",
|
||||||
|
DeliveryMode: amqp.Persistent,
|
||||||
|
MessageId: id,
|
||||||
|
Timestamp: time.Now().UTC(),
|
||||||
|
Body: body,
|
||||||
|
Headers: amqp.Table{
|
||||||
|
"x-event-id": id,
|
||||||
|
"x-source": source,
|
||||||
|
"x-node": node,
|
||||||
|
"x-time": time.Now().UTC().Format(time.RFC3339),
|
||||||
|
"content-type": "application/json",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// AskTool is implemented over the existing reply-queue machinery in ask.go; this seam does not
|
||||||
|
// change how it works today.
|
||||||
|
func (b OverCurrent) AskTool(ctx context.Context, module, tool string, args []byte, timeout time.Duration) ([]byte, error) {
|
||||||
|
answer, err := Ask(ctx, b.Channel, module, tool, args, timeout)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return answer.Result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b OverCurrent) PublishDeclaration(ctx context.Context, node string, body []byte) error {
|
||||||
|
// To the queue directly rather than through an exchange: a declaration is for one node, and
|
||||||
|
// routing it by name through a shared exchange would mean a binding per node that nothing
|
||||||
|
// removes when a node is retired.
|
||||||
|
return b.Channel.PublishWithContext(ctx, "", QueueFor(node), false, false, amqp.Publishing{
|
||||||
|
ContentType: "application/json",
|
||||||
|
DeliveryMode: amqp.Persistent,
|
||||||
|
Body: body,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- NATS, the bus being built ----------------------------------------------------------------
|
||||||
|
|
||||||
|
// OverNATS is the bus as a JetStream context.
|
||||||
|
type OverNATS struct {
|
||||||
|
Conn *nats.Conn
|
||||||
|
JS nats.JetStreamContext
|
||||||
|
}
|
||||||
|
|
||||||
|
// The subjects a node publishes on, and the controller listens to.
|
||||||
|
//
|
||||||
|
// One tree, and each name says who it is about: `mesh.control.<node>.…` is a node's own, which is
|
||||||
|
// what lets a node's account be granted exactly its own prefix and nothing of any other node's
|
||||||
|
// (design 25 §2, §4). The two that belong to no node — an enrolment, because a machine enrolling
|
||||||
|
// has no name the mesh has agreed to yet, and a build's outcome, because a builder is not
|
||||||
|
// reporting about itself — are named directly.
|
||||||
|
const (
|
||||||
|
// EnrolSubject is where a joining machine asks. Its enrolment user may publish here and
|
||||||
|
// nowhere else, so a leaked token buys nothing but the chance to enrol.
|
||||||
|
EnrolSubject = "mesh.control.enrol"
|
||||||
|
|
||||||
|
// BuiltSubject is where a build's outcome lands, for results nobody was waiting for.
|
||||||
|
BuiltSubject = "mesh.control.built"
|
||||||
|
|
||||||
|
// AliveSubjects is every node's heartbeat. Core NATS, never a stream: a lost heartbeat is the
|
||||||
|
// next heartbeat, and a stream of them is the mesh's least valuable message competing for
|
||||||
|
// retention with its most valuable (design 25 §3).
|
||||||
|
AliveSubjects = "mesh.control.*.alive"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ReportSubject is where one node says what it did. On the CONTROL stream, because it is the
|
||||||
|
// message the store-window guarantee is about (ADR 0083).
|
||||||
|
func ReportSubject(node string) string { return "mesh.control." + node + ".report" }
|
||||||
|
|
||||||
|
// AliveSubject is one node's heartbeat.
|
||||||
|
func AliveSubject(node string) string { return "mesh.control." + node + ".alive" }
|
||||||
|
|
||||||
|
// EventSubject is where a module's event lands. Derived from the emitter, never taken from the
|
||||||
|
// caller: a source that could differ from the subject is an envelope that can lie about its
|
||||||
|
// origin, and on NATS the account's permissions make the subject the authority (design 29 §2).
|
||||||
|
func EventSubject(source, key string) string {
|
||||||
|
return "mesh.mod." + source + ".event." + key
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeclareSubject is where one node's declaration lands. Last-per-subject on the NODES stream, so
|
||||||
|
// a node that was away gets exactly the current one and a replayed older one is refused by
|
||||||
|
// sequence — the wire-level answer to novox/hq issue 107.
|
||||||
|
func DeclareSubject(node string) string { return "mesh.node." + node + ".declare" }
|
||||||
|
|
||||||
|
func (b OverNATS) PublishEvent(ctx context.Context, key, source, node string, body []byte) error {
|
||||||
|
id, err := eventID()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
h := nats.Header{}
|
||||||
|
h.Set("x-event-id", id)
|
||||||
|
h.Set("x-source", source)
|
||||||
|
h.Set("x-node", node)
|
||||||
|
h.Set("x-time", time.Now().UTC().Format(time.RFC3339))
|
||||||
|
h.Set("content-type", "application/json")
|
||||||
|
|
||||||
|
// The id is also the publish's message id, so the server refuses a duplicate inside its
|
||||||
|
// window. That narrows the window a consumer must deduplicate in; it does not remove the
|
||||||
|
// requirement, because the window is finite (design 19, delivery).
|
||||||
|
_, err = b.JS.PublishMsg(&nats.Msg{
|
||||||
|
Subject: EventSubject(source, key),
|
||||||
|
Header: h,
|
||||||
|
Data: body,
|
||||||
|
}, nats.MsgId(id), nats.Context(ctx))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("emitting %s: %w", key, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b OverNATS) PublishDeclaration(ctx context.Context, node string, body []byte) error {
|
||||||
|
_, err := b.JS.Publish(DeclareSubject(node), body, nats.Context(ctx))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("declaring to %s: %w", node, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ToolSubject is where a module answers. Derived from the module and the tool, so a caller names
|
||||||
|
// what it wants rather than where it lives.
|
||||||
|
func ToolSubject(module, tool string) string { return "mesh.mod." + module + ".tool." + tool }
|
||||||
|
|
||||||
|
func (b OverNATS) AskTool(ctx context.Context, module, tool string, args []byte, timeout time.Duration) ([]byte, error) {
|
||||||
|
if len(args) == 0 {
|
||||||
|
args = []byte(`{}`)
|
||||||
|
}
|
||||||
|
ask, cancel := context.WithTimeout(ctx, timeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
// **No reply queue, and no correlation to check.** The caller's inbox is its own — each
|
||||||
|
// account is granted one prefix and no other (design 25 §4) — so an answer cannot reach the
|
||||||
|
// wrong asker and there is nothing to correlate against. That also settles a cost recorded
|
||||||
|
// in build.go: on a shared reply exchange every asker saw every result.
|
||||||
|
msg, err := b.Conn.RequestWithContext(ask, ToolSubject(module, tool), args)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, nats.ErrNoResponders) {
|
||||||
|
// Said at once rather than after the whole wait: nothing is subscribed to that
|
||||||
|
// subject, which is a different fact from a tool being slow.
|
||||||
|
return nil, fmt.Errorf("nothing serves %s.%s", module, tool)
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("asking %s.%s: %w", module, tool, err)
|
||||||
|
}
|
||||||
|
return msg.Data, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,154 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **Both implementations, one fixture.** The point of the seam is not that the transport can be
|
||||||
|
// swapped — it is that the two can be held to the same envelope while both ship, so the day the
|
||||||
|
// bus moves is a configuration change rather than a discovery.
|
||||||
|
//
|
||||||
|
// Against a real server, because what the fixture pins is what reaches the wire:
|
||||||
|
//
|
||||||
|
// docker run -d --rm --name t -p 14222:4222 nats:2.10-alpine -js
|
||||||
|
// MESH_TEST_NATS=nats://127.0.0.1:14222 go test ./internal/link/ -run TestTheNatsBus
|
||||||
|
func TestTheNatsBusEmitsTheEnvelopeTheFixturePins(t *testing.T) {
|
||||||
|
url := os.Getenv("MESH_TEST_NATS")
|
||||||
|
if url == "" {
|
||||||
|
t.Skip("MESH_TEST_NATS unset")
|
||||||
|
}
|
||||||
|
f := loadFixture(t, "events/module-event.json")
|
||||||
|
|
||||||
|
conn, err := nats.Connect(url)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
js, err := conn.JetStream()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := js.AddStream(&nats.StreamConfig{
|
||||||
|
Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"},
|
||||||
|
}); err != nil && err != nats.ErrStreamNameAlreadyInUse {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
bus := OverNATS{JS: js}
|
||||||
|
body, _ := json.Marshal(f.Given.Body)
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if err := EmitEvent(ctx, bus, f.Given.Key, f.Given.Module, f.Given.Node, f.Given.Body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read back from the stream, not from the thing that wrote it.
|
||||||
|
raw, err := js.GetLastMsg("EVENTS", f.Wire.Subject)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("nothing landed on %s, which the fixture names: %v", f.Wire.Subject, err)
|
||||||
|
}
|
||||||
|
for _, h := range f.Wire.RequiredHeaders {
|
||||||
|
if raw.Header.Get(h) == "" {
|
||||||
|
t.Errorf("%s is not set on the wire, and the fixture requires it", h)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if got := raw.Header.Get("x-source"); got != f.Given.Module {
|
||||||
|
t.Errorf("x-source is %q; the subject says %q", got, f.Given.Module)
|
||||||
|
}
|
||||||
|
if string(raw.Data) != string(body) {
|
||||||
|
t.Errorf("the payload is %s, expected the body alone: %s", raw.Data, body)
|
||||||
|
}
|
||||||
|
// The envelope must not also be nested inside the payload.
|
||||||
|
var nested map[string]any
|
||||||
|
if json.Unmarshal(raw.Data, &nested) == nil {
|
||||||
|
if _, has := nested["key"]; has {
|
||||||
|
t.Error("the payload carries the envelope's own fields, which the fixture refuses")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The subject a declaration lands on is one node's, and nothing else's — the state shape.
|
||||||
|
func TestADeclarationIsAddressedToOneNode(t *testing.T) {
|
||||||
|
if got := DeclareSubject("anchor"); got != "mesh.node.anchor.declare" {
|
||||||
|
t.Fatalf("a declaration would go to %q", got)
|
||||||
|
}
|
||||||
|
if DeclareSubject("anchor") == DeclareSubject("laptop") {
|
||||||
|
t.Fatal("two nodes share a declaration subject, so each would apply the other's")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A module cannot emit under another's name: the subject is derived from the source, and the
|
||||||
|
// server's permissions make that subject the authority.
|
||||||
|
func TestAnEventsSubjectIsDerivedFromItsSource(t *testing.T) {
|
||||||
|
if got := EventSubject("shop", "order.placed"); got != "mesh.mod.shop.event.order.placed" {
|
||||||
|
t.Fatalf("an event would land on %q", got)
|
||||||
|
}
|
||||||
|
if EventSubject("shop", "x") == EventSubject("billing", "x") {
|
||||||
|
t.Fatal("two modules share an event subject, so neither owns its own name")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A tool nobody serves says so at once. The difference between "that module is down" and "that
|
||||||
|
// tool is slow" is the first thing a person asking wants, and waiting out the timeout to say it
|
||||||
|
// is how a fast answer becomes a slow non-answer.
|
||||||
|
func TestAskingAToolNobodyServesFailsAtOnce(t *testing.T) {
|
||||||
|
url := os.Getenv("MESH_TEST_NATS")
|
||||||
|
if url == "" {
|
||||||
|
t.Skip("MESH_TEST_NATS unset")
|
||||||
|
}
|
||||||
|
conn, err := nats.Connect(url)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
|
||||||
|
bus := OverNATS{Conn: conn}
|
||||||
|
began := time.Now()
|
||||||
|
_, err = bus.AskTool(context.Background(), "nobody", "home", nil, 30*time.Second)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("asking a tool nothing serves succeeded")
|
||||||
|
}
|
||||||
|
if took := time.Since(began); took > 2*time.Second {
|
||||||
|
t.Errorf("took %v to say nothing serves it; the caller waited out the timeout", took)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "nothing serves") {
|
||||||
|
t.Errorf("the refusal does not say nobody is there: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// And a served tool answers, with no reply queue to declare and no correlation to check.
|
||||||
|
func TestAskingAServedToolAnswers(t *testing.T) {
|
||||||
|
url := os.Getenv("MESH_TEST_NATS")
|
||||||
|
if url == "" {
|
||||||
|
t.Skip("MESH_TEST_NATS unset")
|
||||||
|
}
|
||||||
|
conn, err := nats.Connect(url)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
|
||||||
|
sub, err := conn.Subscribe(ToolSubject("shop", "price"), func(m *nats.Msg) {
|
||||||
|
_ = m.Respond([]byte(`{"total":12}`))
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer sub.Unsubscribe()
|
||||||
|
|
||||||
|
got, err := OverNATS{Conn: conn}.AskTool(context.Background(), "shop", "price",
|
||||||
|
[]byte(`{"qty":4}`), 5*time.Second)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if string(got) != `{"total":12}` {
|
||||||
|
t.Fatalf("the answer came back as %s", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,107 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The Go implementation, held to the shared fixtures (novox/hq ADR 0074, design 19).
|
||||||
|
//
|
||||||
|
// **Read from the sdk's conformance directory by sibling path**, the way the lab finds its
|
||||||
|
// siblings — deliberately not copied here. A fixture copied into each implementation is two
|
||||||
|
// fixtures, and two fixtures drift, which is the exact failure the suite exists to prevent.
|
||||||
|
type fixture struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Given struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
Node string `json:"node"`
|
||||||
|
Key string `json:"key"`
|
||||||
|
Body map[string]any `json:"body"`
|
||||||
|
Headers map[string]string `json:"headers"`
|
||||||
|
} `json:"given"`
|
||||||
|
Wire struct {
|
||||||
|
Subject string `json:"subject"`
|
||||||
|
RequiredHeaders []string `json:"requiredHeaders"`
|
||||||
|
HeaderFormats map[string]string `json:"headerFormats"`
|
||||||
|
} `json:"wire"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadFixture(t *testing.T, name string) fixture {
|
||||||
|
t.Helper()
|
||||||
|
path := filepath.Join("..", "..", "..", "mesh-sdk", "conformance", name)
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Skipf("the sdk's conformance fixtures are not beside this checkout: %v", err)
|
||||||
|
}
|
||||||
|
var f fixture
|
||||||
|
if err := json.Unmarshal(raw, &f); err != nil {
|
||||||
|
t.Fatalf("%s: %v", name, err)
|
||||||
|
}
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every header the fixture requires is one this implementation actually sets.
|
||||||
|
func TestTheGoEmitterSetsEveryRequiredHeader(t *testing.T) {
|
||||||
|
f := loadFixture(t, "events/module-event.json")
|
||||||
|
sent := goEventHeaders(f.Given.Key, f.Given.Module, f.Given.Node)
|
||||||
|
for _, want := range f.Wire.RequiredHeaders {
|
||||||
|
if _, ok := sent[want]; !ok {
|
||||||
|
t.Errorf("the Go emitter does not set %q, which the fixture requires — an event it "+
|
||||||
|
"emits is one a conforming consumer refuses", want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// And each value is in the shape the fixture pins, because a header present but differently
|
||||||
|
// formatted is the disagreement that does not announce itself.
|
||||||
|
func TestTheGoEmittersHeaderFormatsMatch(t *testing.T) {
|
||||||
|
f := loadFixture(t, "events/module-event.json")
|
||||||
|
sent := goEventHeaders(f.Given.Key, f.Given.Module, f.Given.Node)
|
||||||
|
|
||||||
|
if got := sent["content-type"]; got != f.Wire.HeaderFormats["content-type"] {
|
||||||
|
t.Errorf("content-type is %q, the fixture says %q", got, f.Wire.HeaderFormats["content-type"])
|
||||||
|
}
|
||||||
|
if _, err := time.Parse(time.RFC3339, sent["x-time"]); err != nil {
|
||||||
|
t.Errorf("x-time %q is not RFC3339, which the fixture requires: %v", sent["x-time"], err)
|
||||||
|
}
|
||||||
|
if pattern := f.Wire.HeaderFormats["x-event-id"]; pattern != "" {
|
||||||
|
if !regexp.MustCompile(pattern).MatchString(sent["x-event-id"]) {
|
||||||
|
t.Errorf("x-event-id %q does not match %q", sent["x-event-id"], pattern)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The origin the envelope claims is the one the bus enforces by namespace. A disagreement
|
||||||
|
// here means the envelope is lying about where it came from.
|
||||||
|
if sent["x-source"] != f.Given.Module {
|
||||||
|
t.Errorf("x-source is %q for module %q", sent["x-source"], f.Given.Module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The subject a module's event lands on is derived, not carried — so this implementation must
|
||||||
|
// derive the same one the fixture names.
|
||||||
|
func TestTheGoSubjectMatchesTheFixture(t *testing.T) {
|
||||||
|
f := loadFixture(t, "events/module-event.json")
|
||||||
|
got := "mesh.mod." + f.Given.Module + ".event." + f.Given.Key
|
||||||
|
if got != f.Wire.Subject {
|
||||||
|
t.Errorf("this implementation would publish on %q; the fixture says %q", got, f.Wire.Subject)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// goEventHeaders is the header set EmitEvent produces, factored so conformance can see it
|
||||||
|
// without a broker. Kept beside the emitter so the two cannot drift apart silently.
|
||||||
|
func goEventHeaders(eventType, source, node string) map[string]string {
|
||||||
|
id, err := eventID()
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
return map[string]string{
|
||||||
|
"x-event-id": id,
|
||||||
|
"x-source": source,
|
||||||
|
"x-node": node,
|
||||||
|
"x-time": time.Now().UTC().Format(time.RFC3339),
|
||||||
|
"content-type": "application/json",
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,8 +5,6 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
amqp "github.com/rabbitmq/amqp091-go"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Signer is whatever holds the control plane's signing key.
|
// Signer is whatever holds the control plane's signing key.
|
||||||
@@ -21,7 +19,7 @@ type Signer interface {
|
|||||||
// something else, and the node would refuse a declaration that was genuinely the mesh's.
|
// something else, and the node would refuse a declaration that was genuinely the mesh's.
|
||||||
//
|
//
|
||||||
// Published to the node's own queue, which its account alone may read.
|
// Published to the node's own queue, which its account alone may read.
|
||||||
func Declare(ctx context.Context, channel *amqp.Channel, signer Signer, node string,
|
func Declare(ctx context.Context, bus Bus, signer Signer, node string,
|
||||||
declaration []byte, timeout time.Duration) error {
|
declaration []byte, timeout time.Duration) error {
|
||||||
|
|
||||||
if !json.Valid(declaration) {
|
if !json.Valid(declaration) {
|
||||||
@@ -41,13 +39,5 @@ func Declare(ctx context.Context, channel *amqp.Channel, signer Signer, node str
|
|||||||
publish, cancel := context.WithTimeout(ctx, timeout)
|
publish, cancel := context.WithTimeout(ctx, timeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
// Published to the queue directly rather than through the exchange: a declaration is for one
|
return bus.PublishDeclaration(publish, node, body)
|
||||||
// node, and routing it by name through a shared exchange would mean a binding per node that
|
|
||||||
// nothing removes when a node is retired.
|
|
||||||
return channel.PublishWithContext(publish, "", QueueFor(node), false, false,
|
|
||||||
amqp.Publishing{
|
|
||||||
ContentType: "application/json",
|
|
||||||
DeliveryMode: amqp.Persistent,
|
|
||||||
Body: body,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,107 @@
|
|||||||
|
package link_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/ed25519"
|
||||||
|
"crypto/rand"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/crypto/bcrypt"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/identity"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What a node is given to come back with, on the bus being built.
|
||||||
|
//
|
||||||
|
// **The credential becomes usable at the next composition, not when it is made**, which is the one
|
||||||
|
// real difference from the bus the mesh runs on today: there a management call makes it live at once.
|
||||||
|
// So what has to be true here is that the mesh recorded it and told the node, and the rest is a push.
|
||||||
|
|
||||||
|
// aMeshReadyToEnrol is both stores with a signing key established, which a control plane does at
|
||||||
|
// start: one that cannot sign is one whose declarations every node correctly refuses.
|
||||||
|
func aMeshReadyToEnrol(t *testing.T) (*inventory.Inventory, *identity.Identity) {
|
||||||
|
t.Helper()
|
||||||
|
inv := inventory.ForTest(t)
|
||||||
|
ident := identity.ForTest(t)
|
||||||
|
if _, err := ident.Establish(t.Context()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return inv, ident
|
||||||
|
}
|
||||||
|
|
||||||
|
func aTokenFor(t *testing.T, inv *inventory.Inventory, node string) (string, ed25519.PublicKey) {
|
||||||
|
t.Helper()
|
||||||
|
ctx := t.Context()
|
||||||
|
if _, err := inv.AddNode(ctx, node); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
issued, err := inv.IssueToken(ctx, node, time.Hour)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
public, _, err := ed25519.GenerateKey(rand.Reader)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return issued.Secret, public
|
||||||
|
}
|
||||||
|
|
||||||
|
// A node enrolling onto the bus being built is told a password of its own, and the mesh keeps only
|
||||||
|
// its hash — which is what the next composition writes into the bus's user list.
|
||||||
|
func TestANodeEnrollingOnTheNewBusIsMintedACredentialTheMeshOnlyHashes(t *testing.T) {
|
||||||
|
inv, ident := aMeshReadyToEnrol(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
secret, public := aTokenFor(t, inv, "anchor")
|
||||||
|
|
||||||
|
reply, err := link.Enrolment{Inventory: inv, Identity: ident, OnNATS: true}.Enrol(ctx, link.EnrolRequest{
|
||||||
|
Node: "anchor", Secret: secret, PublicKey: public})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if reply.Password == "" {
|
||||||
|
t.Fatal("the node was told no password, so it keeps a one-time secret as a credential")
|
||||||
|
}
|
||||||
|
if reply.Password == secret {
|
||||||
|
t.Fatal("the node was handed the token's own secret back: a credential that lives for years " +
|
||||||
|
"must not be the string that was pasted into a terminal")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recorded under the name the composed file will use, and as a hash: a credential recoverable
|
||||||
|
// from the mesh's store is one whose blast radius is the store's.
|
||||||
|
hash, known, err := inv.BusUserHash(ctx, "node.anchor")
|
||||||
|
if err != nil || !known {
|
||||||
|
t.Fatalf("the mesh kept no credential for the node it enrolled: %v %v", known, err)
|
||||||
|
}
|
||||||
|
if hash == reply.Password {
|
||||||
|
t.Fatal("the store holds the password itself")
|
||||||
|
}
|
||||||
|
if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(reply.Password)); err != nil {
|
||||||
|
t.Fatalf("what the mesh kept does not verify what it told the node: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// On the bus the mesh runs on today, with no management configured, nothing is minted and the node is
|
||||||
|
// told so by being given no password — it keeps the token's secret, which it says out loud.
|
||||||
|
//
|
||||||
|
// **This is the check that the switch is a switch.** A node enrolling on one bus must not come away
|
||||||
|
// with a credential for the other: it would be half-moved, and nothing anywhere would say which half.
|
||||||
|
func TestANodeEnrollingOnTheOldBusIsMintedNoCredentialForTheNewOne(t *testing.T) {
|
||||||
|
inv, ident := aMeshReadyToEnrol(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
secret, public := aTokenFor(t, inv, "anchor")
|
||||||
|
|
||||||
|
reply, err := link.Enrolment{Inventory: inv, Identity: ident}.Enrol(ctx, link.EnrolRequest{
|
||||||
|
Node: "anchor", Secret: secret, PublicKey: public})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if reply.Password != "" {
|
||||||
|
t.Fatalf("a node on the old bus was given a password from nowhere: %q", reply.Password)
|
||||||
|
}
|
||||||
|
if _, known, err := inv.BusUserHash(ctx, "node.anchor"); err != nil || known {
|
||||||
|
t.Fatalf("a node enrolling on the old bus was given a credential for the new one: %v %v",
|
||||||
|
known, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **Verified 2026-09-27**: the caller asked for a reply to `_INBOX.LCr3M83q…` and the consumer
|
||||||
|
// saw `$JS.ACK.PROBE.probe_consumer.1.1.1…`. The design was right, and enrolment's payload-borne
|
||||||
|
// reply subject is necessary rather than defensive.
|
||||||
|
//
|
||||||
|
// Design 25 §2 asserts that a reply address is **eaten** when a message crosses a stream: core
|
||||||
|
// request/reply puts the requester's inbox in the message's Reply field, but a JetStream consumer
|
||||||
|
// has already claimed that field for its own ack address by the time a handler sees it. The whole
|
||||||
|
// enrolment design rests on it — the reply subject travels in the payload instead — so it is
|
||||||
|
// checked rather than believed.
|
||||||
|
//
|
||||||
|
// docker run -d --rm --name t -p 14222:4222 nats:2.10-alpine -js
|
||||||
|
// MESH_TEST_NATS=nats://127.0.0.1:14222 go test ./internal/link/ -run TestAReplyAddress
|
||||||
|
func TestAReplyAddressDoesNotSurviveAStream(t *testing.T) {
|
||||||
|
url := os.Getenv("MESH_TEST_NATS")
|
||||||
|
if url == "" {
|
||||||
|
t.Skip("MESH_TEST_NATS unset")
|
||||||
|
}
|
||||||
|
conn, err := nats.Connect(url)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
js, err := conn.JetStream()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := js.AddStream(&nats.StreamConfig{
|
||||||
|
Name: "PROBE", Subjects: []string{"probe.>"}, Retention: nats.WorkQueuePolicy,
|
||||||
|
}); err != nil && err != nats.ErrStreamNameAlreadyInUse {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer js.DeleteStream("PROBE")
|
||||||
|
|
||||||
|
seen := make(chan *nats.Msg, 1)
|
||||||
|
sub, err := js.Subscribe("probe.enrol", func(m *nats.Msg) { seen <- m },
|
||||||
|
nats.Durable("probe_consumer"), nats.ManualAck())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer sub.Unsubscribe()
|
||||||
|
|
||||||
|
// A caller doing what core request/reply does: publish with its own inbox as the reply.
|
||||||
|
inbox := nats.NewInbox()
|
||||||
|
if err := conn.PublishMsg(&nats.Msg{Subject: "probe.enrol", Reply: inbox, Data: []byte("{}")}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case m := <-seen:
|
||||||
|
t.Logf("the caller asked for a reply to %s", inbox)
|
||||||
|
t.Logf("the consumer sees a Reply field of %s", m.Reply)
|
||||||
|
if m.Reply == inbox {
|
||||||
|
t.Fatalf("the reply address SURVIVED the stream. Design 25 §2 says it does not, and " +
|
||||||
|
"builds enrolment around carrying the reply subject in the payload to work " +
|
||||||
|
"around it. If this holds generally, that work is unnecessary and the design " +
|
||||||
|
"should say so.")
|
||||||
|
}
|
||||||
|
if m.Reply == "" {
|
||||||
|
t.Fatal("the Reply field is empty rather than claimed; the design says it carries " +
|
||||||
|
"the consumer's ack address, which is a different fact")
|
||||||
|
}
|
||||||
|
// Answering it would send the enrolling node's credentials to an ack subject.
|
||||||
|
if len(m.Reply) < 7 || m.Reply[:7] != "$JS.ACK" {
|
||||||
|
t.Errorf("the Reply field is %q, which is neither the caller's inbox nor an ack "+
|
||||||
|
"address — the design's reasoning assumes one of the two", m.Reply)
|
||||||
|
}
|
||||||
|
m.Ack()
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("nothing was delivered")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -28,6 +28,15 @@ type Enrolment struct {
|
|||||||
Identity *identity.Identity
|
Identity *identity.Identity
|
||||||
Management *broker.Management
|
Management *broker.Management
|
||||||
Broker broker.Broker
|
Broker broker.Broker
|
||||||
|
|
||||||
|
// OnNATS says the mesh's own traffic is on the bus being built, so a node's credential is
|
||||||
|
// minted into the mesh's records and composed into the bus's user list rather than pushed
|
||||||
|
// through a management call (novox/hq design 25 §4).
|
||||||
|
//
|
||||||
|
// **One bus, and a node gets a credential for exactly one** — refused at start if the
|
||||||
|
// controller is told about both (broker.MustBeOneBus), because a node holding a credential for
|
||||||
|
// each is one that could be half-moved, and nothing would say which half.
|
||||||
|
OnNATS bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// Enrol records what the node presented and spends the token.
|
// Enrol records what the node presented and spends the token.
|
||||||
@@ -140,7 +149,7 @@ func (e Enrolment) Enrol(ctx context.Context, request EnrolRequest) (reply Enrol
|
|||||||
}
|
}
|
||||||
if err := e.Inventory.RecordTunnel(ctx, node.ID, inventory.Tunnel{
|
if err := e.Inventory.RecordTunnel(ctx, node.ID, inventory.Tunnel{
|
||||||
Interface: request.Tunnel.Interface, Unit: request.Tunnel.Unit,
|
Interface: request.Tunnel.Interface, Unit: request.Tunnel.Unit,
|
||||||
Config: request.Tunnel.Config, Port: request.Tunnel.Port,
|
Config: request.Tunnel.Config, Port: request.Tunnel.Port, MTU: request.Tunnel.MTU,
|
||||||
Address: request.Tunnel.Address, Range: request.Tunnel.Range,
|
Address: request.Tunnel.Address, Range: request.Tunnel.Range,
|
||||||
PublicKey: request.Tunnel.PublicKey, Peers: peers,
|
PublicKey: request.Tunnel.PublicKey, Peers: peers,
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
@@ -159,7 +168,33 @@ func (e Enrolment) Enrol(ctx context.Context, request EnrolRequest) (reply Enrol
|
|||||||
// the spend and not before: a replaced password on an attempt that failed would be held by
|
// the spend and not before: a replaced password on an attempt that failed would be held by
|
||||||
// nobody. If the broker will not take it now, the enrolment still stands — the node keeps
|
// nobody. If the broker will not take it now, the enrolment still stands — the node keeps
|
||||||
// the token's secret as its password, which it is told, and which is said here.
|
// the token's secret as its password, which it is told, and which is said here.
|
||||||
if e.Management != nil {
|
switch {
|
||||||
|
case e.OnNATS:
|
||||||
|
// **Minted into the mesh's records, not pushed to a server.** The bus's users are a file
|
||||||
|
// the controller composes, so a credential becomes usable at the next composition rather
|
||||||
|
// than at the moment it is made — and the plaintext is returned once, here, and then exists
|
||||||
|
// only on the machine it was sealed to.
|
||||||
|
//
|
||||||
|
// The node reconnects as itself and may be refused until that composition reaches the
|
||||||
|
// machine running the bus. That is what the host's reconnect backoff is for and it is
|
||||||
|
// survivable by design (ADR 0004: disconnection is an ordinary situation); waiting for the
|
||||||
|
// push here would hold an enrolment open for as long as a declaration takes to apply.
|
||||||
|
password, err := e.Inventory.MintBusPassword(ctx, inventory.BusUser{
|
||||||
|
Username: broker.Principal{Kind: broker.KindNode, Node: node.Name}.Username(),
|
||||||
|
Kind: inventory.BusNode,
|
||||||
|
Node: node.Name,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
// Not fatal to the enrolment: the node is recorded and the token is spent, and a node
|
||||||
|
// that keeps the token's secret is told so. Said loudly, because until this is minted
|
||||||
|
// the machine has no credential of its own.
|
||||||
|
log.Printf("%s is enrolled and the mesh could not mint its bus credential, so it keeps "+
|
||||||
|
"the token's secret as its password: %v", node.Name, err)
|
||||||
|
} else {
|
||||||
|
reply.Password = password
|
||||||
|
}
|
||||||
|
|
||||||
|
case e.Management != nil:
|
||||||
password, err := freshPassword()
|
password, err := freshPassword()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return EnrolReply{}, err
|
return EnrolReply{}, err
|
||||||
@@ -199,7 +234,7 @@ func (e Enrolment) rekey(ctx context.Context, node inventory.Node, r Rekey) erro
|
|||||||
peers = append(peers, inventory.TunnelPeer{PublicKey: p.PublicKey, Address: p.Address})
|
peers = append(peers, inventory.TunnelPeer{PublicKey: p.PublicKey, Address: p.Address})
|
||||||
}
|
}
|
||||||
err := e.Inventory.Rekey(ctx, node.ID, r.Previous, r.OverlayKey, inventory.Tunnel{
|
err := e.Inventory.Rekey(ctx, node.ID, r.Previous, r.OverlayKey, inventory.Tunnel{
|
||||||
Interface: r.Tunnel.Interface, Unit: r.Tunnel.Unit, Config: r.Tunnel.Config, Port: r.Tunnel.Port,
|
Interface: r.Tunnel.Interface, Unit: r.Tunnel.Unit, Config: r.Tunnel.Config, Port: r.Tunnel.Port, MTU: r.Tunnel.MTU,
|
||||||
Address: r.Tunnel.Address, Range: r.Tunnel.Range, PublicKey: r.Tunnel.PublicKey, Peers: peers,
|
Address: r.Tunnel.Address, Range: r.Tunnel.Range, PublicKey: r.Tunnel.PublicKey, Peers: peers,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -234,6 +269,17 @@ var ErrNoBrokerManagement = errors.New("no broker management configured")
|
|||||||
// A node states; the owning context writes (novox/hq ADR 0006). What a node says it applied is
|
// A node states; the owning context writes (novox/hq ADR 0006). What a node says it applied is
|
||||||
// its own account of its own machine, kept as a copy for recovery — so this writes it down and
|
// its own account of its own machine, kept as a copy for recovery — so this writes it down and
|
||||||
// decides nothing from it.
|
// decides nothing from it.
|
||||||
|
// Outstanding is the declaration the mesh last sent a node, so a report about an older one is not
|
||||||
|
// acted on (design 25 §3, window.go).
|
||||||
|
//
|
||||||
|
// **Here rather than on Listener.** A report is recorded by whatever keeps records, and a great
|
||||||
|
// many things that record reports have no idea what was sent — every test in this package among
|
||||||
|
// them. So the serving loop asks for this when the listener happens to be able to answer, and
|
||||||
|
// where it cannot, a report has nothing to be stale against and is simply acted on.
|
||||||
|
func (e Enrolment) Outstanding(ctx context.Context, node string) (string, error) {
|
||||||
|
return e.Inventory.Outstanding(ctx, node)
|
||||||
|
}
|
||||||
|
|
||||||
func (e Enrolment) Heard(ctx context.Context, report Report) (err error) {
|
func (e Enrolment) Heard(ctx context.Context, report Report) (err error) {
|
||||||
// A store that could not be asked right now is said as such, so the report is kept for
|
// A store that could not be asked right now is said as such, so the report is kept for
|
||||||
// another attempt rather than acknowledged and lost (novox/hq issue 082).
|
// another attempt rather than acknowledged and lost (novox/hq issue 082).
|
||||||
|
|||||||
+7
-25
@@ -6,9 +6,6 @@ import (
|
|||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
|
||||||
|
|
||||||
amqp "github.com/rabbitmq/amqp091-go"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Emitting a module event from Go.
|
// Emitting a module event from Go.
|
||||||
@@ -29,32 +26,17 @@ const (
|
|||||||
|
|
||||||
// EmitEvent publishes one module event, in the envelope the sdk's consumers expect.
|
// EmitEvent publishes one module event, in the envelope the sdk's consumers expect.
|
||||||
//
|
//
|
||||||
// Persistent, because an event that a broker restart loses is not an announcement. The publish is
|
// The envelope is the transport's to write (bus.go) and this is only what goes in it, which is
|
||||||
// not confirmed here: the caller has already done the work the event describes, and a build that
|
// what lets one conformance fixture hold both implementations to the same headers.
|
||||||
// succeeded must not be reported as failed because saying so failed.
|
func EmitEvent(ctx context.Context, bus Bus, eventType, source, node string, body any) error {
|
||||||
func EmitEvent(ctx context.Context, channel *amqp.Channel, eventType, source, node string, body any) error {
|
|
||||||
payload, err := json.Marshal(body)
|
payload, err := json.Marshal(body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("cannot serialise a %s event: %w", eventType, err)
|
return fmt.Errorf("cannot serialise a %s event: %w", eventType, err)
|
||||||
}
|
}
|
||||||
id, err := eventID()
|
// The publish is not confirmed by the caller: it has already done the work the event
|
||||||
if err != nil {
|
// describes, and a build that succeeded must not be reported as failed because saying so
|
||||||
return err
|
// failed. Each transport decides what "published" means for it.
|
||||||
}
|
return bus.PublishEvent(ctx, eventType, source, node, payload)
|
||||||
return channel.PublishWithContext(ctx, EventsExchange, eventType, false, false, amqp.Publishing{
|
|
||||||
ContentType: "application/json",
|
|
||||||
DeliveryMode: amqp.Persistent,
|
|
||||||
MessageId: id,
|
|
||||||
Timestamp: time.Now().UTC(),
|
|
||||||
Body: payload,
|
|
||||||
Headers: amqp.Table{
|
|
||||||
"x-event-id": id,
|
|
||||||
"x-source": source,
|
|
||||||
"x-node": node,
|
|
||||||
"x-time": time.Now().UTC().Format(time.RFC3339),
|
|
||||||
"content-type": "application/json",
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// eventID is what a consumer deduplicates on: delivery is at-least-once, so a handler must be able
|
// eventID is what a consumer deduplicates on: delivery is at-least-once, so a handler must be able
|
||||||
|
|||||||
@@ -80,6 +80,20 @@ type EnrolRequest struct {
|
|||||||
// it. Nil from a node that found none, which is every converged one.
|
// it. Nil from a node that found none, which is every converged one.
|
||||||
Tunnel *Tunnel `json:"tunnel,omitempty"`
|
Tunnel *Tunnel `json:"tunnel,omitempty"`
|
||||||
|
|
||||||
|
// ReplyTo is where the answer goes, as a field of the request rather than the transport's own
|
||||||
|
// reply address.
|
||||||
|
//
|
||||||
|
// **Because a stream eats the transport's field** (design 25 §2, verified against a running
|
||||||
|
// server): a message a JetStream consumer delivers has had its reply field claimed for that
|
||||||
|
// consumer's own ack address, so by the time the controller sees an enrolment, the field names
|
||||||
|
// where the *controller* must acknowledge, not where the node is waiting. An enrolment is the
|
||||||
|
// case that matters — a caller waiting on an ephemeral inbox, over a subject the store window
|
||||||
|
// may legitimately delay by several nak cycles.
|
||||||
|
//
|
||||||
|
// Empty on the bus the mesh runs on today, where the delivery carries the reply queue and the
|
||||||
|
// field means what it has always meant.
|
||||||
|
ReplyTo string `json:"reply_to,omitempty"`
|
||||||
|
|
||||||
// Redelivered is set by the control plane, never sent: the broker handed this request over a
|
// Redelivered is set by the control plane, never sent: the broker handed this request over a
|
||||||
// second time. Such a request does not finish an enrolment already spent — the first time may
|
// second time. Such a request does not finish an enrolment already spent — the first time may
|
||||||
// have answered, and the node holds what it was told.
|
// have answered, and the node holds what it was told.
|
||||||
@@ -93,6 +107,7 @@ type Tunnel struct {
|
|||||||
Unit string `json:"unit"`
|
Unit string `json:"unit"`
|
||||||
Config string `json:"config"`
|
Config string `json:"config"`
|
||||||
Port int `json:"port"`
|
Port int `json:"port"`
|
||||||
|
MTU int `json:"mtu,omitempty"`
|
||||||
Address string `json:"address"`
|
Address string `json:"address"`
|
||||||
Range string `json:"range"`
|
Range string `json:"range"`
|
||||||
PublicKey string `json:"public_key"`
|
PublicKey string `json:"public_key"`
|
||||||
|
|||||||
@@ -0,0 +1,125 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The consume side of the bus, in the mesh's own words.
|
||||||
|
//
|
||||||
|
// The outbound half went behind `Bus` (bus.go) and the transport stopped reaching its callers.
|
||||||
|
// This is the other half, and it is the larger one: everything a node or a module says arrives
|
||||||
|
// here, and until now every handler took the transport's own delivery type — so the serving loop
|
||||||
|
// could not be moved to another bus without moving enrolment, reports, builds, upgrades and
|
||||||
|
// catch-up with it in one breath.
|
||||||
|
//
|
||||||
|
// **Two implementations, both shipping** (novox/hq ADR 0116: nothing moves a node's bus before
|
||||||
|
// step 5). Both shipping is what makes them comparable, and it is what lets the store-window
|
||||||
|
// guarantee (ADR 0083) be stated once — in window.go, pure — rather than twice, once per
|
||||||
|
// transport, where the two would eventually disagree about the thing that matters most.
|
||||||
|
|
||||||
|
// The kinds of message the controller acts on.
|
||||||
|
//
|
||||||
|
// A transport maps its own addressing onto these — a routing key on the bus the mesh has, a
|
||||||
|
// subject on the one being built — and nothing past this point knows which it was. They are never
|
||||||
|
// on the wire: the wire is the transport's business, and a kind that travelled would be a third
|
||||||
|
// name for the same thing.
|
||||||
|
const (
|
||||||
|
KindEnrolment = "enrolment"
|
||||||
|
KindReport = "report"
|
||||||
|
KindHeartbeat = "heartbeat"
|
||||||
|
KindBuilt = "built"
|
||||||
|
KindModuleMoved = "module-moved"
|
||||||
|
KindCatchUp = "catch-up"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Control is one thing a node or a module said, as the controller must act on it.
|
||||||
|
//
|
||||||
|
// **Settling is stated as what the mesh means, not as the transport's verbs.** The two buses
|
||||||
|
// spell them differently — an ack and a reject against a delivery tag, an ack and a term against
|
||||||
|
// a stream sequence — and the guarantee is the same either way: `Took` is done with, `Drop` is
|
||||||
|
// understood and not worth another attempt, and `Hold` is the store window, where the message is
|
||||||
|
// kept and comes back.
|
||||||
|
//
|
||||||
|
// A handler that returns without calling any of the three leaves the message unsettled on
|
||||||
|
// purpose. That is the right answer while shutting down: a cancelled context is not an answer
|
||||||
|
// about a message, and the bus should hand it to whatever consumes next (novox/hq issue 083).
|
||||||
|
type Control interface {
|
||||||
|
// Kind is which of the constants above this is.
|
||||||
|
Kind() string
|
||||||
|
|
||||||
|
// Body is the message itself — the payload alone, never the envelope.
|
||||||
|
Body() []byte
|
||||||
|
|
||||||
|
// Redelivered says the bus has handed this message over before. An enrolment cares and
|
||||||
|
// nothing else does: one already spent is not finished a second time.
|
||||||
|
Redelivered() bool
|
||||||
|
|
||||||
|
// HeldFor is how long this message has been waiting to be taken. Zero on a first delivery.
|
||||||
|
//
|
||||||
|
// **Read from the message rather than remembered by the controller.** On the bus being built
|
||||||
|
// it is the age of the publish, which a controller that restarted mid-window still reads
|
||||||
|
// correctly — the whole reason the holding moves into the server. On the bus the mesh has it
|
||||||
|
// is how long this process has held it, which is the most that transport can say.
|
||||||
|
HeldFor() time.Duration
|
||||||
|
|
||||||
|
// Answer replies to whoever is waiting on this message; only an enrolment expects one.
|
||||||
|
//
|
||||||
|
// Each transport knows where its own answer goes, and they do not agree about it: one carries
|
||||||
|
// a reply queue in the delivery, and on the other the field that would have carried it has
|
||||||
|
// been claimed by the consumer's own ack subject, so the address travels in the payload
|
||||||
|
// (design 25 §2, verified). That difference is exactly what this seam exists to keep out of
|
||||||
|
// the handler.
|
||||||
|
Answer(ctx context.Context, body []byte) error
|
||||||
|
|
||||||
|
// Took settles the message: acted on, or understood and needing no action.
|
||||||
|
Took() error
|
||||||
|
|
||||||
|
// About names what this message is about — a node's report, one module's move, one build's
|
||||||
|
// outcome — and is said before the store is asked.
|
||||||
|
//
|
||||||
|
// A transport that holds messages **in memory** uses it to set aside anything older it is
|
||||||
|
// holding about the same thing: the older is the past, and letting it come back after the
|
||||||
|
// newer was acted on would undo the newer.
|
||||||
|
//
|
||||||
|
// **This is the one thing holding-in-memory can do that holding-in-the-server cannot**, and
|
||||||
|
// naming it here rather than hiding it is deliberate. On the bus being built the message
|
||||||
|
// belongs to the server and comes back whatever happened meanwhile, so this is ignored and the
|
||||||
|
// digest a report carries answers the same question instead (window.go, design 25 §3).
|
||||||
|
About(what string)
|
||||||
|
|
||||||
|
// Hold keeps the message and asks for it again after the delay — the store window.
|
||||||
|
Hold(after time.Duration) error
|
||||||
|
|
||||||
|
// Drop settles the message without acting on it: refused, stale, or given up on. It is not
|
||||||
|
// delivered again.
|
||||||
|
Drop() error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Inbound is where control messages come from.
|
||||||
|
type Inbound interface {
|
||||||
|
// Also asks for one more kind to be delivered.
|
||||||
|
//
|
||||||
|
// **Nothing is subscribed unless something is listening for it.** A durable queue or a
|
||||||
|
// durable stream consumer that nobody reads fills quietly, and the first symptom is a bus out
|
||||||
|
// of disk rather than anything about modules.
|
||||||
|
Also(kind string) error
|
||||||
|
|
||||||
|
// Receive delivers every message to act until the context ends, and says why it stopped.
|
||||||
|
Receive(ctx context.Context, act func(context.Context, Control)) error
|
||||||
|
|
||||||
|
// Close lets go of whatever the implementation holds.
|
||||||
|
Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Outstanding answers which declaration the mesh last sent a node — the digest, not the
|
||||||
|
// declaration.
|
||||||
|
//
|
||||||
|
// **Asked before the store is waited on** (design 25 §3): a report about a declaration the mesh
|
||||||
|
// has already moved past is not worth holding a slot in the window that a current message needs.
|
||||||
|
// It is a separate interface from Listener rather than a method on it, because a controller that
|
||||||
|
// only publishes needs neither and something that records reports need not also be able to say
|
||||||
|
// what was sent.
|
||||||
|
type Outstanding interface {
|
||||||
|
Outstanding(ctx context.Context, node string) (string, error)
|
||||||
|
}
|
||||||
@@ -0,0 +1,317 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
amqp "github.com/rabbitmq/amqp091-go"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The consume side on the bus the mesh runs on today.
|
||||||
|
//
|
||||||
|
// Everything here was the serving loop's until the seam went in: the queues, the binds, the
|
||||||
|
// prefetch, and the list of messages the store could not take yet. It moved rather than changed —
|
||||||
|
// the behaviour this transport has is the behaviour it had, because the mesh is running on it and
|
||||||
|
// a bus nothing speaks yet is no reason to alter the one every node is on (ADR 0116).
|
||||||
|
|
||||||
|
// Prefetch is how many messages the bus hands the controller before it has settled them.
|
||||||
|
//
|
||||||
|
// More than one because a message the store could not take is held, unsettled, while the loop
|
||||||
|
// goes on answering others — an enrolment above all, which a host is waiting on (novox/hq issue
|
||||||
|
// 083). Bounded, because what is held is also what the bus has not kept on its own disk as
|
||||||
|
// pending.
|
||||||
|
const Prefetch = 64
|
||||||
|
|
||||||
|
// PrefetchHeadroom is how much of the prefetch is never held, so the loop always has messages to
|
||||||
|
// answer — an enrolment above all — while others wait for the store.
|
||||||
|
const PrefetchHeadroom = 8
|
||||||
|
|
||||||
|
// TryAgainAfter is how often the held are looked at. A store comes back in seconds, and a report a
|
||||||
|
// few seconds late is still current.
|
||||||
|
const TryAgainAfter = 2 * time.Second
|
||||||
|
|
||||||
|
// currentInbound consumes what nodes say over the bus the mesh has.
|
||||||
|
type currentInbound struct {
|
||||||
|
conn *amqp.Connection
|
||||||
|
channel *amqp.Channel
|
||||||
|
// upgrades and catchups are bound only when something is listening (Also).
|
||||||
|
upgrades bool
|
||||||
|
catchups bool
|
||||||
|
// held is every message the store could not take, by the bus's own delivery tag. Kept here
|
||||||
|
// rather than in the serving loop because holding a delivery unacknowledged is this
|
||||||
|
// transport's way of keeping it, and the other's is to hand it back to the server.
|
||||||
|
held map[uint64]*holding
|
||||||
|
// again is how often the held are looked at; zero means TryAgainAfter. Set by tests.
|
||||||
|
again time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// holding is one message kept for the store, and when to try it again.
|
||||||
|
type holding struct {
|
||||||
|
message *currentControl
|
||||||
|
due time.Time
|
||||||
|
about string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Current is the consume side of the bus the mesh runs on today.
|
||||||
|
func Current(conn *amqp.Connection, channel *amqp.Channel) Inbound {
|
||||||
|
return ¤tInbound{conn: conn, channel: channel, held: map[uint64]*holding{}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Also binds the queue one more kind arrives on.
|
||||||
|
//
|
||||||
|
// The kinds nodes publish all share one queue and are bound at Connect, because a node may
|
||||||
|
// publish any of them and binding one while forgetting another is a message the bus accepts, finds
|
||||||
|
// no queue for, and drops — the publisher sees success and the consumer sees nothing. The two that
|
||||||
|
// are events get their own queue each, and only when something is listening.
|
||||||
|
func (c *currentInbound) Also(kind string) error {
|
||||||
|
switch kind {
|
||||||
|
case KindModuleMoved:
|
||||||
|
if err := c.bindEvent(UpgradeQueue, KeyModuleUpgraded); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
c.upgrades = true
|
||||||
|
case KindCatchUp:
|
||||||
|
if err := c.bindEvent(CatchUpQueue, KeyCatchingUp); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
c.catchups = true
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("nothing binds a queue for %s on this bus", kind)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *currentInbound) bindEvent(queue, key string) error {
|
||||||
|
if _, err := c.channel.QueueDeclare(queue, true, false, false, false, nil); err != nil {
|
||||||
|
return fmt.Errorf("cannot declare the %s queue: %w", queue, err)
|
||||||
|
}
|
||||||
|
if err := c.channel.QueueBind(queue, key, EventsExchange, false, nil); err != nil {
|
||||||
|
return fmt.Errorf("cannot bind %s to %s/%s: %w", queue, EventsExchange, key, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *currentInbound) Close() {}
|
||||||
|
|
||||||
|
// Receive consumes until the context ends.
|
||||||
|
//
|
||||||
|
// One consumer per queue, deliberately: with two on one queue the bus would round-robin between
|
||||||
|
// them and each would receive half of what it expects — a fault this project has already had,
|
||||||
|
// between a module's daemon and its capability server.
|
||||||
|
func (c *currentInbound) Receive(ctx context.Context, act func(context.Context, Control)) error {
|
||||||
|
// A bounded prefetch rather than one. The loop still takes messages one at a time; what the
|
||||||
|
// prefetch buys is that a message the store could not take can be held while the loop goes on
|
||||||
|
// to the next, instead of every enrolment waiting behind it (novox/hq issue 083). Anything
|
||||||
|
// held goes back to the bus if the controller stops, because nothing held is acknowledged.
|
||||||
|
if err := c.channel.Qos(Prefetch, 0, false); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
deliveries, err := c.channel.ConsumeWithContext(ctx, ControlQueue, "control-plane",
|
||||||
|
false, false, false, false, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Its own queue and its own consumer for each event, for the reason above: two consumers on
|
||||||
|
// one queue split its messages between them, and an upgrade or a catch-up request going to
|
||||||
|
// whichever half was not listening is a gap that looks like a working mesh.
|
||||||
|
var upgrades, catchups <-chan amqp.Delivery
|
||||||
|
if c.upgrades {
|
||||||
|
upgrades, err = c.channel.ConsumeWithContext(ctx, UpgradeQueue, "control-plane-upgrades",
|
||||||
|
false, false, false, false, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if c.catchups {
|
||||||
|
catchups, err = c.channel.ConsumeWithContext(ctx, CatchUpQueue, "control-plane-catchup",
|
||||||
|
false, false, false, false, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
closed := c.conn.NotifyClose(make(chan *amqp.Error, 1))
|
||||||
|
|
||||||
|
again := c.again
|
||||||
|
if again == 0 {
|
||||||
|
again = TryAgainAfter
|
||||||
|
}
|
||||||
|
ticker := time.NewTicker(again)
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil
|
||||||
|
case <-ticker.C:
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
c.retryHeld(ctx, act)
|
||||||
|
case delivery, ok := <-catchups:
|
||||||
|
if !ok {
|
||||||
|
if catchups != nil {
|
||||||
|
return errors.New("the bus stopped delivering catch-up requests")
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
act(ctx, c.wrap(KindCatchUp, delivery))
|
||||||
|
case delivery, ok := <-upgrades:
|
||||||
|
// A nil channel blocks for ever, so this case simply never fires when nothing is
|
||||||
|
// listening for upgrades. Closed is different, and means the bus stopped.
|
||||||
|
if !ok {
|
||||||
|
if upgrades != nil {
|
||||||
|
return errors.New("the bus stopped delivering upgrades")
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
act(ctx, c.wrap(KindModuleMoved, delivery))
|
||||||
|
case reason := <-closed:
|
||||||
|
// Said rather than returned quietly. A controller whose bus connection dropped is a
|
||||||
|
// mesh where nothing can be told anything, and the reason is the first thing anybody
|
||||||
|
// will want.
|
||||||
|
return fmt.Errorf("the bus connection closed: %v", reason)
|
||||||
|
case delivery, ok := <-deliveries:
|
||||||
|
if !ok {
|
||||||
|
return errors.New("the bus stopped delivering")
|
||||||
|
}
|
||||||
|
kind, known := kindOfKey[delivery.RoutingKey]
|
||||||
|
if !known {
|
||||||
|
// Rejected without requeue: a message nothing understands will not be understood
|
||||||
|
// on the next attempt either, and requeuing it would spin.
|
||||||
|
_ = delivery.Reject(false)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
act(ctx, c.wrap(kind, delivery))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// kindOfKey is how this transport's addressing becomes what the mesh calls a message.
|
||||||
|
var kindOfKey = map[string]string{
|
||||||
|
KeyEnrol: KindEnrolment,
|
||||||
|
KeyReport: KindReport,
|
||||||
|
KeyAlive: KindHeartbeat,
|
||||||
|
KeyBuilt: KindBuilt,
|
||||||
|
KeyModuleUpgraded: KindModuleMoved,
|
||||||
|
KeyCatchingUp: KindCatchUp,
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *currentInbound) wrap(kind string, delivery amqp.Delivery) *currentControl {
|
||||||
|
return ¤tControl{kind: kind, delivery: delivery, on: c}
|
||||||
|
}
|
||||||
|
|
||||||
|
// retryHeld hands every message whose delay has passed back to the loop. Each handler holds it
|
||||||
|
// again, settles it, or lets it go past the bound.
|
||||||
|
func (c *currentInbound) retryHeld(ctx context.Context, act func(context.Context, Control)) {
|
||||||
|
now := time.Now()
|
||||||
|
due := make([]*currentControl, 0, len(c.held))
|
||||||
|
for _, h := range c.held {
|
||||||
|
if !h.due.After(now) {
|
||||||
|
due = append(due, h.message)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, m := range due {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
act(ctx, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// currentControl is one delivery from the bus the mesh has, as the controller reads it.
|
||||||
|
type currentControl struct {
|
||||||
|
kind string
|
||||||
|
delivery amqp.Delivery
|
||||||
|
on *currentInbound
|
||||||
|
// about is what this message is about, as the handler named it; empty until it does.
|
||||||
|
about string
|
||||||
|
// first is when this message was first held for the store; zero while it has not been.
|
||||||
|
first time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *currentControl) Kind() string { return m.kind }
|
||||||
|
func (m *currentControl) Body() []byte { return m.delivery.Body }
|
||||||
|
func (m *currentControl) Redelivered() bool { return m.delivery.Redelivered }
|
||||||
|
|
||||||
|
func (m *currentControl) HeldFor() time.Duration {
|
||||||
|
if m.first.IsZero() {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return time.Since(m.first)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Answer publishes to the reply queue the request named.
|
||||||
|
func (m *currentControl) Answer(ctx context.Context, body []byte) error {
|
||||||
|
if m.delivery.ReplyTo == "" {
|
||||||
|
return errors.New("that request named no reply queue, so nothing can be told the answer")
|
||||||
|
}
|
||||||
|
return m.on.channel.PublishWithContext(ctx, "", m.delivery.ReplyTo, false, false,
|
||||||
|
amqp.Publishing{
|
||||||
|
ContentType: "application/json",
|
||||||
|
CorrelationId: m.delivery.CorrelationId,
|
||||||
|
Body: body,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *currentControl) Took() error {
|
||||||
|
m.forget()
|
||||||
|
return m.delivery.Ack(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Drop rejects without requeue: on this bus that is what "understood, and not worth another
|
||||||
|
// attempt" is spelled as, and it is what feeds a dead-letter queue where one is configured.
|
||||||
|
func (m *currentControl) Drop() error {
|
||||||
|
m.forget()
|
||||||
|
return m.delivery.Reject(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
// About names what this message is about, and lets go of whatever is held about the same thing:
|
||||||
|
// the held one is the past, and acting on it after this one would undo this one. Acknowledged
|
||||||
|
// rather than left to come back, because a held message nothing will act on is a place in the
|
||||||
|
// prefetch nothing gets back.
|
||||||
|
func (m *currentControl) About(what string) {
|
||||||
|
m.about = what
|
||||||
|
if what == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for tag, h := range m.on.held {
|
||||||
|
if h.about != what || tag == m.delivery.DeliveryTag {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
delete(m.on.held, tag)
|
||||||
|
_ = h.message.delivery.Ack(false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Hold keeps the message unacknowledged and sets it aside to be handed back after the delay.
|
||||||
|
//
|
||||||
|
// Held no further than the prefetch leaves room: past that the bus would hand the loop nothing
|
||||||
|
// new — enrolments included — until something held was let go. A message that cannot be held says
|
||||||
|
// so, and the handler settles it its own way.
|
||||||
|
func (m *currentControl) Hold(after time.Duration) error {
|
||||||
|
if m.on.held == nil {
|
||||||
|
m.on.held = map[uint64]*holding{}
|
||||||
|
}
|
||||||
|
if _, already := m.on.held[m.delivery.DeliveryTag]; !already {
|
||||||
|
if len(m.on.held) >= Prefetch-PrefetchHeadroom {
|
||||||
|
return fmt.Errorf("%d messages are already held for the store, and holding more "+
|
||||||
|
"would stop the queue", len(m.on.held))
|
||||||
|
}
|
||||||
|
m.first = time.Now()
|
||||||
|
}
|
||||||
|
m.on.held[m.delivery.DeliveryTag] = &holding{
|
||||||
|
message: m, due: time.Now().Add(after), about: m.about,
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *currentControl) forget() {
|
||||||
|
if m.on != nil {
|
||||||
|
delete(m.on.held, m.delivery.DeliveryTag)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
amqp "github.com/rabbitmq/amqp091-go"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The harness for the consume side on the bus the mesh runs on today.
|
||||||
|
//
|
||||||
|
// Messages arrive through the seam, so what these tests exercise is the controller's decision
|
||||||
|
// about a message and this transport's way of keeping one — which is what the seam separated. A
|
||||||
|
// fake acknowledger stands in for the bus, because what is asserted is how a message was settled
|
||||||
|
// and that needs no server.
|
||||||
|
|
||||||
|
// settled is how the bus was told to settle one message.
|
||||||
|
type settled struct{ acked, nacked, requeued, rejected bool }
|
||||||
|
|
||||||
|
func (a *settled) Ack(uint64, bool) error { a.acked = true; return nil }
|
||||||
|
func (a *settled) Nack(_ uint64, _ bool, requeue bool) error {
|
||||||
|
a.nacked, a.requeued = true, requeue
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (a *settled) Reject(uint64, bool) error { a.rejected = true; return nil }
|
||||||
|
|
||||||
|
// unsettled is a message the controller has neither taken nor let go: it is held, and the bus will
|
||||||
|
// hand it to whatever consumes next if the controller stops.
|
||||||
|
func (a *settled) unsettled() bool { return !a.acked && !a.nacked && !a.rejected }
|
||||||
|
|
||||||
|
var tag uint64
|
||||||
|
|
||||||
|
func quiet() *log.Logger { return log.New(io.Discard, "", 0) }
|
||||||
|
|
||||||
|
// serving is a controller with nothing but a way of receiving, ready for a listener, a recorder,
|
||||||
|
// an upgrader or a replayer to be set on it.
|
||||||
|
func serving() (*Server, *currentInbound) {
|
||||||
|
in := ¤tInbound{held: map[uint64]*holding{}}
|
||||||
|
return &Server{inbound: in, bus: OverCurrent{}, log: quiet()}, in
|
||||||
|
}
|
||||||
|
|
||||||
|
// sends is one message arriving over this transport, as the controller reads it.
|
||||||
|
func (c *currentInbound) sends(t *testing.T, to *settled, kind string, v any) Control {
|
||||||
|
t.Helper()
|
||||||
|
body, err := json.Marshal(v)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
tag++
|
||||||
|
return ¤tControl{kind: kind, on: c, delivery: amqp.Delivery{
|
||||||
|
Acknowledger: to, Body: body, DeliveryTag: tag,
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// dueNow brings every held message forward, so a test need not wait out the backoff a real store
|
||||||
|
// restart would be given (RedeliverAfter).
|
||||||
|
func (c *currentInbound) dueNow() {
|
||||||
|
for _, h := range c.held {
|
||||||
|
h.due = time.Now().Add(-time.Second)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// retries hands every held message back to the controller, the way the ticker does.
|
||||||
|
func (c *currentInbound) retries(ctx context.Context, s *Server) {
|
||||||
|
c.dueNow()
|
||||||
|
c.retryHeld(ctx, s.act)
|
||||||
|
}
|
||||||
@@ -0,0 +1,290 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The consume side on the bus being built.
|
||||||
|
//
|
||||||
|
// The shape is the AMQP one's, because the seam made them comparable: one loop, one message at a
|
||||||
|
// time, and the same window deciding. What differs is where a held message lives — and that is the
|
||||||
|
// whole point of the move. On the bus the mesh has, holding one means keeping an unacknowledged
|
||||||
|
// delivery in this process, bounded by the prefetch and lost if the controller stops. Here it is a
|
||||||
|
// `nak` with a delay: the message stays the server's, the controller keeps nothing but the moment
|
||||||
|
// it first could not take it, and a controller that restarts mid-window has nothing to lose.
|
||||||
|
|
||||||
|
// natsInbound consumes what nodes and modules say over NATS.
|
||||||
|
type natsInbound struct {
|
||||||
|
js *broker.JetStream
|
||||||
|
// follows is the kinds asked for beyond what nodes say (Also). The events those are are the
|
||||||
|
// only ones the controller subscribes, and only when something is listening.
|
||||||
|
follows map[string]bool
|
||||||
|
// since is when the controller first could not take a message, by that message's place in its
|
||||||
|
// stream.
|
||||||
|
//
|
||||||
|
// **A timestamp, not a message.** This is the whole difference the move buys: the AMQP side
|
||||||
|
// keeps the delivery, and this keeps eight bytes saying when the window opened. A controller
|
||||||
|
// that restarts loses these and starts the window again, which is correct — it is holding
|
||||||
|
// nothing, and the messages are all still on the server.
|
||||||
|
since map[uint64]time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// Nats is the consume side of the bus being built.
|
||||||
|
func Nats(js *broker.JetStream) Inbound {
|
||||||
|
return &natsInbound{js: js, follows: map[string]bool{}, since: map[uint64]time.Time{}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Also records one more kind to subscribe. Nothing is subscribed here: the controller's consumer on
|
||||||
|
// the events stream carries both of these as filters, so it is created once, in Receive, with
|
||||||
|
// whatever was asked for — and not at all when nothing was.
|
||||||
|
func (n *natsInbound) Also(kind string) error {
|
||||||
|
switch kind {
|
||||||
|
case KindModuleMoved, KindCatchUp:
|
||||||
|
n.follows[kind] = true
|
||||||
|
return nil
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("nothing subscribes %s separately on this bus", kind)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *natsInbound) Close() {}
|
||||||
|
|
||||||
|
// Receive consumes until the context ends.
|
||||||
|
//
|
||||||
|
// Three subscriptions, and each is a channel the one loop selects on. **Channels rather than
|
||||||
|
// callbacks**: the library would run a handler on its own goroutine, and the window's bookkeeping —
|
||||||
|
// which message is held, and since when — is read and written without a lock because the AMQP loop
|
||||||
|
// never had two. A second goroutine would make that wrong in a way no test would catch.
|
||||||
|
func (n *natsInbound) Receive(ctx context.Context, act func(context.Context, Control)) error {
|
||||||
|
if err := broker.AssertMeshConsumers(n.js); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
js, conn := n.js.Context(), n.js.Conn()
|
||||||
|
|
||||||
|
// What nodes say, off the CONTROL stream. Bound to the durable the controller asserted rather
|
||||||
|
// than creating one here: the consumer is an object with a configuration — ack policy, ack
|
||||||
|
// wait, redelivery — and a client that creates its own would be a second opinion about it.
|
||||||
|
control := make(chan *nats.Msg, Prefetch)
|
||||||
|
said, err := js.ChanSubscribe("", control, nats.Bind("CONTROL", broker.ControllerName))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("subscribing to what nodes say: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = said.Unsubscribe() }()
|
||||||
|
|
||||||
|
// Heartbeats, on core NATS and off any stream (design 25 §3). Their own subscription because
|
||||||
|
// they are their own guarantee: a lost one is the next one.
|
||||||
|
beats := make(chan *nats.Msg, Prefetch)
|
||||||
|
alive, err := conn.ChanSubscribe(AliveSubjects, beats)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("subscribing to heartbeats: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = alive.Unsubscribe() }()
|
||||||
|
|
||||||
|
// The events the controller follows, when something is listening for them.
|
||||||
|
var events chan *nats.Msg
|
||||||
|
if len(n.follows) > 0 {
|
||||||
|
events = make(chan *nats.Msg, Prefetch)
|
||||||
|
followed, err := js.ChanSubscribe("", events, nats.Bind("EVENTS", broker.ControllerName))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("subscribing to what the catalogue says: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = followed.Unsubscribe() }()
|
||||||
|
}
|
||||||
|
|
||||||
|
// A connection that dropped is said, not discovered. A controller whose bus connection is gone
|
||||||
|
// is a mesh where nothing can be told anything.
|
||||||
|
gone := make(chan error, 1)
|
||||||
|
conn.SetDisconnectErrHandler(func(_ *nats.Conn, err error) {
|
||||||
|
select {
|
||||||
|
case gone <- err:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil
|
||||||
|
case err := <-gone:
|
||||||
|
return fmt.Errorf("the bus connection dropped: %w", err)
|
||||||
|
case msg := <-beats:
|
||||||
|
n.deliver(ctx, act, msg, false)
|
||||||
|
case msg := <-events:
|
||||||
|
n.deliver(ctx, act, msg, true)
|
||||||
|
case msg, ok := <-control:
|
||||||
|
if !ok {
|
||||||
|
return errors.New("the bus stopped delivering")
|
||||||
|
}
|
||||||
|
n.deliver(ctx, act, msg, true)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// deliver names one message and hands it to the loop, or drops it where the mesh has no name for
|
||||||
|
// its subject — which cannot happen through a filter the controller wrote, and is said rather than
|
||||||
|
// ignored for exactly that reason.
|
||||||
|
func (n *natsInbound) deliver(ctx context.Context, act func(context.Context, Control),
|
||||||
|
msg *nats.Msg, streamed bool) {
|
||||||
|
|
||||||
|
kind, known := kindOfSubject(msg.Subject)
|
||||||
|
if !known {
|
||||||
|
if streamed {
|
||||||
|
_ = msg.Term()
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
m := &natsControl{kind: kind, msg: msg, on: n}
|
||||||
|
if streamed {
|
||||||
|
// A message with no metadata is not from a stream, whatever it was delivered on, and the
|
||||||
|
// window has nothing to hold it by. Said by leaving the sequence at zero.
|
||||||
|
if meta, err := msg.Metadata(); err == nil {
|
||||||
|
m.seq = meta.Sequence.Stream
|
||||||
|
m.delivered = meta.NumDelivered
|
||||||
|
}
|
||||||
|
}
|
||||||
|
act(ctx, m)
|
||||||
|
}
|
||||||
|
|
||||||
|
// kindOfSubject is how this transport's addressing becomes what the mesh calls a message.
|
||||||
|
//
|
||||||
|
// By subject, which is the only thing the server enforces: a body claiming to be a report does not
|
||||||
|
// make it one, and on this bus the subject an account may publish *is* its authority (design 29
|
||||||
|
// §2). The mirror of the routing-key table on the bus the mesh has.
|
||||||
|
func kindOfSubject(subject string) (string, bool) {
|
||||||
|
switch subject {
|
||||||
|
case EnrolSubject:
|
||||||
|
return KindEnrolment, true
|
||||||
|
case BuiltSubject:
|
||||||
|
return KindBuilt, true
|
||||||
|
}
|
||||||
|
if node, rest, ok := strings.Cut(strings.TrimPrefix(subject, "mesh.control."), "."); ok &&
|
||||||
|
node != "" && !strings.Contains(node, ".") {
|
||||||
|
switch rest {
|
||||||
|
case "report":
|
||||||
|
return KindReport, true
|
||||||
|
case "alive":
|
||||||
|
return KindHeartbeat, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
switch subject {
|
||||||
|
case broker.ControllerFollows[0]:
|
||||||
|
return KindModuleMoved, true
|
||||||
|
case broker.ControllerFollows[1]:
|
||||||
|
return KindCatchUp, true
|
||||||
|
case BuildOutcome():
|
||||||
|
// A build's outcome is the role's event now, so it arrives on the events stream rather than
|
||||||
|
// the control branch — and is acted on by the same handler, because what the controller does
|
||||||
|
// with it did not change (novox/hq ADR 0121).
|
||||||
|
return KindBuilt, true
|
||||||
|
}
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
|
||||||
|
// natsControl is one message from the bus being built, as the controller reads it.
|
||||||
|
type natsControl struct {
|
||||||
|
kind string
|
||||||
|
msg *nats.Msg
|
||||||
|
on *natsInbound
|
||||||
|
// seq is this message's place in its stream; zero for a core message, which has none and
|
||||||
|
// cannot be held.
|
||||||
|
seq uint64
|
||||||
|
// delivered is how many times the server has handed this message over, this time included.
|
||||||
|
delivered uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *natsControl) Kind() string { return m.kind }
|
||||||
|
func (m *natsControl) Body() []byte { return m.msg.Data }
|
||||||
|
|
||||||
|
// Redelivered is what the server counted, not what the controller remembers. Which is the answer to
|
||||||
|
// a question the AMQP side could only guess at across a restart: an enrolment redelivered because
|
||||||
|
// the controller stopped mid-answer reads as redelivered to the controller that comes back.
|
||||||
|
func (m *natsControl) Redelivered() bool { return m.delivered > 1 }
|
||||||
|
|
||||||
|
func (m *natsControl) HeldFor() time.Duration {
|
||||||
|
if m.seq == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
first, held := m.on.since[m.seq]
|
||||||
|
if !held {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return time.Since(first)
|
||||||
|
}
|
||||||
|
|
||||||
|
// About is nothing here, and that is the point.
|
||||||
|
//
|
||||||
|
// Setting a held message aside when a newer one about the same thing arrives is what a controller
|
||||||
|
// holding deliveries in memory can do. A naked message belongs to the server and comes back
|
||||||
|
// whatever happened meanwhile, so the question "is this the past?" is answered by what the message
|
||||||
|
// says instead — the digest of the declaration a report is about (window.go, design 25 §3).
|
||||||
|
func (m *natsControl) About(string) {}
|
||||||
|
|
||||||
|
// Answer publishes to the reply subject the request carries **in its payload**.
|
||||||
|
//
|
||||||
|
// Not `Respond`, and not the message's reply field: a message a JetStream consumer delivers has had
|
||||||
|
// that field claimed for the consumer's own ack address, so answering it would send the reply to
|
||||||
|
// `$JS.ACK.CONTROL.controller.…` and the enrolling node would wait out its timeout. Verified
|
||||||
|
// against a running server (design 25 §2), which is why it is a field of the request and this reads
|
||||||
|
// it from there.
|
||||||
|
func (m *natsControl) Answer(ctx context.Context, body []byte) error {
|
||||||
|
var addressed replyAddressed
|
||||||
|
if err := json.Unmarshal(m.msg.Data, &addressed); err != nil {
|
||||||
|
return fmt.Errorf("that request cannot be read, so its reply address cannot be: %w", err)
|
||||||
|
}
|
||||||
|
if addressed.ReplyTo == "" {
|
||||||
|
return errors.New("that request named no reply subject in its payload, so nothing can be " +
|
||||||
|
"told the answer")
|
||||||
|
}
|
||||||
|
return m.on.js.Conn().PublishMsg(&nats.Msg{Subject: addressed.ReplyTo, Data: body})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *natsControl) Took() error {
|
||||||
|
m.forget()
|
||||||
|
if m.seq == 0 {
|
||||||
|
// Core NATS: nothing is keeping it, so there is nothing to settle.
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return m.msg.Ack(nats.Context(context.Background()))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Drop terminates the delivery: understood, and the server is told not to send it again. Different
|
||||||
|
// from an ack only in the server's own accounting, which is where somebody asking "what happened to
|
||||||
|
// that message" will look.
|
||||||
|
func (m *natsControl) Drop() error {
|
||||||
|
m.forget()
|
||||||
|
if m.seq == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return m.msg.Term()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Hold hands the message back with a delay, and remembers when the window opened.
|
||||||
|
func (m *natsControl) Hold(after time.Duration) error {
|
||||||
|
if m.seq == 0 {
|
||||||
|
return errors.New("a message that is not in a stream cannot be held: nothing is keeping it")
|
||||||
|
}
|
||||||
|
if _, already := m.on.since[m.seq]; !already {
|
||||||
|
m.on.since[m.seq] = time.Now()
|
||||||
|
}
|
||||||
|
return m.msg.NakWithDelay(after)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *natsControl) forget() {
|
||||||
|
if m.on != nil && m.seq != 0 {
|
||||||
|
delete(m.on.since, m.seq)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// replyAddressed is the one field every message that expects an answer carries.
|
||||||
|
type replyAddressed struct {
|
||||||
|
ReplyTo string `json:"reply_to,omitempty"`
|
||||||
|
}
|
||||||
@@ -0,0 +1,376 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The consume side against a real server, because what is being checked is what the server does.
|
||||||
|
//
|
||||||
|
// Reasoning cannot answer any of these: whether a nak-with-delay really comes back, whether the
|
||||||
|
// delay is honoured, whether terminating a delivery really stops it, or whether an answer published
|
||||||
|
// to an address carried in the payload reaches a caller waiting on its own inbox. Each is a claim
|
||||||
|
// about a server, so each is asked of one:
|
||||||
|
//
|
||||||
|
// docker run -d --rm --name t -p 14222:4222 nats:2.10-alpine -js
|
||||||
|
// MESH_TEST_NATS=nats://127.0.0.1:14222 go test ./internal/link/ -run TestNats
|
||||||
|
|
||||||
|
func aBus(t *testing.T) *broker.JetStream {
|
||||||
|
t.Helper()
|
||||||
|
url := os.Getenv("MESH_TEST_NATS")
|
||||||
|
if url == "" {
|
||||||
|
t.Skip("MESH_TEST_NATS unset")
|
||||||
|
}
|
||||||
|
js, err := broker.Dial(url)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(js.Close)
|
||||||
|
|
||||||
|
// **Streams purged, consumers removed.** Both halves, and each was learned by getting it wrong.
|
||||||
|
//
|
||||||
|
// The streams are emptied rather than deleted and recreated, because delete-then-add is not a
|
||||||
|
// reset: the server's teardown races the creation, and a test then inherits the previous one's
|
||||||
|
// messages — which reads as a redelivery bug in the code under test.
|
||||||
|
//
|
||||||
|
// The consumers are removed, because deleting a stream used to take them with it and purging
|
||||||
|
// does not. A durable *push* consumer that survives between tests keeps pushing to a delivery
|
||||||
|
// subject the previous test's subscription has gone from: the messages count as delivered, go
|
||||||
|
// nowhere, and the next test waits out its timeout for an announcement the server believes it
|
||||||
|
// already sent. The controller recreates what it needs on start, so leaving none is correct.
|
||||||
|
if err := broker.AssertMeshStreams(js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
clean := func() {
|
||||||
|
for _, c := range broker.MeshConsumers() {
|
||||||
|
_ = js.Context().DeleteConsumer(c.Stream, c.Name)
|
||||||
|
}
|
||||||
|
for _, s := range broker.MeshStreams() {
|
||||||
|
_ = js.Context().PurgeStream(s.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
clean()
|
||||||
|
t.Cleanup(clean)
|
||||||
|
return js
|
||||||
|
}
|
||||||
|
|
||||||
|
// serving1 is a controller reading from a real bus, and a way to stop it.
|
||||||
|
func servingOn(t *testing.T, js *broker.JetStream, l Listener) (*Server, func()) {
|
||||||
|
t.Helper()
|
||||||
|
s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()},
|
||||||
|
listener: l, log: quiet()}
|
||||||
|
ctx, stop := context.WithCancel(context.Background())
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() { defer close(done); _ = s.Serve(ctx) }()
|
||||||
|
return s, func() {
|
||||||
|
stop()
|
||||||
|
<-done
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// counted records reports and can be told to refuse them, from another goroutine.
|
||||||
|
type counted struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
err error
|
||||||
|
heard []Report
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *counted) Heard(_ context.Context, r Report) error {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
if c.err != nil {
|
||||||
|
return c.err
|
||||||
|
}
|
||||||
|
c.heard = append(c.heard, r)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *counted) refusing(err error) {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
c.err = err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *counted) count() int {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
return len(c.heard)
|
||||||
|
}
|
||||||
|
|
||||||
|
func eventually(t *testing.T, what string, is func() bool) {
|
||||||
|
t.Helper()
|
||||||
|
deadline := time.Now().Add(8 * time.Second)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if is() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
|
}
|
||||||
|
t.Fatalf("%s did not happen within the wait", what)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A report published by a node reaches the controller, is recorded, and is acknowledged — so the
|
||||||
|
// stream does not hold it. A work queue is the check: what is acknowledged leaves it.
|
||||||
|
func TestNatsAReportIsHeardAndLeavesTheStream(t *testing.T) {
|
||||||
|
js := aBus(t)
|
||||||
|
store := &counted{}
|
||||||
|
_, stop := servingOn(t, js, store)
|
||||||
|
defer stop()
|
||||||
|
|
||||||
|
body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}})
|
||||||
|
if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
eventually(t, "a report being recorded", func() bool { return store.count() == 1 })
|
||||||
|
eventually(t, "the report leaving the work queue", func() bool {
|
||||||
|
info, err := js.Context().StreamInfo("CONTROL")
|
||||||
|
return err == nil && info.State.Msgs == 0
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The store window, in the server.** A report the store cannot take is naked with a delay and
|
||||||
|
// comes back; once the store is there it is recorded and leaves the stream. The controller holds
|
||||||
|
// nothing in the meantime — which is what the sequence check below is for: the message is still on
|
||||||
|
// the server while it waits.
|
||||||
|
func TestNatsAReportTheStoreCannotTakeIsHeldByTheServerAndComesBack(t *testing.T) {
|
||||||
|
js := aBus(t)
|
||||||
|
store := &counted{}
|
||||||
|
store.refusing(errors.Join(ErrTryAgain, errors.New("the database system is starting up")))
|
||||||
|
_, stop := servingOn(t, js, store)
|
||||||
|
defer stop()
|
||||||
|
|
||||||
|
body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}})
|
||||||
|
if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Held: the message is the server's, unacknowledged, and still in the stream.
|
||||||
|
eventually(t, "the report being redelivered at least once", func() bool {
|
||||||
|
info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName)
|
||||||
|
return err == nil && info.NumRedelivered >= 1
|
||||||
|
})
|
||||||
|
info, err := js.Context().StreamInfo("CONTROL")
|
||||||
|
if err != nil || info.State.Msgs != 1 {
|
||||||
|
t.Fatalf("a held report did not stay on the server: %+v, %v", info, err)
|
||||||
|
}
|
||||||
|
if store.count() != 0 {
|
||||||
|
t.Fatalf("a report was recorded by a store that was refusing it")
|
||||||
|
}
|
||||||
|
|
||||||
|
store.refusing(nil)
|
||||||
|
eventually(t, "the report being recorded once the store was back",
|
||||||
|
func() bool { return store.count() == 1 })
|
||||||
|
eventually(t, "the recorded report leaving the work queue", func() bool {
|
||||||
|
info, err := js.Context().StreamInfo("CONTROL")
|
||||||
|
return err == nil && info.State.Msgs == 0
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// A report about a declaration the mesh has moved past is settled without being acted on, and
|
||||||
|
// leaves the stream rather than coming back for ever.
|
||||||
|
func TestNatsASupersededReportIsSettledAndNotActedOn(t *testing.T) {
|
||||||
|
js := aBus(t)
|
||||||
|
store := &sentAndHeardSafely{sent: "d2"}
|
||||||
|
_, stop := servingOn(t, js, store)
|
||||||
|
defer stop()
|
||||||
|
|
||||||
|
body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}})
|
||||||
|
if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
eventually(t, "the superseded report leaving the stream", func() bool {
|
||||||
|
info, err := js.Context().StreamInfo("CONTROL")
|
||||||
|
return err == nil && info.State.Msgs == 0
|
||||||
|
})
|
||||||
|
if store.count() != 0 {
|
||||||
|
t.Fatalf("a report about a superseded declaration was acted on")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// sentAndHeardSafely is sentAndHeard, read from two goroutines.
|
||||||
|
type sentAndHeardSafely struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
sent string
|
||||||
|
heard []Report
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *sentAndHeardSafely) Heard(_ context.Context, r Report) error {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
s.heard = append(s.heard, r)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *sentAndHeardSafely) Outstanding(context.Context, string) (string, error) {
|
||||||
|
return s.sent, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *sentAndHeardSafely) count() int {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
return len(s.heard)
|
||||||
|
}
|
||||||
|
|
||||||
|
// **An enrolment answered through a reply address the stream would have eaten.**
|
||||||
|
//
|
||||||
|
// The caller waits on its own inbox and states that address in the request's payload. The check is
|
||||||
|
// that the answer arrives there — which is the whole reason the address is a field rather than the
|
||||||
|
// transport's reply, and this is the test design 25 §2 asks for so the reason cannot quietly become
|
||||||
|
// folklore.
|
||||||
|
func TestNatsAnEnrolmentIsAnsweredOnTheAddressInItsPayload(t *testing.T) {
|
||||||
|
js := aBus(t)
|
||||||
|
s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()},
|
||||||
|
enroller: enrolsAs{reply: EnrolReply{Accepted: true, Node: "anchor"}}, log: quiet()}
|
||||||
|
ctx, stop := context.WithCancel(context.Background())
|
||||||
|
defer stop()
|
||||||
|
go func() { _ = s.Serve(ctx) }()
|
||||||
|
|
||||||
|
inbox := nats.NewInbox()
|
||||||
|
answers, err := js.Conn().SubscribeSync(inbox)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
body, _ := json.Marshal(EnrolRequest{Node: "anchor", Secret: "t", ReplyTo: inbox})
|
||||||
|
if _, err := js.Context().Publish(EnrolSubject, body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
msg, err := answers.NextMsg(8 * time.Second)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("no answer reached the address the request named: %v", err)
|
||||||
|
}
|
||||||
|
var reply EnrolReply
|
||||||
|
if err := json.Unmarshal(msg.Data, &reply); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !reply.Accepted || reply.Node != "anchor" {
|
||||||
|
t.Fatalf("the answer was not the mesh's: %+v", reply)
|
||||||
|
}
|
||||||
|
// And the address really is not the one the transport carried: what the consumer saw was its
|
||||||
|
// own ack subject, which is why this had to travel in the payload.
|
||||||
|
if msg.Subject != inbox {
|
||||||
|
t.Fatalf("the answer arrived on %s, not the address the request named", msg.Subject)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// enrolsAs answers every request the same way.
|
||||||
|
type enrolsAs struct{ reply EnrolReply }
|
||||||
|
|
||||||
|
func (e enrolsAs) Enrol(context.Context, EnrolRequest) (EnrolReply, error) { return e.reply, nil }
|
||||||
|
|
||||||
|
// A heartbeat is core NATS: it reaches the controller and nothing is persisted, so the stream the
|
||||||
|
// reports live in stays empty.
|
||||||
|
func TestNatsAHeartbeatIsHeardAndNothingIsKept(t *testing.T) {
|
||||||
|
js := aBus(t)
|
||||||
|
store := &counted{}
|
||||||
|
_, stop := servingOn(t, js, store)
|
||||||
|
defer stop()
|
||||||
|
|
||||||
|
// Given time to subscribe: a core subscription that is not yet up misses what is published,
|
||||||
|
// which is the guarantee a heartbeat has and not a fault.
|
||||||
|
eventually(t, "the heartbeat subscription coming up", func() bool {
|
||||||
|
body, _ := json.Marshal(Alive{Node: "anchor"})
|
||||||
|
_ = js.Conn().Publish(AliveSubject("anchor"), body)
|
||||||
|
_ = js.Conn().Flush()
|
||||||
|
return store.count() >= 1
|
||||||
|
})
|
||||||
|
info, err := js.Context().StreamInfo("CONTROL")
|
||||||
|
if err != nil || info.State.Msgs != 0 {
|
||||||
|
t.Fatalf("a heartbeat was persisted, and the mesh's least valuable message now competes "+
|
||||||
|
"for retention with its most valuable: %+v, %v", info, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The two events the controller follows arrive over one durable consumer with two filters, and it
|
||||||
|
// can acknowledge them.
|
||||||
|
//
|
||||||
|
// **Both halves are the point.** A consumer with several filter subjects is a 2.10 feature and this
|
||||||
|
// is the first thing in the mesh to use one; and a delivery from the events stream is acknowledged
|
||||||
|
// on a different ack subject from a delivery from the control stream, which the controller's own
|
||||||
|
// permission list has to cover or every announcement is redelivered for ever.
|
||||||
|
func TestNatsTheEventsTheControllerFollowsArriveAndAreAcknowledged(t *testing.T) {
|
||||||
|
js := aBus(t)
|
||||||
|
told := &toldAbout{}
|
||||||
|
s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()}
|
||||||
|
if err := s.Follows(told); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := s.Answers(replaysWith{}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, stop := context.WithCancel(context.Background())
|
||||||
|
defer stop()
|
||||||
|
go func() { _ = s.Serve(ctx) }()
|
||||||
|
|
||||||
|
moved, _ := json.Marshal(Upgraded{Module: "gitea", Commit: "abcdef0123"})
|
||||||
|
if _, err := js.Context().Publish(broker.ControllerFollows[0], moved); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := js.Context().Publish(broker.ControllerFollows[1], []byte(`{}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
eventually(t, "the catalogue's upgrade reaching the controller",
|
||||||
|
func() bool { return told.count() == 1 })
|
||||||
|
eventually(t, "both announcements being acknowledged", func() bool {
|
||||||
|
info, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName)
|
||||||
|
return err == nil && info.NumAckPending == 0 && info.Delivered.Consumer == 2
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
type toldAbout struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
saw []Upgraded
|
||||||
|
fail error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u *toldAbout) Upgraded(_ context.Context, m Upgraded) error {
|
||||||
|
u.mu.Lock()
|
||||||
|
defer u.mu.Unlock()
|
||||||
|
if u.fail != nil {
|
||||||
|
return u.fail
|
||||||
|
}
|
||||||
|
u.saw = append(u.saw, m)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u *toldAbout) count() int {
|
||||||
|
u.mu.Lock()
|
||||||
|
defer u.mu.Unlock()
|
||||||
|
return len(u.saw)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A store that never comes back: the report is let go once the bound passes, and it leaves the
|
||||||
|
// stream rather than being held for ever. The bound is the controller's, not the server's — nothing
|
||||||
|
// here sets max-deliver, and that is deliberate (streams.go).
|
||||||
|
func TestNatsAReportIsLetGoOnceTheStoreHasBeenGoneTooLong(t *testing.T) {
|
||||||
|
js := aBus(t)
|
||||||
|
store := &counted{}
|
||||||
|
store.refusing(errors.Join(ErrTryAgain, errors.New("connection refused")))
|
||||||
|
s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()},
|
||||||
|
listener: store, log: quiet(), giveUp: 1500 * time.Millisecond}
|
||||||
|
ctx, stop := context.WithCancel(context.Background())
|
||||||
|
defer stop()
|
||||||
|
go func() { _ = s.Serve(ctx) }()
|
||||||
|
|
||||||
|
body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}})
|
||||||
|
if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
eventually(t, "the report being let go once the bound passed", func() bool {
|
||||||
|
info, err := js.Context().StreamInfo("CONTROL")
|
||||||
|
return err == nil && info.State.Msgs == 0
|
||||||
|
})
|
||||||
|
if store.count() != 0 {
|
||||||
|
t.Fatalf("a report was recorded by a store that never came back")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,26 +2,11 @@ package link
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
"errors"
|
||||||
"io"
|
|
||||||
"log"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
amqp "github.com/rabbitmq/amqp091-go"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type saidTo struct{ acked, nacked, requeued bool }
|
|
||||||
|
|
||||||
func (a *saidTo) Ack(uint64, bool) error { a.acked = true; return nil }
|
|
||||||
func (a *saidTo) Nack(_ uint64, _ bool, requeue bool) error {
|
|
||||||
a.nacked, a.requeued = true, requeue
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
func (a *saidTo) Reject(uint64, bool) error { return nil }
|
|
||||||
func (a *saidTo) unsettled() bool { return !a.acked && !a.nacked }
|
|
||||||
|
|
||||||
type heardWith struct{ err error }
|
type heardWith struct{ err error }
|
||||||
|
|
||||||
func (h heardWith) Heard(context.Context, Report) error { return h.err }
|
func (h heardWith) Heard(context.Context, Report) error { return h.err }
|
||||||
@@ -31,39 +16,31 @@ type switchable struct{ err error }
|
|||||||
|
|
||||||
func (h *switchable) Heard(context.Context, Report) error { return h.err }
|
func (h *switchable) Heard(context.Context, Report) error { return h.err }
|
||||||
|
|
||||||
var tag uint64
|
func aReport(node, declared string) Report {
|
||||||
|
return Report{Node: node, Declared: declared, Applied: []string{"store"}}
|
||||||
func aReport(t *testing.T, to *saidTo, node, declared string) amqp.Delivery {
|
|
||||||
t.Helper()
|
|
||||||
body, err := json.Marshal(Report{Node: node, Declared: declared, Applied: []string{"store"}})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
tag++
|
|
||||||
return amqp.Delivery{Acknowledger: to, RoutingKey: KeyReport, Body: body, DeliveryTag: tag}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func quiet() *log.Logger { return log.New(io.Discard, "", 0) }
|
|
||||||
|
|
||||||
// A report the store could not take right now is held, unsettled, and recorded when the store is
|
// A report the store could not take right now is held, unsettled, and recorded when the store is
|
||||||
// back; one the store answered no to is acknowledged; one recorded is acknowledged (issue 082, 083).
|
// back; one the store answered no to is acknowledged; one recorded is acknowledged (issue 082, 083).
|
||||||
func TestAReportTheStoreCouldNotTakeIsHeldAndOneItRefusedIsNot(t *testing.T) {
|
func TestAReportTheStoreCouldNotTakeIsHeldAndOneItRefusedIsNot(t *testing.T) {
|
||||||
store := &switchable{err: errors.Join(ErrTryAgain, errors.New("starting up"))}
|
store := &switchable{err: errors.Join(ErrTryAgain, errors.New("starting up"))}
|
||||||
s := &Server{listener: store, log: quiet()}
|
s, in := serving()
|
||||||
held := &saidTo{}
|
s.listener = store
|
||||||
s.handleReport(context.Background(), aReport(t, held, "anchor", "d1"))
|
held := &settled{}
|
||||||
if !held.unsettled() || len(s.parked) != 1 {
|
s.act(context.Background(), in.sends(t, held, KindReport, aReport("anchor", "d1")))
|
||||||
t.Fatalf("a report the store could not take was not held: %+v, %d held", held, len(s.parked))
|
if !held.unsettled() || len(in.held) != 1 {
|
||||||
|
t.Fatalf("a report the store could not take was not held: %+v, %d held", held, len(in.held))
|
||||||
}
|
}
|
||||||
store.err = nil
|
store.err = nil
|
||||||
s.retryHeld(context.Background())
|
in.retries(context.Background(), s)
|
||||||
if !held.acked || len(s.parked) != 0 {
|
if !held.acked || len(in.held) != 0 {
|
||||||
t.Fatalf("a held report was not recorded once the store was back: %+v, %d held", held, len(s.parked))
|
t.Fatalf("a held report was not recorded once the store was back: %+v, %d held", held, len(in.held))
|
||||||
}
|
}
|
||||||
|
|
||||||
refused := &saidTo{}
|
refused := &settled{}
|
||||||
s = &Server{listener: heardWith{err: errors.New("a report named no node")}, log: quiet()}
|
s, in = serving()
|
||||||
s.handleReport(context.Background(), aReport(t, refused, "anchor", "d1"))
|
s.listener = heardWith{err: errors.New("a report named no node")}
|
||||||
|
s.act(context.Background(), in.sends(t, refused, KindReport, aReport("anchor", "d1")))
|
||||||
if !refused.acked || refused.nacked {
|
if !refused.acked || refused.nacked {
|
||||||
t.Fatalf("a report the store answered no to was not acknowledged: %+v", refused)
|
t.Fatalf("a report the store answered no to was not acknowledged: %+v", refused)
|
||||||
}
|
}
|
||||||
@@ -72,33 +49,35 @@ func TestAReportTheStoreCouldNotTakeIsHeldAndOneItRefusedIsNot(t *testing.T) {
|
|||||||
// A newer report from the same node supersedes one of its reports still held: recorded after the
|
// A newer report from the same node supersedes one of its reports still held: recorded after the
|
||||||
// newer, the older would overwrite what the node is doing now.
|
// newer, the older would overwrite what the node is doing now.
|
||||||
func TestANewerReportSupersedesAHeldOneFromTheSameNode(t *testing.T) {
|
func TestANewerReportSupersedesAHeldOneFromTheSameNode(t *testing.T) {
|
||||||
s := &Server{listener: heardWith{err: errors.Join(ErrTryAgain, errors.New("starting up"))}, log: quiet()}
|
s, in := serving()
|
||||||
older, newer, other := &saidTo{}, &saidTo{}, &saidTo{}
|
s.listener = heardWith{err: errors.Join(ErrTryAgain, errors.New("starting up"))}
|
||||||
s.handleReport(context.Background(), aReport(t, older, "anchor", "d1"))
|
older, newer, other := &settled{}, &settled{}, &settled{}
|
||||||
s.handleReport(context.Background(), aReport(t, other, "laptop", "d7"))
|
s.act(context.Background(), in.sends(t, older, KindReport, aReport("anchor", "d1")))
|
||||||
s.handleReport(context.Background(), aReport(t, newer, "anchor", "d2"))
|
s.act(context.Background(), in.sends(t, other, KindReport, aReport("laptop", "d7")))
|
||||||
|
s.act(context.Background(), in.sends(t, newer, KindReport, aReport("anchor", "d2")))
|
||||||
if !older.acked {
|
if !older.acked {
|
||||||
t.Fatalf("the older report was not set aside by the newer: %+v", older)
|
t.Fatalf("the older report was not set aside by the newer: %+v", older)
|
||||||
}
|
}
|
||||||
if !newer.unsettled() || !other.unsettled() || len(s.parked) != 2 {
|
if !newer.unsettled() || !other.unsettled() || len(in.held) != 2 {
|
||||||
t.Fatalf("the newer report and another node's were not both held: newer %+v other %+v, %d held",
|
t.Fatalf("the newer report and another node's were not both held: newer %+v other %+v, %d held",
|
||||||
newer, other, len(s.parked))
|
newer, other, len(in.held))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A store that has not come back within the bound is not restarting: the report is let go, loudly,
|
// A store that has not come back within the bound is not restarting: the report is let go, loudly,
|
||||||
// rather than held for ever.
|
// rather than held for ever.
|
||||||
func TestAReportIsLetGoOnceTheStoreHasBeenGoneTooLong(t *testing.T) {
|
func TestAReportIsLetGoOnceTheStoreHasBeenGoneTooLong(t *testing.T) {
|
||||||
s := &Server{listener: heardWith{err: errors.Join(ErrTryAgain, errors.New("connection refused"))},
|
s, in := serving()
|
||||||
log: quiet(), giveUp: time.Millisecond}
|
s.listener = heardWith{err: errors.Join(ErrTryAgain, errors.New("connection refused"))}
|
||||||
held := &saidTo{}
|
s.giveUp = time.Millisecond
|
||||||
s.handleReport(context.Background(), aReport(t, held, "anchor", "d1"))
|
held := &settled{}
|
||||||
|
s.act(context.Background(), in.sends(t, held, KindReport, aReport("anchor", "d1")))
|
||||||
if !held.unsettled() {
|
if !held.unsettled() {
|
||||||
t.Fatalf("the first failure was not held: %+v", held)
|
t.Fatalf("the first failure was not held: %+v", held)
|
||||||
}
|
}
|
||||||
time.Sleep(5 * time.Millisecond)
|
time.Sleep(5 * time.Millisecond)
|
||||||
s.retryHeld(context.Background())
|
in.retries(context.Background(), s)
|
||||||
if !held.acked || len(s.parked) != 0 {
|
if !held.acked || len(in.held) != 0 {
|
||||||
t.Fatalf("a report past the bound was not let go: %+v, %d held", held, len(s.parked))
|
t.Fatalf("a report past the bound was not let go: %+v, %d held", held, len(in.held))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user