Compare commits

..
Author SHA1 Message Date
jschoubben ab74988f1c The filter constrains what arrives from outside, and names no network
The forward chain blocked everything passing through the machine and then allowed
the machine's own containers back by naming their address ranges: 172.16.0.0/12 and
192.168.128.0/17 fixed here, the rest recorded per machine by 0043. Every way of
keeping that list correct fails — a constant describes one machine, a recorded range
goes stale in silence and cannot tell a network the mesh made from one a predecessor
left behind, and generating it from the modules would put half the rule set on the
machine.

The mesh has no position on a container reaching outward: that is not a port opened
to anybody. So both chains are written around the links traffic arrives on. What did
not arrive from outside is accepted in one line; what did meets the declared rules.
The tunnel is named beside the outward links rather than treated as inside, or a port
nothing declares would be reachable from every machine in the mesh.

A machine that has not reported an outward link is sent no filter and keeps the one
it has, refused where a person reads it rather than as a rule set that will not load.

Removes the two constants, `node networks`, and the column behind it. novox/hq ADR
0140, superseding 0137 and 0139.
2026-09-28 23:49:14 +02:00
202 changed files with 912 additions and 18113 deletions
+6 -17
View File
@@ -27,18 +27,8 @@ build:
IMAGE ?= mesh-controller:$(VERSION)
DEV_TAG ?= mesh-controller:development
# The base the module declares, read from the manifest rather than written here twice.
#
# **`make image` was broken and stayed broken**, because the Dockerfile's fallback base was a Go
# older than go.mod asks for: every build died at `go mod download` with "go.mod requires go >=
# 1.26.0", and the pipeline never saw it because the pipeline passes the declared base in. Anybody
# building the image by hand hit it and had to find the digest themselves (novox/hq 04-ISSUES/146,
# what it cost).
GO_BASE ?= $(shell python3 -c "import json;print(next(o['image'] for o in json.load(open('module.json'))['build']['on'] if o['arg']=='GO_BASE'))" 2>/dev/null)
image:
@test -n "$(GO_BASE)" || { echo "module.json declares no GO_BASE; pass GO_BASE=<image> or fix the manifest"; exit 1; }
docker build --build-arg GO_BASE=$(GO_BASE) --build-arg VERSION=$(VERSION) -t $(IMAGE) -t $(DEV_TAG) .
docker build --build-arg VERSION=$(VERSION) -t $(IMAGE) -t $(DEV_TAG) .
@echo
@docker image inspect $(IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
@@ -48,8 +38,7 @@ BUILDER_IMAGE ?= mesh-builder:$(VERSION)
BUILDER_DEV_TAG ?= mesh-builder:development
builder-image:
@test -n "$(GO_BASE)" || { echo "module.json declares no GO_BASE; pass GO_BASE=<image> or fix the manifest"; exit 1; }
docker build --build-arg GO_BASE=$(GO_BASE) -f cmd/mesh-builder/Dockerfile -t $(BUILDER_IMAGE) -t $(BUILDER_DEV_TAG) .
docker build -f cmd/mesh-builder/Dockerfile -t $(BUILDER_IMAGE) -t $(BUILDER_DEV_TAG) .
@echo
@docker image inspect $(BUILDER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
@@ -59,7 +48,7 @@ PROVISIONER_IMAGE ?= mesh-provision-postgres:$(VERSION)
PROVISIONER_DEV_TAG ?= mesh-provision-postgres:development
provisioner-image:
docker build --build-arg GO_BASE=$(GO_BASE) -f examples/postgres-provisioner/Dockerfile \
docker build -f examples/postgres-provisioner/Dockerfile \
-t $(PROVISIONER_IMAGE) -t $(PROVISIONER_DEV_TAG) .
@echo
@docker image inspect $(PROVISIONER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
@@ -70,7 +59,7 @@ OBJECTSTORE_IMAGE ?= mesh-provision-objectstore:$(VERSION)
OBJECTSTORE_DEV_TAG ?= mesh-provision-objectstore:development
objectstore-image:
docker build --build-arg GO_BASE=$(GO_BASE) -f examples/objectstore-provisioner/Dockerfile \
docker build -f examples/objectstore-provisioner/Dockerfile \
-t $(OBJECTSTORE_IMAGE) -t $(OBJECTSTORE_DEV_TAG) .
@echo
@docker image inspect $(OBJECTSTORE_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
@@ -81,7 +70,7 @@ REDIS_PROVISIONER_IMAGE ?= mesh-provision-redis:$(VERSION)
REDIS_PROVISIONER_DEV_TAG ?= mesh-provision-redis:development
redis-provisioner-image:
docker build --build-arg GO_BASE=$(GO_BASE) -f examples/redis-provisioner/Dockerfile \
docker build -f examples/redis-provisioner/Dockerfile \
-t $(REDIS_PROVISIONER_IMAGE) -t $(REDIS_PROVISIONER_DEV_TAG) .
@echo
@docker image inspect $(REDIS_PROVISIONER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
@@ -91,7 +80,7 @@ PROXY_IMAGE ?= mesh-route-proxy:$(VERSION)
PROXY_DEV_TAG ?= mesh-route-proxy:development
proxy-image:
docker build --build-arg GO_BASE=$(GO_BASE) -f examples/route-proxy/Dockerfile -t $(PROXY_IMAGE) -t $(PROXY_DEV_TAG) .
docker build -f examples/route-proxy/Dockerfile -t $(PROXY_IMAGE) -t $(PROXY_DEV_TAG) .
@echo
@docker image inspect $(PROXY_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
+13 -44
View File
@@ -137,12 +137,7 @@ func takeWorkFrom(credential Credential, on string) (link.BuildMachine, error) {
if err != nil {
return nil, err
}
// **The seat this machine serves is the one its credential claims** (novox/hq ADR 0190, the
// handover): the mesh issues a build machine's credential naming the seat its module claims,
// and one binary serves the old role as `builder` and the new as `build-agent` from that alone.
seat := link.BuildSeatClaimed(credential.seatsClaimed())
fmt.Fprintf(os.Stderr, "taking build work as a holder of %s\n", seat)
return link.MachineOverNATSOn(js, on, seat), nil
return link.MachineOverNATS(js, on), nil
}
// answer does one build and says what happened, whichever way it went.
@@ -153,34 +148,20 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
// it either finishes or fails is indistinguishable from one that never arrived — which cost a long
// diagnosis against a running mesh, chasing "the handler never fired" when the truth was only that
// the handler said nothing until the end.
fmt.Fprintf(os.Stderr, "a build request arrived for %s (%s)\n", request.Repository, request.ID)
// **Everything a build says goes two ways**: to stderr, as always, and onto the bus as the
// role's own events under the build's id (novox/hq ADR 0157) — so whoever asked, and anybody
// watching, reads the same lines this container's log holds, live, and after the fact from the
// stream. Said first, before anything runs, so a build that hangs is one that visibly started.
say := func(step, message string) {
fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message)
work.Say(step, message)
}
builder.Said = say
defer func() { builder.Said = nil }()
if err := work.Began(ctx); err != nil {
fmt.Fprintf(os.Stderr, "cannot say a build started: %v\n", err)
}
fmt.Fprintf(os.Stderr, "a build request arrived for %s\n", request.Repository)
result := link.BuildResult{
ID: request.ID, Repository: request.Repository, Path: request.Path,
Ref: request.Ref, On: on, Source: request.Source,
Ref: request.Ref, On: on,
}
what := "building " + request.Repository
fmt.Fprintf(os.Stderr, "building %s", request.Repository)
if request.Path != "" {
what += " at " + request.Path
fmt.Fprintf(os.Stderr, " at %s", request.Path)
}
if request.Ref != "" {
what += " on " + request.Ref
fmt.Fprintf(os.Stderr, " at %s", request.Ref)
}
say("build", what)
fmt.Fprintln(os.Stderr)
npmrc, err := packagesFrom()
var built builder.Result
@@ -190,13 +171,16 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
// after a clone that then fails at npm ci.
built, err = builder.Build(ctx, builder.Command, publisher,
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
forgeFrom(), say, request.Seats)
forgeFrom(),
func(step, message string) {
fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message)
})
}
if err != nil {
// A failure is a result. A build that fails and says nothing is indistinguishable from a
// builder that is not running, and those want completely different responses.
result.Failed = err.Error()
say("failed", err.Error())
fmt.Fprintf(os.Stderr, " failed: %v\n", err)
} else {
manifest, marshalErr := json.Marshal(built.Manifest)
if marshalErr != nil {
@@ -213,7 +197,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
for _, r := range built.Read {
result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref})
}
say("built", built.Manifest.Module+" from "+short(built.Commit))
fmt.Fprintf(os.Stderr, " built %s from %s\n", built.Manifest.Module, short(built.Commit))
}
}
@@ -458,21 +442,6 @@ type Credential struct {
// as two fields and this machine joins them once, here, to dial.
User string `json:"user,omitempty"`
Password string `json:"password,omitempty"`
// Claims are the seats the module this credential was issued for claims, as the mesh writes
// them beside the credential (novox/hq ADR 0159). The first is the build role this machine
// serves; a credential naming none is from before claims travelled in it.
Claims []struct {
Seat string `json:"seat"`
} `json:"claims,omitempty"`
}
// seatsClaimed is the seats the credential names, in order.
func (c Credential) seatsClaimed() []string {
out := make([]string, 0, len(c.Claims))
for _, claim := range c.Claims {
out = append(out, claim.Seat)
}
return out
}
// onTheNewBus is whether a credential is for the bus being built: its address says so, and the
-27
View File
@@ -1,27 +0,0 @@
package main
import (
"encoding/json"
"testing"
"github.com/novox/mesh-controller/internal/link"
)
// The seat a build machine serves comes from its credential (novox/hq ADR 0190 handover).
func TestTheCredentialSaysWhichBuildRoleThisMachineServes(t *testing.T) {
var held Credential
if err := json.Unmarshal([]byte(`{"url":"nats://bus:4222","user":"anchor.builder","password":"x",
"claims":[{"seat":"mesh-build-machine","scope":"mesh","serves":[]}]}`), &held); err != nil {
t.Fatal(err)
}
if got := link.BuildSeatClaimed(held.seatsClaimed()); got != "mesh-build-machine" {
t.Errorf("the old builder's credential serves %q", got)
}
var bare Credential
if err := json.Unmarshal([]byte(`{"url":"nats://bus:4222","user":"anchor.build-agent","password":"x"}`), &bare); err != nil {
t.Fatal(err)
}
if got := link.BuildSeatClaimed(bare.seatsClaimed()); got != link.TheBuildMachine {
t.Errorf("a credential without claims serves %q, want %s", got, link.TheBuildMachine)
}
}
-48
View File
@@ -3,7 +3,6 @@ package main
import (
"context"
"fmt"
"github.com/novox/mesh-controller/internal/broker"
"sort"
"strings"
@@ -47,13 +46,6 @@ func assign(ctx context.Context, open *stores, node, module string) (string, err
return "", err
}
defer release()
// **Before the new assignment can unsettle a seat somebody holds only by being alone**
// (novox/hq 04-ISSUES/170): what the mesh derived so far is written down, and then the
// assignment resolves against a record rather than against a coincidence.
settled, err := recordDerivedHolders(ctx, open)
if err != nil {
return "", err
}
fresh, err := open.inventory.Assign(ctx, node, module)
if err != nil {
return "", err
@@ -65,16 +57,6 @@ func assign(ctx context.Context, open *stores, node, module string) (string, err
node, module), nil
}
said := fmt.Sprintf("%s is assigned %s", node, module)
for _, line := range settled {
said += "\n " + line
}
// Its bus credential, in the same act (novox/hq issue 203): an assignment pushed before its
// credential exists delivers a process that cannot authenticate and crash-loops until somebody
// runs a second verb and a second push. Issued here when the module speaks on the bus and has
// no credential yet; kept when it has one, so re-assigning rotates nothing.
if line := issueOnAssign(ctx, open, node, module); line != "" {
said += "\n " + line
}
plan, _, err := planFor(ctx, open, node)
if err != nil {
// Kept, and still refused. Both halves are the answer, and the rest of the mesh is still
@@ -160,33 +142,3 @@ func blockedElsewhere(ctx context.Context, open *stores, except string) string {
out.WriteString("\nThis may or may not be what just changed — it is what is true now.")
return out.String()
}
// issueOnAssign gives a newly assigned module its bus credential, the way `module issue` does, and
// says what it did in one line. Nothing for a module that declares no broker secret; nothing for one
// whose user is already minted (a credential is rotated on purpose, never by re-assigning); and when
// the bus cannot be reached from here, the line names the verb and the push that would refuse the
// module until it is run — never a silent placeholder (novox/hq issue 203).
func issueOnAssign(ctx context.Context, open *stores, node, module string) string {
inv := open.inventory
shelf, err := inv.Catalogue(ctx)
if err != nil {
return ""
}
m, known := shelf[module]
if !known || mayIssue(m) != nil {
return ""
}
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username()
if _, minted, err := inv.BusUserHash(ctx, user); err != nil || minted {
return ""
}
busAddress, err := broker.BusAddress()
if err == nil {
err = issueOnTheNewBus(ctx, inv, m, node, busAddress)
}
if err != nil {
return fmt.Sprintf("its bus credential is not issued (%v): `module issue %s --node %s` first — "+
"`push %s` refuses to send %s until it is", err, module, node, node, module)
}
return fmt.Sprintf("its bus credential is issued and sealed to %s, and arrives with the push", node)
}
-6
View File
@@ -175,12 +175,6 @@ func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T)
Guards: []int{15672},
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-broker",
"ports": []any{"5671:5671", "5672:5672", "127.0.0.1:15672:15672"}, "image": "mq@" + aDigest}}})
// The control plane's own bus user is the installer's, seeded at genesis before the controller
// runs (SeedBusUser); without it a push now refuses the credential nobody issued (issue 203).
if err := open.inventory.SeedBusUser(ctx, inventory.BusUser{Username: "anchor.mesh-controller",
Kind: inventory.BusController, Node: "anchor", Module: "mesh-controller"}, "bootstrap"); err != nil {
t.Fatal(err)
}
if _, err := assign(ctx, open, "anchor", "mesh-controller"); err != nil {
t.Fatal(err)
}
+9 -116
View File
@@ -91,26 +91,11 @@ func reportsReaching(t *testing.T, open *stores, reachable []link.Reach, held ..
if _, err := (link.Enrolment{Inventory: open.inventory}).Heard(ctx, link.Report{
Node: "anchor", Applied: []string{"hello-web.x"}, Declared: digestOf(body),
Firewall: "ufw", Held: held, Reachable: reachable,
// A machine says which of its links face outside on every apply (novox/hq ADR 0140), and a
// filter is not sent to one that has not. The anchor reports one, as a real host does; this
// fixture lacked it from 2026-09-28 and nothing ran the test (issue 177).
Outward: []string{"eth0"},
// And what filters it (ADR 0168): its front end, the runtime's own, and a chain a
// predecessor left in the runtime's user chain.
Filters: anchorFilters,
}); err != nil {
t.Fatal(err)
}
}
// anchorFilters is what the adopted anchor says filters it: ufw's chains, the runtime's, and a
// predecessor's chain the mesh did not write.
var anchorFilters = []link.Filter{
{Where: "table ip filter, chain ufw-reject-input", Owner: "found-firewall", Refuses: "reject"},
{Where: "table ip filter, chain DOCKER", Owner: "runtime", Refuses: `iifname != "docker0" oifname "docker0" drop`},
{Where: "table ip filter, chain DOCKER-USER", Owner: "other", Refuses: `iifname "eth0" tcp dport 6000 drop`},
}
var (
heldContainer = link.Held{ID: "hello-web.server", Module: "hello-web", Kind: "container",
Target: "hello-web", Since: time.Now()}
@@ -120,10 +105,10 @@ var (
func TestTakingAModuleNotOnTheNodeIsRefused(t *testing.T) {
open, _ := anAdoptedAnchor(t)
if _, err := take(t.Context(), open, "anchor", "nftables", takeOptions{Yes: true}); !errors.Is(err, inventory.ErrNotAssigned) {
if _, err := take(t.Context(), open, "anchor", "nftables"); !errors.Is(err, inventory.ErrNotAssigned) {
t.Fatalf("taking an unassigned module gave %v", err)
}
if _, err := take(t.Context(), open, "laptop", "network", takeOptions{Yes: true}); !errors.Is(err, inventory.ErrNotAdopted) {
if _, err := take(t.Context(), open, "laptop", "network"); !errors.Is(err, inventory.ErrNotAdopted) {
t.Fatalf("taking on a converged node gave %v", err)
}
}
@@ -154,24 +139,7 @@ func TestTheFlipIsRefusedWhileAFoundContainerIsHeld(t *testing.T) {
func TestTakingNamesWhatItReplaces(t *testing.T) {
open, _ := anAdoptedAnchor(t)
reportsHolding(t, open, heldContainer, heldFile)
ctx := t.Context()
// The machine holds something for the module, so the take acts on the preview the operator
// saw and names its digest (novox/hq ADR 0163).
preview, err := take(ctx, open, "anchor", "hello-web", takeOptions{})
if err != nil {
t.Fatal(err)
}
saw := takeDigestIn(t, preview)
if !strings.Contains(preview, "nothing taken; `take anchor hello-web --yes "+saw+"`") {
t.Fatalf("the preview does not say how to act on it:\n%s", preview)
}
if taken, _ := open.inventory.Taken(ctx, "anchor"); len(taken) != 0 {
t.Fatal("the preview took something")
}
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err == nil || !strings.Contains(err.Error(), "name its digest") {
t.Fatalf("--yes without the digest was not refused: %v", err)
}
said, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: saw})
said, err := take(t.Context(), open, "anchor", "hello-web")
if err != nil {
t.Fatal(err)
}
@@ -181,71 +149,10 @@ func TestTakingNamesWhatItReplaces(t *testing.T) {
}
}
// takeDigestIn is the digest a take's preview printed.
func takeDigestIn(t *testing.T, preview string) string {
t.Helper()
for _, line := range strings.Split(preview, "\n") {
if fields := strings.Fields(line); len(fields) == 2 && fields[0] == "preview" {
return fields[1]
}
}
t.Fatalf("the preview printed no digest:\n%s", preview)
return ""
}
// A take acts on the preview the operator saw, and on an account of the machine that is still the
// machine: a changed preview and a stale account refuse (novox/hq ADR 0163, rule 1).
func TestATakeIsRefusedOnAChangedPreviewOrAStaleAccount(t *testing.T) {
open, _ := anAdoptedAnchor(t)
ctx := t.Context()
reportsHolding(t, open, heldContainer, heldFile)
preview, err := take(ctx, open, "anchor", "hello-web", takeOptions{})
if err != nil {
t.Fatal(err)
}
saw := takeDigestIn(t, preview)
// The machine reports again, and what it holds has changed: the found container now carries
// facts the preview never showed.
changed := heldContainer
changed.Facts = map[string]any{"image": "hello:2", "declared_image": "registry.example/hello"}
reportsHolding(t, open, changed, heldFile)
_, err = take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: saw})
if err == nil || !strings.Contains(err.Error(), "has changed since preview "+saw) {
t.Fatalf("a changed preview was acted on: %v", err)
}
if taken, _ := open.inventory.Taken(ctx, "anchor"); len(taken) != 0 {
t.Fatal("a refused take took something")
}
// And an account older than the flip allows.
preview, err = take(ctx, open, "anchor", "hello-web", takeOptions{})
if err != nil {
t.Fatal(err)
}
saw = takeDigestIn(t, preview)
saved := reportFreshFor
reportFreshFor = -time.Second
defer func() { reportFreshFor = saved }()
_, err = take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: saw})
if err == nil || !strings.Contains(err.Error(), "a take acts only on an account newer than") {
t.Fatalf("a stale account was acted on: %v", err)
}
reportFreshFor = saved
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: saw}); err != nil {
t.Fatal(err)
}
// A module the machine holds nothing for has nothing to compare: --yes alone suffices.
if _, err := take(ctx, open, "anchor", "notes", takeOptions{Yes: true}); err == nil {
// notes holds a file, so this one needs the digest too.
t.Fatal("notes holds a found file and was taken without a digest")
}
}
func TestConvergingPreviewsThenChangesAndAdoptingKeepsWhatWasTaken(t *testing.T) {
open, sent := anAdoptedAnchor(t)
ctx := t.Context()
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
if _, err := take(ctx, open, "anchor", "hello-web"); err != nil {
t.Fatal(err)
}
reportsHolding(t, open, heldFile)
@@ -275,20 +182,6 @@ func TestConvergingPreviewsThenChangesAndAdoptingKeepsWhatWasTaken(t *testing.T)
if strings.Contains(preview, "15672") {
t.Errorf("a loopback listener is in the preview:\n%s", preview)
}
// What filters the machine now, and the fate of each (novox/hq ADR 0168): the predecessor's
// chain is named as not the mesh's and left, so the reader knows before the flip.
for _, want := range []string{
"table ip filter, chain DOCKER-USER",
"NOT THE MESH'S; left in force",
`iifname "eth0" tcp dport 6000 drop`,
"table ip filter, chain ufw-reject-input",
"the found firewall's; retired with it",
"the container runtime's own; left",
} {
if !strings.Contains(preview, want) {
t.Errorf("the preview does not say %q:\n%s", want, preview)
}
}
for _, line := range strings.Split(preview, "\n") {
if strings.Contains(line, "5000") && !strings.Contains(line, "WILL CLOSE") {
t.Errorf("an undeclared published port is not said to close: %s", line)
@@ -433,7 +326,7 @@ func digestIn(t *testing.T, preview string) string {
func TestTheFlipActsOnlyOnThePreviewTheOperatorSaw(t *testing.T) {
open, sent := anAdoptedAnchor(t)
ctx := t.Context()
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
if _, err := take(ctx, open, "anchor", "hello-web"); err != nil {
t.Fatal(err)
}
reportsHolding(t, open, heldFile)
@@ -499,7 +392,7 @@ func TestTheFlipActsOnlyOnThePreviewTheOperatorSaw(t *testing.T) {
func TestTheFlipHoldsTheNodeWhileItSends(t *testing.T) {
open, _ := anAdoptedAnchor(t)
ctx := t.Context()
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
if _, err := take(ctx, open, "anchor", "hello-web"); err != nil {
t.Fatal(err)
}
reportsHolding(t, open, heldFile)
@@ -544,7 +437,7 @@ func TestTheFlipHoldsTheNodeWhileItSends(t *testing.T) {
func TestThePreviewNamesEveryHeldKind(t *testing.T) {
open, _ := anAdoptedAnchor(t)
ctx := t.Context()
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
if _, err := take(ctx, open, "anchor", "hello-web"); err != nil {
t.Fatal(err)
}
since := time.Now()
@@ -587,7 +480,7 @@ func TestThePreviewNamesEveryHeldKind(t *testing.T) {
func TestTheFlipIsRefusedOnAnAccountNamingNothingReachable(t *testing.T) {
open, sent := anAdoptedAnchor(t)
ctx := t.Context()
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
if _, err := take(ctx, open, "anchor", "hello-web"); err != nil {
t.Fatal(err)
}
// Only a loopback listener: nothing off the machine, which is the same silence.
@@ -615,7 +508,7 @@ func TestTheFlipIsRefusedOnAnAccountNamingNothingReachable(t *testing.T) {
func TestAssigningWaitsForWhateverIsConvergingTheNode(t *testing.T) {
open, _ := anAdoptedAnchor(t)
ctx := t.Context()
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
if _, err := take(ctx, open, "anchor", "hello-web"); err != nil {
t.Fatal(err)
}
reportsHolding(t, open, heldFile)
+14 -479
View File
@@ -24,15 +24,6 @@ import (
func showMode(ctx context.Context, inv *inventory.Inventory, node inventory.Node) error {
if !node.Adopted {
fmt.Printf(" mode converged\n")
// A converged machine holds nothing, and can still run what nobody asked for
// (novox/hq ADR 0163): what it reports as strays is said whatever its mode.
if said, err := inv.AdoptionOf(ctx, node.Name); err == nil && len(said.Strays) > 0 {
showStrays(said.Strays)
}
// And what filters it, truthfully (novox/hq ADR 0168): the mesh alone, or not.
if filtering, err := inv.FilteringOf(ctx, node.Name); err == nil {
showFiltering(filtering, false)
}
return nil
}
fmt.Printf(" mode adopted since %s\n",
@@ -71,81 +62,11 @@ func showMode(ctx context.Context, inv *inventory.Inventory, node inventory.Node
if h.Kept != "" {
fmt.Printf(" %-17s original kept at %s\n", "", h.Kept)
}
for _, f := range comparisonLines(h) {
fmt.Printf(" %-17s %s\n", "", f)
}
}
showStrays(said.Strays)
if filtering, err := inv.FilteringOf(ctx, node.Name); err == nil {
showFiltering(filtering, true)
}
fmt.Printf(" as of %s\n", said.At.Local().Format(time.DateTime))
return nil
}
// showFiltering says what filters a machine, with owners (novox/hq ADR 0168), and for a converged
// machine the state of the firewall it was found with. A machine that has not said is not said to
// be filtered by anything.
func showFiltering(f inventory.Filtering, adopted bool) {
if len(f.Filters) == 0 && f.FoundFirewall == nil {
return
}
if fw := f.FoundFirewall; fw != nil && !adopted {
switch {
case fw.Active:
fmt.Printf(" found firewall %s is ACTIVE on this converged machine; the next apply retires it again\n", fw.Kind)
case fw.RetiredBy == "removed":
fmt.Printf(" found firewall %s, removed: the mesh's filter is what filters this machine (novox/hq ADR 0180)\n", fw.Kind)
case fw.RetiredBy == inventory.FilterMesh || fw.RetiredBy == "mesh":
fmt.Printf(" found firewall %s, retired by the mesh; its configuration stays on disk\n", fw.Kind)
case fw.RetiredBy != "":
fmt.Printf(" found firewall %s, found inactive — not by the mesh\n", fw.Kind)
default:
fmt.Printf(" found firewall %s, inactive\n", fw.Kind)
}
}
if len(f.Filters) == 0 {
return
}
if f.Alone() {
fmt.Printf(" filtered by the mesh alone (%s)\n", filterSummary(f.Filters))
return
}
fmt.Printf(" filtered by NOT the mesh alone: %d rule set(s) the mesh did not write refuse traffic here\n", len(f.Others()))
for _, x := range f.Filters {
if x.Owner == inventory.FilterOther || x.Owner == inventory.FilterFoundFirewall {
fmt.Printf(" %-17s %s — %s: %s\n", "", x.Where, x.Owner, x.Refuses)
}
}
fmt.Printf(" %-17s and its own: %s\n", "", filterSummary(f.Filters))
}
// filterSummary counts a machine's filters by owner: "mesh 2, runtime 3, ban 1".
func filterSummary(filters []inventory.Filter) string {
counts := map[string]int{}
for _, x := range filters {
counts[x.Owner]++
}
var parts []string
for _, owner := range []string{inventory.FilterMesh, inventory.FilterRuntime, inventory.FilterBan, inventory.FilterFoundFirewall, inventory.FilterOther} {
if n := counts[owner]; n > 0 {
parts = append(parts, fmt.Sprintf("%s %d", owner, n))
}
}
return strings.Join(parts, ", ")
}
// showStrays says what a machine runs that the mesh neither wrote nor holds (ADR 0163).
func showStrays(strays []inventory.Stray) {
if len(strays) == 0 {
return
}
fmt.Printf(" strays %d container(s) the mesh neither wrote nor holds:\n", len(strays))
for _, s := range strays {
fmt.Printf(" %-17s %s (%s)\n", "", s.Name, s.Detail)
}
}
// showTunnel is the node show lines about the tunnel an adopted node found and carried (novox/hq
// ADR 0105): what it presented at enrolment, and what it last said about taking it over.
func showTunnel(ctx context.Context, inv *inventory.Inventory, name string) error {
@@ -215,28 +136,7 @@ const DefaultFilter = "nftables"
// take is a module's cutover on an adopted node: the operator's act, done when that module's data
// has moved. From the next push its resources converge there like any other, replacing what the
// node found and holds for it.
//
// **Previewed, and the preview is a comparison** (novox/hq ADR 0163): for every held thing the
// module would replace, what runs beside what the module declares, and the difference; the
// module's secrets on the machine and where each came from; its settings on the machine. Without
// --yes the comparison is printed and nothing changes. `--yes <digest>` cuts over exactly what was
// previewed, the way the flip is confirmed: the preview ends with a digest of what it said, and a
// take naming an older one, or acting on an account of the machine older than the flip allows, is
// refused. A module the machine holds nothing for has nothing to compare, and `--yes` suffices.
// takeOptions is what a take was told about the differences it may pass (novox/hq ADR 0163).
type takeOptions struct {
Yes bool
// Digest is the preview's, named with --yes; required whenever the machine holds something
// for the module.
Digest string
Downgrade bool
Replace map[string]bool
// Mint names the secrets the service shall take a new value for, although the mesh minted
// one and the service already has its own (rule 2).
Mint map[string]bool
}
func take(ctx context.Context, open *stores, node, module string, opts takeOptions) (string, error) {
func take(ctx context.Context, open *stores, node, module string) (string, error) {
inv := open.inventory
assigned, err := inv.Assigned(ctx, node)
if err != nil {
@@ -250,335 +150,25 @@ func take(ctx context.Context, open *stores, node, module string, opts takeOptio
}
}
}
// The comparison first (novox/hq ADR 0163): every held thing the module would replace, beside
// what the module declares, and the differences that refuse unless named.
c, err := comparisonFor(ctx, open, node, module)
if err != nil {
return "", err
}
preview, refusals, saw := comparisonOf(module, c, opts)
if len(refusals) > 0 {
return "", fmt.Errorf("taking %s on %s is refused:\n %s\n%s", module, node,
strings.Join(refusals, "\n "), preview)
}
holds := len(heldOf(c.reported, module)) > 0
if holds {
preview += "\n preview " + saw
}
if !opts.Yes {
if !holds {
return preview + fmt.Sprintf("\nnothing taken; `take %s %s --yes` declares it as the mesh's own", node, module), nil
}
return preview + fmt.Sprintf("\nnothing taken; `take %s %s --yes %s` cuts it over as previewed", node, module, saw), nil
}
if holds {
// The take acts on the preview the operator saw, and on an account of the machine that
// is still the machine: the same two refusals the flip makes.
if age := time.Since(c.reported.At); age > reportFreshFor {
return preview, fmt.Errorf("%s last said what it holds %s ago, and a take acts only on "+
"an account newer than %s: run `push %s --wait 2m`, then preview again",
node, age.Round(time.Second), reportFreshFor, node)
}
if opts.Digest == "" {
return preview, fmt.Errorf("taking %s on %s acts on the preview you saw: name its digest, "+
"`take %s %s --yes %s`, once you have read it", module, node, node, module, saw)
}
if opts.Digest != saw {
return preview, fmt.Errorf("what taking %s on %s would replace has changed since preview %s "+
"(it is now %s): read the preview above, and run `take %s %s --yes %s` if it is "+
"what you want", module, node, opts.Digest, saw, node, module, saw)
}
}
if err := inv.Take(ctx, node, module); err != nil {
return "", err
}
said := fmt.Sprintf("%s is taken on %s", module, node)
if holds {
said += "; the next push replaces what the node found and holds for it:\n" + preview
}
return said + fmt.Sprintf("\n run `push %s` to cut it over", node), nil
}
// comparison is everything a take puts beside what the module declares: the machine's account of
// what it holds and what is reachable on it, the module's secrets on the machine, its settings
// there, and which found networks a setting keeps for each of its containers (by held id).
type comparison struct {
reported inventory.Adoption
secrets []inventory.SecretState
layers []catalogue.Layer
keeps map[string][]string
// settingsRefused is why the module's settings cannot compose with its definition, when
// they cannot — the module would be left out of the declaration (rule 6).
settingsRefused string
}
func comparisonFor(ctx context.Context, open *stores, node, module string) (comparison, error) {
inv := open.inventory
var c comparison
var err error
if c.reported, err = inv.AdoptionOf(ctx, node); err != nil {
return c, err
}
if c.secrets, err = inv.SecretsOf(ctx, node, module); err != nil {
return c, err
}
if c.layers, err = inv.SettingsFor(ctx, node, module); err != nil {
return c, err
}
shelf, err := inv.Catalogue(ctx)
reported, err := inv.AdoptionOf(ctx, node)
if err != nil {
return c, err
return "", err
}
if m, known := shelf[module]; known && len(c.layers) > 0 {
if err := catalogue.JudgeSettings(m, c.layers, true); err != nil {
c.settingsRefused = err.Error()
}
if kept, err := catalogue.KeptNetworks(m, c.layers, true); err == nil && len(kept) > 0 {
c.keeps = map[string][]string{}
for id, networks := range kept {
c.keeps[module+"."+id] = networks
}
}
}
return c, nil
}
// heldOf is what a node holds for one module.
func heldOf(reported inventory.Adoption, module string) []inventory.Held {
var out []inventory.Held
var replaces []string
for _, h := range reported.Held {
if h.Module == module {
out = append(out, h)
replaces = append(replaces, " "+heldLine(h))
}
}
return out
}
// comparisonOf is a take's preview: for every held thing of the module, what runs beside what the
// module declares; its secrets and its settings on the machine; and the refusals the differences
// earn unless the take named them (novox/hq ADR 0163): an image older than the one running, a
// declared file that differs from the found one, a secret the mesh minted for a service whose data
// was found. A narrowed port and a shared network are said and not refused. The digest is of what
// the preview says, so anything in it changing changes the digest.
func comparisonOf(module string, c comparison, opts takeOptions) (preview string, refusals []string, digest string) {
var b strings.Builder
held := heldOf(c.reported, module)
foundData := false
for _, h := range held {
if h.Kind == "container" || h.Kind == "directory" {
foundData = true
}
fmt.Fprintf(&b, " %s", heldLine(h))
if h.Kept != "" {
fmt.Fprintf(&b, ", original kept at %s", h.Kept)
}
b.WriteString("\n")
for _, line := range comparisonLinesWith(h, c.keeps[h.ID], c.reported) {
fmt.Fprintf(&b, " %s\n", line)
}
f := factsOf(h)
if f.downgrade && !opts.Downgrade {
refusals = append(refusals, fmt.Sprintf("%s: the module's image (%s, made %s) is older than the one running (%s, made %s) — "+
"a service that migrated its data forward may not start on it; `--downgrade` to take it anyway",
h.Target, f.declaredImage, day(f.declaredCreated), f.image, day(f.imageCreated)))
}
if f.differs && !opts.Replace[h.Target] && !opts.Replace["*"] {
refusals = append(refusals, fmt.Sprintf("%s: the module's content differs from the file found; the lines above "+
"marked - are lost by taking it; `--replace %s` to replace it anyway, or declare the file partially",
h.Target, h.Target))
}
if len(replaces) > 0 {
said += "; the next push replaces what the node found and holds for it:\n" +
strings.Join(replaces, "\n")
}
// The module's secrets on the machine (rule 2 and 3): a service whose data was found already
// has a value for each, so one the mesh minted and nobody accepted refuses unless --mint says
// the service shall take a new one.
for _, sec := range c.secrets {
name := sec.Name
if sec.Local != "" {
name += " (" + sec.Local + ")"
}
what := "own secret"
accept := fmt.Sprintf("`secret accept <node> %s %s`", module, sec.Name)
if !sec.Own() {
what = "secret from " + sec.Provider
accept = fmt.Sprintf("`secret accept <node> %s %s --provider %s`", module, sec.Name, sec.Provider)
if sec.Local != "" {
accept = strings.TrimSuffix(accept, "`") + " --local " + sec.Local + "`"
}
}
switch {
case sec.Origin == inventory.OriginAccepted:
fmt.Fprintf(&b, " %s %s: accepted from a person, carried in as it is\n", what, name)
case opts.Mint[sec.Name]:
fmt.Fprintf(&b, " %s %s: minted by the mesh; the service takes the new value, as --mint said\n", what, name)
case foundData:
fmt.Fprintf(&b, " %s %s: MINTED by the mesh and not accepted — the running service already has one\n", what, name)
refusals = append(refusals, fmt.Sprintf("%s: the mesh minted a value and the service whose data was found "+
"already uses its own; %s carries the existing value in, or `--mint %s` says the service shall take "+
"the new one", name, accept, sec.Name))
default:
fmt.Fprintf(&b, " %s %s: minted by the mesh\n", what, name)
}
}
// And its settings on this machine, composed against its definition (rule 1, rule 6).
for _, layer := range c.layers {
keys := make([]string, 0, len(layer.Values))
for k := range layer.Values {
keys = append(keys, k)
}
sort.Strings(keys)
fmt.Fprintf(&b, " settings from %s: %s\n", layer.From, strings.Join(keys, ", "))
}
if c.settingsRefused != "" {
fmt.Fprintf(&b, " SETTINGS DO NOT COMPOSE with the module's definition, so the push leaves it out: %s\n", c.settingsRefused)
}
preview = strings.TrimRight(b.String(), "\n")
sum := sha256.Sum256([]byte(preview))
return preview, refusals, hex.EncodeToString(sum[:])[:12]
}
// facts is a held thing's facts as the preview reads them.
type facts struct {
image, imageCreated, declaredImage, declaredCreated string
downgrade, differs bool
networks map[string][]string
mounts, ports, declaredPorts, declaredVolumes []string
difference []string
}
func factsOf(h inventory.Held) facts {
var f facts
if h.Facts == nil {
return f
}
str := func(k string) string { s, _ := h.Facts[k].(string); return s }
list := func(k string) []string {
var out []string
if raw, ok := h.Facts[k].([]any); ok {
for _, x := range raw {
if s, ok := x.(string); ok {
out = append(out, s)
}
}
}
return out
}
f.image, f.imageCreated = str("image"), str("image_created")
f.declaredImage, f.declaredCreated = str("declared_image"), str("declared_image_created")
f.downgrade, _ = h.Facts["downgrade"].(bool)
f.differs, _ = h.Facts["differs"].(bool)
f.mounts, f.ports = list("mounts"), list("ports")
f.declaredPorts, f.declaredVolumes, f.difference = list("declared_ports"), list("declared_volumes"), list("difference")
if raw, ok := h.Facts["networks"].(map[string]any); ok {
f.networks = map[string][]string{}
for name, members := range raw {
var out []string
if ms, ok := members.([]any); ok {
for _, m := range ms {
if s, ok := m.(string); ok {
out = append(out, s)
}
}
}
f.networks[name] = out
}
}
return f
}
// comparisonLines says a held thing's facts the way a person weighs them.
func comparisonLines(h inventory.Held) []string {
return comparisonLinesWith(h, nil, inventory.Adoption{})
}
// comparisonLinesWith is comparisonLines knowing which found networks this machine's setting keeps
// for the container (rule 4) and what the machine reports reachable, so a published port's reach
// is said beside the port (rule 1).
func comparisonLinesWith(h inventory.Held, keeps []string, reported inventory.Adoption) []string {
f := factsOf(h)
var out []string
if f.image != "" || f.declaredImage != "" {
line := fmt.Sprintf("runs %s", orNone(f.image))
if f.imageCreated != "" {
line += " (made " + day(f.imageCreated) + ")"
}
line += "; the module declares " + orNone(f.declaredImage)
switch {
case f.declaredCreated != "":
line += " (made " + day(f.declaredCreated) + ")"
case f.declaredImage != "":
line += " (not on the machine yet, so its age is unknown)"
}
if f.downgrade {
line += " — DOWNGRADE"
}
out = append(out, line)
}
names := make([]string, 0, len(f.networks))
for n := range f.networks {
names = append(names, n)
}
sort.Strings(names)
for _, n := range names {
members := f.networks[n]
if len(members) == 0 {
continue
}
if slices.Contains(keeps, n) {
out = append(out, fmt.Sprintf("on the network %s with %s — kept by this machine's setting, so they still reach it by name once taken",
n, strings.Join(members, ", ")))
continue
}
out = append(out, fmt.Sprintf("on the network %s with %s, which may reach it by name and will not once it moves to the module's own network"+
" (`settings set %s --node <node>` with {%q: {<container>: [%q]}} keeps it)",
n, strings.Join(members, ", "), h.Module, catalogue.NetworksSetting, n))
}
for _, n := range keeps {
if _, found := f.networks[n]; !found {
out = append(out, fmt.Sprintf("keeps the network %s by this machine's setting, which the found container is not on", n))
}
}
if len(f.ports) > 0 || len(f.declaredPorts) > 0 {
out = append(out, fmt.Sprintf("publishes %s; the module declares %s",
orNone(strings.Join(f.ports, " ")), orNone(strings.Join(f.declaredPorts, " "))))
// How far each published port reaches now, as the machine reported it: the listener the
// runtime publishes for this container. The found firewall's and the guard's rules are
// not read; what they let through is said as what was reported reachable.
var reach []string
for _, r := range reported.Reachable {
if r.By == h.Target && r.Published {
reach = append(reach, fmt.Sprintf("%s:%d (%s, container port %d)", r.Address, r.Port, r.Protocol, r.ContainerPort))
}
}
switch {
case len(reach) > 0:
line := "reachable now at " + strings.Join(reach, ", ")
if reported.Firewall != "" && reported.Firewall != "none" {
line += ", behind the found firewall (" + reported.Firewall + "), whose rules are not read"
}
out = append(out, line)
case len(f.ports) > 0 && len(reported.Reachable) > 0:
out = append(out, "not reported reachable on the machine")
}
}
if len(f.mounts) > 0 || len(f.declaredVolumes) > 0 {
out = append(out, fmt.Sprintf("mounts %s; the module declares %s",
orNone(strings.Join(f.mounts, " ")), orNone(strings.Join(f.declaredVolumes, " "))))
}
if f.differs {
out = append(out, "the declared content differs from the file found (- lost, + new):")
for _, d := range f.difference {
out = append(out, " "+d)
}
}
return out
}
// day is a timestamp as a person reads it in a preview: its date.
func day(stamp string) string {
if len(stamp) >= 10 {
return stamp[:10]
}
return stamp
return said + fmt.Sprintf("\n run `push %s` to cut it over", node), nil
}
// reportFreshFor is how old a node's account of itself may be for the flip to act on it. A
@@ -720,11 +310,7 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s
}
derived := derivedFilter{rules: rules, foundation: with.Foundation, mesh: with.Mesh,
outward: plan.PublicDomain != "", outwardLinks: with.OutwardLinks}
filtering, err := inv.FilteringOf(ctx, node)
if err != nil {
return "", err
}
preview, saw := previewOf(node, reported, filtering, derived, plan, taken, filter, runs[filter])
preview, saw := previewOf(node, reported, derived, plan, taken, filter, runs[filter])
preview += "\n\n preview " + saw
if !yes {
return preview + fmt.Sprintf("\n\nNothing has changed. Run `converge %s --yes %s` to do "+
@@ -794,7 +380,7 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s
// previewOf is what converging a node will change, before it changes it, and a short digest of
// what it said: every reachable thing and its fate, the modules the flip takes and the filter. The
// digest is what the flip is asked to act on, so it changes whenever any of those would.
func previewOf(node string, reported inventory.Adoption, filtering inventory.Filtering, derived derivedFilter,
func previewOf(node string, reported inventory.Adoption, derived derivedFilter,
plan catalogue.Resolution, taken []string, filter string, filterAssigned bool) (string, string) {
var said []string
var b strings.Builder
@@ -886,30 +472,6 @@ func previewOf(node string, reported inventory.Adoption, filtering inventory.Fil
fmt.Fprintf(&b, " the found firewall (%s) is disabled, never flushed: its configuration stays on disk\n", fw)
}
said = append(said, fmt.Sprintf("filter %s assigned=%t firewall=%s", filter, filterAssigned, fw))
// What filters the machine now, and the fate of each (novox/hq ADR 0168): the found firewall
// retired, the runtime's own and bans left, and what the mesh did not write left and named —
// so the reader knows before the flip that the machine will not be filtered by the mesh alone.
if len(filtering.Filters) > 0 {
b.WriteString("\n what filters the machine now, and what the flip does to each:\n")
for _, x := range filtering.Filters {
fate := "left: " + x.Owner + "'s"
switch x.Owner {
case inventory.FilterMesh:
fate = "the mesh's guard; replaced by its filter"
case inventory.FilterFoundFirewall:
fate = "the found firewall's; retired with it"
case inventory.FilterRuntime:
fate = "the container runtime's own; left"
case inventory.FilterBan:
fate = "a ban list; left"
case inventory.FilterOther:
fate = "NOT THE MESH'S; left in force — the machine is not filtered by the mesh alone until you remove it"
}
fmt.Fprintf(&b, " %-50s %s\n", x.Where, fate)
fmt.Fprintf(&b, " %-50s %s\n", "", x.Refuses)
said = append(said, "filter "+x.Owner+" "+x.Where)
}
}
// Sorted: the same account, reported in another order, is the same preview.
sort.Strings(said)
sum := sha256.Sum256([]byte(strings.Join(said, "\n")))
@@ -1034,39 +596,12 @@ func adopt(ctx context.Context, open *stores, node string) (string, error) {
// takeCommand, convergeCommand and adoptCommand are the command line's adapters to the acts above.
func takeCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("take", flag.ContinueOnError)
yes := set.Bool("yes", false, "cut over as previewed, naming the digest the preview printed after it; "+
"without it the comparison is printed and nothing is taken")
downgrade := set.Bool("downgrade", false, "take it although the module's image is older than the one running")
var replace, mint stringList
set.Var(&replace, "replace", "a found file's path whose content the module may replace although it differs (repeatable; * for every one)")
set.Var(&mint, "mint", "a secret the service shall take the mesh's minted value for, although it already has its own (repeatable)")
positionals, err := parseAround(set, args)
if err != nil {
return err
if len(args) != 2 {
return errors.New("take <node> <module>")
}
if len(positionals) < 2 || len(positionals) > 3 || (len(positionals) == 3 && !*yes) {
return errors.New("take <node> <module> [--yes <digest>] [--downgrade] [--replace <path>]... [--mint <secret>]...")
}
opts := takeOptions{Yes: *yes, Downgrade: *downgrade, Replace: map[string]bool{}, Mint: map[string]bool{}}
if len(positionals) == 3 {
opts.Digest = positionals[2]
}
for _, r := range replace {
opts.Replace[r] = true
}
for _, m := range mint {
opts.Mint[m] = true
}
return runAct(ctx, func(open *stores) (string, error) { return take(ctx, open, positionals[0], positionals[1], opts) })
return runAct(ctx, func(open *stores) (string, error) { return take(ctx, open, args[0], args[1]) })
}
// stringList is a repeatable flag.
type stringList []string
func (l *stringList) String() string { return strings.Join(*l, ",") }
func (l *stringList) Set(v string) error { *l = append(*l, v); return nil }
func convergeCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("converge", flag.ContinueOnError)
yes := set.String("yes", "", "do it, naming the digest the preview printed; without it, only "+
+3 -3
View File
@@ -102,7 +102,7 @@ func commands(who Authenticator) http.Handler {
}))
// Adoption (novox/hq ADR 0100): the same acts as `take`, `converge` and `adopt`.
mux.HandleFunc("POST /take", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
return take(ctx, open, in.Node, in.Module, takeOptions{Yes: in.Yes, Digest: in.Digest})
return take(ctx, open, in.Node, in.Module)
}))
mux.HandleFunc("POST /converge", acting(who, false, func(ctx context.Context, open *stores, in request) (string, error) {
return converge(ctx, open, in.Node, in.Yes, in.Digest, in.Filter)
@@ -124,8 +124,8 @@ func commands(who Authenticator) http.Handler {
type request struct {
Node string `json:"node"`
Module string `json:"module"`
// Yes, Digest and Filter are converge's and take's: do it rather than preview it, the digest
// of the preview it acts on, and (converge) which module loads the mesh's filter.
// Yes, Digest and Filter are converge's: do it rather than preview it, the digest of the
// preview it acts on, and which module loads the mesh's filter.
Yes bool `json:"yes,omitempty"`
Digest string `json:"digest,omitempty"`
Filter string `json:"filter,omitempty"`
+37 -207
View File
@@ -10,8 +10,6 @@ import (
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
@@ -177,14 +175,10 @@ func buildFrom(result link.BuildResult) inventory.Build {
func buildsCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("builds", flag.ContinueOnError)
limit := set.Int("n", 20, "how many to show")
logOf := set.String("log", "", "a build's id: print what the build machine said, line by line")
positionals, err := parseAround(set, args)
if err != nil {
return err
}
if *logOf != "" {
return buildLog(ctx, *logOf)
}
module := ""
if len(positionals) == 1 {
module = positionals[0]
@@ -225,8 +219,8 @@ func buildsCommand(ctx context.Context, args []string) error {
if !b.Worked() {
outcome = "failed"
}
fmt.Printf("%-18s %-14s %-10s %s %s\n",
what, outcome, b.On, b.At.Local().Format("2006-01-02 15:04"), b.ID)
fmt.Printf("%-18s %-14s %-10s %s\n",
what, outcome, b.On, b.At.Local().Format("2006-01-02 15:04"))
fmt.Printf(" %s", b.Repository)
if b.Ref != "" {
fmt.Printf(" at %s", b.Ref)
@@ -414,14 +408,11 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti
Path: path,
Ref: ref,
Held: heldBy(ctx),
Seats: seatBases(ctx),
}
fmt.Printf("asked for %s", source)
if source.Seat != "" {
fmt.Printf(" (%s)", repository)
}
// The id is how a person follows this build while it runs: `builds --log <id>`.
fmt.Printf(" as %s", request.ID)
if path != "" {
fmt.Printf(" at %s", path)
}
@@ -430,114 +421,70 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti
}
fmt.Println()
seat := buildSeatHeld(ctx)
ask, err := askOverOn(seat)
ask, err := askOver(server)
if err != nil {
return err
}
defer ask.Close()
fmt.Printf(" of %s\n", seat)
if wait == 0 {
// Asked and not waited for (novox/hq issue 176): the outcome is the role's event, and the
// controller takes it in — records the build, registers the module — whether or not anybody
// is still here. A tool call cannot hold a connection for the minutes a build takes; it
// follows the build by its id instead.
if err := ask.Ask(ctx, request); err != nil {
return err
}
fmt.Printf("asked, not waited for: `builds --log %s` follows it as it runs, and `builds` "+
"shows what came of it; the module is registered when the outcome comes\n", request.ID)
return nil
}
result, err := ask.Submit(ctx, request, wait)
if err != nil {
return err
}
// Kept before it is judged. A failed build that leaves no trace is indistinguishable from one
// nobody asked for, and the difference is the whole of whether somebody should be looking at
// something.
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
manifest, kept, err := takeIn(ctx, open.inventory, result)
if err != nil {
inv := open.inventory
kept := buildFrom(result)
if err := inv.RecordBuild(ctx, kept); err != nil {
return err
}
if result.Failed != "" {
// The builder's own words. Wrapping them in something about the control plane would put
// two explanations between a person and a build log.
return fmt.Errorf("%s could not build %s:\n%s", result.On, result.Repository, result.Failed)
}
// Said as recorded: what each artifact is, not where this builder happened to push it.
for _, made := range kept.Made {
fmt.Printf(" %-12s %s %s\n", made.Name, made.Kind, made.Reference)
}
fmt.Printf("\n%s %s, built on %s from %s\n",
manifest.Module, manifest.Version, result.On, short(result.Commit))
saysWhenThePolicyActs(ctx, open.inventory, manifest.Module)
fmt.Printf(" run `assign <node> %s` to put it somewhere\n", manifest.Module)
return nil
}
// saysWhenThePolicyActs tells whoever built a module that its upgrade policy will send the
// result on at once (novox/hq issue 126, ADR 0163): a person choreographing a data move must
// know which module will not wait for them.
func saysWhenThePolicyActs(ctx context.Context, inv *inventory.Inventory, module string) {
if u, err := inv.UpgradeOf(ctx, module); err == nil && u.RollOut {
how := "one machine at a time"
if u.Together {
how = "every machine at once"
}
fmt.Printf(" %s rolls out on build: the machines running it are sent this now, %s — "+
"`upgrade %s record` first if something must move before it does\n", module, how, module)
}
}
// takeIn is what the mesh does with a build's outcome, whoever hears it: the waiting command and
// the daemon that follows the role's events both come here (novox/hq issue 176), so a build's
// result reaches the catalogue whether or not the asker was still listening.
//
// Kept before it is judged. A failed build that leaves no trace is indistinguishable from one
// nobody asked for, and the difference is the whole of whether somebody should be looking at
// something. Then parsed with the same parser a hand-written manifest goes through — a second path
// would be a second thing to disagree about what a manifest is — and registered with where it came
// from: **for a source on a seat, as the path and the seat, never the URL just cloned** (ADR 0111),
// which the request carried and the outcome echoes. A definition naming an installation is refused
// here, where it would enter the catalogue; the build stays recorded and the refusal says which.
//
// Idempotent: the same outcome taken in twice registers the same module twice, which is one row
// written with the same values.
func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResult) (
catalogue.Manifest, inventory.Build, error) {
kept := buildFrom(result)
if err := inv.RecordBuild(ctx, kept); err != nil {
return catalogue.Manifest{}, kept, err
}
if result.Failed != "" {
// The builder's own words. Wrapping them in something about the control plane would put
// two explanations between a person and a build log.
return catalogue.Manifest{}, kept, fmt.Errorf("%s could not build %s:\n%s",
result.On, result.Repository, result.Failed)
}
// Parsed with the same parser a hand-written manifest goes through. A second path would be a
// second thing to disagree about what a manifest is. The manifest as recorded, so the catalogue
// holds references by digest and path and every declaration composes the store's address in.
manifest, err := catalogue.ParseManifest(kept.Manifest)
if err != nil {
return catalogue.Manifest{}, kept, fmt.Errorf("%s built %s and what came back is not a manifest: %w",
return fmt.Errorf("%s built %s and what came back is not a manifest: %w",
result.On, result.Repository, err)
}
// Recorded with where it came from, so "is this current?" is answerable without building it
// again (novox/hq ADR 0009). **For a source on a seat, as the path and the seat, never the URL
// just cloned** (ADR 0111): the URL is where the forge runs today, and recording it would put
// the forge's address back into every module built from it. The build log above keeps the URL,
// because that is what was cloned.
recorded := inventory.Source{
Repository: result.Repository, Path: result.Path, Ref: result.Ref,
BuiltFrom: result.Commit, Head: result.Commit,
// What it stood on, so registration can judge a built manifest's base (to-be 38 WP2.4).
Against: kept.Against,
}
if result.Source != nil && result.Source.Seat != "" {
recorded.Repository, recorded.Seat = result.Source.Repository, result.Source.Seat
}
if err := namesNoInstallation(manifest); err != nil {
return manifest, kept, fmt.Errorf("%s built %s (%s), and the mesh does not register it: %w",
result.On, result.Repository, short(result.Commit), err)
if source.Seat != "" {
recorded.Repository, recorded.Seat = source.Repository, source.Seat
}
if err := inv.RegisterModule(ctx, manifest, recorded); err != nil {
return manifest, kept, err
return err
}
return manifest, kept, nil
fmt.Printf("\n%s %s, built on %s from %s\n",
manifest.Module, manifest.Version, result.On, short(result.Commit))
fmt.Printf(" run `assign <node> %s` to put it somewhere\n", manifest.Module)
return nil
}
// buildAndShow builds and prints the manifest without recording anything.
@@ -557,7 +504,7 @@ func buildAndShow(ctx context.Context, source buildSource, path, ref string, wai
}
defer server.Close()
ask, err := askOverOn(buildSeatHeld(ctx))
ask, err := askOver(server)
if err != nil {
return err
}
@@ -566,7 +513,7 @@ func buildAndShow(ctx context.Context, source buildSource, path, ref string, wai
result, err := ask.Submit(ctx, link.BuildRequest{
ID: fmt.Sprintf("%s-%d", "build", time.Now().UnixNano()),
Repository: repository, Path: path, Ref: ref,
Held: heldBy(ctx), Seats: seatBases(ctx),
Held: heldBy(ctx),
}, wait)
if err != nil {
return err
@@ -600,8 +547,6 @@ type answers struct {
// pair that answers "has it caught up", which waiting alone cannot (the sent digest is
// recorded at send, not at apply).
reported []inventory.Reported
// plans is what the last merges produced and where each stands (novox/hq ADR 0162).
plans []inventory.Plan
// refused is why a machine cannot be worked out at all, by name. A different thing from every
// other answer here: those are about a machine that was told something, and this is about one
// that cannot be told anything — it never reaches waiting, because nothing was computed for it
@@ -611,20 +556,6 @@ type answers struct {
// a consequence of the refusals above: a node that does not resolve is not on the network, and
// a mesh whose hub is that node has no hub.
network string
// filtered is every converged machine that is not filtered by the mesh alone (novox/hq ADR
// 0168): what filters it beyond the mesh's own, the runtime's plumbing and bans, by name — a
// predecessor's chain, a found firewall in force again. Such a machine is not "all well".
filtered map[string]inventory.Filtering
// untaken is, per machine, each assigned module whose resources the machine is holding as it
// found them, and how many — a module that was assigned, sent, and is running none of what it
// declares because nothing has taken it (novox/hq ADR 0100, 04-ISSUES/125).
//
// **Its absence cost an outage.** The module was assigned, the push reported success, this
// command said the machine was doing everything it was told, and the module's three containers
// did not exist. On the strength of those reports the predecessor's proxy was stopped and every
// public name on the machine went dark. The holds were correct; they were recorded only in the
// machine's own state file, and the one visible symptom was a count that did not add up.
untaken map[string]map[string]int
}
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
@@ -670,111 +601,10 @@ func heldBy(ctx context.Context) map[string]string {
// **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5). On the bus
// the mesh runs on today this needs the controller's own connection, so it is handed one; on the bus
// being built it dials, because a build request is a one-shot and holds nothing else.
func askOverOn(seat string) (link.Builders, error) {
func askOver(_ *link.Server) (link.Builders, error) {
address, err := broker.BusAddress()
if err != nil {
return nil, err
}
return link.BuildsOverNATSOn(address, seat)
}
// buildSeatHeld is the build role to ask: the one some assigned module claims (novox/hq ADR 0190,
// the handover). Read from the catalogue at ask time, because the answer changes exactly once, the
// moment the first build-agent is assigned — and a controller that asked the new role before then
// would queue work nothing takes, while the outcome that registers build-agent itself has to come
// from the old builder. When the catalogue cannot be read the current role is asked, said aloud.
func buildSeatHeld(ctx context.Context) string {
open, err := openStores(ctx)
if err != nil {
fmt.Fprintf(os.Stderr, "could not read what is assigned, so the build is asked of %s: %v\n",
link.TheBuildMachine, err)
return link.TheBuildMachine
}
defer open.Close()
entries, err := open.inventory.Catalogued(ctx)
if err != nil {
fmt.Fprintf(os.Stderr, "could not read the catalogue, so the build is asked of %s: %v\n",
link.TheBuildMachine, err)
return link.TheBuildMachine
}
return buildSeatAmong(entries)
}
// buildSeatAmong is the rule, over what the catalogue holds: the current build role when any
// assigned module claims it; else the retired role while an assigned module still claims that; else
// the current role, which is where every ask goes once the handover is done.
func buildSeatAmong(entries []inventory.Entry) string {
heldBefore := false
for _, e := range entries {
if len(e.On) == 0 {
continue
}
if e.Manifest.ClaimsSeat(link.TheBuildMachine) {
return link.TheBuildMachine
}
if e.Manifest.ClaimsSeat(link.TheBuildMachineBefore) {
heldBefore = true
}
}
if heldBefore {
return link.TheBuildMachineBefore
}
return link.TheBuildMachine
}
// buildLog prints everything a build machine said about one build, read back from the bus.
//
// **From the stream, not from a record** (novox/hq ADR 0157). A build's lines are the role's own
// events under the build's id, retained with every other event; the mesh keeps no second copy. Read
// with a consumer of its own that is gone when this returns, so nothing accumulates in the server
// for the reading, and filtered by subject, so one build's lines are all that travel.
func buildLog(ctx context.Context, id string) error {
address, err := broker.BusAddress()
if err != nil {
return err
}
js, err := broker.Dial(address)
if err != nil {
return fmt.Errorf("cannot reach the bus to read a build's log: %w", err)
}
defer js.Close()
// Under whichever build role did it: a build asked of the retired role during the handover
// (ADR 0190) said its lines as that role's events, and a reader should not have to know which.
lines := link.BuildLogOf("*", id)
sub, err := js.Context().PullSubscribe(lines, "",
nats.BindStream(broker.EventsStream), nats.DeliverAll(), nats.AckNone())
if err != nil {
return fmt.Errorf("cannot read %s from the bus: %w", lines, err)
}
defer func() { _ = sub.Unsubscribe() }()
printed := 0
for {
batch, err := sub.Fetch(200, nats.MaxWait(2*time.Second))
if err != nil && !errors.Is(err, nats.ErrTimeout) && !errors.Is(err, context.DeadlineExceeded) {
return fmt.Errorf("reading a build's log: %w", err)
}
for _, msg := range batch {
var line link.BuildLine
if err := json.Unmarshal(msg.Data, &line); err != nil {
fmt.Printf(" ? %s\n", string(msg.Data))
continue
}
at := line.At
if t, err := time.Parse(time.RFC3339Nano, line.At); err == nil {
at = t.Local().Format("15:04:05")
}
fmt.Printf("%s %4d [%s] %s\n", at, line.Seq, line.Step, line.Message)
printed++
}
if len(batch) < 200 {
break
}
}
if printed == 0 {
fmt.Printf("nothing on the bus for build %s: no build by that id in the last week, or a build "+
"machine older than this that said nothing while building\n", id)
}
return nil
return link.BuildsOverNATS(address)
}
-43
View File
@@ -1,43 +0,0 @@
package main
import (
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
func claiming(module, seat string, on ...string) inventory.Entry {
return inventory.Entry{
Manifest: catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: seat}}},
On: on,
}
}
// The controller asks the build role that has a holder (novox/hq ADR 0190 handover): the retired
// one while only the builder is assigned, the current one from the first build-agent on, and the
// current one when nothing holds either — where every ask goes once the handover is done.
func TestTheControllerAsksTheBuildRoleThatHasAHolder(t *testing.T) {
onlyTheBuilder := []inventory.Entry{
claiming("builder", link.TheBuildMachineBefore, "anchor"),
claiming("build-agent", link.TheBuildMachine), // registered, assigned nowhere yet
}
if got := buildSeatAmong(onlyTheBuilder); got != link.TheBuildMachineBefore {
t.Errorf("with only the builder assigned, asked %q", got)
}
bothHeld := []inventory.Entry{
claiming("builder", link.TheBuildMachineBefore, "anchor"),
claiming("build-agent", link.TheBuildMachine, "home-server"),
}
if got := buildSeatAmong(bothHeld); got != link.TheBuildMachine {
t.Errorf("with a build-agent assigned anywhere, asked %q", got)
}
neither := []inventory.Entry{claiming("builder", link.TheBuildMachineBefore)}
if got := buildSeatAmong(neither); got != link.TheBuildMachine {
t.Errorf("with no holder of either, asked %q, want the current role", got)
}
if got := buildSeatAmong(nil); got != link.TheBuildMachine {
t.Errorf("an empty catalogue asks %q", got)
}
}
-65
View File
@@ -1,65 +0,0 @@
package main
import (
"encoding/json"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/link"
)
// A build's outcome is taken in the same way whoever hears it (novox/hq issue 176): recorded, and
// the module registered with its source as the seat and path when the request said so — never the
// URL. A definition naming an installation is recorded and not registered; a failure is recorded
// and said.
func TestABuildHeardIsRecordedAndRegistered(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
manifest, _ := json.Marshal(map[string]any{"module": "shop", "version": "3"})
m, _, err := takeIn(ctx, open.inventory, link.BuildResult{
ID: "b-1", Repository: "http://forge.internal:20000/novox/shop.git", Path: "modules/shop",
Ref: "main", On: "anchor", Commit: "abcdef0123", Manifest: manifest,
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/shop"},
})
if err != nil {
t.Fatal(err)
}
if m.Module != "shop" {
t.Fatalf("registered %q", m.Module)
}
shelf, err := open.inventory.Catalogue(ctx)
if err != nil {
t.Fatal(err)
}
if _, held := shelf["shop"]; !held {
t.Fatal("the module a heard build produced is not in the catalogue")
}
src, err := open.inventory.SourceOf(ctx, "shop")
if err != nil || src.Seat != "git" || src.Repository != "novox/shop" || src.BuiltFrom != "abcdef0123" {
t.Fatalf("the source is the seat and the path, never the URL: %+v %v", src, err)
}
builds, err := open.inventory.Builds(ctx, "shop", 5)
if err != nil || len(builds) != 1 || builds[0].ID != "b-1" {
t.Fatalf("the build is not recorded once: %v %v", builds, err)
}
named, _ := json.Marshal(map[string]any{"module": "idp", "version": "1", "resources": []any{
map[string]any{"id": "server", "type": "container", "image": "x@sha256:aa",
"env": map[string]any{"KC_HOSTNAME": "https://login.mesh-one.be"}}}})
_, _, err = takeIn(ctx, open.inventory, link.BuildResult{
ID: "b-2", Repository: "/r", On: "anchor", Commit: "0123456789", Manifest: named})
if err == nil || !strings.Contains(err.Error(), "does not register it") {
t.Fatalf("a definition naming an installation was taken in: %v", err)
}
if shelf, _ := open.inventory.Catalogue(ctx); shelf["idp"].Module != "" {
t.Fatal("the refused module was registered anyway")
}
if builds, _ := open.inventory.Builds(ctx, "idp", 5); len(builds) != 1 {
t.Fatalf("the refused build was not recorded: %v", builds)
}
_, _, err = takeIn(ctx, open.inventory, link.BuildResult{ID: "b-3", Repository: "/r", On: "anchor", Failed: "no compiler"})
if err == nil || !strings.Contains(err.Error(), "no compiler") {
t.Fatalf("a failure is said in the builder's words: %v", err)
}
}
-258
View File
@@ -1,258 +0,0 @@
package main
import (
"context"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"errors"
"fmt"
"math/big"
"net"
"os"
"path/filepath"
"strings"
"time"
"github.com/novox/mesh-controller/internal/broker"
)
// The bus's own certificate, made by the mesh rather than borrowed from an image.
//
// **The foundation asked a third-party image for a tool it never said must be there** (novox/hq
// 04-ISSUES/146). The bootstrap made this certificate by running `openssl` inside the broker's
// image, which worked while the broker was one that happened to carry it and stopped the day the
// bus changed: the new one has a shell and no openssl, so the step exited 127 and no mesh could be
// raised. Substituting another image the bundle names does not help — none of them carry it
// either.
//
// So the program that needs a certificate makes one. It is the mesh's own binary, already on the
// machine at this point in the bootstrap (the schema step ran it), and it needs nothing from the
// image it writes into but a mounted directory.
//
// **Self-signed, and that is the design** — a host pins this server's exact certificate and
// authenticates with a password (novox/hq ADR 0004). There is no authority above it to ask, and at
// this moment in a bootstrap there is no mesh to ask one of.
//
// Idempotent, because the step is applied again on every reconcile and a second certificate would
// be one the hosts that pinned the first no longer believe.
// busCertificateNames is what the bus is reached by: the container name on a mesh network, and the
// loopback address the machine's own foundation dials.
var busCertificateNames = []string{"mesh-broker"}
const busCertificateLife = 10 * 365 * 24 * time.Hour
// busCertificate makes the bus's certificate in a directory, or says whether one is there.
//
// broker certificate --into /tls make it if it is not there
// broker certificate --check --into /tls exit non-zero unless a usable pair is
func busCertificate(args []string) error {
into, check := "", false
for i := 0; i < len(args); i++ {
switch args[i] {
case "--check":
check = true
case "--into":
if i+1 >= len(args) {
return errors.New("--into needs a directory")
}
into = args[i+1]
i++
default:
return fmt.Errorf("broker certificate [--check] --into <directory>: %q", args[i])
}
}
if into == "" {
return errors.New("broker certificate [--check] --into <directory>")
}
crt, key := filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key")
if usable, err := busCertificateUsable(crt, key); err != nil {
return err
} else if usable {
fmt.Printf("the bus already has a certificate at %s, and it was left alone\n", crt)
return nil
}
if check {
// Said as a failure, because that is what the caller asked: a bootstrap's verify runs
// this and a false answer is what makes the step run.
return fmt.Errorf("no usable certificate and key at %s", into)
}
return writeBusCertificate(crt, key)
}
// busCertificateUsable says whether a certificate and its key are both there and parse.
//
// Both, and parsed rather than stat'ed: a half-written pair is the state a bootstrap interrupted
// between the two files leaves behind, and a step that treated it as done would hand the server a
// certificate with no key and report success.
func busCertificateUsable(crt, key string) (bool, error) {
certPEM, err := os.ReadFile(crt)
if errors.Is(err, os.ErrNotExist) {
return false, nil
}
if err != nil {
return false, err
}
keyPEM, err := os.ReadFile(key)
if errors.Is(err, os.ErrNotExist) {
return false, nil
}
if err != nil {
return false, err
}
if _, err := tlsPairParses(certPEM, keyPEM); err != nil {
return false, nil
}
return true, nil
}
func tlsPairParses(certPEM, keyPEM []byte) (*x509.Certificate, error) {
block, _ := pem.Decode(certPEM)
if block == nil || block.Type != "CERTIFICATE" {
return nil, errors.New("not a certificate")
}
certificate, err := x509.ParseCertificate(block.Bytes)
if err != nil {
return nil, err
}
keyBlock, _ := pem.Decode(keyPEM)
if keyBlock == nil {
return nil, errors.New("not a key")
}
if _, err := x509.ParsePKCS8PrivateKey(keyBlock.Bytes); err != nil {
if _, err := x509.ParsePKCS1PrivateKey(keyBlock.Bytes); err != nil {
return nil, err
}
}
return certificate, nil
}
func writeBusCertificate(crt, key string) error {
private, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
return err
}
serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
if err != nil {
return err
}
template := &x509.Certificate{
SerialNumber: serial,
Subject: pkix.Name{CommonName: busCertificateNames[0]},
DNSNames: busCertificateNames,
IPAddresses: []net.IP{net.ParseIP("127.0.0.1")},
NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().Add(busCertificateLife),
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
BasicConstraintsValid: true,
}
der, err := x509.CreateCertificate(rand.Reader, template, template, &private.PublicKey, private)
if err != nil {
return err
}
pkcs8, err := x509.MarshalPKCS8PrivateKey(private)
if err != nil {
return err
}
// **The key first, and only then the certificate**, so the pair a reader finds is never a
// certificate whose key has not been written yet — the one order in which an interruption
// leaves something that looks finished (novox/hq 04-ISSUES/014, a key present and unusable).
if err := os.WriteFile(key, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8}), 0o600); err != nil {
return err
}
if err := os.WriteFile(crt, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644); err != nil {
return err
}
fmt.Printf("made the bus a certificate for %v, valid until %s\n %s\n %s\n",
busCertificateNames, template.NotAfter.Format(time.RFC3339), crt, key)
return nil
}
// busAccounts writes the mesh's composed user list to a file.
//
// **For genesis, where no declaration can deliver it** (novox/hq 04-ISSUES/146). Everywhere else
// the list reaches the machine running the bus as a resource of the module that holds it — which
// requires that machine to be an enrolled node, and at genesis it is not: the first node cannot
// enrol because the account it would enrol with cannot be composed onto a bus it has no declaration
// for. The installer breaks that circle by placing the file itself, once, and the module takes the
// file over from its first push.
//
// The same composition, not a second one: this asks the store for the same records and renders them
// with the same composer the declaration uses. A genesis that hand-wrote an account would be a
// second statement of who may say what, able to disagree with the first.
//
// **It writes to standard output unless told a file**, and that is the point: the control plane
// composes and says what it composed, and whoever is raising the machine puts it where that
// machine's bus reads it. A control plane that wrote into the bus's own directory would have to
// know where that is and how to make the server re-read it — which is the module's knowledge, and
// the module is what takes this over on the first push.
//
// broker accounts > /var/lib/mesh-bus-conf/accounts.conf
func busAccounts(ctx context.Context, args []string) error {
into := ""
for i := 0; i < len(args); i++ {
switch args[i] {
case "--into":
if i+1 >= len(args) {
return errors.New("--into needs a file")
}
into = args[i+1]
i++
default:
return fmt.Errorf("broker accounts --into <file>: %q", args[i])
}
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
records, err := open.inventory.BusRecords(ctx)
if err != nil {
return err
}
users, err := broker.Users(records)
if err != nil {
return err
}
kept, err := open.inventory.BusUsers(ctx)
if err != nil {
return err
}
hashes := make(map[string]string, len(kept))
for name, u := range kept {
hashes[name] = u.PasswordHash
}
filled, missing := broker.WithPasswords(users, hashes)
if len(missing) > 0 {
// To standard error, always: the composed file may be going to standard output, and a
// remark in the middle of it is a configuration the server refuses to parse.
fmt.Fprintf(os.Stderr, "leaving out %d user(s) the mesh has minted no credential for: %s\n",
len(missing), strings.Join(missing, ", "))
}
if len(filled) == 0 {
return errors.New("not one user has a credential, so this list would refuse every " +
"connection in the mesh")
}
accounts, err := broker.ComposeAccounts(filled)
if err != nil {
return err
}
if into == "" {
fmt.Print(accounts)
return nil
}
if err := os.WriteFile(into, []byte(accounts), 0o600); err != nil {
return err
}
fmt.Printf("wrote %d user(s) to %s\n", len(filled), into)
return nil
}
-121
View File
@@ -1,121 +0,0 @@
package main
import (
"crypto/tls"
"crypto/x509"
"os"
"path/filepath"
"strings"
"testing"
)
// novox/hq 04-ISSUES/146. The bootstrap could not make the bus a certificate: it asked an image for
// `openssl` and the image it asks has none. What replaces it is this command, so what is checked is
// what the bootstrap needs from it — a pair a TLS server can actually load, made once and only once.
func TestTheBusCertificateLoadsAsAServersWould(t *testing.T) {
into := t.TempDir()
if err := busCertificate([]string{"--into", into}); err != nil {
t.Fatalf("the bus could not be given a certificate: %v", err)
}
// The check a cheaper test would not make. The key was present and valid and the server could
// not start, once, because nothing loaded the pair the way a server loads it
// (novox/hq 04-ISSUES/014).
pair, err := tls.LoadX509KeyPair(filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key"))
if err != nil {
t.Fatalf("a TLS server cannot load what was written: %v", err)
}
leaf := pair.Leaf
if leaf == nil {
if leaf, err = x509.ParseCertificate(pair.Certificate[0]); err != nil {
t.Fatal(err)
}
}
if err := leaf.VerifyHostname("mesh-broker"); err != nil {
t.Errorf("the certificate is not for the name the bus is reached by: %v", err)
}
if len(leaf.IPAddresses) == 0 || leaf.IPAddresses[0].String() != "127.0.0.1" {
t.Errorf("the certificate does not cover the loopback address the foundation dials: %v", leaf.IPAddresses)
}
// The key is not readable by anything else on the machine; the certificate is public and is.
key, err := os.Stat(filepath.Join(into, "tls.key"))
if err != nil {
t.Fatal(err)
}
if key.Mode().Perm() != 0o600 {
t.Errorf("the key is %v", key.Mode().Perm())
}
crt, err := os.Stat(filepath.Join(into, "tls.crt"))
if err != nil {
t.Fatal(err)
}
if crt.Mode().Perm() != 0o644 {
t.Errorf("the certificate is %v, which the server runs as another user cannot read", crt.Mode().Perm())
}
}
// **Made once.** The step is applied again on every reconcile, and a second certificate is one the
// hosts that pinned the first no longer believe (novox/hq ADR 0004).
func TestTheBusCertificateIsMadeOnce(t *testing.T) {
into := t.TempDir()
if err := busCertificate([]string{"--into", into}); err != nil {
t.Fatal(err)
}
first, err := os.ReadFile(filepath.Join(into, "tls.crt"))
if err != nil {
t.Fatal(err)
}
if err := busCertificate([]string{"--into", into}); err != nil {
t.Fatal(err)
}
again, err := os.ReadFile(filepath.Join(into, "tls.crt"))
if err != nil {
t.Fatal(err)
}
if string(first) != string(again) {
t.Fatal("running it twice replaced the certificate every host had pinned")
}
}
// The verify half: false before, true after, which is what makes the bootstrap run the step at all.
func TestTheCheckIsFalseUntilThereIsAPair(t *testing.T) {
into := t.TempDir()
if err := busCertificate([]string{"--check", "--into", into}); err == nil {
t.Fatal("an empty directory reported a usable certificate")
}
if err := busCertificate([]string{"--into", into}); err != nil {
t.Fatal(err)
}
if err := busCertificate([]string{"--check", "--into", into}); err != nil {
t.Fatalf("the certificate it just made does not satisfy its own check: %v", err)
}
}
// A half-written pair is not a pair. An interrupted bootstrap leaves exactly this, and a step that
// called it done would hand the server a certificate with no key and report success.
func TestACertificateWithoutItsKeyIsNotUsable(t *testing.T) {
into := t.TempDir()
if err := busCertificate([]string{"--into", into}); err != nil {
t.Fatal(err)
}
if err := os.Remove(filepath.Join(into, "tls.key")); err != nil {
t.Fatal(err)
}
if err := busCertificate([]string{"--check", "--into", into}); err == nil {
t.Fatal("a certificate with no key passed the check")
}
if err := busCertificate([]string{"--into", into}); err != nil {
t.Fatal(err)
}
if _, err := tls.LoadX509KeyPair(filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key")); err != nil {
t.Fatalf("it did not replace the unusable pair: %v", err)
}
}
func TestWhereToWriteIsRequired(t *testing.T) {
if err := busCertificate(nil); err == nil || !strings.Contains(err.Error(), "--into") {
t.Fatalf("it did not ask where to write: %v", err)
}
}
-135
View File
@@ -1,135 +0,0 @@
package main
import (
"errors"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"github.com/novox/mesh-controller/internal/catalogue"
)
// moduleCheck judges manifests where they are written, with no mesh (novox/hq ADR 0037, issue 148).
//
// **The same functions registration runs, and nothing the command line adds** (ADR 0035): the strict
// parse with every per-manifest problem, then the rules no single manifest can be judged against,
// over exactly the manifests given. Somebody describing their own application in their own
// repository runs this before pushing and finds out there, rather than when a running mesh refuses
// the registration or, later, when a machine applies something that resolved and should not have.
//
// **What it cannot know without a store, it says.** The mesh's own seat set is the store's (ADR
// 0122); this binary carries a compiled copy that the store overrides when loaded, so a claim on a
// mesh seat is judged fully only at registration. A seat another module declares is unknown unless
// that module's manifest is passed too. Both are printed as a note, not as a problem — a check that
// refused what it could not see would teach people to ignore it.
func moduleCheck(paths []string, out io.Writer) error {
if len(paths) == 0 {
return errors.New("module check <manifest.json>... — one file per module; pass every " +
"manifest of a repository together so the rules between them are checked too")
}
shelf := catalogue.Shelf{}
faulted := map[string]bool{}
failed := 0
for _, path := range paths {
raw, err := os.ReadFile(path)
if err != nil {
fmt.Fprintf(out, "%s: %v\n", path, err)
failed++
continue
}
m, err := catalogue.ParseManifest(raw)
if err != nil {
fmt.Fprintf(out, "%s: %v\n", path, err)
failed++
continue
}
if first, twice := shelf[m.Module]; twice {
_ = first
fmt.Fprintf(out, "%s: %s was already given; two manifests name one module\n", path, m.Module)
failed++
continue
}
// A definition names no installation (novox/hq ADR 0112, ADR 0155): judged here, in the
// catalogue-wide test, and at registration, which refuses in the same words.
if named := catalogue.InstallationProblems(m); len(named) > 0 {
for _, p := range named {
fmt.Fprintf(out, "%s: %s\n", path, p)
}
failed += len(named)
faulted[m.Module] = true
}
shelf[m.Module] = m
}
// Between the manifests: a seat declared twice, a use of a seat nothing declares, a claim on
// a seat that does not exist. Run only over what parsed, because a problem inside one manifest
// has already been said and would be said again here in a worse form.
problems := catalogue.CatalogueProblems(shelf)
sort.Strings(problems)
for _, p := range problems {
fmt.Fprintln(out, p)
}
failed += len(problems)
var names []string
for name := range shelf {
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
m := shelf[name]
if faulted[name] {
continue
}
fmt.Fprintf(out, "%s: ok", name)
if n := len(m.Tools); n > 0 {
fmt.Fprintf(out, ", %d tool(s)", n)
}
if len(m.Invokes) > 0 {
fmt.Fprintf(out, ", invokes %s", joinInvokes(m.Invokes))
}
fmt.Fprintln(out)
}
if failed > 0 {
return fmt.Errorf("%d problem(s) in %d manifest(s)", failed, len(paths))
}
fmt.Fprintf(out, "%d manifest(s) checked. Judged against the seats this binary carries; a claim on "+
"one of the mesh's own seats is judged fully at registration, and a seat declared by a "+
"module not given here reads as unknown\n", len(paths))
return nil
}
func joinInvokes(invokes []string) string {
if len(invokes) == 1 && invokes[0] == "*" {
return "every tool"
}
s := ""
for i, t := range invokes {
if i > 0 {
s += ", "
}
s += t
}
return s
}
// manifestsUnder lists every module.json below a directory, for `module check <dir>`.
func manifestsUnder(dir string) ([]string, error) {
var found []string
err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() && (d.Name() == "node_modules" || d.Name() == ".git" || d.Name() == "dist") {
return filepath.SkipDir
}
if !d.IsDir() && d.Name() == "module.json" {
found = append(found, path)
}
return nil
})
sort.Strings(found)
return found, err
}
-94
View File
@@ -1,94 +0,0 @@
package main
import (
"bytes"
"os"
"path/filepath"
"strings"
"github.com/novox/mesh-controller/internal/catalogue"
"testing"
)
// The check anybody can run is the check registration runs (novox/hq issue 148, ADR 0037): a manifest
// with a known fault is named, and one without passes, with no store opened.
func TestModuleCheckNamesAFaultAndNeedsNoMesh(t *testing.T) {
dir := t.TempDir()
good := filepath.Join(dir, "good.json")
bad := filepath.Join(dir, "bad.json")
os.WriteFile(good, []byte(`{"module":"shop","version":"1","tools":["price"],"invokes":["mesh-catalog.catalog_modules"]}`), 0o600)
os.WriteFile(bad, []byte(`{"module":"till","version":"1","invokes":["shop"]}`), 0o600)
var out bytes.Buffer
if err := moduleCheck([]string{good}, &out); err != nil {
t.Fatalf("a sound manifest was refused: %v\n%s", err, out.String())
}
if !strings.Contains(out.String(), "shop: ok, 1 tool(s), invokes mesh-catalog.catalog_modules") {
t.Fatalf("the report does not say what it checked:\n%s", out.String())
}
out.Reset()
err := moduleCheck([]string{good, bad}, &out)
if err == nil {
t.Fatal("a manifest invoking a module and no tool passed")
}
if !strings.Contains(out.String(), `till invokes "shop", which does not name a tool`) {
t.Fatalf("the fault is not named in the manifest's words:\n%s", out.String())
}
}
// The rules between manifests run over what was given together: a seat two modules declare is
// refused, which no single-manifest check can see.
func TestModuleCheckJudgesBetweenTheManifestsGiven(t *testing.T) {
dir := t.TempDir()
a := filepath.Join(dir, "a.json")
b := filepath.Join(dir, "b.json")
os.WriteFile(a, []byte(`{"module":"a","version":"1","seats":[{"name":"printer","scope":"mesh"}]}`), 0o600)
os.WriteFile(b, []byte(`{"module":"b","version":"1","seats":[{"name":"printer","scope":"mesh"}]}`), 0o600)
var out bytes.Buffer
if err := moduleCheck([]string{a, b}, &out); err == nil {
t.Fatalf("two declarations of one seat passed:\n%s", out.String())
}
if !strings.Contains(out.String(), "a seat name means one protocol") {
t.Fatalf("the cross-manifest rule was not the one named:\n%s", out.String())
}
}
// The real catalogue passes the command, the way it passes the test that used to be the only check.
func TestModuleCheckPassesTheCatalogue(t *testing.T) {
root := filepath.Join("..", "..", "..", "mesh-catalog", "modules")
if _, err := os.Stat(root); err != nil {
t.Skipf("catalogue sibling not present: %v", err)
}
paths, err := manifestsUnder(root)
if err != nil || len(paths) == 0 {
t.Fatalf("no manifests under %s: %v", root, err)
}
var out bytes.Buffer
if err := moduleCheck(paths, &out); err != nil {
t.Fatalf("the catalogue does not pass its own check: %v\n%s", err, out.String())
}
}
func TestRegistrationRefusesADefinitionNamingAnInstallation(t *testing.T) {
// novox/hq ADR 0155: the check moves to registration once the catalogue passes it. Both
// ways in — `module add` and a build's result — go through this, and a name declared on
// purpose passes with its reason.
named := catalogue.Manifest{Module: "idp", Resources: []map[string]any{
{"id": "server", "type": "container", "image": "x@sha256:aa",
"env": map[string]any{"KC_HOSTNAME": "https://login.mesh-one.be"}},
}}
err := namesNoInstallation(named)
if err == nil || !strings.Contains(err.Error(), "login.mesh-one.be") ||
!strings.Contains(err.Error(), catalogue.NamesOnPurpose) {
t.Fatalf("a definition naming an installation is refused with the name and the way out; got %v", err)
}
meant := catalogue.Manifest{Module: "site", Resources: []map[string]any{
{"id": "server", "type": "container", "image": "registry.mesh-one.be/org/site@sha256:cc",
catalogue.NamesOnPurpose: map[string]any{
"registry.mesh-one.be": "built outside the mesh until its repository is a build source here"}},
}}
if err := namesNoInstallation(meant); err != nil {
t.Fatalf("a name declared on purpose passes; got %v", err)
}
}
@@ -1,90 +0,0 @@
package main
import (
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
)
// A fresh assignment is pushed before its credential exists (novox/hq issue 203): `assign` recorded
// the module, `push` sealed a random own secret where the bus credential belongs, and the process
// crash-looped until a person ran `module issue` and pushed again. Now assigning a module that speaks
// on the bus issues its credential in the same act — or, when the bus cannot be reached from here,
// says which verb to run — and a push never seals a placeholder in a credential's place.
func aTalker() catalogue.Manifest {
return catalogue.Manifest{Module: "talker", Version: "1",
OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/var/lib/mesh/talker/broker"}},
Resources: []map[string]any{
{"id": "state", "type": "directory", "path": "/var/lib/mesh/talker", "mode": "0700"},
}}
}
func TestAssigningAModuleThatSpeaksOnTheBusNamesItsCredential(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, aTalker())
// No bus is known to this process, so the credential cannot be issued here: the assignment
// stands and says exactly what must happen before a push — never silently.
said, err := assign(ctx, open, "laptop", "talker")
if err != nil {
t.Fatal(err)
}
if !strings.Contains(said, "module issue talker --node laptop") {
t.Fatalf("an assignment whose credential could not be issued does not name the verb:\n%s", said)
}
// And the push refuses to send it, naming the same verb, rather than sealing a placeholder.
plan, settings, err := planFor(ctx, open, "laptop")
if err != nil {
t.Fatal(err)
}
_, err = declarationFor(ctx, open, "laptop", plan, settings)
if err == nil {
t.Fatal("a push sealed a placeholder where talker's bus credential belongs")
}
if !strings.Contains(err.Error(), "module issue talker --node laptop") || !strings.Contains(err.Error(), "issue 203") {
t.Fatalf("the refusal does not say what to run: %v", err)
}
// Once the user is minted, the push goes on to the credential the mesh sealed, and re-assigning
// does not mint again: a credential rotates on purpose, never by habit.
if _, err := open.inventory.MintBusPassword(ctx, inventory.BusUser{
Username: "laptop.talker", Kind: inventory.BusModule, Node: "laptop", Module: "talker"}); err != nil {
t.Fatal(err)
}
hash, _, err := open.inventory.BusUserHash(ctx, "laptop.talker")
if err != nil {
t.Fatal(err)
}
said, err = assign(ctx, open, "laptop", "talker")
if err != nil {
t.Fatal(err)
}
if strings.Contains(said, "module issue") {
t.Fatalf("a module with a minted credential was told to issue one:\n%s", said)
}
again, _, err := open.inventory.BusUserHash(ctx, "laptop.talker")
if err != nil {
t.Fatal(err)
}
if again != hash {
t.Fatal("re-assigning rotated the credential")
}
}
// A module that declares no broker secret is left alone: nothing to issue, nothing said.
func TestAssigningAModuleThatDoesNotSpeakSaysNothingOfCredentials(t *testing.T) {
open := aMesh(t)
register(t, open, helloWeb())
said, err := assign(t.Context(), open, "laptop", "hello-web")
if err != nil {
t.Fatal(err)
}
if strings.Contains(said, "credential") {
t.Fatalf("a module without a broker secret was told about credentials:\n%s", said)
}
}
-52
View File
@@ -1,52 +0,0 @@
package main
import (
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
)
// A merge that rebuilds a base rebuilds what stands on it, through every layer, and nothing else
// (novox/hq issue 186): the runtime image moving means every module built on it moves too, and a
// module built on one of those moves as well.
func TestAMergeOfABaseTakesWhatStandsOnItAlong(t *testing.T) {
entry := func(name string) inventory.Entry {
return inventory.Entry{Manifest: catalogue.Manifest{Module: name}}
}
entries := []inventory.Entry{entry("mesh-tools"), entry("shop"), entry("shop-plugin"), entry("postgres"), entry("unrelated")}
against := map[string][]string{
"shop": {catalogue.ArtifactStoreScheme + "mesh-tools/runtime@sha256:a"},
"shop-plugin": {catalogue.ArtifactStoreScheme + "shop/runtime@sha256:b"},
"postgres": {catalogue.ArtifactStoreScheme + "mesh-tools/runtime@sha256:a"},
"unrelated": {catalogue.ArtifactStoreScheme + "alpine/base@sha256:c"},
}
got := dependentsOf([]inventory.Entry{entry("mesh-tools")}, entries, against)
var names []string
for _, e := range got {
names = append(names, e.Manifest.Module)
}
want := map[string]bool{"shop": true, "shop-plugin": true, "postgres": true}
if len(names) != len(want) {
t.Fatalf("rebuilt %v; wanted exactly the three that stand on the runtime, directly or through shop", names)
}
for _, n := range names {
if !want[n] {
t.Fatalf("%s was rebuilt and stands on nothing that moved (%v)", n, names)
}
}
// The dependents come in base order when the merge orders them: the runtime, then shop, then
// the plugin that stands on shop.
ordered := orderByBases(append([]inventory.Entry{entry("mesh-tools")}, got...), against)
pos := map[string]int{}
for i, e := range ordered {
pos[e.Manifest.Module] = i
}
if !(pos["mesh-tools"] < pos["shop"] && pos["shop"] < pos["shop-plugin"]) {
t.Fatalf("not in base order: %v", ordered)
}
// Nothing moved: nothing follows.
if more := dependentsOf(nil, entries, against); len(more) != 0 {
t.Fatalf("with nothing moved, %d module(s) were rebuilt", len(more))
}
}
@@ -0,0 +1,33 @@
package main
// The broker opening belongs only on the node that listens on it (novox/hq: it leaked onto
// every enrolled node's declaration, opening a from-anywhere hole for a port nothing there
// serves). foundationPortsFor is the scope.
import (
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
func TestTheBrokerHostGetsTheFoundationOpening(t *testing.T) {
broker := catalogue.Manifest{Module: "lavinmq", Listens: []catalogue.Listening{
{Port: 5671, Protocol: "tcp", From: "mesh"},
{Port: 5672, Protocol: "tcp", From: "mesh"},
}}
got := foundationPortsFor(5671, []catalogue.Manifest{broker})
if len(got) != 1 || got[0] != 5671 {
t.Fatalf("the node that listens on the broker port keeps it; got %v", got)
}
}
func TestANodeThatOnlyDialsTheBrokerGetsNoOpening(t *testing.T) {
// ace's set: things that reach the broker as a client, none listening on 5671.
ace := []catalogue.Manifest{
{Module: "plex", Listens: []catalogue.Listening{{Port: 32400, Protocol: "tcp", From: "anywhere"}}},
{Module: "postgres", Listens: []catalogue.Listening{{Port: 5432, Protocol: "tcp", From: "mesh"}}},
}
if got := foundationPortsFor(5671, ace); got != nil {
t.Fatalf("a node that only dials out opens nothing for the broker; got %v", got)
}
}
-92
View File
@@ -1,92 +0,0 @@
package main
import (
"context"
"fmt"
"sort"
"github.com/novox/mesh-controller/internal/catalogue"
)
// recordDerivedHolders writes down who holds each mesh-scoped seat that nobody was ever recorded
// as holding.
//
// **A seat held by derivation is a seat held by accident of being alone** (novox/hq
// 04-ISSUES/170). ADR 0131 lets a holder on record settle a seat, and lets any other assignment
// whose module could hold it stand beside the holder, eligible and silent. But a seat nobody
// ever handed over has no record, so its holder is whichever assignment happened to be the sole
// claimant — and the day a second one is assigned, both claim, both are refused, and the first
// one's whole machine stops resolving. That is what assigning a second postgres did to the
// control plane's own store.
//
// So the mesh writes the derived answer down before it acts on an assignment: for every
// mesh-scoped seat with exactly one resolved holder and nothing on record, that holder is
// recorded as the standing one — the same record `seat <name> --to <node>/<module>` makes by
// hand, made from what the mesh already resolved. A seat with two derived claimants is left
// alone: that is the ambiguity a person settles, and recording either would be guessing.
//
// Node-scoped seats are untouched: a record is one holder per seat, and a node-scoped seat has
// one holder per machine (ADR 0121), so there is nothing for a record to settle there.
func recordDerivedHolders(ctx context.Context, open *stores) ([]string, error) {
inv := open.inventory
shelf, err := inv.Catalogue(ctx)
if err != nil {
return nil, err
}
// exclude nobody: every node's claims, resolved with the holdings on record.
world, err := theRestOfTheMesh(ctx, inv, shelf, "")
if err != nil {
return nil, err
}
recorded, err := inv.Holdings(ctx)
if err != nil {
return nil, err
}
// A record is a row against a seat the store knows. A seat it does not — a mesh whose seats
// were never seeded, a seat a module declares for itself — stays held by derivation, as it
// always was; a missing row is not a reason an assignment fails.
known, err := inv.Seats(ctx)
if err != nil {
return nil, err
}
recordable := map[string]bool{}
for _, s := range known {
recordable[s.Name] = true
}
onRecord := map[string]bool{}
for _, h := range recorded {
if s, ok := catalogue.SeatNamed(h.Claim); ok {
onRecord[s.Name] = true
}
}
holders := map[string][]catalogue.Held{}
for _, h := range world.Held {
if h.Scope != catalogue.ScopeMesh {
continue
}
s, ok := catalogue.SeatNamed(h.Claim)
if !ok || onRecord[s.Name] || !recordable[s.Name] {
continue
}
holders[s.Name] = append(holders[s.Name], h)
}
names := make([]string, 0, len(holders))
for name := range holders {
names = append(names, name)
}
sort.Strings(names)
var said []string
for _, name := range names {
if len(holders[name]) != 1 {
continue
}
h := holders[name][0]
if err := inv.HoldSeat(ctx, name, catalogue.ScopeMesh, h.Node, h.Module); err != nil {
return said, err
}
said = append(said, fmt.Sprintf(
"recorded %s on %s as the standing holder of %s, which it held only by being alone",
h.Module, h.Node, name))
}
return said, nil
}
-110
View File
@@ -1,110 +0,0 @@
package main
import (
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// A seat nobody ever handed over is held by whichever assignment happened to be alone — and the
// day a second module able to hold it is assigned, both claimed, both were refused, and the first
// one's machine stopped resolving (novox/hq 04-ISSUES/170). The mesh now writes the derived holder
// down before it acts, so the second assignment stands beside the holder on record.
func aSeatedStore() catalogue.Manifest {
return catalogue.Manifest{Module: "store", Version: "1",
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}},
Serves: map[string]map[string]any{"postgres-database": {"port": 5432}},
Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}}}
}
func TestASecondEligibleHolderStandsBesideTheOneHeldByBeingAlone(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
// Every deploy seeds the mesh's own seats; a record is a row against one of them.
if _, err := open.inventory.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
t.Fatal(err)
}
register(t, open, aSeatedStore())
if _, err := assign(ctx, open, "anchor", "store"); err != nil {
t.Fatal(err)
}
said, err := assign(ctx, open, "laptop", "store")
if err != nil {
t.Fatalf("a second store, eligible for the seat, was refused:\n%s\n%v", said, err)
}
if strings.Contains(said, "cannot be worked out") {
t.Fatalf("assigning a second store unsettled the first one's machine:\n%s", said)
}
if !strings.Contains(said, "recorded store on anchor as the standing holder of mesh-store") {
t.Fatalf("the holder by derivation was not written down:\n%s", said)
}
holdings, err := open.inventory.Holdings(ctx)
if err != nil {
t.Fatal(err)
}
var found bool
for _, h := range holdings {
if h.Claim == "mesh-store" {
found = true
if h.Node != "anchor" || h.Module != "store" {
t.Fatalf("mesh-store is recorded on %s/%s, not on the one that held it", h.Node, h.Module)
}
}
}
if !found {
t.Fatalf("mesh-store has no holder on record after assigning: %v", holdings)
}
// And the record decides from here: the anchor's plan holds the seat, the laptop's does not.
for node, holds := range map[string]bool{"anchor": true, "laptop": false} {
plan, _, err := planFor(ctx, open, node)
if err != nil {
t.Fatalf("%s no longer resolves: %v", node, err)
}
var claimed bool
for _, c := range plan.Claims {
if c.Claim == "mesh-store" {
claimed = true
}
}
if claimed != holds {
t.Fatalf("%s holds mesh-store: %v, want %v", node, claimed, holds)
}
}
}
func TestAHolderOnRecordIsNotRewrittenByDerivation(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
if _, err := open.inventory.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
t.Fatal(err)
}
register(t, open, aSeatedStore())
for _, node := range []string{"anchor", "laptop"} {
if _, err := assign(ctx, open, node, "store"); err != nil {
t.Fatal(err)
}
}
// A person hands the seat to the laptop. From here the record decides, and what the mesh
// derives must never write over it.
if err := open.inventory.HoldSeat(ctx, "mesh-store", catalogue.ScopeMesh, "laptop", "store"); err != nil {
t.Fatal(err)
}
said, err := recordDerivedHolders(ctx, open)
if err != nil {
t.Fatal(err)
}
if len(said) != 0 {
t.Fatalf("a seat on record was written again from derivation: %v", said)
}
holdings, _ := open.inventory.Holdings(ctx)
for _, h := range holdings {
if h.Claim == "mesh-store" && h.Node != "laptop" {
t.Fatalf("the record moved to %s", h.Node)
}
}
}
-111
View File
@@ -1,111 +0,0 @@
package main
import (
"strings"
"testing"
"github.com/novox/mesh-controller/internal/inventory"
)
// A host refuses a declaration carrying a field it does not know, and refuses it whole — so every new
// field is a flag day, and the mesh had no record of which host any machine ran (novox/hq
// 04-ISSUES/087). The order was kept by somebody remembering it.
func TestTheMeshNamesWhichMachinesRunWhichHost(t *testing.T) {
split := hostSplit([]inventory.Node{
{Name: "anchor", HostVersion: "04a27ca"},
{Name: "laptop", HostVersion: "ced54d4"},
{Name: "spare", HostVersion: "04a27ca"},
})
if len(split) != 2 {
t.Fatalf("two versions were reported and the split has %d: %v", len(split), split)
}
if got := strings.Join(split["04a27ca"], ","); got != "anchor,spare" && got != "spare,anchor" {
t.Fatalf("04a27ca is held by %q", got)
}
if got := strings.Join(split["ced54d4"], ","); got != "laptop" {
t.Fatalf("ced54d4 is held by %q", got)
}
}
func TestTheMeshDoesNotClaimWhichHostIsNewer(t *testing.T) {
// **The fault this replaced.** A host reports its version as a commit, and commits have no order.
// The first version compared them as strings and, on the live mesh, named the three machines
// running the NEWER host as the ones behind: `ced54d4` sorts above `04a27ca` and means nothing.
//
// There is no assertion to make about which is newer, and that is the point — the type says so.
// hostSplit returns who runs what, and nothing that could be read as an ordering.
split := hostSplit([]inventory.Node{
{Name: "old-but-sorts-high", HostVersion: "ced54d4"},
{Name: "new-but-sorts-low", HostVersion: "04a27ca"},
})
for version, machines := range split {
if len(machines) != 1 {
t.Fatalf("%s is held by %v", version, machines)
}
}
}
func TestAMachineThatHasNotSaidIsNotAVersion(t *testing.T) {
// It may be running anything. Counting it as a version would invent a disagreement; `node show`
// says per machine that it has not said.
split := hostSplit([]inventory.Node{
{Name: "anchor", HostVersion: "04a27ca"},
{Name: "quiet"},
})
if split != nil {
t.Fatalf("one reported version and one silence read as a disagreement: %v", split)
}
}
func TestMachinesAgreeingOnTheirHostAreNotADisagreement(t *testing.T) {
if split := hostSplit([]inventory.Node{
{Name: "anchor", HostVersion: "v2"},
{Name: "laptop", HostVersion: "v2"},
}); split != nil {
t.Fatalf("machines agreeing reported a split: %v", split)
}
}
func TestAMeshWhereNothingReportedAHostStatesNoDisagreement(t *testing.T) {
if split := hostSplit([]inventory.Node{{Name: "anchor"}, {Name: "laptop"}}); split != nil {
t.Fatalf("a mesh told no host version reported a split: %v", split)
}
}
func TestAReportedHostVersionIsKeptAndReadBack(t *testing.T) {
// The machine has sent this since ADR 0141 and the controller's own copy of the report did not
// have the field, so it was unmarshalled into nothing. End to end through the store, because the
// fault was a field that existed on one side of the wire only.
open := aMesh(t)
record, err := open.inventory.NodeByName(t.Context(), "anchor")
if err != nil {
t.Fatal(err)
}
if record.HostVersion != "" {
t.Fatalf("a machine that never reported one has host version %q", record.HostVersion)
}
if err := open.inventory.RecordHostVersion(t.Context(), record.ID, "ced54d4"); err != nil {
t.Fatal(err)
}
again, err := open.inventory.NodeByName(t.Context(), "anchor")
if err != nil {
t.Fatal(err)
}
if again.HostVersion != "ced54d4" {
t.Fatalf("the reported host version read back as %q", again.HostVersion)
}
// An empty report never clears what a machine last said: a bare word that the node is there says
// nothing about its host.
if err := open.inventory.RecordHostVersion(t.Context(), record.ID, " "); err != nil {
t.Fatal(err)
}
kept, err := open.inventory.NodeByName(t.Context(), "anchor")
if err != nil {
t.Fatal(err)
}
if kept.HostVersion != "ced54d4" {
t.Fatalf("a report carrying no host version cleared what the machine had said: %q",
kept.HostVersion)
}
}
-93
View File
@@ -1,93 +0,0 @@
package main
import (
"context"
"testing"
"github.com/novox/mesh-controller/internal/inventory"
)
// A declaration composed earlier is numbered lower than one composed later, whatever order the two
// are sent in (novox/hq issue 204). The number used to be taken at send time, after composing, so a
// declaration composed before an assignment changed and sent after a newer one carried the higher
// number — and the machine, which refuses a lower number, took the older content as the mesh's
// newest word. Taken before the composition reads anything, the order of numbers is the order of
// compositions, and the host's refusal does what it is for.
func TestADeclarationComposedEarlierIsNumberedLowerWhateverOrderItIsSent(t *testing.T) {
allot := numbered()
var composed []string
compose := func(stamp string) func(string) (sendable, error) {
return func(node string) (sendable, error) {
composed = append(composed, stamp)
return sendable{Resources: []map[string]any{{"id": node + "." + stamp}}}, nil
}
}
// Composed first — before an assignment changed — and sent last.
stale, _ := composeEach([]string{"anchor"}, allot, compose("before"))
// Composed after the change, sent first.
fresh, _ := composeEach([]string{"anchor"}, allot, compose("after"))
if stale[0].declared.Sequence != 1 || fresh[0].declared.Sequence != 2 {
t.Fatalf("the numbers do not follow the compositions: before=%d after=%d",
stale[0].declared.Sequence, fresh[0].declared.Sequence)
}
// Sent in the other order, the numbers do not change — so the machine that has applied the
// fresh one (2) refuses the stale one (1) when it arrives late.
if !(stale[0].declared.Sequence < fresh[0].declared.Sequence) {
t.Fatal("a declaration composed earlier must carry the lower number, however late it is sent")
}
if len(composed) != 2 || composed[0] != "before" {
t.Fatalf("compositions happened in an unexpected order: %v", composed)
}
}
// The number is taken before the first read of the composition, not after it: an allotter that
// fails leaves nothing composed for that machine, and the others are still composed.
func TestTheNumberIsTakenBeforeComposingAndItsFailureIsARefusal(t *testing.T) {
calls := 0
allot := func(node string) (int64, error) {
if node == "anchor" {
return 0, context.DeadlineExceeded
}
return 7, nil
}
sending, refusals := composeEach([]string{"anchor", "laptop"}, allot, func(node string) (sendable, error) {
calls++
if node == "anchor" {
t.Fatal("anchor was composed although its number could not be taken")
}
return sendable{}, nil
})
if calls != 1 || len(sending) != 1 || sending[0].node != "laptop" || sending[0].declared.Sequence != 7 {
t.Fatalf("laptop should be composed with its number and anchor refused: %v / %v", sending, refusals)
}
if len(refusals) != 1 {
t.Fatalf("anchor's failed number should be a refusal naming it: %v", refusals)
}
}
// What was sent is written down even when the sender's context is already cancelled (issue 204): a
// controller replaced mid-send had told the machine and never recorded it, so status read "applied,
// current" over a machine that had just been sent something else.
func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) {
inv := inventory.ForTest(t)
ctx, cancel := context.WithCancel(t.Context())
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
t.Fatal(err)
}
cancel() // the sender is going away: its context is cancelled between the send and the record
body := []byte(`{"declaration":1,"resources":[]}`)
digest, err := recordSent(ctx, inv, "anchor", body)
if err != nil {
// NodeByName on the cancelled context may itself refuse; the record must still be possible
// through the detached context, so look the node up again on a live one.
t.Fatalf("recording a send after cancellation failed: %v", err)
}
outstanding, err := inv.Outstanding(t.Context(), "anchor")
if err != nil {
t.Fatal(err)
}
if outstanding != digest || digest != digestOf(body) {
t.Fatalf("the send was not recorded: outstanding %q, sent %q", outstanding, digest)
}
}
+4 -36
View File
@@ -72,8 +72,6 @@ func run() error {
return askCommand(ctx, args[1:])
case "builds":
return buildsCommand(ctx, args[1:])
case "plans":
return plansCommand(ctx, args[1:])
case "pin":
return pinCommand(ctx, args[1:], true)
case "unpin":
@@ -90,7 +88,7 @@ func run() error {
case "identity":
return identityCommand(ctx, args[1:])
case "broker":
return brokerCommand(ctx, args[1:])
return brokerCommand(args[1:])
case "serve":
return serve(ctx)
case "upgrade":
@@ -163,7 +161,6 @@ func usage() {
overlay place <node> [flags] say where a node is and how it is reached
overlay show the private network, as the mesh computes it
module add <file> register a module from its manifest
module check <file|dir>... judge manifests where they are written, with no mesh (exit 1 on any problem)
module list what modules this mesh knows about
module moved <name> <commit> the source has a newer commit than the mesh built
module forget <name> remove one, unless a node runs it or the mesh holds things for it
@@ -174,14 +171,11 @@ func usage() {
upgrade <name> record ...record that they are behind, and send nothing
status [--json] what is wrong, what is quiet, and what is out of date
seats [--json] every seat this mesh defines, what it delivers, and who holds it
seat rename <from> <to> rename a seat; its former name still resolves (ADR 0122)
seat <name> --to <node>/<module> hand a seat to that assignment as one act; never empty in between (ADR 0131)
board [--listen ADDR] the same three questions, as a page that holds nothing
api --issuer URL [--listen A] assign and unassign over http, for a surface that is not here
assign <node> <module> put a module on a node
unassign <node> <module> take it off
take <node> <module> preview a module's cutover on an adopted node: what runs beside
what it declares; --yes <digest> cuts it over as previewed
take <node> <module> cut a module over on an adopted node, once its data has moved
converge <node> [--yes <digest>] [--filter nftables] preview, then make, an adopted node converged
adopt <node> return a converged node to adopted; what was taken stays taken
settings set <module> <file> what a module's config should say, for the whole mesh
@@ -206,8 +200,7 @@ func usage() {
licence refresh <name> mint a new access token and seal it to every holder
rotate <provision> [--consumer <n>] a new credential for every holder, both ends at once
ask <module> <tool> [json] call one of a module's tools over the broker, and print its answer
pin <node> <provision> <from-node> <module>
which provider this one gets a provision from: the module, and its node
pin <node> <provision> <from> which node this one gets a provision from
unpin <node> <provision> put that question back
plan <node> [--files|--json] what that node would run, and why
push [<node>] [--behind] send a node everything it should be, or only those that need it
@@ -247,31 +240,6 @@ func parseAround(set *flag.FlagSet, args []string) ([]string, error) {
}
}
// Built is the daemon hearing a build's outcome on the bus — its own asking, an announcement's, or
// a tool's that did not wait (novox/hq issue 176) — and taking it in: recorded, and the module
// registered, the same as the waiting command does. Said either way, so the daemon's log tells what
// became of a build nobody was watching.
func (b builds) Built(ctx context.Context, result link.BuildResult) error {
manifest, _, err := takeIn(ctx, b.inv, result)
switch {
case err != nil && result.Failed != "":
fmt.Printf("%s: %v\n", result.ID, err)
if result.Module != "" {
planBuilt(ctx, b.open, result.Module, result.Commit, result.Failed)
} else {
planFailedBuild(ctx, b.open, result)
}
return nil
case err != nil:
fmt.Printf("%s: heard and recorded, and not registered: %v\n", result.ID, err)
if manifest.Module != "" {
planBuilt(ctx, b.open, manifest.Module, result.Commit, err.Error())
}
return nil
}
fmt.Printf("%s: %s %s registered, built on %s from %s\n",
result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit))
saysWhenThePolicyActs(ctx, b.inv, manifest.Module)
planBuilt(ctx, b.open, manifest.Module, result.Commit, "")
return nil
return b.inv.RecordBuild(ctx, buildFrom(result))
}
+24 -166
View File
@@ -54,24 +54,7 @@ var provided = providedModules()
func moduleCommand(ctx context.Context, args []string) error {
if len(args) == 0 {
return errors.New("module add <file>, module check <file>..., module list, or module forget <name>")
}
// `check` needs no mesh, and must not: it is what somebody runs in their own repository before
// there is a mesh in reach (novox/hq issue 148). A directory expands to every manifest under it.
if args[0] == "check" {
var paths []string
for _, a := range args[1:] {
if info, err := os.Stat(a); err == nil && info.IsDir() {
under, err := manifestsUnder(a)
if err != nil {
return err
}
paths = append(paths, under...)
continue
}
paths = append(paths, a)
}
return moduleCheck(paths, os.Stdout)
return errors.New("module add <file>, module list, or module forget <name>")
}
open, err := openStores(ctx)
if err != nil {
@@ -113,9 +96,6 @@ func moduleCommand(ctx context.Context, args []string) error {
if err != nil {
return err
}
if err := namesNoInstallation(m); err != nil {
return err
}
if err := inv.RegisterModule(ctx, m, from); err != nil {
return err
}
@@ -147,40 +127,6 @@ func moduleCommand(ctx context.Context, args []string) error {
if err != nil {
return err
}
// **The same list, for something other than a person** (novox/hq ADR 0195): what each module
// is, where it runs, whether it is current, and what it says of itself.
if len(args) > 1 && args[1] == "--json" {
type listed struct {
Module string `json:"module"`
Version string `json:"version"`
Built string `json:"built,omitempty"`
Head string `json:"head,omitempty"`
Current bool `json:"current"`
Provided bool `json:"provided,omitempty"`
// Tools says whether the module answers tools anywhere it runs: a list of its own,
// a bundle the runtime serves, or a seat's verbs it claims (novox/hq ADR 0197) —
// what the console checks the bus's answers against.
Tools bool `json:"tools"`
On []string `json:"on"`
Provides []string `json:"provides,omitempty"`
Requires []string `json:"requires,omitempty"`
Claims []string `json:"claims,omitempty"`
Capabilities []string `json:"capabilities,omitempty"`
}
out := make([]listed, 0, len(entries))
for _, e := range entries {
m := e.Manifest
l := listed{Module: m.Module, Version: m.Version, Built: e.Source.BuiltFrom, Head: e.Source.Head,
Current: e.Provided || e.Source.Repository == "" || e.Source.Current(), Provided: e.Provided,
On: append([]string{}, e.On...), Provides: m.Offers(), Requires: m.Requires,
Capabilities: m.Capabilities, Tools: declaresTools(m)}
for _, c := range m.Claims {
l.Claims = append(l.Claims, c.At()+"/"+c.Name)
}
out = append(out, l)
}
return printJSON(out)
}
if len(entries) == 0 {
fmt.Println("this mesh knows about no modules yet")
return nil
@@ -329,7 +275,7 @@ func moduleCommand(ctx context.Context, args []string) error {
return issueOnTheNewBus(ctx, inv, m, *forNode, busAddress)
default:
return fmt.Errorf("module has no %q; it has add, check, list, moved, forget and issue", args[0])
return fmt.Errorf("module has no %q; it has add, list, moved, forget and issue", args[0])
}
}
@@ -386,14 +332,10 @@ func settingsCommand(ctx context.Context, args []string) error {
switch args[0] {
case "set":
if len(positionals) != 2 {
return errors.New("settings set <module> <settings.json | {…}> [--node <node>]")
return errors.New("settings set <module> <settings.json> [--node <node>]")
}
// A file, or the values themselves when they begin with `{` — which is how the mesh's own
// `settings` tool passes them, having no file to hand over (novox/hq issue 198).
var raw []byte
if strings.HasPrefix(strings.TrimSpace(positionals[1]), "{") {
raw = []byte(positionals[1])
} else if raw, err = os.ReadFile(positionals[1]); err != nil {
raw, err := os.ReadFile(positionals[1])
if err != nil {
return err
}
var values map[string]any
@@ -449,8 +391,8 @@ func describeOffers(offers []catalogue.Offer) string {
// database should not change where an existing machine gets its data the day a second one
// arrives.
func pinCommand(ctx context.Context, args []string, setting bool) error {
if setting && len(args) != 4 {
return errors.New("pin <node> <provision> <from-node> <module>")
if setting && len(args) != 3 {
return errors.New("pin <node> <provision> <from-node>")
}
if !setting && len(args) != 2 {
return errors.New("unpin <node> <provision>")
@@ -469,12 +411,17 @@ func pinCommand(ctx context.Context, args []string, setting bool) error {
fmt.Printf("%s is no longer told where to get %s from\n", args[0], args[1])
return nil
}
// The provider's node may be this same machine: two modules beside the consumer can both
// answer a provision, and then the module is the whole question (novox/hq #258).
if err := inv.PinProvision(ctx, args[0], args[1], args[2], args[3]); err != nil {
if args[0] == args[2] {
// Allowed by nothing here, and worth saying rather than resolving into a confusing
// refusal later: a node providing something to itself is a node-scoped provision, and
// this field is for the other kind.
return fmt.Errorf("%s cannot get %s from itself; that would be a provision this machine "+
"provides, which does not need saying", args[0], args[1])
}
if err := inv.PinProvision(ctx, args[0], args[1], args[2]); err != nil {
return err
}
fmt.Printf("%s gets %s from %s/%s\n", args[0], args[1], args[2], args[3])
fmt.Printf("%s gets %s from %s\n", args[0], args[1], args[2])
fmt.Printf(" run `push %s` to send it\n", args[0])
return nil
}
@@ -591,7 +538,7 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: m.Module}.Username()
password, err := inv.MintBusPassword(ctx, inventory.BusUser{
Username: user, Kind: busKindOf(m.Module), Node: node, Module: m.Module,
Username: user, Kind: inventory.BusModule, Node: node, Module: m.Module,
})
if err != nil {
return err
@@ -611,41 +558,22 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue
return issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password)
}
// busKindOf is what a module's bus user is recorded as: the node's tool runtime where the module is
// the runtime (novox/hq ADR 0175), a module otherwise. The username is the same either way — the
// runtime is issued through this same path — and the kind is what a reader of the records sees.
func busKindOf(module string) string {
if module == catalogue.RuntimeModule {
return inventory.BusNodeTools
}
return inventory.BusModule
}
// issueWith is the delivery half: the minted password sealed to the machine as the module's broker
// secret, and the module's consumer created where the bus can be reached. Split from the minting
// so the move can issue every module against a bus whose address it worked out itself
// (`rollout mint`, design 28 task 5.2) rather than the one in this process's environment.
func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest,
node, busAddress string, known broker.Broker, reachable, user, password string) error {
// The seats this module claims, with the verbs each promises (novox/hq ADR 0159): the runtime
// serves a claimed seat's verbs with its tools of the same name, and the bus admits only the
// holder's subscription — so the runtime tries each claim and the grant decides. Written here
// because this file is the one thing the mesh writes that the runtime reads before it speaks.
claims, err := claimsFor(ctx, inv, m)
if err != nil {
return err
}
held, err := json.Marshal(struct {
URL string `json:"url"`
Fingerprint string `json:"fingerprint,omitempty"`
Node string `json:"node"`
Module string `json:"module"`
User string `json:"user"`
Password string `json:"password"`
Claims []seatClaimed `json:"claims,omitempty"`
URL string `json:"url"`
Fingerprint string `json:"fingerprint,omitempty"`
Node string `json:"node"`
Module string `json:"module"`
User string `json:"user"`
Password string `json:"password"`
}{
URL: "nats://" + reachable, Fingerprint: known.Fingerprint,
Node: node, Module: m.Module, User: user, Password: password, Claims: claims,
Node: node, Module: m.Module, User: user, Password: password,
})
if err != nil {
return err
@@ -717,73 +645,3 @@ func whereItComesFrom(repository, ref, commit, path string, self bool) (inventor
}
return from, nil
}
// namesNoInstallation is the mesh refusing a definition that names an installation, at the moment
// it would enter the catalogue (novox/hq ADR 0112, ADR 0155). `module check` says the same thing
// earlier, where the author is; this is the last moment the mesh can still say no, and a
// definition that got past the check — written elsewhere, or checked by nobody — is refused here
// in the same words. A name meant on purpose is declared with its reason and passes.
func namesNoInstallation(m catalogue.Manifest) error {
named := catalogue.InstallationProblems(m)
if len(named) == 0 {
return nil
}
return fmt.Errorf("%s names an installation, and a definition names none — declare a name meant "+
"on purpose under %s with its reason, or take it out:\n - %s",
m.Module, catalogue.NamesOnPurpose, strings.Join(named, "\n - "))
}
// seatClaimed is one seat a module claims, as its runtime needs it: the name, the scope (a
// node-scoped seat's verb carries the machine, design 33 §4) and the verbs the seat promises.
type seatClaimed struct {
Seat string `json:"seat"`
Scope string `json:"scope"`
Serves []string `json:"serves,omitempty"`
}
// claimsFor joins a module's claims with the seats' protocols from the mesh's records.
func claimsFor(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest) ([]seatClaimed, error) {
if len(m.Claims) == 0 {
return nil, nil
}
seats, err := inv.Seats(ctx)
if err != nil {
return nil, err
}
byName := map[string]catalogue.Seat{}
for _, s := range seats {
byName[s.Name] = s
}
var out []seatClaimed
for _, c := range m.Claims {
claimed := seatClaimed{Seat: c.Name, Scope: c.At()}
if s, known := byName[c.Name]; known {
claimed.Scope = s.Scope
// The verbs the runtime serves for the seat: the claim's own when it names them
// (ADR 0160), else every verb the seat promises, which its tools then answer.
claimed.Serves = c.ServesFor(catalogue.Manifest{Tools: catalogue.VerbNames(s.Serves)})
}
out = append(out, claimed)
}
return out, nil
}
// declaresTools is whether a module answers tools wherever it runs (novox/hq ADR 0197): it names
// tools of its own, its build delivers a bundle the node's runtime serves, or it claims a seat
// whose verbs it serves. A module with none is never expected to announce anything.
func declaresTools(m catalogue.Manifest) bool {
if len(m.Tools) > 0 {
return true
}
for _, b := range m.Bundles {
if len(b.Loads) > 0 {
return true
}
}
for _, c := range m.Claims {
if len(c.Serves) > 0 {
return true
}
}
return false
}
+2 -2
View File
@@ -11,7 +11,7 @@ import (
// A module that declares none is refused before the account exists, so the bus never carries an
// account nothing reads (novox/hq 04-ISSUES/078).
func TestAModuleWithNoBrokerSecretCannotBeIssued(t *testing.T) {
err := mayIssue(catalogue.Manifest{Module: "step-ca", OwnSecrets: catalogue.OwnSecrets{"password": {Path: "/run/password"}}})
err := mayIssue(catalogue.Manifest{Module: "step-ca", OwnSecrets: map[string]string{"password": "/run/password"}})
if err == nil {
t.Fatal("a module with no broker own secret was issued an account")
}
@@ -20,7 +20,7 @@ func TestAModuleWithNoBrokerSecretCannotBeIssued(t *testing.T) {
t.Errorf("the refusal does not say %q: %v", want, err)
}
}
if err := mayIssue(catalogue.Manifest{Module: "redis", OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/run/broker"}}}); err != nil {
if err := mayIssue(catalogue.Manifest{Module: "redis", OwnSecrets: map[string]string{"broker": "/run/broker"}}); err != nil {
t.Errorf("a module declaring its broker secret was refused: %v", err)
}
}
+1 -11
View File
@@ -346,16 +346,9 @@ func whoResolves(ctx context.Context, open *stores, requirement string) (
refused := map[string]string{}
for _, n := range nodes {
plan, _, err := planFor(ctx, open, n.Name)
switch {
case unresolvable(err):
if err != nil {
refused[n.Name] = err.Error()
continue
case err != nil:
// Not a node that does not resolve — a question that went unanswered. Recording it as a
// refusal would take the machine off the private network, and the generator that reads
// this would then write a roster and a filter without it (novox/hq 04-ISSUES/152).
return nil, nil, fmt.Errorf("whether %s answers %q cannot be read: %w",
n.Name, requirement, err)
}
for _, m := range plan.Modules {
for _, offered := range m.Offers() {
@@ -567,9 +560,6 @@ func onTheNetwork(ctx context.Context, inv *inventory.Inventory,
catalogue.Node{Name: p.Name, Site: p.Site, Capabilities: caps},
catalogue.World{Unchecked: true, Holdings: holdings})
if err != nil {
// Said, not skipped in silence: a machine dropped here loses its address, and every
// plan that names it fails in another module's words (novox/hq issue 188).
fmt.Fprintf(os.Stderr, "%s is not counted as on the network: it does not resolve: %v\n", p.Name, err)
continue
}
for _, m := range got.Modules {
-26
View File
@@ -3,7 +3,6 @@ package main
import (
"context"
"os"
"reflect"
"strings"
"testing"
@@ -304,28 +303,3 @@ func TestTheResolverIsToldEveryMachineOnTheNetworkAndToldAgainWhenOneLeaves(t *t
t.Fatalf("a machine that left the network is still a wildcard, or the one that stayed is not:\n%s", after)
}
}
// The roster is the machines and nothing else (novox/hq ADR 0191): each node's internal domain covers
// every route on it, and a node's public domains are public DNS's. A routed name in `.Names` was a
// private answer for a public name, handed by a resolver serving a LAN to a phone that could not use it.
func TestTheRosterNamesOnlyTheMachines(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
gens, err := generators(ctx, open)
if err != nil {
t.Fatal(err)
}
for _, node := range []string{"anchor", "laptop"} {
plan, settings, err := planFor(ctx, open, node)
if err != nil {
t.Fatal(err)
}
with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Reading)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(with.Names, with.Machines) {
t.Fatalf("%s's roster names more than the machines:\n names %v\n machines %v", node, with.Names, with.Machines)
}
}
}
+2 -48
View File
@@ -2,7 +2,6 @@ package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
@@ -45,21 +44,6 @@ func nodeCommand(ctx context.Context, args []string) error {
if err != nil {
return err
}
// **The same list, for something other than a person** — the console's discovery reads it
// (novox/hq ADR 0195), and a reader that parses a printed column breaks when it is reworded.
if len(args) > 1 && args[1] == "--json" {
type listed struct {
Name string `json:"name"`
Heard string `json:"heard"`
Mode string `json:"mode"`
ID string `json:"id"`
}
out := make([]listed, 0, len(nodes))
for _, n := range nodes {
out = append(out, listed{Name: n.Name, Heard: heardFrom(n), Mode: modeOf(n), ID: n.ID})
}
return printJSON(out)
}
if len(nodes) == 0 {
// Said rather than printed as nothing: an empty list and a failed read must never
// look the same, and this command answering "none" is only honest because getting
@@ -379,15 +363,9 @@ func identityCommand(ctx context.Context, args []string) error {
return nil
}
func brokerCommand(ctx context.Context, args []string) error {
if len(args) > 0 && args[0] == "certificate" {
return busCertificate(args[1:])
}
if len(args) > 0 && args[0] == "accounts" {
return busAccounts(ctx, args[1:])
}
func brokerCommand(args []string) error {
if len(args) == 0 || args[0] != "show" {
return errors.New("broker show | broker certificate [--check] --into <directory> | broker accounts --into <file>")
return errors.New("broker show")
}
known, err := broker.FromEnvironment()
if errors.Is(err, broker.ErrNotConfigured) {
@@ -452,12 +430,6 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
}
fmt.Printf("%s\n", node.Name)
fmt.Printf(" last heard from %s\n", heardFrom(node))
// Which host runs it, as it reported (novox/hq 04-ISSUES/087). Said whenever known, because a
// host refuses a declaration carrying a field it does not understand and refuses it WHOLE — so
// which host a machine runs is what decides whether the mesh can send it anything new, and
// nothing could say it. "not reported" rather than blank: a machine that has not said is a
// different thing from one running nothing.
fmt.Printf(" host %s\n", orNotReported(node.HostVersion))
if err := showMode(ctx, inv, node); err != nil {
return err
}
@@ -508,21 +480,3 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
}
return nil
}
// orNotReported is a fact a machine states about itself, or the fact that it has not.
func orNotReported(s string) string {
if strings.TrimSpace(s) == "" {
return "not reported — this machine has not said since the mesh began keeping it"
}
return s
}
// printJSON prints a value as indented JSON, the shape every `--json` answers in.
func printJSON(v any) error {
body, err := json.MarshalIndent(v, "", " ")
if err != nil {
return err
}
fmt.Println(string(body))
return nil
}
+1 -2
View File
@@ -1,7 +1,6 @@
package main
import (
"reflect"
"strings"
"testing"
"time"
@@ -192,7 +191,7 @@ func TestWhatAHandedOverModuleRecordsAboutItsSource(t *testing.T) {
t.Fatalf("the source records as %+v", from)
}
// A manifest with no provenance at all is legitimate: fixing something in a hurry.
if from, err := whereItComesFrom("", "", "", "", false); err != nil || !reflect.DeepEqual(from, inventory.Source{}) {
if from, err := whereItComesFrom("", "", "", "", false); err != nil || from != (inventory.Source{}) {
t.Fatalf("a manifest handed over with no provenance was refused: %+v, %v", from, err)
}
for _, c := range []struct {
+136 -161
View File
@@ -7,7 +7,6 @@ import (
"errors"
"flag"
"fmt"
"os"
"sort"
"strings"
@@ -16,6 +15,8 @@ import (
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/licences"
"github.com/novox/mesh-controller/internal/overlay"
"net"
"strconv"
)
// working out what one machine should be.
@@ -24,36 +25,7 @@ import (
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
// nothing in it was wrong, and no one edit was the one that should have been a new file.
// notResolvable marks the one failure in planFor that is a statement about the node: its assigned
// modules do not compose. Every other failure means the mesh could not be *asked* — the store was
// unreachable, a key could not be read — and says nothing about the node at all.
//
// The distinction exists because three callers gather something across every machine and must carry
// on when one machine's set is broken. Each of them read a plain error as "their set does not
// resolve", and so read a store that was briefly unreachable as a machine that runs nothing. On the
// roster of routed names that is not a degraded answer but a false one: it states, to every machine
// at once, that another machine's names do not exist. A control node spent hours replacing every
// container it ran, on a six-minute cycle, because each pass restarted the store this is read from,
// the read failed, one name left the roster, and the roster is part of every container's identity
// (novox/hq 04-ISSUES/152, and 04-ISSUES/151 for why a changed roster is a changed container).
//
// So: skip a node that cannot resolve, and never a node that could not be read.
type notResolvable struct{ err error }
func (n notResolvable) Error() string { return n.err.Error() }
func (n notResolvable) Unwrap() error { return n.err }
// unresolvable reports whether err is a node's own set failing to compose, rather than the mesh
// being unable to answer.
func unresolvable(err error) bool {
var n notResolvable
return errors.As(err, &n)
}
// planFor works out everything a node should run, from what was assigned to it.
//
// A failure to compose the node's own modules is wrapped as notResolvable; every other failure is
// returned as it is. Callers gathering across the mesh must tell them apart — see notResolvable.
func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Resolution, catalogue.SettingsBy, error) {
inv := open.inventory
shelf, err := inv.Catalogue(ctx)
@@ -123,9 +95,7 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
At: onNetwork[nodeName], PublicDomain: publicDomain,
Account: who.Account, AccountHome: who.AccountHome}, world)
if err != nil {
// The node's own set does not compose. Marked, because this is the only failure here that
// a mesh-wide gatherer may pass over — see notResolvable.
return catalogue.Resolution{}, nil, notResolvable{err}
return catalogue.Resolution{}, nil, err
}
// The credential for each thing this node takes from elsewhere. Made once and kept, so the
@@ -162,14 +132,7 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
}
continue
}
var secret inventory.Secret
var err error
if n.SharedOwn != "" {
// The provider's one credential, sealed to this consumer too (novox/hq ADR 0158).
secret, err = inv.SharedSecretFor(ctx, n.Name, nodeName, n.For, n.From, providerModuleOf(resolved, open, ctx, n), n.Local, n.SharedOwn)
} else {
secret, err = inv.SecretFor(ctx, n.Name, nodeName, n.For, n.From, n.Local)
}
secret, err := inv.SecretFor(ctx, n.Name, nodeName, n.For, n.From, n.Local)
if err != nil {
// Said rather than skipped. A machine that resolves cleanly and receives no
// credential is one that will fail to authenticate at some later, less obvious
@@ -184,12 +147,8 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
// Settings for everything that resolved, including modules nobody assigned directly: a
// requirement pulled in by something else is still configurable, and finding out that it is
// not only when you try would be an arbitrary line nobody could predict.
//
// A setting that reaches nothing, or cannot compose with the definition it was stored for,
// no longer refuses the machine here: it is judged where it is stored, and a definition that
// moved under it costs that module its place in the declaration, said by name (novox/hq ADR
// 0163, rule 6 — see Compose).
settings := catalogue.SettingsBy{}
var stray []string
for _, m := range resolved.Modules {
layers, err := inv.SettingsFor(ctx, nodeName, m.Module)
if err != nil {
@@ -199,6 +158,13 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
continue
}
settings[m.Module] = layers
stray = append(stray, catalogue.UnusedSettings(m, layers)...)
}
if len(stray) > 0 {
// Somebody set something that reaches no file. Said here rather than discovered by the
// machine not behaving differently, which is the slowest way there is.
return catalogue.Resolution{}, nil, fmt.Errorf(
"these settings reach nothing:\n - %s", strings.Join(stray, "\n - "))
}
return resolved, settings, nil
}
@@ -272,9 +238,8 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
for _, o := range others {
got, err := catalogue.Resolve(shelf, o.assigned, o.node, catalogue.World{Unchecked: true, Holdings: holdings})
if err != nil {
// Said, not skipped: a machine dropped here offers nothing and holds nothing as far
// as every other machine's plan can tell (novox/hq issue 188).
fmt.Fprintf(os.Stderr, "%s is left out of the rest of the mesh: it does not resolve: %v\n", o.node.Name, err)
// Their set does not resolve for some other reason. Not this node's problem to
// report, and nothing of theirs is running, so it offers nothing.
continue
}
firstHeld = append(firstHeld, got.Claims...)
@@ -305,22 +270,8 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
world := catalogue.World{Offered: offered, Held: firstHeld, Holdings: holdings}
var held []catalogue.Held
for _, o := range others {
// Each machine is resolved with its own pins, as its plan is: a machine that needs one to
// settle two providers would otherwise be refused here and vanish from the mesh — every
// seat it holds unheld, every build that needs one refused (2026-10-01, the control node;
// novox/hq issue 188).
theirs := world
if pins, err := inv.PinsFor(ctx, o.node.Name); err == nil {
theirs.Pinned = pins
}
got, err := catalogue.Resolve(shelf, o.assigned, o.node, theirs)
got, err := catalogue.Resolve(shelf, o.assigned, o.node, world)
if err != nil {
// Said only for the whole-mesh view. With one machine excluded, the others are
// resolved without its offers, and one that consumes them cannot resolve here by
// design — that is not the machine being dropped, it is the view being partial.
if exclude == "" {
fmt.Fprintf(os.Stderr, "%s is left out of the rest of the mesh: it does not resolve: %v\n", o.node.Name, err)
}
continue
}
held = append(held, got.Claims...)
@@ -394,33 +345,7 @@ func declarationWith(ctx context.Context, open *stores, node string,
if err != nil {
return sendable{}, err
}
return sendable{Resources: composed.Resources, Adoption: adoption,
Received: composed.Received, Mesh: with.Mesh,
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut}, nil
}
// sortedKeysOf is a map's keys, sorted — so what a declaration says it left out does not move
// for a reordering nobody made.
func sortedKeysOf(m map[string]string) []string {
if len(m) == 0 {
return nil
}
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
// reportLeftOut says which of a machine's modules its declaration leaves out and why (novox/hq ADR
// 0163, rule 6), one line each: the machine is told everything else, and is told it was left out.
func reportLeftOut(node string, declared sendable) {
for _, m := range declared.LeftOut {
fmt.Printf("%s: %s left out — a setting stored for it cannot compose with its definition; "+
"what the machine holds for it is kept and its containers are untouched. %s\n",
node, m, declared.leftOutWhy[m])
}
return sendable{Resources: composed.Resources, Adoption: adoption}, nil
}
// renderingFor is everything a node's declaration is composed with, and the node's record.
@@ -460,10 +385,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
for _, m := range plan.Modules {
g, err := catalogue.GivenPorts(m, settings[m.Module])
if err != nil {
// A given port its definition no longer publishes: the module is left out of the
// declaration, by name, when it is composed (novox/hq ADR 0163, rule 6) — never the
// machine refused here for it.
continue
return catalogue.Rendering{}, inventory.Node{}, err
}
if g != nil {
given[m.Module] = g
@@ -533,23 +455,6 @@ func renderingFor(ctx context.Context, open *stores, node string,
var sealed string
var err error
if choosing == Allocating {
// **The broker credential is never invented here** (novox/hq issue 203). Every other
// own secret is the mesh's to make — a password nobody else knows — but this one
// is an account on the bus, minted by `module issue` and sealed by it; a push that
// made a random one would deliver a file the process cannot read and report the
// machine applied. Refused by name, with the verb.
if name == "broker" {
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: m.Module}.Username()
if _, minted, err := inv.BusUserHash(ctx, user); err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
} else if !minted {
return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf(
"%s on %s has no bus credential: nothing was issued for %s, and a push "+
"would seal a placeholder its process cannot read (novox/hq issue 203). "+
"`module issue %s --node %s`, then push again",
m.Module, node, user, m.Module, node)
}
}
sealed, err = inv.SecretForModule(ctx, node, m.Module, name)
} else {
var held bool
@@ -645,24 +550,43 @@ func renderingFor(ctx context.Context, open *stores, node string,
}
}
// **The roster is the machines and nothing else** (novox/hq ADR 0191). Each node has one internal
// domain, `<node>.internal`, and every route on it is a name under that domain (ADR 0151), which
// the resolver answers with one wildcard per machine — so no route needs a line of its own. A
// node's public domains are the operator's and public DNS answers them; the mesh gives no private
// answer for any of them. The roster once carried every routed name, public ones included, and a
// resolver that also serves a LAN handed a phone a tunnel address for the mail server.
// `.Names` and `.Machines` stay two fields so a module's template keeps rendering (issue 111).
// And every routed name → the node that serves it (novox/hq ADR 0066). Alongside the
// `<node>.internal` names above, so a container — or an internal ACME validator — resolves a
// routed name to the proxy that serves it, mesh-wide. The mesh publishes the names it was told
// to serve and knows nothing about what they mean.
// Kept apart from the machines, because a fact about the machines must not be handed the names
// the mesh merely serves (novox/hq 04-ISSUES/111).
machines := make(map[string]string, len(names))
for name, at := range names {
machines[name] = at
}
routes, err := routeNamesInTheMesh(ctx, open)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
for name, at := range routes {
names[name] = at
}
// **The bus is never public** (novox/hq ADR 0169). It was a foundation port — widened from the
// broker's own `from: mesh` to from-anywhere on the broker's host, so a machine could enrol
// before it had an address on the private network. A machine joins through the tunnel now, and
// every link to the bus crosses it, so its reach is what the `nats` module declares: the mesh.
// Nothing the mesh itself needs is opened beyond what a module declares.
// The ports the mesh itself needs open, which no module declares. Read from the broker this
// control plane was told about rather than written down twice: the address a node is handed in
// its token and the port its machine must accept on are the same fact.
//
// **Only on the node that listens on it** (novox/hq issue: the broker opening leaked onto
// every node). The opening exists to WIDEN the broker's port to from-anywhere — a machine
// enrolling is not on the mesh yet, so the broker's own `from: mesh` listen would refuse its
// first dial. That widening belongs on the broker's host and nowhere else: a node that only
// dials out needs no incoming rule, and an opening for a port nothing here listens on is a
// from-anywhere hole for a dead port. So the foundation port is kept only when a module
// resolved onto THIS node actually listens on it.
var foundation []int
if b, err := broker.FromEnvironment(); err == nil {
if _, port, err := net.SplitHostPort(b.Address); err == nil {
if n, err := strconv.Atoi(port); err == nil {
foundation = foundationPortsFor(n, plan.Modules)
}
}
}
// And, for a module that keeps them, every operator-sealed secret in the mesh — the vault's
// copy, outside the store (novox/hq ADR 0085, amended). Read only; nothing here mints. The
@@ -734,6 +658,80 @@ func renderingFor(ctx context.Context, open *stores, node string,
}, record, nil
}
// routeNamesInTheMesh is every routed name and the address of the node that serves it (novox/hq
// ADR 0066).
//
// **Mesh-wide, so any container resolves any routed name to its proxy** — including an internal
// ACME validator, which cannot complete a challenge for a name it cannot reach. A routed name is
// composed on the consumer's node (from its label and that node's public domain) and served by the
// node answering the consumer's route requirement; this gathers both.
//
// It reads route names off resolutions rather than a table because there is no table: a route is a
// contribution, computed from what each node runs. Name-agnostic — a contribution counts as a
// routed name only because it carried a label the mesh composed, never because the mesh knows what
// "route" means. A node that does not resolve is skipped, so one machine's broken set does not cost
// the rest their names.
func routeNamesInTheMesh(ctx context.Context, open *stores) (map[string]string, error) {
inv := open.inventory
places, err := inv.Overlays(ctx)
if err != nil {
return nil, err
}
address := map[string]string{}
for _, p := range places {
if strings.TrimSpace(p.Address) != "" {
address[p.Name] = p.Address
}
}
nodes, err := inv.Nodes(ctx)
if err != nil {
return nil, err
}
out := map[string]string{}
for _, n := range nodes {
plan, settings, err := planFor(ctx, open, n.Name)
if err != nil {
continue
}
for _, m := range plan.Modules {
for to := range m.Contributes {
values, asks, err := plan.ContributionsFrom(to, m.Module, settings)
if err != nil {
return nil, err
}
if !asks {
continue
}
// A routed name, and only that: a contribution the mesh composed a name for from a
// label it was given. A grant that happens to carry a `name` of its own — a database
// name — carries no label and is left alone.
if _, labelled := values["label"]; !labelled {
continue
}
name, _ := values["name"].(string)
if name == "" {
continue
}
// The node that serves it: whoever answers this consumer's route requirement, or
// this same node when the proxy is beside the consumer.
serving := n.Name
for _, need := range plan.Needs {
if need.Name == to && need.For == m.Module {
serving = need.From
break
}
}
if at := address[serving]; at != "" {
out[strings.ToLower(name)] = at
}
}
}
}
return out, nil
}
// certificateFor is what the mesh certifies about one machine's internal name.
//
// It reaches across two contexts and reads neither one's store from the other: `inventory` knows
@@ -825,17 +823,11 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
out := make([]catalogue.Grant, 0, len(issued))
for _, s := range issued {
plan, settings, err := planFor(ctx, open, s.Consumer)
switch {
case unresolvable(err):
if err != nil {
// Their set does not resolve. Skipped rather than fatal: this node is not the place
// to report another machine's problem, and a grant for something that is not going to
// run would have the provider create a user nothing uses.
continue
case err != nil:
// The mesh could not be asked what they wanted, which is not the same as their wanting
// nothing — and withholding a grant on that reading takes a consumer's access away
// (novox/hq 04-ISSUES/152).
return nil, fmt.Errorf("what %s asked of %s cannot be read: %w", s.Consumer, s.Name, err)
}
values, asks, err := plan.ContributionsFrom(s.Name, s.ConsumerModule, settings)
if err != nil {
@@ -946,20 +938,6 @@ func planCommand(ctx context.Context, args []string) error {
return nil
}
// Which modules a push would leave out, and why — said before the plan, since the plan is of
// what the machine would be told (novox/hq ADR 0163, rule 6). Judged, never composed: `plan`
// without --json allocates nothing.
if record, err := open.inventory.NodeByName(ctx, args[0]); err == nil {
left := plan.LeftOut(settings, record.Adopted)
reportLeftOut(args[0], sendable{LeftOut: sortedKeysOf(left), leftOutWhy: left})
}
// And a setting that reaches nothing — refused where it is stored, and said here for one
// stored before its definition moved from under it.
for _, m := range plan.Modules {
for _, stray := range catalogue.UnusedSettings(m, settings[m.Module]) {
fmt.Printf("%s: a setting reaches nothing — %s\n", args[0], stray)
}
}
fmt.Printf("%s would run:\n", args[0])
for _, m := range plan.Modules {
fmt.Printf(" %-20s %s\n", m.Module, plan.Because[m.Module])
@@ -1306,22 +1284,19 @@ func composeBusUsers(ctx context.Context, inv *inventory.Inventory,
return broker.ComposeAccounts(filled)
}
// providerModuleOf is which module answers a need on the providing node: the one in this node's
// own set when the provider is here, else the one the catalogue says offers it.
func providerModuleOf(resolved catalogue.Resolution, open *stores, ctx context.Context, n catalogue.Needed) string {
for _, m := range resolved.Modules {
if _, shared := m.SharedCredentialOf(n.Name); shared {
return m.Module
// foundationPortsFor is the broker port, kept only when a module resolved onto this node listens
// on it (novox/hq issue: the broker opening leaked onto every node). The foundation opening
// exists to WIDEN the broker's `from: mesh` port to from-anywhere, because a machine enrolling is
// not on the mesh yet and its first dial would be refused. That widening belongs on the broker's
// host alone: a node that only dials out needs no incoming rule, and an opening for a port
// nothing here listens on is a from-anywhere hole for a dead port.
func foundationPortsFor(brokerPort int, modules []catalogue.Manifest) []int {
for _, m := range modules {
for _, l := range m.Listens {
if l.Port == brokerPort {
return []int{brokerPort}
}
}
}
shelf, err := open.inventory.Catalogue(ctx)
if err != nil {
return ""
}
for name, m := range shelf {
if _, shared := m.SharedCredentialOf(n.Name); shared {
return name
}
}
return ""
return nil
}
+21 -197
View File
@@ -4,11 +4,9 @@ import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"flag"
"fmt"
"log"
"os"
"sort"
"strings"
@@ -113,10 +111,7 @@ func serve(ctx context.Context) error {
// while everything else about it looks correct.
// And build results nobody was waiting for. A build triggered any other way than `build`
// would otherwise be reported into the void, which is the same as not reporting it.
server.Records(builds{inv, open})
// Open plans move on a timer as well as on outcomes (novox/hq ADR 0162): a tier waiting for
// machines to report moves when they have, and a plan left by a replaced controller resumes.
go planTicker(ctx, open)
server.Records(builds{inv})
// And what the catalogue decided a build meant. The builder's own result is already handled
// above; this is the other half — the control plane is the only one of the three that knows
// which machines run the thing, so it is the one that acts (novox/hq ADR 0072).
@@ -129,35 +124,6 @@ func serve(ctx context.Context) error {
return err
}
// And the mesh's own verbs, as the seat this control plane holds (novox/hq ADR 0154). Served
// from the store's row, so what the seat declares is what is answered.
handlers, behind, err := seatToolHandlers()
if err != nil {
return err
}
if len(behind) > 0 {
// Said once, loudly, and then served anyway (novox/hq ADR 0185): the mesh keeps answering
// while whatever put an older control plane here is undone.
fmt.Printf("this control plane is behind the %s row: it cannot run %s. "+
"Those answer the reason when called; everything else is served as usual\n",
catalogue.ControllerSeatName, strings.Join(behind, ", "))
}
bus, isNATS := server.Bus().(link.OverNATS)
if !isNATS {
return errors.New("the mesh's verbs are served over the bus, and this control plane is not on it")
}
stopServing, err := bus.ServeSeatTools(catalogue.ControllerSeatName, handlers, log.New(os.Stdout, "", log.LstdFlags))
if err != nil {
return err
}
defer stopServing()
// And says so on the bus (novox/hq ADR 0197): what it serves, as the NATS services protocol asks.
stopAnnouncing, err := bus.Announce(seatAnnouncement(handlers), log.New(os.Stdout, "", log.LstdFlags))
if err != nil {
return err
}
defer stopAnnouncing()
return server.Serve(ctx)
}
@@ -211,12 +177,6 @@ func declare(ctx context.Context, args []string) error {
if err := link.Declare(ctx, server.Bus(), ident, node, raw, 15*time.Second); err != nil {
return err
}
// Written down like every other send (novox/hq issue 204): a declaration a person sent by hand
// is still what the machine was last told, and status must not read it as current for the one
// the mesh would compose.
if _, err := recordSent(ctx, inv, node, raw); err != nil {
return err
}
fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw))
return nil
}
@@ -360,7 +320,7 @@ func pushCommand(ctx context.Context, args []string) error {
if err != nil {
return err
}
sending, refusals := composeEach(asked, allotting(held, inv), func(node string) (sendable, error) {
sending, refusals := composeEach(asked, func(node string) (sendable, error) {
plan, settings, err := planFor(held, open, node)
if err != nil {
return sendable{}, err
@@ -372,18 +332,12 @@ func pushCommand(ctx context.Context, args []string) error {
// The private network is in here with everything else. It used to be composed separately
// and prepended, which meant every machine with an address was on it and no machine could
// be kept off. It is a module now, so it arrives the way a module does.
declared, err := declarationWith(held, open, node, plan, settings, gens, Allocating)
if err == nil {
reportLeftOut(node, declared)
}
return declared, err
return declarationWith(held, open, node, plan, settings, gens, Allocating)
})
sentDigest := map[string]string{}
defer release()
for _, s := range sending {
// The number is inside the signed bytes, so a replayed older declaration cannot borrow a
// newer one's (novox/hq 04-ISSUES/107); it was taken when the composition began (issue 204).
body, err := s.declared.Body()
if err != nil {
return err
@@ -393,20 +347,19 @@ func pushCommand(ctx context.Context, args []string) error {
}
// After it is away, not before. A digest recorded for something that failed to send would
// make the machine look current for a declaration it never received.
digest, err := recordSent(ctx, inv, s.node, body)
record, err := inv.NodeByName(ctx, s.node)
if err != nil {
return err
}
digest := digestOf(body)
if err := inv.RecordSent(ctx, record.ID, digest); err != nil {
return err
}
sentDigest[s.node] = digest
fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.declared.Resources))
}
release()
fmt.Printf("\n%d node(s) told\n", len(sending))
// And each machine's memberships, as every other send does (ADR 0160): a push is the one most
// operators run, and on 2026-10-01 it was the one path that issued none.
if err := issueMemberships(ctx, open, server, sending); err != nil {
return err
}
// **A named push leaves the mesh consistent, not just the machine it named** (novox/hq
// issue 057, ADR 0083). Assigning a cross-node consumer mints a provision, and the PROVIDER's
@@ -469,18 +422,18 @@ func pushCommand(ctx context.Context, args []string) error {
return sendable{}, err
}
reportUnhostable(node, plan)
declared, err := declarationWith(held, open, node, plan, settings, gens, Allocating)
if err == nil {
reportLeftOut(node, declared)
}
return declared, err
return declarationWith(held, open, node, plan, settings, gens, Allocating)
},
func(s readyNode, body []byte) error {
if err := link.Declare(ctx, server.Bus(), ident, s.node, body,
15*time.Second); err != nil {
return err
}
if _, err := recordSent(ctx, inv, s.node, body); err != nil {
record, err := inv.NodeByName(ctx, s.node)
if err != nil {
return err
}
if err := inv.RecordSent(ctx, record.ID, digestOf(body)); err != nil {
return err
}
fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.declared.Resources))
@@ -569,31 +522,17 @@ type readyNode struct {
//
// The all-or-nothing rule is kept where it means something — sendTo, which rotates a credential
// across two machines that must agree — and dropped here, where it never did.
func composeEach(names []string, allot func(node string) (int64, error),
func composeEach(names []string,
compose func(node string) (sendable, error)) ([]readyNode, []string) {
var sending []readyNode
var refusals []string
for _, name := range names {
// **Numbered before it is composed, not before it is sent** (novox/hq issue 204). The
// number says where this declaration stands against every other the mesh composed for the
// machine, and the host refuses one lower than the last it applied. Taken at send time, as
// it was, a declaration composed a minute ago — before an assignment changed — went out with
// a number higher than one composed after the change and sent before it, and the machine
// took the older content as the newer word: on 2026-10-02 a runtime assigned and applied on
// two machines was undone two seconds later by exactly that. Taken here, before the first
// read, what was composed earlier is numbered lower whatever order the sends happen in.
seq, err := allot(name)
if err != nil {
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
continue
}
declared, err := compose(name)
if err != nil {
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
continue
}
declared.Sequence = seq
if len(declared.Resources) == 0 {
// Sent, not skipped (novox/hq issue 127). A node whose declaration composes to
// nothing may have HELD something before — the broker opening a placement gave it,
@@ -621,7 +560,7 @@ func sendRound(ctx context.Context, open *stores, names []string,
return nil, err
}
defer release()
sending, refused := composeEach(names, allotting(held, open.inventory), func(node string) (sendable, error) {
sending, refused := composeEach(names, func(node string) (sendable, error) {
return compose(held, node)
})
for _, s := range sending {
@@ -681,12 +620,6 @@ func sendTo(ctx context.Context, open *stores, names []string) error {
var sending []readyNode
var refusals []string
for _, name := range names {
// Numbered before composing, for the reason composeEach gives (novox/hq issue 204).
seq, err := allot(ctx, inv, name)
if err != nil {
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
continue
}
plan, settings, err := planFor(ctx, open, name)
if err != nil {
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
@@ -698,8 +631,6 @@ func sendTo(ctx context.Context, open *stores, names []string) error {
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
continue
}
declared.Sequence = seq
reportLeftOut(name, declared)
sending = append(sending, readyNode{name, declared})
}
if len(refusals) > 0 {
@@ -721,73 +652,15 @@ func sendTo(ctx context.Context, open *stores, names []string) error {
if err := link.Declare(ctx, server.Bus(), ident, s.node, body, 15*time.Second); err != nil {
return err
}
if _, err := recordSent(ctx, inv, s.node, body); err != nil {
record, err := inv.NodeByName(ctx, s.node)
if err != nil {
return err
}
if err := inv.RecordSent(ctx, record.ID, digestOf(body)); err != nil {
return err
}
fmt.Printf(" sent %s %d resource(s)\n", s.node, len(s.declared.Resources))
}
// And every assignment on those machines its membership (novox/hq ADR 0160): composed from the
// same records the bus's accounts are, so what a runtime serves and what its account may are one
// composition. Issued after the declaration, because the runtime it is for arrives with it.
return issueMemberships(ctx, open, server, sending)
}
// issueMemberships publishes the membership of every module on the machines just sent.
//
// Each carries what its module receives and the private network's addresses, from the same
// composition as the declaration it was sent (novox/hq ADR 0167): a provider reads what it is
// given on the bus, and the file written beside it says the same thing.
func issueMemberships(ctx context.Context, open *stores, server *link.Server, sent []readyNode) error {
records, err := open.inventory.BusRecords(ctx)
if err != nil {
return err
}
where := broker.PlacementsOf(records, records.Interchangeable)
bus, ok := server.Bus().(link.OverNATS)
if !ok {
return nil
}
// The declarations are sent and recorded by now; a membership that cannot be issued is said
// and does not unsay them. Every runtime without one serves the shape it derives (ADR 0160), so
// the push stands, the first failure is named once, and the next push tries again.
issued, failed := 0, 0
var first error
for _, s := range sent {
node := s.node
for _, d := range records.Assigned[node] {
membership := broker.MembershipFor(node, d, where)
membership.Mesh = s.declared.Mesh
for requirement, given := range s.declared.Received[d.Module] {
raw, err := json.Marshal(given)
if err != nil {
return err
}
if membership.Receives == nil {
membership.Receives = map[string]json.RawMessage{}
}
membership.Receives[requirement] = raw
}
body, err := json.Marshal(membership)
if err != nil {
return err
}
if err := bus.PublishMembership(ctx, node, d.Module, body); err != nil {
if first == nil {
first = err
}
failed++
continue
}
issued++
}
}
if issued > 0 {
fmt.Printf(" issued %d membership(s)\n", issued)
}
if failed > 0 {
fmt.Printf(" %d membership(s) could not be issued; the first: %v — the machines keep what "+
"they derive until the next push\n", failed, first)
}
return nil
}
@@ -821,12 +694,6 @@ func wouldSend(ctx context.Context, open *stores,
if err != nil {
continue
}
// Composed with the number the machine was LAST sent, so this is byte for byte what it was
// sent when nothing else changed. A fresh number here would make every machine read as
// behind for ever (novox/hq 04-ISSUES/107).
if declared.Sequence, err = open.inventory.Sequence(ctx, n.ID); err != nil {
return nil, err
}
body, err := declared.Body()
if err != nil {
return nil, err
@@ -850,12 +717,6 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string)
broker.BareAddress(address), err)
}
defer js.Close()
// What the raise decided not to fail over. Said, for the reason everything else here is said:
// a consumer kept as it was is a difference between what the mesh asked for and what the bus
// holds, and one nobody would find by reading either (novox/hq 04-ISSUES/156).
js.Note = func(format string, args ...any) {
fmt.Printf(" "+format+"\n", args...)
}
// **Its own user, before anything else.** The controller's account is created by the installer at
// a bootstrap password, before there is a controller to mint one — so nothing recorded a hash for
@@ -960,40 +821,3 @@ func seatHolders(ctx context.Context, inv *inventory.Inventory) (map[string]brok
}
return out, nil
}
// number gives one send the next sequence for its node (novox/hq 04-ISSUES/107).
// allotting is allot over one inventory, in the shape composeEach takes.
func allotting(ctx context.Context, inv *inventory.Inventory) func(node string) (int64, error) {
return func(node string) (int64, error) { return allot(ctx, inv, node) }
}
// allot takes the next sequence for a machine — the number its next declaration carries.
func allot(ctx context.Context, inv *inventory.Inventory, node string) (int64, error) {
record, err := inv.NodeByName(ctx, node)
if err != nil {
return 0, err
}
return inv.NextSequence(ctx, record.ID)
}
// recordSent writes down what a machine was just sent, and returns the digest.
//
// **On a context that outlives the caller's** (novox/hq issue 204). The record is written after the
// declaration is away, so a send that failed is never recorded as current — and a controller being
// replaced mid-send had its context cancelled between the two, so the machine was told and the mesh
// never wrote it down: status read "applied, current" over a machine that had just been sent
// something else. What was sent was sent; the record of it must not depend on the sender living
// another second. Bounded, so a store that is away does not hold a dying process open for ever.
func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte) (string, error) {
kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
record, err := inv.NodeByName(kept, node)
if err != nil {
return "", err
}
digest := digestOf(body)
if err := inv.RecordSent(kept, record.ID, digest); err != nil {
return "", err
}
return digest, nil
}
+2 -8
View File
@@ -17,7 +17,7 @@ import (
// the wrong machine no longer refuses the whole node), applied one level up.
func TestOneUnresolvableNodeStillLetsTheRestBeSent(t *testing.T) {
sending, refusals := composeEach(
[]string{"anchor", "home-server", "laptop"}, numbered(),
[]string{"anchor", "home-server", "laptop"},
func(node string) (sendable, error) {
if node == "anchor" {
return sendable{}, errors.New(`nothing provides "acme-ca", wanted by route-proxy`)
@@ -43,7 +43,7 @@ func TestOneUnresolvableNodeStillLetsTheRestBeSent(t *testing.T) {
// (novox/hq issue 127): it may have held something before, and only sending the empty
// declaration tells it to drop what the mesh owned. It is never a refusal.
func TestAnEmptyDeclarationIsSentSoTheNodeDropsWhatItHeld(t *testing.T) {
sending, refusals := composeEach([]string{"spare"}, numbered(),
sending, refusals := composeEach([]string{"spare"},
func(string) (sendable, error) { return sendable{}, nil })
if len(sending) != 1 || len(refusals) != 0 {
t.Errorf("an empty declaration must be sent, not skipped or refused: %v / %v", sending, refusals)
@@ -74,9 +74,3 @@ func TestASkippedMachineIsStillAnError(t *testing.T) {
}
}
}
// numbered is an allotter for tests: one higher per call, as the inventory's is per machine.
func numbered() func(string) (int64, error) {
var n int64
return func(string) (int64, error) { n++; return n, nil }
}
+1 -69
View File
@@ -3,7 +3,6 @@ package main
import (
"encoding/json"
"fmt"
"github.com/novox/mesh-controller/internal/inventory"
"sort"
"time"
)
@@ -40,9 +39,6 @@ type meshStatus struct {
// whose is older is still working — and Waiting cannot tell those apart, because the sent
// digest is recorded at send, not at apply.
Reported []machineReported `json:"reported"`
// Plans is what the last merges produced and where each stands (novox/hq ADR 0162): the
// open ones first, each saying its tier, what it waits for, and whether it has waited too long.
Plans []planStatus `json:"plans"`
// Unresolved is every machine that cannot be worked out at all, with what the mesh said when
// it tried. **A machine here is in none of the lists above**: nothing was computed for it, so
// there is nothing to compare it against and nothing it can be behind — which is why a
@@ -60,38 +56,6 @@ type meshStatus struct {
Machines int `json:"machines"`
// Adopted is every node still adopted (novox/hq ADR 0100); absent when none is.
Adopted []string `json:"adopted,omitempty"`
// Untaken is every module assigned to a machine that is holding what it found rather than
// running what the module declares, because nothing took it (novox/hq 04-ISSUES/125). Absent
// when nothing is held.
//
// **A document without this said an outage was a well mesh.** Read from what each machine
// reported, so it is the machine's account and not the mesh's take-time listing.
Untaken []machineUntaken `json:"untaken,omitempty"`
// Filtered is every converged machine that is not filtered by the mesh alone (novox/hq ADR
// 0168), one entry per rule set the mesh did not write — the found firewall in force again,
// or a chain nobody speaks for. Absent when every converged machine is filtered by the mesh
// alone. A document without this called a machine well while a predecessor's chain refused
// what the mesh declared open.
Filtered []machineFiltered `json:"filtered,omitempty"`
}
// machineFiltered is one rule set on a converged machine that the mesh did not write and that
// refuses traffic: where it is, whose the host reads it as, and what it refuses.
type machineFiltered struct {
Node string `json:"node"`
Where string `json:"where"`
Owner string `json:"owner"`
Refuses string `json:"refuses"`
}
// machineUntaken is one module a machine is holding rather than running, and how many resources of
// it are held.
type machineUntaken struct {
Node string `json:"node"`
Module string `json:"module"`
// Held is how many of the module's resources the machine is keeping as it found them. Zero is
// impossible here: a module with nothing held is not in this list.
Held int `json:"held"`
}
type machineUnresolved struct {
@@ -171,39 +135,7 @@ func statusAsJSON(asked answers) ([]byte, error) {
out := meshStatus{Machines: len(nodes), Wrong: []machineDoing{},
Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{},
Reported: []machineReported{}, Unresolved: []machineUnresolved{},
Network: asked.network, Adopted: adoptedNodes(nodes), Plans: planStatuses(asked.plans, time.Now())}
// In a stated order, so two readings of an unchanged mesh are the same document.
untakenNodes := make([]string, 0, len(asked.untaken))
for name := range asked.untaken {
untakenNodes = append(untakenNodes, name)
}
sort.Strings(untakenNodes)
for _, name := range untakenNodes {
modules := make([]string, 0, len(asked.untaken[name]))
for m := range asked.untaken[name] {
modules = append(modules, m)
}
sort.Strings(modules)
for _, m := range modules {
out.Untaken = append(out.Untaken,
machineUntaken{Node: name, Module: m, Held: asked.untaken[name][m]})
}
}
filteredNodes := make([]string, 0, len(asked.filtered))
for name := range asked.filtered {
filteredNodes = append(filteredNodes, name)
}
sort.Strings(filteredNodes)
for _, name := range filteredNodes {
f := asked.filtered[name]
if fw := f.FoundFirewall; fw != nil && fw.Active {
out.Filtered = append(out.Filtered, machineFiltered{Node: name, Where: "the found firewall",
Owner: inventory.FilterFoundFirewall, Refuses: fw.Kind + " is in force again"})
}
for _, x := range f.Others() {
out.Filtered = append(out.Filtered, machineFiltered{Node: name, Where: x.Where, Owner: x.Owner, Refuses: x.Refuses})
}
}
Network: asked.network, Adopted: adoptedNodes(nodes)}
for name := range asked.refused {
out.Unresolved = append(out.Unresolved, machineUnresolved{
Node: name, Problem: asked.refused[name]})
-43
View File
@@ -167,46 +167,3 @@ func TestAMachineFailingTheSameWayIsSaidToBeStuck(t *testing.T) {
t.Fatalf("one failure is not stuck: %v", once)
}
}
// A converged machine something other than the mesh filters is named, per rule set, and is not
// well (novox/hq ADR 0168); one filtered by the mesh alone is not in the list.
func TestAMachineNotFilteredByTheMeshAloneIsNamedAndNotWell(t *testing.T) {
alone := inventory.Filtering{Filters: []inventory.Filter{
{Where: "table inet mesh, chain input", Owner: inventory.FilterMesh, Refuses: "policy drop"},
{Where: "table ip filter, chain DOCKER", Owner: inventory.FilterRuntime, Refuses: "drop"},
{Where: "table ip filter, chain f2b-sshd", Owner: inventory.FilterBan, Refuses: "ip saddr 192.0.2.1 reject"},
}}
if !alone.Alone() {
t.Fatal("the mesh's own, the runtime's and a ban are not the mesh alone")
}
notAlone := inventory.Filtering{
Filters: append(alone.Filters, inventory.Filter{Where: "chain HAL-MESH-ONLY (iptables-legacy)",
Owner: inventory.FilterOther, Refuses: `-A HAL-MESH-ONLY -m comment --comment "not public" -j DROP`}),
FoundFirewall: &inventory.FoundFirewall{Kind: "ufw", Active: true},
}
asked := answers{nodes: []inventory.Node{{Name: "home-server"}, {Name: "laptop"}},
filtered: map[string]inventory.Filtering{"home-server": notAlone}}
if asked.well() {
t.Fatal("a machine not filtered by the mesh alone reads as well")
}
body, err := statusAsJSON(asked)
if err != nil {
t.Fatal(err)
}
var parsed struct {
Filtered []map[string]string `json:"filtered"`
}
if err := json.Unmarshal(body, &parsed); err != nil {
t.Fatal(err)
}
if len(parsed.Filtered) != 2 {
t.Fatalf("filtered: %v", parsed.Filtered)
}
if parsed.Filtered[0]["node"] != "home-server" || parsed.Filtered[0]["owner"] != inventory.FilterFoundFirewall ||
parsed.Filtered[1]["where"] != "chain HAL-MESH-ONLY (iptables-legacy)" || parsed.Filtered[1]["owner"] != inventory.FilterOther {
t.Fatalf("filtered: %v", parsed.Filtered)
}
if body, _ := statusAsJSON(answers{nodes: asked.nodes}); strings.Contains(string(body), `"filtered"`) {
t.Fatal("a mesh filtered by itself alone carries a filtered list")
}
}
-757
View File
@@ -1,757 +0,0 @@
package main
import (
"context"
"flag"
"fmt"
"sort"
"strings"
"time"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// A merge produces a tiered plan the mesh keeps (novox/hq ADR 0162).
//
// The handler that hears the merge computes the plan from the catalogue's one dependency relation,
// writes it to the store, asks the first tier and returns — the receive loop is never held by a
// build. Every outcome taken in advances the plan it belongs to; a ticker advances what outcomes
// alone cannot (a tier waiting for machines to report); a controller replaced mid-plan finds the
// plan where it left it.
// planWaitBound is how long a plan may wait on one thing before `status` names it red.
const planWaitBound = 30 * time.Minute
// tiersOf sorts a set of modules into tiers along the ordering edges among them: tier 0 depends
// on nothing else in the set, tier 1 only on tier 0, and so on. An edge to a module outside the set says
// nothing about the order inside it. A cycle — which the catalogue should never produce — puts
// what remains in one last tier rather than losing it, and is said by the caller.
func tiersOf(set []string, edges []inventory.Edge) [][]string {
in := map[string]bool{}
for _, m := range set {
in[m] = true
}
deps := map[string]map[string]bool{}
for _, m := range set {
deps[m] = map[string]bool{}
}
// The build seat's holders follow the controller that defines their worker (EdgeWorkerOf,
// novox/hq issue 206), so the built-by edge from that controller to such a holder yields: the
// controller is built by whichever build machine is running, as the runtime image always was.
worker := map[string]map[string]bool{}
for _, e := range edges {
if e.Kind == inventory.EdgeWorkerOf && in[e.From] && in[e.To] {
if worker[e.To] == nil {
worker[e.To] = map[string]bool{}
}
worker[e.To][e.From] = true
}
}
for _, e := range edges {
// A code dependency — B packages A's source — rebuilds B with A, in the same tier: B's
// build needs nothing of A's first. The other kinds order: stands-on and declared after
// the base is built, built-by after the build machine is built and running — except for
// what the build machine itself stands on, and for the controller whose worker the build
// machine binds. The runtime image is built by the builder and the builder is built on the
// runtime image; the image comes first, built by the builder that is running.
if !in[e.From] || !in[e.To] || e.From == e.To || e.Kind == inventory.EdgePackages {
continue
}
if e.Kind == inventory.EdgeBuiltBy && (isBaseOf(e.From, e.To, edges, in) || worker[e.From][e.To]) {
continue
}
deps[e.From][e.To] = true
}
placed := map[string]bool{}
var tiers [][]string
for len(placed) < len(set) {
var tier []string
for _, m := range set {
if placed[m] {
continue
}
free := true
for d := range deps[m] {
if !placed[d] {
free = false
break
}
}
if free {
tier = append(tier, m)
}
}
if len(tier) == 0 {
// A cycle: everything left, together, and the caller says so.
for _, m := range set {
if !placed[m] {
tier = append(tier, m)
}
}
}
sort.Strings(tier)
for _, m := range tier {
placed[m] = true
}
tiers = append(tiers, tier)
}
return tiers
}
// isBaseOf says whether `to` stands on `base`, directly or through other bases in the set, along
// the build edges alone.
func isBaseOf(base, to string, edges []inventory.Edge, in map[string]bool) bool {
seen := map[string]bool{}
var walk func(string) bool
walk = func(m string) bool {
if m == base {
return true
}
if seen[m] {
return false
}
seen[m] = true
for _, e := range edges {
if e.From == m && in[e.To] && (e.Kind == inventory.EdgeStandsOn || e.Kind == inventory.EdgeDeclared) && walk(e.To) {
return true
}
}
return false
}
return walk(to)
}
// reachableFrom is the moved modules plus everything that depends on them, through every layer:
// what a merge rebuilds. Along the code and build edges only: a module *built by* the build machine
// is not changed by a new build machine, so a built-by edge orders and gates a plan and never
// widens it — the first plan of 2026-10-01 took the whole catalogue along for a controller change.
func reachableFrom(moved []string, edges []inventory.Edge) []string {
in := map[string]bool{}
for _, m := range moved {
in[m] = true
}
for grew := true; grew; {
grew = false
for _, e := range edges {
// Built-by and worker-of order a plan; neither widens it. A new build machine changes
// nothing it builds, and a new controller changes nothing about the holder it orders —
// what packages the controller's source is already a code edge.
if e.Kind == inventory.EdgeBuiltBy || e.Kind == inventory.EdgeWorkerOf {
continue
}
if in[e.To] && !in[e.From] {
in[e.From] = true
grew = true
}
}
}
out := make([]string, 0, len(in))
for m := range in {
out = append(out, m)
}
sort.Strings(out)
return out
}
// hasCycle says whether the tiers' last tier holds modules that still depend on each other.
func hasCycle(tiers [][]string, edges []inventory.Edge) bool {
if len(tiers) == 0 {
return false
}
last := map[string]bool{}
for _, m := range tiers[len(tiers)-1] {
last[m] = true
}
for _, e := range edges {
if last[e.From] && last[e.To] {
return true
}
}
return false
}
// planFor is the plan a merge produces: the moved modules and everything reachable from them,
// tiered, with the merge it answers.
func planOfMerge(m link.SourceMoved, moved []string, edges []inventory.Edge) inventory.Plan {
set := reachableFrom(moved, edges)
tiers := tiersOf(set, edges)
modules := map[string]*inventory.PlanModule{}
for _, name := range set {
modules[name] = &inventory.PlanModule{}
}
return inventory.Plan{
ID: fmt.Sprintf("plan-%d", time.Now().UnixNano()),
Repository: m.Owner + "/" + m.Repo,
Commit: m.Commit,
Created: time.Now().UTC(),
State: inventory.PlanBuilding,
Tiers: tiers,
Modules: modules,
}
}
// gates is what the next tier needs running from this one: a module of the tier that a later
// tier is built by — the runtime dependency — and whose policy rolls it out, must be applied by
// the machines running it before the next tier is asked. A base an image stands on need only be
// built; a source another module packages need not even be that.
func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool) []string {
if p.Tier >= len(p.Tiers) {
return nil
}
inTier := map[string]bool{}
all := map[string]bool{}
for _, tier := range p.Tiers {
for _, m := range tier {
all[m] = true
}
}
for _, m := range p.Tiers[p.Tier] {
inTier[m] = true
}
later := map[string]bool{}
for _, tier := range p.Tiers[p.Tier+1:] {
for _, m := range tier {
later[m] = true
}
}
seen := map[string]bool{}
var out []string
for _, e := range edges {
if later[e.From] && inTier[e.To] && e.Kind == inventory.EdgeBuiltBy && !seen[e.To] && rollsOut(e.To) &&
!isBaseOf(e.From, e.To, edges, all) {
seen[e.To] = true
out = append(out, e.To)
}
}
sort.Strings(out)
return out
}
// applied says whether every machine running the module has reported since the module was built.
func applied(module string, builtAt time.Time, running []string, reports []inventory.Reported) (bool, []string) {
at := map[string]*time.Time{}
for _, r := range reports {
at[r.Node] = r.At
}
var waiting []string
for _, n := range running {
if t := at[n]; t == nil || t.Before(builtAt) {
waiting = append(waiting, n)
}
}
return len(waiting) == 0, waiting
}
// askTier asks the build machine for every module of the tier, and marks each asked. A module
// the catalogue no longer holds, or whose ask could not be made, is a failure of the plan: a tier
// half asked is a tier that will never complete.
func askTier(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) error {
entries, err := inv.Catalogued(ctx)
if err != nil {
return err
}
byName := map[string]inventory.Entry{}
for _, e := range entries {
byName[e.Manifest.Module] = e
}
now := time.Now().UTC()
for _, name := range p.Tiers[p.Tier] {
state := p.Modules[name]
if state == nil {
state = &inventory.PlanModule{}
p.Modules[name] = state
}
e, known := byName[name]
if !known {
state.State = "failed"
state.Why = "no longer in the catalogue"
p.State = inventory.PlanFailed
p.Note = name + " is no longer in the catalogue"
continue
}
source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
fmt.Printf(" tier %d: ", p.Tier)
if err := buildOne(ctx, source, e.Source.Path, e.Source.Ref, 0); err != nil {
state.State = "failed"
state.Why = err.Error()
p.State = inventory.PlanFailed
p.Note = fmt.Sprintf("%s could not be asked for: %v", name, err)
continue
}
state.State = "asked"
state.AskedAt = &now
}
return nil
}
// planBuilt marks a module built (or failed) in every open plan whose current tier holds it, and
// advances what that completes. Called from the daemon's take-in of every outcome.
func planBuilt(ctx context.Context, open *stores, module, commit, failed string) {
inv := open.inventory
plans, err := inv.OpenPlans(ctx)
if err != nil {
fmt.Printf("plans: cannot read them: %v\n", err)
return
}
now := time.Now().UTC()
for i := range plans {
p := &plans[i]
if p.Tier >= len(p.Tiers) {
continue
}
inTier := false
for _, m := range p.Tiers[p.Tier] {
if m == module {
inTier = true
}
}
if !inTier {
continue
}
state := p.Modules[module]
if state == nil {
state = &inventory.PlanModule{}
p.Modules[module] = state
}
if failed != "" {
state.State = "failed"
state.Why = failed
p.State = inventory.PlanFailed
p.Note = fmt.Sprintf("%s failed to build in tier %d", module, p.Tier)
} else {
state.State = "built"
state.BuiltAt = &now
state.Commit = commit
}
if err := inv.SavePlan(ctx, *p); err != nil {
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
continue
}
if p.State == inventory.PlanFailed {
fmt.Printf("%s: %s; the tiers after it are not asked\n", p.ID, p.Note)
}
}
advancePlans(ctx, open)
}
// advancePlans moves every open plan as far as the facts allow: a tier whose modules are all built
// and whose gates are applied gives way to the next; the last tier done is the plan done. Called
// after every outcome and on a timer, so a plan waiting on a machine's report moves when it comes.
func advancePlans(ctx context.Context, open *stores) {
inv := open.inventory
plans, err := inv.OpenPlans(ctx)
if err != nil {
fmt.Printf("plans: cannot read them: %v\n", err)
return
}
if len(plans) == 0 {
return
}
edges, err := inv.Dependencies(ctx)
if err != nil {
fmt.Printf("plans: cannot read the dependencies: %v\n", err)
return
}
rollsOut := func(module string) bool {
u, err := inv.UpgradeOf(ctx, module)
return err == nil && u.RollOut
}
for i := range plans {
p := &plans[i]
for p.Open() {
moved, err := advanceOnce(ctx, open, p, edges, rollsOut)
if err != nil {
fmt.Printf("%s: %v\n", p.ID, err)
break
}
if err := inv.SavePlan(ctx, *p); err != nil {
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
break
}
if !moved {
break
}
}
}
}
// advanceOnce takes one step of one plan and says whether anything changed.
func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
edges []inventory.Edge, rollsOut func(string) bool) (bool, error) {
inv := open.inventory
if p.Tier >= len(p.Tiers) {
p.State = inventory.PlanDone
fmt.Printf("%s: done — %s at %s, %d tier(s)\n", p.ID, p.Repository, short(p.Commit), len(p.Tiers))
return true, nil
}
tier := p.Tiers[p.Tier]
// Not yet asked: ask.
unasked := 0
for _, m := range tier {
if s := p.Modules[m]; s == nil || s.State == "" {
unasked++
}
}
if unasked == len(tier) {
if err := askTier(ctx, inv, p); err != nil {
return false, err
}
return true, nil
}
// Asked: wait for every build.
var latest time.Time
for _, m := range tier {
s := p.Modules[m]
if s == nil || s.State != "built" {
return false, nil
}
if s.BuiltAt != nil && s.BuiltAt.After(latest) {
latest = *s.BuiltAt
}
}
// Built: send every module of the tier whose policy rolls out, once, to the machines running
// it — whether or not its source commit moved. A dependent rebuilt because its base moved, or
// a module that packages another repository's source, keeps its commit; the catalogue announces
// no move for it and its machines would keep the old image until somebody pushed (novox/hq
// issue 189). A module whose policy records is built and left, as its policy says.
for _, m := range tier {
state := p.Modules[m]
if state == nil || state.SentAt != nil || !rollsOut(m) {
continue
}
running, err := inv.Running(ctx, m)
if err != nil {
return false, err
}
now := time.Now().UTC()
state.SentAt = &now
if len(running) == 0 {
continue
}
if err := sendTo(ctx, open, running); err != nil {
return false, fmt.Errorf("sending %s to %s after tier %d: %w", m, strings.Join(running, ", "), p.Tier, err)
}
fmt.Printf("%s: tier %d built; sent %s to %s\n", p.ID, p.Tier, m, strings.Join(running, ", "))
return true, nil
}
// And wait for what the next tier needs running.
needed := gates(*p, edges, rollsOut)
if len(needed) > 0 {
reports, err := inv.LastReports(ctx)
if err != nil {
return false, err
}
var waiting []string
for _, m := range needed {
running, err := inv.Running(ctx, m)
if err != nil {
return false, err
}
state := p.Modules[m]
if state == nil {
state = &inventory.PlanModule{}
p.Modules[m] = state
}
// The plan sends what it waits for. A rebuild from the same source commit is not a
// move the catalogue announces — the build machine rebuilt for a controller change
// is one — so the roll-out that opens this gate is the plan's to make, once, and
// the reports that open it are the ones after the send.
since := latest
if state.BuiltAt != nil {
since = *state.BuiltAt
}
if state.SentAt != nil && state.SentAt.After(since) {
since = *state.SentAt
}
if ok, on := applied(m, since, running, reports); !ok {
waiting = append(waiting, fmt.Sprintf("%s on %s", m, strings.Join(on, ", ")))
}
}
if len(waiting) > 0 {
note := "tier " + fmt.Sprint(p.Tier) + " built; waiting for " + strings.Join(waiting, "; ") + " to be applied"
changed := p.State != inventory.PlanRolling || p.Note != note
p.State = inventory.PlanRolling
p.Note = note
return changed, nil
}
}
p.Tier++
p.State = inventory.PlanBuilding
p.Note = ""
if p.Tier < len(p.Tiers) {
fmt.Printf("%s: tier %d done; asking tier %d: %s\n", p.ID, p.Tier-1, p.Tier, strings.Join(p.Tiers[p.Tier], ", "))
}
return true, nil
}
// planTicker advances open plans on a timer, for the steps outcomes alone cannot take.
func planTicker(ctx context.Context, open *stores) {
advancePlans(ctx, open)
tick := time.NewTicker(30 * time.Second)
defer tick.Stop()
for {
select {
case <-ctx.Done():
return
case <-tick.C:
advancePlans(ctx, open)
}
}
}
// planLine is one plan as `status` says it.
func planLine(p inventory.Plan, now time.Time) string {
where := fmt.Sprintf("tier %d of %d", min(p.Tier+1, len(p.Tiers)), len(p.Tiers))
switch p.State {
case inventory.PlanDone:
return fmt.Sprintf("%s %s done, %d tier(s)", p.Repository, short(p.Commit), len(p.Tiers))
case inventory.PlanFailed:
return fmt.Sprintf("%s %s FAILED at %s: %s", p.Repository, short(p.Commit), where, p.Note)
}
since := now.Sub(p.Updated).Round(time.Minute)
late := ""
if since > planWaitBound {
late = " — LATE"
}
what := "building"
if p.State == inventory.PlanRolling {
what = p.Note
}
return fmt.Sprintf("%s %s %s, %s for %s%s", p.Repository, short(p.Commit), where, what, since, late)
}
// planFailedBuild marks the module a failed build was for when the result names no module: by the
// repository and path the plan's modules were asked at.
func planFailedBuild(ctx context.Context, open *stores, result link.BuildResult) {
entries, err := open.inventory.Catalogued(ctx)
if err != nil {
return
}
for _, e := range entries {
if repositoryMatches(e.Source.Repository, result.Repository) && e.Source.Path == result.Path {
planBuilt(ctx, open, e.Manifest.Module, result.Commit, result.Failed)
return
}
}
}
func repositoryMatches(a, b string) bool {
trim := func(s string) string { return strings.ToLower(strings.TrimSuffix(s, ".git")) }
return trim(a) == trim(b) || strings.HasSuffix(trim(a), "/"+trim(b)) || strings.HasSuffix(trim(b), "/"+trim(a))
}
// planStatus is one plan as `status --json` says it.
type planStatus struct {
ID string `json:"id"`
Repository string `json:"repository"`
Commit string `json:"commit"`
State string `json:"state"`
Tier int `json:"tier"`
Tiers int `json:"tiers"`
Waiting string `json:"waiting,omitempty"`
Since time.Time `json:"since"`
Late bool `json:"late"`
}
func planStatuses(plans []inventory.Plan, now time.Time) []planStatus {
out := make([]planStatus, 0, len(plans))
for _, p := range plans {
ps := planStatus{ID: p.ID, Repository: p.Repository, Commit: p.Commit, State: p.State,
Tier: p.Tier, Tiers: len(p.Tiers), Since: p.Updated}
if p.Open() {
ps.Waiting = p.Note
if ps.Waiting == "" {
ps.Waiting = "builds of tier " + fmt.Sprint(p.Tier)
}
ps.Late = now.Sub(p.Updated) > planWaitBound
}
out = append(out, ps)
}
return out
}
// openPlans is the open plans among the recent ones, and how many have waited past the bound.
func openPlans(plans []inventory.Plan) ([]inventory.Plan, int) {
var open []inventory.Plan
late := 0
for _, p := range plans {
if p.Open() {
open = append(open, p)
if time.Since(p.Updated) > planWaitBound {
late++
}
}
}
return open, late
}
// plansCommand says what the last merges produced and where each stands; given an id, one plan
// tier by tier with every module's state.
func plansCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("plans", flag.ContinueOnError)
limit := set.Int("n", 10, "how many to show")
whatIf := set.String("what-if", "", "owner/repository: the plan a merge there would produce, saving nothing — with --paths or --modules")
paths := set.String("paths", "", "the files the merge would change, comma-separated, from the repository's root")
modules := set.String("modules", "", "or the modules it would change, comma-separated")
positionals, err := parseAround(set, args)
if err != nil {
return err
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
now := time.Now()
if len(positionals) == 1 {
p, err := inv.PlanByID(ctx, positionals[0])
if err != nil {
return err
}
fmt.Printf("%s — %s\n", p.ID, planLine(p, now))
for i, tier := range p.Tiers {
marker := " "
if i == p.Tier && p.Open() {
marker = ">"
}
fmt.Printf("%s tier %d\n", marker, i)
for _, m := range tier {
s := p.Modules[m]
state := "not yet asked"
if s != nil && s.State != "" {
state = s.State
if s.Commit != "" {
state += " from " + short(s.Commit)
}
if s.Why != "" {
state += ": " + s.Why
}
}
fmt.Printf(" %-22s %s\n", m, state)
}
}
return nil
}
if *whatIf != "" {
return planWhatIf(ctx, inv, *whatIf, splitList(*paths), splitList(*modules))
}
if len(positionals) == 2 && positionals[0] == "stop" {
p, err := inv.PlanByID(ctx, positionals[1])
if err != nil {
return err
}
if !p.Open() {
return fmt.Errorf("%s is already %s", p.ID, p.State)
}
p.State = inventory.PlanFailed
p.Note = "stopped by hand at tier " + fmt.Sprint(p.Tier)
if err := inv.SavePlan(ctx, p); err != nil {
return err
}
fmt.Printf("%s stopped at tier %d of %d; what was asked still builds and registers, nothing further is asked\n",
p.ID, p.Tier, len(p.Tiers))
return nil
}
plans, err := inv.RecentPlans(ctx, *limit)
if err != nil {
return err
}
if len(plans) == 0 {
fmt.Println("no merge has produced a plan yet")
return nil
}
for _, p := range plans {
fmt.Printf("%-28s %s\n", p.ID, planLine(p, now))
}
return nil
}
// planWhatIf is the plan a merge would produce, computed the way the merge handler computes one
// and saved nowhere: the modules the repository's changed files touch (or the modules named), what
// packages their source, everything reachable from them, in tiers. For reading before merging.
func planWhatIf(ctx context.Context, inv *inventory.Inventory, repository string, paths, modules []string) error {
owner, repo, found := strings.Cut(repository, "/")
if !found {
return fmt.Errorf("--what-if takes owner/repository, not %q", repository)
}
m := link.SourceMoved{Owner: owner, Repo: repo, Base: "main", Commit: "what-if", Paths: paths}
entries, err := inv.Catalogued(ctx)
if err != nil {
return err
}
read, err := inv.ReadRepositories(ctx)
if err != nil {
return err
}
var from, packaging []inventory.Entry
named := map[string]bool{}
for _, name := range modules {
named[name] = true
}
for _, e := range entries {
switch {
case named[e.Manifest.Module]:
from = append(from, e)
case len(named) == 0 && sourceIs(e.Source, m):
from = append(from, e)
case readsFrom(read[e.Manifest.Module], m):
packaging = append(packaging, e)
}
}
if len(named) == 0 {
from = whatTheMergeTouched(from, entries, m)
}
moved := append(append([]inventory.Entry{}, from...), packaging...)
if len(moved) == 0 {
fmt.Printf("a merge of %s changing %s would build nothing the mesh holds\n", repository,
orNone(strings.Join(append(paths, modules...), ", ")))
return nil
}
edges, err := inv.Dependencies(ctx)
if err != nil {
return err
}
var names []string
for _, e := range moved {
names = append(names, e.Manifest.Module)
}
p := planOfMerge(m, names, edges)
fmt.Printf("a merge of %s would build %d module(s) in %d tier(s):\n", repository, len(p.Modules), len(p.Tiers))
rolls := map[string]string{}
for i, tier := range p.Tiers {
fmt.Printf(" tier %d\n", i)
for _, name := range tier {
how := "built; its policy records, so nothing is sent"
if u, err := inv.UpgradeOf(ctx, name); err == nil && u.RollOut {
running, _ := inv.Running(ctx, name)
how = "built, then sent to " + orNone(strings.Join(running, ", "))
rolls[name] = how
}
fmt.Printf(" %-22s %s\n", name, how)
}
}
if hasCycle(p.Tiers, edges) {
fmt.Println(" the last tier depends on itself and would be built together, in no order")
}
if len(packaging) > 0 {
var also []string
for _, e := range packaging {
also = append(also, e.Manifest.Module)
}
fmt.Printf(" %s package source from %s, so they are rebuilt without their own source moving\n",
strings.Join(also, ", "), repository)
}
return nil
}
func splitList(s string) []string {
var out []string
for _, part := range strings.Split(s, ",") {
if part = strings.TrimSpace(part); part != "" {
out = append(out, part)
}
}
return out
}
-128
View File
@@ -1,128 +0,0 @@
package main
import (
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// A merge produces a tiered plan (novox/hq ADR 0162): what moved and everything reachable from it,
// sorted so a tier depends only on earlier ones — with the three kinds of dependency told apart.
func TestAMergeIsPlannedInTiersAlongTheThreeKindsOfDependency(t *testing.T) {
edges := []inventory.Edge{
// build dependencies: images on the runtime, a plugin on one of them
{From: "shop", To: "mesh-tools", Kind: inventory.EdgeStandsOn},
{From: "postgres", To: "mesh-tools", Kind: inventory.EdgeStandsOn},
{From: "shop-plugin", To: "shop", Kind: inventory.EdgeDeclared},
// a code dependency: the proxy packages the controller's source — same tier
{From: "route-proxy", To: "mesh-controller", Kind: inventory.EdgePackages},
{From: "builder", To: "mesh-controller", Kind: inventory.EdgePackages},
// runtime dependencies: everything source-built is built by the builder
{From: "shop", To: "builder", Kind: inventory.EdgeBuiltBy},
{From: "postgres", To: "builder", Kind: inventory.EdgeBuiltBy},
{From: "shop-plugin", To: "builder", Kind: inventory.EdgeBuiltBy},
{From: "route-proxy", To: "builder", Kind: inventory.EdgeBuiltBy},
{From: "mesh-controller", To: "builder", Kind: inventory.EdgeBuiltBy},
{From: "mesh-tools", To: "builder", Kind: inventory.EdgeBuiltBy},
{From: "builder", To: "mesh-tools", Kind: inventory.EdgeStandsOn},
{From: "unrelated", To: "alpine", Kind: inventory.EdgeStandsOn},
}
// The runtime image moved: everything on it, and what is built by what is on it.
set := reachableFrom([]string{"mesh-tools"}, edges)
// What stands on the runtime, and the builder that stands on it; not the controller, which the
// builder merely builds, nor the proxy that packages the controller.
want := []string{"builder", "mesh-tools", "postgres", "shop", "shop-plugin"}
if len(set) != len(want) {
t.Fatalf("reachable from the runtime: %v, want %v", set, want)
}
tiers := tiersOf(set, edges)
pos := map[string]int{}
for i, tier := range tiers {
for _, m := range tier {
pos[m] = i
}
}
if pos["mesh-tools"] != 0 || pos["builder"] != 1 {
t.Fatalf("the runtime then the builder: %v", tiers)
}
if !(pos["shop"] > pos["builder"] && pos["postgres"] > pos["builder"]) {
t.Fatalf("what the builder builds comes after the builder: %v", tiers)
}
if pos["shop-plugin"] <= pos["shop"] {
t.Fatalf("a plugin after what it is declared on: %v", tiers)
}
if hasCycle(tiers, edges) {
t.Fatalf("no cycle here: %v", tiers)
}
// The controller alone moved: the proxy with it, nothing else.
small := reachableFrom([]string{"mesh-controller"}, edges)
if len(small) != 3 {
t.Fatalf("a controller merge rebuilds the controller and what packages it: %v", small)
}
// The builder packages the controller's source (same tier by that edge) and the controller is
// built by the builder (next tier by that one): the builder first, then the controller and the
// proxy together — a code dependency in one tier, a runtime dependency across tiers.
smallTiers := tiersOf(small, edges)
if len(smallTiers) != 2 || smallTiers[0][0] != "builder" || len(smallTiers[1]) != 2 {
t.Fatalf("the builder, then the controller and the proxy together: %v", smallTiers)
}
// The builder alone moved: the builder, and nothing it builds.
if only := reachableFrom([]string{"builder"}, edges); len(only) != 1 {
t.Fatalf("a build machine change rebuilds the build machine alone: %v", only)
}
// Only a runtime dependency gates on deployment, and only when the module rolls out.
p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-tools", Commit: "abc"}, []string{"mesh-tools"}, edges)
p.Tier = pos["builder"]
rollsOut := func(m string) bool { return m == "builder" }
if g := gates(p, edges, rollsOut); len(g) != 1 || g[0] != "builder" {
t.Fatalf("the builder gates the tier after it: %v", g)
}
p.Tier = 0
if g := gates(p, edges, rollsOut); len(g) != 0 {
t.Fatalf("the runtime image is a build dependency and gates nothing: %v", g)
}
if g := gates(p, edges, func(string) bool { return false }); len(g) != 0 {
t.Fatalf("a module that only records its upgrade gates nothing: %v", g)
}
}
// A gate is open once every machine running the module has reported after it was built.
func TestAGateOpensWhenTheMachinesHaveReportedSinceTheBuild(t *testing.T) {
built := time.Date(2026, 10, 1, 15, 0, 0, 0, time.UTC)
before, after := built.Add(-time.Minute), built.Add(time.Minute)
reports := []inventory.Reported{{Node: "anchor", At: &after}, {Node: "home-server", At: &before}}
ok, waiting := applied("builder", built, []string{"anchor", "home-server"}, reports)
if ok || len(waiting) != 1 || waiting[0] != "home-server" {
t.Fatalf("one machine has not reported since the build: ok=%v waiting=%v", ok, waiting)
}
if ok, _ := applied("builder", built, []string{"anchor"}, reports); !ok {
t.Fatal("the machine that reported after the build holds the gate open")
}
if ok, _ := applied("builder", built, nil, reports); !ok {
t.Fatal("a module running nowhere gates nothing")
}
}
// A cycle is not lost: what remains is one last tier, and the caller says so.
func TestACycleIsOneLastTierAndSaidSo(t *testing.T) {
edges := []inventory.Edge{{From: "a", To: "b", Kind: inventory.EdgeStandsOn}, {From: "b", To: "a", Kind: inventory.EdgeStandsOn}}
tiers := tiersOf([]string{"a", "b"}, edges)
if len(tiers) != 1 || len(tiers[0]) != 2 || !hasCycle(tiers, edges) {
t.Fatalf("a cycle should be one tier of two, said: %v", tiers)
}
}
// novox/hq 04-ISSUES/211: a merge moving the toolchain and a bundle compiled in it builds the
// bundle a tier after the toolchain, not beside it.
func TestABundleIsPlannedAfterTheToolchainItIsCompiledIn(t *testing.T) {
edges := []inventory.Edge{{From: "node-tools", To: "mesh-tools", Kind: inventory.EdgeStandsOn}}
p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-tools", Commit: "abc"},
[]string{"mesh-tools", "node-tools"}, edges)
if len(p.Tiers) != 2 || p.Tiers[0][0] != "mesh-tools" || p.Tiers[1][0] != "node-tools" {
t.Fatalf("the toolchain, then the bundle: %v", p.Tiers)
}
}
+2 -4
View File
@@ -346,9 +346,7 @@ func rolloutMint(ctx context.Context, again bool) error {
}
machines++
case broker.KindModule, broker.KindNodeTools:
// The runtime is minted and delivered exactly as a module is (novox/hq ADR 0175): it is
// issued as the module it stands for, to that module's `broker` secret.
case broker.KindModule:
if p.Module == "mesh-controller" {
// The control plane is a module too, and its `broker` secret is the old bus's
// credential it is still using while this runs. Writing the new bus's blob there
@@ -367,7 +365,7 @@ func rolloutMint(ctx context.Context, again bool) error {
skipped++
continue
}
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: busKindOf(p.Module), Node: p.Node, Module: p.Module})
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusModule, Node: p.Node, Module: p.Module})
if err != nil {
return err
}
@@ -1,46 +0,0 @@
package main
import (
"context"
"testing"
)
// A node's own set failing to compose, and the mesh being unable to answer at all, are different
// things, and only the first may be passed over when something is gathered across every machine
// (novox/hq 04-ISSUES/152). These pin that distinction where the gatherers rely on it.
func TestASetThatDoesNotComposeIsMarkedAsTheNodesOwnProblem(t *testing.T) {
open := aMesh(t)
one, two := rivals()
register(t, open, one)
register(t, open, two)
for _, m := range []string{one.Module, two.Module} {
if _, err := open.inventory.Assign(t.Context(), "laptop", m); err != nil {
t.Fatal(err)
}
}
_, _, err := planFor(t.Context(), open, "laptop")
if err == nil {
t.Fatal("two modules claiming one seat composed anyway")
}
if !unresolvable(err) {
t.Fatalf("a set that cannot compose was not marked as the node's own problem: %v", err)
}
}
func TestAStoreThatCannotBeReadIsNotANodeThatDoesNotCompose(t *testing.T) {
open := aMesh(t)
// Nothing is wrong with anchor. The question simply cannot be asked.
stopped, cancel := context.WithCancel(t.Context())
cancel()
_, _, err := planFor(stopped, open, "anchor")
if err == nil {
t.Fatal("a plan composed against a store that could not be read")
}
if unresolvable(err) {
t.Fatalf("a question the mesh could not answer was read as a node that runs nothing: %v", err)
}
}
-426
View File
@@ -1,426 +0,0 @@
package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"github.com/nats-io/nats.go/micro"
"os"
"os/exec"
"sort"
"strings"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/link"
)
// The mesh's own verbs, served as the mesh-controller seat's tools (novox/hq ADR 0154, design 33).
//
// **Each tool runs the command it names, in this same binary, and answers what it printed.** That is
// ADR 0035 taken literally: the logic lives once, in the command, and a surface is an adapter with no
// decisions in it. Running a fresh process rather than calling the function keeps two things true
// that calling it would not — every command opens and closes its own stores the way it does from a
// shell, and nothing a command prints to the process's standard output can leak into another call's
// answer. It also means a refusal is the same refusal in the same words, because it is the same
// output.
// verbAnswer is what a verb answers: what the command printed, whether it succeeded, and — where the
// command speaks JSON — the same as data.
type verbAnswer struct {
Output string `json:"output"`
OK bool `json:"ok"`
Answer any `json:"answer,omitempty"`
}
// argvFor is the command line a verb and its arguments become. Only the verbs the seat declares, and
// only the arguments each declares: a caller cannot reach a flag the schema did not name.
func argvFor(verb string, args map[string]any) ([]string, error) {
str := func(key string) string {
v, _ := args[key].(string)
return strings.TrimSpace(v)
}
need := func(keys ...string) error {
for _, k := range keys {
if str(k) == "" {
return fmt.Errorf("%s needs %q", verb, k)
}
}
return nil
}
switch verb {
case "command":
// The generic verb: the command line as given, split as a shell would split it, with
// nothing added — the named verbs add flags a caller cannot reach; this one is the whole
// binary and says so in its description (novox/hq ADR 0154, 0175).
if err := need("command"); err != nil {
return nil, err
}
argv, err := splitCommandLine(str("command"))
if err != nil {
return nil, err
}
if len(argv) == 0 {
return nil, errors.New("command names no command")
}
return argv, nil
case "status":
return []string{"status", "--json"}, nil
case "nodes":
return []string{"node", "list", "--json"}, nil
case "node":
if err := need("node"); err != nil {
return nil, err
}
return []string{"node", "show", str("node")}, nil
case "modules":
return []string{"module", "list", "--json"}, nil
case "seats":
return []string{"seats", "--json"}, nil
case "builds":
if id := str("log"); id != "" {
return []string{"builds", "--log", id}, nil
}
if m := str("module"); m != "" {
return []string{"builds", m}, nil
}
return []string{"builds"}, nil
case "plans":
if r := str("repository"); r != "" {
argv := []string{"plans", "--what-if", r}
if p := str("paths"); p != "" {
argv = append(argv, "--paths", p)
}
if m := str("modules"); m != "" {
argv = append(argv, "--modules", m)
}
return argv, nil
}
if id := str("stop"); id != "" {
return []string{"plans", "stop", id}, nil
}
if id := str("id"); id != "" {
return []string{"plans", id}, nil
}
return []string{"plans"}, nil
case "plan":
if err := need("node"); err != nil {
return nil, err
}
return []string{"plan", str("node"), "--json"}, nil
case "assign", "unassign":
if err := need("node", "module"); err != nil {
return nil, err
}
return []string{verb, str("node"), str("module")}, nil
case "pin":
if err := need("node", "provision", "from", "module"); err != nil {
return nil, err
}
return []string{"pin", str("node"), str("provision"), str("from"), str("module")}, nil
case "unpin":
if err := need("node", "provision"); err != nil {
return nil, err
}
return []string{"unpin", str("node"), str("provision")}, nil
case "push":
// Sent and not waited for: the asker reads `status` for what the machine did, which is
// what a person at a shell does too. A tool call that blocked for a push's whole apply would
// time out on every machine that takes a minute, and say nothing about the ones that did not.
if n := str("node"); n != "" {
return []string{"push", n, "--wait", "0"}, nil
}
return []string{"push", "--behind", "--wait", "0"}, nil
case "rotate":
if p := str("provision"); p != "" {
argv := []string{"rotate", p}
if c := str("consumer"); c != "" {
argv = append(argv, "--consumer", c)
}
return argv, nil
}
if str("node") != "" && str("module") != "" && str("secret") != "" {
return []string{"secret", "rotate", str("node"), str("module"), str("secret")}, nil
}
// Half of either shape: the command says its usage, which names both shapes, and that is
// the answer the caller needs.
return []string{"rotate"}, nil
case "settings":
// `settings set|clear` at a shell (novox/hq issue 198). The values travel as an argument
// because a tool has no file to hand the command; the command reads either.
if err := need("module"); err != nil {
return nil, err
}
argv := []string{"settings", "set", str("module")}
switch {
case str("clear") == "true":
argv = []string{"settings", "clear", str("module")}
case str("values") != "":
argv = append(argv, str("values"))
}
// Neither values nor clear: the command says its usage, which names both, and that is the
// answer the caller needs — the same as `rotate` given half of either shape.
if n := str("node"); n != "" {
argv = append(argv, "--node", n)
}
return argv, nil
case "issue":
// The same act as `module issue` at a shell (novox/hq design 25 §4): the account is minted
// into the mesh's records and delivered at the machine's next push, which is the caller's to
// ask for — so the mesh is never pushed as a side effect of a credential.
if err := need("node", "module"); err != nil {
return nil, err
}
return []string{"module", "issue", str("module"), "--node", str("node")}, nil
case "build":
if err := need("repository"); err != nil {
return nil, err
}
// Not waited for: a tool call cannot hold a connection for the minutes a build takes; the
// daemon takes the outcome in when it comes and the id follows the build (issue 176). A
// repository given without a scheme is a path on the forge holding the git seat.
argv := []string{"build", str("repository"), "--wait", "0"}
if !strings.Contains(str("repository"), "://") && !strings.HasPrefix(str("repository"), "git@") {
argv = append(argv, "--self")
}
if p := str("path"); p != "" {
argv = append(argv, "--path", p)
}
if r := str("ref"); r != "" {
argv = append(argv, "--ref", r)
}
return argv, nil
}
return nil, fmt.Errorf("%q is not a verb the %s seat serves", verb, catalogue.ControllerSeatName)
}
// jsonVerbs are the verbs whose command speaks JSON, so the answer carries it as data as well.
var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true}
// runVerb runs this binary with the given command line and gathers what it said.
func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
self, err := os.Executable()
if err != nil {
return verbAnswer{}, err
}
cmd := exec.CommandContext(ctx, self, argv...)
// The same environment: the stores' credentials, the bus, the broker — everything a command run
// from a shell in this container would have, because it is that.
cmd.Env = os.Environ()
// Two buffers, one answer. What the command *says* is both streams, in the order a person at
// a shell would read them; what it *answers as data* is standard output alone — `status --json`
// prints its warnings beside the document, and a JSON parsed from the two together parsed
// nothing (2026-09-30, the first status asked through the console had no `answer`).
var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
runErr := cmd.Run()
answer := verbAnswer{Output: stdout.String() + stderr.String(), OK: runErr == nil}
if jsonVerbs[argv[0]] && runErr == nil {
var parsed any
if json.Unmarshal(bytes.TrimSpace(stdout.Bytes()), &parsed) == nil {
answer.Answer = parsed
}
}
var exit *exec.ExitError
if runErr != nil && !errors.As(runErr, &exit) {
// Not the command refusing — the command not running at all, which is this process's fault.
return answer, fmt.Errorf("could not run %s: %w", strings.Join(argv, " "), runErr)
}
return answer, nil
}
// seatToolHandlers are the handlers for every verb the mesh-controller seat declares, from the
// store's row, so a verb the row does not carry is not served. A verb it carries that this binary
// cannot run is named at start and answers the reason when called — never a refusal to serve, which
// would take the whole control plane down for one word (novox/hq ADR 0185).
func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
seat, known := catalogue.SeatNamed(catalogue.ControllerSeatName)
if !known {
return nil, nil, fmt.Errorf("this mesh defines no %s seat", catalogue.ControllerSeatName)
}
var behind []string
handlers := map[string]link.ToolHandler{}
for _, v := range seat.Serves {
verb := v.Name
if verb == "tools" {
handlers[verb] = func(ctx context.Context, _ json.RawMessage) (any, error) {
return seatTools(), nil
}
continue
}
if _, err := argvFor(verb, sampleArguments(v)); err != nil {
// **A row ahead of this binary is not a reason to go silent.**
//
// The row is the store's and a control plane follows it (novox/hq ADR 0154), so a verb
// this build does not know means the row was widened by a newer one — the ordinary
// state of a roll-out, and of a push that put an older control plane back. Refusing to
// serve at all made that transient fatal: on 2026-10-02 one unknown verb took the whole
// mesh off the bus for ten minutes, and the way back was a human running the binary by
// hand, because the thing that would have repaired it is the thing that was down
// (novox/hq 04-ISSUES/201, ADR 0185).
//
// So the verbs this binary knows are served, and this one answers the reason instead of
// nothing: a caller gets a sentence naming the fault, and everything else keeps working
// — including the push that replaces this binary with the one whose verb it is.
behind = append(behind, verb)
reason := err
handlers[verb] = func(context.Context, json.RawMessage) (any, error) {
return nil, fmt.Errorf("%s is in this mesh's %s row and the control plane running "+
"here cannot run it: %w. It is a verb of a newer build; this one is behind",
verb, catalogue.ControllerSeatName, reason)
}
continue
}
handlers[verb] = func(ctx context.Context, raw json.RawMessage) (any, error) {
args := map[string]any{}
if len(raw) > 0 {
if err := json.Unmarshal(raw, &args); err != nil {
return nil, fmt.Errorf("the arguments are not a JSON object: %w", err)
}
}
argv, err := argvFor(verb, args)
if err != nil {
return nil, err
}
return runVerb(ctx, argv)
}
}
return handlers, behind, nil
}
// seatTools is what `tools` answers: every seat with a protocol, and the tools each serves, from the
// mesh's own records — no holder in the path, so it is true while a holder restarts (design 33 §5).
func seatTools() map[string]any {
var seats []map[string]any
for _, s := range catalogue.SeatsWithAProtocol() {
if len(s.Serves) == 0 {
continue
}
var tools []map[string]any
for _, v := range s.Serves {
tools = append(tools, map[string]any{
"name": v.Name, "description": v.Description, "input": v.Input, "output": v.Output,
})
}
seats = append(seats, map[string]any{"seat": s.Name, "scope": s.Scope, "tools": tools})
}
return map[string]any{"seats": seats}
}
// sampleArguments is one of every argument a verb's schema requires, so the check at start proves the
// verb runnable rather than that it happens to want the arguments the check guessed.
func sampleArguments(v catalogue.Verb) map[string]any {
sample := map[string]any{"node": "x", "module": "x", "repository": "x"}
switch required := v.Input["required"].(type) {
case []string:
for _, k := range required {
sample[k] = "x"
}
case []any:
for _, k := range required {
if name, ok := k.(string); ok {
sample[name] = "x"
}
}
}
return sample
}
// splitCommandLine splits a command line into words the way a POSIX shell does for the simple
// cases a controller command needs: spaces separate, single or double quotes group, a backslash
// escapes the next character inside double quotes or outside any. No expansion of anything.
func splitCommandLine(line string) ([]string, error) {
var words []string
var cur strings.Builder
inWord := false
quote := rune(0)
runes := []rune(line)
for i := 0; i < len(runes); i++ {
r := runes[i]
switch {
case quote == '\'':
if r == '\'' {
quote = 0
} else {
cur.WriteRune(r)
}
case quote == '"':
if r == '"' {
quote = 0
} else if r == '\\' && i+1 < len(runes) {
i++
cur.WriteRune(runes[i])
} else {
cur.WriteRune(r)
}
case r == '\'' || r == '"':
quote = r
inWord = true
case r == '\\' && i+1 < len(runes):
i++
cur.WriteRune(runes[i])
inWord = true
case r == ' ' || r == '\t' || r == '\n':
if inWord {
words = append(words, cur.String())
cur.Reset()
inWord = false
}
default:
cur.WriteRune(r)
inWord = true
}
}
if quote != 0 {
return nil, fmt.Errorf("command has an unclosed %c quote", quote)
}
if inWord {
words = append(words, cur.String())
}
return words, nil
}
// seatAnnouncement is what the controller says it serves on the bus (novox/hq ADR 0197): the
// mesh-controller seat, one endpoint per verb it answers, each with the seat's own description and
// argument schema — the same facts `tools` answers from the records, as NATS's services format.
func seatAnnouncement(handlers map[string]link.ToolHandler) micro.Info {
about := map[string]catalogue.Verb{}
for _, s := range catalogue.SeatsWithAProtocol() {
if s.Name == catalogue.ControllerSeatName {
for _, v := range s.Serves {
about[v.Name] = v
}
}
}
verbs := make([]string, 0, len(handlers))
for verb := range handlers {
verbs = append(verbs, verb)
}
sort.Strings(verbs)
var endpoints []micro.EndpointInfo
for _, verb := range verbs {
schema, _ := json.Marshal(about[verb].Input)
// The same shape every tool runtime announces in (node-tools' announce package): the name is
// `<seat>__<verb>`, as the protocol's characters allow; the metadata is what identifies it.
endpoints = append(endpoints, micro.EndpointInfo{
Name: catalogue.ControllerSeatName + "__" + verb,
Subject: link.SeatToolSubject(catalogue.ControllerSeatName, verb),
QueueGroup: "seat." + catalogue.ControllerSeatName,
Metadata: map[string]string{
"kind": "seat", "module": catalogue.ControllerSeatName, "tool": verb,
"seat": catalogue.ControllerSeatName, "scope": "mesh", "interchangeable": "false",
"description": about[verb].Description, "schema": string(schema),
},
})
}
return micro.Info{
ServiceIdentity: micro.ServiceIdentity{
Name: catalogue.ControllerSeatName, ID: "controller", Version: "0.1.0",
Metadata: map[string]string{"seat": catalogue.ControllerSeatName, "scope": "mesh"},
},
Description: "the mesh's own verbs, answered by the holder of the mesh-controller seat",
Endpoints: endpoints,
}
}
-298
View File
@@ -1,298 +0,0 @@
package main
import (
"context"
"fmt"
"github.com/novox/mesh-controller/internal/link"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// Every verb the mesh-controller seat declares is one this binary can run, with the arguments the
// schema names and no other (novox/hq ADR 0154, ADR 0035).
func TestEveryDeclaredVerbHasACommandLine(t *testing.T) {
for _, v := range catalogue.ControllerVerbs {
if v.Name == "tools" {
continue
}
args := map[string]any{}
props, _ := v.Input["properties"].(map[string]any)
for name := range props {
args[name] = "x"
}
argv, err := argvFor(v.Name, args)
if err != nil {
t.Errorf("%s: %v", v.Name, err)
continue
}
if argv[0] == "" {
t.Errorf("%s: empty command", v.Name)
}
}
}
// `builds` given a build's id reads that build's log from the bus rather than listing builds
// (novox/hq ADR 0157).
func TestBuildsWithAnIdReadsThatBuildsLog(t *testing.T) {
argv, err := argvFor("builds", map[string]any{"log": "build-17"})
if err != nil {
t.Fatal(err)
}
if strings.Join(argv, " ") != "builds --log build-17" {
t.Fatalf("builds with a log id became %q", strings.Join(argv, " "))
}
}
// The build tool takes a repository as a URL or as its path on the forge holding the git seat, and
// says which it was given, so the command reads the path as a seat source rather than handing it to
// git as written (novox/hq issue 176). And it never waits: the id follows the build.
func TestTheBuildToolTellsAForgePathFromAURL(t *testing.T) {
argv, _ := argvFor("build", map[string]any{"repository": "novox/mesh-catalog", "path": "modules/x"})
if line := strings.Join(argv, " "); !strings.Contains(line, "--self") || !strings.Contains(line, "--wait 0") {
t.Fatalf("a forge path is a seat source, not waited for; got %q", line)
}
argv, _ = argvFor("build", map[string]any{"repository": "https://example.tld/o/r.git"})
if line := strings.Join(argv, " "); strings.Contains(line, "--self") {
t.Fatalf("a URL is cloned as given; got %q", line)
}
}
// `rotate` is one verb with two shapes (ADR 0114, issue 180): a pair credential by provision, or a
// module's own secret by machine, module and name.
func TestRotateTakesAProvisionOrAnOwnSecret(t *testing.T) {
argv, _ := argvFor("rotate", map[string]any{"provision": "postgres-database", "consumer": "ace"})
if strings.Join(argv, " ") != "rotate postgres-database --consumer ace" {
t.Fatalf("a pair credential: %v", argv)
}
argv, _ = argvFor("rotate", map[string]any{"node": "ace", "module": "nodered", "secret": "api-token"})
if strings.Join(argv, " ") != "secret rotate ace nodered api-token" {
t.Fatalf("an own secret: %v", argv)
}
argv, _ = argvFor("rotate", map[string]any{"node": "ace"})
if strings.Join(argv, " ") != "rotate" {
t.Fatalf("half an own secret falls to the command's usage: %v", argv)
}
}
// `settings` is `settings set|clear` at a shell, with the values passed inline (novox/hq issue 198).
func TestSettingsSetsOrClearsALayer(t *testing.T) {
argv, err := argvFor("settings", map[string]any{"module": "dnsmasq", "values": `{"a":1}`, "node": "ace"})
if err != nil || strings.Join(argv, " ") != `settings set dnsmasq {"a":1} --node ace` {
t.Fatalf("set on a machine: %v %v", argv, err)
}
argv, _ = argvFor("settings", map[string]any{"module": "dnsmasq", "clear": "true"})
if strings.Join(argv, " ") != "settings clear dnsmasq" {
t.Fatalf("clear for the mesh: %v", argv)
}
argv, _ = argvFor("settings", map[string]any{"module": "dnsmasq"})
if strings.Join(argv, " ") != "settings set dnsmasq" {
t.Fatalf("a set with no values falls to the command's usage: %v", argv)
}
}
// `issue` is `module issue` at a shell: the module and the machine, and nothing that would push. A
// module's bus account was mintable only from the controller's command line, so an agent working
// through the tools could not finish a rollout that gave a module one (novox/hq issue 191).
func TestIssueGivesAModuleItsAccountOnAMachine(t *testing.T) {
argv, err := argvFor("issue", map[string]any{"node": "ace", "module": "route-proxy"})
if err != nil {
t.Fatal(err)
}
if strings.Join(argv, " ") != "module issue route-proxy --node ace" {
t.Fatalf("issue runs %v", argv)
}
if _, err := argvFor("issue", map[string]any{"module": "route-proxy"}); err == nil {
t.Error("an account was issued without saying which machine reads it")
}
}
// A required argument missing is refused in the verb's own words, before anything runs.
func TestAVerbMissingWhatItNeedsIsRefused(t *testing.T) {
if _, err := argvFor("node", map[string]any{}); err == nil || !strings.Contains(err.Error(), `node needs "node"`) {
t.Fatalf("node without a machine was accepted: %v", err)
}
if _, err := argvFor("upgrade", map[string]any{}); err == nil {
t.Fatal("a verb the seat does not serve was accepted")
}
}
// A push and a build are sent, not waited for: the asker reads status, or the build's log by its
// id, for what happened. A repository given as a forge path is said to be one (issue 176).
func TestActsDoNotBlockTheCall(t *testing.T) {
argv, _ := argvFor("push", map[string]any{"node": "one"})
if strings.Join(argv, " ") != "push one --wait 0" {
t.Fatalf("push waits: %v", argv)
}
argv, _ = argvFor("build", map[string]any{"repository": "novox/x", "path": "modules/x"})
if strings.Join(argv, " ") != "build novox/x --wait 0 --self --path modules/x" {
t.Fatalf("build: %v", argv)
}
}
// What `tools` answers is the seats' records, with each verb's schema.
func TestToolsAnswersTheSeatsRecords(t *testing.T) {
handlers, behind, err := seatToolHandlers()
if err != nil {
t.Fatal(err)
}
if len(behind) != 0 {
t.Fatalf("this build cannot run %v of its own seat's verbs", behind)
}
if len(handlers) != len(catalogue.ControllerVerbs) {
t.Fatalf("%d handlers for %d verbs", len(handlers), len(catalogue.ControllerVerbs))
}
answer := seatTools()
seats, _ := answer["seats"].([]map[string]any)
var found bool
for _, s := range seats {
if s["seat"] == catalogue.ControllerSeatName {
found = true
tools, _ := s["tools"].([]map[string]any)
if len(tools) != len(catalogue.ControllerVerbs) || tools[0]["input"] == nil {
t.Fatalf("the controller seat's tools are not listed in full: %v", tools)
}
}
}
if !found {
t.Fatal("the mesh-controller seat is not in the listing")
}
}
// A JSON verb's answer is parsed from what the command wrote to standard output alone; a warning it
// printed beside the document does not take the document away. The test binary stands in for the
// controller: `-test.run` with a name that matches nothing prints `ok` and a warning about no tests.
func TestAJSONVerbsAnswerIsItsStandardOutput(t *testing.T) {
jsonVerbs["-test.run"] = true
t.Cleanup(func() { delete(jsonVerbs, "-test.run") })
answer, err := runVerb(t.Context(), []string{"-test.run", "TestAnswerEcho", "-test.v"})
if err != nil {
t.Fatal(err)
}
if !answer.OK {
t.Fatalf("the command failed: %s", answer.Output)
}
if !strings.Contains(answer.Output, "PASS") {
t.Fatalf("stderr and stdout are both what the command said: %s", answer.Output)
}
}
// `command` is the generic verb: the command line as given, split as a shell would, nothing added —
// so an operator's `node account g14 jochen` is one call through the console rather than a shell on
// the control node (novox/hq ADR 0154, ADR 0175).
func TestCommandRunsTheLineAsGiven(t *testing.T) {
argv, err := argvFor("command", map[string]any{"command": "node account g14 jochen"})
if err != nil || strings.Join(argv, " ") != "node account g14 jochen" {
t.Fatalf("a plain line: %v %v", argv, err)
}
argv, err = argvFor("command", map[string]any{"command": `settings set dnsmasq '{"a": "b c"}' --node ace`})
if err != nil || len(argv) != 6 || argv[3] != `{"a": "b c"}` {
t.Fatalf("a quoted word stays one word: %q %v", argv, err)
}
argv, err = argvFor("command", map[string]any{"command": `node add "the box" --adopted`})
if err != nil || len(argv) != 4 || argv[2] != "the box" {
t.Fatalf("double quotes group: %q %v", argv, err)
}
if _, err := argvFor("command", map[string]any{"command": " "}); err == nil {
t.Fatal("an empty line was accepted")
}
if _, err := argvFor("command", map[string]any{"command": `node "unclosed`}); err == nil {
t.Fatal("an unclosed quote was accepted")
}
}
// A verb in the row that this binary cannot run does not take the control plane off the bus: the
// rest are served, the unknown one answers the reason, and the start-up names it (novox/hq ADR
// 0185). One unknown word cost the mesh ten minutes of silence on 2026-10-02, recoverable only by
// a person running the binary by hand — the push that would have repaired it needs the control
// plane that was down.
func TestARowAheadOfThisBuildIsServedAnyway(t *testing.T) {
seat, known := catalogue.SeatNamed(catalogue.ControllerSeatName)
if !known {
t.Fatal("no controller seat")
}
// The row as a newer control plane would have written it: every verb this build knows, and one
// it does not.
widened := seat
widened.Serves = append(append([]catalogue.Verb{}, seat.Serves...),
catalogue.Verb{Name: "teleport", Description: "a verb from a build that does not exist yet"})
rows := catalogue.DefaultSeats()
for i := range rows {
if rows[i].Name == catalogue.ControllerSeatName {
rows[i] = widened
}
}
catalogue.UseSeats(rows)
t.Cleanup(func() { catalogue.UseSeats(catalogue.DefaultSeats()) })
handlers, behind, err := seatToolHandlers()
if err != nil {
t.Fatalf("a row with one unknown verb refused to serve at all: %v", err)
}
if len(behind) != 1 || behind[0] != "teleport" {
t.Fatalf("the verbs this build cannot run were reported as %v", behind)
}
if len(handlers) != len(widened.Serves) {
t.Fatalf("%d handlers for %d verbs in the row", len(handlers), len(widened.Serves))
}
for _, known := range []string{"status", "nodes", "push"} {
if handlers[known] == nil {
t.Errorf("%s is not served although this build knows it", known)
}
}
_, err = handlers["teleport"](context.Background(), nil)
if err == nil {
t.Fatal("the unknown verb answered as though it had run")
}
for _, want := range []string{"teleport", "cannot run it", "behind"} {
if !strings.Contains(err.Error(), want) {
t.Errorf("the answer does not say %q: %v", want, err)
}
}
}
// novox/hq ADR 0195: the console's discovery reads the machines and the modules; they answer as JSON,
// as status and seats do, so nothing parses a printed column.
func TestTheNodesAndModulesVerbsAnswerAsJSON(t *testing.T) {
for verb, want := range map[string]string{"nodes": "[node list --json]", "modules": "[module list --json]"} {
argv, err := argvFor(verb, map[string]any{})
if err != nil {
t.Fatal(err)
}
if fmt.Sprint(argv) != want {
t.Errorf("%s runs %v, want %s", verb, argv, want)
}
}
}
// novox/hq ADR 0197: the controller announces exactly the verbs it serves, each on the subject and
// queue it serves it on, with the seat's own description and schema, in NATS's services format.
func TestTheControllerAnnouncesTheVerbsItServes(t *testing.T) {
handlers, _, err := seatToolHandlers()
if err != nil {
t.Fatal(err)
}
info := seatAnnouncement(handlers)
if info.Name != catalogue.ControllerSeatName || info.ID == "" || info.Version == "" {
t.Fatalf("the service is not named for the seat: %+v", info.ServiceIdentity)
}
if len(info.Endpoints) != len(handlers) {
t.Fatalf("%d endpoints announced for %d verbs served", len(info.Endpoints), len(handlers))
}
for _, e := range info.Endpoints {
verb := e.Metadata["tool"]
if _, served := handlers[verb]; !served || e.Name != catalogue.ControllerSeatName+"__"+verb {
t.Errorf("%s (%s) is announced and not served under that name", e.Name, verb)
}
if e.Metadata["kind"] != "seat" || e.Metadata["seat"] != catalogue.ControllerSeatName {
t.Errorf("%s is not announced as the seat's verb: %v", e.Name, e.Metadata)
}
if e.Subject != link.SeatToolSubject(catalogue.ControllerSeatName, verb) || e.QueueGroup != "seat."+catalogue.ControllerSeatName {
t.Errorf("%s is announced on %s/%s, not where it is served", e.Name, e.Subject, e.QueueGroup)
}
if e.Metadata["description"] == "" || e.Metadata["schema"] == "" || e.Metadata["scope"] != "mesh" {
t.Errorf("%s is announced without its description, schema or scope: %v", e.Name, e.Metadata)
}
}
}
+1 -61
View File
@@ -10,7 +10,6 @@ import (
"io"
"os"
"strings"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
@@ -39,8 +38,6 @@ func secretCommand(ctx context.Context, args []string) error {
}
switch args[0] {
case "accept":
case "rotate":
return secretRotate(ctx, args[1:])
case "recover":
return secretRecover(ctx, args[1:])
case "export":
@@ -104,8 +101,7 @@ func secretCommand(ctx context.Context, args []string) error {
return nil
}
const secretUsage = "secret rotate <node> <module> <name>\n" +
"secret accept <node> <module> <name> [--from <file>] [--provider <node> [--local <name>]]\n" +
const secretUsage = "secret accept <node> <module> <name> [--from <file>] [--provider <node> [--local <name>]]\n" +
"secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>] [--provider <node>]\n" +
"secret export [--out <file>]"
@@ -363,59 +359,3 @@ func valueFor(node, module, name, from string) (string, error) {
return line, nil
}
}
// secretRotate makes a module's own secret anew and sends the machine, so the module starts again on
// the new value (novox/hq ADR 0114, issue 180). A pair credential rotates with `rotate <provision>`;
// this is the secret with one party. Said in the log with who asked and when, never the value.
func secretRotate(ctx context.Context, args []string) error {
rest, _ := split(args)
if len(rest) != 3 {
return errors.New(secretUsage)
}
node, module, name := rest[0], rest[1], rest[2]
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
if err := open.inventory.RotateModuleSecret(ctx, node, module, name); err != nil {
var refused inventory.ErrNotRotatable
if errors.As(err, &refused) {
return fmt.Errorf("not rotated: %s", refused.Why)
}
return err
}
fmt.Printf("rotated %q of %s on %s at %s, asked by %s; the value is sealed and not shown\n",
name, module, node, time.Now().UTC().Format(time.RFC3339), whoAsked())
// A shared credential (ADR 0158) has as many holders as the provision has consumers, and all
// of them are sent in one act, so no machine is left reading a value the provider no longer takes.
machines, err := open.inventory.SharedHolders(ctx, node, module, name)
if err != nil {
return err
}
if len(machines) == 0 {
machines = []string{node}
}
if len(machines) == 1 {
fmt.Printf("sending %s, so %s starts again on the new value:\n", node, module)
} else {
fmt.Printf("shared with every consumer; sending %s together:\n", strings.Join(machines, ", "))
}
if err := sendTo(ctx, open, machines); err != nil {
return fmt.Errorf("%w\n\nThe new value is sealed and not yet delivered; what runs keeps the old "+
"one until the machines next apply. Fix the cause and run `push --behind`", err)
}
return nil
}
// whoAsked names the caller for the log: the account the command runs as, which for a tool call
// through the console is the mesh's own.
func whoAsked() string {
if u := os.Getenv("SUDO_USER"); u != "" {
return u
}
if u := os.Getenv("USER"); u != "" {
return u
}
return "the mesh"
}
-24
View File
@@ -18,27 +18,9 @@ import (
// make a machine look out of date for ever, or send something `plan` never showed.
type sendable struct {
Resources []map[string]any
// Sequence orders this send against every other to the same node: one higher each time, taken
// under the node's hold just before the body is made (novox/hq 04-ISSUES/107). Zero is not sent
// at all, which a host reads as "no order claimed" — the shape of every declaration before this.
Sequence int64
// Adoption is nil for a converged node, and then the body is byte for byte what it was before
// adoption existed: an older host parses the envelope strictly and would refuse the key.
Adoption *adoptionEnvelope
// Received and Mesh are not sent in the declaration. They are what this machine's memberships
// are issued with on the bus (novox/hq ADR 0167): each module's received contributions, from
// the same composition as its received files, and every machine's private-network address.
Received map[string]map[string][]catalogue.Contribution
Mesh []string
// LeftOut is every module of the machine's set left out of this declaration because a stored
// setting cannot compose with its definition (novox/hq ADR 0163, rule 6), sorted. The host
// keeps that module's held things and touches none of its containers; a machine is told
// everything or nothing about what it IS told, and what it is not told is said. Absent from
// the body when empty, so a declaration that leaves nothing out is byte for byte what it was.
LeftOut []string
// leftOutWhy is why each was, for push and plan to say; never on the wire.
leftOutWhy map[string]string
}
// adoptionEnvelope is what an adopted node is told about its mode. Taken is every module taken on
@@ -56,12 +38,6 @@ func (s sendable) Body() ([]byte, error) {
if s.Adoption != nil {
envelope["adoption"] = s.Adoption
}
if s.Sequence > 0 {
envelope["sequence"] = s.Sequence
}
if len(s.LeftOut) > 0 {
envelope["left_out"] = s.LeftOut
}
// An empty declaration is deliberate here — the node owns nothing the mesh put there
// (novox/hq issue 127) — and the host refuses an empty body unless it is told the emptiness
// is meant, so a truncated or mis-composed body is never mistaken for "own nothing".
+1 -65
View File
@@ -47,12 +47,7 @@ func composed(t *testing.T, open *stores, node string) sendable {
// aMesh's laptop with the private network taken off it, so nothing in the declaration is random:
// what changes this string is a change to what a converged machine is sent, which is the thing an
// older host would refuse.
//
// Re-captured 2026-10-01 (novox/hq issue 177): c978aa7 took `hosts` off every container — a
// machine's own resolver knows the mesh's names now — and left this string carrying it, so the
// guard failed for a day and nothing ran it. A field an older host never sees is the one change
// this guard permits; a field it would refuse is the one it exists to catch.
const convergedBefore = `{"declaration":1,"resources":[{"content":"hello","id":"hello-web.page","path":"/var/lib/hello-web/index.html","type":"file"},{"id":"hello-web.server","image":"registry.example/hello@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","name":"hello-web","type":"container"},{"id":"hello-web.served","path":"/var/lib/hello-web","type":"directory"}]}`
const convergedBefore = `{"declaration":1,"resources":[{"content":"hello","id":"hello-web.page","path":"/var/lib/hello-web/index.html","type":"file"},{"hosts":["anchor.internal:10.77.0.1"],"id":"hello-web.server","image":"registry.example/hello@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","name":"hello-web","type":"container"},{"id":"hello-web.served","path":"/var/lib/hello-web","type":"directory"}]}`
func TestAConvergedDeclarationIsByteForByteWhatItWas(t *testing.T) {
open := aMesh(t)
@@ -382,62 +377,3 @@ func TestAnEmptyDeclarationSaysOwnsNothing(t *testing.T) {
t.Fatalf("a non-empty declaration must not mark owns_nothing; got %v", env)
}
}
// A setting is judged where it is stored, and an impossible one costs a module, not a machine
// (novox/hq ADR 0163, rule 6): stored while it composed, a setting whose definition then moved from
// under it leaves that module out of the declaration — said in the envelope, so the host keeps the
// module's things — and the machine is told everything else.
func TestADefinitionMovingUnderAStoredSettingLeavesThatModuleOutNotTheMachine(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
web := helloWeb()
web.Resources[1]["ports"] = []any{"8080"}
register(t, open, web)
register(t, open, catalogue.Manifest{Module: "notes", Version: "1",
Resources: []map[string]any{{"id": "conf", "type": "file", "path": "/etc/notes.conf", "content": "x"}}})
for _, m := range []string{"hello-web", "notes"} {
if _, err := assign(ctx, open, "laptop", m); err != nil {
t.Fatal(err)
}
}
// Judged where it is stored: a port the module does not publish is refused by name.
err := open.inventory.SetSettings(ctx, "laptop", "hello-web",
map[string]any{catalogue.PortsSetting: map[string]any{"9999": 10000}})
if err == nil || !strings.Contains(err.Error(), "hello-web on laptop") || !strings.Contains(err.Error(), "9999") {
t.Fatalf("an impossible setting was stored: %v", err)
}
if err := open.inventory.SetSettings(ctx, "laptop", "hello-web",
map[string]any{catalogue.PortsSetting: map[string]any{"8080": 10000}}); err != nil {
t.Fatal(err)
}
if declared := composed(t, open, "laptop"); len(declared.LeftOut) != 0 {
t.Fatalf("a setting that composes left a module out: %v", declared.LeftOut)
}
// The definition moves: the container publishes another port now.
web.Version = "2"
web.Resources[1]["ports"] = []any{"9090"}
register(t, open, web)
declared := composed(t, open, "laptop")
if len(declared.LeftOut) != 1 || declared.LeftOut[0] != "hello-web" {
t.Fatalf("hello-web is not left out: %v", declared.LeftOut)
}
if !strings.Contains(declared.leftOutWhy["hello-web"], "no container of its publishes 8080") {
t.Fatalf("why it was left out is not said: %v", declared.leftOutWhy)
}
if hasID(declared.Resources, "hello-web.server") || !hasID(declared.Resources, "notes.conf") {
t.Fatalf("the machine was not told everything else: %v", declared.Resources)
}
body, err := declared.Body()
if err != nil {
t.Fatal(err)
}
var env map[string]any
if err := json.Unmarshal(body, &env); err != nil {
t.Fatal(err)
}
left, _ := env["left_out"].([]any)
if len(left) != 1 || left[0] != "hello-web" {
t.Fatalf("the envelope does not say what was left out: %v", env)
}
}
-77
View File
@@ -1,77 +0,0 @@
package main
import (
"encoding/json"
"testing"
)
// A declaration's only identity was the digest of its bytes; the controller already held a per-node
// lock and recorded each send, so the order existed and was thrown away at the wire (novox/hq
// 04-ISSUES/107).
func TestASendCarriesItsNumberInsideTheSignedBytes(t *testing.T) {
body, err := sendable{Resources: []map[string]any{{"id": "x", "type": "file"}}, Sequence: 7}.Body()
if err != nil {
t.Fatal(err)
}
var env map[string]any
if err := json.Unmarshal(body, &env); err != nil {
t.Fatal(err)
}
if got, _ := env["sequence"].(float64); got != 7 {
t.Fatalf("the body carries sequence %v, wanted 7", env["sequence"])
}
}
func TestAnUnnumberedSendIsByteForByteWhatItWasBefore(t *testing.T) {
// Zero is not sent at all. A host reads absence as "no order claimed" — the shape of every
// declaration before this — so an older host, or the read-only comparison against a machine
// sent nothing since sends were numbered, sees exactly the bytes it always saw.
body, err := sendable{Resources: []map[string]any{{"id": "x", "type": "file"}}}.Body()
if err != nil {
t.Fatal(err)
}
var env map[string]any
if err := json.Unmarshal(body, &env); err != nil {
t.Fatal(err)
}
if _, present := env["sequence"]; present {
t.Fatalf("a send numbered zero put a sequence on the wire: %s", body)
}
}
func TestEachSendToANodeIsOneHigherAndReadable(t *testing.T) {
open := aMesh(t)
record, err := open.inventory.NodeByName(t.Context(), "anchor")
if err != nil {
t.Fatal(err)
}
// Sent nothing since numbering existed: what it would be sent is composed with zero, which is
// not on the wire, which is what it was actually sent.
if n, err := open.inventory.Sequence(t.Context(), record.ID); err != nil || n != 0 {
t.Fatalf("a fresh node reads sequence %d, %v", n, err)
}
first, err := open.inventory.NextSequence(t.Context(), record.ID)
if err != nil {
t.Fatal(err)
}
second, err := open.inventory.NextSequence(t.Context(), record.ID)
if err != nil {
t.Fatal(err)
}
if first != 1 || second != 2 {
t.Fatalf("two sends were numbered %d and %d", first, second)
}
// And the read path sees the last one taken, so the comparison composes what was sent.
if n, err := open.inventory.Sequence(t.Context(), record.ID); err != nil || n != 2 {
t.Fatalf("after two sends the node reads sequence %d, %v", n, err)
}
// Another node counts on its own.
other, err := open.inventory.NodeByName(t.Context(), "laptop")
if err != nil {
t.Fatal(err)
}
if n, err := open.inventory.NextSequence(t.Context(), other.ID); err != nil || n != 1 {
t.Fatalf("a second node's first send was numbered %d, %v", n, err)
}
}
+4 -39
View File
@@ -82,16 +82,6 @@ func cloneFrom(ctx context.Context, source buildSource) (string, error) {
// serves no scheme or port has nothing to compose from — a default port here would be the forge's
// address guessed, which is the thing this exists to stop.
func clonedFromSeat(world catalogue.World, seatName, repository string) (string, error) {
base, err := seatBase(world, seatName)
if err != nil {
return "", err
}
path := strings.TrimSuffix(strings.Trim(repository, "/"), ".git")
return fmt.Sprintf("%s/%s.git", base, path), nil
}
// seatBase is `scheme://host:port` of a seat's holder as the mesh reaches it, for cloning.
func seatBase(world catalogue.World, seatName string) (string, error) {
seat, known := catalogue.SeatNamed(seatName)
if !known || seat.Delivers == "" {
return "", fmt.Errorf("%q is not a seat a repository can live on", seatName)
@@ -104,9 +94,9 @@ func seatBase(world catalogue.World, seatName string) (string, error) {
}
}
if holder == nil {
return "", fmt.Errorf("nobody holds the %s seat, so nothing can be cloned from this mesh's "+
return "", fmt.Errorf("nobody holds the %s seat, so %s cannot be cloned from this mesh's "+
"forge — assign a module that claims it, or build from the repository's URL without --self",
seat.Name)
seat.Name, repository)
}
var provider *catalogue.Provider
for i, p := range world.Offered[seat.Delivers] {
@@ -128,33 +118,8 @@ func seatBase(world catalogue.World, seatName string) (string, error) {
return "", fmt.Errorf("%s on %s holds the %s seat and does not serve a scheme and a port for %q",
holder.Module, holder.Node, seat.Name, seat.Delivers)
}
return fmt.Sprintf("%s://%s:%s", scheme, provider.At, port), nil
}
// seatBases is the clone base of every seat a recipe's context may name, for a build request
// (novox/hq ADR 0155). A seat nobody holds is left out rather than refused here: the build may not
// name it at all, and if it does the builder refuses with the seat's name.
func seatBases(ctx context.Context) map[string]string {
open, err := openStores(ctx)
if err != nil {
return nil
}
defer open.Close()
shelf, err := open.inventory.Catalogue(ctx)
if err != nil {
return nil
}
world, err := theRestOfTheMesh(ctx, open.inventory, shelf, "")
if err != nil {
return nil
}
bases := map[string]string{}
for _, seatName := range []string{gitSeat} {
if base, err := seatBase(world, seatName); err == nil {
bases[seatName] = base
}
}
return bases
path := strings.TrimSuffix(strings.Trim(repository, "/"), ".git")
return fmt.Sprintf("%s://%s:%s/%s.git", scheme, provider.At, port, path), nil
}
// servedPort is a served port as text, however the manifest and the node's settings carried it.
+6 -253
View File
@@ -46,21 +46,15 @@ func statusCommand(ctx context.Context, args []string) error {
return err
}
defer open.Close()
return statusFor(ctx, open, *asJSON)
}
// statusFor asks and answers, against stores somebody else opened.
//
// Split from the command so what it prints can be read by a test. The sentence it prints when nothing
// is wrong has been acted on and been misleading (novox/hq 04-ISSUES/145, 125), which makes its exact
// words the thing worth holding still.
func statusFor(ctx context.Context, open *stores, asJSON bool) error {
asked, err := theThreeQuestions(ctx, open)
if err != nil {
return err
}
wrong, nodes, quiet := asked.wrong, asked.nodes, asked.quiet
behind, sources := asked.behind, asked.sources
if asJSON {
if *asJSON {
body, err := statusAsJSON(asked)
if err != nil {
return err
@@ -68,18 +62,6 @@ func statusFor(ctx context.Context, open *stores, asJSON bool) error {
fmt.Println(string(body))
return nil
}
return printStatus(asked)
}
// printStatus is the words, separated from the questions.
//
// **Its exact sentences have been acted on and been misleading twice** — a held module reading as a
// machine doing what it was told (novox/hq 04-ISSUES/125), and "all doing what they were told" being
// true of a mesh in which no module could reach another (04-ISSUES/145). So they are written where a
// test can read them without a store, a bus or a machine.
func printStatus(asked answers) error {
wrong, nodes, quiet := asked.wrong, asked.nodes, asked.quiet
behind, sources := asked.behind, asked.sources
if len(asked.refused) > 0 {
// First, above everything else. A machine that cannot be worked out is not running an old
@@ -138,18 +120,6 @@ func printStatus(asked answers) error {
len(quiet), strings.Join(said, "\n "))
}
if open, late := openPlans(asked.plans); len(open) > 0 {
fmt.Printf("%d plan(s) open", len(open))
if late > 0 {
fmt.Printf(", %d waiting past %s", late, planWaitBound)
}
fmt.Println(":")
for _, p := range open {
fmt.Printf(" %s\n", planLine(p, time.Now()))
}
fmt.Println()
}
if len(behind) > 0 {
var names []string
for m := range behind {
@@ -201,87 +171,6 @@ func printStatus(asked answers) error {
fmt.Printf("\n `push --behind` sends them\n\n")
}
if split := hostSplit(nodes); len(split) > 1 {
// **Before a declaration gains a field, every machine has to understand it** (novox/hq
// 04-ISSUES/087). A host refuses a declaration carrying a field it does not know, and refuses
// it whole, so every new field is a flag day: hosts first, then the controller. The mesh had
// no record of which host any machine ran, so that order was kept by somebody remembering it.
//
// **Disagreement, and deliberately not "behind".** A host reports its version as a commit, and
// commits have no order — the first version of this said "N machines run an older host" and
// named the three that were newer, because it compared two hashes as strings. What the mesh
// can say truthfully is that the machines do not all run the same host, and which machines
// hold which. Ordering needs a version that is ordered, and that is the host's to report.
versions := make([]string, 0, len(split))
for v := range split {
versions = append(versions, v)
}
sort.Strings(versions)
fmt.Printf("%d machine(s) do not all run the same host:\n", len(nodes))
for _, v := range versions {
sort.Strings(split[v])
fmt.Printf(" %-12s %s\n", v, strings.Join(split[v], ", "))
}
fmt.Printf("\n a host refuses a declaration carrying a field it does not know, whole — so the\n" +
" mesh may send only what every one of these understands. Which of them is newer is\n" +
" not readable from a commit; that needs a version the host reports as ordered\n\n")
}
if len(asked.filtered) > 0 {
// A converged machine is filtered by the mesh alone, and the mesh says truthfully which
// (novox/hq ADR 0168). One that is not — a predecessor's chain still refusing, a found
// firewall in force again — is said here, and is not well.
machines := make([]string, 0, len(asked.filtered))
for name := range asked.filtered {
machines = append(machines, name)
}
sort.Strings(machines)
fmt.Printf("%d converged machine(s) are not filtered by the mesh alone:\n", len(machines))
for _, name := range machines {
f := asked.filtered[name]
if fw := f.FoundFirewall; fw != nil && fw.Active {
fmt.Printf(" %-12s the found firewall (%s) is in force again; the next apply retires it\n", name, fw.Kind)
}
for _, x := range f.Others() {
fmt.Printf(" %-12s %s (%s): %s\n", name, x.Where, x.Owner, x.Refuses)
}
}
fmt.Printf("\n the mesh wrote none of these and removes none; `node show <node>` lists every filter with its owner\n\n")
}
if len(asked.untaken) > 0 {
// **Before the adopted line, and it breaks "all well".** An adopted machine is a state
// somebody chose and can leave alone; a module assigned to one and never taken is work
// outstanding that reads exactly like work finished. That reading is what stopped a
// predecessor's proxy on the strength of four green surfaces (novox/hq 04-ISSUES/125).
machines := make([]string, 0, len(asked.untaken))
for name := range asked.untaken {
machines = append(machines, name)
}
sort.Strings(machines)
total := 0
for _, held := range asked.untaken {
for _, n := range held {
total += n
}
}
fmt.Printf("%d resource(s) are held as found, because their module was assigned and never "+
"taken — so it is running none of what it declares:\n", total)
for _, name := range machines {
modules := make([]string, 0, len(asked.untaken[name]))
for m := range asked.untaken[name] {
modules = append(modules, m)
}
sort.Strings(modules)
parts := make([]string, 0, len(modules))
for _, m := range modules {
parts = append(parts, fmt.Sprintf("%s (%d)", m, asked.untaken[name][m]))
}
fmt.Printf(" %-12s %s\n", name, strings.Join(parts, ", "))
}
fmt.Printf("\n `take <node> <module>` compares what runs against what it declares, and runs it\n\n")
}
if adopted := adoptedNodes(nodes); len(adopted) > 0 {
// Said, because nothing forces the flip: a node left adopted is visible here rather than
// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
@@ -289,23 +178,12 @@ func printStatus(asked answers) error {
fmt.Printf("\n `converge <node>` previews the flip\n\n")
}
if asked.well() {
if len(wrong) == 0 && len(quiet) == 0 && len(behind) == 0 && len(asked.waiting) == 0 &&
len(asked.refused) == 0 && asked.network == "" {
// Said plainly. "Nothing to report" and "nothing was checked" must never look the same,
// and getting here means every question was asked and answered.
fmt.Printf("%d machine(s), all doing what they were told, all heard from, running what "+
"the mesh would send them, and every module current with its source\n", len(nodes))
// **And what that sentence does not cover**, because for eleven hours it was true of a mesh
// in which no module could reach another (novox/hq 04-ISSUES/145). Every question above is
// about the relationship between the mesh and a machine — applied what it was sent, matches
// what would be sent, built from what the source has. None of them asks whether a module can
// reach what it requires, and the mesh composes every one of those grants itself.
//
// Said here rather than left to be inferred. A reader who acts on the line above is acting on
// "the machines are as the mesh described them", and the distance between that and "it works"
// is where the eleven hours went.
fmt.Printf("\n That is the mesh and the machines agreeing. Nothing here dials a provision:\n" +
" no grant the mesh composed has been tested, so a module unable to reach what it\n" +
" requires would not appear above (04-ISSUES/145)\n")
}
return nil
}
@@ -323,10 +201,7 @@ func firstLine(s string) string {
// A type of its own rather than a method on the enrolment, because they are unrelated things
// arriving on one queue and an implementation of one should not have to say anything about the
// other.
type builds struct {
inv *inventory.Inventory
open *stores
}
type builds struct{ inv *inventory.Inventory }
// theThreeQuestions reads what anything answering "is the mesh alright" needs.
//
@@ -372,24 +247,6 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
if err != nil {
return answers{}, err
}
// And what each machine is holding rather than running, by the module that would run it. Read
// from what the machine itself last reported, not from what take-time computed: the machine is
// the only thing that knows what it found (novox/hq 04-ISSUES/125).
out.untaken, err = untakenModules(ctx, inv, out.nodes)
if err != nil {
return answers{}, err
}
// And which converged machines something other than the mesh filters (novox/hq ADR 0168), as
// each last reported — the account that was missing when a predecessor's chain refused what the
// mesh declared open for eleven hours (04-ISSUES/144, 145).
out.filtered, err = filteredMachines(ctx, inv, out.nodes)
if err != nil {
return answers{}, err
}
out.plans, err = inv.RecentPlans(ctx, 5)
if err != nil {
return answers{}, err
}
// And which machines are not running what the mesh would send them. The same question as a
// module being behind its source, one level down: that one says the catalogue is out of date,
@@ -424,107 +281,3 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
}
return out, nil
}
// untakenModules is, per machine, each module whose resources that machine is holding as found, and
// how many.
//
// **The machine's own account, not the mesh's.** An adopted node decides at apply time what it found
// and reports it; the mesh's take-time listing is a different thing and was the one this command used
// to have, which is why a module assigned after the listing showed nothing at all
// (novox/hq 04-ISSUES/125).
//
// A machine that reports no holds contributes nothing, so a converged mesh answers an empty map and
// the caller prints nothing.
// filteredMachines is every converged machine not filtered by the mesh alone, with what it last said
// filters it (novox/hq ADR 0168). An adopted machine keeps its found firewall by design and is not
// counted; a machine that has not said is not said to be filtered by anything.
func filteredMachines(ctx context.Context, inv *inventory.Inventory, nodes []inventory.Node) (
map[string]inventory.Filtering, error) {
out := map[string]inventory.Filtering{}
for _, n := range nodes {
if n.Adopted {
continue
}
f, err := inv.FilteringOf(ctx, n.Name)
if err != nil {
return nil, fmt.Errorf("what filters %s cannot be read: %w", n.Name, err)
}
if len(f.Filters) == 0 && f.FoundFirewall == nil {
continue
}
if !f.Alone() {
out[n.Name] = f
}
}
return out, nil
}
func untakenModules(ctx context.Context, inv *inventory.Inventory, nodes []inventory.Node) (
map[string]map[string]int, error) {
out := map[string]map[string]int{}
for _, n := range nodes {
said, err := inv.AdoptionOf(ctx, n.Name)
if err != nil {
// A machine whose record cannot be read is not a machine holding nothing. Said, because
// answering "nothing held" from a failed read is the shape this whole issue is about.
return nil, fmt.Errorf("what %s is holding cannot be read: %w", n.Name, err)
}
for _, h := range said.Held {
if h.Module == "" {
continue // a hold the mesh cannot attribute to a module has nothing to take
}
if out[n.Name] == nil {
out[n.Name] = map[string]int{}
}
out[n.Name][h.Module]++
}
}
return out, nil
}
// well is whether every question this command asks came back with nothing to say.
//
// Named, and in one place, because it is the sentence an operator acts on and it has been wrong
// twice. It is deliberately NOT "nothing is broken": a machine holding what it found is not broken
// and is not doing what it was told either.
//
// **A hold suppresses it; being adopted does not.** Adopted is a mode somebody chose and can leave
// alone. A module assigned to a machine and never taken is a half-finished action with nothing left
// to finish it — it runs none of what it declares, and "all doing what they were told" was true and
// read as success for the whole of the edge cut-over outage (novox/hq 04-ISSUES/125).
func (a answers) well() bool {
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 &&
len(a.filtered) == 0
}
// hostSplit is which machines report which host version, for every version more than one machine
// could disagree about.
//
// **It does not say which is newer, because it cannot.** A host reports its version as a commit, and
// commits have no order. The first version of this returned "the machines behind the newest" by
// comparing versions as strings, and on the live mesh it named the three machines running the NEWER
// host as the ones behind — an arbitrary lexicographic result presented as a fact
// (novox/hq 04-ISSUES/087). A report that confidently says the opposite of the truth is worse than one
// that says less, which is the whole subject of 04-ISSUES/145.
//
// So this answers what is checkable: who runs what. The reader sees the split and the mesh claims no
// ordering. Ordering wants an ordered version, and that is the host's to report rather than this
// function's to infer.
//
// Machines that have not reported a version are left out entirely: they are not a version, and
// counting them as one would invent a disagreement. `node show` says per machine that it has not said.
func hostSplit(nodes []inventory.Node) map[string][]string {
out := map[string][]string{}
for _, n := range nodes {
if n.HostVersion == "" {
continue
}
out[n.HostVersion] = append(out[n.HostVersion], n.Name)
}
if len(out) < 2 {
return nil // one version, or none reported: nothing to disagree about
}
return out
}
-143
View File
@@ -1,143 +0,0 @@
package main
import (
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
)
// What the forge's take compares, as a machine would report it.
func aForgeComparison() comparison {
return comparison{reported: inventory.Adoption{
Firewall: "ufw",
Held: []inventory.Held{
{ID: "forge.server", Module: "forge", Kind: "container", Target: "forge", Facts: map[string]any{
"image": "forge:1.27.3", "image_created": "2026-09-17T10:00:00Z",
"declared_image": "forge:1.22.6", "declared_image_created": "2026-08-20T10:00:00Z", "downgrade": true,
"networks": map[string]any{"predecessor_default": []any{"office", "db"}},
"ports": []any{"3000/tcp>0.0.0.0:3000"}, "declared_ports": []any{"3000:3000"},
}},
{ID: "forge.config", Module: "forge", Kind: "file", Target: "/etc/forge/app.ini", Kept: "/var/lib/mesh/kept/app.ini",
Facts: map[string]any{"differs": true, "difference": []any{"- private scope: local", "+ upstream: public"}}},
{ID: "other.server", Module: "other", Kind: "container", Target: "other"},
},
Reachable: []inventory.Reach{
{Protocol: "tcp", Address: "0.0.0.0", Port: 3000, By: "forge", Published: true, ContainerPort: 3000},
{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"},
},
}}
}
// A take is a comparison (novox/hq ADR 0163): the preview puts what runs beside what the module
// declares, and an older image or a differing file refuses unless named.
func TestATakePreviewsTheComparisonAndRefusesWhatIsNotNamed(t *testing.T) {
c := aForgeComparison()
preview, refusals, saw := comparisonOf("forge", c, takeOptions{})
for _, want := range []string{"runs forge:1.27.3 (made 2026-09-17)", "declares forge:1.22.6 (made 2026-08-20)", "DOWNGRADE",
"on the network predecessor_default with office, db", "will not once it moves to the module's own network",
"publishes 3000/tcp>0.0.0.0:3000; the module declares 3000:3000",
// How far the port reaches now, as the machine reported it (rule 1).
"reachable now at 0.0.0.0:3000 (tcp, container port 3000), behind the found firewall (ufw)",
"- private scope: local", "original kept at /var/lib/mesh/kept/app.ini"} {
if !strings.Contains(preview, want) {
t.Errorf("the preview lacks %q:\n%s", want, preview)
}
}
if strings.Contains(preview, "other") {
t.Errorf("another module's held things are in the preview:\n%s", preview)
}
if len(refusals) != 2 || !strings.Contains(refusals[0], "--downgrade") || !strings.Contains(refusals[1], "--replace /etc/forge/app.ini") {
t.Fatalf("the downgrade and the differing file refuse, each naming its override: %v", refusals)
}
if len(saw) != 12 {
t.Fatalf("the preview's digest is %q", saw)
}
// Named, they pass.
if _, refusals, _ := comparisonOf("forge", c, takeOptions{Downgrade: true, Replace: map[string]bool{"/etc/forge/app.ini": true}}); len(refusals) != 0 {
t.Fatalf("named differences still refused: %v", refusals)
}
if _, refusals, _ := comparisonOf("forge", c, takeOptions{Downgrade: true, Replace: map[string]bool{"*": true}}); len(refusals) != 0 {
t.Fatalf("replace * did not cover the file: %v", refusals)
}
// A held thing with no facts yet — a host older than this — refuses nothing and says what it can.
if preview, refusals, _ := comparisonOf("other", c, takeOptions{}); len(refusals) != 0 || !strings.Contains(preview, "container other") {
t.Fatalf("a factless hold: %q %v", preview, refusals)
}
// The digest is of what the preview says: a fact changing changes it.
c.reported.Held[0].Facts["image"] = "forge:1.27.4"
if _, _, again := comparisonOf("forge", c, takeOptions{}); again == saw {
t.Fatal("the found image changed and the digest did not")
}
}
// A secret the mesh minted for a service whose data was found refuses: the running service already
// has a value (rule 2). Accepted, it is carried in; `--mint` says the service shall take the new one.
func TestAMintedSecretForFoundDataRefusesUnlessAcceptedOrMinted(t *testing.T) {
c := aForgeComparison()
c.secrets = []inventory.SecretState{
{Name: "admin", Origin: inventory.OriginMade},
{Name: "postgres-database", Origin: inventory.OriginMade, Provider: "anchor"},
{Name: "broker", Origin: inventory.OriginAccepted},
}
preview, refusals, _ := comparisonOf("forge", c, takeOptions{Downgrade: true, Replace: map[string]bool{"*": true}})
for _, want := range []string{
"own secret admin: MINTED by the mesh and not accepted",
"secret from anchor postgres-database: MINTED by the mesh and not accepted",
"own secret broker: accepted from a person, carried in as it is",
} {
if !strings.Contains(preview, want) {
t.Errorf("the preview lacks %q:\n%s", want, preview)
}
}
if len(refusals) != 2 {
t.Fatalf("two minted secrets refuse: %v", refusals)
}
if !strings.Contains(refusals[0], "`secret accept <node> forge admin`") || !strings.Contains(refusals[0], "`--mint admin`") {
t.Errorf("the own secret's refusal names accepting it and minting it: %s", refusals[0])
}
if !strings.Contains(refusals[1], "`secret accept <node> forge postgres-database --provider anchor`") {
t.Errorf("the required secret's refusal names its provider: %s", refusals[1])
}
preview, refusals, _ = comparisonOf("forge", c, takeOptions{Downgrade: true, Replace: map[string]bool{"*": true},
Mint: map[string]bool{"admin": true, "postgres-database": true}})
if len(refusals) != 0 || !strings.Contains(preview, "admin: minted by the mesh; the service takes the new value, as --mint said") {
t.Fatalf("--mint did not pass the minted secrets: %v\n%s", refusals, preview)
}
// With no found data — only a file held — the service has no value of its own, and a minted
// secret is simply said.
c.reported.Held = c.reported.Held[1:2]
if _, refusals, _ := comparisonOf("forge", c, takeOptions{Replace: map[string]bool{"*": true}}); len(refusals) != 0 {
t.Fatalf("a minted secret refused with no data found: %v", refusals)
}
}
// A found network a per-machine setting keeps is named in the preview (rule 4), and the module's
// settings are said with where each came from, composed or not (rules 1 and 6).
func TestTheKeptNetworkAndTheSettingsAreInThePreview(t *testing.T) {
c := aForgeComparison()
c.keeps = map[string][]string{"forge.server": {"predecessor_default"}}
c.layers = []catalogue.Layer{
{From: catalogue.MeshWideLayer, Values: map[string]any{"site": "x"}},
{From: "anchor", Values: map[string]any{catalogue.NetworksSetting: map[string]any{"server": []any{"predecessor_default"}}}},
}
preview, _, _ := comparisonOf("forge", c, takeOptions{Downgrade: true, Replace: map[string]bool{"*": true}})
for _, want := range []string{
"on the network predecessor_default with office, db — kept by this machine's setting, so they still reach it by name once taken",
"settings from the mesh: site",
"settings from anchor: networks",
} {
if !strings.Contains(preview, want) {
t.Errorf("the preview lacks %q:\n%s", want, preview)
}
}
if strings.Contains(preview, "will not once it moves") {
t.Errorf("a kept network is still said to be lost:\n%s", preview)
}
c.settingsRefused = "forge: ports is a { port: machine-port } map"
preview, _, _ = comparisonOf("forge", c, takeOptions{Downgrade: true, Replace: map[string]bool{"*": true}})
if !strings.Contains(preview, "SETTINGS DO NOT COMPOSE with the module's definition, so the push leaves it out: forge: ports") {
t.Errorf("settings that cannot compose are not said:\n%s", preview)
}
}
-122
View File
@@ -1,122 +0,0 @@
package main
import (
"encoding/json"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/inventory"
)
// A module assigned to an adopted machine and never taken runs none of what it declares, and every
// surface called that success — a push reporting sent, a journal reporting applied, status reporting
// a machine doing what it was told (novox/hq 04-ISSUES/125). The holds were only ever in the
// machine's own state file.
// heldOn makes a machine report that it is holding resources for a module, the way an adopted node
// does after an apply.
func heldOn(t *testing.T, open *stores, node, module string, ids ...string) {
t.Helper()
record, err := open.inventory.NodeByName(t.Context(), node)
if err != nil {
t.Fatal(err)
}
held := make([]inventory.Held, 0, len(ids))
for _, id := range ids {
held = append(held, inventory.Held{ID: id, Module: module, Kind: "container", Target: id})
}
if err := open.inventory.RecordAdoption(t.Context(), record.ID, held, "ufw", nil); err != nil {
t.Fatal(err)
}
}
func TestStatusNamesAModuleHeldBecauseNothingTookIt(t *testing.T) {
open := aMesh(t)
heldOn(t, open, "anchor", "route-proxy", "ca", "certs", "server")
asked, err := theThreeQuestions(t.Context(), open)
if err != nil {
t.Fatal(err)
}
if got := asked.untaken["anchor"]["route-proxy"]; got != 3 {
t.Fatalf("status counted %d resources held for route-proxy, wanted 3", got)
}
}
func TestAHeldModuleStopsTheMeshReadingAsWell(t *testing.T) {
// The whole of the fault. "all doing what they were told" was true throughout the outage, and
// true is not the same as safe to act on: the machine was doing what it was told, and what it
// was told had not started. Asserted against the production condition, not a copy of it.
quiet := answers{}
if !quiet.well() {
t.Fatal("a mesh with nothing to say does not read as well, so nothing below means anything")
}
holding := answers{untaken: map[string]map[string]int{"anchor": {"route-proxy": 3}}}
if holding.well() {
t.Fatal("a machine holding a module's resources still reads as doing what it was told, " +
"which is the sentence that cost every public name on the machine")
}
// And being adopted does not suppress it: that is a mode somebody chose, not work outstanding.
// Kept as an assertion so the difference between the two is deliberate rather than incidental.
if !quiet.well() {
t.Fatal("the well condition is not stable")
}
}
func TestAHeldModuleIsFoundFromWhatTheMachineReported(t *testing.T) {
// End to end through the store, so the condition above is reached by real data and not only by
// a constructed value: the machine reports, the mesh records, status asks.
open := aMesh(t)
heldOn(t, open, "anchor", "route-proxy", "ca", "server")
asked, err := theThreeQuestions(t.Context(), open)
if err != nil {
t.Fatal(err)
}
if len(asked.untaken) == 0 {
t.Fatal("what the machine reported holding did not reach status")
}
if asked.well() {
t.Fatal("a mesh whose machine reported holds reads as well")
}
}
func TestTheJSONStatusCarriesWhatIsHeldAndForWhichModule(t *testing.T) {
open := aMesh(t)
heldOn(t, open, "anchor", "route-proxy", "ca", "certs")
asked, err := theThreeQuestions(t.Context(), open)
if err != nil {
t.Fatal(err)
}
body, err := statusAsJSON(asked)
if err != nil {
t.Fatal(err)
}
var doc struct {
Untaken []struct {
Node string `json:"node"`
Module string `json:"module"`
Held int `json:"held"`
} `json:"untaken"`
}
if err := json.Unmarshal(body, &doc); err != nil {
t.Fatal(err)
}
if len(doc.Untaken) != 1 {
t.Fatalf("the document carries %d untaken rows, wanted 1: %s", len(doc.Untaken), body)
}
row := doc.Untaken[0]
if row.Node != "anchor" || row.Module != "route-proxy" || row.Held != 2 {
t.Fatalf("the row is %+v, wanted anchor/route-proxy/2", row)
}
// Absent rather than empty when nothing is held, so a well mesh's document does not carry a
// field a reader has to interpret.
clean, err := statusAsJSON(answers{})
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(clean), "untaken") {
t.Fatalf("a mesh holding nothing still names untaken: %s", clean)
}
}
+22 -59
View File
@@ -295,31 +295,17 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
"built from\n", m.Owner, m.Repo, m.Base, m.Commit)
return nil
}
// A merge produces a plan the mesh keeps (novox/hq ADR 0162): what moved and everything that
// depends on it, along the catalogue's one dependency relation, sorted into tiers. The plan is
// written before any build is asked; the first tier is asked; this returns. Outcomes advance it.
edges, err := inv.Dependencies(ctx)
against, err := inv.BuiltAgainst(ctx)
if err != nil {
return notNow(err)
}
var movedNames []string
for _, e := range moved {
movedNames = append(movedNames, e.Manifest.Module)
ordered := orderByBases(moved, against)
names := make([]string, 0, len(ordered))
for _, e := range ordered {
names = append(names, e.Manifest.Module)
}
plan := planOfMerge(m, movedNames, edges)
if hasCycle(plan.Tiers, edges) {
fmt.Printf(" the last tier depends on itself: %s — built together, in no order\n",
strings.Join(plan.Tiers[len(plan.Tiers)-1], ", "))
}
if err := inv.SavePlan(ctx, plan); err != nil {
return notNow(err)
}
var tiers []string
for i, t := range plan.Tiers {
tiers = append(tiers, fmt.Sprintf("%d: %s", i, strings.Join(t, ", ")))
}
fmt.Printf("%s/%s merged into %s (%.8s); plan %s, %d module(s) in %d tier(s)\n %s\n",
m.Owner, m.Repo, m.Base, m.Commit, plan.ID, len(plan.Modules), len(plan.Tiers), strings.Join(tiers, "\n "))
fmt.Printf("%s/%s merged into %s (%.8s); building %s\n",
m.Owner, m.Repo, m.Base, m.Commit, strings.Join(names, ", "))
if len(packaging) > 0 {
var also []string
for _, e := range packaging {
@@ -328,11 +314,22 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
fmt.Printf(" %s package source from it, so they are rebuilt and their own source record "+
"is left where it is\n", strings.Join(also, ", "))
}
if err := askTier(ctx, inv, &plan); err != nil {
return notNow(err)
var failed []string
for _, e := range ordered {
source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
if err := buildOne(ctx, source, e.Source.Path, e.Source.Ref, 20*time.Minute); err != nil {
fmt.Printf(" %s: %v\n", e.Manifest.Module, err)
failed = append(failed, e.Manifest.Module)
// A base that failed is a reason to stop: what stands on it would be built against
// the old one, and report success (novox/hq 04-ISSUES/131).
if standsOn(ordered, e.Manifest.Module, against) {
fmt.Printf(" stopping: %s is a base of what was still to build\n", e.Manifest.Module)
break
}
}
}
if err := inv.SavePlan(ctx, plan); err != nil {
return notNow(err)
if len(failed) > 0 {
fmt.Printf("%d of %d not built: %s\n", len(failed), len(ordered), strings.Join(failed, ", "))
}
return nil
}
@@ -525,37 +522,3 @@ func isHistory(mergedAt string, seen time.Time) bool {
}
return at.Before(seen)
}
// dependentsOf is every catalogued module that stands on one of the moved modules, directly or
// through another dependent, and is not itself among them — in the catalogue's order, so the
// answer is the same each time. A module standing on nothing that moved is left alone: a merge
// rebuilds what it changed and what is built on top of that, not the catalogue.
func dependentsOf(moved, entries []inventory.Entry, against map[string][]string) []inventory.Entry {
bases := map[string]bool{}
for _, e := range moved {
bases[e.Manifest.Module] = true
}
var out []inventory.Entry
taken := map[string]bool{}
for grew := true; grew; {
grew = false
for _, e := range entries {
name := e.Manifest.Module
if bases[name] || taken[name] {
continue
}
for base := range bases {
if standsOnModule(e, base, against) {
taken[name] = true
out = append(out, e)
grew = true
break
}
}
}
for _, e := range out {
bases[e.Manifest.Module] = true
}
}
return out
}
-75
View File
@@ -1,75 +0,0 @@
package main
import (
"bytes"
"io"
"os"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/inventory"
)
// "4 machine(s), all doing what they were told, all heard from, running what the mesh would send
// them, and every module current with its source" was true for eleven hours of a mesh in which no
// module could reach another (novox/hq 04-ISSUES/145). Every question it answers is about the mesh
// and a machine agreeing; none of them dials anything.
// printed captures what a function writes to stdout.
func printed(t *testing.T, f func() error) string {
t.Helper()
old := os.Stdout
r, w, err := os.Pipe()
if err != nil {
t.Fatal(err)
}
os.Stdout = w
runErr := f()
_ = w.Close()
os.Stdout = old
var buf bytes.Buffer
if _, err := io.Copy(&buf, r); err != nil {
t.Fatal(err)
}
if runErr != nil {
t.Fatal(runErr)
}
return buf.String()
}
func TestTheAllWellSentenceSaysWhatItDoesNotCover(t *testing.T) {
// A mesh with nothing to say. The sentence below was true of a mesh in which no module could
// reach another, for eleven hours.
got := printed(t, func() error {
return printStatus(answers{nodes: []inventory.Node{{Name: "anchor"}, {Name: "laptop"}}})
})
if !strings.Contains(got, "all doing what they were told") {
t.Fatalf("a mesh with nothing to say did not print the all-well sentence:\n%s", got)
}
// And now says what it is not a claim about.
for _, want := range []string{"Nothing here dials a provision", "04-ISSUES/145"} {
if !strings.Contains(got, want) {
t.Fatalf("the all-well sentence does not say %q:\n%s", want, got)
}
}
}
func TestAMeshWithSomethingToSayDoesNotPrintTheScopeLine(t *testing.T) {
// The scope belongs to the all-well sentence. A mesh with something wrong has specific things to
// read, and appending a caveat to those is noise.
got := printed(t, func() error {
return printStatus(answers{
nodes: []inventory.Node{{Name: "anchor"}},
untaken: map[string]map[string]int{"anchor": {"route-proxy": 3}},
})
})
if strings.Contains(got, "Nothing here dials a provision") {
t.Fatalf("a mesh with a held module printed the all-well scope line:\n%s", got)
}
if strings.Contains(got, "all doing what they were told") {
t.Fatalf("a mesh with a held module printed the all-well sentence:\n%s", got)
}
if !strings.Contains(got, "route-proxy") {
t.Fatalf("the held module is not named:\n%s", got)
}
}
-45
View File
@@ -1,45 +0,0 @@
package main
import (
"testing"
"github.com/novox/mesh-controller/internal/inventory"
)
// The holder of the build seat follows the controller that defines its worker (novox/hq issue 206).
// On 2026-10-03 a plan put the build machine in tier 0 and the controller in tier 1; the new build
// machine could not bind the worker the old controller had defined, and nothing could build the
// controller that would have redefined it. The built-by edge from the controller to its build
// machine yields to that order: the controller is built by whichever build machine is running.
func TestTheBuildSeatsHolderFollowsTheControllerThatDefinesItsWorker(t *testing.T) {
edges := []inventory.Edge{
{From: "build-agent", To: "mesh-controller", Kind: inventory.EdgePackages},
{From: "build-agent", To: "mesh-controller", Kind: inventory.EdgeWorkerOf},
{From: "mesh-controller", To: "build-agent", Kind: inventory.EdgeBuiltBy},
{From: "route-proxy", To: "mesh-controller", Kind: inventory.EdgePackages},
{From: "route-proxy", To: "build-agent", Kind: inventory.EdgeBuiltBy},
}
set := reachableFrom([]string{"mesh-controller"}, edges)
if len(set) != 3 {
t.Fatalf("the controller, what packages it, and nothing more: %v", set)
}
tiers := tiersOf(set, edges)
pos := map[string]int{}
for i, tier := range tiers {
for _, m := range tier {
pos[m] = i
}
}
if pos["mesh-controller"] != 0 {
t.Fatalf("the controller first, built by the build machine that is running: %v", tiers)
}
if pos["build-agent"] <= pos["mesh-controller"] {
t.Fatalf("the build machine after the controller that defines its worker: %v", tiers)
}
if pos["route-proxy"] <= pos["build-agent"] {
t.Fatalf("what the build machine builds comes after it: %v", tiers)
}
if hasCycle(tiers, edges) {
t.Fatalf("no cycle here: %v", tiers)
}
}
+1 -4
View File
@@ -10,10 +10,7 @@
# The client is copied from the vendor's own image rather than installed from a distribution:
# `apk add mc` on Alpine installs Midnight Commander, which is a different program with the same
# name, and the failure would be a provisioner that starts cleanly and cannot do anything.
# The Go it builds with is the one the manifest pins (build.on GO_BASE), passed by the Makefile and the
# build machine alike; the default only serves a hand build, and matches go.mod.
ARG GO_BASE=golang:1.26-alpine
FROM ${GO_BASE} AS build
FROM golang:1.25-alpine AS build
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
+1 -4
View File
@@ -3,10 +3,7 @@
# Built here so a machine can be given it by the mesh rather than by somebody putting a binary on
# it. Static and FROM scratch for the same reason the control plane's image is: it is fetched by
# digest and run on a machine, and everything in it is something a person would have to audit.
# The Go it builds with is the one the manifest pins (build.on GO_BASE), passed by the Makefile and the
# build machine alike; the default only serves a hand build, and matches go.mod.
ARG GO_BASE=golang:1.26-alpine
FROM ${GO_BASE} AS build
FROM golang:1.25-alpine AS build
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
+1 -4
View File
@@ -2,10 +2,7 @@
#
# FROM scratch, like the postgres one and unlike the bucket one: it speaks the store's own wire
# protocol directly and needs no client in the image.
# The Go it builds with is the one the manifest pins (build.on GO_BASE), passed by the Makefile and the
# build machine alike; the default only serves a hand build, and matches go.mod.
ARG GO_BASE=golang:1.26-alpine
FROM ${GO_BASE} AS build
FROM golang:1.25-alpine AS build
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
+1 -4
View File
@@ -2,10 +2,7 @@
#
# Static and FROM scratch like the control plane's image, and for the same reason: it is fetched
# by digest and run on a machine, so everything in it is something a person would have to audit.
# The Go it builds with is the one the manifest pins (build.on GO_BASE), passed by the Makefile and the
# build machine alike; the default only serves a hand build, and matches go.mod.
ARG GO_BASE=golang:1.26-alpine
FROM ${GO_BASE} AS build
FROM golang:1.25-alpine AS build
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
-132
View File
@@ -1,132 +0,0 @@
package main
import (
"encoding/json"
"fmt"
"log"
"os"
"strings"
"sync/atomic"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
)
// What the mesh issued this proxy, read on the bus (novox/hq ADR 0160, ADR 0167).
//
// **The proxy is told, not left to work it out.** Its membership carries the routes it is given —
// the same contributions its file is written from — and every machine's address on the private
// network, which is who may be served an internal name. Read once at connect and followed, so a
// route added or a machine joining reaches a running proxy without a restart.
// credential is the bus account the mesh delivered as this module's own secret named broker.
type credential struct {
URL string `json:"url"`
Fingerprint string `json:"fingerprint"`
Node string `json:"node"`
Module string `json:"module"`
User string `json:"user"`
Password string `json:"password"`
}
// followMembership connects with the credential in path and applies every membership the mesh
// issues this proxy. It retries the first connection for as long as it takes: a proxy that started
// before the bus keeps serving the file, and takes the bus when it answers.
func followMembership(path string, held *table, fromBus *atomic.Bool) {
for {
err := followOnce(path, held, fromBus)
if err == nil {
return
}
log.Printf("cannot follow this proxy's membership, serving the file meanwhile: %v", err)
time.Sleep(30 * time.Second)
}
}
func followOnce(path string, held *table, fromBus *atomic.Bool) error {
raw, err := os.ReadFile(path)
if err != nil {
return err
}
var cred credential
if err := json.Unmarshal(raw, &cred); err != nil {
return fmt.Errorf("the broker credential is not one: %w", err)
}
if cred.Node == "" || cred.Module == "" {
return fmt.Errorf("the broker credential names no node or module, so it has no membership")
}
opts := []nats.Option{
nats.Name(cred.Node + "." + cred.Module),
nats.UserInfo(cred.User, cred.Password),
// Its own inbox, and nothing wider: every principal is granted `_INBOX.<its user>.>` alone.
nats.CustomInboxPrefix("_INBOX." + cred.User),
// The bus being restarted is an upgrade, not a reason to stop following.
nats.MaxReconnects(-1),
}
if strings.TrimSpace(cred.Fingerprint) != "" {
opts = append(opts, nats.Secure(broker.PinnedToFingerprint(cred.Fingerprint)))
}
conn, err := nats.Connect(cred.URL, opts...)
if err != nil {
return fmt.Errorf("connecting to the bus at %s: %w", broker.BareAddress(cred.URL), err)
}
subject := broker.MembershipSubject(cred.Node, cred.Module)
apply := func(body []byte) {
var issued broker.Membership
if err := json.Unmarshal(body, &issued); err != nil {
log.Printf("a membership arrived that is not one: %v", err)
return
}
if took := applyMembership(issued, held); took && !fromBus.Swap(true) {
log.Printf("routes now come from this proxy's membership on %s", subject)
}
}
// Followed first, read second: an issue landing between the two is applied, not missed.
if _, err := conn.Subscribe(subject, func(m *nats.Msg) { apply(m.Data) }); err != nil {
conn.Close()
return fmt.Errorf("cannot follow %s: %w", subject, err)
}
// The subject-addressed direct get: the one request this account may make of the stream.
got, err := conn.Request("$JS.API.DIRECT.GET."+broker.AssignmentsStream+"."+subject, nil, 5*time.Second)
switch {
case err != nil:
log.Printf("cannot read the membership issued on %s yet (%v); following it", subject, err)
case got.Header.Get("Status") != "" || len(got.Data) == 0:
log.Printf("no membership issued on %s yet; serving the file until one is", subject)
default:
apply(got.Data)
}
return nil
}
// applyMembership serves what a membership says, and says whether it said anything about routes.
//
// A membership with no routes in it is one from a controller older than ADR 0167, and the file stays
// the source rather than every route being withdrawn because a field was absent.
func applyMembership(issued broker.Membership, held *table) bool {
raw, carries := issued.Receives["route"]
if !carries {
return false
}
var contributions []contribution
if err := json.Unmarshal(raw, &contributions); err != nil {
log.Printf("the routes in this proxy's membership are not contributions, keeping what is served: %v", err)
return false
}
inside, err := sourcesOf(issued.Mesh)
if err != nil {
log.Printf("the mesh in this proxy's membership is unreadable, keeping what is served: %v", err)
return false
}
routes, public := routesOf(contributions)
held.set(routes, public)
held.setInside(inside)
log.Printf("serving %d route(s) from the membership, internal names to %d machine(s): %s",
len(routes), len(inside), strings.Join(held.names(), ", "))
return true
}
-29
View File
@@ -1,29 +0,0 @@
package main
import (
"crypto/tls"
"net/http"
"net/http/httptest"
"net/url"
"testing"
)
// A backend behind the proxy learns the client used TLS and which name it asked for, so the addresses
// it writes into its own pages are the ones a client can use (2026-10-03: a forge's Go import tag
// named an http clone URL, and Go refused the module path).
func TestABackendIsToldTheRequestWasHTTPSAndForWhichName(t *testing.T) {
var proto, host, fwdHost, fwdFor string
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
proto, host, fwdHost, fwdFor = r.Header.Get("X-Forwarded-Proto"), r.Host, r.Header.Get("X-Forwarded-Host"), r.Header.Get("X-Forwarded-For")
}))
defer backend.Close()
where, _ := url.Parse(backend.URL)
req := httptest.NewRequest(http.MethodGet, "https://git.example.org/novox/mesh-sdk/go?go-get=1", nil)
req.TLS = &tls.ConnectionState{}
req.Host = "git.example.org"
req.RemoteAddr = "192.0.2.7:51000"
towards(where).ServeHTTP(httptest.NewRecorder(), req)
if proto != "https" || fwdHost != "git.example.org" || host != "git.example.org" || fwdFor != "192.0.2.7" {
t.Errorf("the backend was told proto=%q host=%q forwarded-host=%q for=%q", proto, host, fwdHost, fwdFor)
}
}
+30 -195
View File
@@ -54,14 +54,12 @@ import (
"net"
"net/http"
"net/http/httputil"
"net/netip"
"net/url"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"golang.org/x/crypto/acme"
@@ -198,63 +196,6 @@ type table struct {
// pass ACME's own validation (it has no public DNS to prove it against), so asking for it is
// not merely pointless but the failing order onlyWhatTheMeshSaid exists to prevent.
public map[string]bool
// inside is where a request must come from to be served a name that is only internal: every
// machine's address on the private network, as the mesh issued it in this proxy's membership
// (novox/hq ADR 0167). Empty until it is issued, and then only the machine itself is inside.
inside sources
}
// sources is who may be served an internal name: the private network's addresses as the mesh
// issued them. The machine itself is always inside — anything on a machine may call anything on it
// (novox/hq ADR 0144) — so loopback needs no entry.
type sources []netip.Prefix
// sourcesOf reads the addresses the mesh issued, each a single address or a range. One that does
// not parse is an error, not an entry skipped: the proxy would otherwise serve internal names to
// fewer machines than the mesh said, and say nothing.
func sourcesOf(mesh []string) (sources, error) {
var out sources
for _, entry := range mesh {
entry = strings.TrimSpace(entry)
if prefix, err := netip.ParsePrefix(entry); err == nil {
out = append(out, prefix.Masked())
continue
}
addr, err := netip.ParseAddr(entry)
if err != nil {
return nil, fmt.Errorf("%q is not an address on the private network", entry)
}
addr = addr.Unmap()
out = append(out, netip.PrefixFrom(addr, addr.BitLen()))
}
return out, nil
}
// holds says whether a request from this remote address came from the mesh or the machine itself.
//
// **By source, which the mesh's guard deliberately is not** — it names interfaces, because a source
// address can be claimed by whoever sends the packet. The proxy cannot see the interface a request
// arrived on, and here the claim does not carry: a connection needs its replies, and replies to a
// mesh address leave by the tunnel, never back to the claimant.
func (s sources) holds(remote string) bool {
host := remote
if h, _, err := net.SplitHostPort(remote); err == nil {
host = h
}
addr, err := netip.ParseAddr(host)
if err != nil {
return false
}
addr = addr.Unmap()
if addr.IsLoopback() {
return true
}
for _, prefix := range s {
if prefix.Contains(addr) {
return true
}
}
return false
}
func (t *table) set(routes map[string][]rule, public map[string]bool) {
@@ -273,7 +214,7 @@ func (t *table) set(routes map[string][]rule, public map[string]bool) {
log.Printf("route %s points at %q, which is not a URL: %v", host, r.target, err)
continue
}
r.to = towards(where)
r.to = httputil.NewSingleHostReverseProxy(where)
if r.insecure {
r.to.Transport = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
}
@@ -373,41 +314,6 @@ func bareHost(host string) string {
return strings.ToLower(host)
}
// hiddenFrom says whether this host must look unrouted to a request from this address: it is
// only an internal name, and the request did not come from the private network.
//
// **The proxy is the only way in to a routed endpoint, so it is what makes `internal` true**
// (novox/hq ADR 0138, issue 191). It answers public names on the same listeners, so a request from
// anywhere can carry any Host header; a name being internal keeps nobody out unless this check does.
// Answered exactly as a name that was never routed, so an outsider learns nothing from asking.
func (t *table) hiddenFrom(host, remote string) bool {
if !t.eligibleForInternalACME(host) {
return false
}
t.mu.RLock()
defer t.mu.RUnlock()
return !t.inside.holds(remote)
}
// setInside replaces who the mesh is, as the membership said.
func (t *table) setInside(inside sources) {
t.mu.Lock()
t.inside = inside
t.mu.Unlock()
}
// namesSeenFrom is what this proxy says it serves to a request from this address — every routed
// name, less the internal-only ones when the request came from outside.
func (t *table) namesSeenFrom(remote string) []string {
out := []string{}
for _, name := range t.names() {
if !t.hiddenFrom(name, remote) {
out = append(out, name)
}
}
return out
}
func (t *table) names() []string {
t.mu.RLock()
defer t.mu.RUnlock()
@@ -437,20 +343,7 @@ func run() error {
}
held := newTable()
// **The bus first, the file until it has spoken** (novox/hq ADR 0167). The membership carries
// the routes and who the mesh is; the file carries the routes alone, so while the proxy reads
// it an internal name is served to this machine and to nobody else — refused, never opened.
fromBus := &atomic.Bool{}
if credential := strings.TrimSpace(os.Getenv("MESH_BROKER_FILE")); credential != "" {
go followMembership(credential, held, fromBus)
} else {
log.Printf("MESH_BROKER_FILE is not set: routes come from %s alone, and a name that is only "+
"internal is served to this machine alone", path)
}
read := func() {
if fromBus.Load() {
return
}
routes, public, err := routesFrom(path)
if err != nil {
// Kept serving what it had. A file being rewritten is momentarily unreadable, and
@@ -529,7 +422,19 @@ func run() error {
}()
tlsConfig := publicManager.TLSConfig()
tlsConfig.GetCertificate = certificateFor(held, tlsConfig.GetCertificate, internalManager)
if internalManager != nil {
// Dispatched by which authority may certify this name at all — the same question
// eligibleForInternalACME already answers, asked once more at handshake time rather than
// only when an order is placed, since a cached certificate is served here on every request
// and never goes through HostPolicy again.
fromPublic, fromInternal := tlsConfig.GetCertificate, internalManager.TLSConfig().GetCertificate
tlsConfig.GetCertificate = func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
if held.eligibleForInternalACME(hello.ServerName) {
return fromInternal(hello)
}
return fromPublic(hello)
}
}
server := &http.Server{
Addr: secure,
@@ -687,33 +592,6 @@ func forThisAuthority(cache, directory string, root []byte) string {
return filepath.Join(cache, hex.EncodeToString(sum[:])[:16])
}
// certificateFor picks the certificate a handshake is answered with.
//
// Dispatched by which authority may certify this name at all — the same question
// eligibleForInternalACME already answers, asked once more at handshake time rather than only when
// an order is placed, since a cached certificate is served here on every request and never goes
// through HostPolicy again. And refused, exactly as an unrouted name is, to a client outside the
// private network asking for a name that is only internal: the certificate would name it.
func certificateFor(held *table, fromPublic func(*tls.ClientHelloInfo) (*tls.Certificate, error),
internalManager *autocert.Manager) func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
var fromInternal func(*tls.ClientHelloInfo) (*tls.Certificate, error)
if internalManager != nil {
fromInternal = internalManager.TLSConfig().GetCertificate
}
return func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
if held.eligibleForInternalACME(hello.ServerName) {
if hello.Conn != nil && held.hiddenFrom(hello.ServerName, hello.Conn.RemoteAddr().String()) {
return nil, fmt.Errorf("no public route for %q in this mesh, so no certificate is asked for",
hello.ServerName)
}
if fromInternal != nil {
return fromInternal(hello)
}
}
return fromPublic(hello)
}
}
// newTable is an empty routing table.
func newTable() *table {
return &table{to: map[string][]rule{}}
@@ -722,9 +600,8 @@ func newTable() *table {
// handler is the proxy itself, separated so it can be driven by a test without a listener.
func handler(held *table) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hidden := held.hiddenFrom(r.Host, r.RemoteAddr)
matched, known := held.find(r.Host, r.URL.Path)
if hidden || !known {
if !known {
// **Named, not a bare 404.** A route that was withdrawn and a name that never existed
// are different things, and a proxy that says only "not found" makes an operator go
// and read the mesh to tell them apart. What it is serving is the answer to both.
@@ -732,20 +609,15 @@ func handler(held *table) http.Handler {
// And since a host may now be routed only on some paths, those are a third thing:
// saying "no route for this name" while listing that very name as served is a
// contradiction an operator would have to disbelieve the proxy to get past.
// **Said in the log as well as to the client.** A name this mesh does not serve, asked
// for from outside, is what a scanner does, and the machine's intrusion prevention reads
// this proxy's log for exactly that line (novox/hq ADR 0179): the address last, as the
// jail's filter expects it.
log.Printf("refused: no route for %q, asked from %s", r.Host, r.RemoteAddr)
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.WriteHeader(http.StatusNotFound)
if !hidden && held.routed(r.Host) {
if held.routed(r.Host) {
fmt.Fprintf(w, "%s is served here, but no route covers %q.\n",
bareHost(r.Host), r.URL.Path)
return
}
fmt.Fprintf(w, "no route for %q in this mesh.\nserving: %s\n",
r.Host, strings.Join(held.namesSeenFrom(r.RemoteAddr), ", "))
r.Host, strings.Join(held.names(), ", "))
return
}
@@ -844,12 +716,6 @@ func boolByte(b bool) byte {
// routesFrom reads what the mesh wrote and turns it into host → the rules for that host, and
// which of those hosts is a public name — the second is `name`, ACME-eligible; a host reached
// only through `internal-name` never appears there.
//
// **A route may carry either name, or both** (novox/hq ADR 0138). How far an endpoint reaches
// decides which names the mesh composes, so an endpoint that reaches only the private network
// arrives with an `internal-name` and no `name`. That is a whole route, not a malformed one: it is
// served under its internal name and certified by the internal authority. Only a route with
// neither name has nothing to be served under (novox/hq issue 191).
func routesFrom(path string) (map[string][]rule, map[string]bool, error) {
raw, err := os.ReadFile(path)
if err != nil {
@@ -859,29 +725,17 @@ func routesFrom(path string) (map[string][]rule, map[string]bool, error) {
if err := json.Unmarshal(raw, &said); err != nil {
return nil, nil, err
}
routes, public := routesOf(said.Given)
return routes, public, nil
}
// routesOf turns what the mesh gave into host → the rules for that host, and which hosts are public
// names — the same whether the contributions came in the file or in the membership.
func routesOf(contributions []contribution) (map[string][]rule, map[string]bool) {
out := map[string][]rule{}
public := map[string]bool{}
for _, c := range contributions {
for _, c := range said.Given {
name, _ := c.Values["name"].(string)
name = strings.TrimSpace(name)
internal, _ := c.Values["internal-name"].(string)
internal = strings.TrimSpace(internal)
if name == "" && internal == "" {
if name == "" {
log.Printf("%s on %s asked for a route and named nothing; skipped", c.From, c.Node)
continue
}
// What the route is called in a log line: its public name when it has one.
called := name
if called == "" {
called = internal
}
host := strings.ToLower(name)
public[host] = true
made := rule{path: asPath(c.Values["path"])}
if p, ok := asWhole(c.Values["priority"]); ok {
@@ -898,7 +752,7 @@ func routesOf(contributions []contribution) (map[string][]rule, map[string]bool)
if looksLikeACredential(named) {
log.Printf("%s on %s declared route %q with a credential in the declaration rather "+
"than the name of a secret; the whole route is refused (novox/hq ADR 0108)",
c.From, c.Node, called)
c.From, c.Node, name)
continue
}
users, err := usersFrom(named)
@@ -916,7 +770,7 @@ func routesOf(contributions []contribution) (map[string][]rule, map[string]bool)
port, ok := asPort(c.Values["port"])
if !ok {
log.Printf("%s on %s asked for route %q and gave no usable port; skipped",
c.From, c.Node, called)
c.From, c.Node, name)
continue
}
// Where the mesh says that machine is. Empty means it is this one — a workload beside
@@ -937,7 +791,7 @@ func routesOf(contributions []contribution) (map[string][]rule, map[string]bool)
}
if scheme != "http" && scheme != "https" {
log.Printf("%s on %s asked for route %q with scheme %q, which is neither http "+
"nor https; skipped", c.From, c.Node, called, scheme)
"nor https; skipped", c.From, c.Node, name, scheme)
continue
}
made.insecure, _ = c.Values["insecure"].(bool)
@@ -948,7 +802,7 @@ func routesOf(contributions []contribution) (map[string][]rule, map[string]bool)
bytes, whole := asWhole(asked)
if !whole || bytes <= 0 {
log.Printf("%s on %s asked for route %q with a max-request-body of %v, which is "+
"not a whole positive number of bytes; skipped", c.From, c.Node, called, asked)
"not a whole positive number of bytes; skipped", c.From, c.Node, name, asked)
continue
}
made.maxRequestBody = int64(bytes)
@@ -956,24 +810,19 @@ func routesOf(contributions []contribution) (map[string][]rule, map[string]bool)
made.target = fmt.Sprintf("%s://%s:%d", scheme, at, port)
}
if name != "" {
host := strings.ToLower(name)
out[host] = append(out[host], made)
public[host] = true
}
out[host] = append(out[host], made)
// The internal-network name, the same rule under a second host — a predecessor proxy
// The internal-network alias, the same rule under a second host — a predecessor proxy
// answered both for one route, as a convenience (reaching a service over the VPN without a
// public TLS round trip), not as an access boundary; composing it here restores exactly
// that, nothing more. Absent whenever the node composed no internal name (novox/hq ADR
// 0056's internalDomain half) — the same "nothing to join a label to" case the public name
// already has. And the only name, when the endpoint reaches no further than the private
// network.
if internal != "" {
// already has.
if internal, _ := c.Values["internal-name"].(string); strings.TrimSpace(internal) != "" {
out[strings.ToLower(internal)] = append(out[strings.ToLower(internal)], made)
}
}
return out, public
return out, public, nil
}
// asWhole is any whole number the mesh wrote, whatever its magnitude.
@@ -1060,17 +909,3 @@ func asPort(v any) (int, bool) {
}
return 0, false
}
// towards proxies to one backend and tells it what the client asked: **X-Forwarded-Proto, -Host and
// -For**, set from the request this proxy received. A backend that builds its own addresses — a forge
// writing its clone URL into a page, a login redirect — otherwise sees the plain HTTP hop from this
// proxy and writes `http://`, though every client reached it over TLS: Go refused the forge's module
// path for exactly that on 2026-10-03, its import tag naming an http clone URL.
// The standard library's NewSingleHostReverseProxy sets only X-Forwarded-For.
func towards(where *url.URL) *httputil.ReverseProxy {
return &httputil.ReverseProxy{Rewrite: func(pr *httputil.ProxyRequest) {
pr.SetURL(where)
pr.Out.Host = pr.In.Host
pr.SetXForwarded()
}}
}
-206
View File
@@ -1,206 +0,0 @@
package main
import (
"crypto/tls"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/broker"
)
// behind is a workload the proxy can send to, and a table routing one public name and one
// internal-only name to it, with the mesh's machines as the membership would issue them.
func behind(t *testing.T, mesh ...string) *table {
t.Helper()
workload := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
io.WriteString(w, "the workload")
}))
t.Cleanup(workload.Close)
at, _ := url.Parse(workload.URL)
host, port, _ := net.SplitHostPort(at.Host)
routes, public, err := routesFrom(write(t, fmt.Sprintf(`{"given":[
{"from":"app","node":"anchor","at":%q,
"values":{"name":"app.example","internal-name":"app.anchor.internal","port":%s}},
{"from":"admin","node":"anchor","at":%q,
"values":{"internal-name":"admin.anchor.internal","port":%s}}
]}`, host, port, host, port)))
if err != nil {
t.Fatal(err)
}
held := newTable()
inside, err := sourcesOf(mesh)
if err != nil {
t.Fatal(err)
}
held.setInside(inside)
held.set(routes, public)
return held
}
// askFrom is what the proxy answers a request for host coming from remote.
func askFrom(held *table, host, remote string) (int, string) {
r := httptest.NewRequest(http.MethodGet, "http://"+host+"/", nil)
r.RemoteAddr = remote
w := httptest.NewRecorder()
handler(held).ServeHTTP(w, r)
return w.Code, w.Body.String()
}
// **An internal-only name is served to the private network and to nobody else** (novox/hq ADR
// 0138, issue 191). The proxy answers public names on the same listeners, so without this a name
// being internal kept nobody out: a request from the internet only had to carry it.
func TestAnInternalOnlyNameIsServedOnlyInsideThePrivateNetwork(t *testing.T) {
held := behind(t, "10.10.0.1", "10.10.0.7")
if code, body := askFrom(held, "admin.anchor.internal", "10.10.0.7:51000"); code != http.StatusOK ||
body != "the workload" {
t.Errorf("a request from the private network was not served: %d %q", code, body)
}
if code, body := askFrom(held, "admin.anchor.internal", "127.0.0.1:51000"); code != http.StatusOK {
t.Errorf("a request from the machine itself was not served: %d %q", code, body)
}
code, body := askFrom(held, "admin.anchor.internal", "203.0.113.9:51000")
if code != http.StatusNotFound {
t.Fatalf("a request from outside the private network reached an internal-only name: %d %q",
code, body)
}
// Answered as a name never routed, and the list of what is served does not name it either —
// otherwise the refusal would tell an outsider exactly what to ask for from inside.
if strings.Contains(strings.SplitN(body, "\n", 2)[1], "admin.anchor.internal") {
t.Errorf("the refusal names the internal-only route to an outsider: %q", body)
}
if !strings.Contains(body, "app.example") {
t.Errorf("the refusal stopped listing the public names: %q", body)
}
}
// The internal name of a route that also has a public one is internal too: served inside, and to
// an outsider only under the public name. Nothing is lost — the outsider has the public name — and a
// name stays one thing whichever route it came from.
func TestAnInternalAliasOfAPublicRouteIsServedInsideOnly(t *testing.T) {
held := behind(t, "10.10.0.1", "10.10.0.7")
if code, body := askFrom(held, "app.anchor.internal", "10.10.0.7:51000"); code != http.StatusOK {
t.Errorf("the internal alias stopped answering the private network: %d %q", code, body)
}
if code, _ := askFrom(held, "app.anchor.internal", "203.0.113.9:51000"); code != http.StatusNotFound {
t.Errorf("the internal alias was served to an outsider: %d", code)
}
if code, _ := askFrom(held, "app.example", "203.0.113.9:51000"); code != http.StatusOK {
t.Errorf("the public name was refused to an outsider: %d", code)
}
}
// Before a membership has said who the mesh is, only the machine itself is inside — refused to
// everyone else, never served to everyone.
func TestUntilTheMeshIsIssuedAnInternalOnlyNameIsServedToTheMachineAlone(t *testing.T) {
held := behind(t)
if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.7:51000"); code != http.StatusNotFound {
t.Errorf("an internal-only name was served with no private network said: %d", code)
}
if code, _ := askFrom(held, "admin.anchor.internal", "[::1]:51000"); code != http.StatusOK {
t.Errorf("an internal-only name was refused to the machine itself: %d", code)
}
}
type from struct {
net.Conn
remote net.Addr
}
func (c from) RemoteAddr() net.Addr { return c.remote }
// The handshake refuses an internal-only name to an outsider too: the certificate would name it,
// and serving it would answer the question the routing refuses to.
func TestTheHandshakeRefusesAnInternalOnlyNameToAnOutsider(t *testing.T) {
held := behind(t, "10.10.0.1", "10.10.0.7")
served := &tls.Certificate{}
pick := certificateFor(held, func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return served, nil }, nil)
hello := func(name, remote string) *tls.ClientHelloInfo {
addr, _ := net.ResolveTCPAddr("tcp", remote)
return &tls.ClientHelloInfo{ServerName: name, Conn: from{remote: addr}}
}
if _, err := pick(hello("admin.anchor.internal", "203.0.113.9:443")); err == nil {
t.Error("an outsider was handed a certificate for an internal-only name")
}
if got, err := pick(hello("admin.anchor.internal", "10.10.0.7:443")); err != nil || got != served {
t.Errorf("a client on the private network was refused: %v", err)
}
if got, err := pick(hello("app.example", "203.0.113.9:443")); err != nil || got != served {
t.Errorf("a public name was refused to an outsider: %v", err)
}
}
// The mesh is issued as machines' addresses; a range is read as well. One that does not parse is
// refused rather than skipped, so a typo never quietly narrows or widens who is inside.
func TestTheMeshIsReadAsAddressesAndRanges(t *testing.T) {
if _, err := sourcesOf([]string{"10.10.0.1", "not-an-address"}); err == nil {
t.Error("an entry that is not an address was accepted")
}
inside, err := sourcesOf([]string{"10.10.0.1", "fd00::1", "10.20.0.0/24"})
if err != nil {
t.Fatal(err)
}
for remote, want := range map[string]bool{
"10.10.0.1:1": true,
"[::ffff:10.10.0.1]:1": true,
"[fd00::1]:1": true,
"10.20.0.200:1": true,
"10.10.0.2:1": false,
"192.168.1.10:1": false,
"not-an-address": false,
} {
if inside.holds(remote) != want {
t.Errorf("%s inside the mesh: got %v, want %v", remote, !want, want)
}
}
}
// What the mesh issues is what is served: the routes in the membership, internal names to the
// machines it names (novox/hq ADR 0167).
func TestAMembershipIsServedAsIssued(t *testing.T) {
held := newTable()
took := applyMembership(broker.Membership{
Receives: map[string]json.RawMessage{"route": json.RawMessage(`[
{"from":"admin","node":"anchor","at":"anchor.internal",
"values":{"internal-name":"admin.anchor.internal","port":8080}}]`)},
Mesh: []string{"10.10.0.7"},
}, held)
if !took {
t.Fatal("a membership carrying routes was not applied")
}
if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.7:1"); code == http.StatusNotFound {
t.Error("a machine the membership names was refused the internal-only route")
}
if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.9:1"); code != http.StatusNotFound {
t.Errorf("a machine the membership does not name was served the internal-only route: %d", code)
}
}
// A membership that says nothing about routes is one from a controller that does not issue them,
// and changes nothing: the file stays the source rather than every route being withdrawn.
func TestAMembershipWithoutRoutesLeavesTheFileServing(t *testing.T) {
held := behind(t, "10.10.0.7")
before := held.names()
if applyMembership(broker.Membership{Mesh: []string{"10.10.0.7"}}, held) {
t.Error("a membership without routes was taken as the source of routes")
}
if got := held.names(); strings.Join(got, ",") != strings.Join(before, ",") {
t.Errorf("a membership without routes changed what is served: %v, was %v", got, before)
}
if applyMembership(broker.Membership{
Receives: map[string]json.RawMessage{"route": json.RawMessage(`[]`)},
Mesh: []string{"not-an-address"},
}, held) {
t.Error("a membership whose mesh cannot be read was applied")
}
}
-44
View File
@@ -90,50 +90,6 @@ func TestARouteWithAnInternalNameIsReachableUnderBoth(t *testing.T) {
}
}
// A route whose endpoint reaches only the private network carries an internal name and no public
// one (novox/hq ADR 0138), and is served under that name rather than skipped as naming nothing —
// skipping it left every internal-only module unreachable by name (novox/hq issue 191).
func TestARouteWithOnlyAnInternalNameIsServed(t *testing.T) {
routes, public, err := routesFrom(write(t, `{"given":[
{"from":"app","node":"anchor","at":"anchor.internal",
"values":{"internal-name":"App.Anchor.Internal","port":8443,"scheme":"https","insecure":true}}
]}`))
if err != nil {
t.Fatal(err)
}
if targetOf(routes, "app.anchor.internal") != "https://anchor.internal:8443" {
t.Fatalf("the internal-only route is not served: %v", routes)
}
if len(routes) != 1 {
t.Errorf("an internal-only route made hosts it never named: %v", routes)
}
if len(public) != 0 {
t.Errorf("an internal-only route made a name eligible for a public certificate: %v", public)
}
held := newTable()
held.set(routes, public)
if err := onlyInternalNamesTheMeshSaid(held)(context.Background(), "app.anchor.internal"); err != nil {
t.Errorf("the internal authority refused the internal-only route's name: %v", err)
}
if err := onlyWhatTheMeshSaid(held)(context.Background(), "app.anchor.internal"); err == nil {
t.Error("a public certificate was ordered for an internal-only name")
}
}
// A route with neither name has nothing to be served under, and is still skipped.
func TestARouteWithNeitherNameIsSkipped(t *testing.T) {
routes, public, err := routesFrom(write(t, `{"given":[
{"from":"app","node":"anchor","at":"anchor.internal","values":{"internal-name":" ","port":8080}}
]}`))
if err != nil {
t.Fatal(err)
}
if len(routes) != 0 || len(public) != 0 {
t.Errorf("a route that named nothing was served: %v %v", routes, public)
}
}
// A route with no internal-name composed gets no second host — the ordinary case, unchanged.
func TestARouteWithNoInternalNameGetsNoAlias(t *testing.T) {
routes, _, err := routesFrom(write(t, `{"given":[
+1 -11
View File
@@ -67,7 +67,7 @@ func TestTheAgreementCheckCatchesASubscriptionThatMatchesNothing(t *testing.T) {
// An event published under a seat's name is real even though no module declares it as its own.
if bad := Disagreements(nil,
[]AConsumer{{Module: "watcher", Consumes: []string{"mesh-artifact-store.image.pushed"}}},
[]DeclaredSeat{{Name: "mesh-artifact-store", Emits: []string{"image.pushed"}}}); len(bad) != 0 {
[]DeclaredSeat{{Name: "the-artifact-store", Emits: []string{"image.pushed"}}}); len(bad) != 0 {
t.Fatalf("an event a seat emits was reported as matching nothing: %v", bad)
}
}
@@ -234,13 +234,3 @@ func admitsSubject(pattern, subject []string) bool {
}
return len(pattern) == len(subject)
}
// The two packages name the runtime module separately — the broker's types stay free of the
// catalogue's on purpose — so this is what holds them to one string. A rename that reached only one
// side would compose a runtime principal for a module nobody assigns, silently, and leave the one
// that is assigned with a module's own grants.
func TestTheBrokerAndTheCatalogueAgreeOnTheRuntimeModule(t *testing.T) {
if RuntimeModule != catalogue.RuntimeModule {
t.Fatalf("the broker calls the runtime %q and the catalogue %q", RuntimeModule, catalogue.RuntimeModule)
}
}
-178
View File
@@ -1,178 +0,0 @@
package broker
import (
"fmt"
"os"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
)
const twoSeconds = 2 * time.Second
// A running mesh already holds consumers made before the delivery subject carried the stream
// (novox/hq 04-ISSUES/146). The server will not change a push consumer's delivery subject in place,
// so bringing one to match must replace it — and must not replay what it already acknowledged
// (novox/hq 04-ISSUES/156).
//
// docker run -d --rm --name t -p 14231:4222 nats:2.10-alpine -js
// MESH_TEST_NATS=nats://127.0.0.1:14231 go test ./internal/broker/ -run TestUpgrading
func TestUpgradingAConsumerWhoseDeliverySubjectMoved(t *testing.T) {
url := os.Getenv("MESH_TEST_NATS")
if url == "" {
t.Skip("MESH_TEST_NATS unset")
}
js, err := Dial(url)
if err != nil {
t.Fatal(err)
}
defer js.Close()
// The stream exactly as the mesh's own is — one declaration per node, always the newest.
// Reproduced rather than approximated: the first version of this test used a plain stream and
// a plain consumer, and the server accepted the update it refuses in a running mesh, so the
// test passed against the very code that was crash-looping on the control node.
const stream, name = "NODES", "novox"
subject := "mesh.node." + name + ".declare"
_ = js.js.DeleteStream(stream)
if _, err := js.js.AddStream(&nats.StreamConfig{
Name: stream, Subjects: []string{"mesh.node.*.declare"},
MaxMsgsPerSubject: 1, Storage: nats.MemoryStorage,
}); err != nil {
t.Fatal(err)
}
defer func() { _ = js.js.DeleteStream(stream) }()
for i := 0; i < 6; i++ {
if _, err := js.js.Publish(subject, []byte(fmt.Sprint(i))); err != nil {
t.Fatal(err)
}
}
// The consumer as a running mesh holds it: made before the subject carried the stream, and
// otherwise exactly what NodeConsumer asks for.
if _, err := js.js.AddConsumer(stream, &nats.ConsumerConfig{
Durable: name, AckPolicy: nats.AckExplicitPolicy,
AckWait: 300 * time.Second, MaxDeliver: -1,
FilterSubject: subject,
DeliverSubject: "_DELIVER." + name,
}); err != nil {
t.Fatal(err)
}
// It acknowledged the first four. Those must not come back.
sub, err := js.js.SubscribeSync(subject, nats.Bind(stream, name))
if err != nil {
t.Fatal(err)
}
for i := 0; i < 1; i++ {
m, err := sub.NextMsg(twoSeconds)
if err != nil {
t.Fatalf("message %d never arrived: %v", i, err)
}
if err := m.AckSync(); err != nil {
t.Fatal(err)
}
}
// **The subscription stays up.** In a running mesh the machine is attached to this consumer
// the whole time — that is what a node listening for its declaration IS. The first version of
// this test unsubscribed first, and the server then accepted an update it refuses while a
// subscriber is bound, so the test passed against the code that was crash-looping.
defer func() { _ = sub.Unsubscribe() }()
// Now the upgrade: the consumer the controller asserts on every start, with the subject that
// carries the stream.
want := NodeConsumer(name)
var notes []string
js.Note = func(f string, a ...any) { notes = append(notes, fmt.Sprintf(f, a...)) }
if err := js.EnsureConsumer(want); err != nil {
t.Fatalf("a consumer the mesh already held could not be brought to match, which is the "+
"control plane failing to start: %v", err)
}
info, err := js.js.ConsumerInfo(stream, name)
if err != nil {
t.Fatal(err)
}
// It KEEPS the subject it had. Moving it would need the holder's grant to have widened first,
// and that grant travels in the bus's user list, which a machine applies minutes later.
if got := info.Config.DeliverSubject; got != "_DELIVER."+name {
t.Fatalf("the consumer a machine is bound to was moved to %q; a machine not yet allowed "+
"to subscribe there is a machine that hears nothing", got)
}
if len(notes) != 1 {
t.Fatalf("keeping it was not reported, so it would be invisible: %v", notes)
}
if !strings.Contains(notes[0], "keeps working") {
t.Fatalf("the note does not say the consumer still works: %q", notes[0])
}
// And the machine bound to it is still being delivered to — the point of keeping it.
if _, err := js.js.Publish(subject, []byte("after the assertion")); err != nil {
t.Fatal(err)
}
m, err := sub.NextMsg(twoSeconds)
if err != nil {
t.Fatalf("the machine stopped hearing its declarations after the assertion: %v", err)
}
if string(m.Data) != "after the assertion" {
t.Fatalf("delivered %q", m.Data)
}
// Asserting again is a no-op, or the controller crash-loops on its own restart.
if err := js.EnsureConsumer(want); err != nil {
t.Fatalf("the second assertion failed: %v", err)
}
}
// And where nothing is bound, the subject DOES move — that is 04-ISSUES/146's fix, which this must
// not undo. The controller's own two consumers are in exactly this position: it asserts them before
// it subscribes.
func TestAConsumerNothingIsBoundToDoesMove(t *testing.T) {
url := os.Getenv("MESH_TEST_NATS")
if url == "" {
t.Skip("MESH_TEST_NATS unset")
}
js, err := Dial(url)
if err != nil {
t.Fatal(err)
}
defer js.Close()
const stream, name = "NODES", "shanks"
subject := "mesh.node." + name + ".declare"
_ = js.js.DeleteStream(stream)
if _, err := js.js.AddStream(&nats.StreamConfig{
Name: stream, Subjects: []string{"mesh.node.*.declare"},
MaxMsgsPerSubject: 1, Storage: nats.MemoryStorage,
}); err != nil {
t.Fatal(err)
}
defer func() { _ = js.js.DeleteStream(stream) }()
if _, err := js.js.AddConsumer(stream, &nats.ConsumerConfig{
Durable: name, AckPolicy: nats.AckExplicitPolicy,
AckWait: 300 * time.Second, MaxDeliver: -1,
FilterSubject: subject,
DeliverSubject: "_DELIVER." + name,
}); err != nil {
t.Fatal(err)
}
want := NodeConsumer(name)
if err := js.EnsureConsumer(want); err != nil {
t.Fatal(err)
}
info, err := js.js.ConsumerInfo(stream, name)
if err != nil {
t.Fatal(err)
}
if got := info.Config.DeliverSubject; got != DeliverSubjectFor(want) {
t.Fatalf("delivery subject is %q, wanted %q -- issue 146's fix no longer applies to a "+
"consumer nothing is holding", got, DeliverSubjectFor(want))
}
}
+9 -24
View File
@@ -40,14 +40,7 @@ type Consumer struct {
AckWaitSeconds int
// MaxDeliver before the message is dead-lettered; zero for the mesh's default.
MaxDeliver int
// MaxAckPending is how many deliveries the server lets stand unacknowledged at once; zero for
// the server's default, which is many. **One, for a consumer handled one at a time**
// (novox/hq issue 175): a handler that builds for minutes keeps its own message alive with a
// heartbeat, but everything handed over behind it times out unacknowledged and comes back —
// and a merge that came back rebuilt what it had just built, five times over on 2026-09-30.
// With one outstanding, the server holds the rest, and the heartbeat is keeping the message.
MaxAckPending int
Why string
Why string
}
// seatStreamName is the stream holding a seat's inbound work. Named after the seat rather than
@@ -144,18 +137,12 @@ func ConsumerFor(p Principal) (Consumer, bool) {
}, true
}
// HolderConsumerFor is the worker a seat's holders share on that seat's work queue.
// HolderConsumerFor is the worker a seat's holder gets on that seat's work queue.
//
// **One worker for every holder, and each holder pulls one ask when it is idle** (novox/hq ADR
// 0190). The seat is *authority* — who may be the telegram sender — and the worker is *delivery*,
// kept separate so that relaxing one changes nothing about the other: a node-scoped seat has a
// holder per machine, and all of them take from this one consumer, so the work is shared without
// any holder knowing about the others. Pulled rather than pushed because a push consumer hands the
// next ask to whichever subscriber the server picks, busy or not, and a pulled one is asked for by
// a holder that has just become free. Which is also what ends the race issue 186 describes — asks
// delivered behind the one being worked, expiring unacknowledged and dropped after the fifth
// redelivery: nothing is delivered that nobody asked for. A long build keeps its own ask alive
// (stillWorking); the ack wait is for a holder that died.
// **A queue group even though the seat guarantees one holder.** The seat is *authority* — who may
// be the telegram sender — and the queue group is *delivery*. Tie delivery to the seat and the
// day somebody allows two holders for throughput, every message is processed twice with nothing
// reporting it. Kept separate, relaxing one changes nothing about the other.
func HolderConsumerFor(node, module string, seat DeclaredSeat) (Consumer, bool) {
if len(seat.Accepts) == 0 {
return Consumer{}, false
@@ -164,13 +151,11 @@ func HolderConsumerFor(node, module string, seat DeclaredSeat) (Consumer, bool)
Name: "SEAT_" + upperSnake(seat.Name) + "_worker",
Stream: seatStreamName(seat.Name),
Filters: []string{"mesh.seat." + seat.Name + ".accept.>"},
Queue: "holders",
AckWaitSeconds: 60,
MaxDeliver: 5,
// As many in flight as there are holders working, which pulling bounds by itself: a holder
// fetches one and fetches again only after it acknowledged. The server's default stands.
Why: fmt.Sprintf("%s on %s holds %s; every holder pulls one ask at a time from this worker "+
"and acknowledges after the work is done, so a crash mid-work redelivers rather than "+
"loses and an idle holder is the one that takes the next ask", module, node, seat.Name),
Why: fmt.Sprintf("%s on %s holds %s; it acknowledges after the work is done, so a "+
"crash mid-work redelivers rather than loses", module, node, seat.Name),
}, true
}
+5 -31
View File
@@ -88,20 +88,15 @@ func TestAModuleThatConsumesNothingGetsNoConsumer(t *testing.T) {
}
}
// The seat is authority and the worker is delivery (novox/hq ADR 0190): one worker per seat, shared
// by every holder and pulled from, so a second holder takes the next ask rather than a copy of the
// same one — which is what a queue group used to guard, and what pulling one durable gives outright.
func TestAHoldersWorkerIsOneSharedByItsHolders(t *testing.T) {
// The seat is authority and the queue group is delivery. Tie them together and the day somebody
// allows two holders, every message is processed twice with nothing reporting it.
func TestAHoldersWorkerUsesAQueueGroupAnyway(t *testing.T) {
c, ok := HolderConsumerFor("one", "telegram", telegramSeat())
if !ok {
t.Fatal("the holder of a seat with inbound work got no worker")
}
two, _ := HolderConsumerFor("two", "telegram", telegramSeat())
if c.Name != two.Name || c.Stream != two.Stream {
t.Fatal("two holders got two workers, so each would process every ask")
}
if c.Push || c.Queue != "" {
t.Fatal("the worker is pushed, so the server would hand an ask to a busy holder")
if c.Queue == "" {
t.Fatal("the worker is not in a queue group, so a second holder would double-process")
}
if c.Stream != "SEAT_TELEGRAM_SENDER" {
t.Fatalf("the worker reads %q, not the seat's own stream", c.Stream)
@@ -158,24 +153,3 @@ func TestANodesDeclarationConsumerIsWhatItsOwnGrantAllows(t *testing.T) {
has(t, perms.Publish, "$JS.ACK.NODES."+c.Name+".>")
has(t, perms.Subscribe, c.Filters[0])
}
// Every holder of a seat shares one worker and pulls from it (novox/hq ADR 0190): no queue group
// and no delivery subject, because a push consumer hands the next ask to whichever subscriber the
// server picks, busy or not; and no cap of one in flight, because pulling bounds the asks in flight
// by the holders that are free — which is what ended the race of issue 186, where asks delivered
// behind the one being worked expired and were dropped.
func TestAHoldersWorkerIsPulledByEveryHolder(t *testing.T) {
c, found := HolderConsumerFor("anchor", "build-agent", DeclaredSeat{Name: "node-build-agent", Accepts: []string{"build"}})
if !found {
t.Fatal("a seat that accepts work has no worker")
}
if c.Queue != "" || c.Push {
t.Fatalf("the worker is pushed (queue %q, push %v); a holder pulls when it is free", c.Queue, c.Push)
}
if c.MaxAckPending != 0 {
t.Fatalf("the worker caps asks in flight at %d; pulling bounds them by the holders working", c.MaxAckPending)
}
if c.Name != "SEAT_NODE_BUILD_AGENT_worker" || c.Stream != "SEAT_NODE_BUILD_AGENT" {
t.Fatalf("the worker is %s on %s; one per seat, shared by its holders", c.Name, c.Stream)
}
}
+11 -17
View File
@@ -62,22 +62,6 @@ func TestTheInstallersFirstUserListIsWhatTheControllerWouldCompose(t *testing.T)
// theCarriedAccounts is the accounts file the installer's template writes at genesis.
func theCarriedAccounts(t *testing.T) string {
t.Helper()
for _, r := range theTemplate(t) {
if r["id"] == "bus-accounts" {
content, _ := r["content"].(string)
if content == "" {
t.Fatal("the template's accounts file is empty, so the bus would refuse every connection")
}
return content
}
}
t.Fatal("the template carries no accounts file, so a mesh raised from it has a bus nobody may use")
return ""
}
// theTemplate is the installer's bundle, as resources.
func theTemplate(t *testing.T) []map[string]any {
t.Helper()
path := filepath.Join("..", "..", "..", "mesh-host", "examples", "foundation-first-node-nats.lock")
raw, err := os.ReadFile(path)
@@ -97,7 +81,17 @@ func theTemplate(t *testing.T) []map[string]any {
if err := json.Unmarshal([]byte(strings.Join(lines, "\n")), &bundle); err != nil {
t.Fatalf("the template is not readable: %v", err)
}
return bundle.Resources
for _, r := range bundle.Resources {
if r["id"] == "bus-accounts" {
content, _ := r["content"].(string)
if content == "" {
t.Fatal("the template's accounts file is empty, so the bus would refuse every connection")
}
return content
}
}
t.Fatal("the template carries no accounts file, so a mesh raised from it has a bus nobody may use")
return ""
}
// subjectsIn reads one allow-list out of a composed accounts file.
-106
View File
@@ -1,106 +0,0 @@
package broker
import (
"strings"
"testing"
)
// A module that says it calls a tool may publish exactly that subject (novox/hq ADR 0152): the same
// grant a person gets, derived the same way, so one list answers "what may this ask" for everybody.
func TestAModuleMayAskOnlyTheToolsItInvokes(t *testing.T) {
perms, err := PermissionsFor(Principal{Kind: KindModule, Node: "desk", Module: "mesh-console",
Invokes: []string{"shop.price"}, PasswordHash: "x"})
if err != nil {
t.Fatal(err)
}
has(t, perms.Publish, "mesh.mod.shop.tool.price")
hasNot(t, perms.Publish, "mesh.mod.shop.tool.refund")
hasNot(t, perms.Publish, "mesh.mod.*.tool.>")
}
// The console's grant: every tool, as one subject, and it reads as one.
func TestAModuleInvokingEverythingMayAskAnyTool(t *testing.T) {
perms, err := PermissionsFor(Principal{Kind: KindModule, Node: "desk", Module: "mesh-console",
Invokes: []string{"*"}, PasswordHash: "x"})
if err != nil {
t.Fatal(err)
}
has(t, perms.Publish, "mesh.mod.*.tool.>")
}
// **A grant to call widens nothing else.** A module that invokes may not publish an event it did not
// declare, may not answer as another module, and subscribes nothing it did not consume — the
// difference between the console and a person is that the console is on a machine, not that it may
// do more.
func TestInvokingGrantsNothingButTheCall(t *testing.T) {
perms, err := PermissionsFor(Principal{Kind: KindModule, Node: "desk", Module: "mesh-console",
Invokes: []string{"*"}, PasswordHash: "x"})
if err != nil {
t.Fatal(err)
}
for _, p := range perms.Publish {
if strings.Contains(p, ".event.") {
t.Errorf("a module that only invokes may publish %q, an event it never declared", p)
}
// A role's tools are tools (ADR 0132); a role's work queue and events are not.
if strings.HasPrefix(p, "mesh.seat.") && !strings.Contains(p, ".tool.") {
t.Errorf("a module that only invokes may publish %q, a seat it neither holds nor uses", p)
}
}
for _, s := range perms.Subscribe {
if strings.Contains(s, ".tool.") && !strings.HasPrefix(s, "mesh.mod.mesh-console.") {
t.Errorf("a module that invokes may subscribe %q, another module's tools", s)
}
}
}
// A module that declares no invokes calls nothing, which is every module but the console.
func TestAModuleThatInvokesNothingCallsNothing(t *testing.T) {
perms, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "shop",
Emits: []string{"order.placed"}, PasswordHash: "x"})
if err != nil {
t.Fatal(err)
}
for _, p := range perms.Publish {
if strings.Contains(p, ".tool.") {
t.Errorf("a module with no invokes may publish %q", p)
}
}
}
// The malformed entry is refused for a module as it is for a person, and in the same words.
func TestAModulesToolGrantThatNamesNoToolIsRefused(t *testing.T) {
if _, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "shop",
Invokes: []string{"telegram"}, PasswordHash: "x"}); err == nil {
t.Fatal("a grant naming a module but no tool was accepted")
}
}
// What a declaration says reaches the composed user, so a manifest's `invokes` is the grant.
func TestADeclaredInvokeReachesTheComposedUser(t *testing.T) {
users, err := Users(Records{
Nodes: []string{"desk"},
Assigned: map[string][]Declared{"desk": {{Module: "mesh-console", Invokes: []string{"*"}}}},
})
if err != nil {
t.Fatal(err)
}
perms, err := PermissionsFor(users[len(users)-1])
if err != nil {
t.Fatal(err)
}
has(t, perms.Publish, "mesh.mod.*.tool.>")
}
// A module's tool is addressed two ways (novox/hq ADR 0159): to whichever instance answers, and to
// the instance on one machine. A grant for the tool covers both and nothing wider.
func TestInvokingAToolMayAddressTheMachineToo(t *testing.T) {
got, err := invokedSubjects([]string{"postgres.postgres_query"})
if err != nil {
t.Fatal(err)
}
want := []string{"mesh.mod.postgres.tool.postgres_query", "mesh.mod.postgres.tool.postgres_query.*"}
if len(got) != 2 || got[0] != want[0] || got[1] != want[1] {
t.Fatalf("the grant is %v, want %v", got, want)
}
}
+2 -107
View File
@@ -26,16 +26,6 @@ import (
type JetStream struct {
conn *nats.Conn
js nats.JetStreamContext
// Note is how this says something it decided not to fail over. Nil is silent, which is only
// right for a caller that has no way to report; the controller sets it.
Note func(string, ...any)
}
// note reports without requiring a caller to have set one.
func (j *JetStream) note(format string, args ...any) {
if j.Note != nil {
j.Note(format, args...)
}
}
// Dial connects and returns the controller's JetStream handle.
@@ -144,7 +134,6 @@ func (j *JetStream) EnsureStream(s Stream) error {
MaxMsgsPerSubject: int64(s.MaxMsgsPerSubject),
Description: s.Why,
}
want.AllowDirect = s.Direct
if s.Retention == RetentionLastPerSubject {
// Last-per-subject is a limits stream with one message kept per subject, not a
// retention policy of its own — the state shape, spelled the way the server spells it.
@@ -180,7 +169,6 @@ func (j *JetStream) EnsureConsumer(c Consumer) error {
AckPolicy: nats.AckExplicitPolicy,
AckWait: time.Duration(c.AckWaitSeconds) * time.Second,
MaxDeliver: c.MaxDeliver,
MaxAckPending: c.MaxAckPending,
DeliverGroup: c.Queue,
DeliverSubject: "",
Description: c.Why,
@@ -196,105 +184,12 @@ func (j *JetStream) EnsureConsumer(c Consumer) error {
// without the other is refused by the server with a message that does not say which half is
// missing.
if c.Queue != "" || c.Push {
// **Per consumer, which means per stream as well as per name** (novox/hq 04-ISSUES/146).
// A push consumer delivers onto an ordinary subject, and everything subscribed to that
// subject gets a copy. The controller holds a consumer called `controller` on CONTROL and
// another called `controller` on EVENTS, and both were given `_DELIVER.controller` — so the
// one process, holding both subscriptions, acted on every message twice. It enrolled a
// joining machine twice from one request, minting a second credential that replaced the one
// the machine had just been given; the same doubling applied to every report and every
// event the controller follows.
//
// The stream is in the name because the pair is what identifies a consumer — the server
// scopes a durable's name to its stream, and this subject is the only place that scoping
// was dropped. Already within what the controller may subscribe (`_DELIVER.controller.>`),
// so no permission moves.
want.DeliverSubject = DeliverSubjectFor(c)
want.DeliverSubject = "_DELIVER." + c.Name
}
switch have, err := j.js.ConsumerInfo(c.Stream, c.Name); {
switch _, err := j.js.ConsumerInfo(c.Stream, c.Name); {
case err == nil:
// **The controller owns the worker's shape, type included** (novox/hq issue 206). A holder
// built for a pull worker cannot bind a push one — `cannot pull subscribe to push based
// consumer` — and on 2026-10-03 the build machine rolled before the controller that would
// have redefined its worker, restarted on that for an hour, and nothing could build the
// controller that would have ended it. The server cannot change a consumer's type in place,
// so one of the wrong type is re-made: on a work queue nothing is lost, because what was
// acknowledged is gone from the stream and what was not is delivered again from the start.
// On any other stream a re-made consumer would replay what this one acknowledged (issue
// 156), so there it is said and left, and the person re-makes it knowing the cost.
if havePush, wantPush := have.Config.DeliverSubject != "", want.DeliverSubject != ""; havePush != wantPush {
shape := func(push bool) string {
if push {
return "push"
}
return "pull"
}
info, err := j.js.StreamInfo(c.Stream)
if err != nil {
return fmt.Errorf("asking about stream %s to re-make consumer %s: %w", c.Stream, c.Name, err)
}
if info.Config.Retention != nats.WorkQueuePolicy {
// **A stream that keeps its history is re-made from now on, never from the start.**
// Left for a hand, the hand re-makes it with the server's default — everything the
// stream holds — which on 2026-10-03 replayed every build ask since 1 October and
// re-registered nine modules from the past (novox/hq issue 207). What this consumer
// had not yet acknowledged is lost with it, and said: on a history stream that is
// the smaller cost, and the asks in flight are visible to whoever asked.
j.note("consumer %s on %s changes from %s to %s delivery on a stream that keeps its history: "+
"re-made to deliver from now on, so nothing this one acknowledged comes back (novox/hq issue "+
"207); %d ask(s) it had not acknowledged are not carried over and must be asked again",
c.Name, c.Stream, shape(havePush), shape(wantPush), have.NumPending+uint64(have.NumAckPending))
want.DeliverPolicy = nats.DeliverNewPolicy
} else {
j.note("consumer %s on %s changes from %s to %s delivery: re-made where it left off, nothing "+
"acknowledged comes back and nothing pending is lost (novox/hq issue 206); a holder bound to "+
"the old shape binds again", c.Name, c.Stream, shape(havePush), shape(wantPush))
}
if err := j.js.DeleteConsumer(c.Stream, c.Name); err != nil {
return fmt.Errorf("re-making consumer %s on %s as %s: %w", c.Name, c.Stream, shape(wantPush), err)
}
if _, err := j.js.AddConsumer(c.Stream, want); err != nil {
return fmt.Errorf("re-making consumer %s on %s as %s: %w", c.Name, c.Stream, shape(wantPush), err)
}
return nil
}
// Where an existing consumer starts is its history, not something an assertion may move:
// the server refuses a changed deliver policy outright. Carried across, so asserting twice
// is the no-op a restart depends on.
want.DeliverPolicy = have.Config.DeliverPolicy
want.OptStartSeq = have.Config.OptStartSeq
want.OptStartTime = have.Config.OptStartTime
if _, err := j.js.UpdateConsumer(c.Stream, want); err != nil {
// **A consumer that works is not replaced to make its name tidier**
// (novox/hq 04-ISSUES/156).
//
// The server will not move a push consumer's delivery subject while a subscriber is
// bound to it, and answers `consumer name already in use` — a message about the name,
// for a conflict about the subject. A node is bound to its declaration consumer the
// whole time it is up; that IS a node listening. So when 04-ISSUES/146 put the stream
// into the subject, every node consumer in a running mesh became one this could not
// bring to match, and the control plane crash-looped on the assertion it makes before
// it serves. A fresh mesh showed nothing: nothing was bound.
//
// Kept rather than deleted and re-made. Re-making moves the subject, and a holder may
// not be allowed to subscribe to the new one yet — the wider grant travels in the bus's
// user list, which this same control plane composes and a machine applies minutes
// later. Re-making here would have silenced every machine in the mesh, which is worse
// than the collision it was fixing and harder to undo.
//
// Kept rather than fatal, which is what 146's change intended and did not do: the bare
// subject it replaces still delivers, and it collides only where one holder has two
// consumers of one name. That is the controller's own pair, and the controller is not
// bound to them while it asserts, so those do move. A node has one consumer and nothing
// to collide with.
if have.Config.DeliverSubject != want.DeliverSubject {
j.note("consumer %s on %s still delivers to %q and not %q: %v. It keeps working; "+
"the subject moves on an assertion made while nothing is bound to it",
c.Name, c.Stream, have.Config.DeliverSubject, want.DeliverSubject, err)
return nil
}
return fmt.Errorf("bringing consumer %s on %s to match: %w", c.Name, c.Stream, err)
}
return nil
-143
View File
@@ -1,143 +0,0 @@
package broker
import (
"encoding/json"
"sort"
"strings"
)
// What the mesh issues an assignment to serve and to reach (novox/hq ADR 0160).
//
// A module's code names its tools and its events; **where they land is the mesh's to decide**, and
// it decided it twice — once in the runtime, once here, by one rule compiled into both. Now the
// controller composes a membership for every module on every machine and publishes it to a subject
// only that assignment reads; the runtime serves exactly what the membership says, and the account's
// grant is the same composition read the other way. The shape issued today is the shape the mesh
// already had, so nothing moves when a membership first arrives; only who decides it moves.
// Membership is one assignment's subjects: what this instance of a module on this machine serves,
// and what it may reach.
type Membership struct {
Node string `json:"node"`
Module string `json:"module"`
// Serves is every address a tool of this instance answers on. `{tool}` stands for the tool's
// own name, which the module knows and the mesh does not need to: the mesh issues the address,
// the runtime fills the name. An address with a queue is shared with the module's other
// instances, and the bus hands each call to one of them; an address without is this instance's.
Serves []Served `json:"serves"`
// Seats is every verb of a seat this instance holds, at the subject the seat's callers use.
Seats []SeatServed `json:"seats,omitempty"`
// Emits is where an event of this module lands; `{event}` stands for the event's name.
Emits string `json:"emits"`
// Reaches is each tool this module may call, `<module>.<tool>`, to the subjects that reach it:
// the first is whichever instance answers, when the mesh issued one; the rest name a machine.
Reaches map[string][]string `json:"reaches,omitempty"`
// Tools is where this instance answers what it serves — the runtime's one verb of its own.
Tools string `json:"tools"`
// Receives is what this assignment is given for each requirement it receives, by requirement:
// the contributions of every module that asked for it, as the catalogue composed them (novox/hq
// ADR 0167). The same list its received file is written from, so the two cannot disagree; a
// requirement nobody contributed to is an empty list, never absent. Kept as JSON because the
// catalogue owns the shape of a contribution and the bus only carries it.
Receives map[string]json.RawMessage `json:"receives,omitempty"`
// Mesh is every machine's address on the private network — what a rule saying "from the mesh"
// resolves to in the packet filter, issued here from the same list (novox/hq ADR 0167). A
// module that must tell the mesh from the world, the route proxy serving an internal name, reads
// it here rather than keeping a definition of its own.
Mesh []string `json:"mesh,omitempty"`
}
// Served is one address a tool is answered on.
type Served struct {
Subject string `json:"subject"`
Queue string `json:"queue,omitempty"`
}
// SeatServed is one verb of a held seat, where its callers ask.
type SeatServed struct {
Seat string `json:"seat"`
Verb string `json:"verb"`
Subject string `json:"subject"`
}
// MembershipSubject is the one address a runtime derives for itself: where its own membership is
// published, from the two names its credential carries. Everything else is in the membership.
func MembershipSubject(node, module string) string {
return "mesh.assignment." + node + "." + module
}
// Placements is where every module runs, for deciding which instance answers for the module.
type Placements struct {
// Nodes is each module's machines.
Nodes map[string][]string
// Interchangeable is each module whose definition says its instances are the same anywhere,
// so the module's plain subject is issued to all of them in one queue.
Interchangeable map[string]bool
}
// AnswersForTheModule says whether an instance of a module on one machine is issued the module's
// plain subject: when it is the only instance, or when the definition says instances are
// interchangeable. A stateful module on two machines gets only its machines' subjects, so a call
// that names none reaches nothing rather than the wrong store.
func (p Placements) AnswersForTheModule(module string) bool {
return len(p.Nodes[module]) <= 1 || p.Interchangeable[module]
}
// MembershipFor composes one assignment's membership from what it declared and where everything
// runs. The subjects are the ones PermissionsFor grants, derived here once more only until the
// grant itself is read from the membership — which is the next step, not this one.
func MembershipFor(node string, d Declared, where Placements) Membership {
own := "mesh.mod." + d.Module
m := Membership{
Node: node, Module: d.Module,
Emits: own + ".event.{event}",
Tools: own + ".tool.tools",
}
// This machine's address always; the module's when this instance answers for the module.
m.Serves = append(m.Serves, Served{Subject: own + ".tool.{tool}." + node})
if where.AnswersForTheModule(d.Module) {
m.Serves = append(m.Serves, Served{Subject: own + ".tool.{tool}", Queue: "serve." + d.Module})
}
for _, s := range d.Holds {
for _, verb := range s.Serves {
m.Seats = append(m.Seats, SeatServed{Seat: s.Name, Verb: verb, Subject: seatToolSubject(s, verb, node)})
}
}
if len(d.Invokes) > 0 {
m.Reaches = map[string][]string{}
for _, t := range d.Invokes {
if t == "*" || strings.HasPrefix(t, "seat:") {
continue // every tool, or a role's: addressed by name, not resolved per instance
}
module, tool, ok := strings.Cut(t, ".")
if !ok {
continue
}
var reach []string
if where.AnswersForTheModule(module) {
reach = append(reach, "mesh.mod."+module+".tool."+tool)
}
nodes := append([]string{}, where.Nodes[module]...)
sort.Strings(nodes)
for _, n := range nodes {
reach = append(reach, "mesh.mod."+module+".tool."+tool+"."+n)
}
m.Reaches[t] = reach
}
}
return m
}
// PlacementsOf reads where everything runs from the records the bus's accounts are composed from.
func PlacementsOf(r Records, interchangeable map[string]bool) Placements {
p := Placements{Nodes: map[string][]string{}, Interchangeable: interchangeable}
for node, declared := range r.Assigned {
for _, d := range declared {
p.Nodes[d.Module] = append(p.Nodes[d.Module], node)
}
}
for _, nodes := range p.Nodes {
sort.Strings(nodes)
}
return p
}
-109
View File
@@ -1,109 +0,0 @@
package broker
import (
"reflect"
"testing"
)
// The mesh issues an assignment's subjects (novox/hq ADR 0160): a module alone on one machine
// answers for the module and for its machine; a stateful module on two machines answers only for
// each machine; one that says its instances are interchangeable answers for the module everywhere;
// a holder serves its seat's verbs; and what a module may reach is resolved the same way.
func TestAMembershipIsIssuedFromWhereEverythingRuns(t *testing.T) {
records := Records{Assigned: map[string][]Declared{
"anchor": {
{Module: "postgres", Serves: []string{"query"}, Holds: []Seat{{Name: "mesh-store", Scope: "mesh", Serves: []string{"databases", "query"}}}},
{Module: "catalog", Invokes: []string{"postgres.query", "search.find"}},
},
"home-server": {
{Module: "postgres"},
{Module: "search"},
{Module: "dashboard", Invokes: []string{"postgres.query"}},
},
"laptop": {{Module: "search"}},
}, Interchangeable: map[string]bool{"search": true}}
where := PlacementsOf(records, records.Interchangeable)
pg := MembershipFor("anchor", records.Assigned["anchor"][0], where)
if !reflect.DeepEqual(pg.Serves, []Served{{Subject: "mesh.mod.postgres.tool.{tool}.anchor"}}) {
t.Fatalf("a stateful module on two machines answers only for its machine: %+v", pg.Serves)
}
if len(pg.Seats) != 2 || pg.Seats[0].Subject != "mesh.seat.mesh-store.tool.databases" {
t.Fatalf("the holder serves the seat's verbs at the seat's subjects: %+v", pg.Seats)
}
if pg.Emits != "mesh.mod.postgres.event.{event}" || pg.Tools != "mesh.mod.postgres.tool.tools" {
t.Fatalf("events and the tools verb: %+v", pg)
}
search := MembershipFor("laptop", records.Assigned["laptop"][0], where)
if !reflect.DeepEqual(search.Serves, []Served{
{Subject: "mesh.mod.search.tool.{tool}.laptop"},
{Subject: "mesh.mod.search.tool.{tool}", Queue: "serve.search"},
}) {
t.Fatalf("an interchangeable module answers for the module in the queue too: %+v", search.Serves)
}
dashboard := MembershipFor("home-server", records.Assigned["home-server"][2], where)
if !reflect.DeepEqual(dashboard.Serves, []Served{
{Subject: "mesh.mod.dashboard.tool.{tool}.home-server"},
{Subject: "mesh.mod.dashboard.tool.{tool}", Queue: "serve.dashboard"},
}) {
t.Fatalf("a module alone on one machine answers for the module: %+v", dashboard.Serves)
}
if !reflect.DeepEqual(dashboard.Reaches["postgres.query"],
[]string{"mesh.mod.postgres.tool.query.anchor", "mesh.mod.postgres.tool.query.home-server"}) {
t.Fatalf("reaching a stateful module names each machine and no plain subject: %v", dashboard.Reaches)
}
catalog := MembershipFor("anchor", records.Assigned["anchor"][1], where)
if !reflect.DeepEqual(catalog.Reaches["search.find"],
[]string{"mesh.mod.search.tool.find", "mesh.mod.search.tool.find.home-server", "mesh.mod.search.tool.find.laptop"}) {
t.Fatalf("reaching an interchangeable module offers the plain subject first: %v", catalog.Reaches)
}
if MembershipSubject("anchor", "postgres") != "mesh.assignment.anchor.postgres" {
t.Fatal("the one subject a runtime derives for itself")
}
}
func TestAnAccountMayReadItsOwnMembershipAndNoOthers(t *testing.T) {
perms, err := PermissionsFor(Principal{Kind: KindModule, Node: "anchor", Module: "postgres", PasswordHash: "x"})
if err != nil {
t.Fatal(err)
}
has(t, perms.Subscribe, "mesh.assignment.anchor.postgres")
has(t, perms.Publish, "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.anchor.postgres")
hasNot(t, perms.Subscribe, "mesh.assignment.>")
}
// The runtime arriving on a machine changes nothing about what each module is issued (to-be 38 WP2):
// the memberships are composed as before and the runtime reads several of them. What the machine's
// user list gains is one runtime principal, and loses nothing but the runtime module's own.
func TestTheRuntimeArrivingLeavesEveryMembershipAsItWas(t *testing.T) {
filter := Seat{Name: "node-packet-filter", Scope: "node", Serves: []string{"rules", "reload"}}
three := []Declared{
{Module: "nftables", Holds: []Seat{filter}, Serves: []string{"firewall_rules"}},
{Module: "zsh", Serves: []string{"execute"}},
{Module: "systemd", Serves: []string{"units"}},
}
before := Records{Nodes: []string{"anchor"}, Assigned: map[string][]Declared{"anchor": three}}
after := Records{Nodes: []string{"anchor"}, Assigned: map[string][]Declared{
"anchor": append(append([]Declared{}, three...), Declared{Module: RuntimeModule}),
}}
for _, d := range three {
was := MembershipFor("anchor", d, PlacementsOf(before, nil))
is := MembershipFor("anchor", d, PlacementsOf(after, nil))
if !reflect.DeepEqual(was, is) {
t.Errorf("%s's membership changed when the runtime arrived:\n%+v\n%+v", d.Module, was, is)
}
}
users, err := Users(after)
if err != nil {
t.Fatal(err)
}
kinds := map[Kind]int{}
for _, p := range users {
kinds[p.Kind]++
}
if kinds[KindNodeTools] != 1 || kinds[KindModule] != 3 || kinds[KindNode] != 1 || kinds[KindController] != 1 {
t.Errorf("the machine's users are %v; one runtime, the three modules, the host and the controller", kinds)
}
}
+30 -263
View File
@@ -34,28 +34,12 @@ const (
// authority is a list of tools and nothing else — not control, not declarations, not builds,
// and no ability to answer anything, because a person asks.
KindPerson Kind = "person"
// KindNodeTools is a machine's tool runtime (novox/hq ADR 0175, to-be 38): one process per
// node, on the host side, serving every assigned module's tools and every held seat's verbs.
// Its authority is the union of what the modules it carries would each have had for their
// tools — and nothing of what they consume, because tools are what it runs, not reactions.
KindNodeTools Kind = "node-tools"
)
// RuntimeModule is the module that IS the node's tool runtime (novox/hq ADR 0175). Where it is
// assigned, the mesh composes one runtime principal for the machine in place of that module's own,
// and the per-module containers that served tools until then stop being the way tools reach a node.
// Mirrored in the catalogue package, which the agreement test holds to the same string; one
// constant, so a rename is one edit and the two packages cannot drift.
const RuntimeModule = "node-tools"
// Seat is a role on the bus as a principal relates to it: the subjects it accepts, and those it
// emits (novox/hq ADR 0118, design 29 §5).
type Seat struct {
Name string
// Scope is where the seat has one holder. A node-scoped seat's tool carries the node in its
// subject, because one subject reaching six machines' holders is not an address
// (novox/hq ADR 0132, design 33 §4). Empty reads as mesh.
Scope string
Name string
Accepts []string
Emits []string
Serves []string
@@ -86,21 +70,12 @@ type Principal struct {
// a namespace no such module owns. Every service started and the graph stayed empty.
Watches []Seat
// Carries are the modules whose tools this principal serves, for a KindNodeTools principal
// (novox/hq ADR 0175): every module assigned to its node, as each declares itself. Its
// serving authority is the union of theirs — each module's own tool namespace and each held
// seat's verbs on this node — derived from the same declarations the modules' own principals
// are, so the runtime can serve nothing a module could not have served for itself.
Carries []Declared
// Invokes are the tools this principal may call, as `<module>.<tool>`; a single `*` is every
// tool. A person's whole authority (design 25 §7), and a module's only if its manifest says so
// (novox/hq ADR 0152) — the console's does, and nothing else's.
// Invokes are the tools a person may call, as `<module>.<tool>`; a single `*` is every tool,
// for an administrator. Only meaningful for KindPerson.
//
// **A list, not a role.** A person is not a module and holds no seat: nothing is addressed
// to them, nothing is delivered to them, and they have no durable consumer to acknowledge.
// What they have is permission to ask. A module that invokes gains exactly the same
// permission and nothing beside it.
// What they have is permission to ask.
Invokes []string
// PasswordHash is the bcrypt hash the mesh minted. The plaintext is sealed to the principal
@@ -110,13 +85,10 @@ type Principal struct {
PasswordHash string
}
// seatsTheControllerAsks are the roles the mesh's own flows submit work to. Named rather than
// meshSeatsTheControllerUses are the roles the mesh's own flows submit work to. Named rather than
// derived from the seat set: the controller is not a module and declares no `uses`, so its side of a
// seat has to be stated, and a list is what makes "which roles does the mesh itself talk to" answerable.
// Both build roles while the handover runs (novox/hq ADR 0190): the controller asks whichever has a
// holder, and the retired one has one until build-agent replaces the builder. The second entry
// goes with the retired seat row.
var seatsTheControllerAsks = []string{"node-build-agent", "mesh-build-machine"}
var meshSeatsTheControllerUses = []string{"mesh-build-machine"}
// enrolmentPrefix is the space every enrolling node's user and inbox live under, so the one place the
// controller may answer an enrolment is derived from the same constant the user is named from.
@@ -134,10 +106,7 @@ func (p Principal) Username() string {
switch p.Kind {
case KindPerson:
return "person." + p.Module
case KindModule, KindNodeTools:
// The runtime is named exactly as the module it stands for would have been: the mesh
// issues its credential through the same path a module's takes (`module issue`), and
// that path knows the node and the module, not the kind.
case KindModule:
return p.Node + "." + p.Module
case KindNode:
return "node." + p.Node
@@ -198,10 +167,8 @@ func PermissionsFor(p Principal) (Permissions, error) {
switch p.Kind {
case KindController:
// The controller owns the mesh's own traffic and the streams. It is the only writer of
// stream definitions (design 25 §3), so it alone reaches the JetStream API — and it alone
// issues memberships (novox/hq ADR 0160), which it publishes into the assignments stream
// after each push; refused by the server on 2026-10-01 until this line named them.
pub = []string{"mesh.control.>", "mesh.node.>", "mesh.assignment.>", "$JS.API.>"}
// stream definitions (design 25 §3), so it alone reaches the JetStream API.
pub = []string{"mesh.control.>", "mesh.node.>", "$JS.API.>"}
// **And where its consumers deliver.** A push consumer delivers on `_DELIVER.<its name>`,
// and a client bound to it subscribes exactly that; the server refused it for every
// principal the first time one bound a consumer (2026-09-28). Each kind below is granted
@@ -211,9 +178,7 @@ func PermissionsFor(p Principal) (Permissions, error) {
// Work the mesh's own flows submit to a role, and the outcomes they wait on (ADR 0121). A
// build is the one today: the controller asks, and reads the answer from the seat's event
// like the catalogue does — which is why no holder needs to publish into anybody's inbox.
// A node-scoped seat's work subject carries no node (novox/hq ADR 0190): the ask goes to
// the role, and whichever machine holding it is idle takes it.
for _, seat := range seatsTheControllerAsks {
for _, seat := range meshSeatsTheControllerUses {
pub = append(pub, "mesh.seat."+seat+".accept.>")
}
// **And what the mesh says it did** (novox/hq ADR 0134). The control plane states its own
@@ -230,15 +195,6 @@ func PermissionsFor(p Principal) (Permissions, error) {
// the new bus was refused the publish (2026-09-28).
pub = append(pub, "mesh.mod.*.tool.>")
// **And the mesh's own verbs, as the seat it holds** (novox/hq ADR 0132, ADR 0154):
// `status`, `push`, `assign` are the mesh-controller seat's tools, served by its holder. The
// whole verb namespace of its own seat rather than a list: the list is the seat's protocol,
// which this package mirrors rather than reads, and a verb the seat does not declare is a
// subject nothing publishes.
sub = append(sub, "mesh.seat."+ControllerSeat+".tool.>")
// And says so (novox/hq ADR 0197): it answers discovery for the seat it serves.
sub = append(sub, announcing(ControllerSeat)...)
// The two events it reacts to, and its ack subject on the stream they arrive from
// (streams.go). **Each named, not a pattern**: `mesh.mod.*.event.>` would make the
// controller a subscriber to every event in the mesh, and its permission list would stop
@@ -268,14 +224,18 @@ func PermissionsFor(p Principal) (Permissions, error) {
case KindPerson:
// Tools, and nothing else. Every subject a person may publish is a tool call; a person
// who could publish an event would be able to claim a module said something.
invoked, err := invokedSubjects(p.Invokes)
if err != nil {
return Permissions{}, err
for _, t := range p.Invokes {
if t == "*" {
pub = append(pub, "mesh.mod.*.tool.>")
continue
}
module, tool, ok := strings.Cut(t, ".")
if !ok {
return Permissions{}, fmt.Errorf(
"%q does not name a tool: a person invokes <module>.<tool>, or * for every one", t)
}
pub = append(pub, "mesh.mod."+module+".tool."+tool)
}
pub = append(pub, invoked...)
// And may ask what answers (novox/hq ADR 0197): a question every service answers about
// itself, its replies to the asker's own inbox.
pub = append(pub, discovering()...)
case KindEnrolment:
// A leaked token is useless for anything but enrolling: it cannot read a declaration, hear
@@ -309,13 +269,7 @@ func PermissionsFor(p Principal) (Permissions, error) {
"mesh.control." + p.Node + ".>",
"$JS.API.CONSUMER.INFO.NODES." + p.Node,
}
// The deliver subject carries the stream as well as the consumer's name, so what a
// subscriber is permitted has to carry it too (novox/hq 04-ISSUES/146). The bare name
// stays: an existing consumer keeps delivering where it always did until the controller's
// next assertion moves it, and a permission that only allowed the new shape would refuse
// every node in the mesh for exactly as long as that took.
sub = []string{"mesh.node." + p.Node + ".declare",
"_DELIVER." + p.Node, "_DELIVER." + p.Node + ".>"}
sub = []string{"mesh.node." + p.Node + ".declare", "_DELIVER." + p.Node}
case KindModule:
// 1. Its own namespace: it publishes its events there and serves its tools there. Nothing
@@ -332,29 +286,6 @@ func PermissionsFor(p Principal) (Permissions, error) {
// away — no other principal may subscribe this namespace, and a caller's authority is
// still granted per tool, by name, on the publish side.
sub = append(sub, own+".tool.>")
// It says what it serves (novox/hq ADR 0197): discovery for its own name and every seat it
// holds a verb of, answered by the runtime that serves them.
announced := []string{p.Module}
for _, s := range p.Holds {
if len(s.Serves) > 0 {
announced = append(announced, s.Name)
}
}
sub = append(sub, announcing(announced...)...)
// Its own membership (ADR 0160): the one subject a runtime derives for itself, read
// directly from the stream and followed live. Nothing else's.
sub = append(sub, MembershipSubject(p.Node, p.Module))
pub = append(pub, "$JS.API.DIRECT.GET."+AssignmentsStream+"."+MembershipSubject(p.Node, p.Module))
// 1b. The tools it calls, if its manifest says it calls any (novox/hq ADR 0152). The same
// grant a person gets and derived the same way, so "what may this module ask" is
// answered by the one list that answers it for everybody. Publish only: an answer
// arrives on its own inbox, which every principal has below.
invoked, err := invokedSubjects(p.Invokes)
if err != nil {
return Permissions{}, err
}
pub = append(pub, invoked...)
// 2. What it consumes, by the emitter's own subject — an event is addressed to its
// emitter, because the emitter's identity is the meaning (ADR 0118).
@@ -387,17 +318,13 @@ func PermissionsFor(p Principal) (Permissions, error) {
// 3. Seats it holds: full participation.
for _, s := range p.Holds {
// Taking work from the role's queue: the worker consumer every holder shares (asked
// about, pulled from, acknowledged), on the seat's own stream (novox/hq ADR 0190). A
// holder pulls — asks the consumer for its next message, answered on its own inbox —
// so what it needs is MSG.NEXT on that worker and nothing delivered to it. The first
// machine to take work over the new bus was refused the asking (2026-09-28).
// Taking work from the role's queue: the worker consumer it binds (asked about,
// delivered on, acknowledged), each on the seat's own stream. The first machine to
// take work over the new bus was refused the asking (2026-09-28).
worker := "SEAT_" + upperSnake(s.Name) + "_worker"
stream := seatStreamName(s.Name)
pub = append(pub,
"$JS.API.CONSUMER.INFO."+stream+"."+worker,
"$JS.API.CONSUMER.MSG.NEXT."+stream+"."+worker,
"$JS.ACK."+stream+"."+worker+".>")
sub = append(sub, "_DELIVER."+worker)
pub = append(pub, "$JS.API.CONSUMER.INFO."+stream+"."+worker, "$JS.ACK."+stream+"."+worker+".>")
for _, a := range s.Accepts {
sub = append(sub, seatSubject(s, "accept", a))
}
@@ -405,7 +332,7 @@ func PermissionsFor(p Principal) (Permissions, error) {
pub = append(pub, seatSubject(s, "event", e))
}
for _, t := range s.Serves {
sub = append(sub, seatToolSubject(s, t, p.Node))
sub = append(sub, seatSubject(s, "tool", t))
}
}
@@ -417,78 +344,9 @@ func PermissionsFor(p Principal) (Permissions, error) {
pub = append(pub, seatSubject(s, "accept", a))
}
for _, t := range s.Serves {
pub = append(pub, seatToolSubject(s, t, "*"))
pub = append(pub, seatSubject(s, "tool", t))
}
}
case KindNodeTools:
// **One process serves what every module on the machine would have served for itself**
// (novox/hq ADR 0175). Each carried module's whole tool namespace — the same grant that
// module's own principal has, for the same reason: the tools a module serves are what its
// code answers, and a list here would be a second copy of it. Each held seat's verbs on
// this node, as the holder's own principal would be granted them.
var serves []string
for _, d := range p.Carries {
if !safeSubject.MatchString(d.Module) {
return Permissions{}, fmt.Errorf(
"%q cannot be part of a subject: a permission is a subject pattern, and this would widen it", d.Module)
}
serves = append(serves, d.Module)
for _, s := range d.Holds {
serves = append(serves, s.Name)
}
own := "mesh.mod." + d.Module
sub = append(sub, own+".tool.>")
// A tool that emits an event is the module's code and emits under the module's name
// (ADR 0042); the runtime carrying that code may publish what the module declared it
// emits, and nothing it did not.
for _, e := range d.Emits {
pub = append(pub, own+".event."+e)
}
for _, s := range d.Holds {
for _, t := range s.Serves {
sub = append(sub, seatToolSubject(s, t, p.Node))
}
}
}
// Every assigned module's membership on this node (ADR 0160): one per module, read
// directly from the stream and followed live. This node's and no other's — the one token
// that varies is the module, so the pattern is the machine's own assignments.
sub = append(sub, "mesh.assignment."+p.Node+".*")
pub = append(pub, "$JS.API.DIRECT.GET."+AssignmentsStream+".mesh.assignment."+p.Node+".*")
// And every tool on the mesh (ADR 0175, decision 5): any node may call any tool on any
// node, as the console already could — the runtime is the console's serving mode.
invoked, err := invokedSubjects([]string{"*"})
if err != nil {
return Permissions{}, err
}
pub = append(pub, invoked...)
// It says what it serves and may ask what answers (novox/hq ADR 0197): the runtime answers
// discovery for each module and seat it carries, and the console it is asks the bus.
// One service per runtime process, named for the runtime: the bus lets a principal answer each
// request once, so the runtime announces everything it carries under its own name.
sub = append(sub, announcing(append([]string{RuntimeModule}, serves...)...)...)
pub = append(pub, discovering()...)
// **And it consumes for the modules it carries** (novox/hq ADR 0198, which changes ADR 0175's
// "it consumes nothing"): a module's long-running code is a bundle this runtime launches, and
// the runtime is its bus — it reads the module's own durable consumer and acknowledges what
// the module's code took. Exactly the grants the module's own principal has for that consumer,
// on its name and no other's: asking about it, pulling from it, acknowledging it. The
// consumer is still the controller's to make, from the module's own principal.
for _, d := range p.Carries {
own := Principal{Kind: KindModule, Node: p.Node, Module: d.Module, Emits: d.Emits,
Consumes: d.Consumes, Serves: d.Serves, Holds: d.Holds, Uses: d.Uses, Watches: d.Watches}
if _, consumes := ConsumerFor(own); !consumes {
continue
}
stream, durable := consumerStream(own), consumerDurable(own)
pub = append(pub,
"$JS.API.CONSUMER.INFO."+stream+"."+durable,
"$JS.API.CONSUMER.MSG.NEXT."+stream+"."+durable,
"$JS.ACK."+stream+"."+durable+".>")
}
sub = unique(sub)
pub = unique(pub)
}
if p.Kind == KindPerson {
@@ -496,11 +354,6 @@ func PermissionsFor(p Principal) (Permissions, error) {
// consumer, because nothing is delivered to a person — they ask and are answered.
sub = append(sub, p.inbox())
}
if p.Kind == KindNodeTools {
// Its reply space, so the answers to what its tools call come back to it. No ack subject
// for the same reason a person has none: nothing is delivered to it.
sub = append(sub, p.inbox())
}
if p.Kind == KindModule || p.Kind == KindNode || p.Kind == KindController {
// Its own reply space, and nothing wider.
@@ -522,7 +375,7 @@ func PermissionsFor(p Principal) (Permissions, error) {
// A module answers what it was asked — a tool call reaches it on its own namespace, so the
// authority is bounded by having been asked — and so does the controller. A node and a
// person are never asked anything, and are granted nothing here.
AllowResponses: p.Kind == KindModule || p.Kind == KindController || p.Kind == KindNodeTools,
AllowResponses: p.Kind == KindModule || p.Kind == KindController,
}, nil
}
@@ -541,18 +394,6 @@ func seatSubject(s Seat, kind, verb string) string {
return "mesh.seat." + s.Name + "." + kind + "." + verb
}
// seatToolSubject is where a role's tool is asked. Mesh-wide for a mesh-scoped seat; a node-scoped
// seat carries the node it is asked of, because a flat subject would reach every machine's holder
// and the queue group would silently pick a winner (novox/hq ADR 0132, design 33 §4). A holder
// subscribes its own node's; a user publishes any node's (`*`) and names the machine in the subject.
func seatToolSubject(s Seat, verb, node string) string {
base := seatSubject(s, "tool", verb)
if s.Scope == "node" && node != "" {
return base + "." + node
}
return base
}
// consumerStream and consumerDurable are the two halves of a consumer's identity, and they are
// two functions because conflating them was a real bug.
//
@@ -744,20 +585,6 @@ func ComposeAccounts(principals []Principal) (string, error) {
return b.String(), nil
}
// unique is a sorted list with each subject once. Two carried modules holding seats with the same
// verb, or the runtime module itself carried beside the others, would otherwise write a grant twice
// — harmless to the server, and noise in a file that is read as the mesh's authority model.
func unique(values []string) []string {
sort.Strings(values)
out := values[:0]
for i, v := range values {
if i == 0 || v != values[i-1] {
out = append(out, v)
}
}
return out
}
func quoted(values []string) string {
if len(values) == 0 {
return ""
@@ -768,63 +595,3 @@ func quoted(values []string) string {
}
return strings.Join(out, ", ")
}
// invokedSubjects is the publish side of a grant to call tools: one subject per `<module>.<tool>`,
// or the whole tool namespace for `*`. A person's authority and a module's `invokes` are both this
// (novox/hq ADR 0152), so a malformed entry is refused in one place, before it could be widened into
// something that happens to parse.
func invokedSubjects(invokes []string) ([]string, error) {
var out []string
for _, t := range invokes {
if t == "*" {
// Every module's tools and every role's (novox/hq ADR 0132): a role's verb is a tool
// like any other, addressed to the seat instead of a module.
out = append(out, "mesh.mod.*.tool.>", "mesh.seat.*.tool.>")
continue
}
if rest, isSeat := strings.CutPrefix(t, "seat:"); isSeat {
// A role's tool, `seat:<seat>.<verb>`. Both address shapes, because the grant is
// written without knowing the seat's scope: a mesh seat's verb is flat and a node
// seat's carries the machine (design 33 §4).
seat, verb, ok := strings.Cut(rest, ".")
if !ok || seat == "" || verb == "" {
return nil, fmt.Errorf(
"%q does not name a role's tool: one invokes seat:<seat>.<verb>", t)
}
out = append(out, "mesh.seat."+seat+".tool."+verb, "mesh.seat."+seat+".tool."+verb+".*")
continue
}
module, tool, ok := strings.Cut(t, ".")
if !ok || module == "" || tool == "" {
return nil, fmt.Errorf(
"%q does not name a tool: one invokes <module>.<tool>, seat:<seat>.<verb>, or * for every one", t)
}
// Both ways a module's tool is addressed (novox/hq ADR 0159): to whichever instance
// answers, and to the instance on one machine, which is the same subject with the machine
// as its last token.
out = append(out, "mesh.mod."+module+".tool."+tool, "mesh.mod."+module+".tool."+tool+".*")
}
return out, nil
}
// announcing is what a principal that serves tools subscribes to answer the NATS services
// protocol's discovery (novox/hq ADR 0197): the questions asked of every service, and those asked of
// each name it serves — its own and no other's, so it cannot answer for a service it is not.
func announcing(names ...string) []string {
out := []string{"$SRV.PING", "$SRV.INFO", "$SRV.STATS"}
for _, n := range names {
if !safeSubject.MatchString(n) {
continue
}
for _, verb := range []string{"PING", "INFO", "STATS"} {
out = append(out, "$SRV."+verb+"."+n, "$SRV."+verb+"."+n+".>")
}
}
return out
}
// discovering is what a principal publishes to ask what answers (novox/hq ADR 0197): the services
// protocol's discovery requests, whose replies come to its own inbox.
func discovering() []string {
return []string{"$SRV.PING", "$SRV.PING.>", "$SRV.INFO", "$SRV.INFO.>"}
}
+1 -121
View File
@@ -71,18 +71,6 @@ func TestHoldingASeatIsTheMirrorOfUsingIt(t *testing.T) {
hasNot(t, perms.Publish, "mesh.seat.telegram-sender.accept.send")
}
// A build machine may say everything about a build as it happens (novox/hq ADR 0157): that it
// started, and every line under the build's own id — the seat's `log.*` becomes a publish over
// one token, so a reader follows one build by subject and the holder can name no other subject.
func TestTheBuildMachineMaySayWhatItDoesUnderTheBuildsId(t *testing.T) {
seat := Seat{Name: "mesh-build-machine", Accepts: []string{"build"}, Emits: []string{"started", "built", "log.*"}}
perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "anchor", Module: "builder",
Holds: []Seat{seat}, PasswordHash: "x"})
has(t, perms.Publish, "mesh.seat.mesh-build-machine.event.started")
has(t, perms.Publish, "mesh.seat.mesh-build-machine.event.log.*")
hasNot(t, perms.Publish, "mesh.seat.mesh-build-machine.event.>")
}
// Without an ack permission a durable consumer never really consumes: every message it receives is
// redelivered forever, refused by the permission list it already has (design 25 §4).
func TestAModuleMayAckItsOwnDeliveriesAndNoOthers(t *testing.T) {
@@ -233,9 +221,7 @@ func TestAPersonReachesNothingButTools(t *testing.T) {
perms, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo",
Invokes: []string{"*"}, PasswordHash: "x"})
for _, p := range perms.Publish {
// A tool call, or asking what answers (novox/hq ADR 0197) — a question every service
// answers about itself, which claims nothing and controls nothing.
if !strings.Contains(p, ".tool.") && !strings.HasPrefix(p, "$SRV.") {
if !strings.Contains(p, ".tool.") {
t.Errorf("a person may publish %q, which is not a tool call", p)
}
}
@@ -373,109 +359,3 @@ func TestAModulePullsItsOwnConsumerAndNoOthers(t *testing.T) {
}
}
}
// The runtime's authority is the union of what the modules it carries would have been granted for
// their tools (novox/hq ADR 0175): every carried module's tool namespace, every held seat's verbs
// on this node, every module's membership on this node, and a call to anything. Nothing it
// consumes, because it reacts to nothing.
func TestTheRuntimeServesTheUnionAndConsumesForItsModules(t *testing.T) {
filter := Seat{Name: "node-packet-filter", Scope: "node", Serves: []string{"rules", "reload"}}
p := Principal{Kind: KindNodeTools, Node: "anchor", Module: RuntimeModule, Carries: []Declared{
{Module: "nftables", Holds: []Seat{filter}, Serves: []string{"firewall_rules"}},
{Module: "zsh", Emits: []string{"shell.opened"}, Consumes: []string{"shop.order.placed"}},
{Module: RuntimeModule},
}}
perms, err := PermissionsFor(p)
if err != nil {
t.Fatal(err)
}
for _, want := range []string{
"mesh.mod.nftables.tool.>", "mesh.mod.zsh.tool.>", "mesh.mod." + RuntimeModule + ".tool.>",
"mesh.seat.node-packet-filter.tool.rules.anchor", "mesh.seat.node-packet-filter.tool.reload.anchor",
"mesh.assignment.anchor.*",
"_INBOX.anchor." + RuntimeModule + ".>",
} {
if !contains(perms.Subscribe, want) {
t.Errorf("the runtime may not subscribe %s: %v", want, perms.Subscribe)
}
}
for _, want := range []string{
"mesh.mod.*.tool.>", "mesh.seat.*.tool.>",
"$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.anchor.*",
"mesh.mod.zsh.event.shell.opened",
} {
if !contains(perms.Publish, want) {
t.Errorf("the runtime may not publish %s: %v", want, perms.Publish)
}
}
// It reads the consumer of every carried module that consumes — that module's, by its name, as
// the module's own principal could (novox/hq ADR 0198) — and of no module that consumes nothing.
for _, want := range []string{
"$JS.API.CONSUMER.INFO.EVENTS.anchor_zsh",
"$JS.API.CONSUMER.MSG.NEXT.EVENTS.anchor_zsh",
"$JS.ACK.EVENTS.anchor_zsh.>",
} {
if !contains(perms.Publish, want) {
t.Errorf("the runtime may not read zsh's consumer: %s missing from %v", want, perms.Publish)
}
}
for _, s := range perms.Publish {
if (strings.HasPrefix(s, "$JS.ACK.") || strings.Contains(s, "CONSUMER")) && !strings.Contains(s, "anchor_zsh") {
t.Errorf("the runtime was granted a consumer no carried module of it consumes on: %s", s)
}
}
// It pulls; nothing is pushed to it, and it subscribes no event subject directly.
for _, s := range perms.Subscribe {
if strings.Contains(s, ".event.") || strings.HasPrefix(s, "_DELIVER.") {
t.Errorf("the runtime was granted a delivery: %s", s)
}
}
if !perms.AllowResponses {
t.Error("the runtime answers what it is asked, and may not reply")
}
if _, needed := ConsumerFor(p); needed {
t.Error("a consumer would be made for the runtime itself; it reads its modules' consumers, never one of its own")
}
// Each subject once in each list: the file is read as the mesh's authority model. One subject may
// stand in both — the runtime answers discovery on `$SRV.INFO` and, as the console, asks it
// (novox/hq ADR 0197) — because subscribing and publishing are two different grants.
for _, list := range [][]string{perms.Subscribe, perms.Publish} {
seen := map[string]bool{}
for _, s := range list {
if seen[s] {
t.Errorf("%s is granted twice", s)
}
seen[s] = true
}
}
}
func contains(list []string, want string) bool {
for _, s := range list {
if s == want {
return true
}
}
return false
}
// A node-scoped seat's work is shared (novox/hq ADR 0190): its holder on any machine subscribes the
// seat's one work subject, with no node in it, so holders on several machines read one queue. The
// node token belongs to a seat's tools, which are asked of one machine (design 33 §4), not to its work.
func TestANodeSeatsWorkSubjectCarriesNoNode(t *testing.T) {
seat := Seat{Name: "node-build-agent", Scope: "node", Accepts: []string{"build"}, Serves: []string{"status"}}
perms, err := PermissionsFor(Principal{Kind: KindModule, Node: "anchor", Module: "build-agent", Holds: []Seat{seat}})
if err != nil {
t.Fatal(err)
}
has(t, perms.Subscribe, "mesh.seat.node-build-agent.accept.build")
hasNot(t, perms.Subscribe, "mesh.seat.node-build-agent.accept.build.anchor")
// And its tools still carry the machine.
has(t, perms.Subscribe, "mesh.seat.node-build-agent.tool.status.anchor")
// The controller asks the role, not a machine.
controller, err := PermissionsFor(Principal{Kind: KindController})
if err != nil {
t.Fatal(err)
}
has(t, controller.Publish, "mesh.seat.node-build-agent.accept.>")
}
-57
View File
@@ -1,57 +0,0 @@
package broker
import "testing"
// A node-scoped seat's tool carries the node (novox/hq ADR 0132, design 33 §4): two nodes holding one
// node-scoped seat derive two addresses, and a user of the seat may publish any node's.
func TestTwoNodesHoldingOneNodeSeatDeriveTwoToolAddresses(t *testing.T) {
seat := Seat{Name: "node-dns-resolver", Scope: "node", Serves: []string{"lookup"}}
one, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "dnsmasq", Holds: []Seat{seat}, PasswordHash: "x"})
two, _ := PermissionsFor(Principal{Kind: KindModule, Node: "two", Module: "dnsmasq", Holds: []Seat{seat}, PasswordHash: "x"})
has(t, one.Subscribe, "mesh.seat.node-dns-resolver.tool.lookup.one")
has(t, two.Subscribe, "mesh.seat.node-dns-resolver.tool.lookup.two")
hasNot(t, one.Subscribe, "mesh.seat.node-dns-resolver.tool.lookup")
hasNot(t, one.Subscribe, "mesh.seat.node-dns-resolver.tool.lookup.two")
user, _ := PermissionsFor(Principal{Kind: KindModule, Node: "three", Module: "asker", Uses: []Seat{seat}, PasswordHash: "x"})
has(t, user.Publish, "mesh.seat.node-dns-resolver.tool.lookup.*")
}
// A mesh-scoped seat's tool stays flat: nothing about it changes.
func TestAMeshSeatsToolIsAddressedToTheSeatAlone(t *testing.T) {
seat := Seat{Name: "git", Scope: "mesh", Serves: []string{"list_repos"}}
holder, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "gitea", Holds: []Seat{seat}, PasswordHash: "x"})
has(t, holder.Subscribe, "mesh.seat.git.tool.list_repos")
user, _ := PermissionsFor(Principal{Kind: KindModule, Node: "two", Module: "asker", Uses: []Seat{seat}, PasswordHash: "x"})
has(t, user.Publish, "mesh.seat.git.tool.list_repos")
}
// The controller serves its own seat's verbs and may answer them (novox/hq ADR 0154).
func TestTheControllerServesItsSeatsToolsAndMayAnswer(t *testing.T) {
perms, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
if err != nil {
t.Fatal(err)
}
has(t, perms.Subscribe, "mesh.seat.mesh-controller.tool.>")
if !perms.AllowResponses {
t.Fatal("the controller serves tools and may not answer one")
}
}
// A grant to every tool reaches a role's tools too, and a role's tool is granted by name.
func TestAGrantReachesARolesTools(t *testing.T) {
all, _ := PermissionsFor(Principal{Kind: KindModule, Node: "desk", Module: "mesh-console", Invokes: []string{"*"}, PasswordHash: "x"})
has(t, all.Publish, "mesh.seat.*.tool.>")
one, err := PermissionsFor(Principal{Kind: KindPerson, Module: "jo", Invokes: []string{"seat:mesh-controller.status"}, PasswordHash: "x"})
if err != nil {
t.Fatal(err)
}
has(t, one.Publish, "mesh.seat.mesh-controller.tool.status")
hasNot(t, one.Publish, "mesh.seat.mesh-controller.tool.push")
hasNot(t, one.Publish, "mesh.mod.*.tool.>")
if _, err := PermissionsFor(Principal{Kind: KindPerson, Module: "jo", Invokes: []string{"seat:mesh-controller"}, PasswordHash: "x"}); err == nil {
t.Fatal("a role grant naming no verb was accepted")
}
}
+4 -49
View File
@@ -47,14 +47,8 @@ type Stream struct {
// Why is carried into the assertion so an operator reading the server's own state finds the
// reason there, rather than only in a repository they may not have.
Why string
// Direct lets a client read a subject's last message without a consumer, which is how a
// runtime reads its own membership with no JetStream API beyond one request (ADR 0160).
Direct bool
}
// AssignmentsStream holds every assignment's membership, the newest per subject.
const AssignmentsStream = "ASSIGNMENTS"
// MeshStreams is the foundation set, in the order a person reads it.
//
// **CONTROL names its subjects rather than taking `mesh.control.>`**, because heartbeats live
@@ -85,15 +79,7 @@ func MeshStreams() []Stream {
"n-1 by construction (issue 107)",
},
{
Name: AssignmentsStream,
Subjects: []string{"mesh.assignment.*.*"},
Retention: RetentionLastPerSubject,
Direct: true,
Why: "one membership per assignment, always the newest: what the mesh issued this module " +
"on this machine to serve and to reach (ADR 0160); read directly by the runtime it is for",
},
{
Name: EventsStream,
Name: "EVENTS",
// A seat's own events ride here too: they are 1:many like any event, and the
// `event` token keeps them clear of both the seat's work queue (`accept`) and its
// tools (`tool`), which must not be persisted.
@@ -108,24 +94,6 @@ func MeshStreams() []Stream {
}
}
// DeliverSubjectFor is where a push consumer's messages land.
//
// **Per consumer, which means per stream as well as per name** (novox/hq 04-ISSUES/146). A push
// consumer delivers onto an ordinary subject, and everything subscribed to that subject gets a
// copy. The controller holds a consumer called `controller` on CONTROL and another called
// `controller` on EVENTS; while both were given `_DELIVER.controller`, the one process holding
// both subscriptions acted on every message twice — a joining machine was enrolled twice from one
// request, and the second enrolment minted a credential that replaced the one the machine had just
// been handed. Every report and every followed event doubled the same way, silently: nothing is
// redelivered, no count is wrong, the work simply happens twice.
//
// The stream belongs in it because the pair is what identifies a consumer — the server scopes a
// durable's name to its stream, and this subject was the one place that scoping was dropped. It
// stays inside what a controller may already subscribe (`_DELIVER.controller.>`).
func DeliverSubjectFor(c Consumer) string {
return "_DELIVER." + c.Name + "." + c.Stream
}
// An Asserter is the part of a JetStream connection stream assertion needs. Narrow on purpose: it
// keeps this testable without a server, and keeps the client library out of everything that only
// wants to know what the streams are.
@@ -217,15 +185,10 @@ var ControllerFollows = []string{
// A build's outcome, which is the build-machine role's own event now (ADR 0121) rather than a
// message on the control branch. Same three audiences, one publish: whoever asked, this, and the
// catalogue.
seatEventSubject("node-build-agent", "built"),
seatEventSubject("mesh-build-machine", "built"),
// The forge's merges: what moved a source, so the mesh builds what that source produces
// without anybody telling it (novox/hq 04-ISSUES/131). Appended, because the index is a name.
moduleEventSubject("gitea", "pull.merged"),
// The retired build role's outcome too, while the handover runs (novox/hq ADR 0190): the one
// build machine keeps answering on its seat until build-agent replaces it, and the outcome that
// registers build-agent itself comes from there. Appended, for the same reason as above; goes
// with the retired seat row.
seatEventSubject("mesh-build-machine", "built"),
}
// moduleEventSubject is where one module's event lands. The same derivation PermissionsFor uses, so
@@ -239,9 +202,6 @@ func seatEventSubject(seat, verb string) string {
return "mesh.seat." + seat + ".event." + verb
}
// EventsStream holds every module's and every role's events, a build's log among them.
const EventsStream = "EVENTS"
// MeshConsumers is what the controller consumes, in the order a person reads it.
//
// **Unlimited redelivery on CONTROL, deliberately.** The store window's bound is the controller's,
@@ -266,13 +226,8 @@ func MeshConsumers() []Consumer {
Push: true,
AckWaitSeconds: 30,
MaxDeliver: 5,
// One at a time (novox/hq issue 175): acting on a merge builds for minutes, and an
// announcement handed over behind it must wait on the server, not time out on the
// client and come back to be acted on again.
MaxAckPending: 1,
Why: "the two events the mesh's own controller reacts to, one at a time; after " +
"max-deliver it dead-letters, because an announcement it cannot act on will not " +
"become actionable",
Why: "the two events the mesh's own controller reacts to; after max-deliver it " +
"dead-letters, because an announcement it cannot act on will not become actionable",
},
}
}
+3 -42
View File
@@ -123,10 +123,9 @@ func subjectMatches(filter, subject string) bool {
// Each relationship's retention is the thing that makes it what it is (design 29 §4).
func TestEachStreamCarriesTheRetentionItsShapeNeeds(t *testing.T) {
want := map[string]Retention{
"CONTROL": RetentionWorkQueue,
"NODES": RetentionLastPerSubject,
"EVENTS": RetentionLimits,
"ASSIGNMENTS": RetentionLastPerSubject,
"CONTROL": RetentionWorkQueue,
"NODES": RetentionLastPerSubject,
"EVENTS": RetentionLimits,
}
got := map[string]Retention{}
for _, s := range MeshStreams() {
@@ -234,41 +233,3 @@ func containsStep(steps []string, want string) bool {
}
return false
}
// **Two consumers may share a name, and must not share a delivery subject** (novox/hq
// 04-ISSUES/146).
//
// A push consumer delivers onto an ordinary subject and everything subscribed to it gets a copy.
// The controller holds a consumer called `controller` on CONTROL and another called `controller` on
// EVENTS; while both were given `_DELIVER.controller`, the one process holding both subscriptions
// acted on every message twice — a joining machine enrolled twice from one request, with the second
// enrolment minting a credential that replaced the one the machine had just been handed.
//
// Checked here rather than against a server because it is a property of what the mesh asks for, and
// because the failure it produces is silent: every count is right, nothing is redelivered, and the
// work simply happens twice.
func TestNoTwoConsumersDeliverOntoTheSameSubject(t *testing.T) {
seen := map[string]string{}
for _, c := range MeshConsumers() {
if !c.Push && c.Queue == "" {
continue
}
subject := DeliverSubjectFor(c)
if other, taken := seen[subject]; taken {
t.Errorf("%s on %s and %s deliver onto %s, so whoever holds both acts on every "+
"message twice", c.Name, c.Stream, other, subject)
}
seen[subject] = c.Name + " on " + c.Stream
}
}
// The controller's events consumer is handed one announcement at a time (novox/hq issue 175): a
// merge's handler builds for minutes, and what is queued behind it must wait on the server rather
// than time out on the client and be acted on twice.
func TestTheControllerTakesOneAnnouncementAtATime(t *testing.T) {
for _, c := range MeshConsumers() {
if c.Stream == "EVENTS" && c.Name == ControllerName && c.MaxAckPending != 1 {
t.Fatalf("the events consumer may have %d outstanding; one announcement at a time", c.MaxAckPending)
}
}
}
+9 -9
View File
@@ -24,8 +24,8 @@ accounts {
jetstream: enabled
users = [
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>"] }
subscribe: { allow: ["$JS.API.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.refused"] }
subscribe: { allow: ["$JS.API.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built"] }
allow_responses: { max: 1, ttl: "1m" }
} }
{ user: "enrol.one", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: {
@@ -34,21 +34,21 @@ accounts {
} }
{ user: "node.one", password: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn", permissions: {
publish: { allow: ["$JS.ACK.NODES.one.>", "$JS.API.CONSUMER.INFO.NODES.one", "mesh.control.one.>"] }
subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.node.one.declare"] }
subscribe: { allow: ["_DELIVER.one", "_INBOX.node.one.>", "mesh.node.one.declare"] }
} }
{ user: "one.telegram", password: "$2a$11$tttttttttttttttttttttt", permissions: {
publish: { allow: ["$JS.ACK.EVENTS.one_telegram.>", "$JS.ACK.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker.>", "$JS.API.CONSUMER.INFO.EVENTS.one_telegram", "$JS.API.CONSUMER.INFO.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_telegram", "$JS.API.CONSUMER.MSG.NEXT.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.one.telegram", "mesh.seat.telegram-sender.event.delivered", "mesh.seat.telegram-sender.event.failed"] }
subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.telegram", "$SRV.INFO.telegram.>", "$SRV.PING", "$SRV.PING.telegram", "$SRV.PING.telegram.>", "$SRV.STATS", "$SRV.STATS.telegram", "$SRV.STATS.telegram.>", "_INBOX.one.telegram.>", "mesh.assignment.one.telegram", "mesh.mod.telegram.tool.>", "mesh.seat.telegram-sender.accept.send"] }
publish: { allow: ["$JS.ACK.EVENTS.one_telegram.>", "$JS.ACK.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker.>", "$JS.API.CONSUMER.INFO.EVENTS.one_telegram", "$JS.API.CONSUMER.INFO.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_telegram", "mesh.seat.telegram-sender.event.delivered", "mesh.seat.telegram-sender.event.failed"] }
subscribe: { allow: ["_DELIVER.SEAT_TELEGRAM_SENDER_worker", "_INBOX.one.telegram.>", "mesh.mod.telegram.tool.>", "mesh.seat.telegram-sender.accept.send"] }
allow_responses: { max: 1, ttl: "1m" }
} }
{ user: "two.audit", password: "$2a$11$aaaaaaaaaaaaaaaaaaaaaa", permissions: {
publish: { allow: ["$JS.ACK.EVENTS.two_audit.>", "$JS.API.CONSUMER.INFO.EVENTS.two_audit", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_audit", "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.two.audit"] }
subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.audit", "$SRV.INFO.audit.>", "$SRV.PING", "$SRV.PING.audit", "$SRV.PING.audit.>", "$SRV.STATS", "$SRV.STATS.audit", "$SRV.STATS.audit.>", "_INBOX.two.audit.>", "mesh.assignment.two.audit", "mesh.mod.audit.tool.>", "mesh.mod.shop.event.order.placed"] }
publish: { allow: ["$JS.ACK.EVENTS.two_audit.>", "$JS.API.CONSUMER.INFO.EVENTS.two_audit", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_audit"] }
subscribe: { allow: ["_INBOX.two.audit.>", "mesh.mod.audit.tool.>", "mesh.mod.shop.event.order.placed"] }
allow_responses: { max: 1, ttl: "1m" }
} }
{ user: "two.shop", password: "$2a$11$ssssssssssssssssssssss", permissions: {
publish: { allow: ["$JS.ACK.EVENTS.two_shop.>", "$JS.API.CONSUMER.INFO.EVENTS.two_shop", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_shop", "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.two.shop", "mesh.mod.shop.event.order.placed", "mesh.seat.telegram-sender.accept.send"] }
subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.shop", "$SRV.INFO.shop.>", "$SRV.PING", "$SRV.PING.shop", "$SRV.PING.shop.>", "$SRV.STATS", "$SRV.STATS.shop", "$SRV.STATS.shop.>", "_INBOX.two.shop.>", "mesh.assignment.two.shop", "mesh.mod.shop.tool.>"] }
publish: { allow: ["$JS.ACK.EVENTS.two_shop.>", "$JS.API.CONSUMER.INFO.EVENTS.two_shop", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_shop", "mesh.mod.shop.event.order.placed", "mesh.seat.telegram-sender.accept.send"] }
subscribe: { allow: ["_INBOX.two.shop.>", "mesh.mod.shop.tool.>"] }
allow_responses: { max: 1, ttl: "1m" }
} }
]
+1 -26
View File
@@ -31,8 +31,6 @@ type Declared struct {
Uses []Seat
// Watches are the seats whose events it consumes.
Watches []Seat
// Invokes are the tools it calls, `<module>.<tool>` or `*` (novox/hq ADR 0152).
Invokes []string
}
// Records is what composing a user list needs to know about the mesh, and nothing more.
@@ -47,9 +45,6 @@ type Records struct {
Enrolling []string
// People is each person's name against the tools they may invoke, `*` for an administrator.
People map[string][]string
// Interchangeable is each module whose definition says its instances are the same anywhere
// (ADR 0160), which decides whether the module's plain subject is issued to every instance.
Interchangeable map[string]bool
}
// Users is every user the composed file should contain, in the order it will be written.
@@ -62,31 +57,11 @@ func Users(r Records) ([]Principal, error) {
for _, node := range sortedCopy(r.Nodes) {
out = append(out, Principal{Kind: KindNode, Node: node})
// **Where the runtime is assigned, the machine gets one runtime principal in place of the
// runtime module's own** (novox/hq ADR 0175, to-be 38). It carries every module on the
// node: its serving grants are the union of theirs. Every other module keeps its own
// principal — a module still serving tools from its own container holds its own
// credential until it moves, and the two serve side by side in the meantime.
runtimeHere := false
for _, d := range r.Assigned[node] {
if d.Module == RuntimeModule {
runtimeHere = true
}
}
for _, d := range r.Assigned[node] {
if runtimeHere && d.Module == RuntimeModule {
continue
}
out = append(out, Principal{
Kind: KindModule, Node: node, Module: d.Module,
Emits: d.Emits, Consumes: d.Consumes, Serves: d.Serves,
Holds: d.Holds, Uses: d.Uses, Watches: d.Watches, Invokes: d.Invokes,
})
}
if runtimeHere {
out = append(out, Principal{
Kind: KindNodeTools, Node: node, Module: RuntimeModule,
Carries: append([]Declared(nil), r.Assigned[node]...),
Holds: d.Holds, Uses: d.Uses, Watches: d.Watches,
})
}
}
-51
View File
@@ -245,54 +245,3 @@ func TestAUserListIsComposedBeforeAnythingMovesOntoTheBus(t *testing.T) {
t.Errorf("the composed list does not contain the machine running the bus")
}
}
// Where the runtime module is assigned, the machine gets one runtime principal in place of the
// runtime module's own (novox/hq ADR 0175, to-be 38). Every other module keeps its own: a module
// still serving tools from its own container holds its own credential until it moves.
func TestTheRuntimeModuleBecomesTheMachinesRuntimePrincipal(t *testing.T) {
r := someRecords()
r.Assigned["one"] = append(r.Assigned["one"], Declared{Module: RuntimeModule})
users, err := Users(r)
if err != nil {
t.Fatal(err)
}
var runtime *Principal
for i := range users {
p := &users[i]
if p.Node == "one" && p.Module == RuntimeModule {
if p.Kind == KindModule {
t.Fatalf("%s on one was composed as an ordinary module beside the runtime", RuntimeModule)
}
runtime = p
}
}
if runtime == nil || runtime.Kind != KindNodeTools {
t.Fatalf("one runs %s and got no runtime principal: %v", RuntimeModule, namesOf(t, r))
}
if runtime.Username() != "one."+RuntimeModule {
t.Errorf("the runtime is named %q; `module issue` names it as the module it stands for", runtime.Username())
}
carried := map[string]bool{}
for _, d := range runtime.Carries {
carried[d.Module] = true
}
if !carried["telegram"] || !carried[RuntimeModule] {
t.Errorf("the runtime carries %v; it carries every module on its node", carried)
}
// And the other node, where the runtime is not assigned, is exactly as before.
for _, p := range users {
if p.Node == "two" && p.Kind == KindNodeTools {
t.Fatal("two runs no runtime and was given a runtime principal")
}
}
// A module serving its own tools beside the runtime keeps its own principal.
found := false
for _, p := range users {
if p.Kind == KindModule && p.Node == "one" && p.Module == "telegram" {
found = true
}
}
if !found {
t.Error("telegram lost its own principal when the runtime arrived on its node")
}
}
-167
View File
@@ -1,167 +0,0 @@
package broker
import (
"os"
"testing"
"time"
"github.com/nats-io/nats.go"
)
// A seat's worker that changed from push to pull delivery strands a holder built for the new shape
// (novox/hq issue 206): the server refuses a pull subscription on a push consumer, and the controller
// that would redefine it was the build that nobody could take. The controller owns the worker's
// shape, type included: on a work queue it re-makes one of the wrong type, losing nothing, and a
// pull subscription then binds and takes what was pending.
//
// docker run -d --rm --name t -p 14231:4222 nats:2.10-alpine -js
// MESH_TEST_NATS=nats://127.0.0.1:14231 go test ./internal/broker/ -run TestAWorker
func TestAWorkerOfTheWrongTypeIsRemadeOnAWorkQueueAndAPullThenBinds(t *testing.T) {
url := os.Getenv("MESH_TEST_NATS")
if url == "" {
t.Skip("MESH_TEST_NATS unset")
}
js, err := Dial(url)
if err != nil {
t.Fatal(err)
}
defer js.Close()
const stream, worker, filter = "SEAT_T_SHELF", "SEAT_T_SHELF_worker", "mesh.seat.t-shelf.accept.>"
_ = js.js.DeleteStream(stream)
if _, err := js.js.AddStream(&nats.StreamConfig{
Name: stream, Subjects: []string{filter}, Retention: nats.WorkQueuePolicy, Storage: nats.MemoryStorage,
}); err != nil {
t.Fatal(err)
}
defer func() { _ = js.js.DeleteStream(stream) }()
// The worker as the previous controller defined it: push, in a queue group.
if _, err := js.js.AddConsumer(stream, &nats.ConsumerConfig{
Durable: worker, AckPolicy: nats.AckExplicitPolicy, AckWait: 60 * time.Second, MaxDeliver: 5,
FilterSubject: filter, DeliverSubject: "_DELIVER." + worker, DeliverGroup: "holders",
}); err != nil {
t.Fatal(err)
}
for _, body := range []string{"one", "two", "three"} {
if _, err := js.js.Publish("mesh.seat.t-shelf.accept.build", []byte(body)); err != nil {
t.Fatal(err)
}
}
// The old holder took and acknowledged the first ask, then went away.
old, err := js.js.QueueSubscribeSync(filter, "holders", nats.Bind(stream, worker))
if err != nil {
t.Fatal(err)
}
m, err := old.NextMsg(twoSeconds)
if err != nil {
t.Fatal(err)
}
if string(m.Data) != "one" {
t.Fatalf("the first ask is %q", m.Data)
}
if err := m.AckSync(); err != nil {
t.Fatal(err)
}
if err := old.Unsubscribe(); err != nil {
t.Fatal(err)
}
// The new controller asserts the worker as the mesh derives it now: pull.
if err := js.EnsureConsumer(Consumer{
Name: worker, Stream: stream, Filters: []string{filter}, AckWaitSeconds: 60, MaxDeliver: 5,
Why: "the test's worker",
}); err != nil {
t.Fatal(err)
}
have, err := js.js.ConsumerInfo(stream, worker)
if err != nil {
t.Fatal(err)
}
if have.Config.DeliverSubject != "" || have.Config.DeliverGroup != "" {
t.Fatalf("the worker is still push: %+v", have.Config)
}
// A holder built for the new shape binds, and takes exactly what the old one left.
sub, err := js.js.PullSubscribe(filter, worker, nats.Bind(stream, worker), nats.ManualAck())
if err != nil {
t.Fatalf("a pull subscription does not bind the re-made worker: %v", err)
}
got, err := sub.Fetch(3, nats.MaxWait(twoSeconds))
if err != nil && len(got) == 0 {
t.Fatalf("nothing pending was delivered: %v", err)
}
var bodies []string
for _, g := range got {
bodies = append(bodies, string(g.Data))
_ = g.Ack()
}
if len(bodies) != 2 || bodies[0] != "two" || bodies[1] != "three" {
t.Fatalf("the pending asks after the acknowledged one, in order: %v", bodies)
}
// Asserted again, the pull worker is the no-op a restart depends on.
if err := js.EnsureConsumer(Consumer{
Name: worker, Stream: stream, Filters: []string{filter}, AckWaitSeconds: 60, MaxDeliver: 5,
}); err != nil {
t.Fatal(err)
}
}
// On a stream that keeps its history, a worker of the wrong type is re-made to deliver from now on:
// re-making it from the start would replay what it acknowledged (novox/hq issue 156), and leaving it
// for a hand re-made it exactly that way on 2026-10-03 (issue 207).
func TestAWorkerOfTheWrongTypeOnAHistoryStreamIsRemadeFromNowOn(t *testing.T) {
url := os.Getenv("MESH_TEST_NATS")
if url == "" {
t.Skip("MESH_TEST_NATS unset")
}
js, err := Dial(url)
if err != nil {
t.Fatal(err)
}
defer js.Close()
const stream, worker, filter = "EVENTS_T", "EVENTS_T_reader", "mesh.t.event.>"
_ = js.js.DeleteStream(stream)
if _, err := js.js.AddStream(&nats.StreamConfig{Name: stream, Subjects: []string{filter}, Storage: nats.MemoryStorage}); err != nil {
t.Fatal(err)
}
defer func() { _ = js.js.DeleteStream(stream) }()
if _, err := js.js.AddConsumer(stream, &nats.ConsumerConfig{
Durable: worker, AckPolicy: nats.AckExplicitPolicy, AckWait: 60 * time.Second,
FilterSubject: filter, DeliverSubject: "_DELIVER." + worker,
}); err != nil {
t.Fatal(err)
}
// History the old consumer would have acknowledged long ago, and must not come back.
for i := 0; i < 3; i++ {
if _, err := js.js.Publish("mesh.t.event.old", []byte("old")); err != nil {
t.Fatal(err)
}
}
if err := js.EnsureConsumer(Consumer{Name: worker, Stream: stream, Filters: []string{filter}, AckWaitSeconds: 60}); err != nil {
t.Fatal(err)
}
have, err := js.js.ConsumerInfo(stream, worker)
if err != nil {
t.Fatal(err)
}
if have.Config.DeliverSubject != "" {
t.Fatal("a history stream's consumer of the wrong type was left as it was")
}
if have.Config.DeliverPolicy != nats.DeliverNewPolicy || have.NumPending != 0 {
t.Fatalf("re-made consumer delivers %v with %d pending; it must deliver from now on with nothing of the past", have.Config.DeliverPolicy, have.NumPending)
}
// And what arrives from now on is delivered.
if _, err := js.js.Publish("mesh.t.event.new", []byte("new")); err != nil {
t.Fatal(err)
}
sub, err := js.js.PullSubscribe(filter, worker, nats.Bind(stream, worker))
if err != nil {
t.Fatal(err)
}
got, err := sub.Fetch(1, nats.MaxWait(3*time.Second))
if err != nil || len(got) != 1 || string(got[0].Data) != "new" {
t.Fatalf("the re-made consumer delivered %v, %v; want the one new message", got, err)
}
}
-44
View File
@@ -1,44 +0,0 @@
package builder
import (
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// The name a machine runs a binary by is not always the name of the package that built it. The host's
// command is cmd/mesh-host and every machine runs it as nox-mesh-host — the path it is installed at,
// the name in its unit, and the name its launcher looks for inside a delivered version.
//
// A bundle carrying the package's name was delivered to a machine correctly, reported "created … 1
// file(s)", and was invisible to the launcher (novox/hq 04-ISSUES/142). Found by reading the delivered
// directory rather than by trusting the line that said it worked.
func TestACompiledArtifactNamesTheBinaryAMachineWillRun(t *testing.T) {
got := binaryName(catalogue.Artifact{
Name: "host-arch", From: "cmd/mesh-host", Binary: "nox-mesh-host",
})
if got != "nox-mesh-host" {
t.Fatalf("the binary is named %q, and the launcher looks for nox-mesh-host", got)
}
}
func TestSayingNothingKeepsWhatTheCompilerWouldHaveChosen(t *testing.T) {
// go build names its output after the package, so an artifact that says nothing gets the same
// thing it got before this existed.
if got := binaryName(catalogue.Artifact{Name: "host-arch", From: "cmd/mesh-host"}); got != "mesh-host" {
t.Fatalf("an artifact naming no binary produced %q", got)
}
if got := binaryName(catalogue.Artifact{Name: "host-arch", From: "./cmd/agent/"}); got != "agent" {
t.Fatalf("a from with slashes produced %q", got)
}
}
func TestABundleBuiltFromTheModuleRootFallsBackToItsArtifactName(t *testing.T) {
// A single-command repository names no package, and `go build -o <dir>` would then write a file
// named after the module directory — which is not something the manifest states. The artifact's
// own name is what the manifest does state.
if got := binaryName(catalogue.Artifact{Name: "tool"}); got != "tool" {
t.Fatalf("a bundle built from the root produced %q", got)
}
}
+18 -192
View File
@@ -90,13 +90,7 @@ type GitCredential struct {
// records — reachable, unreferenced, and indistinguishable from something in use.
func Build(ctx context.Context, run Runner, publish Publisher,
repository, path, ref, workspace string, held map[string]string, npmrc Npmrc,
forge GitCredential, log Log, seats ...map[string]string) (Result, error) {
// The clone base of each seat a context may name (novox/hq ADR 0155); variadic so the callers
// that hand none — tests of everything but contexts — read as they did.
var seatBases map[string]string
if len(seats) > 0 {
seatBases = seats[0]
}
forge GitCredential, log Log) (Result, error) {
say := logging(log)
say("clone", "%s%s at %s", repository, describePath(path), refOrHead(ref))
@@ -215,7 +209,7 @@ func Build(ctx context.Context, run Runner, publish Publisher,
sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name })
for _, a := range artifacts {
say("artifact", "%s (%s%s) — starting", a.Name, a.Kind, langSuffix(a))
made, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, npmrcPath, seatBases, say)
made, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, npmrcPath, say)
if err != nil {
say("artifact", "%s FAILED: %v", a.Name, err)
return Result{}, err
@@ -252,18 +246,14 @@ func logging(log Log) func(step, format string, args ...any) {
// module's own repository — a fresh tree, the same way the module's own is, keyed by artifact
// name so two artifacts of one module naming different contexts do not collide.
func contextFrom(ctx context.Context, run Runner, workspace, artifact, credentials string,
from catalogue.ArtifactContext, seats map[string]string, say func(step, format string, args ...any)) (string, error) {
url, err := contextURL(from, seats)
if err != nil {
return "", err
}
say("context", "cloning %s at %s for %s", url, refOrHead(from.Ref), artifact)
from catalogue.ArtifactContext, say func(step, format string, args ...any)) (string, error) {
say("context", "cloning %s at %s for %s", from.Repository, refOrHead(from.Ref), artifact)
dir := filepath.Join(workspace, "context-"+artifact)
if err := os.RemoveAll(dir); err != nil {
return "", err
}
if _, err := run(ctx, workspace, "git", cloneWith(credentials, "clone", "--quiet", url, dir)...); err != nil {
return "", fmt.Errorf("cannot clone %s: %w", url, err)
if _, err := run(ctx, workspace, "git", cloneWith(credentials, "clone", "--quiet", from.Repository, dir)...); err != nil {
return "", fmt.Errorf("cannot clone %s: %w", from.Repository, err)
}
if from.Ref != "" {
if _, err := run(ctx, dir, "git", "checkout", "--quiet", from.Ref); err != nil {
@@ -274,23 +264,6 @@ func contextFrom(ctx context.Context, run Runner, workspace, artifact, credentia
return dir, nil
}
// contextURL is what a context is cloned from: its URL, or — for a context on a seat — the seat's
// clone base the mesh sent with the request joined to the repository's path (novox/hq ADR 0155).
// Refused, never guessed, when the mesh sent no base for that seat: a builder that guessed a forge
// would be the literal this removes, one layer down.
func contextURL(from catalogue.ArtifactContext, seats map[string]string) (string, error) {
if from.Seat == "" {
return from.Repository, nil
}
base, told := seats[from.Seat]
if !told || base == "" {
return "", fmt.Errorf("the context is %s on the %s seat, and this build was told no clone "+
"base for that seat — nothing holds it in this mesh, or the control plane predates the word",
from.Repository, from.Seat)
}
return strings.TrimRight(base, "/") + "/" + strings.TrimSuffix(strings.Trim(from.Repository, "/"), ".git") + ".git", nil
}
// cloneWith is a git invocation that may offer a stored credential.
//
// The first `-c credential.helper=` clears every helper the environment might carry, so exactly
@@ -443,8 +416,7 @@ func wantsPackages(manifest catalogue.Manifest, within string) bool {
func one(ctx context.Context, run Runner, publish Publisher,
module, tree, workspace, commit, credentials string, a catalogue.Artifact, args []string,
held map[string]string, npmrc string, seats map[string]string,
say func(step, format string, args ...any)) (catalogue.Built, error) {
held map[string]string, npmrc string, say func(step, format string, args ...any)) (catalogue.Built, error) {
switch a.Kind {
case catalogue.ArtifactUpstream:
@@ -521,7 +493,7 @@ func one(ctx context.Context, run Runner, publish Publisher,
recipePath := a.From
buildDir := tree
if a.Context != nil {
cloned, err := contextFrom(ctx, run, workspace, a.Name, credentials, *a.Context, seats, say)
cloned, err := contextFrom(ctx, run, workspace, a.Name, credentials, *a.Context, say)
if err != nil {
return catalogue.Built{}, fmt.Errorf("%s: %s's context: %w", module, a.Name, err)
}
@@ -587,12 +559,6 @@ func one(ctx context.Context, run Runner, publish Publisher,
if err != nil {
return catalogue.Built{}, fmt.Errorf("%s: compiling %s failed: %w", module, a.Name, err)
}
// **Every entrypoint the runtime may serve is executable** (novox/hq ADR 0193). The runtime
// knows no language; for one that runs through an interpreter the build writes the launcher.
launchers, err := writeLaunchers(compiled, chain, a)
if err != nil {
return catalogue.Built{}, fmt.Errorf("%s: writing %s's launchers failed: %w", module, a.Name, err)
}
say("bundle", "compiled, packing")
body, err := pack(compiled)
if err != nil {
@@ -604,7 +570,7 @@ func one(ctx context.Context, run Runner, publish Publisher,
if err != nil {
return catalogue.Built{}, err
}
return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: where, Digest: digest, Launchers: launchers}, nil
return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: where, Digest: digest}, nil
case catalogue.ArtifactPackage:
// Built and published on a public base, to the mesh's package registry, by version
@@ -725,21 +691,6 @@ func short(commit string) string {
return commit
}
// Said is where the lines Command speaks go, beside the build's own Log: what runs, how long it
// took, and that it failed. Nil prints them to stderr, as a build machine with nobody listening
// should. The machine sets it per build so every line reaches the bus too (novox/hq ADR 0157) —
// the step is "run", and the message is the line as it has always been printed.
var Said Log
func tell(step, format string, args ...any) {
message := fmt.Sprintf(format, args...)
if Said == nil {
fmt.Fprintf(os.Stderr, " %s\n", message)
return
}
Said(step, message)
}
// Command is a Runner that actually runs things.
func Command(ctx context.Context, dir, name string, args ...string) (string, error) {
// **Every command is echoed before it runs**, with where. On a build that hangs, the last line
@@ -747,23 +698,16 @@ func Command(ctx context.Context, dir, name string, args ...string) (string, err
// nothing" and "git clone is waiting on a network that will not answer". Silent on success is
// what made an empty workspace unreadable.
started := timeNow()
tell("run", "$ (%s) %s %s", short(filepath.Base(dir)), name, strings.Join(args, " "))
fmt.Fprintf(os.Stderr, " $ (%s) %s %s\n", short(filepath.Base(dir)), name, strings.Join(args, " "))
cmd := exec.CommandContext(ctx, name, args...)
cmd.Dir = dir
out, err := cmd.CombinedOutput()
if err != nil {
tell("run", "! %s %s failed after %s", name, args[0], since(started))
// The command's own output is part of what a reader needs — the compiler's error, the
// clone's refusal — and a line per output line keeps it readable on the bus.
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
if line != "" {
tell("output", "%s", line)
}
}
fmt.Fprintf(os.Stderr, " ! %s %s failed after %s\n", name, args[0], since(started))
return string(out), fmt.Errorf("%s %s: %w\n%s",
name, strings.Join(args, " "), err, strings.TrimSpace(string(out)))
}
tell("run", "✓ %s %s (%s)", name, firstArg(args), since(started))
fmt.Fprintf(os.Stderr, " ✓ %s %s (%s)\n", name, firstArg(args), since(started))
return string(out), nil
}
@@ -917,15 +861,6 @@ func compile(ctx context.Context, run Runner, tree string, chain Toolchain,
// each other and then be packed together, so each bundle compiles and packs alone.
out := Out(a.Name)
// **The output directory exists before the compiler is told about it.** `tsc --outDir` makes
// one; `go build -o` writes a file into a directory and does not create it, failing with a
// message about a path rather than about a build. Made here for every toolchain, because which
// compilers happen to be forgiving is not a thing a reader should have to know
// (novox/hq 04-ISSUES/142).
if err := os.MkdirAll(filepath.Join(tree, out), 0o755); err != nil {
return "", fmt.Errorf("making the output directory for %s: %w", a.Name, err)
}
invocation := []string{
"run", "--rm",
"--volume", tree + ":" + within,
@@ -933,67 +868,17 @@ func compile(ctx context.Context, run Runner, tree string, chain Toolchain,
base,
}
invocation = append(invocation, chain.Compile...)
// **One `-ldflags`, composed here.** A repeated flag is not a merged one: the Go command takes
// the last and drops the first, so passing the toolchain's flags and then the system stamp as a
// second `-ldflags` produced a binary that knew its system and had lost `-s -w` — half again the
// size, with its debug info (novox/hq 04-ISSUES/161).
//
// What it was built for is the one thing taken from the artifact, and ADR 0142 says why: the
// target is a property of the artifact rather than of the recipe. A host with no system refuses
// every declaration before it applies anything.
linker := append([]string(nil), chain.LinkerFlags...)
if chain.SystemStamp != "" && strings.TrimSpace(a.System) != "" {
linker = append(linker, "-X", chain.SystemStamp+"="+strings.TrimSpace(a.System))
}
if len(linker) > 0 {
invocation = append(invocation, "-ldflags", strings.Join(linker, " "))
}
if chain.OutputFlag != "" {
// A compiler pointed at a package is told the file to write, not the directory: the name a
// machine runs it by is not always the name of the package that built it. The host's command
// is `cmd/mesh-host` and every machine runs it as `nox-mesh-host` — so a bundle carrying the
// package's name lands correctly, reports success, and is invisible to whatever looks for it
// (novox/hq 04-ISSUES/142).
target := out
if chain.Unit == UnitPackage {
target = filepath.Join(out, binaryName(a))
}
invocation = append(invocation, chain.OutputFlag, target)
invocation = append(invocation, chain.OutputFlag, out)
}
// What to compile. Named by the module rather than discovered, so adding a file does not
// silently change what a build produces.
switch {
case chain.Unit == UnitPackage:
// One directory, compiled whole: the thing the artifact is built `from`. Relative, because
// the compiler runs with the module's own root as its working directory and a package path
// that looked absolute would name one inside the toolchain image.
invocation = append(invocation, "./"+strings.Trim(a.From, "./"))
case len(a.Entrypoints) > 0:
invocation = append(invocation, sourcesFor(a.Entrypoints, out, chain.SourceExt)...)
if len(a.Entrypoints) > 0 {
invocation = append(invocation, sourcesFor(a.Entrypoints, out)...)
}
if _, err := run(ctx, tree, "docker", invocation...); err != nil {
return "", err
}
if chain.Dependencies != "" {
// **What the bundle runs with, from the image it was compiled in** (Toolchain.Dependencies).
// A second run in the same image rather than a shell wrapped around the compiler: the
// compile line stays a plain command a reader can run by hand, and the copy is one more
// plain command beside it. Refused by name when the image carries no such directory — an
// older toolchain image — because a bundle packed without its dependencies starts nowhere
// and says so three layers away from here.
copying := []string{
"run", "--rm",
"--volume", tree + ":" + within,
"--workdir", within,
base,
"sh", "-c",
`test -d "$1" || { echo "the toolchain image carries no $1: it predates the mesh shipping a bundle's dependencies, rebuild $2 first" >&2; exit 1; }; cp -a "$1/." "$3/"`,
"dependencies", chain.Dependencies, chain.Base, out,
}
if _, err := run(ctx, tree, "docker", copying...); err != nil {
return "", fmt.Errorf("copying the %s dependencies a bundle runs with: %w", chain.Language, err)
}
}
return filepath.Join(tree, out), nil
}
@@ -1002,16 +887,14 @@ func compile(ctx context.Context, run Runner, tree string, chain Toolchain,
// A module names what a tool host should LOAD — compiled paths under the bundle's root — because
// that is the thing anything else needs to know. What to compile is the same list with the
// language's own extension, which is the toolchain's business rather than the module's.
func sourcesFor(entrypoints []string, out, ext string) []string {
func sourcesFor(entrypoints []string, out string) []string {
sources := make([]string, 0, len(entrypoints))
for _, e := range entrypoints {
// An entrypoint is named as it will be FOUND — a path inside the unpacked bundle — so the
// source is the same path with the output directory taken off the front and the language's
// own extension on the end. **The extension is the toolchain's**, where it used to be the
// literal `.ts`: one language's file extension written into the code that serves every
// language is a wall the next one hits (novox/hq 04-ISSUES/142).
// own extension on the end.
at := strings.TrimPrefix(strings.TrimPrefix(e, out), "/")
sources = append(sources, strings.TrimSuffix(at, filepath.Ext(at))+ext)
sources = append(sources, strings.TrimSuffix(at, filepath.Ext(at))+".ts")
}
return sources
}
@@ -1167,60 +1050,3 @@ func readBy(manifest catalogue.Manifest) []catalogue.ArtifactContext {
})
return out
}
// binaryName is what a compiled bundle's executable is called: what the artifact says, or the name of
// the package it is built from, which is what a compiler would have chosen anyway.
func binaryName(a catalogue.Artifact) string {
if name := catalogue.BinaryOf(a); name != "" {
return name
}
if name := strings.TrimSpace(a.Binary); name != "" {
return name
}
if from := strings.Trim(a.From, "./"); from != "" {
return filepath.Base(from)
}
return a.Name
}
// launcherSuffix is what a TypeScript entrypoint's launcher is called beside it: index.js is
// started as index.serve.mjs (novox/hq ADR 0193). An ES module by its own extension, whatever the
// bundle's package.json says.
const launcherSuffix = ".serve.mjs"
// writeLaunchers writes, beside every entrypoint of a TypeScript bundle, an executable that
// imports the entrypoint and serves what it registered over MCP on stdio — through the bundle's
// own copy of the SDK, so registering and serving meet in one registry (novox/hq ADR 0193). Its
// answer is each entrypoint's launcher, by entrypoint, relative to the bundle's root; nothing for
// a language whose build is already executable.
func writeLaunchers(root string, chain Toolchain, a catalogue.Artifact) (map[string]string, error) {
if chain.Language != "typescript" {
return nil, nil
}
out := map[string]string{}
for _, entry := range a.Entrypoints {
if !strings.HasSuffix(entry, ".js") {
continue
}
launcher := strings.TrimSuffix(entry, ".js") + launcherSuffix
body := "#!/usr/bin/env node\n" +
"// Written by the mesh's builder (novox/hq ADR 0193): serve what " + entry + " registers,\n" +
"// over MCP on stdio, as the module the node's runtime names in MESH_SERVED_MODULE.\n" +
"import { serveRegisteredOverStdio } from \"@novox/mesh-sdk/stdio\";\n" +
"await import(\"./" + filepath.Base(entry) + "\");\n" +
"await serveRegisteredOverStdio();\n"
path := filepath.Join(root, filepath.FromSlash(launcher))
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return nil, err
}
if err := os.WriteFile(path, []byte(body), 0o755); err != nil {
return nil, err
}
// WriteFile honours the umask; the mode a machine unpacks is the one packed, so it is set.
if err := os.Chmod(path, 0o755); err != nil {
return nil, err
}
out[entry] = launcher
}
return out, nil
}
+1 -40
View File
@@ -82,41 +82,6 @@ func TestABundleIsCompiledAndPackedWithNoDockerfile(t *testing.T) {
if !strings.HasPrefix(digest, "sha256:") {
t.Fatalf("the bundle was not pinned: %v", got.Manifest.Resources[0])
}
// **And what it runs with, from the image it was compiled in** (novox/hq to-be 38 WP3). A
// second run in the same toolchain image copies the toolchain's runtime directory — the
// `"type": "module"` package.json and the pruned node_modules — into the output's root, and
// refuses by name when the image carries none rather than packing a bundle that starts nowhere.
var copied string
for _, line := range r.ran {
if strings.HasPrefix(line, "docker run") && strings.Contains(line, "/app/runtime") {
copied = line
}
}
if copied == "" {
t.Fatalf("the bundle's dependencies were not copied in after the compile:\n%s", strings.Join(r.ran, "\n"))
}
if !strings.Contains(copied, "mesh-tools/build@sha256:") || !strings.Contains(copied, "predates") ||
!strings.Contains(copied, Out("code")) {
t.Fatalf("the copy does not run in the same toolchain, refuse an older image by name, or land in the artifact's output: %s", copied)
}
if strings.Index(strings.Join(r.ran, "\n"), "--outDir") > strings.Index(strings.Join(r.ran, "\n"), "/app/runtime") {
t.Fatal("the dependencies were copied before the compile wrote its output")
}
}
// A language whose bundle carries its own dependencies copies nothing in: a Go binary is static.
func TestOnlyALanguageWithARuntimeDirectoryCopiesDependenciesIn(t *testing.T) {
ts, _ := ToolchainFor("typescript")
if ts.Dependencies != "/app/runtime" {
t.Fatalf("typescript bundles run with %q", ts.Dependencies)
}
for _, language := range []string{"go", "python"} {
chain, _ := ToolchainFor(language)
if chain.Dependencies != "" {
t.Fatalf("%s copies %q into every bundle, and its bundles carry their own", language, chain.Dependencies)
}
}
}
// **Refused before anything is built, naming what to build first.** A base the mesh has not built
@@ -180,13 +145,9 @@ func TestTwoBundlesInOneModuleArePackedSeparately(t *testing.T) {
t.Fatalf("a module with two bundles did not build: %v", err)
}
// Compiled into two different places. Only the compile lines: the copy of each bundle's
// dependencies names the same directory again, deliberately.
// Compiled into two different places.
var outputs []string
for _, line := range r.ran {
if !strings.Contains(line, "--outDir") {
continue
}
for _, part := range strings.Fields(line) {
if strings.HasPrefix(part, ".mesh-build/") {
outputs = append(outputs, part)
-26
View File
@@ -1,26 +0,0 @@
package builder
import (
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// A context on a seat is cloned from the base the mesh sent, joined to the repository's path; a
// context by URL is itself; a seat the mesh sent no base for is refused by name (novox/hq ADR 0155).
func TestAContextOnASeatIsClonedFromTheBaseTheMeshSent(t *testing.T) {
seats := map[string]string{"git": "http://forge.example.tld:3000"}
got, err := contextURL(catalogue.ArtifactContext{Seat: "git", Repository: "org/controller"}, seats)
if err != nil || got != "http://forge.example.tld:3000/org/controller.git" {
t.Fatalf("got %q, %v", got, err)
}
got, err = contextURL(catalogue.ArtifactContext{Repository: "https://elsewhere.example/x.git"}, seats)
if err != nil || got != "https://elsewhere.example/x.git" {
t.Fatalf("a URL context was changed: %q, %v", got, err)
}
_, err = contextURL(catalogue.ArtifactContext{Seat: "git", Repository: "org/controller"}, nil)
if err == nil || !strings.Contains(err.Error(), "git seat") {
t.Fatalf("a seat with no base was not refused by name: %v", err)
}
}
-71
View File
@@ -1,71 +0,0 @@
package builder
import (
"strings"
"testing"
)
// Nothing could compile the mesh's own components, which is why nothing delivers the host
// (novox/hq 04-ISSUES/142, and ADR 0141's own insight naming it). The toolchain list was a closed
// set of typescript and python, and two things in the path beyond it assumed TypeScript.
func TestTheMeshCanCompileGo(t *testing.T) {
chain, err := ToolchainFor("go")
if err != nil {
t.Fatal(err)
}
// Named, not pinned: the mesh answers with the copy it holds, so moving compiler is a build
// rather than an edit to this source (ADR 0044, 0142).
if chain.Base != "mesh-tools-go" || chain.Artifact != "build" {
t.Fatalf("the go toolchain is based on %s/%s", chain.Base, chain.Artifact)
}
joined := strings.Join(chain.Compile, " ")
// Static, because what a machine holds is a file and not a container: a binary needing a libc
// it did not bring is a delivery that works until a machine differs.
if !strings.Contains(joined, "CGO_ENABLED=0") {
t.Fatalf("the go toolchain does not build statically: %q", joined)
}
// Reproducible: a version comes from where a component sits, not from its linker (ADR 0142),
// so two builds of one commit should produce the same bytes.
if !strings.Contains(joined, "-trimpath") {
t.Fatalf("the go toolchain leaves build paths in the binary: %q", joined)
}
if chain.Unit != UnitPackage {
t.Fatalf("the go toolchain compiles %q, wanted a package", chain.Unit)
}
}
func TestEveryToolchainSaysWhatItIsPointedAt(t *testing.T) {
// The field exists because the compile path used to assume one language. A toolchain that says
// nothing would fall through to the entrypoint branch and compile a file list, which for a
// compiled language builds a program out of exactly those files and ignores the rest of the
// package — a missing symbol rather than a legible refusal.
for _, chain := range toolchains {
switch chain.Unit {
case UnitPackage:
case UnitSources:
if chain.SourceExt == "" {
t.Fatalf("%s compiles a file list and names no source extension", chain.Language)
}
if !strings.HasPrefix(chain.SourceExt, ".") {
t.Fatalf("%s's source extension %q is not an extension", chain.Language, chain.SourceExt)
}
default:
t.Fatalf("%s says it is pointed at %q, which is neither sources nor a package",
chain.Language, chain.Unit)
}
}
}
func TestAnEntrypointBecomesASourceInItsOwnLanguage(t *testing.T) {
// It used to become a `.ts` whatever the language was.
out := Out("build")
got := sourcesFor([]string{out + "/tools/index.js"}, out, ".ts")
if len(got) != 1 || got[0] != "tools/index.ts" {
t.Fatalf("a typescript entrypoint became %v", got)
}
got = sourcesFor([]string{out + "/tools/index.js"}, out, ".py")
if len(got) != 1 || got[0] != "tools/index.py" {
t.Fatalf("a python entrypoint became %v", got)
}
}
-49
View File
@@ -1,49 +0,0 @@
package builder
import (
"os"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// novox/hq ADR 0193: every entrypoint the runtime may serve is executable, and the runtime knows no
// language — so a TypeScript bundle carries a launcher beside each entrypoint.
func TestATypeScriptBundleCarriesAnExecutableLauncherBesideEachEntrypoint(t *testing.T) {
root := t.TempDir()
chain, err := ToolchainFor("typescript")
if err != nil {
t.Fatal(err)
}
got, err := writeLaunchers(root, chain, catalogue.Artifact{Name: "tools", Kind: catalogue.ArtifactBundle,
Language: "typescript", Entrypoints: []string{"tools/index.js", "index.js"}})
if err != nil {
t.Fatal(err)
}
if got["tools/index.js"] != "tools/index.serve.mjs" || got["index.js"] != "index.serve.mjs" {
t.Fatalf("launchers: %v", got)
}
path := filepath.Join(root, "tools", "index.serve.mjs")
info, err := os.Stat(path)
if err != nil {
t.Fatal(err)
}
if info.Mode().Perm() != 0o755 {
t.Errorf("the launcher is %v, not executable 0755", info.Mode().Perm())
}
body, _ := os.ReadFile(path)
for _, want := range []string{"#!/usr/bin/env node\n", `from "@novox/mesh-sdk/stdio"`, `await import("./index.js")`, "serveRegisteredOverStdio()"} {
if !strings.Contains(string(body), want) {
t.Errorf("the launcher lacks %q:\n%s", want, body)
}
}
// A compiled language's build is executable already: no launcher.
goChain, _ := ToolchainFor("go")
none, err := writeLaunchers(t.TempDir(), goChain, catalogue.Artifact{Name: "tools", Kind: catalogue.ArtifactBundle, Language: "go"})
if err != nil || len(none) != 0 {
t.Errorf("a Go bundle was given launchers: %v %v", none, err)
}
}
-76
View File
@@ -1,76 +0,0 @@
package builder
import (
"strings"
"testing"
)
// A host built without knowing its system refuses every declaration before applying anything —
// safely, totally, and with nothing reporting it. The mesh built one, delivered it, started it, and
// it would have refused the first thing it was asked to do (novox/hq 04-ISSUES/161).
func TestTheGoToolchainStampsTheArtifactsSystem(t *testing.T) {
chain, err := ToolchainFor("go")
if err != nil {
t.Fatal(err)
}
if chain.SystemStamp != "main.builtFor" {
t.Fatalf("the go toolchain fills %q", chain.SystemStamp)
}
}
func TestALanguageWithNoPinnedSystemStampsNothing(t *testing.T) {
// Interpreted output is not pinned to a system, and a manifest declaring one for it is already
// refused. Nothing to fill.
for _, language := range []string{"typescript", "python"} {
chain, err := ToolchainFor(language)
if err != nil {
t.Fatal(err)
}
if chain.SystemStamp != "" {
t.Fatalf("%s fills %q, and its output is not pinned to a system",
language, chain.SystemStamp)
}
}
}
func TestTheStampIsTheOneThingTakenFromTheArtifact(t *testing.T) {
// The toolchain accepts nothing else from the module — anything it could override it would be
// writing a Dockerfile to override. The system is the stated exception, because a compiled
// binary is per system and the artifact is what declares one (ADR 0142).
chain, err := ToolchainFor("go")
if err != nil {
t.Fatal(err)
}
joined := strings.Join(chain.Compile, " ")
if strings.Contains(joined, "${") || strings.Contains(joined, "%s") {
t.Fatalf("the compile line takes something from the module: %q", joined)
}
}
func TestTheLinkerIsToldOnceNotTwice(t *testing.T) {
// A repeated flag is not a merged one: the Go command takes the last -ldflags and drops the
// first. Passing the toolchain's flags and then the stamp separately produced a binary that knew
// its system and had lost -s -w — 12.2MB against 8.5MB, with its debug info (04-ISSUES/161).
chain, err := ToolchainFor("go")
if err != nil {
t.Fatal(err)
}
for _, arg := range chain.Compile {
if arg == "-ldflags" {
t.Fatal("the compile line carries -ldflags, so composing one here makes two")
}
}
if len(chain.LinkerFlags) == 0 {
t.Fatal("the go toolchain passes no linker flags, so the binary keeps its debug info")
}
var stripped bool
for _, f := range chain.LinkerFlags {
if f == "-s" {
stripped = true
}
}
if !stripped {
t.Fatalf("the go toolchain does not strip: %v", chain.LinkerFlags)
}
}
+2 -105
View File
@@ -35,67 +35,8 @@ type Toolchain struct {
Compile []string
// OutputFlag is how this compiler is told where to put its output.
OutputFlag string
// Unit is what this compiler is pointed at: UnitSources, the entrypoint files the module named,
// or UnitPackage, the one directory the artifact is built `from`.
//
// **The difference is the language and not the module.** A TypeScript bundle is a set of files
// compiled into a set of files, so what to compile is the module's entrypoints with their source
// extension. A Go bundle is a package compiled into one binary, and there is no per-file
// compilation to name — pointing `go build` at a file list builds a program out of exactly those
// files and ignores the rest of the package, which fails as a missing symbol rather than as a
// wrong instruction.
Unit string
// SourceExt is the extension an entrypoint has in the repository, for UnitSources. An entrypoint
// is named as it will be FOUND, inside the unpacked bundle, so the source is the same path with
// the output directory taken off the front and this on the end.
SourceExt string
// LinkerFlags are passed to the linker as one flag, together with the system stamp below.
//
// **Separate from Compile because a repeated flag is not a merged one.** They were in the compile
// line, and appending the stamp as a second `-ldflags` meant the Go command took the last and
// dropped the first — so the binary gained its system and lost `-s -w`, growing by half and
// carrying its debug info. The mistake was believing a comment rather than reading the file it
// produced (novox/hq 04-ISSUES/161).
LinkerFlags []string
// Dependencies is a directory inside the toolchain image whose contents a bundle in this
// language runs with, copied whole into the compiled output's root after the compile.
//
// **A bundle that compiles is not yet a bundle that runs.** The compiler resolves `import
// "nats"` from the toolchain image's own node_modules and the pack takes only what the compiler
// wrote, so what a machine unpacked could not find a single dependency — and no TypeScript bundle
// had ever run live to show it (novox/hq to-be 38 WP3). For TypeScript the directory holds a
// `package.json` saying `"type": "module"` — Node reads a bare `.js` as CommonJS otherwise, so a
// bundle with its dependencies and without that line still fails to start — and the pruned,
// production-only node_modules the runtime itself ships with: the SDK's and the runtime's
// dependencies, and nothing module-specific yet (novox/hq ADR 0188 §5: a skeleton; a module's
// own npm dependencies are a later step). Empty for a language whose bundle carries its own —
// a Go binary is static, a Python bundle is installed with its dependencies.
//
// A toolchain image without the directory fails the build by name rather than packing a bundle
// that starts nowhere: the image predates this and must be rebuilt first.
Dependencies string
// SystemStamp is the variable this language's linker fills with the artifact's declared system,
// for a language whose binaries are pinned to one at link time (novox/hq ADR 0005).
//
// **The one thing a toolchain takes from the artifact, and 0142 says why**: the target is a
// property of the artifact rather than of the recipe, because a compiled binary is per system
// and a toolchain that accepted it from the module would be accepting a build instruction. This
// is the narrow exception, named here rather than inferred.
//
// Empty for a language that compiles to nothing pinned. A host built without it refuses every
// declaration before applying anything — safely, totally, and with nothing reporting it
// (novox/hq 04-ISSUES/161).
SystemStamp string
}
// What a toolchain is pointed at.
const (
// UnitSources is a list of files, derived from the module's entrypoints.
UnitSources = "sources"
// UnitPackage is the single directory the artifact is built `from`, compiled whole.
UnitPackage = "package"
)
// Out is where one artifact's compiled output lands, inside the module's own directory.
//
// **Per artifact, never per toolchain.** A module is one piece of software and may still be
@@ -124,54 +65,12 @@ var toolchains = []Toolchain{
// symlinks to a launcher that requires its library relatively — and the base image's own
// assembly resolves them away, leaving a launcher whose relative require points nowhere.
// Every module's hand-written Dockerfile had to know this. Now none of them does.
// **Rooted at the module, so an entrypoint lands where it is named.** Without a root the
// compiler takes the common directory of the files it is given: a module compiling only
// `tools/index.ts` had its output at `index.js`, and the entrypoint it declared —
// `tools/index.js`, "named as it will be found" — named a file the bundle did not
// contain. The runtime that loads bundles by their declared entrypoints (novox/hq ADR
// 0175) is what made this visible.
Compile: []string{
"node", "/app/node_modules/typescript/bin/tsc",
"--module", "NodeNext", "--moduleResolution", "NodeNext",
"--target", "ES2022", "--rootDir", ".",
"--target", "ES2022",
},
OutputFlag: "--outDir",
Unit: UnitSources,
SourceExt: ".ts",
Dependencies: "/app/runtime",
},
{
Language: "go",
Base: "mesh-tools-go",
Artifact: "build",
// **The mesh's own components, and not modules.** The warning above this list — that every
// language is another implementation of the contracts modules share, so adding one commits
// to keeping N implementations in step — does not attach here. Go is how the host, the
// control plane and the builder are written, and none of them is a module in that sense:
// the host is what APPLIES modules. So there is no SDK obligation, and the reason this
// entry did not exist was that nothing needed to compile the mesh itself
// (novox/hq ADR 0142, and 04-ISSUES/142 where that is why nothing delivers the host).
//
// Static, because what a machine ends up holding is a file rather than a container, and a
// binary that needs a libc it did not bring is a delivery that works until a machine
// differs. Trimmed of its own paths for the same reason a version comes from where it sits
// rather than from the linker: two builds of one commit produce the same bytes.
Compile: []string{
"env", "CGO_ENABLED=0", "GOFLAGS=-trimpath",
"go", "build",
},
// Stripped of symbols and debug info: what a machine holds is a file it runs, not one it
// debugs, and the difference measured 12.2MB against 8.5MB.
LinkerFlags: []string{"-s", "-w"},
OutputFlag: "-o",
// Pointed at the package the artifact is built `from`, compiled whole. Go writes the binary
// into the output directory, named after the package — so the bundle a machine unpacks is a
// directory holding one executable, which is what the delivery mechanism expects
// (novox/hq ADR 0141).
Unit: UnitPackage,
// The mesh's own Go components read the system they were built for from this variable, and
// refuse to touch a machine without one.
SystemStamp: "main.builtFor",
OutputFlag: "--outDir",
},
{
Language: "python",
@@ -183,8 +82,6 @@ var toolchains = []Toolchain{
// each actually does.
Compile: []string{"python", "-m", "pip", "install", "--no-compile", "--target"},
OutputFlag: "",
Unit: UnitSources,
SourceExt: ".py",
},
}
@@ -1,18 +0,0 @@
package catalogue
import "testing"
// A claim written before the rename still holds (novox/hq ADR 0122, ADR 0156): with the store's
// aliases loaded, the former name resolves to the seat.
func TestTheArtifactStoresFormerNameResolvesToIt(t *testing.T) {
was := aliases
t.Cleanup(func() { aliases = was })
UseAliases(map[string]string{"the-artifact-store": "mesh-artifact-store"})
seat, known := SeatNamed("the-artifact-store")
if !known || seat.Name != "mesh-artifact-store" || seat.Delivers != "artifact-store" {
t.Fatalf("the former name did not resolve: %+v %v", seat, known)
}
if _, known := SeatNamed("mesh-artifact-store"); !known {
t.Fatal("the seat is not in the set under its name")
}
}
@@ -23,7 +23,7 @@ func TestASecondArtifactStoreAnywhereIsRefusedByName(t *testing.T) {
}
// A second one, on any other machine, is refused — and the refusal names the seat.
elsewhere := World{Held: []Held{{Claim: "mesh-artifact-store", Scope: ScopeMesh,
elsewhere := World{Held: []Held{{Claim: "the-artifact-store", Scope: ScopeMesh,
Node: "anchor", Module: "distribution"}}}
other := workstation()
other.Name = "laptop"
@@ -32,7 +32,7 @@ func TestASecondArtifactStoreAnywhereIsRefusedByName(t *testing.T) {
t.Fatal("a second store was accepted on another machine; it would offer artifact-store a " +
"second time and every consumer elsewhere would refuse to choose")
}
if !strings.Contains(err.Error(), "mesh-artifact-store") || !strings.Contains(err.Error(), "one per mesh") {
if !strings.Contains(err.Error(), "the-artifact-store") || !strings.Contains(err.Error(), "one per mesh") {
t.Fatalf("refused without naming the seat: %v", err)
}
}
-21
View File
@@ -70,27 +70,6 @@ func knownFor(m Manifest, needs []Needed, node string) map[string]map[string]str
return out
}
// withOwnNames adds a module's own composed names to what it may name from one binding:
// `${bound:<provision>:name}` and `:internal-name`, and for several contributions to one requirement
// `:name-<local>` / `:internal-name-<local>`. Set over anything the provider serves under those keys:
// what the module is called is the mesh's statement, not the provider's.
func withOwnNames(values map[string]string, own map[string]any) {
for _, key := range []string{"name", "internal-name"} {
if v, ok := own[key].(string); ok {
values[key] = v
}
}
many, _ := own["names"].(map[string]any)
for local, raw := range many {
names, _ := raw.(map[string]any)
for _, key := range []string{"name", "internal-name"} {
if v, ok := names[key].(string); ok {
values[key+"-"+local] = v
}
}
}
}
// plainly renders a served value as a program would expect to read it.
func plainly(value any) string {
switch v := value.(type) {
+1 -1
View File
@@ -59,7 +59,7 @@ func TestRenamingASeatDidNotRenameTheInterfaceItDelivers(t *testing.T) {
for _, pair := range []struct{ seat, delivers string }{
{"git", "git"},
{"npm-package-registry", "npm-package-registry"},
{"mesh-artifact-store", "artifact-store"},
{"the-artifact-store", "artifact-store"},
{"mesh-store", "postgres-database"},
{"mesh-broker", "mesh-bus"},
} {
+5 -5
View File
@@ -25,7 +25,7 @@ func reachable() Node {
func onNetwork(nodes ...string) map[string][]Provider {
out := make([]Provider, 0, len(nodes))
for _, n := range nodes {
out = append(out, Provider{Node: n, At: n + ".internal", Module: "postgres"})
out = append(out, Provider{Node: n, At: n + ".internal"})
}
return map[string][]Provider{"postgres-database": out}
}
@@ -99,7 +99,7 @@ func TestSayingWhichOneSettlesIt(t *testing.T) {
got, err := Resolve(brokeredShelf(), []string{"meshboard"}, reachable(),
World{
Offered: onNetwork("anchor", "archive"),
Pinned: map[string]Chosen{"postgres-database": {Node: "archive", Module: "postgres"}},
Pinned: map[string]string{"postgres-database": "archive"},
})
if err != nil {
t.Fatal(err)
@@ -115,7 +115,7 @@ func TestBeingPointedAtAMachineThatDoesNotProvideItIsRefused(t *testing.T) {
_, err := Resolve(brokeredShelf(), []string{"meshboard"}, reachable(),
World{
Offered: onNetwork("anchor", "archive"),
Pinned: map[string]Chosen{"postgres-database": {Node: "somewhere-else", Module: "postgres"}},
Pinned: map[string]string{"postgres-database": "somewhere-else"},
})
if err == nil {
t.Fatal("a machine was silently given a different database from the one chosen")
@@ -131,12 +131,12 @@ func TestOneProviderDoesNotOverruleAChoice(t *testing.T) {
_, err := Resolve(brokeredShelf(), []string{"meshboard"}, reachable(),
World{
Offered: onNetwork("anchor"),
Pinned: map[string]Chosen{"postgres-database": {Node: "archive", Module: "postgres"}},
Pinned: map[string]string{"postgres-database": "archive"},
})
if err == nil {
t.Fatal("the only database was used although another was chosen")
}
if !strings.Contains(err.Error(), "only anchor/postgres provides it") {
if !strings.Contains(err.Error(), "only anchor provides it") {
t.Fatalf("the refusal does not say what is available: %v", err)
}
}
+2 -245
View File
@@ -2,7 +2,6 @@ package catalogue
import (
"fmt"
"path"
"sort"
"strings"
)
@@ -29,9 +28,6 @@ type Built struct {
Reference string
// Digest is "sha256:<hex>", for an archive. An image reference already ends in one.
Digest string
// Launchers are, for a bundle in an interpreted language, the executable the build wrote beside
// each entrypoint, by entrypoint (novox/hq ADR 0193): what the node's runtime starts to serve it.
Launchers map[string]string
}
// Resolve fills a manifest's resources in from what was built.
@@ -71,37 +67,6 @@ func (m Manifest) Resolve(built []Built) (Manifest, error) {
out := m
out.Build = nil
out.Resources = nil
// What the build compiled, kept on the resolved manifest (novox/hq ADR 0175): a tools bundle is
// named by no resource of the module's own — the node's runtime loads it — so this is the only
// place the mesh would otherwise not have it. In artifact order, so two resolutions of one
// build compare equal.
out.Bundles = nil
if m.Build != nil {
for _, a := range m.Build.Artifacts {
if a.Kind != ArtifactBundle {
continue
}
made := by[a.Name]
// What the runtime loads: what the artifact said, else every entrypoint of a module
// that declares tools, else nothing (the field's own rule; see Artifact.Loads).
loads := append([]string(nil), a.Loads...)
if a.Loads == nil && len(m.Tools) > 0 {
loads = append([]string(nil), a.Entrypoints...)
}
// **Kept, never routed** (ADR 0155): the builder publishes to the store at the address
// it reached it by, and a manifest carrying that address names an installation —
// registration refused node-tools for exactly this on 2026-10-02. The build record
// already keeps the store-relative form; the resolved manifest keeps the same, and
// composition routes it through the store a machine reaches (Routed).
out.Bundles = append(out.Bundles, Bundle{
Name: a.Name, Source: Recorded(made.Reference), Digest: made.Digest,
Language: a.Language, Entrypoints: append([]string(nil), a.Entrypoints...),
Loads: loads, Env: copyWords(a.Env), Launchers: copyWords(made.Launchers),
Binary: BinaryOf(a),
})
}
sort.Slice(out.Bundles, func(i, j int) bool { return out.Bundles[i].Name < out.Bundles[j].Name })
}
for _, r := range m.Resources {
named, _ := r["artifact"].(string)
if named == "" {
@@ -129,44 +94,12 @@ func (m Manifest) Resolve(built []Built) (Manifest, error) {
m.Module, r["id"], named)
case ArtifactImage, ArtifactUpstream:
filled["image"] = artifact.Reference
// An image is not unpacked anywhere, so it has no directory to be named for its
// version and `${version}` has nothing to mean. Refused rather than left as literal
// text in a path, which is how it would reach a machine and be created as a directory
// called `${version}`.
for key, value := range filled {
if text, isText := value.(string); isText && strings.Contains(text, versionRef) {
return Manifest{}, fmt.Errorf(
"%s: %v says %s in %q, and %q is an image — an image is not unpacked, so "+
"it has no versioned place. %s is for an archive or a bundle",
m.Module, r["id"], versionRef, key, named, versionRef)
}
}
case ArtifactArchive, ArtifactBundle:
// The same on the wire: both are bytes fetched by digest and unpacked. They differ in
// how they were made — one packed as it stood, the other compiled first — and a
// machine has no reason to care which.
// Kept, not routed, for the reason the bundles above are (ADR 0155).
filled["source"] = Recorded(artifact.Reference)
filled["source"] = artifact.Reference
filled["digest"] = artifact.Digest
// **And `${version}`, so a resource can name a place that is this build's alone**
// (novox/hq ADR 0141, 04-ISSUES/142). A component is unpacked into a directory named
// for its version so it can read its own version from its path — and until this,
// nothing could compose that path: an archive named a fixed one in the manifest and
// nothing interpolated the build into it, so nothing could ask for
// `…/versions/<version>/` and every machine took a hand-placed fallback.
//
// The version is the artifact's own digest, short. Not the commit: two builds of one
// commit are meant to be the same bytes (the toolchains are `-trimpath` for this), and
// a content-addressed version means an unchanged build resolves to the path it already
// had — so re-composing a declaration moves nothing, where a commit would move the
// path of an identical binary and recreate everything that reads it.
for key, value := range filled {
text, isText := value.(string)
if !isText || !strings.Contains(text, versionRef) {
continue
}
filled[key] = strings.ReplaceAll(text, versionRef, versionOf(artifact.Digest))
}
default:
return Manifest{}, fmt.Errorf("%s: %q is a %q, and an artifact is %q, %q, %q or %q",
m.Module, named, artifact.Kind, ArtifactImage, ArtifactArchive, ArtifactUpstream,
@@ -208,21 +141,8 @@ func (b *Build) problems(module string) []string {
// A bundle's source is the module's own directory by definition, and what it needs to say
// is which compiler — because the mesh chooses that, and cannot choose for a module that
// has not said.
if len(a.Env) > 0 && a.Kind != ArtifactBundle {
problems = append(problems, fmt.Sprintf(
"%s: %q is a %q and says what it is given (env). Only a bundle the node's runtime "+
"serves is given words (novox/hq ADR 0192); a container says its own environment",
module, a.Name, a.Kind))
}
if a.Kind == ArtifactBundle || a.Kind == ArtifactPackage {
// **Except for a language that compiles to a binary, where it names which one**
// (novox/hq 04-ISSUES/142). A bundle in an interpreted language is the module's own
// directory compiled whole, and naming a source would be describing its own build. A
// repository written in a compiled language holds several commands — the host and its
// bootstrap live in one, and the mesh needs the host — and "the module's own directory"
// is then not a package at all. So the compiled case may say which package, and says
// the module root by saying nothing.
if a.From != "" && !compilesToABinary(a.Language) {
if a.From != "" {
problems = append(problems, fmt.Sprintf(
"%s: %q is a bundle and names what it is built from (%q). A bundle is built "+
"from the module's own directory; what it says is the language",
@@ -233,41 +153,6 @@ func (b *Build) problems(module string) []string {
"%s: %q is a bundle and says no language, so nothing can choose a compiler "+
"for it", module, a.Name))
}
problems = append(problems, bundleEnvProblems(module, a)...)
// What the runtime loads is among what was compiled (ADR 0175): a name here that is
// not an entrypoint is a file the bundle does not contain, and the runtime would
// fail to import it on every machine rather than here.
for _, load := range a.Loads {
found := false
for _, e := range a.Entrypoints {
found = found || e == load
}
if !found {
problems = append(problems, fmt.Sprintf(
"%s: %q says the runtime loads %q, which is not among its entrypoints — "+
"what is loaded is compiled, so it is named there too", module, a.Name, load))
}
}
// **A system, for a language that compiles to a binary** (novox/hq ADR 0142). A binary
// is pinned to one operating system at link time so a host refuses to touch a machine
// it was not built for (novox/hq ADR 0005); an artifact that says nothing would be
// compiled for whatever the build machine happened to be, which reads as portable and
// is not.
if compiled := compilesToABinary(a.Language); compiled && strings.TrimSpace(a.System) == "" {
problems = append(problems, fmt.Sprintf(
"%s: %q is compiled to a binary and says no system, so it would be built for "+
"whatever the build machine happens to be. Declare one artifact per "+
"system: %s", module, a.Name, spokenSystems()))
} else if !compiled && strings.TrimSpace(a.System) != "" {
problems = append(problems, fmt.Sprintf(
"%s: %q names the system %q and is written in %q, which compiles to code that "+
"runs anywhere — a system that decides nothing reads as though it did",
module, a.Name, a.System, a.Language))
} else if compiled && !knownSystem(a.System) {
problems = append(problems, fmt.Sprintf(
"%s: %q is built for %q, and a system is %s",
module, a.Name, a.System, spokenSystems()))
}
} else {
if a.From == "" {
problems = append(problems, fmt.Sprintf(
@@ -308,131 +193,3 @@ func oneOrOther(n int) string {
}
return "them"
}
// Systems the mesh builds binaries for, which is the set a host may be pinned to (novox/hq ADR 0005).
//
// **A closed list, and the host's own, not the compiler's.** These are not the values a Go toolchain
// would call an operating system — the difference between two of them is a C library, not a kernel.
// They are what a machine reports itself to be and what a host is linked to refuse, so the list that
// matters is the one the host understands.
var systems = []string{"alpine", "android", "arch"}
// knownSystem is whether the mesh builds for it.
func knownSystem(system string) bool {
want := strings.ToLower(strings.TrimSpace(system))
for _, s := range systems {
if s == want {
return true
}
}
return false
}
// spokenSystems is the list as a refusal says it, so a reader is one edit from right.
func spokenSystems() string {
return strings.Join(systems, ", ")
}
// compilesToABinary is whether this language's bundle is a binary for one operating system rather
// than code that runs wherever its interpreter does.
//
// **Asked of the language, not of the artifact.** A module says what it is written in; what that
// implies is the mesh's to know, exactly as the compiler is (novox/hq ADR 0142). Asking the artifact
// would let two artifacts in one language disagree about whether they are portable.
func compilesToABinary(language string) bool {
switch strings.ToLower(strings.TrimSpace(language)) {
case "go":
return true
default:
return false
}
}
// versionRef is how a resource names the version of the artifact it uses: ${version}.
//
// No artifact name in it, because the resource already says which artifact it is for — a second
// name would be a second thing to keep in step with the first.
const versionRef = "${version}"
// versionOf is an artifact's version as a path names it: its digest, short.
//
// **Content-addressed on purpose.** The alternative is the commit a build came from, and two builds
// of one commit are meant to produce the same bytes — every toolchain here is `-trimpath` for that
// reason. A commit-named path would move for an identical binary, and everything reading that path
// would be recreated for a change that is not one. A digest-named path moves exactly when the bytes
// do.
//
// Twelve hex characters: enough that two of this mesh's builds will not collide, short enough to
// read in a path and in a journal line. The `sha256:` prefix goes, because a directory name carrying
// a colon is a directory name people quote wrong.
func versionOf(digest string) string {
hex := strings.TrimPrefix(strings.TrimSpace(digest), "sha256:")
if len(hex) > 12 {
return hex[:12]
}
return hex
}
// bundleEnvWords are the words the runtime sets for itself; a bundle that named one would be
// telling the runtime what it is, which is the mesh's to say (novox/hq ADR 0192).
var bundleEnvWords = map[string]bool{
RuntimeToolModules: true, RuntimeBrokerFile: true, RuntimeOperatorAccount: true,
RuntimeOperatorHome: true, RuntimeToolEnv: true,
}
// bundleEnvProblems says what is wrong with what a bundle says it is given (novox/hq ADR 0192):
// a value is a path or a constant written with the references a container's environment may use
// for a place or a port, and never a secret's content or another module's binding — a secret
// reaches a tool as a file whose path is named.
func bundleEnvProblems(module string, a Artifact) []string {
if len(a.Env) == 0 {
return nil
}
var problems []string
for _, word := range sortedKeys(a.Env) {
value := a.Env[word]
if bundleEnvWords[word] {
problems = append(problems, fmt.Sprintf(
"%s: %q gives itself %s, which the node's runtime sets for itself; a bundle is "+
"given its own words beside the runtime's, never in place of them (novox/hq ADR 0192)",
module, a.Name, word))
}
rest := ofPort.ReplaceAllString(dirRef.ReplaceAllString(value, ""), "")
if strings.Contains(rest, "${") {
problems = append(problems, fmt.Sprintf(
"%s: %q gives %s the value %q. A bundle's word is a path or a constant, written with "+
"${dir:…} and ${port:…} only; a secret reaches a tool as a file the mesh places, "+
"named by its path, never as its content (novox/hq ADR 0192)",
module, a.Name, word, value))
}
}
return problems
}
func copyWords(in map[string]string) map[string]string {
if len(in) == 0 {
return nil
}
out := make(map[string]string, len(in))
for k, v := range in {
out[k] = v
}
return out
}
// BinaryOf is what a bundle compiled to a binary is called once built: what the artifact names, else
// the package it is built from, else the artifact's own name (novox/hq 04-ISSUES/142). Empty for a
// language that does not compile to one. The builder writes the binary under this name, and the
// composer runs it by it, so both ask here.
func BinaryOf(a Artifact) string {
if !compilesToABinary(a.Language) {
return ""
}
if name := strings.TrimSpace(a.Binary); name != "" {
return name
}
if from := strings.Trim(a.From, "./"); from != "" {
return path.Base(from)
}
return a.Name
}
-82
View File
@@ -1,82 +0,0 @@
package catalogue
import (
"strings"
"testing"
)
// bundleFor is a manifest whose one artifact is a bundle in the given language and system.
func bundleFor(language, system string) Manifest {
return Manifest{Module: "a-component", Build: &Build{Artifacts: []Artifact{
{Name: "binary", Kind: ArtifactBundle, Language: language, System: system},
}}}
}
func problemsOf(t *testing.T, m Manifest) string {
t.Helper()
return strings.Join(m.Build.problems(m.Module), "\n")
}
// **A language that compiles to a binary must say which system.**
//
// A binary is pinned to one operating system at link time, so a host refuses to touch a machine it
// was not built for. An artifact that says nothing would be compiled for whatever the build machine
// happened to be — which reads as portable and is not, and is the fault this check exists for.
func TestABinaryMustSayWhichSystemItIsFor(t *testing.T) {
got := problemsOf(t, bundleFor("go", ""))
if !strings.Contains(got, "says no system") {
t.Fatalf("a compiled bundle with no system was accepted:\n%s", got)
}
// And the refusal names what it could have said, so a reader is one edit from right.
for _, system := range []string{"alpine", "android", "arch"} {
if !strings.Contains(got, system) {
t.Fatalf("the refusal does not name %q as a choice:\n%s", system, got)
}
}
}
func TestABinaryThatNamesASystemIsAccepted(t *testing.T) {
if got := problemsOf(t, bundleFor("go", "arch")); got != "" {
t.Fatalf("a compiled bundle naming a system was refused:\n%s", got)
}
}
// A system the mesh does not build for is refused where it is written. These are the host's own
// names, not a compiler's: the difference between two of them is a C library rather than a kernel,
// so a value that looks like an operating system to a toolchain is still wrong here.
func TestASystemTheMeshDoesNotBuildForIsRefused(t *testing.T) {
for _, wrong := range []string{"linux", "debian", "darwin"} {
got := problemsOf(t, bundleFor("go", wrong))
if !strings.Contains(got, "and a system is") {
t.Fatalf("%q was accepted as a system:\n%s", wrong, got)
}
}
}
// **And a language that runs anywhere must not name one.** A system that decides nothing reads as
// though it did, which is the same fault as a restriction that restricts nothing (novox/hq ADR 0045).
func TestAPortableBundleMayNotNameASystem(t *testing.T) {
got := problemsOf(t, bundleFor("typescript", "arch"))
if !strings.Contains(got, "runs anywhere") {
t.Fatalf("a portable bundle was allowed to name a system:\n%s", got)
}
}
func TestAPortableBundleNamingNoSystemIsAccepted(t *testing.T) {
if got := problemsOf(t, bundleFor("typescript", "")); got != "" {
t.Fatalf("an ordinary bundle was refused:\n%s", got)
}
}
// One component, one artifact per system: the shape the mesh's own binaries are declared in, and the
// reason the target is the artifact's rather than the recipe's.
func TestOneArtifactPerSystemIsAccepted(t *testing.T) {
m := Manifest{Module: "the-host", Build: &Build{Artifacts: []Artifact{
{Name: "arch", Kind: ArtifactBundle, Language: "go", System: "arch"},
{Name: "alpine", Kind: ArtifactBundle, Language: "go", System: "alpine"},
{Name: "android", Kind: ArtifactBundle, Language: "go", System: "android"},
}}}
if got := problemsOf(t, m); got != "" {
t.Fatalf("one artifact per system was refused:\n%s", got)
}
}
@@ -1,88 +0,0 @@
package catalogue
import (
"os"
"path/filepath"
"strings"
"testing"
)
// TestEveryCatalogueManifestParses runs the real catalogue through the real gate.
//
// Not a fixture: the point is whether the manifests as written are accepted by the control plane that
// will read them, and a copy of one manifest proves nothing about the other seventy-one.
// catalogueRoot is the catalogue these checks run over: MESH_CATALOGUE when set, else the checkout
// beside this one, the way the main layout has it. A check that only ran when somebody remembered a
// variable was a check nobody ran (novox/hq issue 134, 2026-09-30); it skips only when there is no
// catalogue to be found at all.
func catalogueRoot(t *testing.T) string {
t.Helper()
if root := os.Getenv("MESH_CATALOGUE"); root != "" {
return root
}
sibling := filepath.Join("..", "..", "..", "mesh-catalog")
if _, err := os.Stat(filepath.Join(sibling, "modules")); err != nil {
t.Skip("no catalogue beside this checkout and MESH_CATALOGUE unset")
}
return sibling
}
func TestEveryCatalogueManifestParses(t *testing.T) {
root := catalogueRoot(t)
found, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
if err != nil || len(found) == 0 {
t.Fatalf("no manifests under %s: %v", root, err)
}
named, routed := 0, 0
for _, p := range found {
raw, err := os.ReadFile(p)
if err != nil {
t.Fatalf("%s: %v", p, err)
}
m, err := ParseManifest(raw)
if err != nil {
t.Errorf("%s: %v", filepath.Base(filepath.Dir(p)), err)
continue
}
for _, l := range m.Listens {
if l.Name != "" {
named++
}
}
for port := range RoutedPorts(m) {
_ = port
routed++
}
}
t.Logf("%d manifests, %d named endpoints, %d routed endpoints resolved", len(found), named, routed)
if named == 0 {
t.Fatal("no endpoint in the catalogue is named, so this proved nothing")
}
}
// TestNoCatalogueManifestNamesAnInstallation is ADR 0112's check, run over the real catalogue: no
// definition names a domain or a public address the mesh acts on, and every value that must for now
// carries its reason (novox/hq ADR 0155, issue 134). The list it prints is the one that shrinks.
func TestNoCatalogueManifestNamesAnInstallation(t *testing.T) {
root := catalogueRoot(t)
found, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
if err != nil || len(found) == 0 {
t.Fatalf("no manifests under %s: %v", root, err)
}
var named []string
for _, p := range found {
raw, err := os.ReadFile(p)
if err != nil {
t.Fatalf("%s: %v", p, err)
}
m, err := ParseManifest(raw)
if err != nil {
t.Errorf("%s: %v", p, err)
continue
}
named = append(named, InstallationProblems(m)...)
}
if len(named) > 0 {
t.Fatalf("%d value(s) name an installation:\n %s", len(named), strings.Join(named, "\n "))
}
}
@@ -1,71 +0,0 @@
package catalogue
import (
"os"
"strings"
"testing"
)
// **A machine trusts the mesh's authority because a module put its root there** (novox/hq ADR
// 0147, issue 129). The module carries a shell script and a unit, and both are worthless unless
// the mesh fills in where the authority is — which is the one thing about it the manifest cannot
// state, because the authority's address is a fact about the mesh and not about the module.
//
// So what is checked here is the rendering, not the parsing: the script the machine will run
// names the authority it was bound to, and the unit runs that script both ways. The verification
// itself — a plain client trusting an internal name on a machine holding this, and failing on one
// that does not — is the lab's, and cannot be had here.
func TestCaTrustRendersTheAuthorityItWasBoundTo(t *testing.T) {
raw, err := os.ReadFile("../../../mesh-catalog/modules/ca-trust/module.json")
if err != nil {
t.Skipf("the catalogue is not beside this checkout: %v", err)
}
m, err := ParseManifest(raw)
if err != nil {
t.Fatalf("the trust module does not parse:\n%v", err)
}
r := Resolution{
Node: "workstation",
Modules: []Manifest{m},
Needs: []Needed{{
Name: "internal-acme-ca", From: "anchor", At: "anchor.internal", For: "ca-trust",
Serves: map[string]any{
"port": float64(9000), "path": "/acme/acme/directory", "roots": "/roots.pem",
},
}},
}
out, err := r.Declaration(Rendering{})
if err != nil {
t.Fatalf("the trust module could not be composed for a machine: %v", err)
}
script := fileNamed(out, "ca-trust.anchor")
if script == nil {
t.Fatalf("nothing writes the script the unit runs: %v", out)
}
body, _ := script["content"].(string)
if !strings.Contains(body, "https://anchor.internal:9000/roots.pem") {
t.Errorf("the script does not fetch from the authority it was bound to:\n%s", body)
}
if script["mode"] != "0755" {
t.Errorf("the script is written %v, which systemd cannot execute", script["mode"])
}
unit := fileNamed(out, "ca-trust.unit")
if unit == nil {
t.Fatalf("no unit: %v", out)
}
text, _ := unit["content"].(string)
// Both halves. A unit that only installs the anchor leaves a machine trusting an authority
// nobody assigned it to any more, which is the half issue 129 asked for by name.
for _, want := range []string{
"ExecStart=" + script["path"].(string) + " install",
"ExecStop=" + script["path"].(string) + " remove",
"RemainAfterExit=yes",
} {
if !strings.Contains(text, want) {
t.Errorf("the unit does not say %q:\n%s", want, text)
}
}
}
-100
View File
@@ -1,100 +0,0 @@
package catalogue
import (
"sort"
)
// Chosen is the provider somebody named for a provision: the module, and the node it runs on. Both,
// always (novox/hq #258) — a provision comes from a module, and the same module on two machines is
// two answers, so neither half alone says which. Module is empty only on a record made before this
// was asked, and such a record is honoured exactly as long as it is unambiguous.
type Chosen struct {
Node string
Module string
}
func (c Chosen) String() string {
if c.Module == "" {
return c.Node
}
return c.Node + "/" + c.Module
}
// matches is whether this provider is the one chosen.
func (c Chosen) matches(p Provider) bool {
return p.Node == c.Node && (c.Module == "" || p.Module == c.Module)
}
// among is every offered provider the choice names — one, when the choice is whole.
func (c Chosen) among(where []Provider) []Provider {
var out []Provider
for _, p := range where {
if c.matches(p) {
out = append(out, p)
}
}
return out
}
// nameOf is how a refusal names a provider: the node and the module on it.
func nameOf(p Provider) string {
return Chosen{Node: p.Node, Module: p.Module}.String()
}
// providerNames is every provider named, sorted, for a refusal to list.
func providerNames(where []Provider) []string {
out := make([]string, 0, len(where))
for _, p := range where {
out = append(out, nameOf(p))
}
sort.Strings(out)
return out
}
// providersHere is which modules in this node's own set offer a provision, sorted.
func providersHere(catalogue map[string]Manifest, here func(string) bool, want string) []string {
var out []string
for name, m := range catalogue {
if !here(name) {
continue
}
for _, o := range m.Offers() {
if o == want {
out = append(out, name)
break
}
}
}
sort.Strings(out)
return out
}
// servedByOne is what one provider beside the consumer says a consumer needs to know, or nothing.
//
// Serving is *whether* a need is created at all when the provider is on this same machine (novox/hq
// 04-ISSUES/038's sibling): a need never created is a binding the consumer never gets. The manifest
// alone answers that; the values are settled later, with the node's settings.
func servedByOne(m Manifest, want string) map[string]any {
if _, ok := m.Serves[want]; ok {
return ServedOn(m, want, nil)
}
return nil
}
// sharedByOne is the own secret that provider names as its credential (ADR 0158), or "" when it
// gives each consumer its own.
func sharedByOne(m Manifest, want string) string {
if own, shared := m.SharedCredentialOf(want); shared {
return own
}
return ""
}
func oneOf(list []string, s string) bool {
for _, x := range list {
if x == s {
return true
}
}
return false
}
-21
View File
@@ -197,27 +197,6 @@ func TestARouteCanBeSetPerMesh(t *testing.T) {
}
}
func TestASettingReachesAContributionOnlyWhereItDeclaresTheKey(t *testing.T) {
// novox/hq 04-ISSUES/173: the mail module's site name, set so its environment file could read
// it, arrived in every route it contributed. A setting overrides a key the contribution
// declares and adds none — the provider reads the contribution as a contract.
got, _ := Resolve(shelf(proxy(), published("board", "board", 8080)), []string{"board"}, workstation(), World{})
out, err := got.Declaration(Rendering{Settings: SettingsBy{
"board": {{From: "the mesh", Values: map[string]any{"host": "dashboard", "sitename": "Board"}}},
}})
if err != nil {
t.Fatal(err)
}
given := received(t, out)
if given[0].Values["host"] != "dashboard" {
t.Fatalf("the setting did not override the route's host: %v", given[0].Values)
}
if _, leaked := given[0].Values["sitename"]; leaked {
t.Fatalf("a setting the route never declared reached the proxy: %v", given[0].Values)
}
}
func TestReceivingWhatYouDoNotProvideIsRefused(t *testing.T) {
// It would create a file nobody ever writes to, on a machine where nothing asked for it.
_, err := ParseManifest([]byte(`{"module":"traefik","version":"1",
+68 -531
View File
@@ -241,40 +241,15 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) {
// Owner is kept beside the resources because a resource id cannot be split back into its module:
// a module's name may itself contain a dot. What the mesh adds of its own — an opening, the guard —
// has no owner.
//
// Received is what each module on the machine is given for each requirement it receives — the same
// contributions its received file is written from, kept beside it so the mesh can also issue them
// on the bus in the module's membership (novox/hq ADR 0167). By module, then requirement.
type Composed struct {
Resources []map[string]any
Owner map[string]string
Received map[string]map[string][]Contribution
// LeftOut is every module of this machine's set that was left out of its declaration, and
// why (novox/hq ADR 0163, rule 6): a setting stored for it that its definition can no longer
// compose. Its held things are kept and its containers untouched — the machine is told so —
// and it is told everything else.
LeftOut map[string]string
}
// LeftOut is which of this machine's modules a declaration composed with these settings leaves
// out, and why (novox/hq ADR 0163, rule 6): each whose stored settings its definition can no longer
// compose. Empty when every module composes. The same judgement SetSettings makes before storing.
func (r Resolution) LeftOut(settings SettingsBy, adopted bool) map[string]string {
out := map[string]string{}
for _, m := range r.Modules {
if err := JudgeSettings(m, settings[m.Module], adopted); err != nil {
out[m.Module] = err.Error()
}
}
return out
}
// Compose is Declaration with the owner of every resource said.
func (r Resolution) Compose(with Rendering) (Composed, error) {
owner := map[string]string{}
received := map[string]map[string][]Contribution{}
leftOut := map[string]string{}
resources, err := r.compose(with, owner, received, leftOut)
resources, err := r.compose(with, owner)
if err != nil {
return Composed{}, err
}
@@ -286,7 +261,7 @@ func (r Resolution) Compose(with Rendering) (Composed, error) {
"sealed": with.BusMembership, "mode": "0600",
})
}
return Composed{Resources: resources, Owner: owner, Received: received, LeftOut: leftOut}, nil
return Composed{Resources: resources, Owner: owner}, nil
}
// BusMembershipID names the resource carrying a machine's membership for the new bus, and
@@ -295,26 +270,7 @@ func BusMembershipID() string { return "bus-membership" }
const BusMembershipPath = "/var/lib/mesh/membership-next.json"
func (r Resolution) compose(with Rendering, owner map[string]string,
received map[string]map[string][]Contribution, leftOut map[string]string) ([]map[string]any, error) {
// **A setting is judged where it is stored, and an impossible one costs a module, not a
// machine** (novox/hq ADR 0163, rule 6). A definition that moved under a stored setting makes
// this module uncomposable; it is left out of the declaration — its held things kept, its
// containers untouched, the machine told so by name — and the machine is told everything else.
// Before placing, because a placement is a setting too.
left := r.LeftOut(with.Settings, with.Adopted)
kept := make([]Manifest, 0, len(r.Modules))
for _, m := range r.Modules {
if why, isLeft := left[m.Module]; isLeft {
if leftOut != nil {
leftOut[m.Module] = why
}
continue
}
kept = append(kept, m)
}
r.Modules = kept
func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[string]any, error) {
// Every manifest is placed first (novox/hq ADR 0112): the maps naming where its bindings,
// credentials and contributions land are resolved against this node's directories, so every
// reader below — the binding files, the sealed secrets, the grant paths a contribution
@@ -508,41 +464,18 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
return nil, fmt.Errorf(
"%s needs a secret called %q and none was made for it", m.Module, name)
}
// The runtime's credential belongs to the account the runtime runs as (novox/hq ADR 0175,
// to-be 38 WP3): its process is composed `user: <account>` where the node has one, and a
// root-owned 0600 file is one that process cannot read. Composed here rather than said in
// the manifest, because a manifest cannot say ${machine:account} safely — a node with no
// account has nothing to resolve it to, and then the runtime runs as root and the file
// stays root's.
owner := m.SecretsOwner
if m.Module == RuntimeModule && r.Account != "" {
owner = r.Account
}
first = append(first, ownedBy(owner, map[string]any{
"id": NeedID(name), "type": "file", "path": m.OwnSecrets[name].Path, "sealed": sealed,
first = append(first, ownedBy(m.SecretsOwner, map[string]any{
"id": NeedID(name), "type": "file", "path": m.OwnSecrets[name], "sealed": sealed,
}))
}
// This module's tools bundles, where the machine runs the node's tool runtime (novox/hq
// ADR 0175, to-be 38 WP2). Mesh-computed like everything above it, and before the module's
// own resources for the same reason: the runtime's process names the files inside these
// and is restarted when one changes, so they are on the machine before it is.
if r.runtimeHere() {
first = append(first, bundleArchives(m)...)
}
// Operator-owned paths this module is granted use of (novox/hq ADR 0051). Written before
// the module's own resources, and so before the container that mounts them: the host must
// find each present — refusing clearly if the operator has not provided it — before it
// starts anything that depends on it. The mesh creates, chowns and reconciles none of it;
// an `access` resource says only *this path must exist, and this module reaches it*.
// Where each is on THIS machine is the assignment's (novox/hq issue 153): placed by id
// where the operator said, the definition's default otherwise, refused where neither.
accesses, accessPaths, err := accessesFor(m, with.Settings[m.Module])
if err != nil {
return nil, err
}
for _, a := range accesses {
for _, a := range m.Accesses {
first = append(first, map[string]any{
"id": AccessID(a.Path), "type": "access", "path": a.Path, "mode": a.Mode,
"id": AccessID(a.Path), "type": "access", "path": a.Path, "mode": a.At(),
})
}
for _, to := range m.SecretRequirements() {
@@ -641,11 +574,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
}
found = here
}
own, err := r.ownNames(m, to, with.Settings[m.Module])
if err != nil {
return nil, err
}
file, err := boundFile(*found, m.Binds[to], ConsumerIdentity(r.Node, IdentitySource(m.Slug, m.Module)), own)
file, err := boundFile(*found, m.Binds[to], ConsumerIdentity(r.Node, IdentitySource(m.Slug, m.Module)))
if err != nil {
return nil, err
}
@@ -657,12 +586,6 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
return nil, err
}
first = append(first, file)
if received[m.Module] == nil {
received[m.Module] = map[string][]Contribution{}
}
// Empty rather than absent when nobody contributed, for the reason the file is
// written empty: "nothing asked" and "never told" want different responses.
received[m.Module][to] = append([]Contribution{}, given[to]...)
}
if m.Keeps != "" && with.Kept != nil {
file, err := keptFile(m.Keeps, with.Kept)
@@ -695,15 +618,17 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
// merging earlier would throw away the files it still needs.
resources = append(append([]map[string]any{}, first...), resources...)
// No container is given the mesh's names (novox/hq ADR 0148). It used to be: every
// container got the whole roster as `--add-host` entries at creation, and a name that
// moved afterwards was wrong inside it for as long as it ran (issues 109, 135) — and once
// the roster was made part of a container's identity so that could be caught, one name
// moving anywhere replaced every container in the mesh (issue 151). A container resolves a
// mesh name through its machine's resolver at the moment it asks, which the runtime is
// told once per machine, as a file, by the resolver's own module. The names a module
// declares for itself are its own and stay exactly as written: they are part of what the
// module is, and the mesh does not know what they mean.
// Every container is given the mesh's names. Not a choice a module makes: a module that
// listed them would go stale the day a machine joins, and one that did not would be a
// module whose containers cannot reach anything by name.
//
// A container that was given names of its own keeps them and gets the mesh's beside them:
// the mesh does not know what else a workload needs to reach, and taking something away
// to add something is not what "also" means.
if len(with.Names) > 0 {
resources = withMeshNames(resources, with.Names)
}
// What this module may name from inside one of its own files. Gathered once per module
// rather than per file, because it is a fact about the module.
sealed, err := sealedFor(m, r.Needs, with)
@@ -712,43 +637,8 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
}
// And what its bindings say, for the half of a connection that is not secret.
known := knownFor(m, r.Needs, r.Node)
// A requirement answered on this same machine is not in r.Needs — its binding file is
// written from `here` (above) — and so `${bound:…}` could not name it, though the file
// beside it said the same facts. Filled from the same answer, so the two cannot disagree.
for _, want := range m.Wants() {
if _, has := known[want]; has {
continue
}
answered, err := here(r, want, with)
if err != nil {
return nil, err
}
if answered == nil {
continue
}
local := *answered
local.For = m.Module
for provision, values := range knownFor(m, []Needed{local}, r.Node) {
known[provision] = values
}
}
// And what the module is called through each requirement it contributes to (novox/hq
// 04-ISSUES/122) — the same composition its binding file carries.
for provision, values := range known {
own, err := r.ownNames(m, provision, with.Settings[m.Module])
if err != nil {
return nil, err
}
withOwnNames(values, own)
}
// And where this node places the directories the module declared without a path
// (novox/hq ADR 0112) — resolved once per module, named by ${dir:…} from any resource.
// — and, on an adopted machine, where the assignment says they already are, with the
// owner the data already has (novox/hq issue 153). Malformed placements are refused here.
placed, err := Places(m, with.Settings[m.Module])
if err != nil {
return nil, err
}
dirs := dirsFor(m, with)
// And the machine underneath, which no binding of its own can tell it.
thisMachine := machineFacts(r, with.Names, with.MeshRange)
@@ -760,10 +650,6 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
// Which of this module's resources its preparation runs before, if it prepares anything.
prepareBefore := preparationTarget(m)
// Which found networks this machine's setting keeps for each of its containers (novox/hq
// ADR 0163, rule 4); judged above, so an invalid one is not here.
keptNetworks, _ := KeptNetworks(m, with.Settings[m.Module], with.Adopted)
for _, unsettled := range resources {
resource, err := ApplySettings(unsettled, with.Settings[m.Module])
if err != nil {
@@ -773,29 +659,12 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
for k, v := range resource {
copied[k] = v
}
if networks, keeps := keptNetworks[fmt.Sprint(copied["id"])]; keeps {
// The container also joins the found network the setting names, so a neighbour
// that resolves it there keeps resolving it. Passed to the host as its own field,
// which it joins after the container is made.
joins := make([]any, 0, len(networks))
for _, n := range networks {
joins = append(joins, n)
}
copied["networks"] = joins
}
if err := refuseSecretsInEnvironment(copied, secretFiles, m.Module); err != nil {
return nil, err
}
// Said in the catalogue, not on the machine: the host parses strictly and knows no
// such field, and the reason is for a reader of the manifest.
delete(copied, SecretsInEnvironment)
delete(copied, NamesOnPurpose)
// **An operator's value, from the assignment** (novox/hq ADR 0112, ADR 0155): what a
// definition may not carry because it is true of one installation only. Filled from
// the same layers a mergeable file takes, and refused when no layer set it.
if err := settingInto(copied, with.Settings[m.Module], m.Module); err != nil {
return nil, err
}
// **Placed before anything reads a path.** A pathless directory receives the path
// this node resolves for it, and every ${dir:…} — in paths, mounts, content and
// environment — becomes that path, so what follows sees only concrete places
@@ -803,12 +672,6 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
if err := dirInto(copied, dirs, m.Module); err != nil {
return nil, err
}
// The operator's data the same way: ${access:…} becomes where this node keeps it,
// and a placed directory takes the owner the assignment said (issue 153).
if err := accessInto(copied, accessPaths, m.Module); err != nil {
return nil, err
}
ownerInto(copied, placed)
// **After settings, and that is the whole reason it is here.** A module's file
// content is where a setting lands, so a placeholder may only exist once the setting
// has been put in — filling secrets first would look at content that is not yet what
@@ -872,15 +735,6 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
if renamed := reflectsRenamed(m.Module, resource["reload-on"]); renamed != nil {
copied["reload-on"] = renamed
}
// **What reads one of this module's own secrets is restarted when it changes** (novox/hq
// issue 203, issue 206). A credential is re-issued by the mesh, and a container that
// mounted the old file keeps the old one open: the build machine ran for an hour on a
// credential the mesh had replaced, because its manifest restarted it on its
// environment file and nobody had thought to name the credential too. Composed here so
// no manifest has to say it, for a container or a daemon that names the secret's path.
if reads := secretsReadBy(copied, m); len(reads) > 0 {
copied["restart-on"] = withRestartOn(copied["restart-on"], reads)
}
// **A version prepares its state before it runs** (novox/hq ADR 0135). Derived from the
// module's own resource rather than declared beside it: what prepares the state is the
// module's own code, so what it is given has to be what that code is given — and a
@@ -911,41 +765,6 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
out = append(out, fact)
}
}
// The node's tool runtime, last (novox/hq ADR 0175, to-be 38 WP2.3): one process loading every
// bundle delivered above and holding the credential sealed above, so both exist before it starts
// — the order written here is the order the machine applies.
if r.runtimeHere() {
process, err := r.runtimeProcess(with)
if err != nil {
return nil, err
}
owner[fmt.Sprint(process["id"])] = RuntimeModule
out = append(out, process)
// What each module's bundles are given is read as the account the runtime runs as.
words := map[string]map[string]string{}
for _, m := range r.Modules {
w, err := bundleWords(m, with)
if err != nil {
return nil, err
}
words[m.Module] = w
}
// And a file a module's words name is one the runtime is restarted for when it changes.
if named := givenTo(out, owner, words, r.Account); len(named) > 0 {
restarts, _ := process["restart-on"].([]any)
seen := map[string]bool{}
for _, id := range restarts {
seen[fmt.Sprint(id)] = true
}
for _, id := range named {
if !seen[id] {
restarts = append(restarts, id)
seen[id] = true
}
}
process["restart-on"] = restarts
}
}
if with.Adopted {
// First, before anything a module declares: what the mesh needs reachable, then its guard.
// The order a machine applies is the order written here.
@@ -1072,48 +891,12 @@ func (r Resolution) filtersHere() string {
// computed for this machine, and each module's per-node exposure. The same answer whether the node
// is adopted or converged — the one loads it as a filter, the other declares it as openings.
func (r Resolution) Rules(with Rendering) ([]Rule, error) {
// A module whose settings cannot compose is left out of the declaration (novox/hq ADR 0163,
// rule 6), and out of the filter with it: nothing of it is declared, so nothing of it is let
// through.
left := r.LeftOut(with.Settings, with.Adopted)
exposure := map[string]map[int]string{}
for _, m := range r.Modules {
if _, isLeft := left[m.Module]; isLeft {
continue
}
e, err := Exposure(m, with.Settings[m.Module])
if err != nil {
return nil, err
}
// And how far each endpoint reaches, which says the same thing to the filter and more
// besides (novox/hq ADR 0138). Folded in here rather than beside: the filter has one
// question — from where — and a reach answers it, so giving it two inputs would let them
// disagree. Reaches refuses a port that both name, so this cannot silently prefer one.
reaches, err := Reaches(m, with.Settings[m.Module])
if err != nil {
return nil, err
}
// **Only for an endpoint the proxy does not serve.** A routed endpoint's port is how the
// proxy reaches it and nothing else (ADR 0045), so `public` there asks for a public name and
// says nothing about the port — opening it to the world as well would undo the arrangement
// the proxy exists for, and would silently reopen a port an operator had narrowed.
//
// Found by trying to express a real module: one whose routed name must be public and whose
// machine-side port must not be. Under one value for both, there was no way to say it.
routed := RoutedPorts(m)
for port, reach := range reaches {
if routed[port] {
continue
}
source, ok := FilterSource(reach)
if !ok {
return nil, fmt.Errorf("%s: %q is not a reach the filter can read", m.Module, reach)
}
if e == nil {
e = map[int]string{}
}
e[port] = source
}
if e != nil {
exposure[m.Module] = e
}
@@ -1277,11 +1060,12 @@ func (r Resolution) contributions(settings SettingsBy, grants []Grant,
// Settings reach a contribution the same way they reach a file. A route's hostname is
// exactly the kind of thing that differs between one mesh and the next, and a module
// that could not have it set would have to be edited to be reused.
values, err := r.composed(m, to, m.Contributes[to], settings[m.Module],
values, err := settle(m.Contributes[to], settings[m.Module], nil,
m.Module+" contributing to "+to)
if err != nil {
return nil, err
return nil, fmt.Errorf("%s contributing to %s: %w", m.Module, to, err)
}
composeName(values, r.PublicDomain, r.At)
out[to] = append(out[to], Contribution{From: m.Module, Values: values})
}
// Several contributions to one requirement (ADR 0094's sibling for `contributes`): an
@@ -1290,11 +1074,12 @@ func (r Resolution) contributions(settings SettingsBy, grants []Grant,
// name always reaches the provider from here.
for _, to := range sortedKeys(m.ContributesMany) {
for _, local := range sortedKeys(m.ContributesMany[to]) {
values, err := r.composed(m, to, m.ContributesMany[to][local], settings[m.Module],
values, err := settle(m.ContributesMany[to][local], settings[m.Module], nil,
m.Module+" contributing "+local+" to "+to)
if err != nil {
return nil, err
return nil, fmt.Errorf("%s contributing %s to %s: %w", m.Module, local, to, err)
}
composeName(values, r.PublicDomain, r.At)
out[to] = append(out[to], Contribution{From: m.Module, Values: values})
}
}
@@ -1302,85 +1087,6 @@ func (r Resolution) contributions(settings SettingsBy, grants []Grant,
return out, nil
}
// composed is one contribution as its provider receives it: settled with this node's settings, its
// endpoint's port filled in, and its names composed from the label.
//
// **One function, because two readers must agree.** The provider is told the names in its received
// file; the contributing module is told the same names in its own binding (novox/hq 04-ISSUES/122).
// Composing them twice, in two places, is how the proxy would come to serve one name while the
// module wrote another into its configuration.
func (r Resolution) composed(m Manifest, to string, raw map[string]any, layers []Layer, what string) (
map[string]any, error) {
// Overridden, not merged: a setting changes a key the contribution declares and adds none.
// The provider reads the contribution as a contract, and a setting made for one of this
// module's files is no part of it (novox/hq 04-ISSUES/173).
values, err := overridden(raw, layers, what)
if err != nil {
return nil, fmt.Errorf("%s: %w", what, err)
}
reaches, err := Reaches(m, layers)
if err != nil {
return nil, fmt.Errorf("%s: %w", what, err)
}
blocks, err := Endpoints(m, layers)
if err != nil {
return nil, fmt.Errorf("%s: %w", what, err)
}
portOfEndpoint(values, endpointPorts(m))
composeName(values, r.PublicDomain, servingAt(r, to), reaches, endpointPorts(m), blocks)
return values, nil
}
// ownNames is what a module is known by through what it contributes to one requirement — the names
// the mesh composed for it, and nothing else of the contribution.
//
// **The half a module could not learn** (novox/hq 04-ISSUES/122). A module contributes a label, the
// mesh joins it with this node's domains, and the provider serves the result — and the module itself
// was never told. Software that must know its own address (a login redirect, a canonical URL, an
// issuer) had it written into the manifest as a literal, which is a domain in a definition and wrong
// on every other machine. `${bound:<requirement>:name}` is the answer, from the same composition the
// provider receives.
//
// Several contributions to one requirement are keyed by their local name under `names`.
func (r Resolution) ownNames(m Manifest, to string, layers []Layer) (map[string]any, error) {
pick := func(values map[string]any) map[string]any {
names := map[string]any{}
for _, key := range []string{"name", "internal-name"} {
if v, ok := values[key].(string); ok && v != "" {
names[key] = v
}
}
return names
}
out := map[string]any{}
if raw, ok := m.Contributes[to]; ok {
values, err := r.composed(m, to, raw, layers, m.Module+" contributing to "+to)
if err != nil {
return nil, err
}
for k, v := range pick(values) {
out[k] = v
}
}
if locals := m.ContributesMany[to]; len(locals) > 0 {
many := map[string]any{}
for _, local := range sortedKeys(locals) {
values, err := r.composed(m, to, locals[local], layers,
m.Module+" contributing "+local+" to "+to)
if err != nil {
return nil, err
}
if names := pick(values); len(names) > 0 {
many[local] = names
}
}
if len(many) > 0 {
out["names"] = many
}
}
return out, nil
}
// composeName joins a contribution's label with a node's public domain, and separately with its
// private one, in place (novox/hq ADR 0056).
//
@@ -1404,44 +1110,10 @@ func (r Resolution) ownNames(m Manifest, to string, layers []Layer) (map[string]
// the running mesh keeps serving the full names it has. And a labelled contribution on a node with
// no public domain composes nothing — there is nothing to join it to — which reads downstream as a
// route that named no host, the same as it would have before this existed.
func composeName(values map[string]any, publicDomain, internalDomain string, reaches map[int]string,
ports map[string]int, blocks map[string]Endpoint) {
func composeName(values map[string]any, publicDomain, internalDomain string) {
if values == nil {
return
}
// **The subdomain an assignment gave this endpoint**, before the name is joined (novox/hq ADR
// 0138). The module contributes a label because it names its own parts; an assignment may say a
// different one, because where a thing lives under a domain is the operator's to choose and used
// to require editing the module to change.
if name, ok := values[RouteEndpoint].(string); ok {
if ep, said := blocks[strings.TrimSpace(name)]; said && ep.Label != "" {
values["label"] = ep.Label
}
}
// **How far the endpoint this route serves reaches decides which names exist** (novox/hq ADR
// 0138). Both were composed whenever the node had both domains, so every routed module got a
// public name and an internal one whether anybody wanted them or not — and a certificate for
// each, because the proxy certifies the names it is given.
//
// Joined by the port: a route entry names the port it serves and the module declares a listen on
// it. An entry with no port is not an endpoint's route but a rule about a name — a path-level
// refusal shadowing another route — and it inherits whatever that route's names turned out to
// be, which is why it is left alone here.
//
// Nothing said is both names, as before. That is what keeps every mesh already running identical
// until an assignment speaks.
wantPublic, wantInternal := true, true
if port, ok := endpointPortOf(values, ports); ok {
if reach, said := reaches[port]; said {
wantPublic, wantInternal = WantsPublicName(reach), WantsInternalName(reach)
}
}
if !wantPublic {
publicDomain = ""
}
if !wantInternal {
internalDomain = ""
}
if _, already := values["name"]; already {
// A full name was given rather than a label. Left as-is: this is the legacy shape, and the
// point of the label is to not have to write the full name — a contribution that wrote both
@@ -1473,24 +1145,6 @@ func composeName(values map[string]any, publicDomain, internalDomain string, rea
}
}
// servingAt is the private-network name of the node a contribution to `to` arrives at: the
// provider's, when the provision is answered elsewhere, and this machine's own when it is answered
// here or not yet settled.
//
// A route's internal name is composed under it (novox/hq ADR 0151, issue 139). `<label>.<node>.internal`
// is answered by every machine's resolver as *anything under that node's name goes to that node* —
// so the node in the name has to be the one whose proxy answers, or the name sends a client to a
// machine with nothing listening while the public name, published at the serving node's address,
// works. Where the proxy runs beside the module the two are the same machine and nothing changes.
func servingAt(r Resolution, to string) string {
for _, n := range r.Needs {
if n.Name == to && n.At != "" {
return n.At
}
}
return r.At
}
// receivedFile is the file a provider is given its consumers' contributions in.
func receivedFile(requirement, path string, given []Contribution) (map[string]any, error) {
if given == nil {
@@ -1595,14 +1249,14 @@ func sortedKeys[V any](m map[string]V) []string {
// Where it is and what the providing module said about using it. **No credential**, and the file
// says so rather than leaving a reader to wonder whether one was meant to be there — a missing
// field looks like a bug, and a stated absence looks like a boundary.
func boundFile(n Needed, path, as string, own map[string]any) (map[string]any, error) {
func boundFile(n Needed, path, as string) (map[string]any, error) {
// A record has no machine and no address. Saying so is the difference between a reader
// concluding "somewhere with no address" and concluding the mesh failed to fill something in.
where := any(n.At)
if n.ByRecord {
where = "a record in this mesh, not a machine"
}
doc := map[string]any{
body, err := json.MarshalIndent(map[string]any{
"binding": 1,
"provision": n.Name,
"from": n.From,
@@ -1618,14 +1272,7 @@ func boundFile(n Needed, path, as string, own map[string]any) (map[string]any, e
"generated": "by the mesh — do not edit; replaced whenever this changes. " +
"The credential is not here: it is sealed, in the file this module's manifest " +
"names under `secrets`",
}
// **What this module is called through what it contributes here** (novox/hq 04-ISSUES/122):
// `name`, `internal-name`, or `names` by local name — composed exactly as the provider receives
// them. Absent when the module contributes nothing named, rather than written empty.
for key, value := range own {
doc[key] = value
}
body, err := json.MarshalIndent(doc, "", " ")
}, "", " ")
if err != nil {
return nil, err
}
@@ -1754,6 +1401,43 @@ func (r Resolution) servedOnThisMachine(provision string, with Rendering) (map[s
return nil, false, nil
}
// withMeshNames gives every container in a set the mesh's names.
//
// Copied rather than edited in place: these maps come from a module's manifest, and mutating one
// would change what the catalogue holds for every other machine running that module.
//
// A host-network container gets the names too. It was once skipped, on the belief that it "shares
// the machine's hosts file already" — but it does not: `docker run --network host` still gives the
// container its own /etc/hosts (localhost and its own id only), so every `<node>.internal` name the
// mesh wrote for the machine is invisible inside it, and a client that dials one gets EAI_AGAIN. The
// remedy is the same `--add-host` every other container gets — the runtime accepts it with
// `--network host` (verified), and without it a host-network consumer cannot reach a provider by the
// `.internal` address the mesh hands it as `${bound:...:at}`.
func withMeshNames(resources []map[string]any, names map[string]string) []map[string]any {
out := make([]map[string]any, 0, len(resources))
for _, r := range resources {
if r["type"] != "container" {
out = append(out, r)
continue
}
copied := map[string]any{}
for k, v := range r {
copied[k] = v
}
var given []any
if already, ok := copied["hosts"].([]any); ok {
given = append(given, already...)
}
for _, name := range sortedKeys(names) {
given = append(given, name+":"+names[name])
}
copied["hosts"] = given
out = append(out, copied)
}
return out
}
// pinned refuses an image that is not really pinned, on its way to a machine.
//
// **Here and not at parse** (novox/hq 04-ISSUES/025). A manifest in a repository names artifacts
@@ -1832,23 +1516,14 @@ func publishedOn(resource map[string]any, module string, with Rendering) {
out = append(out, givenOuter(written, with.Given[module]))
continue
}
// A short form may carry the protocol — `"3478/udp"` — and the number is what the mesh
// assigns for; the protocol rides along. Read as one token, the `/udp` made the whole
// entry "not a port", and passing it through let the runtime publish it wherever it
// liked: unifi's STUN and discovery landed on random machine ports while every TCP pin
// beside them held.
mapping, protocol := written, ""
if cut := strings.LastIndex(written, "/"); cut >= 0 {
mapping, protocol = written[:cut], written[cut:]
}
wanted, err := strconv.Atoi(strings.TrimSpace(mapping))
wanted, err := strconv.Atoi(strings.TrimSpace(written))
if err != nil {
// Not a port at all. Passed through, so the host refuses it with its own words rather
// than this quietly dropping something somebody meant.
out = append(out, written)
continue
}
out = append(out, fmt.Sprintf("%d:%d%s", with.machinePort(module, wanted), wanted, protocol))
out = append(out, fmt.Sprintf("%d:%d", with.machinePort(module, wanted), wanted))
}
resource["ports"] = out
}
@@ -2058,141 +1733,3 @@ func prepared(from map[string]any) map[string]any {
delete(step, "reload-on")
return step
}
// endpointPortOf is the port the endpoint a route serves listens on: looked up by the name the route
// gives, or read from the port it repeats (novox/hq ADR 0138).
//
// `ports` maps this module's endpoint names to their ports, computed once per module rather than
// re-scanned per contribution.
func endpointPortOf(values map[string]any, ports map[string]int) (int, bool) {
if name, ok := values[RouteEndpoint].(string); ok {
if port, found := ports[strings.TrimSpace(name)]; found {
return port, true
}
}
return asPort(values["port"])
}
// endpointPorts is a module's endpoint names against the ports they listen on.
func endpointPorts(m Manifest) map[string]int {
out := map[string]int{}
for _, l := range m.Listens {
if name := strings.TrimSpace(l.Name); name != "" {
out[name] = l.Port
}
}
return out
}
// portOfEndpoint fills in the port of the endpoint a contribution names, in place.
//
// **A contribution that names an endpoint must still carry that endpoint's port**, because everything
// downstream reads the port: the provider is told where to reach the consumer, and the machine-side
// redirection that turns a declared port into the number the machine published is keyed on it
// (atMachinePort). A route that named only its endpoint left the proxy with no port at all, and a
// proxy with no port has nothing to dial.
//
// Found before it shipped and after the catalogue had already been changed to name endpoints — the
// manifests were merged and the mesh had not yet picked them up, so nothing was broken yet. The
// declared port, not the machine one: the redirection happens later and is keyed on the declared
// number, so filling in the machine port here would be redirected a second time or not at all.
func portOfEndpoint(values map[string]any, ports map[string]int) {
if values == nil {
return
}
if _, already := values["port"]; already {
// A route that says both is its own answer; the older shape repeated the port and is still read.
return
}
name, ok := values[RouteEndpoint].(string)
if !ok {
return
}
if port, found := ports[strings.TrimSpace(name)]; found {
values["port"] = port
}
}
// secretsReadBy is the file resources of this module's own secrets that a container or a daemon reads
// — named in its volumes, its environment or its env-files by the secret's placed path — as
// restart-on ids. Nothing for other shapes, and nothing for a scheduled or run-once process, which
// the host refuses a restart-on for (it runs again anyway, and reads the file afresh).
func secretsReadBy(resource map[string]any, m Manifest) []string {
kind := fmt.Sprint(resource["type"])
if kind != "container" && kind != "process" {
return nil
}
if resource["schedule"] != nil || resource["run-once"] == true {
return nil
}
var mentioned []string
for _, key := range []string{"volumes", "env", "env-file"} {
mentioned = append(mentioned, stringsIn(resource[key])...)
}
var out []string
for _, name := range sortedKeys(m.OwnSecrets) {
path := m.OwnSecrets[name].Path
if path == "" {
continue
}
for _, s := range mentioned {
// A volume is `source:destination[:mode]`; an env value or an env-file is the path itself.
if s == path || strings.HasPrefix(s, path+":") {
out = append(out, m.Module+"."+NeedID(name))
break
}
}
}
return out
}
// stringsIn is every string in a list or a map's values; nothing for anything else.
func stringsIn(v any) []string {
switch x := v.(type) {
case []any:
var out []string
for _, item := range x {
if s, ok := item.(string); ok {
out = append(out, s)
}
}
return out
case []string:
return x
case map[string]any:
var out []string
for _, k := range sortedKeys(x) {
if s, ok := x[k].(string); ok {
out = append(out, s)
}
}
return out
case map[string]string:
var out []string
for _, k := range sortedKeys(x) {
out = append(out, x[k])
}
return out
}
return nil
}
// withRestartOn is a resource's restart-on list with these ids added once each.
func withRestartOn(have any, add []string) []any {
var out []any
seen := map[string]bool{}
for _, id := range reflectsRenamed("", have) {
s := fmt.Sprint(id)
if !seen[s] {
seen[s] = true
out = append(out, s)
}
}
for _, id := range add {
if !seen[id] {
seen[id] = true
out = append(out, id)
}
}
return out
}
+13 -64
View File
@@ -20,12 +20,6 @@ import (
// declared with the path as the exception it is, and everything else in the module names it by
// id — so moving it later is one line, not a search.
//
// **The mesh's own files for a module are placed too** (novox/hq issue 174). What the mesh writes
// *for* a module — its sealed bus credential, its merged configuration, its bindings — is the
// mesh's plumbing, not the module's data, and sits under `<root>/mesh/<module>`. A directory
// saying `"place": "mesh"` is that place; the definition names the files beneath it by
// `${dir:<id>}` and states no path.
//
// **Resolved here, not on the machine.** The host receives concrete paths exactly as it always
// has; nothing new reaches it and it learns no field. Which also means a resolved path changing
// is a spec change like any other — and the spec comparison must see it (novox/hq issue 126).
@@ -33,15 +27,6 @@ import (
// defaultDataRoot is where module data lands when a node states no root of its own.
const defaultDataRoot = "/var/lib"
// The two places a pathless directory may name, beside its own id.
const (
// placeOwn is the assignment's own root, <root>/<module> — to-be 27's one directory per
// assignment, which every other placed thing of the module sits beneath.
placeOwn = "."
// placeMesh is where the mesh keeps what it writes for the module, <root>/mesh/<module>.
placeMesh = "mesh"
)
// dirRef is how a module names one of its placed directories: ${dir:<id>}.
var dirRef = regexp.MustCompile(`\$\{dir:([a-z0-9][a-z0-9-]*)\}`)
@@ -55,53 +40,26 @@ func dataRoot(with Rendering) string {
// dirsFor is every placed directory of a module, id → the path it resolves to on this node.
//
// A pathless directory saying `"place": "."` is the assignment's own root, <root>/<module>; one
// saying `"place": "mesh"` is the mesh's directory for the module, <root>/mesh/<module>; one
// saying neither is <root>/<module>/<id>. At most one of each place makes sense; nothing enforces
// one, because two ids resolving to one path is a mistake the module's own files make visible
// immediately.
//
// A stated path may itself begin with a placed reference — `${dir:mesh-state}/state` — and is
// filled after the directories it can name are resolved; one level, because a directory beneath
// a placed one is the whole of what an adopted layout needs (issue 174's `state` and `out`).
// A pathless directory saying `"place": "."` is the assignment's own root, <root>/<module> —
// to-be 27's one directory per assignment, which every other placed thing sits beneath. At most
// one makes sense; nothing enforces one, because two ids resolving to one path is a mistake the
// module's own files make visible immediately.
func dirsFor(m Manifest, with Rendering) map[string]string {
dirs := map[string]string{}
var beneath []map[string]any
// The assignment's placement wins over both (novox/hq issue 153). Refused elsewhere when
// malformed; here an invalid setting simply places nothing.
placed, _ := Places(m, with.Settings[m.Module])
for _, r := range m.Resources {
if fmt.Sprint(r["type"]) != "directory" {
continue
}
id := fmt.Sprint(r["id"])
if p, said := placed[id]; said {
dirs[id] = p.Path
continue
}
if path, stated := r["path"].(string); stated && path != "" {
if strings.HasPrefix(path, "${dir:") {
beneath = append(beneath, r)
continue
}
dirs[id] = strings.TrimRight(path, "/")
continue
}
switch place, _ := r["place"].(string); place {
case placeOwn:
if place, said := r["place"].(string); said && place == "." {
dirs[id] = dataRoot(with) + "/" + m.Module
case placeMesh:
dirs[id] = dataRoot(with) + "/mesh/" + m.Module
default:
dirs[id] = dataRoot(with) + "/" + m.Module + "/" + id
continue
}
}
for _, r := range beneath {
path := strings.TrimRight(r["path"].(string), "/")
// A reference to no directory is left as written and refused where the resource is
// placed (dirInto), with the message that names what exists.
filled, _ := dirFill(path, dirs, m.Module)
dirs[fmt.Sprint(r["id"])] = filled
dirs[id] = dataRoot(with) + "/" + m.Module + "/" + id
}
return dirs
}
@@ -161,16 +119,8 @@ func placedManifest(m Manifest, with Rendering) (Manifest, error) {
if m.Secrets, err = fillMap(m.Secrets); err != nil {
return m, err
}
if len(m.OwnSecrets) > 0 {
own := make(OwnSecrets, len(m.OwnSecrets))
for name, s := range m.OwnSecrets {
filled, err := dirFill(s.Path, dirs, m.Module)
if err != nil {
return m, err
}
own[name] = OwnSecret{Path: filled, Taken: s.Taken}
}
m.OwnSecrets = own
if m.OwnSecrets, err = fillMap(m.OwnSecrets); err != nil {
return m, err
}
if m.Grants, err = fillMap(m.Grants); err != nil {
return m, err
@@ -294,11 +244,10 @@ func (m Manifest) unknownDirRefs() []string {
"%s states both path and place on %v — a stated path IS the placement",
m.Module, r["id"]))
}
if place != placeOwn && place != placeMesh {
if place != "." {
problems = append(problems, fmt.Sprintf(
"%s says place %q on %v, and the places are %q — the assignment's own root — and "+
"%q — where the mesh keeps what it writes for the module",
m.Module, place, r["id"], placeOwn, placeMesh))
"%s says place %q on %v, and the only place is %q — the assignment's own root",
m.Module, place, r["id"], "."))
}
}
seen := map[string]bool{}
@@ -343,7 +292,7 @@ func (m Manifest) unknownDirRefs() []string {
}
maps := map[string]map[string]string{
"receives": m.Receives, "binds": m.Binds, "secrets": m.Secrets,
"own-secrets": m.OwnSecrets.Paths(), "grants": m.Grants,
"own-secrets": m.OwnSecrets, "grants": m.Grants,
}
for field, entries := range maps {
for _, value := range entries {
+4 -77
View File
@@ -164,7 +164,7 @@ func TestTheManifestsMapsArePlaced(t *testing.T) {
},
Binds: map[string]string{"route": "${dir:state}/route.json"},
Secrets: map[string]string{"mongodb-database": "${dir:state}/database.secret"},
OwnSecrets: OwnSecrets{"admin-key": {Path: "${dir:state}/admin-key.secret"}},
OwnSecrets: map[string]string{"admin-key": "${dir:state}/admin-key.secret"},
Receives: map[string]string{"route": "${dir:state}/grants/mesh.json"},
}
placed, err := placedManifest(m, Rendering{})
@@ -177,7 +177,7 @@ func TestTheManifestsMapsArePlaced(t *testing.T) {
if placed.Secrets["mongodb-database"] != "/var/lib/photos/database.secret" {
t.Fatalf("secrets are placed; got %v", placed.Secrets)
}
if placed.OwnSecrets["admin-key"].Path != "/var/lib/photos/admin-key.secret" {
if placed.OwnSecrets["admin-key"] != "/var/lib/photos/admin-key.secret" {
t.Fatalf("own-secrets are placed; got %v", placed.OwnSecrets)
}
if placed.Receives["route"] != "/var/lib/photos/grants/mesh.json" {
@@ -216,48 +216,8 @@ func TestPlaceIsValidatedAtTheManifest(t *testing.T) {
wrong := Manifest{Module: "x", Resources: []map[string]any{
{"id": "d", "type": "directory", "place": "sub/dir"},
}}
if got := wrong.unknownDirRefs(); len(got) != 1 || !strings.Contains(got[0], `the places are "."`) {
t.Fatalf("a place that is neither root refuses; got %v", got)
}
}
func TestTheMeshsDirectoryForAModuleIsAPlace(t *testing.T) {
// novox/hq issue 174. What the mesh writes for a module — its bus credential, its bindings —
// is the mesh's plumbing under <root>/mesh/<module>, and the definition names it by id.
m := Manifest{Module: "umami",
Resources: []map[string]any{
{"id": "mesh-state", "type": "directory", "place": "mesh"},
{"id": "state", "type": "directory", "place": "."},
{"id": "server", "type": "container", "image": "x@sha256:aa",
"volumes": []any{"${dir:mesh-state}/broker:/run/secrets/broker:ro"}},
},
OwnSecrets: OwnSecrets{"broker": {Path: "${dir:mesh-state}/broker"}},
Binds: map[string]string{"route": "${dir:state}/route.json"},
}
if got := m.unknownDirRefs(); len(got) != 0 {
t.Fatalf("place %q is a place; got %v", "mesh", got)
}
dirs := dirsFor(m, Rendering{})
if dirs["mesh-state"] != "/var/lib/mesh/umami" || dirs["state"] != "/var/lib/umami" {
t.Fatalf("the mesh's directory sits beside the module's, not in it; got %v", dirs)
}
dirs = dirsFor(m, Rendering{DataRoot: "/srv"})
if dirs["mesh-state"] != "/srv/mesh/umami" {
t.Fatalf("a node's root moves the mesh's files with the module's; got %v", dirs)
}
placed, err := placedManifest(m, Rendering{})
if err != nil {
t.Fatal(err)
}
if placed.OwnSecrets["broker"].Path != "/var/lib/mesh/umami/broker" {
t.Fatalf("own-secrets are placed under the mesh's directory; got %v", placed.OwnSecrets)
}
container := shallowCopy(m.Resources[2])
if err := dirInto(container, dirsFor(m, Rendering{}), m.Module); err != nil {
t.Fatal(err)
}
if container["volumes"].([]any)[0] != "/var/lib/mesh/umami/broker:/run/secrets/broker:ro" {
t.Fatalf("the mount's host side is placed; got %v", container["volumes"])
if got := wrong.unknownDirRefs(); len(got) != 1 || !strings.Contains(got[0], `the only place is "."`) {
t.Fatalf("a place that is not the root refuses; got %v", got)
}
}
@@ -290,36 +250,3 @@ func shallowCopy(resource map[string]any) map[string]any {
}
return copied
}
func TestADirectoryBeneathAPlacedOneIsPlacedWithIt(t *testing.T) {
// An adopted layout keeps a subdirectory the predecessor made under the mesh's directory
// (issue 174: a forge's runtime state, a manager's output). Stated as beneath the placed one,
// it moves with it — a node's root moves both, and the definition names no host path.
m := Manifest{Module: "gitea", Resources: []map[string]any{
{"id": "mesh-state", "type": "directory", "place": "mesh"},
{"id": "runtime-state", "type": "directory", "path": "${dir:mesh-state}/state"},
{"id": "server", "type": "container", "image": "x@sha256:aa",
"volumes": []any{"${dir:runtime-state}:/data"}},
}}
if got := m.unknownDirRefs(); len(got) != 0 {
t.Fatalf("a path beneath a placed directory is well formed; got %v", got)
}
dirs := dirsFor(m, Rendering{DataRoot: "/srv"})
if dirs["runtime-state"] != "/srv/mesh/gitea/state" {
t.Fatalf("the subdirectory follows the placed one; got %v", dirs)
}
sub := shallowCopy(m.Resources[1])
if err := dirInto(sub, dirs, m.Module); err != nil {
t.Fatal(err)
}
if sub["path"] != "/srv/mesh/gitea/state" {
t.Fatalf("the directory resource itself is resolved; got %v", sub["path"])
}
container := shallowCopy(m.Resources[2])
if err := dirInto(container, dirs, m.Module); err != nil {
t.Fatal(err)
}
if container["volumes"].([]any)[0] != "/srv/mesh/gitea/state:/data" {
t.Fatalf("a reference to the subdirectory resolves whole; got %v", container["volumes"])
}
}

Some files were not shown because too many files have changed in this diff Show More